16cbd5570SChris Mason /* 2d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 36cbd5570SChris Mason * 46cbd5570SChris Mason * This program is free software; you can redistribute it and/or 56cbd5570SChris Mason * modify it under the terms of the GNU General Public 66cbd5570SChris Mason * License v2 as published by the Free Software Foundation. 76cbd5570SChris Mason * 86cbd5570SChris Mason * This program is distributed in the hope that it will be useful, 96cbd5570SChris Mason * but WITHOUT ANY WARRANTY; without even the implied warranty of 106cbd5570SChris Mason * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 116cbd5570SChris Mason * General Public License for more details. 126cbd5570SChris Mason * 136cbd5570SChris Mason * You should have received a copy of the GNU General Public 146cbd5570SChris Mason * License along with this program; if not, write to the 156cbd5570SChris Mason * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 166cbd5570SChris Mason * Boston, MA 021110-1307, USA. 176cbd5570SChris Mason */ 186cbd5570SChris Mason 19a6b6e75eSChris Mason #include <linux/sched.h> 205a0e3ad6STejun Heo #include <linux/slab.h> 21bd989ba3SJan Schmidt #include <linux/rbtree.h> 22adf02123SDavid Sterba #include <linux/mm.h> 23eb60ceacSChris Mason #include "ctree.h" 24eb60ceacSChris Mason #include "disk-io.h" 257f5c1516SChris Mason #include "transaction.h" 265f39d397SChris Mason #include "print-tree.h" 27925baeddSChris Mason #include "locking.h" 289a8dd150SChris Mason 29e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 30e089f05cSChris Mason *root, struct btrfs_path *path, int level); 31310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 32310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 33310712b2SOmar Sandoval int data_size, int extend); 345f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 352ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 362ff7e61eSJeff Mahoney struct extent_buffer *dst, 37971a1f66SChris Mason struct extent_buffer *src, int empty); 385f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 392ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 405f39d397SChris Mason struct extent_buffer *dst_buf, 415f39d397SChris Mason struct extent_buffer *src_buf); 42afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 43afe5fea7STsutomu Itoh int level, int slot); 445de865eeSFilipe David Borba Manana static int tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 45f230475eSJan Schmidt struct extent_buffer *eb); 46d97e63b6SChris Mason 472c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 482c90e5d6SChris Mason { 49e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 502c90e5d6SChris Mason } 512c90e5d6SChris Mason 52b4ce94deSChris Mason /* 53b4ce94deSChris Mason * set all locked nodes in the path to blocking locks. This should 54b4ce94deSChris Mason * be done before scheduling 55b4ce94deSChris Mason */ 56b4ce94deSChris Mason noinline void btrfs_set_path_blocking(struct btrfs_path *p) 57b4ce94deSChris Mason { 58b4ce94deSChris Mason int i; 59b4ce94deSChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 60bd681513SChris Mason if (!p->nodes[i] || !p->locks[i]) 61bd681513SChris Mason continue; 62bd681513SChris Mason btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]); 63bd681513SChris Mason if (p->locks[i] == BTRFS_READ_LOCK) 64bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK_BLOCKING; 65bd681513SChris Mason else if (p->locks[i] == BTRFS_WRITE_LOCK) 66bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING; 67b4ce94deSChris Mason } 68b4ce94deSChris Mason } 69b4ce94deSChris Mason 70b4ce94deSChris Mason /* 71b4ce94deSChris Mason * reset all the locked nodes in the patch to spinning locks. 724008c04aSChris Mason * 734008c04aSChris Mason * held is used to keep lockdep happy, when lockdep is enabled 744008c04aSChris Mason * we set held to a blocking lock before we go around and 754008c04aSChris Mason * retake all the spinlocks in the path. You can safely use NULL 764008c04aSChris Mason * for held 77b4ce94deSChris Mason */ 784008c04aSChris Mason noinline void btrfs_clear_path_blocking(struct btrfs_path *p, 79bd681513SChris Mason struct extent_buffer *held, int held_rw) 80b4ce94deSChris Mason { 81b4ce94deSChris Mason int i; 824008c04aSChris Mason 83bd681513SChris Mason if (held) { 84bd681513SChris Mason btrfs_set_lock_blocking_rw(held, held_rw); 85bd681513SChris Mason if (held_rw == BTRFS_WRITE_LOCK) 86bd681513SChris Mason held_rw = BTRFS_WRITE_LOCK_BLOCKING; 87bd681513SChris Mason else if (held_rw == BTRFS_READ_LOCK) 88bd681513SChris Mason held_rw = BTRFS_READ_LOCK_BLOCKING; 89bd681513SChris Mason } 904008c04aSChris Mason btrfs_set_path_blocking(p); 914008c04aSChris Mason 924008c04aSChris Mason for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) { 93bd681513SChris Mason if (p->nodes[i] && p->locks[i]) { 94bd681513SChris Mason btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]); 95bd681513SChris Mason if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING) 96bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK; 97bd681513SChris Mason else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING) 98bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK; 99bd681513SChris Mason } 100b4ce94deSChris Mason } 1014008c04aSChris Mason 1024008c04aSChris Mason if (held) 103bd681513SChris Mason btrfs_clear_lock_blocking_rw(held, held_rw); 104b4ce94deSChris Mason } 105b4ce94deSChris Mason 106d352ac68SChris Mason /* this also releases the path */ 1072c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 1082c90e5d6SChris Mason { 109ff175d57SJesper Juhl if (!p) 110ff175d57SJesper Juhl return; 111b3b4aa74SDavid Sterba btrfs_release_path(p); 1122c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 1132c90e5d6SChris Mason } 1142c90e5d6SChris Mason 115d352ac68SChris Mason /* 116d352ac68SChris Mason * path release drops references on the extent buffers in the path 117d352ac68SChris Mason * and it drops any locks held by this path 118d352ac68SChris Mason * 119d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 120d352ac68SChris Mason */ 121b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 122eb60ceacSChris Mason { 123eb60ceacSChris Mason int i; 124a2135011SChris Mason 125234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1263f157a2fSChris Mason p->slots[i] = 0; 127eb60ceacSChris Mason if (!p->nodes[i]) 128925baeddSChris Mason continue; 129925baeddSChris Mason if (p->locks[i]) { 130bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 131925baeddSChris Mason p->locks[i] = 0; 132925baeddSChris Mason } 1335f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1343f157a2fSChris Mason p->nodes[i] = NULL; 135eb60ceacSChris Mason } 136eb60ceacSChris Mason } 137eb60ceacSChris Mason 138d352ac68SChris Mason /* 139d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 140d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 141d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 142d352ac68SChris Mason * looping required. 143d352ac68SChris Mason * 144d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 145d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 146d352ac68SChris Mason * at any time because there are no locks held. 147d352ac68SChris Mason */ 148925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 149925baeddSChris Mason { 150925baeddSChris Mason struct extent_buffer *eb; 151240f62c8SChris Mason 1523083ee2eSJosef Bacik while (1) { 153240f62c8SChris Mason rcu_read_lock(); 154240f62c8SChris Mason eb = rcu_dereference(root->node); 1553083ee2eSJosef Bacik 1563083ee2eSJosef Bacik /* 1573083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 15801327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 1593083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1603083ee2eSJosef Bacik * synchronize_rcu and try again. 1613083ee2eSJosef Bacik */ 1623083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 163240f62c8SChris Mason rcu_read_unlock(); 1643083ee2eSJosef Bacik break; 1653083ee2eSJosef Bacik } 1663083ee2eSJosef Bacik rcu_read_unlock(); 1673083ee2eSJosef Bacik synchronize_rcu(); 1683083ee2eSJosef Bacik } 169925baeddSChris Mason return eb; 170925baeddSChris Mason } 171925baeddSChris Mason 172d352ac68SChris Mason /* loop around taking references on and locking the root node of the 173d352ac68SChris Mason * tree until you end up with a lock on the root. A locked buffer 174d352ac68SChris Mason * is returned, with a reference held. 175d352ac68SChris Mason */ 176925baeddSChris Mason struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root) 177925baeddSChris Mason { 178925baeddSChris Mason struct extent_buffer *eb; 179925baeddSChris Mason 180925baeddSChris Mason while (1) { 181925baeddSChris Mason eb = btrfs_root_node(root); 182925baeddSChris Mason btrfs_tree_lock(eb); 183240f62c8SChris Mason if (eb == root->node) 184925baeddSChris Mason break; 185925baeddSChris Mason btrfs_tree_unlock(eb); 186925baeddSChris Mason free_extent_buffer(eb); 187925baeddSChris Mason } 188925baeddSChris Mason return eb; 189925baeddSChris Mason } 190925baeddSChris Mason 191bd681513SChris Mason /* loop around taking references on and locking the root node of the 192bd681513SChris Mason * tree until you end up with a lock on the root. A locked buffer 193bd681513SChris Mason * is returned, with a reference held. 194bd681513SChris Mason */ 19584f7d8e6SJosef Bacik struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) 196bd681513SChris Mason { 197bd681513SChris Mason struct extent_buffer *eb; 198bd681513SChris Mason 199bd681513SChris Mason while (1) { 200bd681513SChris Mason eb = btrfs_root_node(root); 201bd681513SChris Mason btrfs_tree_read_lock(eb); 202bd681513SChris Mason if (eb == root->node) 203bd681513SChris Mason break; 204bd681513SChris Mason btrfs_tree_read_unlock(eb); 205bd681513SChris Mason free_extent_buffer(eb); 206bd681513SChris Mason } 207bd681513SChris Mason return eb; 208bd681513SChris Mason } 209bd681513SChris Mason 210d352ac68SChris Mason /* cowonly root (everything not a reference counted cow subvolume), just get 211d352ac68SChris Mason * put onto a simple dirty list. transaction.c walks this to make sure they 212d352ac68SChris Mason * get properly updated on disk. 213d352ac68SChris Mason */ 2140b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2150b86a832SChris Mason { 2160b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2170b246afaSJeff Mahoney 218e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 219e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 220e7070be1SJosef Bacik return; 221e7070be1SJosef Bacik 2220b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 223e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 224e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 225e7070be1SJosef Bacik if (root->objectid == BTRFS_EXTENT_TREE_OBJECTID) 226e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 2270b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 228e7070be1SJosef Bacik else 229e7070be1SJosef Bacik list_move(&root->dirty_list, 2300b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 2310b86a832SChris Mason } 2320b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 2330b86a832SChris Mason } 2340b86a832SChris Mason 235d352ac68SChris Mason /* 236d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 237d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 238d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 239d352ac68SChris Mason */ 240be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 241be20aa9dSChris Mason struct btrfs_root *root, 242be20aa9dSChris Mason struct extent_buffer *buf, 243be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 244be20aa9dSChris Mason { 2450b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 246be20aa9dSChris Mason struct extent_buffer *cow; 247be20aa9dSChris Mason int ret = 0; 248be20aa9dSChris Mason int level; 2495d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 250be20aa9dSChris Mason 25127cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 2520b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 25327cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 25427cdeb70SMiao Xie trans->transid != root->last_trans); 255be20aa9dSChris Mason 256be20aa9dSChris Mason level = btrfs_header_level(buf); 2575d4f98a2SYan Zheng if (level == 0) 2585d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 2595d4f98a2SYan Zheng else 2605d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 26131840ae1SZheng Yan 2624d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 2634d75f8a9SDavid Sterba &disk_key, level, buf->start, 0); 2645d4f98a2SYan Zheng if (IS_ERR(cow)) 265be20aa9dSChris Mason return PTR_ERR(cow); 266be20aa9dSChris Mason 26758e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 268be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 269be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2705d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2715d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2725d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2735d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2745d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2755d4f98a2SYan Zheng else 276be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 277be20aa9dSChris Mason 2780b246afaSJeff Mahoney write_extent_buffer_fsid(cow, fs_info->fsid); 2792b82032cSYan Zheng 280be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2815d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 282e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 2835d4f98a2SYan Zheng else 284e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 2854aec2b52SChris Mason 286be20aa9dSChris Mason if (ret) 287be20aa9dSChris Mason return ret; 288be20aa9dSChris Mason 289be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 290be20aa9dSChris Mason *cow_ret = cow; 291be20aa9dSChris Mason return 0; 292be20aa9dSChris Mason } 293be20aa9dSChris Mason 294bd989ba3SJan Schmidt enum mod_log_op { 295bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 296bd989ba3SJan Schmidt MOD_LOG_KEY_ADD, 297bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE, 298bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_FREEING, 299bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_MOVING, 300bd989ba3SJan Schmidt MOD_LOG_MOVE_KEYS, 301bd989ba3SJan Schmidt MOD_LOG_ROOT_REPLACE, 302bd989ba3SJan Schmidt }; 303bd989ba3SJan Schmidt 304bd989ba3SJan Schmidt struct tree_mod_move { 305bd989ba3SJan Schmidt int dst_slot; 306bd989ba3SJan Schmidt int nr_items; 307bd989ba3SJan Schmidt }; 308bd989ba3SJan Schmidt 309bd989ba3SJan Schmidt struct tree_mod_root { 310bd989ba3SJan Schmidt u64 logical; 311bd989ba3SJan Schmidt u8 level; 312bd989ba3SJan Schmidt }; 313bd989ba3SJan Schmidt 314bd989ba3SJan Schmidt struct tree_mod_elem { 315bd989ba3SJan Schmidt struct rb_node node; 316298cfd36SChandan Rajendra u64 logical; 317097b8a7cSJan Schmidt u64 seq; 318bd989ba3SJan Schmidt enum mod_log_op op; 319bd989ba3SJan Schmidt 320bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */ 321bd989ba3SJan Schmidt int slot; 322bd989ba3SJan Schmidt 323bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */ 324bd989ba3SJan Schmidt u64 generation; 325bd989ba3SJan Schmidt 326bd989ba3SJan Schmidt /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */ 327bd989ba3SJan Schmidt struct btrfs_disk_key key; 328bd989ba3SJan Schmidt u64 blockptr; 329bd989ba3SJan Schmidt 330bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_MOVE_KEYS */ 331bd989ba3SJan Schmidt struct tree_mod_move move; 332bd989ba3SJan Schmidt 333bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_ROOT_REPLACE */ 334bd989ba3SJan Schmidt struct tree_mod_root old_root; 335bd989ba3SJan Schmidt }; 336bd989ba3SJan Schmidt 337097b8a7cSJan Schmidt static inline void tree_mod_log_read_lock(struct btrfs_fs_info *fs_info) 338bd989ba3SJan Schmidt { 339097b8a7cSJan Schmidt read_lock(&fs_info->tree_mod_log_lock); 340bd989ba3SJan Schmidt } 341bd989ba3SJan Schmidt 342097b8a7cSJan Schmidt static inline void tree_mod_log_read_unlock(struct btrfs_fs_info *fs_info) 343097b8a7cSJan Schmidt { 344097b8a7cSJan Schmidt read_unlock(&fs_info->tree_mod_log_lock); 345097b8a7cSJan Schmidt } 346097b8a7cSJan Schmidt 347097b8a7cSJan Schmidt static inline void tree_mod_log_write_lock(struct btrfs_fs_info *fs_info) 348097b8a7cSJan Schmidt { 349097b8a7cSJan Schmidt write_lock(&fs_info->tree_mod_log_lock); 350097b8a7cSJan Schmidt } 351097b8a7cSJan Schmidt 352097b8a7cSJan Schmidt static inline void tree_mod_log_write_unlock(struct btrfs_fs_info *fs_info) 353097b8a7cSJan Schmidt { 354097b8a7cSJan Schmidt write_unlock(&fs_info->tree_mod_log_lock); 355097b8a7cSJan Schmidt } 356097b8a7cSJan Schmidt 357097b8a7cSJan Schmidt /* 358fcebe456SJosef Bacik * Pull a new tree mod seq number for our operation. 359fc36ed7eSJan Schmidt */ 360fcebe456SJosef Bacik static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info) 361fc36ed7eSJan Schmidt { 362fc36ed7eSJan Schmidt return atomic64_inc_return(&fs_info->tree_mod_seq); 363fc36ed7eSJan Schmidt } 364fc36ed7eSJan Schmidt 365fc36ed7eSJan Schmidt /* 366097b8a7cSJan Schmidt * This adds a new blocker to the tree mod log's blocker list if the @elem 367097b8a7cSJan Schmidt * passed does not already have a sequence number set. So when a caller expects 368097b8a7cSJan Schmidt * to record tree modifications, it should ensure to set elem->seq to zero 369097b8a7cSJan Schmidt * before calling btrfs_get_tree_mod_seq. 370097b8a7cSJan Schmidt * Returns a fresh, unused tree log modification sequence number, even if no new 371097b8a7cSJan Schmidt * blocker was added. 372097b8a7cSJan Schmidt */ 373097b8a7cSJan Schmidt u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, 374bd989ba3SJan Schmidt struct seq_list *elem) 375bd989ba3SJan Schmidt { 376097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 377bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 378097b8a7cSJan Schmidt if (!elem->seq) { 379fcebe456SJosef Bacik elem->seq = btrfs_inc_tree_mod_seq(fs_info); 380097b8a7cSJan Schmidt list_add_tail(&elem->list, &fs_info->tree_mod_seq_list); 381097b8a7cSJan Schmidt } 382bd989ba3SJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 383097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 384097b8a7cSJan Schmidt 385fcebe456SJosef Bacik return elem->seq; 386bd989ba3SJan Schmidt } 387bd989ba3SJan Schmidt 388bd989ba3SJan Schmidt void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, 389bd989ba3SJan Schmidt struct seq_list *elem) 390bd989ba3SJan Schmidt { 391bd989ba3SJan Schmidt struct rb_root *tm_root; 392bd989ba3SJan Schmidt struct rb_node *node; 393bd989ba3SJan Schmidt struct rb_node *next; 394bd989ba3SJan Schmidt struct seq_list *cur_elem; 395bd989ba3SJan Schmidt struct tree_mod_elem *tm; 396bd989ba3SJan Schmidt u64 min_seq = (u64)-1; 397bd989ba3SJan Schmidt u64 seq_putting = elem->seq; 398bd989ba3SJan Schmidt 399bd989ba3SJan Schmidt if (!seq_putting) 400bd989ba3SJan Schmidt return; 401bd989ba3SJan Schmidt 402bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 403bd989ba3SJan Schmidt list_del(&elem->list); 404097b8a7cSJan Schmidt elem->seq = 0; 405bd989ba3SJan Schmidt 406bd989ba3SJan Schmidt list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) { 407097b8a7cSJan Schmidt if (cur_elem->seq < min_seq) { 408bd989ba3SJan Schmidt if (seq_putting > cur_elem->seq) { 409bd989ba3SJan Schmidt /* 410bd989ba3SJan Schmidt * blocker with lower sequence number exists, we 411bd989ba3SJan Schmidt * cannot remove anything from the log 412bd989ba3SJan Schmidt */ 413097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 414097b8a7cSJan Schmidt return; 415bd989ba3SJan Schmidt } 416bd989ba3SJan Schmidt min_seq = cur_elem->seq; 417bd989ba3SJan Schmidt } 418bd989ba3SJan Schmidt } 419097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 420097b8a7cSJan Schmidt 421097b8a7cSJan Schmidt /* 422bd989ba3SJan Schmidt * anything that's lower than the lowest existing (read: blocked) 423bd989ba3SJan Schmidt * sequence number can be removed from the tree. 424bd989ba3SJan Schmidt */ 425097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 426bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 427bd989ba3SJan Schmidt for (node = rb_first(tm_root); node; node = next) { 428bd989ba3SJan Schmidt next = rb_next(node); 4296b4df8b6SGeliang Tang tm = rb_entry(node, struct tree_mod_elem, node); 430097b8a7cSJan Schmidt if (tm->seq > min_seq) 431bd989ba3SJan Schmidt continue; 432bd989ba3SJan Schmidt rb_erase(node, tm_root); 433bd989ba3SJan Schmidt kfree(tm); 434bd989ba3SJan Schmidt } 435097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 436bd989ba3SJan Schmidt } 437bd989ba3SJan Schmidt 438bd989ba3SJan Schmidt /* 439bd989ba3SJan Schmidt * key order of the log: 440298cfd36SChandan Rajendra * node/leaf start address -> sequence 441bd989ba3SJan Schmidt * 442298cfd36SChandan Rajendra * The 'start address' is the logical address of the *new* root node 443298cfd36SChandan Rajendra * for root replace operations, or the logical address of the affected 444298cfd36SChandan Rajendra * block for all other operations. 4455de865eeSFilipe David Borba Manana * 4465de865eeSFilipe David Borba Manana * Note: must be called with write lock (tree_mod_log_write_lock). 447bd989ba3SJan Schmidt */ 448bd989ba3SJan Schmidt static noinline int 449bd989ba3SJan Schmidt __tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm) 450bd989ba3SJan Schmidt { 451bd989ba3SJan Schmidt struct rb_root *tm_root; 452bd989ba3SJan Schmidt struct rb_node **new; 453bd989ba3SJan Schmidt struct rb_node *parent = NULL; 454bd989ba3SJan Schmidt struct tree_mod_elem *cur; 455bd989ba3SJan Schmidt 456fcebe456SJosef Bacik tm->seq = btrfs_inc_tree_mod_seq(fs_info); 457bd989ba3SJan Schmidt 458bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 459bd989ba3SJan Schmidt new = &tm_root->rb_node; 460bd989ba3SJan Schmidt while (*new) { 4616b4df8b6SGeliang Tang cur = rb_entry(*new, struct tree_mod_elem, node); 462bd989ba3SJan Schmidt parent = *new; 463298cfd36SChandan Rajendra if (cur->logical < tm->logical) 464bd989ba3SJan Schmidt new = &((*new)->rb_left); 465298cfd36SChandan Rajendra else if (cur->logical > tm->logical) 466bd989ba3SJan Schmidt new = &((*new)->rb_right); 467097b8a7cSJan Schmidt else if (cur->seq < tm->seq) 468bd989ba3SJan Schmidt new = &((*new)->rb_left); 469097b8a7cSJan Schmidt else if (cur->seq > tm->seq) 470bd989ba3SJan Schmidt new = &((*new)->rb_right); 4715de865eeSFilipe David Borba Manana else 4725de865eeSFilipe David Borba Manana return -EEXIST; 473bd989ba3SJan Schmidt } 474bd989ba3SJan Schmidt 475bd989ba3SJan Schmidt rb_link_node(&tm->node, parent, new); 476bd989ba3SJan Schmidt rb_insert_color(&tm->node, tm_root); 4775de865eeSFilipe David Borba Manana return 0; 478bd989ba3SJan Schmidt } 479bd989ba3SJan Schmidt 480097b8a7cSJan Schmidt /* 481097b8a7cSJan Schmidt * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it 482097b8a7cSJan Schmidt * returns zero with the tree_mod_log_lock acquired. The caller must hold 483097b8a7cSJan Schmidt * this until all tree mod log insertions are recorded in the rb tree and then 484097b8a7cSJan Schmidt * call tree_mod_log_write_unlock() to release. 485097b8a7cSJan Schmidt */ 486e9b7fd4dSJan Schmidt static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info, 487e9b7fd4dSJan Schmidt struct extent_buffer *eb) { 488e9b7fd4dSJan Schmidt smp_mb(); 489e9b7fd4dSJan Schmidt if (list_empty(&(fs_info)->tree_mod_seq_list)) 490e9b7fd4dSJan Schmidt return 1; 491097b8a7cSJan Schmidt if (eb && btrfs_header_level(eb) == 0) 492e9b7fd4dSJan Schmidt return 1; 4935de865eeSFilipe David Borba Manana 4945de865eeSFilipe David Borba Manana tree_mod_log_write_lock(fs_info); 4955de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) { 4965de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 4975de865eeSFilipe David Borba Manana return 1; 4985de865eeSFilipe David Borba Manana } 4995de865eeSFilipe David Borba Manana 500e9b7fd4dSJan Schmidt return 0; 501e9b7fd4dSJan Schmidt } 502e9b7fd4dSJan Schmidt 5035de865eeSFilipe David Borba Manana /* Similar to tree_mod_dont_log, but doesn't acquire any locks. */ 5045de865eeSFilipe David Borba Manana static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info, 5055de865eeSFilipe David Borba Manana struct extent_buffer *eb) 5065de865eeSFilipe David Borba Manana { 5075de865eeSFilipe David Borba Manana smp_mb(); 5085de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) 5095de865eeSFilipe David Borba Manana return 0; 5105de865eeSFilipe David Borba Manana if (eb && btrfs_header_level(eb) == 0) 5115de865eeSFilipe David Borba Manana return 0; 5125de865eeSFilipe David Borba Manana 5135de865eeSFilipe David Borba Manana return 1; 5145de865eeSFilipe David Borba Manana } 5155de865eeSFilipe David Borba Manana 5165de865eeSFilipe David Borba Manana static struct tree_mod_elem * 5175de865eeSFilipe David Borba Manana alloc_tree_mod_elem(struct extent_buffer *eb, int slot, 518bd989ba3SJan Schmidt enum mod_log_op op, gfp_t flags) 519bd989ba3SJan Schmidt { 520097b8a7cSJan Schmidt struct tree_mod_elem *tm; 521bd989ba3SJan Schmidt 522c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 523c8cc6341SJosef Bacik if (!tm) 5245de865eeSFilipe David Borba Manana return NULL; 525bd989ba3SJan Schmidt 526298cfd36SChandan Rajendra tm->logical = eb->start; 527bd989ba3SJan Schmidt if (op != MOD_LOG_KEY_ADD) { 528bd989ba3SJan Schmidt btrfs_node_key(eb, &tm->key, slot); 529bd989ba3SJan Schmidt tm->blockptr = btrfs_node_blockptr(eb, slot); 530bd989ba3SJan Schmidt } 531bd989ba3SJan Schmidt tm->op = op; 532bd989ba3SJan Schmidt tm->slot = slot; 533bd989ba3SJan Schmidt tm->generation = btrfs_node_ptr_generation(eb, slot); 5345de865eeSFilipe David Borba Manana RB_CLEAR_NODE(&tm->node); 535bd989ba3SJan Schmidt 5365de865eeSFilipe David Borba Manana return tm; 537097b8a7cSJan Schmidt } 538097b8a7cSJan Schmidt 539097b8a7cSJan Schmidt static noinline int 540c8cc6341SJosef Bacik tree_mod_log_insert_key(struct btrfs_fs_info *fs_info, 541097b8a7cSJan Schmidt struct extent_buffer *eb, int slot, 542097b8a7cSJan Schmidt enum mod_log_op op, gfp_t flags) 543097b8a7cSJan Schmidt { 5445de865eeSFilipe David Borba Manana struct tree_mod_elem *tm; 5455de865eeSFilipe David Borba Manana int ret; 5465de865eeSFilipe David Borba Manana 5475de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, eb)) 548097b8a7cSJan Schmidt return 0; 549097b8a7cSJan Schmidt 5505de865eeSFilipe David Borba Manana tm = alloc_tree_mod_elem(eb, slot, op, flags); 5515de865eeSFilipe David Borba Manana if (!tm) 5525de865eeSFilipe David Borba Manana return -ENOMEM; 5535de865eeSFilipe David Borba Manana 5545de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, eb)) { 5555de865eeSFilipe David Borba Manana kfree(tm); 5565de865eeSFilipe David Borba Manana return 0; 5575de865eeSFilipe David Borba Manana } 5585de865eeSFilipe David Borba Manana 5595de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 5605de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 5615de865eeSFilipe David Borba Manana if (ret) 5625de865eeSFilipe David Borba Manana kfree(tm); 5635de865eeSFilipe David Borba Manana 5645de865eeSFilipe David Borba Manana return ret; 565097b8a7cSJan Schmidt } 566097b8a7cSJan Schmidt 567097b8a7cSJan Schmidt static noinline int 568bd989ba3SJan Schmidt tree_mod_log_insert_move(struct btrfs_fs_info *fs_info, 569bd989ba3SJan Schmidt struct extent_buffer *eb, int dst_slot, int src_slot, 570176ef8f5SDavid Sterba int nr_items) 571bd989ba3SJan Schmidt { 5725de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 5735de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 5745de865eeSFilipe David Borba Manana int ret = 0; 575bd989ba3SJan Schmidt int i; 5765de865eeSFilipe David Borba Manana int locked = 0; 577bd989ba3SJan Schmidt 5785de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, eb)) 579f395694cSJan Schmidt return 0; 580bd989ba3SJan Schmidt 581176ef8f5SDavid Sterba tm_list = kcalloc(nr_items, sizeof(struct tree_mod_elem *), GFP_NOFS); 5825de865eeSFilipe David Borba Manana if (!tm_list) 5835de865eeSFilipe David Borba Manana return -ENOMEM; 584bd989ba3SJan Schmidt 585176ef8f5SDavid Sterba tm = kzalloc(sizeof(*tm), GFP_NOFS); 5865de865eeSFilipe David Borba Manana if (!tm) { 5875de865eeSFilipe David Borba Manana ret = -ENOMEM; 5885de865eeSFilipe David Borba Manana goto free_tms; 5895de865eeSFilipe David Borba Manana } 590f395694cSJan Schmidt 591298cfd36SChandan Rajendra tm->logical = eb->start; 592bd989ba3SJan Schmidt tm->slot = src_slot; 593bd989ba3SJan Schmidt tm->move.dst_slot = dst_slot; 594bd989ba3SJan Schmidt tm->move.nr_items = nr_items; 595bd989ba3SJan Schmidt tm->op = MOD_LOG_MOVE_KEYS; 596bd989ba3SJan Schmidt 5975de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 5985de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot, 599176ef8f5SDavid Sterba MOD_LOG_KEY_REMOVE_WHILE_MOVING, GFP_NOFS); 6005de865eeSFilipe David Borba Manana if (!tm_list[i]) { 6015de865eeSFilipe David Borba Manana ret = -ENOMEM; 6025de865eeSFilipe David Borba Manana goto free_tms; 6035de865eeSFilipe David Borba Manana } 604bd989ba3SJan Schmidt } 605bd989ba3SJan Schmidt 6065de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, eb)) 6075de865eeSFilipe David Borba Manana goto free_tms; 6085de865eeSFilipe David Borba Manana locked = 1; 6095de865eeSFilipe David Borba Manana 6105de865eeSFilipe David Borba Manana /* 6115de865eeSFilipe David Borba Manana * When we override something during the move, we log these removals. 6125de865eeSFilipe David Borba Manana * This can only happen when we move towards the beginning of the 6135de865eeSFilipe David Borba Manana * buffer, i.e. dst_slot < src_slot. 6145de865eeSFilipe David Borba Manana */ 6155de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 6165de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list[i]); 6175de865eeSFilipe David Borba Manana if (ret) 6185de865eeSFilipe David Borba Manana goto free_tms; 6195de865eeSFilipe David Borba Manana } 6205de865eeSFilipe David Borba Manana 6215de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 6225de865eeSFilipe David Borba Manana if (ret) 6235de865eeSFilipe David Borba Manana goto free_tms; 6245de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 6255de865eeSFilipe David Borba Manana kfree(tm_list); 6265de865eeSFilipe David Borba Manana 6275de865eeSFilipe David Borba Manana return 0; 6285de865eeSFilipe David Borba Manana free_tms: 6295de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 6305de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 6315de865eeSFilipe David Borba Manana rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log); 6325de865eeSFilipe David Borba Manana kfree(tm_list[i]); 6335de865eeSFilipe David Borba Manana } 6345de865eeSFilipe David Borba Manana if (locked) 6355de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 6365de865eeSFilipe David Borba Manana kfree(tm_list); 6375de865eeSFilipe David Borba Manana kfree(tm); 6385de865eeSFilipe David Borba Manana 6395de865eeSFilipe David Borba Manana return ret; 6405de865eeSFilipe David Borba Manana } 6415de865eeSFilipe David Borba Manana 6425de865eeSFilipe David Borba Manana static inline int 6435de865eeSFilipe David Borba Manana __tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 6445de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list, 6455de865eeSFilipe David Borba Manana int nritems) 646097b8a7cSJan Schmidt { 6475de865eeSFilipe David Borba Manana int i, j; 648097b8a7cSJan Schmidt int ret; 649097b8a7cSJan Schmidt 650097b8a7cSJan Schmidt for (i = nritems - 1; i >= 0; i--) { 6515de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list[i]); 6525de865eeSFilipe David Borba Manana if (ret) { 6535de865eeSFilipe David Borba Manana for (j = nritems - 1; j > i; j--) 6545de865eeSFilipe David Borba Manana rb_erase(&tm_list[j]->node, 6555de865eeSFilipe David Borba Manana &fs_info->tree_mod_log); 6565de865eeSFilipe David Borba Manana return ret; 657097b8a7cSJan Schmidt } 658097b8a7cSJan Schmidt } 659097b8a7cSJan Schmidt 6605de865eeSFilipe David Borba Manana return 0; 6615de865eeSFilipe David Borba Manana } 6625de865eeSFilipe David Borba Manana 663bd989ba3SJan Schmidt static noinline int 664bd989ba3SJan Schmidt tree_mod_log_insert_root(struct btrfs_fs_info *fs_info, 665bd989ba3SJan Schmidt struct extent_buffer *old_root, 666bcc8e07fSDavid Sterba struct extent_buffer *new_root, 66790f8d62eSJan Schmidt int log_removal) 668bd989ba3SJan Schmidt { 6695de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 6705de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 6715de865eeSFilipe David Borba Manana int nritems = 0; 6725de865eeSFilipe David Borba Manana int ret = 0; 6735de865eeSFilipe David Borba Manana int i; 674bd989ba3SJan Schmidt 6755de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 676097b8a7cSJan Schmidt return 0; 677097b8a7cSJan Schmidt 6785de865eeSFilipe David Borba Manana if (log_removal && btrfs_header_level(old_root) > 0) { 6795de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(old_root); 68031e818feSDavid Sterba tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), 681bcc8e07fSDavid Sterba GFP_NOFS); 6825de865eeSFilipe David Borba Manana if (!tm_list) { 6835de865eeSFilipe David Borba Manana ret = -ENOMEM; 6845de865eeSFilipe David Borba Manana goto free_tms; 6855de865eeSFilipe David Borba Manana } 6865de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 6875de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(old_root, i, 688bcc8e07fSDavid Sterba MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 6895de865eeSFilipe David Borba Manana if (!tm_list[i]) { 6905de865eeSFilipe David Borba Manana ret = -ENOMEM; 6915de865eeSFilipe David Borba Manana goto free_tms; 6925de865eeSFilipe David Borba Manana } 6935de865eeSFilipe David Borba Manana } 6945de865eeSFilipe David Borba Manana } 695d9abbf1cSJan Schmidt 696bcc8e07fSDavid Sterba tm = kzalloc(sizeof(*tm), GFP_NOFS); 6975de865eeSFilipe David Borba Manana if (!tm) { 6985de865eeSFilipe David Borba Manana ret = -ENOMEM; 6995de865eeSFilipe David Borba Manana goto free_tms; 7005de865eeSFilipe David Borba Manana } 701bd989ba3SJan Schmidt 702298cfd36SChandan Rajendra tm->logical = new_root->start; 703bd989ba3SJan Schmidt tm->old_root.logical = old_root->start; 704bd989ba3SJan Schmidt tm->old_root.level = btrfs_header_level(old_root); 705bd989ba3SJan Schmidt tm->generation = btrfs_header_generation(old_root); 706bd989ba3SJan Schmidt tm->op = MOD_LOG_ROOT_REPLACE; 707bd989ba3SJan Schmidt 7085de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 7095de865eeSFilipe David Borba Manana goto free_tms; 7105de865eeSFilipe David Borba Manana 7115de865eeSFilipe David Borba Manana if (tm_list) 7125de865eeSFilipe David Borba Manana ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems); 7135de865eeSFilipe David Borba Manana if (!ret) 7145de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 7155de865eeSFilipe David Borba Manana 7165de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 7175de865eeSFilipe David Borba Manana if (ret) 7185de865eeSFilipe David Borba Manana goto free_tms; 7195de865eeSFilipe David Borba Manana kfree(tm_list); 7205de865eeSFilipe David Borba Manana 7215de865eeSFilipe David Borba Manana return ret; 7225de865eeSFilipe David Borba Manana 7235de865eeSFilipe David Borba Manana free_tms: 7245de865eeSFilipe David Borba Manana if (tm_list) { 7255de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 7265de865eeSFilipe David Borba Manana kfree(tm_list[i]); 7275de865eeSFilipe David Borba Manana kfree(tm_list); 7285de865eeSFilipe David Borba Manana } 7295de865eeSFilipe David Borba Manana kfree(tm); 7305de865eeSFilipe David Borba Manana 7315de865eeSFilipe David Borba Manana return ret; 732bd989ba3SJan Schmidt } 733bd989ba3SJan Schmidt 734bd989ba3SJan Schmidt static struct tree_mod_elem * 735bd989ba3SJan Schmidt __tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq, 736bd989ba3SJan Schmidt int smallest) 737bd989ba3SJan Schmidt { 738bd989ba3SJan Schmidt struct rb_root *tm_root; 739bd989ba3SJan Schmidt struct rb_node *node; 740bd989ba3SJan Schmidt struct tree_mod_elem *cur = NULL; 741bd989ba3SJan Schmidt struct tree_mod_elem *found = NULL; 742bd989ba3SJan Schmidt 743097b8a7cSJan Schmidt tree_mod_log_read_lock(fs_info); 744bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 745bd989ba3SJan Schmidt node = tm_root->rb_node; 746bd989ba3SJan Schmidt while (node) { 7476b4df8b6SGeliang Tang cur = rb_entry(node, struct tree_mod_elem, node); 748298cfd36SChandan Rajendra if (cur->logical < start) { 749bd989ba3SJan Schmidt node = node->rb_left; 750298cfd36SChandan Rajendra } else if (cur->logical > start) { 751bd989ba3SJan Schmidt node = node->rb_right; 752097b8a7cSJan Schmidt } else if (cur->seq < min_seq) { 753bd989ba3SJan Schmidt node = node->rb_left; 754bd989ba3SJan Schmidt } else if (!smallest) { 755bd989ba3SJan Schmidt /* we want the node with the highest seq */ 756bd989ba3SJan Schmidt if (found) 757097b8a7cSJan Schmidt BUG_ON(found->seq > cur->seq); 758bd989ba3SJan Schmidt found = cur; 759bd989ba3SJan Schmidt node = node->rb_left; 760097b8a7cSJan Schmidt } else if (cur->seq > min_seq) { 761bd989ba3SJan Schmidt /* we want the node with the smallest seq */ 762bd989ba3SJan Schmidt if (found) 763097b8a7cSJan Schmidt BUG_ON(found->seq < cur->seq); 764bd989ba3SJan Schmidt found = cur; 765bd989ba3SJan Schmidt node = node->rb_right; 766bd989ba3SJan Schmidt } else { 767bd989ba3SJan Schmidt found = cur; 768bd989ba3SJan Schmidt break; 769bd989ba3SJan Schmidt } 770bd989ba3SJan Schmidt } 771097b8a7cSJan Schmidt tree_mod_log_read_unlock(fs_info); 772bd989ba3SJan Schmidt 773bd989ba3SJan Schmidt return found; 774bd989ba3SJan Schmidt } 775bd989ba3SJan Schmidt 776bd989ba3SJan Schmidt /* 777bd989ba3SJan Schmidt * this returns the element from the log with the smallest time sequence 778bd989ba3SJan Schmidt * value that's in the log (the oldest log item). any element with a time 779bd989ba3SJan Schmidt * sequence lower than min_seq will be ignored. 780bd989ba3SJan Schmidt */ 781bd989ba3SJan Schmidt static struct tree_mod_elem * 782bd989ba3SJan Schmidt tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start, 783bd989ba3SJan Schmidt u64 min_seq) 784bd989ba3SJan Schmidt { 785bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 1); 786bd989ba3SJan Schmidt } 787bd989ba3SJan Schmidt 788bd989ba3SJan Schmidt /* 789bd989ba3SJan Schmidt * this returns the element from the log with the largest time sequence 790bd989ba3SJan Schmidt * value that's in the log (the most recent log item). any element with 791bd989ba3SJan Schmidt * a time sequence lower than min_seq will be ignored. 792bd989ba3SJan Schmidt */ 793bd989ba3SJan Schmidt static struct tree_mod_elem * 794bd989ba3SJan Schmidt tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq) 795bd989ba3SJan Schmidt { 796bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 0); 797bd989ba3SJan Schmidt } 798bd989ba3SJan Schmidt 7995de865eeSFilipe David Borba Manana static noinline int 800bd989ba3SJan Schmidt tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 801bd989ba3SJan Schmidt struct extent_buffer *src, unsigned long dst_offset, 80290f8d62eSJan Schmidt unsigned long src_offset, int nr_items) 803bd989ba3SJan Schmidt { 8045de865eeSFilipe David Borba Manana int ret = 0; 8055de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 8065de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list_add, **tm_list_rem; 807bd989ba3SJan Schmidt int i; 8085de865eeSFilipe David Borba Manana int locked = 0; 809bd989ba3SJan Schmidt 8105de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 8115de865eeSFilipe David Borba Manana return 0; 812bd989ba3SJan Schmidt 813c8cc6341SJosef Bacik if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0) 8145de865eeSFilipe David Borba Manana return 0; 8155de865eeSFilipe David Borba Manana 81631e818feSDavid Sterba tm_list = kcalloc(nr_items * 2, sizeof(struct tree_mod_elem *), 8175de865eeSFilipe David Borba Manana GFP_NOFS); 8185de865eeSFilipe David Borba Manana if (!tm_list) 8195de865eeSFilipe David Borba Manana return -ENOMEM; 8205de865eeSFilipe David Borba Manana 8215de865eeSFilipe David Borba Manana tm_list_add = tm_list; 8225de865eeSFilipe David Borba Manana tm_list_rem = tm_list + nr_items; 8235de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 8245de865eeSFilipe David Borba Manana tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset, 8255de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE, GFP_NOFS); 8265de865eeSFilipe David Borba Manana if (!tm_list_rem[i]) { 8275de865eeSFilipe David Borba Manana ret = -ENOMEM; 8285de865eeSFilipe David Borba Manana goto free_tms; 8295de865eeSFilipe David Borba Manana } 8305de865eeSFilipe David Borba Manana 8315de865eeSFilipe David Borba Manana tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset, 8325de865eeSFilipe David Borba Manana MOD_LOG_KEY_ADD, GFP_NOFS); 8335de865eeSFilipe David Borba Manana if (!tm_list_add[i]) { 8345de865eeSFilipe David Borba Manana ret = -ENOMEM; 8355de865eeSFilipe David Borba Manana goto free_tms; 8365de865eeSFilipe David Borba Manana } 8375de865eeSFilipe David Borba Manana } 8385de865eeSFilipe David Borba Manana 8395de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 8405de865eeSFilipe David Borba Manana goto free_tms; 8415de865eeSFilipe David Borba Manana locked = 1; 842bd989ba3SJan Schmidt 843bd989ba3SJan Schmidt for (i = 0; i < nr_items; i++) { 8445de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]); 8455de865eeSFilipe David Borba Manana if (ret) 8465de865eeSFilipe David Borba Manana goto free_tms; 8475de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_add[i]); 8485de865eeSFilipe David Borba Manana if (ret) 8495de865eeSFilipe David Borba Manana goto free_tms; 850bd989ba3SJan Schmidt } 8515de865eeSFilipe David Borba Manana 8525de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 8535de865eeSFilipe David Borba Manana kfree(tm_list); 8545de865eeSFilipe David Borba Manana 8555de865eeSFilipe David Borba Manana return 0; 8565de865eeSFilipe David Borba Manana 8575de865eeSFilipe David Borba Manana free_tms: 8585de865eeSFilipe David Borba Manana for (i = 0; i < nr_items * 2; i++) { 8595de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 8605de865eeSFilipe David Borba Manana rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log); 8615de865eeSFilipe David Borba Manana kfree(tm_list[i]); 8625de865eeSFilipe David Borba Manana } 8635de865eeSFilipe David Borba Manana if (locked) 8645de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 8655de865eeSFilipe David Borba Manana kfree(tm_list); 8665de865eeSFilipe David Borba Manana 8675de865eeSFilipe David Borba Manana return ret; 868bd989ba3SJan Schmidt } 869bd989ba3SJan Schmidt 870bd989ba3SJan Schmidt static inline void 871bd989ba3SJan Schmidt tree_mod_log_eb_move(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 872bd989ba3SJan Schmidt int dst_offset, int src_offset, int nr_items) 873bd989ba3SJan Schmidt { 874bd989ba3SJan Schmidt int ret; 875bd989ba3SJan Schmidt ret = tree_mod_log_insert_move(fs_info, dst, dst_offset, src_offset, 876176ef8f5SDavid Sterba nr_items); 877bd989ba3SJan Schmidt BUG_ON(ret < 0); 878bd989ba3SJan Schmidt } 879bd989ba3SJan Schmidt 880097b8a7cSJan Schmidt static noinline void 881bd989ba3SJan Schmidt tree_mod_log_set_node_key(struct btrfs_fs_info *fs_info, 88232adf090SLiu Bo struct extent_buffer *eb, int slot, int atomic) 883bd989ba3SJan Schmidt { 884bd989ba3SJan Schmidt int ret; 885bd989ba3SJan Schmidt 88678357766SFilipe David Borba Manana ret = tree_mod_log_insert_key(fs_info, eb, slot, 887bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 888bd989ba3SJan Schmidt atomic ? GFP_ATOMIC : GFP_NOFS); 889bd989ba3SJan Schmidt BUG_ON(ret < 0); 890bd989ba3SJan Schmidt } 891bd989ba3SJan Schmidt 8925de865eeSFilipe David Borba Manana static noinline int 893097b8a7cSJan Schmidt tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb) 894bd989ba3SJan Schmidt { 8955de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 8965de865eeSFilipe David Borba Manana int nritems = 0; 8975de865eeSFilipe David Borba Manana int i; 8985de865eeSFilipe David Borba Manana int ret = 0; 8995de865eeSFilipe David Borba Manana 9005de865eeSFilipe David Borba Manana if (btrfs_header_level(eb) == 0) 9015de865eeSFilipe David Borba Manana return 0; 9025de865eeSFilipe David Borba Manana 9035de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 9045de865eeSFilipe David Borba Manana return 0; 9055de865eeSFilipe David Borba Manana 9065de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(eb); 90731e818feSDavid Sterba tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), GFP_NOFS); 9085de865eeSFilipe David Borba Manana if (!tm_list) 9095de865eeSFilipe David Borba Manana return -ENOMEM; 9105de865eeSFilipe David Borba Manana 9115de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 9125de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i, 9135de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 9145de865eeSFilipe David Borba Manana if (!tm_list[i]) { 9155de865eeSFilipe David Borba Manana ret = -ENOMEM; 9165de865eeSFilipe David Borba Manana goto free_tms; 9175de865eeSFilipe David Borba Manana } 9185de865eeSFilipe David Borba Manana } 9195de865eeSFilipe David Borba Manana 920e9b7fd4dSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 9215de865eeSFilipe David Borba Manana goto free_tms; 9225de865eeSFilipe David Borba Manana 9235de865eeSFilipe David Borba Manana ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems); 9245de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 9255de865eeSFilipe David Borba Manana if (ret) 9265de865eeSFilipe David Borba Manana goto free_tms; 9275de865eeSFilipe David Borba Manana kfree(tm_list); 9285de865eeSFilipe David Borba Manana 9295de865eeSFilipe David Borba Manana return 0; 9305de865eeSFilipe David Borba Manana 9315de865eeSFilipe David Borba Manana free_tms: 9325de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 9335de865eeSFilipe David Borba Manana kfree(tm_list[i]); 9345de865eeSFilipe David Borba Manana kfree(tm_list); 9355de865eeSFilipe David Borba Manana 9365de865eeSFilipe David Borba Manana return ret; 937bd989ba3SJan Schmidt } 938bd989ba3SJan Schmidt 939097b8a7cSJan Schmidt static noinline void 940bd989ba3SJan Schmidt tree_mod_log_set_root_pointer(struct btrfs_root *root, 94190f8d62eSJan Schmidt struct extent_buffer *new_root_node, 94290f8d62eSJan Schmidt int log_removal) 943bd989ba3SJan Schmidt { 944bd989ba3SJan Schmidt int ret; 945bd989ba3SJan Schmidt ret = tree_mod_log_insert_root(root->fs_info, root->node, 946bcc8e07fSDavid Sterba new_root_node, log_removal); 947bd989ba3SJan Schmidt BUG_ON(ret < 0); 948bd989ba3SJan Schmidt } 949bd989ba3SJan Schmidt 950d352ac68SChris Mason /* 9515d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 9525d4f98a2SYan Zheng */ 9535d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 9545d4f98a2SYan Zheng struct extent_buffer *buf) 9555d4f98a2SYan Zheng { 9565d4f98a2SYan Zheng /* 95701327610SNicholas D Steeves * Tree blocks not in reference counted trees and tree roots 9585d4f98a2SYan Zheng * are never shared. If a block was allocated after the last 9595d4f98a2SYan Zheng * snapshot and the block was not allocated by tree relocation, 9605d4f98a2SYan Zheng * we know the block is not shared. 9615d4f98a2SYan Zheng */ 96227cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 9635d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 9645d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 9655d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 9665d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 9675d4f98a2SYan Zheng return 1; 9685d4f98a2SYan Zheng #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 96927cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 9705d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 9715d4f98a2SYan Zheng return 1; 9725d4f98a2SYan Zheng #endif 9735d4f98a2SYan Zheng return 0; 9745d4f98a2SYan Zheng } 9755d4f98a2SYan Zheng 9765d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 9775d4f98a2SYan Zheng struct btrfs_root *root, 9785d4f98a2SYan Zheng struct extent_buffer *buf, 979f0486c68SYan, Zheng struct extent_buffer *cow, 980f0486c68SYan, Zheng int *last_ref) 9815d4f98a2SYan Zheng { 9820b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 9835d4f98a2SYan Zheng u64 refs; 9845d4f98a2SYan Zheng u64 owner; 9855d4f98a2SYan Zheng u64 flags; 9865d4f98a2SYan Zheng u64 new_flags = 0; 9875d4f98a2SYan Zheng int ret; 9885d4f98a2SYan Zheng 9895d4f98a2SYan Zheng /* 9905d4f98a2SYan Zheng * Backrefs update rules: 9915d4f98a2SYan Zheng * 9925d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 9935d4f98a2SYan Zheng * allocated by tree relocation. 9945d4f98a2SYan Zheng * 9955d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 9965d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 9975d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 9985d4f98a2SYan Zheng * 9995d4f98a2SYan Zheng * If a tree block is been relocating 10005d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 10015d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 10025d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 10035d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 10045d4f98a2SYan Zheng */ 10055d4f98a2SYan Zheng 10065d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 10072ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 10083173a18fSJosef Bacik btrfs_header_level(buf), 1, 10093173a18fSJosef Bacik &refs, &flags); 1010be1a5564SMark Fasheh if (ret) 1011be1a5564SMark Fasheh return ret; 1012e5df9573SMark Fasheh if (refs == 0) { 1013e5df9573SMark Fasheh ret = -EROFS; 10140b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1015e5df9573SMark Fasheh return ret; 1016e5df9573SMark Fasheh } 10175d4f98a2SYan Zheng } else { 10185d4f98a2SYan Zheng refs = 1; 10195d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 10205d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 10215d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 10225d4f98a2SYan Zheng else 10235d4f98a2SYan Zheng flags = 0; 10245d4f98a2SYan Zheng } 10255d4f98a2SYan Zheng 10265d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 10275d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 10285d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 10295d4f98a2SYan Zheng 10305d4f98a2SYan Zheng if (refs > 1) { 10315d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 10325d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 10335d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 1034e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 1035692826b2SJeff Mahoney if (ret) 1036692826b2SJeff Mahoney return ret; 10375d4f98a2SYan Zheng 10385d4f98a2SYan Zheng if (root->root_key.objectid == 10395d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 1040e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 1041692826b2SJeff Mahoney if (ret) 1042692826b2SJeff Mahoney return ret; 1043e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 1044692826b2SJeff Mahoney if (ret) 1045692826b2SJeff Mahoney return ret; 10465d4f98a2SYan Zheng } 10475d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 10485d4f98a2SYan Zheng } else { 10495d4f98a2SYan Zheng 10505d4f98a2SYan Zheng if (root->root_key.objectid == 10515d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 1052e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 10535d4f98a2SYan Zheng else 1054e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 1055692826b2SJeff Mahoney if (ret) 1056692826b2SJeff Mahoney return ret; 10575d4f98a2SYan Zheng } 10585d4f98a2SYan Zheng if (new_flags != 0) { 1059b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 1060b1c79e09SJosef Bacik 10612ff7e61eSJeff Mahoney ret = btrfs_set_disk_extent_flags(trans, fs_info, 10625d4f98a2SYan Zheng buf->start, 10635d4f98a2SYan Zheng buf->len, 1064b1c79e09SJosef Bacik new_flags, level, 0); 1065be1a5564SMark Fasheh if (ret) 1066be1a5564SMark Fasheh return ret; 10675d4f98a2SYan Zheng } 10685d4f98a2SYan Zheng } else { 10695d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 10705d4f98a2SYan Zheng if (root->root_key.objectid == 10715d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 1072e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 10735d4f98a2SYan Zheng else 1074e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 1075692826b2SJeff Mahoney if (ret) 1076692826b2SJeff Mahoney return ret; 1077e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 1078692826b2SJeff Mahoney if (ret) 1079692826b2SJeff Mahoney return ret; 10805d4f98a2SYan Zheng } 10817c302b49SDavid Sterba clean_tree_block(fs_info, buf); 1082f0486c68SYan, Zheng *last_ref = 1; 10835d4f98a2SYan Zheng } 10845d4f98a2SYan Zheng return 0; 10855d4f98a2SYan Zheng } 10865d4f98a2SYan Zheng 10875d4f98a2SYan Zheng /* 1088d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 1089d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 1090d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 1091d397712bSChris Mason * dirty again. 1092d352ac68SChris Mason * 1093d352ac68SChris Mason * search_start -- an allocation hint for the new block 1094d352ac68SChris Mason * 1095d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 1096d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 1097d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 1098d352ac68SChris Mason */ 1099d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 11005f39d397SChris Mason struct btrfs_root *root, 11015f39d397SChris Mason struct extent_buffer *buf, 11025f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 11035f39d397SChris Mason struct extent_buffer **cow_ret, 11049fa8cfe7SChris Mason u64 search_start, u64 empty_size) 11056702ed49SChris Mason { 11060b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 11075d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 11085f39d397SChris Mason struct extent_buffer *cow; 1109be1a5564SMark Fasheh int level, ret; 1110f0486c68SYan, Zheng int last_ref = 0; 1111925baeddSChris Mason int unlock_orig = 0; 11120f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 11136702ed49SChris Mason 1114925baeddSChris Mason if (*cow_ret == buf) 1115925baeddSChris Mason unlock_orig = 1; 1116925baeddSChris Mason 1117b9447ef8SChris Mason btrfs_assert_tree_locked(buf); 1118925baeddSChris Mason 111927cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 11200b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 112127cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 112227cdeb70SMiao Xie trans->transid != root->last_trans); 11235f39d397SChris Mason 11247bb86316SChris Mason level = btrfs_header_level(buf); 112531840ae1SZheng Yan 11265d4f98a2SYan Zheng if (level == 0) 11275d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 11285d4f98a2SYan Zheng else 11295d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 11305d4f98a2SYan Zheng 11310f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 11325d4f98a2SYan Zheng parent_start = parent->start; 11335d4f98a2SYan Zheng 11344d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, parent_start, 11354d75f8a9SDavid Sterba root->root_key.objectid, &disk_key, level, 11364d75f8a9SDavid Sterba search_start, empty_size); 11376702ed49SChris Mason if (IS_ERR(cow)) 11386702ed49SChris Mason return PTR_ERR(cow); 11396702ed49SChris Mason 1140b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 1141b4ce94deSChris Mason 114258e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 1143db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 11445f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 11455d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 11465d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 11475d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 11485d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 11495d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 11505d4f98a2SYan Zheng else 11515f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 11526702ed49SChris Mason 11530b246afaSJeff Mahoney write_extent_buffer_fsid(cow, fs_info->fsid); 11542b82032cSYan Zheng 1155be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 1156b68dc2a9SMark Fasheh if (ret) { 115766642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 1158b68dc2a9SMark Fasheh return ret; 1159b68dc2a9SMark Fasheh } 11601a40e23bSZheng Yan 116127cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) { 116283d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 116393314e3bSZhaolei if (ret) { 116466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 116583d4cfd4SJosef Bacik return ret; 116683d4cfd4SJosef Bacik } 116793314e3bSZhaolei } 11683fd0a558SYan, Zheng 11696702ed49SChris Mason if (buf == root->node) { 1170925baeddSChris Mason WARN_ON(parent && parent != buf); 11715d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 11725d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 11735d4f98a2SYan Zheng parent_start = buf->start; 1174925baeddSChris Mason 11755f39d397SChris Mason extent_buffer_get(cow); 117690f8d62eSJan Schmidt tree_mod_log_set_root_pointer(root, cow, 1); 1177240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 1178925baeddSChris Mason 1179f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 11805581a51aSJan Schmidt last_ref); 11815f39d397SChris Mason free_extent_buffer(buf); 11820b86a832SChris Mason add_root_to_dirty_list(root); 11836702ed49SChris Mason } else { 11845d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 11850b246afaSJeff Mahoney tree_mod_log_insert_key(fs_info, parent, parent_slot, 1186c8cc6341SJosef Bacik MOD_LOG_KEY_REPLACE, GFP_NOFS); 11875f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 1188db94535dSChris Mason cow->start); 118974493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 119074493f7aSChris Mason trans->transid); 11916702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 11925de865eeSFilipe David Borba Manana if (last_ref) { 11930b246afaSJeff Mahoney ret = tree_mod_log_free_eb(fs_info, buf); 11945de865eeSFilipe David Borba Manana if (ret) { 119566642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 11965de865eeSFilipe David Borba Manana return ret; 11975de865eeSFilipe David Borba Manana } 11985de865eeSFilipe David Borba Manana } 1199f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 12005581a51aSJan Schmidt last_ref); 12016702ed49SChris Mason } 1202925baeddSChris Mason if (unlock_orig) 1203925baeddSChris Mason btrfs_tree_unlock(buf); 12043083ee2eSJosef Bacik free_extent_buffer_stale(buf); 12056702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 12066702ed49SChris Mason *cow_ret = cow; 12076702ed49SChris Mason return 0; 12086702ed49SChris Mason } 12096702ed49SChris Mason 12105d9e75c4SJan Schmidt /* 12115d9e75c4SJan Schmidt * returns the logical address of the oldest predecessor of the given root. 12125d9e75c4SJan Schmidt * entries older than time_seq are ignored. 12135d9e75c4SJan Schmidt */ 12145d9e75c4SJan Schmidt static struct tree_mod_elem * 12155d9e75c4SJan Schmidt __tree_mod_log_oldest_root(struct btrfs_fs_info *fs_info, 121630b0463aSJan Schmidt struct extent_buffer *eb_root, u64 time_seq) 12175d9e75c4SJan Schmidt { 12185d9e75c4SJan Schmidt struct tree_mod_elem *tm; 12195d9e75c4SJan Schmidt struct tree_mod_elem *found = NULL; 122030b0463aSJan Schmidt u64 root_logical = eb_root->start; 12215d9e75c4SJan Schmidt int looped = 0; 12225d9e75c4SJan Schmidt 12235d9e75c4SJan Schmidt if (!time_seq) 122435a3621bSStefan Behrens return NULL; 12255d9e75c4SJan Schmidt 12265d9e75c4SJan Schmidt /* 1227298cfd36SChandan Rajendra * the very last operation that's logged for a root is the 1228298cfd36SChandan Rajendra * replacement operation (if it is replaced at all). this has 1229298cfd36SChandan Rajendra * the logical address of the *new* root, making it the very 1230298cfd36SChandan Rajendra * first operation that's logged for this root. 12315d9e75c4SJan Schmidt */ 12325d9e75c4SJan Schmidt while (1) { 12335d9e75c4SJan Schmidt tm = tree_mod_log_search_oldest(fs_info, root_logical, 12345d9e75c4SJan Schmidt time_seq); 12355d9e75c4SJan Schmidt if (!looped && !tm) 123635a3621bSStefan Behrens return NULL; 12375d9e75c4SJan Schmidt /* 123828da9fb4SJan Schmidt * if there are no tree operation for the oldest root, we simply 123928da9fb4SJan Schmidt * return it. this should only happen if that (old) root is at 124028da9fb4SJan Schmidt * level 0. 12415d9e75c4SJan Schmidt */ 124228da9fb4SJan Schmidt if (!tm) 124328da9fb4SJan Schmidt break; 12445d9e75c4SJan Schmidt 124528da9fb4SJan Schmidt /* 124628da9fb4SJan Schmidt * if there's an operation that's not a root replacement, we 124728da9fb4SJan Schmidt * found the oldest version of our root. normally, we'll find a 124828da9fb4SJan Schmidt * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here. 124928da9fb4SJan Schmidt */ 12505d9e75c4SJan Schmidt if (tm->op != MOD_LOG_ROOT_REPLACE) 12515d9e75c4SJan Schmidt break; 12525d9e75c4SJan Schmidt 12535d9e75c4SJan Schmidt found = tm; 12545d9e75c4SJan Schmidt root_logical = tm->old_root.logical; 12555d9e75c4SJan Schmidt looped = 1; 12565d9e75c4SJan Schmidt } 12575d9e75c4SJan Schmidt 1258a95236d9SJan Schmidt /* if there's no old root to return, return what we found instead */ 1259a95236d9SJan Schmidt if (!found) 1260a95236d9SJan Schmidt found = tm; 1261a95236d9SJan Schmidt 12625d9e75c4SJan Schmidt return found; 12635d9e75c4SJan Schmidt } 12645d9e75c4SJan Schmidt 12655d9e75c4SJan Schmidt /* 12665d9e75c4SJan Schmidt * tm is a pointer to the first operation to rewind within eb. then, all 126701327610SNicholas D Steeves * previous operations will be rewound (until we reach something older than 12685d9e75c4SJan Schmidt * time_seq). 12695d9e75c4SJan Schmidt */ 12705d9e75c4SJan Schmidt static void 1271f1ca7e98SJosef Bacik __tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 1272f1ca7e98SJosef Bacik u64 time_seq, struct tree_mod_elem *first_tm) 12735d9e75c4SJan Schmidt { 12745d9e75c4SJan Schmidt u32 n; 12755d9e75c4SJan Schmidt struct rb_node *next; 12765d9e75c4SJan Schmidt struct tree_mod_elem *tm = first_tm; 12775d9e75c4SJan Schmidt unsigned long o_dst; 12785d9e75c4SJan Schmidt unsigned long o_src; 12795d9e75c4SJan Schmidt unsigned long p_size = sizeof(struct btrfs_key_ptr); 12805d9e75c4SJan Schmidt 12815d9e75c4SJan Schmidt n = btrfs_header_nritems(eb); 1282f1ca7e98SJosef Bacik tree_mod_log_read_lock(fs_info); 1283097b8a7cSJan Schmidt while (tm && tm->seq >= time_seq) { 12845d9e75c4SJan Schmidt /* 12855d9e75c4SJan Schmidt * all the operations are recorded with the operator used for 12865d9e75c4SJan Schmidt * the modification. as we're going backwards, we do the 12875d9e75c4SJan Schmidt * opposite of each operation here. 12885d9e75c4SJan Schmidt */ 12895d9e75c4SJan Schmidt switch (tm->op) { 12905d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE_WHILE_FREEING: 12915d9e75c4SJan Schmidt BUG_ON(tm->slot < n); 12921c697d4aSEric Sandeen /* Fallthrough */ 129395c80bb1SLiu Bo case MOD_LOG_KEY_REMOVE_WHILE_MOVING: 12944c3e6969SChris Mason case MOD_LOG_KEY_REMOVE: 12955d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 12965d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 12975d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 12985d9e75c4SJan Schmidt tm->generation); 12994c3e6969SChris Mason n++; 13005d9e75c4SJan Schmidt break; 13015d9e75c4SJan Schmidt case MOD_LOG_KEY_REPLACE: 13025d9e75c4SJan Schmidt BUG_ON(tm->slot >= n); 13035d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 13045d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 13055d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 13065d9e75c4SJan Schmidt tm->generation); 13075d9e75c4SJan Schmidt break; 13085d9e75c4SJan Schmidt case MOD_LOG_KEY_ADD: 130919956c7eSJan Schmidt /* if a move operation is needed it's in the log */ 13105d9e75c4SJan Schmidt n--; 13115d9e75c4SJan Schmidt break; 13125d9e75c4SJan Schmidt case MOD_LOG_MOVE_KEYS: 1313c3193108SJan Schmidt o_dst = btrfs_node_key_ptr_offset(tm->slot); 1314c3193108SJan Schmidt o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot); 1315c3193108SJan Schmidt memmove_extent_buffer(eb, o_dst, o_src, 13165d9e75c4SJan Schmidt tm->move.nr_items * p_size); 13175d9e75c4SJan Schmidt break; 13185d9e75c4SJan Schmidt case MOD_LOG_ROOT_REPLACE: 13195d9e75c4SJan Schmidt /* 13205d9e75c4SJan Schmidt * this operation is special. for roots, this must be 13215d9e75c4SJan Schmidt * handled explicitly before rewinding. 13225d9e75c4SJan Schmidt * for non-roots, this operation may exist if the node 13235d9e75c4SJan Schmidt * was a root: root A -> child B; then A gets empty and 13245d9e75c4SJan Schmidt * B is promoted to the new root. in the mod log, we'll 13255d9e75c4SJan Schmidt * have a root-replace operation for B, a tree block 13265d9e75c4SJan Schmidt * that is no root. we simply ignore that operation. 13275d9e75c4SJan Schmidt */ 13285d9e75c4SJan Schmidt break; 13295d9e75c4SJan Schmidt } 13305d9e75c4SJan Schmidt next = rb_next(&tm->node); 13315d9e75c4SJan Schmidt if (!next) 13325d9e75c4SJan Schmidt break; 13336b4df8b6SGeliang Tang tm = rb_entry(next, struct tree_mod_elem, node); 1334298cfd36SChandan Rajendra if (tm->logical != first_tm->logical) 13355d9e75c4SJan Schmidt break; 13365d9e75c4SJan Schmidt } 1337f1ca7e98SJosef Bacik tree_mod_log_read_unlock(fs_info); 13385d9e75c4SJan Schmidt btrfs_set_header_nritems(eb, n); 13395d9e75c4SJan Schmidt } 13405d9e75c4SJan Schmidt 134147fb091fSJan Schmidt /* 134201327610SNicholas D Steeves * Called with eb read locked. If the buffer cannot be rewound, the same buffer 134347fb091fSJan Schmidt * is returned. If rewind operations happen, a fresh buffer is returned. The 134447fb091fSJan Schmidt * returned buffer is always read-locked. If the returned buffer is not the 134547fb091fSJan Schmidt * input buffer, the lock on the input buffer is released and the input buffer 134647fb091fSJan Schmidt * is freed (its refcount is decremented). 134747fb091fSJan Schmidt */ 13485d9e75c4SJan Schmidt static struct extent_buffer * 13499ec72677SJosef Bacik tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path, 13509ec72677SJosef Bacik struct extent_buffer *eb, u64 time_seq) 13515d9e75c4SJan Schmidt { 13525d9e75c4SJan Schmidt struct extent_buffer *eb_rewin; 13535d9e75c4SJan Schmidt struct tree_mod_elem *tm; 13545d9e75c4SJan Schmidt 13555d9e75c4SJan Schmidt if (!time_seq) 13565d9e75c4SJan Schmidt return eb; 13575d9e75c4SJan Schmidt 13585d9e75c4SJan Schmidt if (btrfs_header_level(eb) == 0) 13595d9e75c4SJan Schmidt return eb; 13605d9e75c4SJan Schmidt 13615d9e75c4SJan Schmidt tm = tree_mod_log_search(fs_info, eb->start, time_seq); 13625d9e75c4SJan Schmidt if (!tm) 13635d9e75c4SJan Schmidt return eb; 13645d9e75c4SJan Schmidt 13659ec72677SJosef Bacik btrfs_set_path_blocking(path); 13669ec72677SJosef Bacik btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK); 13679ec72677SJosef Bacik 13685d9e75c4SJan Schmidt if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 13695d9e75c4SJan Schmidt BUG_ON(tm->slot != 0); 1370da17066cSJeff Mahoney eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start); 1371db7f3436SJosef Bacik if (!eb_rewin) { 13729ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 1373db7f3436SJosef Bacik free_extent_buffer(eb); 1374db7f3436SJosef Bacik return NULL; 1375db7f3436SJosef Bacik } 13765d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb_rewin, eb->start); 13775d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb_rewin, 13785d9e75c4SJan Schmidt btrfs_header_backref_rev(eb)); 13795d9e75c4SJan Schmidt btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb)); 1380c3193108SJan Schmidt btrfs_set_header_level(eb_rewin, btrfs_header_level(eb)); 13815d9e75c4SJan Schmidt } else { 13825d9e75c4SJan Schmidt eb_rewin = btrfs_clone_extent_buffer(eb); 1383db7f3436SJosef Bacik if (!eb_rewin) { 13849ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 1385db7f3436SJosef Bacik free_extent_buffer(eb); 1386db7f3436SJosef Bacik return NULL; 1387db7f3436SJosef Bacik } 13885d9e75c4SJan Schmidt } 13895d9e75c4SJan Schmidt 13909ec72677SJosef Bacik btrfs_clear_path_blocking(path, NULL, BTRFS_READ_LOCK); 13919ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 13925d9e75c4SJan Schmidt free_extent_buffer(eb); 13935d9e75c4SJan Schmidt 139447fb091fSJan Schmidt extent_buffer_get(eb_rewin); 139547fb091fSJan Schmidt btrfs_tree_read_lock(eb_rewin); 1396f1ca7e98SJosef Bacik __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm); 139757911b8bSJan Schmidt WARN_ON(btrfs_header_nritems(eb_rewin) > 1398da17066cSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 13995d9e75c4SJan Schmidt 14005d9e75c4SJan Schmidt return eb_rewin; 14015d9e75c4SJan Schmidt } 14025d9e75c4SJan Schmidt 14038ba97a15SJan Schmidt /* 14048ba97a15SJan Schmidt * get_old_root() rewinds the state of @root's root node to the given @time_seq 14058ba97a15SJan Schmidt * value. If there are no changes, the current root->root_node is returned. If 14068ba97a15SJan Schmidt * anything changed in between, there's a fresh buffer allocated on which the 14078ba97a15SJan Schmidt * rewind operations are done. In any case, the returned buffer is read locked. 14088ba97a15SJan Schmidt * Returns NULL on error (with no locks held). 14098ba97a15SJan Schmidt */ 14105d9e75c4SJan Schmidt static inline struct extent_buffer * 14115d9e75c4SJan Schmidt get_old_root(struct btrfs_root *root, u64 time_seq) 14125d9e75c4SJan Schmidt { 14130b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 14145d9e75c4SJan Schmidt struct tree_mod_elem *tm; 141530b0463aSJan Schmidt struct extent_buffer *eb = NULL; 141630b0463aSJan Schmidt struct extent_buffer *eb_root; 14177bfdcf7fSLiu Bo struct extent_buffer *old; 1418a95236d9SJan Schmidt struct tree_mod_root *old_root = NULL; 14194325edd0SChris Mason u64 old_generation = 0; 1420a95236d9SJan Schmidt u64 logical; 14215d9e75c4SJan Schmidt 142230b0463aSJan Schmidt eb_root = btrfs_read_lock_root_node(root); 14230b246afaSJeff Mahoney tm = __tree_mod_log_oldest_root(fs_info, eb_root, time_seq); 14245d9e75c4SJan Schmidt if (!tm) 142530b0463aSJan Schmidt return eb_root; 14265d9e75c4SJan Schmidt 1427a95236d9SJan Schmidt if (tm->op == MOD_LOG_ROOT_REPLACE) { 14285d9e75c4SJan Schmidt old_root = &tm->old_root; 14295d9e75c4SJan Schmidt old_generation = tm->generation; 1430a95236d9SJan Schmidt logical = old_root->logical; 1431a95236d9SJan Schmidt } else { 143230b0463aSJan Schmidt logical = eb_root->start; 1433a95236d9SJan Schmidt } 14345d9e75c4SJan Schmidt 14350b246afaSJeff Mahoney tm = tree_mod_log_search(fs_info, logical, time_seq); 1436834328a8SJan Schmidt if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 143730b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 143830b0463aSJan Schmidt free_extent_buffer(eb_root); 14392ff7e61eSJeff Mahoney old = read_tree_block(fs_info, logical, 0); 144064c043deSLiu Bo if (WARN_ON(IS_ERR(old) || !extent_buffer_uptodate(old))) { 144164c043deSLiu Bo if (!IS_ERR(old)) 1442416bc658SJosef Bacik free_extent_buffer(old); 14430b246afaSJeff Mahoney btrfs_warn(fs_info, 14440b246afaSJeff Mahoney "failed to read tree block %llu from get_old_root", 14450b246afaSJeff Mahoney logical); 1446834328a8SJan Schmidt } else { 14477bfdcf7fSLiu Bo eb = btrfs_clone_extent_buffer(old); 14487bfdcf7fSLiu Bo free_extent_buffer(old); 1449834328a8SJan Schmidt } 1450834328a8SJan Schmidt } else if (old_root) { 145130b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 145230b0463aSJan Schmidt free_extent_buffer(eb_root); 14530b246afaSJeff Mahoney eb = alloc_dummy_extent_buffer(fs_info, logical); 1454834328a8SJan Schmidt } else { 14559ec72677SJosef Bacik btrfs_set_lock_blocking_rw(eb_root, BTRFS_READ_LOCK); 145630b0463aSJan Schmidt eb = btrfs_clone_extent_buffer(eb_root); 14579ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb_root); 145830b0463aSJan Schmidt free_extent_buffer(eb_root); 1459834328a8SJan Schmidt } 1460834328a8SJan Schmidt 14618ba97a15SJan Schmidt if (!eb) 14628ba97a15SJan Schmidt return NULL; 1463d6381084SJan Schmidt extent_buffer_get(eb); 14648ba97a15SJan Schmidt btrfs_tree_read_lock(eb); 1465a95236d9SJan Schmidt if (old_root) { 14665d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb, eb->start); 14675d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV); 146830b0463aSJan Schmidt btrfs_set_header_owner(eb, btrfs_header_owner(eb_root)); 14695d9e75c4SJan Schmidt btrfs_set_header_level(eb, old_root->level); 14705d9e75c4SJan Schmidt btrfs_set_header_generation(eb, old_generation); 1471a95236d9SJan Schmidt } 147228da9fb4SJan Schmidt if (tm) 14730b246afaSJeff Mahoney __tree_mod_log_rewind(fs_info, eb, time_seq, tm); 147428da9fb4SJan Schmidt else 147528da9fb4SJan Schmidt WARN_ON(btrfs_header_level(eb) != 0); 14760b246afaSJeff Mahoney WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 14775d9e75c4SJan Schmidt 14785d9e75c4SJan Schmidt return eb; 14795d9e75c4SJan Schmidt } 14805d9e75c4SJan Schmidt 14815b6602e7SJan Schmidt int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq) 14825b6602e7SJan Schmidt { 14835b6602e7SJan Schmidt struct tree_mod_elem *tm; 14845b6602e7SJan Schmidt int level; 148530b0463aSJan Schmidt struct extent_buffer *eb_root = btrfs_root_node(root); 14865b6602e7SJan Schmidt 148730b0463aSJan Schmidt tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq); 14885b6602e7SJan Schmidt if (tm && tm->op == MOD_LOG_ROOT_REPLACE) { 14895b6602e7SJan Schmidt level = tm->old_root.level; 14905b6602e7SJan Schmidt } else { 149130b0463aSJan Schmidt level = btrfs_header_level(eb_root); 14925b6602e7SJan Schmidt } 149330b0463aSJan Schmidt free_extent_buffer(eb_root); 14945b6602e7SJan Schmidt 14955b6602e7SJan Schmidt return level; 14965b6602e7SJan Schmidt } 14975b6602e7SJan Schmidt 14985d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 14995d4f98a2SYan Zheng struct btrfs_root *root, 15005d4f98a2SYan Zheng struct extent_buffer *buf) 15015d4f98a2SYan Zheng { 1502f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 1503faa2dbf0SJosef Bacik return 0; 1504fccb84c9SDavid Sterba 1505f1ebcc74SLiu Bo /* ensure we can see the force_cow */ 1506f1ebcc74SLiu Bo smp_rmb(); 1507f1ebcc74SLiu Bo 1508f1ebcc74SLiu Bo /* 1509f1ebcc74SLiu Bo * We do not need to cow a block if 1510f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 1511f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 1512f1ebcc74SLiu Bo * 3) the root is not forced COW. 1513f1ebcc74SLiu Bo * 1514f1ebcc74SLiu Bo * What is forced COW: 151501327610SNicholas D Steeves * when we create snapshot during committing the transaction, 1516f1ebcc74SLiu Bo * after we've finished coping src root, we must COW the shared 1517f1ebcc74SLiu Bo * block to ensure the metadata consistency. 1518f1ebcc74SLiu Bo */ 15195d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 15205d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 15215d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 1522f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 152327cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 15245d4f98a2SYan Zheng return 0; 15255d4f98a2SYan Zheng return 1; 15265d4f98a2SYan Zheng } 15275d4f98a2SYan Zheng 1528d352ac68SChris Mason /* 1529d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 153001327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 1531d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 1532d352ac68SChris Mason */ 1533d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 15345f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 15355f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 15369fa8cfe7SChris Mason struct extent_buffer **cow_ret) 153702217ed2SChris Mason { 15380b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 15396702ed49SChris Mason u64 search_start; 1540f510cfecSChris Mason int ret; 1541dc17ff8fSChris Mason 15420b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 154331b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 1544c1c9ff7cSGeert Uytterhoeven trans->transid, 15450b246afaSJeff Mahoney fs_info->running_transaction->transid); 154631b1a2bdSJulia Lawall 15470b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 154831b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 15490b246afaSJeff Mahoney trans->transid, fs_info->generation); 1550dc17ff8fSChris Mason 15515d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 155264c12921SJeff Mahoney trans->dirty = true; 155302217ed2SChris Mason *cow_ret = buf; 155402217ed2SChris Mason return 0; 155502217ed2SChris Mason } 1556c487685dSChris Mason 1557ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 1558b4ce94deSChris Mason 1559b4ce94deSChris Mason if (parent) 1560b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1561b4ce94deSChris Mason btrfs_set_lock_blocking(buf); 1562b4ce94deSChris Mason 1563f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 15649fa8cfe7SChris Mason parent_slot, cow_ret, search_start, 0); 15651abe9b8aSliubo 15661abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 15671abe9b8aSliubo 1568f510cfecSChris Mason return ret; 15692c90e5d6SChris Mason } 15706702ed49SChris Mason 1571d352ac68SChris Mason /* 1572d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 1573d352ac68SChris Mason * node are actually close by 1574d352ac68SChris Mason */ 15756b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 15766702ed49SChris Mason { 15776b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 15786702ed49SChris Mason return 1; 15796b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 15806702ed49SChris Mason return 1; 158102217ed2SChris Mason return 0; 158202217ed2SChris Mason } 158302217ed2SChris Mason 1584081e9573SChris Mason /* 1585081e9573SChris Mason * compare two keys in a memcmp fashion 1586081e9573SChris Mason */ 1587310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 1588310712b2SOmar Sandoval const struct btrfs_key *k2) 1589081e9573SChris Mason { 1590081e9573SChris Mason struct btrfs_key k1; 1591081e9573SChris Mason 1592081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 1593081e9573SChris Mason 159420736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 1595081e9573SChris Mason } 1596081e9573SChris Mason 1597f3465ca4SJosef Bacik /* 1598f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 1599f3465ca4SJosef Bacik */ 1600310712b2SOmar Sandoval int btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 1601f3465ca4SJosef Bacik { 1602f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 1603f3465ca4SJosef Bacik return 1; 1604f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 1605f3465ca4SJosef Bacik return -1; 1606f3465ca4SJosef Bacik if (k1->type > k2->type) 1607f3465ca4SJosef Bacik return 1; 1608f3465ca4SJosef Bacik if (k1->type < k2->type) 1609f3465ca4SJosef Bacik return -1; 1610f3465ca4SJosef Bacik if (k1->offset > k2->offset) 1611f3465ca4SJosef Bacik return 1; 1612f3465ca4SJosef Bacik if (k1->offset < k2->offset) 1613f3465ca4SJosef Bacik return -1; 1614f3465ca4SJosef Bacik return 0; 1615f3465ca4SJosef Bacik } 1616081e9573SChris Mason 1617d352ac68SChris Mason /* 1618d352ac68SChris Mason * this is used by the defrag code to go through all the 1619d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 1620d352ac68SChris Mason * disk order is close to key order 1621d352ac68SChris Mason */ 16226702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 16235f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 1624de78b51aSEric Sandeen int start_slot, u64 *last_ret, 1625a6b6e75eSChris Mason struct btrfs_key *progress) 16266702ed49SChris Mason { 16270b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 16286b80053dSChris Mason struct extent_buffer *cur; 16296702ed49SChris Mason u64 blocknr; 1630ca7a79adSChris Mason u64 gen; 1631e9d0b13bSChris Mason u64 search_start = *last_ret; 1632e9d0b13bSChris Mason u64 last_block = 0; 16336702ed49SChris Mason u64 other; 16346702ed49SChris Mason u32 parent_nritems; 16356702ed49SChris Mason int end_slot; 16366702ed49SChris Mason int i; 16376702ed49SChris Mason int err = 0; 1638f2183bdeSChris Mason int parent_level; 16396b80053dSChris Mason int uptodate; 16406b80053dSChris Mason u32 blocksize; 1641081e9573SChris Mason int progress_passed = 0; 1642081e9573SChris Mason struct btrfs_disk_key disk_key; 16436702ed49SChris Mason 16445708b959SChris Mason parent_level = btrfs_header_level(parent); 16455708b959SChris Mason 16460b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 16470b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 164886479a04SChris Mason 16496b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 16500b246afaSJeff Mahoney blocksize = fs_info->nodesize; 16515dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 16526702ed49SChris Mason 16535dfe2be7SFilipe Manana if (parent_nritems <= 1) 16546702ed49SChris Mason return 0; 16556702ed49SChris Mason 1656b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1657b4ce94deSChris Mason 16585dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 16596702ed49SChris Mason int close = 1; 1660a6b6e75eSChris Mason 1661081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 1662081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 1663081e9573SChris Mason continue; 1664081e9573SChris Mason 1665081e9573SChris Mason progress_passed = 1; 16666b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 1667ca7a79adSChris Mason gen = btrfs_node_ptr_generation(parent, i); 1668e9d0b13bSChris Mason if (last_block == 0) 1669e9d0b13bSChris Mason last_block = blocknr; 16705708b959SChris Mason 16716702ed49SChris Mason if (i > 0) { 16726b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 16736b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 16746702ed49SChris Mason } 16755dfe2be7SFilipe Manana if (!close && i < end_slot) { 16766b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 16776b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 16786702ed49SChris Mason } 1679e9d0b13bSChris Mason if (close) { 1680e9d0b13bSChris Mason last_block = blocknr; 16816702ed49SChris Mason continue; 1682e9d0b13bSChris Mason } 16836702ed49SChris Mason 16840b246afaSJeff Mahoney cur = find_extent_buffer(fs_info, blocknr); 16856b80053dSChris Mason if (cur) 1686b9fab919SChris Mason uptodate = btrfs_buffer_uptodate(cur, gen, 0); 16876b80053dSChris Mason else 16886b80053dSChris Mason uptodate = 0; 16895708b959SChris Mason if (!cur || !uptodate) { 16906b80053dSChris Mason if (!cur) { 16912ff7e61eSJeff Mahoney cur = read_tree_block(fs_info, blocknr, gen); 169264c043deSLiu Bo if (IS_ERR(cur)) { 169364c043deSLiu Bo return PTR_ERR(cur); 169464c043deSLiu Bo } else if (!extent_buffer_uptodate(cur)) { 1695416bc658SJosef Bacik free_extent_buffer(cur); 169697d9a8a4STsutomu Itoh return -EIO; 1697416bc658SJosef Bacik } 16986b80053dSChris Mason } else if (!uptodate) { 1699018642a1STsutomu Itoh err = btrfs_read_buffer(cur, gen); 1700018642a1STsutomu Itoh if (err) { 1701018642a1STsutomu Itoh free_extent_buffer(cur); 1702018642a1STsutomu Itoh return err; 1703018642a1STsutomu Itoh } 17046702ed49SChris Mason } 1705f2183bdeSChris Mason } 1706e9d0b13bSChris Mason if (search_start == 0) 17076b80053dSChris Mason search_start = last_block; 1708e9d0b13bSChris Mason 1709e7a84565SChris Mason btrfs_tree_lock(cur); 1710b4ce94deSChris Mason btrfs_set_lock_blocking(cur); 17116b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 1712e7a84565SChris Mason &cur, search_start, 17136b80053dSChris Mason min(16 * blocksize, 17149fa8cfe7SChris Mason (end_slot - i) * blocksize)); 1715252c38f0SYan if (err) { 1716e7a84565SChris Mason btrfs_tree_unlock(cur); 17176b80053dSChris Mason free_extent_buffer(cur); 17186702ed49SChris Mason break; 1719252c38f0SYan } 1720e7a84565SChris Mason search_start = cur->start; 1721e7a84565SChris Mason last_block = cur->start; 1722f2183bdeSChris Mason *last_ret = search_start; 1723e7a84565SChris Mason btrfs_tree_unlock(cur); 1724e7a84565SChris Mason free_extent_buffer(cur); 17256702ed49SChris Mason } 17266702ed49SChris Mason return err; 17276702ed49SChris Mason } 17286702ed49SChris Mason 172974123bd7SChris Mason /* 17305f39d397SChris Mason * search for key in the extent_buffer. The items start at offset p, 17315f39d397SChris Mason * and they are item_size apart. There are 'max' items in p. 17325f39d397SChris Mason * 173374123bd7SChris Mason * the slot in the array is returned via slot, and it points to 173474123bd7SChris Mason * the place where you would insert key if it is not found in 173574123bd7SChris Mason * the array. 173674123bd7SChris Mason * 173774123bd7SChris Mason * slot may point to max if the key is bigger than all of the keys 173874123bd7SChris Mason */ 1739e02119d5SChris Mason static noinline int generic_bin_search(struct extent_buffer *eb, 1740310712b2SOmar Sandoval unsigned long p, int item_size, 1741310712b2SOmar Sandoval const struct btrfs_key *key, 1742be0e5c09SChris Mason int max, int *slot) 1743be0e5c09SChris Mason { 1744be0e5c09SChris Mason int low = 0; 1745be0e5c09SChris Mason int high = max; 1746be0e5c09SChris Mason int mid; 1747be0e5c09SChris Mason int ret; 1748479965d6SChris Mason struct btrfs_disk_key *tmp = NULL; 17495f39d397SChris Mason struct btrfs_disk_key unaligned; 17505f39d397SChris Mason unsigned long offset; 17515f39d397SChris Mason char *kaddr = NULL; 17525f39d397SChris Mason unsigned long map_start = 0; 17535f39d397SChris Mason unsigned long map_len = 0; 1754479965d6SChris Mason int err; 1755be0e5c09SChris Mason 17565e24e9afSLiu Bo if (low > high) { 17575e24e9afSLiu Bo btrfs_err(eb->fs_info, 17585e24e9afSLiu Bo "%s: low (%d) > high (%d) eb %llu owner %llu level %d", 17595e24e9afSLiu Bo __func__, low, high, eb->start, 17605e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 17615e24e9afSLiu Bo return -EINVAL; 17625e24e9afSLiu Bo } 17635e24e9afSLiu Bo 1764be0e5c09SChris Mason while (low < high) { 1765be0e5c09SChris Mason mid = (low + high) / 2; 17665f39d397SChris Mason offset = p + mid * item_size; 17675f39d397SChris Mason 1768a6591715SChris Mason if (!kaddr || offset < map_start || 17695f39d397SChris Mason (offset + sizeof(struct btrfs_disk_key)) > 17705f39d397SChris Mason map_start + map_len) { 1771934d375bSChris Mason 1772934d375bSChris Mason err = map_private_extent_buffer(eb, offset, 1773479965d6SChris Mason sizeof(struct btrfs_disk_key), 1774a6591715SChris Mason &kaddr, &map_start, &map_len); 17755f39d397SChris Mason 1776479965d6SChris Mason if (!err) { 1777479965d6SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 1778479965d6SChris Mason map_start); 1779415b35a5SLiu Bo } else if (err == 1) { 17805f39d397SChris Mason read_extent_buffer(eb, &unaligned, 17815f39d397SChris Mason offset, sizeof(unaligned)); 17825f39d397SChris Mason tmp = &unaligned; 1783415b35a5SLiu Bo } else { 1784415b35a5SLiu Bo return err; 1785479965d6SChris Mason } 1786479965d6SChris Mason 17875f39d397SChris Mason } else { 17885f39d397SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 17895f39d397SChris Mason map_start); 17905f39d397SChris Mason } 1791be0e5c09SChris Mason ret = comp_keys(tmp, key); 1792be0e5c09SChris Mason 1793be0e5c09SChris Mason if (ret < 0) 1794be0e5c09SChris Mason low = mid + 1; 1795be0e5c09SChris Mason else if (ret > 0) 1796be0e5c09SChris Mason high = mid; 1797be0e5c09SChris Mason else { 1798be0e5c09SChris Mason *slot = mid; 1799be0e5c09SChris Mason return 0; 1800be0e5c09SChris Mason } 1801be0e5c09SChris Mason } 1802be0e5c09SChris Mason *slot = low; 1803be0e5c09SChris Mason return 1; 1804be0e5c09SChris Mason } 1805be0e5c09SChris Mason 180697571fd0SChris Mason /* 180797571fd0SChris Mason * simple bin_search frontend that does the right thing for 180897571fd0SChris Mason * leaves vs nodes 180997571fd0SChris Mason */ 1810a74b35ecSNikolay Borisov int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, 18115f39d397SChris Mason int level, int *slot) 1812be0e5c09SChris Mason { 1813f775738fSWang Sheng-Hui if (level == 0) 18145f39d397SChris Mason return generic_bin_search(eb, 18155f39d397SChris Mason offsetof(struct btrfs_leaf, items), 18160783fcfcSChris Mason sizeof(struct btrfs_item), 18175f39d397SChris Mason key, btrfs_header_nritems(eb), 18187518a238SChris Mason slot); 1819f775738fSWang Sheng-Hui else 18205f39d397SChris Mason return generic_bin_search(eb, 18215f39d397SChris Mason offsetof(struct btrfs_node, ptrs), 1822123abc88SChris Mason sizeof(struct btrfs_key_ptr), 18235f39d397SChris Mason key, btrfs_header_nritems(eb), 18247518a238SChris Mason slot); 1825be0e5c09SChris Mason } 1826be0e5c09SChris Mason 1827f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 1828f0486c68SYan, Zheng { 1829f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1830f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1831f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 1832f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1833f0486c68SYan, Zheng } 1834f0486c68SYan, Zheng 1835f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 1836f0486c68SYan, Zheng { 1837f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1838f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1839f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 1840f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1841f0486c68SYan, Zheng } 1842f0486c68SYan, Zheng 1843d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 1844d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 1845d352ac68SChris Mason */ 18462ff7e61eSJeff Mahoney static noinline struct extent_buffer * 18472ff7e61eSJeff Mahoney read_node_slot(struct btrfs_fs_info *fs_info, struct extent_buffer *parent, 18482ff7e61eSJeff Mahoney int slot) 1849bb803951SChris Mason { 1850ca7a79adSChris Mason int level = btrfs_header_level(parent); 1851416bc658SJosef Bacik struct extent_buffer *eb; 1852416bc658SJosef Bacik 1853fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 1854fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 1855ca7a79adSChris Mason 1856ca7a79adSChris Mason BUG_ON(level == 0); 1857ca7a79adSChris Mason 18582ff7e61eSJeff Mahoney eb = read_tree_block(fs_info, btrfs_node_blockptr(parent, slot), 1859ca7a79adSChris Mason btrfs_node_ptr_generation(parent, slot)); 1860fb770ae4SLiu Bo if (!IS_ERR(eb) && !extent_buffer_uptodate(eb)) { 1861416bc658SJosef Bacik free_extent_buffer(eb); 1862fb770ae4SLiu Bo eb = ERR_PTR(-EIO); 1863416bc658SJosef Bacik } 1864416bc658SJosef Bacik 1865416bc658SJosef Bacik return eb; 1866bb803951SChris Mason } 1867bb803951SChris Mason 1868d352ac68SChris Mason /* 1869d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 1870d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 1871d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 1872d352ac68SChris Mason */ 1873e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 187498ed5174SChris Mason struct btrfs_root *root, 187598ed5174SChris Mason struct btrfs_path *path, int level) 1876bb803951SChris Mason { 18770b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 18785f39d397SChris Mason struct extent_buffer *right = NULL; 18795f39d397SChris Mason struct extent_buffer *mid; 18805f39d397SChris Mason struct extent_buffer *left = NULL; 18815f39d397SChris Mason struct extent_buffer *parent = NULL; 1882bb803951SChris Mason int ret = 0; 1883bb803951SChris Mason int wret; 1884bb803951SChris Mason int pslot; 1885bb803951SChris Mason int orig_slot = path->slots[level]; 188679f95c82SChris Mason u64 orig_ptr; 1887bb803951SChris Mason 1888bb803951SChris Mason if (level == 0) 1889bb803951SChris Mason return 0; 1890bb803951SChris Mason 18915f39d397SChris Mason mid = path->nodes[level]; 1892b4ce94deSChris Mason 1893bd681513SChris Mason WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK && 1894bd681513SChris Mason path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING); 18957bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 18967bb86316SChris Mason 18971d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 189879f95c82SChris Mason 1899a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 19005f39d397SChris Mason parent = path->nodes[level + 1]; 1901bb803951SChris Mason pslot = path->slots[level + 1]; 1902a05a9bb1SLi Zefan } 1903bb803951SChris Mason 190440689478SChris Mason /* 190540689478SChris Mason * deal with the case where there is only one pointer in the root 190640689478SChris Mason * by promoting the node below to a root 190740689478SChris Mason */ 19085f39d397SChris Mason if (!parent) { 19095f39d397SChris Mason struct extent_buffer *child; 1910bb803951SChris Mason 19115f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1912bb803951SChris Mason return 0; 1913bb803951SChris Mason 1914bb803951SChris Mason /* promote the child to a root */ 19152ff7e61eSJeff Mahoney child = read_node_slot(fs_info, mid, 0); 1916fb770ae4SLiu Bo if (IS_ERR(child)) { 1917fb770ae4SLiu Bo ret = PTR_ERR(child); 19180b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1919305a26afSMark Fasheh goto enospc; 1920305a26afSMark Fasheh } 1921305a26afSMark Fasheh 1922925baeddSChris Mason btrfs_tree_lock(child); 1923b4ce94deSChris Mason btrfs_set_lock_blocking(child); 19249fa8cfe7SChris Mason ret = btrfs_cow_block(trans, root, child, mid, 0, &child); 1925f0486c68SYan, Zheng if (ret) { 1926f0486c68SYan, Zheng btrfs_tree_unlock(child); 1927f0486c68SYan, Zheng free_extent_buffer(child); 1928f0486c68SYan, Zheng goto enospc; 1929f0486c68SYan, Zheng } 19302f375ab9SYan 193190f8d62eSJan Schmidt tree_mod_log_set_root_pointer(root, child, 1); 1932240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1933925baeddSChris Mason 19340b86a832SChris Mason add_root_to_dirty_list(root); 1935925baeddSChris Mason btrfs_tree_unlock(child); 1936b4ce94deSChris Mason 1937925baeddSChris Mason path->locks[level] = 0; 1938bb803951SChris Mason path->nodes[level] = NULL; 19397c302b49SDavid Sterba clean_tree_block(fs_info, mid); 1940925baeddSChris Mason btrfs_tree_unlock(mid); 1941bb803951SChris Mason /* once for the path */ 19425f39d397SChris Mason free_extent_buffer(mid); 1943f0486c68SYan, Zheng 1944f0486c68SYan, Zheng root_sub_used(root, mid->len); 19455581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 1946bb803951SChris Mason /* once for the root ptr */ 19473083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1948f0486c68SYan, Zheng return 0; 1949bb803951SChris Mason } 19505f39d397SChris Mason if (btrfs_header_nritems(mid) > 19510b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 1952bb803951SChris Mason return 0; 1953bb803951SChris Mason 19542ff7e61eSJeff Mahoney left = read_node_slot(fs_info, parent, pslot - 1); 1955fb770ae4SLiu Bo if (IS_ERR(left)) 1956fb770ae4SLiu Bo left = NULL; 1957fb770ae4SLiu Bo 19585f39d397SChris Mason if (left) { 1959925baeddSChris Mason btrfs_tree_lock(left); 1960b4ce94deSChris Mason btrfs_set_lock_blocking(left); 19615f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 19629fa8cfe7SChris Mason parent, pslot - 1, &left); 196354aa1f4dSChris Mason if (wret) { 196454aa1f4dSChris Mason ret = wret; 196554aa1f4dSChris Mason goto enospc; 196654aa1f4dSChris Mason } 19672cc58cf2SChris Mason } 1968fb770ae4SLiu Bo 19692ff7e61eSJeff Mahoney right = read_node_slot(fs_info, parent, pslot + 1); 1970fb770ae4SLiu Bo if (IS_ERR(right)) 1971fb770ae4SLiu Bo right = NULL; 1972fb770ae4SLiu Bo 19735f39d397SChris Mason if (right) { 1974925baeddSChris Mason btrfs_tree_lock(right); 1975b4ce94deSChris Mason btrfs_set_lock_blocking(right); 19765f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 19779fa8cfe7SChris Mason parent, pslot + 1, &right); 19782cc58cf2SChris Mason if (wret) { 19792cc58cf2SChris Mason ret = wret; 19802cc58cf2SChris Mason goto enospc; 19812cc58cf2SChris Mason } 19822cc58cf2SChris Mason } 19832cc58cf2SChris Mason 19842cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 19855f39d397SChris Mason if (left) { 19865f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 19872ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, left, mid, 1); 198879f95c82SChris Mason if (wret < 0) 198979f95c82SChris Mason ret = wret; 1990bb803951SChris Mason } 199179f95c82SChris Mason 199279f95c82SChris Mason /* 199379f95c82SChris Mason * then try to empty the right most buffer into the middle 199479f95c82SChris Mason */ 19955f39d397SChris Mason if (right) { 19962ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, mid, right, 1); 199754aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 199879f95c82SChris Mason ret = wret; 19995f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 20007c302b49SDavid Sterba clean_tree_block(fs_info, right); 2001925baeddSChris Mason btrfs_tree_unlock(right); 2002afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 2003f0486c68SYan, Zheng root_sub_used(root, right->len); 20045581a51aSJan Schmidt btrfs_free_tree_block(trans, root, right, 0, 1); 20053083ee2eSJosef Bacik free_extent_buffer_stale(right); 2006f0486c68SYan, Zheng right = NULL; 2007bb803951SChris Mason } else { 20085f39d397SChris Mason struct btrfs_disk_key right_key; 20095f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 20100b246afaSJeff Mahoney tree_mod_log_set_node_key(fs_info, parent, 201132adf090SLiu Bo pslot + 1, 0); 20125f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 20135f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 2014bb803951SChris Mason } 2015bb803951SChris Mason } 20165f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 201779f95c82SChris Mason /* 201879f95c82SChris Mason * we're not allowed to leave a node with one item in the 201979f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 202079f95c82SChris Mason * could try to delete the only pointer in this node. 202179f95c82SChris Mason * So, pull some keys from the left. 202279f95c82SChris Mason * There has to be a left pointer at this point because 202379f95c82SChris Mason * otherwise we would have pulled some pointers from the 202479f95c82SChris Mason * right 202579f95c82SChris Mason */ 2026305a26afSMark Fasheh if (!left) { 2027305a26afSMark Fasheh ret = -EROFS; 20280b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 2029305a26afSMark Fasheh goto enospc; 2030305a26afSMark Fasheh } 20312ff7e61eSJeff Mahoney wret = balance_node_right(trans, fs_info, mid, left); 203254aa1f4dSChris Mason if (wret < 0) { 203379f95c82SChris Mason ret = wret; 203454aa1f4dSChris Mason goto enospc; 203554aa1f4dSChris Mason } 2036bce4eae9SChris Mason if (wret == 1) { 20372ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, left, mid, 1); 2038bce4eae9SChris Mason if (wret < 0) 2039bce4eae9SChris Mason ret = wret; 2040bce4eae9SChris Mason } 204179f95c82SChris Mason BUG_ON(wret == 1); 204279f95c82SChris Mason } 20435f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 20447c302b49SDavid Sterba clean_tree_block(fs_info, mid); 2045925baeddSChris Mason btrfs_tree_unlock(mid); 2046afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 2047f0486c68SYan, Zheng root_sub_used(root, mid->len); 20485581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 20493083ee2eSJosef Bacik free_extent_buffer_stale(mid); 2050f0486c68SYan, Zheng mid = NULL; 205179f95c82SChris Mason } else { 205279f95c82SChris Mason /* update the parent key to reflect our changes */ 20535f39d397SChris Mason struct btrfs_disk_key mid_key; 20545f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 20550b246afaSJeff Mahoney tree_mod_log_set_node_key(fs_info, parent, pslot, 0); 20565f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 20575f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 205879f95c82SChris Mason } 2059bb803951SChris Mason 206079f95c82SChris Mason /* update the path */ 20615f39d397SChris Mason if (left) { 20625f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 20635f39d397SChris Mason extent_buffer_get(left); 2064925baeddSChris Mason /* left was locked after cow */ 20655f39d397SChris Mason path->nodes[level] = left; 2066bb803951SChris Mason path->slots[level + 1] -= 1; 2067bb803951SChris Mason path->slots[level] = orig_slot; 2068925baeddSChris Mason if (mid) { 2069925baeddSChris Mason btrfs_tree_unlock(mid); 20705f39d397SChris Mason free_extent_buffer(mid); 2071925baeddSChris Mason } 2072bb803951SChris Mason } else { 20735f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 2074bb803951SChris Mason path->slots[level] = orig_slot; 2075bb803951SChris Mason } 2076bb803951SChris Mason } 207779f95c82SChris Mason /* double check we haven't messed things up */ 2078e20d96d6SChris Mason if (orig_ptr != 20795f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 208079f95c82SChris Mason BUG(); 208154aa1f4dSChris Mason enospc: 2082925baeddSChris Mason if (right) { 2083925baeddSChris Mason btrfs_tree_unlock(right); 20845f39d397SChris Mason free_extent_buffer(right); 2085925baeddSChris Mason } 2086925baeddSChris Mason if (left) { 2087925baeddSChris Mason if (path->nodes[level] != left) 2088925baeddSChris Mason btrfs_tree_unlock(left); 20895f39d397SChris Mason free_extent_buffer(left); 2090925baeddSChris Mason } 2091bb803951SChris Mason return ret; 2092bb803951SChris Mason } 2093bb803951SChris Mason 2094d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 2095d352ac68SChris Mason * when they are completely full. This is also done top down, so we 2096d352ac68SChris Mason * have to be pessimistic. 2097d352ac68SChris Mason */ 2098d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 2099e66f709bSChris Mason struct btrfs_root *root, 2100e66f709bSChris Mason struct btrfs_path *path, int level) 2101e66f709bSChris Mason { 21020b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 21035f39d397SChris Mason struct extent_buffer *right = NULL; 21045f39d397SChris Mason struct extent_buffer *mid; 21055f39d397SChris Mason struct extent_buffer *left = NULL; 21065f39d397SChris Mason struct extent_buffer *parent = NULL; 2107e66f709bSChris Mason int ret = 0; 2108e66f709bSChris Mason int wret; 2109e66f709bSChris Mason int pslot; 2110e66f709bSChris Mason int orig_slot = path->slots[level]; 2111e66f709bSChris Mason 2112e66f709bSChris Mason if (level == 0) 2113e66f709bSChris Mason return 1; 2114e66f709bSChris Mason 21155f39d397SChris Mason mid = path->nodes[level]; 21167bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 2117e66f709bSChris Mason 2118a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 21195f39d397SChris Mason parent = path->nodes[level + 1]; 2120e66f709bSChris Mason pslot = path->slots[level + 1]; 2121a05a9bb1SLi Zefan } 2122e66f709bSChris Mason 21235f39d397SChris Mason if (!parent) 2124e66f709bSChris Mason return 1; 2125e66f709bSChris Mason 21262ff7e61eSJeff Mahoney left = read_node_slot(fs_info, parent, pslot - 1); 2127fb770ae4SLiu Bo if (IS_ERR(left)) 2128fb770ae4SLiu Bo left = NULL; 2129e66f709bSChris Mason 2130e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 21315f39d397SChris Mason if (left) { 2132e66f709bSChris Mason u32 left_nr; 2133925baeddSChris Mason 2134925baeddSChris Mason btrfs_tree_lock(left); 2135b4ce94deSChris Mason btrfs_set_lock_blocking(left); 2136b4ce94deSChris Mason 21375f39d397SChris Mason left_nr = btrfs_header_nritems(left); 21380b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 213933ade1f8SChris Mason wret = 1; 214033ade1f8SChris Mason } else { 21415f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 21429fa8cfe7SChris Mason pslot - 1, &left); 214354aa1f4dSChris Mason if (ret) 214454aa1f4dSChris Mason wret = 1; 214554aa1f4dSChris Mason else { 21462ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, 2147971a1f66SChris Mason left, mid, 0); 214854aa1f4dSChris Mason } 214933ade1f8SChris Mason } 2150e66f709bSChris Mason if (wret < 0) 2151e66f709bSChris Mason ret = wret; 2152e66f709bSChris Mason if (wret == 0) { 21535f39d397SChris Mason struct btrfs_disk_key disk_key; 2154e66f709bSChris Mason orig_slot += left_nr; 21555f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 21560b246afaSJeff Mahoney tree_mod_log_set_node_key(fs_info, parent, pslot, 0); 21575f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 21585f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 21595f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 21605f39d397SChris Mason path->nodes[level] = left; 2161e66f709bSChris Mason path->slots[level + 1] -= 1; 2162e66f709bSChris Mason path->slots[level] = orig_slot; 2163925baeddSChris Mason btrfs_tree_unlock(mid); 21645f39d397SChris Mason free_extent_buffer(mid); 2165e66f709bSChris Mason } else { 2166e66f709bSChris Mason orig_slot -= 21675f39d397SChris Mason btrfs_header_nritems(left); 2168e66f709bSChris Mason path->slots[level] = orig_slot; 2169925baeddSChris Mason btrfs_tree_unlock(left); 21705f39d397SChris Mason free_extent_buffer(left); 2171e66f709bSChris Mason } 2172e66f709bSChris Mason return 0; 2173e66f709bSChris Mason } 2174925baeddSChris Mason btrfs_tree_unlock(left); 21755f39d397SChris Mason free_extent_buffer(left); 2176e66f709bSChris Mason } 21772ff7e61eSJeff Mahoney right = read_node_slot(fs_info, parent, pslot + 1); 2178fb770ae4SLiu Bo if (IS_ERR(right)) 2179fb770ae4SLiu Bo right = NULL; 2180e66f709bSChris Mason 2181e66f709bSChris Mason /* 2182e66f709bSChris Mason * then try to empty the right most buffer into the middle 2183e66f709bSChris Mason */ 21845f39d397SChris Mason if (right) { 218533ade1f8SChris Mason u32 right_nr; 2186b4ce94deSChris Mason 2187925baeddSChris Mason btrfs_tree_lock(right); 2188b4ce94deSChris Mason btrfs_set_lock_blocking(right); 2189b4ce94deSChris Mason 21905f39d397SChris Mason right_nr = btrfs_header_nritems(right); 21910b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 219233ade1f8SChris Mason wret = 1; 219333ade1f8SChris Mason } else { 21945f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 21955f39d397SChris Mason parent, pslot + 1, 21969fa8cfe7SChris Mason &right); 219754aa1f4dSChris Mason if (ret) 219854aa1f4dSChris Mason wret = 1; 219954aa1f4dSChris Mason else { 22002ff7e61eSJeff Mahoney wret = balance_node_right(trans, fs_info, 22015f39d397SChris Mason right, mid); 220233ade1f8SChris Mason } 220354aa1f4dSChris Mason } 2204e66f709bSChris Mason if (wret < 0) 2205e66f709bSChris Mason ret = wret; 2206e66f709bSChris Mason if (wret == 0) { 22075f39d397SChris Mason struct btrfs_disk_key disk_key; 22085f39d397SChris Mason 22095f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 22100b246afaSJeff Mahoney tree_mod_log_set_node_key(fs_info, parent, 221132adf090SLiu Bo pslot + 1, 0); 22125f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 22135f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 22145f39d397SChris Mason 22155f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 22165f39d397SChris Mason path->nodes[level] = right; 2217e66f709bSChris Mason path->slots[level + 1] += 1; 2218e66f709bSChris Mason path->slots[level] = orig_slot - 22195f39d397SChris Mason btrfs_header_nritems(mid); 2220925baeddSChris Mason btrfs_tree_unlock(mid); 22215f39d397SChris Mason free_extent_buffer(mid); 2222e66f709bSChris Mason } else { 2223925baeddSChris Mason btrfs_tree_unlock(right); 22245f39d397SChris Mason free_extent_buffer(right); 2225e66f709bSChris Mason } 2226e66f709bSChris Mason return 0; 2227e66f709bSChris Mason } 2228925baeddSChris Mason btrfs_tree_unlock(right); 22295f39d397SChris Mason free_extent_buffer(right); 2230e66f709bSChris Mason } 2231e66f709bSChris Mason return 1; 2232e66f709bSChris Mason } 2233e66f709bSChris Mason 223474123bd7SChris Mason /* 2235d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 2236d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 22373c69faecSChris Mason */ 22382ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 2239e02119d5SChris Mason struct btrfs_path *path, 224001f46658SChris Mason int level, int slot, u64 objectid) 22413c69faecSChris Mason { 22425f39d397SChris Mason struct extent_buffer *node; 224301f46658SChris Mason struct btrfs_disk_key disk_key; 22443c69faecSChris Mason u32 nritems; 22453c69faecSChris Mason u64 search; 2246a7175319SChris Mason u64 target; 22476b80053dSChris Mason u64 nread = 0; 22485f39d397SChris Mason struct extent_buffer *eb; 22496b80053dSChris Mason u32 nr; 22506b80053dSChris Mason u32 blocksize; 22516b80053dSChris Mason u32 nscan = 0; 2252db94535dSChris Mason 2253a6b6e75eSChris Mason if (level != 1) 22543c69faecSChris Mason return; 22553c69faecSChris Mason 22566702ed49SChris Mason if (!path->nodes[level]) 22576702ed49SChris Mason return; 22586702ed49SChris Mason 22595f39d397SChris Mason node = path->nodes[level]; 2260925baeddSChris Mason 22613c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 22620b246afaSJeff Mahoney blocksize = fs_info->nodesize; 22630b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 22645f39d397SChris Mason if (eb) { 22655f39d397SChris Mason free_extent_buffer(eb); 22663c69faecSChris Mason return; 22673c69faecSChris Mason } 22683c69faecSChris Mason 2269a7175319SChris Mason target = search; 22706b80053dSChris Mason 22715f39d397SChris Mason nritems = btrfs_header_nritems(node); 22726b80053dSChris Mason nr = slot; 227325b8b936SJosef Bacik 22743c69faecSChris Mason while (1) { 2275e4058b54SDavid Sterba if (path->reada == READA_BACK) { 22766b80053dSChris Mason if (nr == 0) 22773c69faecSChris Mason break; 22786b80053dSChris Mason nr--; 2279e4058b54SDavid Sterba } else if (path->reada == READA_FORWARD) { 22806b80053dSChris Mason nr++; 22816b80053dSChris Mason if (nr >= nritems) 22826b80053dSChris Mason break; 22833c69faecSChris Mason } 2284e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 228501f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 228601f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 228701f46658SChris Mason break; 228801f46658SChris Mason } 22896b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 2290a7175319SChris Mason if ((search <= target && target - search <= 65536) || 2291a7175319SChris Mason (search > target && search - target <= 65536)) { 22922ff7e61eSJeff Mahoney readahead_tree_block(fs_info, search); 22936b80053dSChris Mason nread += blocksize; 22943c69faecSChris Mason } 22956b80053dSChris Mason nscan++; 2296a7175319SChris Mason if ((nread > 65536 || nscan > 32)) 22976b80053dSChris Mason break; 22983c69faecSChris Mason } 22993c69faecSChris Mason } 2300925baeddSChris Mason 23012ff7e61eSJeff Mahoney static noinline void reada_for_balance(struct btrfs_fs_info *fs_info, 2302b4ce94deSChris Mason struct btrfs_path *path, int level) 2303b4ce94deSChris Mason { 2304b4ce94deSChris Mason int slot; 2305b4ce94deSChris Mason int nritems; 2306b4ce94deSChris Mason struct extent_buffer *parent; 2307b4ce94deSChris Mason struct extent_buffer *eb; 2308b4ce94deSChris Mason u64 gen; 2309b4ce94deSChris Mason u64 block1 = 0; 2310b4ce94deSChris Mason u64 block2 = 0; 2311b4ce94deSChris Mason 23128c594ea8SChris Mason parent = path->nodes[level + 1]; 2313b4ce94deSChris Mason if (!parent) 23140b08851fSJosef Bacik return; 2315b4ce94deSChris Mason 2316b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 23178c594ea8SChris Mason slot = path->slots[level + 1]; 2318b4ce94deSChris Mason 2319b4ce94deSChris Mason if (slot > 0) { 2320b4ce94deSChris Mason block1 = btrfs_node_blockptr(parent, slot - 1); 2321b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot - 1); 23220b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, block1); 2323b9fab919SChris Mason /* 2324b9fab919SChris Mason * if we get -eagain from btrfs_buffer_uptodate, we 2325b9fab919SChris Mason * don't want to return eagain here. That will loop 2326b9fab919SChris Mason * forever 2327b9fab919SChris Mason */ 2328b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2329b4ce94deSChris Mason block1 = 0; 2330b4ce94deSChris Mason free_extent_buffer(eb); 2331b4ce94deSChris Mason } 23328c594ea8SChris Mason if (slot + 1 < nritems) { 2333b4ce94deSChris Mason block2 = btrfs_node_blockptr(parent, slot + 1); 2334b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot + 1); 23350b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, block2); 2336b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2337b4ce94deSChris Mason block2 = 0; 2338b4ce94deSChris Mason free_extent_buffer(eb); 2339b4ce94deSChris Mason } 23408c594ea8SChris Mason 2341b4ce94deSChris Mason if (block1) 23422ff7e61eSJeff Mahoney readahead_tree_block(fs_info, block1); 2343b4ce94deSChris Mason if (block2) 23442ff7e61eSJeff Mahoney readahead_tree_block(fs_info, block2); 2345b4ce94deSChris Mason } 2346b4ce94deSChris Mason 2347b4ce94deSChris Mason 2348b4ce94deSChris Mason /* 2349d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 2350d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 2351d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 2352d397712bSChris Mason * tree. 2353d352ac68SChris Mason * 2354d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 2355d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 2356d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 2357d352ac68SChris Mason * 2358d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 2359d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 2360d352ac68SChris Mason */ 2361e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 2362f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 2363f7c79f30SChris Mason int *write_lock_level) 2364925baeddSChris Mason { 2365925baeddSChris Mason int i; 2366925baeddSChris Mason int skip_level = level; 2367051e1b9fSChris Mason int no_skips = 0; 2368925baeddSChris Mason struct extent_buffer *t; 2369925baeddSChris Mason 2370925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2371925baeddSChris Mason if (!path->nodes[i]) 2372925baeddSChris Mason break; 2373925baeddSChris Mason if (!path->locks[i]) 2374925baeddSChris Mason break; 2375051e1b9fSChris Mason if (!no_skips && path->slots[i] == 0) { 2376925baeddSChris Mason skip_level = i + 1; 2377925baeddSChris Mason continue; 2378925baeddSChris Mason } 2379051e1b9fSChris Mason if (!no_skips && path->keep_locks) { 2380925baeddSChris Mason u32 nritems; 2381925baeddSChris Mason t = path->nodes[i]; 2382925baeddSChris Mason nritems = btrfs_header_nritems(t); 2383051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 2384925baeddSChris Mason skip_level = i + 1; 2385925baeddSChris Mason continue; 2386925baeddSChris Mason } 2387925baeddSChris Mason } 2388051e1b9fSChris Mason if (skip_level < i && i >= lowest_unlock) 2389051e1b9fSChris Mason no_skips = 1; 2390051e1b9fSChris Mason 2391925baeddSChris Mason t = path->nodes[i]; 2392925baeddSChris Mason if (i >= lowest_unlock && i > skip_level && path->locks[i]) { 2393bd681513SChris Mason btrfs_tree_unlock_rw(t, path->locks[i]); 2394925baeddSChris Mason path->locks[i] = 0; 2395f7c79f30SChris Mason if (write_lock_level && 2396f7c79f30SChris Mason i > min_write_lock_level && 2397f7c79f30SChris Mason i <= *write_lock_level) { 2398f7c79f30SChris Mason *write_lock_level = i - 1; 2399f7c79f30SChris Mason } 2400925baeddSChris Mason } 2401925baeddSChris Mason } 2402925baeddSChris Mason } 2403925baeddSChris Mason 24043c69faecSChris Mason /* 2405b4ce94deSChris Mason * This releases any locks held in the path starting at level and 2406b4ce94deSChris Mason * going all the way up to the root. 2407b4ce94deSChris Mason * 2408b4ce94deSChris Mason * btrfs_search_slot will keep the lock held on higher nodes in a few 2409b4ce94deSChris Mason * corner cases, such as COW of the block at slot zero in the node. This 2410b4ce94deSChris Mason * ignores those rules, and it should only be called when there are no 2411b4ce94deSChris Mason * more updates to be done higher up in the tree. 2412b4ce94deSChris Mason */ 2413b4ce94deSChris Mason noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level) 2414b4ce94deSChris Mason { 2415b4ce94deSChris Mason int i; 2416b4ce94deSChris Mason 241709a2a8f9SJosef Bacik if (path->keep_locks) 2418b4ce94deSChris Mason return; 2419b4ce94deSChris Mason 2420b4ce94deSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2421b4ce94deSChris Mason if (!path->nodes[i]) 242212f4daccSChris Mason continue; 2423b4ce94deSChris Mason if (!path->locks[i]) 242412f4daccSChris Mason continue; 2425bd681513SChris Mason btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 2426b4ce94deSChris Mason path->locks[i] = 0; 2427b4ce94deSChris Mason } 2428b4ce94deSChris Mason } 2429b4ce94deSChris Mason 2430b4ce94deSChris Mason /* 2431c8c42864SChris Mason * helper function for btrfs_search_slot. The goal is to find a block 2432c8c42864SChris Mason * in cache without setting the path to blocking. If we find the block 2433c8c42864SChris Mason * we return zero and the path is unchanged. 2434c8c42864SChris Mason * 2435c8c42864SChris Mason * If we can't find the block, we set the path blocking and do some 2436c8c42864SChris Mason * reada. -EAGAIN is returned and the search must be repeated. 2437c8c42864SChris Mason */ 2438c8c42864SChris Mason static int 2439d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 2440c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 2441cda79c54SDavid Sterba const struct btrfs_key *key) 2442c8c42864SChris Mason { 24430b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2444c8c42864SChris Mason u64 blocknr; 2445c8c42864SChris Mason u64 gen; 2446c8c42864SChris Mason struct extent_buffer *b = *eb_ret; 2447c8c42864SChris Mason struct extent_buffer *tmp; 244876a05b35SChris Mason int ret; 2449c8c42864SChris Mason 2450c8c42864SChris Mason blocknr = btrfs_node_blockptr(b, slot); 2451c8c42864SChris Mason gen = btrfs_node_ptr_generation(b, slot); 2452c8c42864SChris Mason 24530b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 2454cb44921aSChris Mason if (tmp) { 2455b9fab919SChris Mason /* first we do an atomic uptodate check */ 2456b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 2457c8c42864SChris Mason *eb_ret = tmp; 2458c8c42864SChris Mason return 0; 2459c8c42864SChris Mason } 2460bdf7c00eSJosef Bacik 2461cb44921aSChris Mason /* the pages were up to date, but we failed 2462cb44921aSChris Mason * the generation number check. Do a full 2463cb44921aSChris Mason * read for the generation number that is correct. 2464cb44921aSChris Mason * We must do this without dropping locks so 2465cb44921aSChris Mason * we can trust our generation number 2466cb44921aSChris Mason */ 2467bd681513SChris Mason btrfs_set_path_blocking(p); 2468bd681513SChris Mason 2469b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 2470bdf7c00eSJosef Bacik ret = btrfs_read_buffer(tmp, gen); 2471bdf7c00eSJosef Bacik if (!ret) { 2472cb44921aSChris Mason *eb_ret = tmp; 2473cb44921aSChris Mason return 0; 2474cb44921aSChris Mason } 2475cb44921aSChris Mason free_extent_buffer(tmp); 2476b3b4aa74SDavid Sterba btrfs_release_path(p); 2477cb44921aSChris Mason return -EIO; 2478cb44921aSChris Mason } 2479c8c42864SChris Mason 2480c8c42864SChris Mason /* 2481c8c42864SChris Mason * reduce lock contention at high levels 2482c8c42864SChris Mason * of the btree by dropping locks before 248376a05b35SChris Mason * we read. Don't release the lock on the current 248476a05b35SChris Mason * level because we need to walk this node to figure 248576a05b35SChris Mason * out which blocks to read. 2486c8c42864SChris Mason */ 24878c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 24888c594ea8SChris Mason btrfs_set_path_blocking(p); 24898c594ea8SChris Mason 2490c8c42864SChris Mason free_extent_buffer(tmp); 2491e4058b54SDavid Sterba if (p->reada != READA_NONE) 24922ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 2493c8c42864SChris Mason 2494b3b4aa74SDavid Sterba btrfs_release_path(p); 249576a05b35SChris Mason 249676a05b35SChris Mason ret = -EAGAIN; 24972ff7e61eSJeff Mahoney tmp = read_tree_block(fs_info, blocknr, 0); 249864c043deSLiu Bo if (!IS_ERR(tmp)) { 249976a05b35SChris Mason /* 250076a05b35SChris Mason * If the read above didn't mark this buffer up to date, 250176a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 250276a05b35SChris Mason * and give up so that our caller doesn't loop forever 250376a05b35SChris Mason * on our EAGAINs. 250476a05b35SChris Mason */ 2505b9fab919SChris Mason if (!btrfs_buffer_uptodate(tmp, 0, 0)) 250676a05b35SChris Mason ret = -EIO; 2507c8c42864SChris Mason free_extent_buffer(tmp); 2508c871b0f2SLiu Bo } else { 2509c871b0f2SLiu Bo ret = PTR_ERR(tmp); 251076a05b35SChris Mason } 251176a05b35SChris Mason return ret; 2512c8c42864SChris Mason } 2513c8c42864SChris Mason 2514c8c42864SChris Mason /* 2515c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 2516c8c42864SChris Mason * for node-level blocks and does any balancing required based on 2517c8c42864SChris Mason * the ins_len. 2518c8c42864SChris Mason * 2519c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 2520c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 2521c8c42864SChris Mason * start over 2522c8c42864SChris Mason */ 2523c8c42864SChris Mason static int 2524c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 2525c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2526bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 2527bd681513SChris Mason int *write_lock_level) 2528c8c42864SChris Mason { 25290b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2530c8c42864SChris Mason int ret; 25310b246afaSJeff Mahoney 2532c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 25330b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 2534c8c42864SChris Mason int sret; 2535c8c42864SChris Mason 2536bd681513SChris Mason if (*write_lock_level < level + 1) { 2537bd681513SChris Mason *write_lock_level = level + 1; 2538bd681513SChris Mason btrfs_release_path(p); 2539bd681513SChris Mason goto again; 2540bd681513SChris Mason } 2541bd681513SChris Mason 2542c8c42864SChris Mason btrfs_set_path_blocking(p); 25432ff7e61eSJeff Mahoney reada_for_balance(fs_info, p, level); 2544c8c42864SChris Mason sret = split_node(trans, root, p, level); 2545bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2546c8c42864SChris Mason 2547c8c42864SChris Mason BUG_ON(sret > 0); 2548c8c42864SChris Mason if (sret) { 2549c8c42864SChris Mason ret = sret; 2550c8c42864SChris Mason goto done; 2551c8c42864SChris Mason } 2552c8c42864SChris Mason b = p->nodes[level]; 2553c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 25540b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 2555c8c42864SChris Mason int sret; 2556c8c42864SChris Mason 2557bd681513SChris Mason if (*write_lock_level < level + 1) { 2558bd681513SChris Mason *write_lock_level = level + 1; 2559bd681513SChris Mason btrfs_release_path(p); 2560bd681513SChris Mason goto again; 2561bd681513SChris Mason } 2562bd681513SChris Mason 2563c8c42864SChris Mason btrfs_set_path_blocking(p); 25642ff7e61eSJeff Mahoney reada_for_balance(fs_info, p, level); 2565c8c42864SChris Mason sret = balance_level(trans, root, p, level); 2566bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2567c8c42864SChris Mason 2568c8c42864SChris Mason if (sret) { 2569c8c42864SChris Mason ret = sret; 2570c8c42864SChris Mason goto done; 2571c8c42864SChris Mason } 2572c8c42864SChris Mason b = p->nodes[level]; 2573c8c42864SChris Mason if (!b) { 2574b3b4aa74SDavid Sterba btrfs_release_path(p); 2575c8c42864SChris Mason goto again; 2576c8c42864SChris Mason } 2577c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 2578c8c42864SChris Mason } 2579c8c42864SChris Mason return 0; 2580c8c42864SChris Mason 2581c8c42864SChris Mason again: 2582c8c42864SChris Mason ret = -EAGAIN; 2583c8c42864SChris Mason done: 2584c8c42864SChris Mason return ret; 2585c8c42864SChris Mason } 2586c8c42864SChris Mason 2587d7396f07SFilipe David Borba Manana static void key_search_validate(struct extent_buffer *b, 2588310712b2SOmar Sandoval const struct btrfs_key *key, 2589d7396f07SFilipe David Borba Manana int level) 2590d7396f07SFilipe David Borba Manana { 2591d7396f07SFilipe David Borba Manana #ifdef CONFIG_BTRFS_ASSERT 2592d7396f07SFilipe David Borba Manana struct btrfs_disk_key disk_key; 2593d7396f07SFilipe David Borba Manana 2594d7396f07SFilipe David Borba Manana btrfs_cpu_key_to_disk(&disk_key, key); 2595d7396f07SFilipe David Borba Manana 2596d7396f07SFilipe David Borba Manana if (level == 0) 2597d7396f07SFilipe David Borba Manana ASSERT(!memcmp_extent_buffer(b, &disk_key, 2598d7396f07SFilipe David Borba Manana offsetof(struct btrfs_leaf, items[0].key), 2599d7396f07SFilipe David Borba Manana sizeof(disk_key))); 2600d7396f07SFilipe David Borba Manana else 2601d7396f07SFilipe David Borba Manana ASSERT(!memcmp_extent_buffer(b, &disk_key, 2602d7396f07SFilipe David Borba Manana offsetof(struct btrfs_node, ptrs[0].key), 2603d7396f07SFilipe David Borba Manana sizeof(disk_key))); 2604d7396f07SFilipe David Borba Manana #endif 2605d7396f07SFilipe David Borba Manana } 2606d7396f07SFilipe David Borba Manana 2607310712b2SOmar Sandoval static int key_search(struct extent_buffer *b, const struct btrfs_key *key, 2608d7396f07SFilipe David Borba Manana int level, int *prev_cmp, int *slot) 2609d7396f07SFilipe David Borba Manana { 2610d7396f07SFilipe David Borba Manana if (*prev_cmp != 0) { 2611a74b35ecSNikolay Borisov *prev_cmp = btrfs_bin_search(b, key, level, slot); 2612d7396f07SFilipe David Borba Manana return *prev_cmp; 2613d7396f07SFilipe David Borba Manana } 2614d7396f07SFilipe David Borba Manana 2615d7396f07SFilipe David Borba Manana key_search_validate(b, key, level); 2616d7396f07SFilipe David Borba Manana *slot = 0; 2617d7396f07SFilipe David Borba Manana 2618d7396f07SFilipe David Borba Manana return 0; 2619d7396f07SFilipe David Borba Manana } 2620d7396f07SFilipe David Borba Manana 2621381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 2622e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 2623e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 2624e33d5c3dSKelley Nielsen { 2625e33d5c3dSKelley Nielsen int ret; 2626e33d5c3dSKelley Nielsen struct btrfs_key key; 2627e33d5c3dSKelley Nielsen struct extent_buffer *eb; 2628381cf658SDavid Sterba 2629381cf658SDavid Sterba ASSERT(path); 26301d4c08e0SDavid Sterba ASSERT(found_key); 2631e33d5c3dSKelley Nielsen 2632e33d5c3dSKelley Nielsen key.type = key_type; 2633e33d5c3dSKelley Nielsen key.objectid = iobjectid; 2634e33d5c3dSKelley Nielsen key.offset = ioff; 2635e33d5c3dSKelley Nielsen 2636e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 26371d4c08e0SDavid Sterba if (ret < 0) 2638e33d5c3dSKelley Nielsen return ret; 2639e33d5c3dSKelley Nielsen 2640e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2641e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 2642e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 2643e33d5c3dSKelley Nielsen if (ret) 2644e33d5c3dSKelley Nielsen return ret; 2645e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2646e33d5c3dSKelley Nielsen } 2647e33d5c3dSKelley Nielsen 2648e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 2649e33d5c3dSKelley Nielsen if (found_key->type != key.type || 2650e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 2651e33d5c3dSKelley Nielsen return 1; 2652e33d5c3dSKelley Nielsen 2653e33d5c3dSKelley Nielsen return 0; 2654e33d5c3dSKelley Nielsen } 2655e33d5c3dSKelley Nielsen 2656c8c42864SChris Mason /* 26574271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 26584271eceaSNikolay Borisov * modifications to preserve tree invariants. 265974123bd7SChris Mason * 26604271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 26614271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 26624271eceaSNikolay Borisov * @root: The root node of the tree 26634271eceaSNikolay Borisov * @key: The key we are looking for 26644271eceaSNikolay Borisov * @ins_len: Indicates purpose of search, for inserts it is 1, for 26654271eceaSNikolay Borisov * deletions it's -1. 0 for plain searches 26664271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 26674271eceaSNikolay Borisov * when modifying the tree. 266897571fd0SChris Mason * 26694271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 26704271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 26714271eceaSNikolay Borisov * 26724271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 26734271eceaSNikolay Borisov * of the path (level 0) 26744271eceaSNikolay Borisov * 26754271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 26764271eceaSNikolay Borisov * points to the slot where it should be inserted 26774271eceaSNikolay Borisov * 26784271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 26794271eceaSNikolay Borisov * is returned 268074123bd7SChris Mason */ 2681310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 2682310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 2683310712b2SOmar Sandoval int ins_len, int cow) 2684be0e5c09SChris Mason { 26850b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 26865f39d397SChris Mason struct extent_buffer *b; 2687be0e5c09SChris Mason int slot; 2688be0e5c09SChris Mason int ret; 268933c66f43SYan Zheng int err; 2690be0e5c09SChris Mason int level; 2691925baeddSChris Mason int lowest_unlock = 1; 2692bd681513SChris Mason int root_lock; 2693bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2694bd681513SChris Mason int write_lock_level = 0; 26959f3a7427SChris Mason u8 lowest_level = 0; 2696f7c79f30SChris Mason int min_write_lock_level; 2697d7396f07SFilipe David Borba Manana int prev_cmp; 26989f3a7427SChris Mason 26996702ed49SChris Mason lowest_level = p->lowest_level; 2700323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 270122b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2702eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 270325179201SJosef Bacik 2704bd681513SChris Mason if (ins_len < 0) { 2705925baeddSChris Mason lowest_unlock = 2; 270665b51a00SChris Mason 2707bd681513SChris Mason /* when we are removing items, we might have to go up to level 2708bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2709bd681513SChris Mason * for those levels as well 2710bd681513SChris Mason */ 2711bd681513SChris Mason write_lock_level = 2; 2712bd681513SChris Mason } else if (ins_len > 0) { 2713bd681513SChris Mason /* 2714bd681513SChris Mason * for inserting items, make sure we have a write lock on 2715bd681513SChris Mason * level 1 so we can update keys 2716bd681513SChris Mason */ 2717bd681513SChris Mason write_lock_level = 1; 2718bd681513SChris Mason } 2719bd681513SChris Mason 2720bd681513SChris Mason if (!cow) 2721bd681513SChris Mason write_lock_level = -1; 2722bd681513SChris Mason 272309a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2724bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2725bd681513SChris Mason 2726f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2727f7c79f30SChris Mason 2728bb803951SChris Mason again: 2729d7396f07SFilipe David Borba Manana prev_cmp = -1; 2730bd681513SChris Mason /* 2731bd681513SChris Mason * we try very hard to do read locks on the root 2732bd681513SChris Mason */ 2733bd681513SChris Mason root_lock = BTRFS_READ_LOCK; 2734bd681513SChris Mason level = 0; 27355d4f98a2SYan Zheng if (p->search_commit_root) { 2736bd681513SChris Mason /* 2737bd681513SChris Mason * the commit roots are read only 2738bd681513SChris Mason * so we always do read locks 2739bd681513SChris Mason */ 27403f8a18ccSJosef Bacik if (p->need_commit_sem) 27410b246afaSJeff Mahoney down_read(&fs_info->commit_root_sem); 27425d4f98a2SYan Zheng b = root->commit_root; 27435d4f98a2SYan Zheng extent_buffer_get(b); 2744bd681513SChris Mason level = btrfs_header_level(b); 27453f8a18ccSJosef Bacik if (p->need_commit_sem) 27460b246afaSJeff Mahoney up_read(&fs_info->commit_root_sem); 27475d4f98a2SYan Zheng if (!p->skip_locking) 2748bd681513SChris Mason btrfs_tree_read_lock(b); 27495d4f98a2SYan Zheng } else { 2750bd681513SChris Mason if (p->skip_locking) { 27515cd57b2cSChris Mason b = btrfs_root_node(root); 2752bd681513SChris Mason level = btrfs_header_level(b); 2753bd681513SChris Mason } else { 2754bd681513SChris Mason /* we don't know the level of the root node 2755bd681513SChris Mason * until we actually have it read locked 2756bd681513SChris Mason */ 2757bd681513SChris Mason b = btrfs_read_lock_root_node(root); 2758bd681513SChris Mason level = btrfs_header_level(b); 2759bd681513SChris Mason if (level <= write_lock_level) { 2760bd681513SChris Mason /* whoops, must trade for write lock */ 2761bd681513SChris Mason btrfs_tree_read_unlock(b); 2762bd681513SChris Mason free_extent_buffer(b); 2763925baeddSChris Mason b = btrfs_lock_root_node(root); 2764bd681513SChris Mason root_lock = BTRFS_WRITE_LOCK; 2765bd681513SChris Mason 2766bd681513SChris Mason /* the level might have changed, check again */ 2767bd681513SChris Mason level = btrfs_header_level(b); 27685d4f98a2SYan Zheng } 2769bd681513SChris Mason } 2770bd681513SChris Mason } 2771bd681513SChris Mason p->nodes[level] = b; 2772bd681513SChris Mason if (!p->skip_locking) 2773bd681513SChris Mason p->locks[level] = root_lock; 2774925baeddSChris Mason 2775eb60ceacSChris Mason while (b) { 27765f39d397SChris Mason level = btrfs_header_level(b); 277765b51a00SChris Mason 277865b51a00SChris Mason /* 277965b51a00SChris Mason * setup the path here so we can release it under lock 278065b51a00SChris Mason * contention with the cow code 278165b51a00SChris Mason */ 278202217ed2SChris Mason if (cow) { 27839ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 27849ea2c7c9SNikolay Borisov 2785c8c42864SChris Mason /* 2786c8c42864SChris Mason * if we don't really need to cow this block 2787c8c42864SChris Mason * then we don't want to set the path blocking, 2788c8c42864SChris Mason * so we test it here 2789c8c42864SChris Mason */ 279064c12921SJeff Mahoney if (!should_cow_block(trans, root, b)) { 279164c12921SJeff Mahoney trans->dirty = true; 279265b51a00SChris Mason goto cow_done; 279364c12921SJeff Mahoney } 27945d4f98a2SYan Zheng 2795bd681513SChris Mason /* 2796bd681513SChris Mason * must have write locks on this node and the 2797bd681513SChris Mason * parent 2798bd681513SChris Mason */ 27995124e00eSJosef Bacik if (level > write_lock_level || 28005124e00eSJosef Bacik (level + 1 > write_lock_level && 28015124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 28025124e00eSJosef Bacik p->nodes[level + 1])) { 2803bd681513SChris Mason write_lock_level = level + 1; 2804bd681513SChris Mason btrfs_release_path(p); 2805bd681513SChris Mason goto again; 2806bd681513SChris Mason } 2807bd681513SChris Mason 2808160f4089SFilipe Manana btrfs_set_path_blocking(p); 28099ea2c7c9SNikolay Borisov if (last_level) 28109ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 28119ea2c7c9SNikolay Borisov &b); 28129ea2c7c9SNikolay Borisov else 281333c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2814e20d96d6SChris Mason p->nodes[level + 1], 28159fa8cfe7SChris Mason p->slots[level + 1], &b); 281633c66f43SYan Zheng if (err) { 281733c66f43SYan Zheng ret = err; 281865b51a00SChris Mason goto done; 281954aa1f4dSChris Mason } 282002217ed2SChris Mason } 282165b51a00SChris Mason cow_done: 2822eb60ceacSChris Mason p->nodes[level] = b; 2823bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2824b4ce94deSChris Mason 2825b4ce94deSChris Mason /* 2826b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2827b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2828b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2829b4ce94deSChris Mason * go through the expensive btree search on b. 2830b4ce94deSChris Mason * 2831eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2832eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2833eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2834eb653de1SFilipe David Borba Manana * we're operating on. 2835b4ce94deSChris Mason */ 2836eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2837eb653de1SFilipe David Borba Manana int u = level + 1; 2838eb653de1SFilipe David Borba Manana 2839eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2840eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2841eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2842eb653de1SFilipe David Borba Manana } 2843eb653de1SFilipe David Borba Manana } 2844b4ce94deSChris Mason 2845d7396f07SFilipe David Borba Manana ret = key_search(b, key, level, &prev_cmp, &slot); 2846415b35a5SLiu Bo if (ret < 0) 2847415b35a5SLiu Bo goto done; 2848b4ce94deSChris Mason 28495f39d397SChris Mason if (level != 0) { 285033c66f43SYan Zheng int dec = 0; 285133c66f43SYan Zheng if (ret && slot > 0) { 285233c66f43SYan Zheng dec = 1; 2853be0e5c09SChris Mason slot -= 1; 285433c66f43SYan Zheng } 2855be0e5c09SChris Mason p->slots[level] = slot; 285633c66f43SYan Zheng err = setup_nodes_for_search(trans, root, p, b, level, 2857bd681513SChris Mason ins_len, &write_lock_level); 285833c66f43SYan Zheng if (err == -EAGAIN) 2859b4ce94deSChris Mason goto again; 286033c66f43SYan Zheng if (err) { 286133c66f43SYan Zheng ret = err; 286265b51a00SChris Mason goto done; 286333c66f43SYan Zheng } 28645c680ed6SChris Mason b = p->nodes[level]; 28655c680ed6SChris Mason slot = p->slots[level]; 2866b4ce94deSChris Mason 2867bd681513SChris Mason /* 2868bd681513SChris Mason * slot 0 is special, if we change the key 2869bd681513SChris Mason * we have to update the parent pointer 2870bd681513SChris Mason * which means we must have a write lock 2871bd681513SChris Mason * on the parent 2872bd681513SChris Mason */ 2873eb653de1SFilipe David Borba Manana if (slot == 0 && ins_len && 2874bd681513SChris Mason write_lock_level < level + 1) { 2875bd681513SChris Mason write_lock_level = level + 1; 2876bd681513SChris Mason btrfs_release_path(p); 2877bd681513SChris Mason goto again; 2878bd681513SChris Mason } 2879bd681513SChris Mason 2880f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2881f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 2882f9efa9c7SChris Mason 2883925baeddSChris Mason if (level == lowest_level) { 288433c66f43SYan Zheng if (dec) 288533c66f43SYan Zheng p->slots[level]++; 28865b21f2edSZheng Yan goto done; 2887925baeddSChris Mason } 2888ca7a79adSChris Mason 2889d07b8528SLiu Bo err = read_block_for_search(root, p, &b, level, 2890cda79c54SDavid Sterba slot, key); 289133c66f43SYan Zheng if (err == -EAGAIN) 2892051e1b9fSChris Mason goto again; 289333c66f43SYan Zheng if (err) { 289433c66f43SYan Zheng ret = err; 289576a05b35SChris Mason goto done; 289633c66f43SYan Zheng } 289776a05b35SChris Mason 2898b4ce94deSChris Mason if (!p->skip_locking) { 2899bd681513SChris Mason level = btrfs_header_level(b); 2900bd681513SChris Mason if (level <= write_lock_level) { 2901bd681513SChris Mason err = btrfs_try_tree_write_lock(b); 290233c66f43SYan Zheng if (!err) { 2903b4ce94deSChris Mason btrfs_set_path_blocking(p); 2904925baeddSChris Mason btrfs_tree_lock(b); 2905bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2906bd681513SChris Mason BTRFS_WRITE_LOCK); 2907b4ce94deSChris Mason } 2908bd681513SChris Mason p->locks[level] = BTRFS_WRITE_LOCK; 2909bd681513SChris Mason } else { 2910f82c458aSChris Mason err = btrfs_tree_read_lock_atomic(b); 2911bd681513SChris Mason if (!err) { 2912bd681513SChris Mason btrfs_set_path_blocking(p); 2913bd681513SChris Mason btrfs_tree_read_lock(b); 2914bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2915bd681513SChris Mason BTRFS_READ_LOCK); 2916bd681513SChris Mason } 2917bd681513SChris Mason p->locks[level] = BTRFS_READ_LOCK; 2918bd681513SChris Mason } 2919bd681513SChris Mason p->nodes[level] = b; 2920b4ce94deSChris Mason } 2921be0e5c09SChris Mason } else { 2922be0e5c09SChris Mason p->slots[level] = slot; 292387b29b20SYan Zheng if (ins_len > 0 && 29242ff7e61eSJeff Mahoney btrfs_leaf_free_space(fs_info, b) < ins_len) { 2925bd681513SChris Mason if (write_lock_level < 1) { 2926bd681513SChris Mason write_lock_level = 1; 2927bd681513SChris Mason btrfs_release_path(p); 2928bd681513SChris Mason goto again; 2929bd681513SChris Mason } 2930bd681513SChris Mason 2931b4ce94deSChris Mason btrfs_set_path_blocking(p); 293233c66f43SYan Zheng err = split_leaf(trans, root, key, 2933cc0c5538SChris Mason p, ins_len, ret == 0); 2934bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2935b4ce94deSChris Mason 293633c66f43SYan Zheng BUG_ON(err > 0); 293733c66f43SYan Zheng if (err) { 293833c66f43SYan Zheng ret = err; 293965b51a00SChris Mason goto done; 294065b51a00SChris Mason } 29415c680ed6SChris Mason } 2942459931ecSChris Mason if (!p->search_for_split) 2943f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2944f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 294565b51a00SChris Mason goto done; 294665b51a00SChris Mason } 294765b51a00SChris Mason } 294865b51a00SChris Mason ret = 1; 294965b51a00SChris Mason done: 2950b4ce94deSChris Mason /* 2951b4ce94deSChris Mason * we don't really know what they plan on doing with the path 2952b4ce94deSChris Mason * from here on, so for now just mark it as blocking 2953b4ce94deSChris Mason */ 2954b9473439SChris Mason if (!p->leave_spinning) 2955b4ce94deSChris Mason btrfs_set_path_blocking(p); 29565f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2957b3b4aa74SDavid Sterba btrfs_release_path(p); 2958be0e5c09SChris Mason return ret; 2959be0e5c09SChris Mason } 2960be0e5c09SChris Mason 296174123bd7SChris Mason /* 29625d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 29635d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 29645d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 29655d9e75c4SJan Schmidt * denoted by the time_seq parameter. 29665d9e75c4SJan Schmidt * 29675d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 29685d9e75c4SJan Schmidt * 29695d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 29705d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 29715d9e75c4SJan Schmidt */ 2972310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 29735d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 29745d9e75c4SJan Schmidt { 29750b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 29765d9e75c4SJan Schmidt struct extent_buffer *b; 29775d9e75c4SJan Schmidt int slot; 29785d9e75c4SJan Schmidt int ret; 29795d9e75c4SJan Schmidt int err; 29805d9e75c4SJan Schmidt int level; 29815d9e75c4SJan Schmidt int lowest_unlock = 1; 29825d9e75c4SJan Schmidt u8 lowest_level = 0; 2983d4b4087cSJosef Bacik int prev_cmp = -1; 29845d9e75c4SJan Schmidt 29855d9e75c4SJan Schmidt lowest_level = p->lowest_level; 29865d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 29875d9e75c4SJan Schmidt 29885d9e75c4SJan Schmidt if (p->search_commit_root) { 29895d9e75c4SJan Schmidt BUG_ON(time_seq); 29905d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 29915d9e75c4SJan Schmidt } 29925d9e75c4SJan Schmidt 29935d9e75c4SJan Schmidt again: 29945d9e75c4SJan Schmidt b = get_old_root(root, time_seq); 29955d9e75c4SJan Schmidt level = btrfs_header_level(b); 29965d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 29975d9e75c4SJan Schmidt 29985d9e75c4SJan Schmidt while (b) { 29995d9e75c4SJan Schmidt level = btrfs_header_level(b); 30005d9e75c4SJan Schmidt p->nodes[level] = b; 30015d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, NULL, 0); 30025d9e75c4SJan Schmidt 30035d9e75c4SJan Schmidt /* 30045d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 30055d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 30065d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 30075d9e75c4SJan Schmidt * go through the expensive btree search on b. 30085d9e75c4SJan Schmidt */ 30095d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 30105d9e75c4SJan Schmidt 3011d4b4087cSJosef Bacik /* 301201327610SNicholas D Steeves * Since we can unwind ebs we want to do a real search every 3013d4b4087cSJosef Bacik * time. 3014d4b4087cSJosef Bacik */ 3015d4b4087cSJosef Bacik prev_cmp = -1; 3016d7396f07SFilipe David Borba Manana ret = key_search(b, key, level, &prev_cmp, &slot); 30175d9e75c4SJan Schmidt 30185d9e75c4SJan Schmidt if (level != 0) { 30195d9e75c4SJan Schmidt int dec = 0; 30205d9e75c4SJan Schmidt if (ret && slot > 0) { 30215d9e75c4SJan Schmidt dec = 1; 30225d9e75c4SJan Schmidt slot -= 1; 30235d9e75c4SJan Schmidt } 30245d9e75c4SJan Schmidt p->slots[level] = slot; 30255d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 30265d9e75c4SJan Schmidt 30275d9e75c4SJan Schmidt if (level == lowest_level) { 30285d9e75c4SJan Schmidt if (dec) 30295d9e75c4SJan Schmidt p->slots[level]++; 30305d9e75c4SJan Schmidt goto done; 30315d9e75c4SJan Schmidt } 30325d9e75c4SJan Schmidt 3033d07b8528SLiu Bo err = read_block_for_search(root, p, &b, level, 3034cda79c54SDavid Sterba slot, key); 30355d9e75c4SJan Schmidt if (err == -EAGAIN) 30365d9e75c4SJan Schmidt goto again; 30375d9e75c4SJan Schmidt if (err) { 30385d9e75c4SJan Schmidt ret = err; 30395d9e75c4SJan Schmidt goto done; 30405d9e75c4SJan Schmidt } 30415d9e75c4SJan Schmidt 30425d9e75c4SJan Schmidt level = btrfs_header_level(b); 3043f82c458aSChris Mason err = btrfs_tree_read_lock_atomic(b); 30445d9e75c4SJan Schmidt if (!err) { 30455d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 30465d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 30475d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, b, 30485d9e75c4SJan Schmidt BTRFS_READ_LOCK); 30495d9e75c4SJan Schmidt } 30500b246afaSJeff Mahoney b = tree_mod_log_rewind(fs_info, p, b, time_seq); 3051db7f3436SJosef Bacik if (!b) { 3052db7f3436SJosef Bacik ret = -ENOMEM; 3053db7f3436SJosef Bacik goto done; 3054db7f3436SJosef Bacik } 30555d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 30565d9e75c4SJan Schmidt p->nodes[level] = b; 30575d9e75c4SJan Schmidt } else { 30585d9e75c4SJan Schmidt p->slots[level] = slot; 30595d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 30605d9e75c4SJan Schmidt goto done; 30615d9e75c4SJan Schmidt } 30625d9e75c4SJan Schmidt } 30635d9e75c4SJan Schmidt ret = 1; 30645d9e75c4SJan Schmidt done: 30655d9e75c4SJan Schmidt if (!p->leave_spinning) 30665d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 30675d9e75c4SJan Schmidt if (ret < 0) 30685d9e75c4SJan Schmidt btrfs_release_path(p); 30695d9e75c4SJan Schmidt 30705d9e75c4SJan Schmidt return ret; 30715d9e75c4SJan Schmidt } 30725d9e75c4SJan Schmidt 30735d9e75c4SJan Schmidt /* 30742f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 30752f38b3e1SArne Jansen * instead the next or previous item should be returned. 30762f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 30772f38b3e1SArne Jansen * otherwise. 30782f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 30792f38b3e1SArne Jansen * return the next lower instead. 30802f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 30812f38b3e1SArne Jansen * return the next higher instead. 30822f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 30832f38b3e1SArne Jansen * < 0 on error 30842f38b3e1SArne Jansen */ 30852f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 3086310712b2SOmar Sandoval const struct btrfs_key *key, 3087310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 3088310712b2SOmar Sandoval int return_any) 30892f38b3e1SArne Jansen { 30902f38b3e1SArne Jansen int ret; 30912f38b3e1SArne Jansen struct extent_buffer *leaf; 30922f38b3e1SArne Jansen 30932f38b3e1SArne Jansen again: 30942f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 30952f38b3e1SArne Jansen if (ret <= 0) 30962f38b3e1SArne Jansen return ret; 30972f38b3e1SArne Jansen /* 30982f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 30992f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 31002f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 31012f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 31022f38b3e1SArne Jansen * item. 31032f38b3e1SArne Jansen */ 31042f38b3e1SArne Jansen leaf = p->nodes[0]; 31052f38b3e1SArne Jansen 31062f38b3e1SArne Jansen if (find_higher) { 31072f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 31082f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 31092f38b3e1SArne Jansen if (ret <= 0) 31102f38b3e1SArne Jansen return ret; 31112f38b3e1SArne Jansen if (!return_any) 31122f38b3e1SArne Jansen return 1; 31132f38b3e1SArne Jansen /* 31142f38b3e1SArne Jansen * no higher item found, return the next 31152f38b3e1SArne Jansen * lower instead 31162f38b3e1SArne Jansen */ 31172f38b3e1SArne Jansen return_any = 0; 31182f38b3e1SArne Jansen find_higher = 0; 31192f38b3e1SArne Jansen btrfs_release_path(p); 31202f38b3e1SArne Jansen goto again; 31212f38b3e1SArne Jansen } 31222f38b3e1SArne Jansen } else { 31232f38b3e1SArne Jansen if (p->slots[0] == 0) { 31242f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 3125e6793769SArne Jansen if (ret < 0) 31262f38b3e1SArne Jansen return ret; 3127e6793769SArne Jansen if (!ret) { 312823c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 312923c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 313023c6bf6aSFilipe David Borba Manana p->slots[0]--; 3131e6793769SArne Jansen return 0; 3132e6793769SArne Jansen } 31332f38b3e1SArne Jansen if (!return_any) 31342f38b3e1SArne Jansen return 1; 31352f38b3e1SArne Jansen /* 31362f38b3e1SArne Jansen * no lower item found, return the next 31372f38b3e1SArne Jansen * higher instead 31382f38b3e1SArne Jansen */ 31392f38b3e1SArne Jansen return_any = 0; 31402f38b3e1SArne Jansen find_higher = 1; 31412f38b3e1SArne Jansen btrfs_release_path(p); 31422f38b3e1SArne Jansen goto again; 3143e6793769SArne Jansen } else { 31442f38b3e1SArne Jansen --p->slots[0]; 31452f38b3e1SArne Jansen } 31462f38b3e1SArne Jansen } 31472f38b3e1SArne Jansen return 0; 31482f38b3e1SArne Jansen } 31492f38b3e1SArne Jansen 31502f38b3e1SArne Jansen /* 315174123bd7SChris Mason * adjust the pointers going up the tree, starting at level 315274123bd7SChris Mason * making sure the right key of each node is points to 'key'. 315374123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 315474123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 315574123bd7SChris Mason * higher levels 3156aa5d6bedSChris Mason * 315774123bd7SChris Mason */ 3158b7a0365eSDaniel Dressler static void fixup_low_keys(struct btrfs_fs_info *fs_info, 3159b7a0365eSDaniel Dressler struct btrfs_path *path, 31605f39d397SChris Mason struct btrfs_disk_key *key, int level) 3161be0e5c09SChris Mason { 3162be0e5c09SChris Mason int i; 31635f39d397SChris Mason struct extent_buffer *t; 31645f39d397SChris Mason 3165234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 3166be0e5c09SChris Mason int tslot = path->slots[i]; 3167eb60ceacSChris Mason if (!path->nodes[i]) 3168be0e5c09SChris Mason break; 31695f39d397SChris Mason t = path->nodes[i]; 3170b7a0365eSDaniel Dressler tree_mod_log_set_node_key(fs_info, t, tslot, 1); 31715f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 3172d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 3173be0e5c09SChris Mason if (tslot != 0) 3174be0e5c09SChris Mason break; 3175be0e5c09SChris Mason } 3176be0e5c09SChris Mason } 3177be0e5c09SChris Mason 317874123bd7SChris Mason /* 317931840ae1SZheng Yan * update item key. 318031840ae1SZheng Yan * 318131840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 318231840ae1SZheng Yan * that the new key won't break the order 318331840ae1SZheng Yan */ 3184b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 3185b7a0365eSDaniel Dressler struct btrfs_path *path, 3186310712b2SOmar Sandoval const struct btrfs_key *new_key) 318731840ae1SZheng Yan { 318831840ae1SZheng Yan struct btrfs_disk_key disk_key; 318931840ae1SZheng Yan struct extent_buffer *eb; 319031840ae1SZheng Yan int slot; 319131840ae1SZheng Yan 319231840ae1SZheng Yan eb = path->nodes[0]; 319331840ae1SZheng Yan slot = path->slots[0]; 319431840ae1SZheng Yan if (slot > 0) { 319531840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 3196143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) >= 0); 319731840ae1SZheng Yan } 319831840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 319931840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 3200143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) <= 0); 320131840ae1SZheng Yan } 320231840ae1SZheng Yan 320331840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 320431840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 320531840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 320631840ae1SZheng Yan if (slot == 0) 3207b7a0365eSDaniel Dressler fixup_low_keys(fs_info, path, &disk_key, 1); 320831840ae1SZheng Yan } 320931840ae1SZheng Yan 321031840ae1SZheng Yan /* 321174123bd7SChris Mason * try to push data from one node into the next node left in the 321279f95c82SChris Mason * tree. 3213aa5d6bedSChris Mason * 3214aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 3215aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 321674123bd7SChris Mason */ 321798ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 32182ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 32192ff7e61eSJeff Mahoney struct extent_buffer *dst, 3220971a1f66SChris Mason struct extent_buffer *src, int empty) 3221be0e5c09SChris Mason { 3222be0e5c09SChris Mason int push_items = 0; 3223bb803951SChris Mason int src_nritems; 3224bb803951SChris Mason int dst_nritems; 3225aa5d6bedSChris Mason int ret = 0; 3226be0e5c09SChris Mason 32275f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 32285f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 32290b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 32307bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 32317bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 323254aa1f4dSChris Mason 3233bce4eae9SChris Mason if (!empty && src_nritems <= 8) 3234971a1f66SChris Mason return 1; 3235971a1f66SChris Mason 3236d397712bSChris Mason if (push_items <= 0) 3237be0e5c09SChris Mason return 1; 3238be0e5c09SChris Mason 3239bce4eae9SChris Mason if (empty) { 3240971a1f66SChris Mason push_items = min(src_nritems, push_items); 3241bce4eae9SChris Mason if (push_items < src_nritems) { 3242bce4eae9SChris Mason /* leave at least 8 pointers in the node if 3243bce4eae9SChris Mason * we aren't going to empty it 3244bce4eae9SChris Mason */ 3245bce4eae9SChris Mason if (src_nritems - push_items < 8) { 3246bce4eae9SChris Mason if (push_items <= 8) 3247bce4eae9SChris Mason return 1; 3248bce4eae9SChris Mason push_items -= 8; 3249bce4eae9SChris Mason } 3250bce4eae9SChris Mason } 3251bce4eae9SChris Mason } else 3252bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 325379f95c82SChris Mason 32540b246afaSJeff Mahoney ret = tree_mod_log_eb_copy(fs_info, dst, src, dst_nritems, 0, 325590f8d62eSJan Schmidt push_items); 32565de865eeSFilipe David Borba Manana if (ret) { 325766642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 32585de865eeSFilipe David Borba Manana return ret; 32595de865eeSFilipe David Borba Manana } 32605f39d397SChris Mason copy_extent_buffer(dst, src, 32615f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 32625f39d397SChris Mason btrfs_node_key_ptr_offset(0), 3263123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 32645f39d397SChris Mason 3265bb803951SChris Mason if (push_items < src_nritems) { 326657911b8bSJan Schmidt /* 326757911b8bSJan Schmidt * don't call tree_mod_log_eb_move here, key removal was already 326857911b8bSJan Schmidt * fully logged by tree_mod_log_eb_copy above. 326957911b8bSJan Schmidt */ 32705f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 32715f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 3272e2fa7227SChris Mason (src_nritems - push_items) * 3273123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 3274bb803951SChris Mason } 32755f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 32765f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 32775f39d397SChris Mason btrfs_mark_buffer_dirty(src); 32785f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 327931840ae1SZheng Yan 3280bb803951SChris Mason return ret; 3281be0e5c09SChris Mason } 3282be0e5c09SChris Mason 328397571fd0SChris Mason /* 328479f95c82SChris Mason * try to push data from one node into the next node right in the 328579f95c82SChris Mason * tree. 328679f95c82SChris Mason * 328779f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 328879f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 328979f95c82SChris Mason * 329079f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 329179f95c82SChris Mason */ 32925f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 32932ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 32945f39d397SChris Mason struct extent_buffer *dst, 32955f39d397SChris Mason struct extent_buffer *src) 329679f95c82SChris Mason { 329779f95c82SChris Mason int push_items = 0; 329879f95c82SChris Mason int max_push; 329979f95c82SChris Mason int src_nritems; 330079f95c82SChris Mason int dst_nritems; 330179f95c82SChris Mason int ret = 0; 330279f95c82SChris Mason 33037bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 33047bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 33057bb86316SChris Mason 33065f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 33075f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 33080b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 3309d397712bSChris Mason if (push_items <= 0) 331079f95c82SChris Mason return 1; 3311bce4eae9SChris Mason 3312d397712bSChris Mason if (src_nritems < 4) 3313bce4eae9SChris Mason return 1; 331479f95c82SChris Mason 331579f95c82SChris Mason max_push = src_nritems / 2 + 1; 331679f95c82SChris Mason /* don't try to empty the node */ 3317d397712bSChris Mason if (max_push >= src_nritems) 331879f95c82SChris Mason return 1; 3319252c38f0SYan 332079f95c82SChris Mason if (max_push < push_items) 332179f95c82SChris Mason push_items = max_push; 332279f95c82SChris Mason 33230b246afaSJeff Mahoney tree_mod_log_eb_move(fs_info, dst, push_items, 0, dst_nritems); 33245f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 33255f39d397SChris Mason btrfs_node_key_ptr_offset(0), 33265f39d397SChris Mason (dst_nritems) * 33275f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 3328d6025579SChris Mason 33290b246afaSJeff Mahoney ret = tree_mod_log_eb_copy(fs_info, dst, src, 0, 333090f8d62eSJan Schmidt src_nritems - push_items, push_items); 33315de865eeSFilipe David Borba Manana if (ret) { 333266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 33335de865eeSFilipe David Borba Manana return ret; 33345de865eeSFilipe David Borba Manana } 33355f39d397SChris Mason copy_extent_buffer(dst, src, 33365f39d397SChris Mason btrfs_node_key_ptr_offset(0), 33375f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 3338123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 333979f95c82SChris Mason 33405f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 33415f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 334279f95c82SChris Mason 33435f39d397SChris Mason btrfs_mark_buffer_dirty(src); 33445f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 334531840ae1SZheng Yan 334679f95c82SChris Mason return ret; 334779f95c82SChris Mason } 334879f95c82SChris Mason 334979f95c82SChris Mason /* 335097571fd0SChris Mason * helper function to insert a new root level in the tree. 335197571fd0SChris Mason * A new node is allocated, and a single item is inserted to 335297571fd0SChris Mason * point to the existing root 3353aa5d6bedSChris Mason * 3354aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 335597571fd0SChris Mason */ 3356d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 33575f39d397SChris Mason struct btrfs_root *root, 3358fdd99c72SLiu Bo struct btrfs_path *path, int level) 335974123bd7SChris Mason { 33600b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 33617bb86316SChris Mason u64 lower_gen; 33625f39d397SChris Mason struct extent_buffer *lower; 33635f39d397SChris Mason struct extent_buffer *c; 3364925baeddSChris Mason struct extent_buffer *old; 33655f39d397SChris Mason struct btrfs_disk_key lower_key; 33665c680ed6SChris Mason 33675c680ed6SChris Mason BUG_ON(path->nodes[level]); 33685c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 33695c680ed6SChris Mason 33707bb86316SChris Mason lower = path->nodes[level-1]; 33717bb86316SChris Mason if (level == 1) 33727bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 33737bb86316SChris Mason else 33747bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 33757bb86316SChris Mason 33764d75f8a9SDavid Sterba c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 33774d75f8a9SDavid Sterba &lower_key, level, root->node->start, 0); 33785f39d397SChris Mason if (IS_ERR(c)) 33795f39d397SChris Mason return PTR_ERR(c); 3380925baeddSChris Mason 33810b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3382f0486c68SYan, Zheng 3383b159fa28SDavid Sterba memzero_extent_buffer(c, 0, sizeof(struct btrfs_header)); 33845f39d397SChris Mason btrfs_set_header_nritems(c, 1); 33855f39d397SChris Mason btrfs_set_header_level(c, level); 3386db94535dSChris Mason btrfs_set_header_bytenr(c, c->start); 33875f39d397SChris Mason btrfs_set_header_generation(c, trans->transid); 33885d4f98a2SYan Zheng btrfs_set_header_backref_rev(c, BTRFS_MIXED_BACKREF_REV); 33895f39d397SChris Mason btrfs_set_header_owner(c, root->root_key.objectid); 3390d5719762SChris Mason 33910b246afaSJeff Mahoney write_extent_buffer_fsid(c, fs_info->fsid); 33920b246afaSJeff Mahoney write_extent_buffer_chunk_tree_uuid(c, fs_info->chunk_tree_uuid); 3393e17cade2SChris Mason 33945f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 3395db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 33967bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 339731840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 33987bb86316SChris Mason 33997bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 34005f39d397SChris Mason 34015f39d397SChris Mason btrfs_mark_buffer_dirty(c); 3402d5719762SChris Mason 3403925baeddSChris Mason old = root->node; 3404fdd99c72SLiu Bo tree_mod_log_set_root_pointer(root, c, 0); 3405240f62c8SChris Mason rcu_assign_pointer(root->node, c); 3406925baeddSChris Mason 3407925baeddSChris Mason /* the super has an extra ref to root->node */ 3408925baeddSChris Mason free_extent_buffer(old); 3409925baeddSChris Mason 34100b86a832SChris Mason add_root_to_dirty_list(root); 34115f39d397SChris Mason extent_buffer_get(c); 34125f39d397SChris Mason path->nodes[level] = c; 341395449a16Schandan path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 341474123bd7SChris Mason path->slots[level] = 0; 341574123bd7SChris Mason return 0; 341674123bd7SChris Mason } 34175c680ed6SChris Mason 34185c680ed6SChris Mason /* 34195c680ed6SChris Mason * worker function to insert a single pointer in a node. 34205c680ed6SChris Mason * the node should have enough room for the pointer already 342197571fd0SChris Mason * 34225c680ed6SChris Mason * slot and level indicate where you want the key to go, and 34235c680ed6SChris Mason * blocknr is the block the key points to. 34245c680ed6SChris Mason */ 3425143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 34262ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, struct btrfs_path *path, 3427143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 3428c3e06965SJan Schmidt int slot, int level) 34295c680ed6SChris Mason { 34305f39d397SChris Mason struct extent_buffer *lower; 34315c680ed6SChris Mason int nritems; 3432f3ea38daSJan Schmidt int ret; 34335c680ed6SChris Mason 34345c680ed6SChris Mason BUG_ON(!path->nodes[level]); 3435f0486c68SYan, Zheng btrfs_assert_tree_locked(path->nodes[level]); 34365f39d397SChris Mason lower = path->nodes[level]; 34375f39d397SChris Mason nritems = btrfs_header_nritems(lower); 3438c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 34390b246afaSJeff Mahoney BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 344074123bd7SChris Mason if (slot != nritems) { 3441c3e06965SJan Schmidt if (level) 34420b246afaSJeff Mahoney tree_mod_log_eb_move(fs_info, lower, slot + 1, 3443f3ea38daSJan Schmidt slot, nritems - slot); 34445f39d397SChris Mason memmove_extent_buffer(lower, 34455f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 34465f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 3447123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 344874123bd7SChris Mason } 3449c3e06965SJan Schmidt if (level) { 34500b246afaSJeff Mahoney ret = tree_mod_log_insert_key(fs_info, lower, slot, 3451c8cc6341SJosef Bacik MOD_LOG_KEY_ADD, GFP_NOFS); 3452f3ea38daSJan Schmidt BUG_ON(ret < 0); 3453f3ea38daSJan Schmidt } 34545f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 3455db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 345674493f7aSChris Mason WARN_ON(trans->transid == 0); 345774493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 34585f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 34595f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 346074123bd7SChris Mason } 346174123bd7SChris Mason 346297571fd0SChris Mason /* 346397571fd0SChris Mason * split the node at the specified level in path in two. 346497571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 346597571fd0SChris Mason * 346697571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 346797571fd0SChris Mason * left and right, if either one works, it returns right away. 3468aa5d6bedSChris Mason * 3469aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 347097571fd0SChris Mason */ 3471e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3472e02119d5SChris Mason struct btrfs_root *root, 3473e02119d5SChris Mason struct btrfs_path *path, int level) 3474be0e5c09SChris Mason { 34750b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 34765f39d397SChris Mason struct extent_buffer *c; 34775f39d397SChris Mason struct extent_buffer *split; 34785f39d397SChris Mason struct btrfs_disk_key disk_key; 3479be0e5c09SChris Mason int mid; 34805c680ed6SChris Mason int ret; 34817518a238SChris Mason u32 c_nritems; 3482be0e5c09SChris Mason 34835f39d397SChris Mason c = path->nodes[level]; 34847bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 34855f39d397SChris Mason if (c == root->node) { 3486d9abbf1cSJan Schmidt /* 348790f8d62eSJan Schmidt * trying to split the root, lets make a new one 348890f8d62eSJan Schmidt * 3489fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 349090f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 349190f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 349290f8d62eSJan Schmidt * elements below with tree_mod_log_eb_copy. We're holding a 349390f8d62eSJan Schmidt * tree lock on the buffer, which is why we cannot race with 349490f8d62eSJan Schmidt * other tree_mod_log users. 3495d9abbf1cSJan Schmidt */ 3496fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 34975c680ed6SChris Mason if (ret) 34985c680ed6SChris Mason return ret; 3499b3612421SChris Mason } else { 3500e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 35015f39d397SChris Mason c = path->nodes[level]; 35025f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 35030b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 3504e66f709bSChris Mason return 0; 350554aa1f4dSChris Mason if (ret < 0) 350654aa1f4dSChris Mason return ret; 35075c680ed6SChris Mason } 3508e66f709bSChris Mason 35095f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 35105d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 35115d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 35127bb86316SChris Mason 35134d75f8a9SDavid Sterba split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 35145581a51aSJan Schmidt &disk_key, level, c->start, 0); 35155f39d397SChris Mason if (IS_ERR(split)) 35165f39d397SChris Mason return PTR_ERR(split); 351754aa1f4dSChris Mason 35180b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3519f0486c68SYan, Zheng 3520b159fa28SDavid Sterba memzero_extent_buffer(split, 0, sizeof(struct btrfs_header)); 35215f39d397SChris Mason btrfs_set_header_level(split, btrfs_header_level(c)); 3522db94535dSChris Mason btrfs_set_header_bytenr(split, split->start); 35235f39d397SChris Mason btrfs_set_header_generation(split, trans->transid); 35245d4f98a2SYan Zheng btrfs_set_header_backref_rev(split, BTRFS_MIXED_BACKREF_REV); 35255f39d397SChris Mason btrfs_set_header_owner(split, root->root_key.objectid); 35260b246afaSJeff Mahoney write_extent_buffer_fsid(split, fs_info->fsid); 35270b246afaSJeff Mahoney write_extent_buffer_chunk_tree_uuid(split, fs_info->chunk_tree_uuid); 35285f39d397SChris Mason 35290b246afaSJeff Mahoney ret = tree_mod_log_eb_copy(fs_info, split, c, 0, mid, c_nritems - mid); 35305de865eeSFilipe David Borba Manana if (ret) { 353166642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 35325de865eeSFilipe David Borba Manana return ret; 35335de865eeSFilipe David Borba Manana } 35345f39d397SChris Mason copy_extent_buffer(split, c, 35355f39d397SChris Mason btrfs_node_key_ptr_offset(0), 35365f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 3537123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 35385f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 35395f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3540aa5d6bedSChris Mason ret = 0; 3541aa5d6bedSChris Mason 35425f39d397SChris Mason btrfs_mark_buffer_dirty(c); 35435f39d397SChris Mason btrfs_mark_buffer_dirty(split); 35445f39d397SChris Mason 35452ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, split->start, 3546c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3547aa5d6bedSChris Mason 35485de08d7dSChris Mason if (path->slots[level] >= mid) { 35495c680ed6SChris Mason path->slots[level] -= mid; 3550925baeddSChris Mason btrfs_tree_unlock(c); 35515f39d397SChris Mason free_extent_buffer(c); 35525f39d397SChris Mason path->nodes[level] = split; 35535c680ed6SChris Mason path->slots[level + 1] += 1; 3554eb60ceacSChris Mason } else { 3555925baeddSChris Mason btrfs_tree_unlock(split); 35565f39d397SChris Mason free_extent_buffer(split); 3557be0e5c09SChris Mason } 3558aa5d6bedSChris Mason return ret; 3559be0e5c09SChris Mason } 3560be0e5c09SChris Mason 356174123bd7SChris Mason /* 356274123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 356374123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 356474123bd7SChris Mason * space used both by the item structs and the item data 356574123bd7SChris Mason */ 35665f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 3567be0e5c09SChris Mason { 356841be1f3bSJosef Bacik struct btrfs_item *start_item; 356941be1f3bSJosef Bacik struct btrfs_item *end_item; 357041be1f3bSJosef Bacik struct btrfs_map_token token; 3571be0e5c09SChris Mason int data_len; 35725f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3573d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3574be0e5c09SChris Mason 3575be0e5c09SChris Mason if (!nr) 3576be0e5c09SChris Mason return 0; 357741be1f3bSJosef Bacik btrfs_init_map_token(&token); 3578dd3cc16bSRoss Kirk start_item = btrfs_item_nr(start); 3579dd3cc16bSRoss Kirk end_item = btrfs_item_nr(end); 358041be1f3bSJosef Bacik data_len = btrfs_token_item_offset(l, start_item, &token) + 358141be1f3bSJosef Bacik btrfs_token_item_size(l, start_item, &token); 358241be1f3bSJosef Bacik data_len = data_len - btrfs_token_item_offset(l, end_item, &token); 35830783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3584d4dbff95SChris Mason WARN_ON(data_len < 0); 3585be0e5c09SChris Mason return data_len; 3586be0e5c09SChris Mason } 3587be0e5c09SChris Mason 358874123bd7SChris Mason /* 3589d4dbff95SChris Mason * The space between the end of the leaf items and 3590d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3591d4dbff95SChris Mason * the leaf has left for both items and data 3592d4dbff95SChris Mason */ 35932ff7e61eSJeff Mahoney noinline int btrfs_leaf_free_space(struct btrfs_fs_info *fs_info, 3594e02119d5SChris Mason struct extent_buffer *leaf) 3595d4dbff95SChris Mason { 35965f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 35975f39d397SChris Mason int ret; 35980b246afaSJeff Mahoney 35990b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 36005f39d397SChris Mason if (ret < 0) { 36010b246afaSJeff Mahoney btrfs_crit(fs_info, 3602efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3603da17066cSJeff Mahoney ret, 36040b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 36055f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 36065f39d397SChris Mason } 36075f39d397SChris Mason return ret; 3608d4dbff95SChris Mason } 3609d4dbff95SChris Mason 361099d8f83cSChris Mason /* 361199d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 361299d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 361399d8f83cSChris Mason */ 36141e47eef2SDavid Sterba static noinline int __push_leaf_right(struct btrfs_fs_info *fs_info, 361544871b1bSChris Mason struct btrfs_path *path, 361644871b1bSChris Mason int data_size, int empty, 361744871b1bSChris Mason struct extent_buffer *right, 361899d8f83cSChris Mason int free_space, u32 left_nritems, 361999d8f83cSChris Mason u32 min_slot) 362000ec4c51SChris Mason { 36215f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 362244871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3623cfed81a0SChris Mason struct btrfs_map_token token; 36245f39d397SChris Mason struct btrfs_disk_key disk_key; 362500ec4c51SChris Mason int slot; 362634a38218SChris Mason u32 i; 362700ec4c51SChris Mason int push_space = 0; 362800ec4c51SChris Mason int push_items = 0; 36290783fcfcSChris Mason struct btrfs_item *item; 363034a38218SChris Mason u32 nr; 36317518a238SChris Mason u32 right_nritems; 36325f39d397SChris Mason u32 data_end; 3633db94535dSChris Mason u32 this_item_size; 363400ec4c51SChris Mason 3635cfed81a0SChris Mason btrfs_init_map_token(&token); 3636cfed81a0SChris Mason 363734a38218SChris Mason if (empty) 363834a38218SChris Mason nr = 0; 363934a38218SChris Mason else 364099d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 364134a38218SChris Mason 364231840ae1SZheng Yan if (path->slots[0] >= left_nritems) 364387b29b20SYan Zheng push_space += data_size; 364431840ae1SZheng Yan 364544871b1bSChris Mason slot = path->slots[1]; 364634a38218SChris Mason i = left_nritems - 1; 364734a38218SChris Mason while (i >= nr) { 3648dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3649db94535dSChris Mason 365031840ae1SZheng Yan if (!empty && push_items > 0) { 365131840ae1SZheng Yan if (path->slots[0] > i) 365231840ae1SZheng Yan break; 365331840ae1SZheng Yan if (path->slots[0] == i) { 36542ff7e61eSJeff Mahoney int space = btrfs_leaf_free_space(fs_info, left); 365531840ae1SZheng Yan if (space + push_space * 2 > free_space) 365631840ae1SZheng Yan break; 365731840ae1SZheng Yan } 365831840ae1SZheng Yan } 365931840ae1SZheng Yan 366000ec4c51SChris Mason if (path->slots[0] == i) 366187b29b20SYan Zheng push_space += data_size; 3662db94535dSChris Mason 3663db94535dSChris Mason this_item_size = btrfs_item_size(left, item); 3664db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 366500ec4c51SChris Mason break; 366631840ae1SZheng Yan 366700ec4c51SChris Mason push_items++; 3668db94535dSChris Mason push_space += this_item_size + sizeof(*item); 366934a38218SChris Mason if (i == 0) 367034a38218SChris Mason break; 367134a38218SChris Mason i--; 3672db94535dSChris Mason } 36735f39d397SChris Mason 3674925baeddSChris Mason if (push_items == 0) 3675925baeddSChris Mason goto out_unlock; 36765f39d397SChris Mason 36776c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 36785f39d397SChris Mason 367900ec4c51SChris Mason /* push left to right */ 36805f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 368134a38218SChris Mason 36825f39d397SChris Mason push_space = btrfs_item_end_nr(left, left_nritems - push_items); 36832ff7e61eSJeff Mahoney push_space -= leaf_data_end(fs_info, left); 36845f39d397SChris Mason 368500ec4c51SChris Mason /* make room in the right data area */ 36862ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, right); 36875f39d397SChris Mason memmove_extent_buffer(right, 36883d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end - push_space, 36893d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 36900b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 36915f39d397SChris Mason 369200ec4c51SChris Mason /* copy from the left data area */ 36933d9ec8c4SNikolay Borisov copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET + 36940b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 36953d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + leaf_data_end(fs_info, left), 3696d6025579SChris Mason push_space); 36975f39d397SChris Mason 36985f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 36995f39d397SChris Mason btrfs_item_nr_offset(0), 37000783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 37015f39d397SChris Mason 370200ec4c51SChris Mason /* copy the items from left to right */ 37035f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 37045f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 37050783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 370600ec4c51SChris Mason 370700ec4c51SChris Mason /* update the item pointers */ 37087518a238SChris Mason right_nritems += push_items; 37095f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 37100b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 37117518a238SChris Mason for (i = 0; i < right_nritems; i++) { 3712dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3713cfed81a0SChris Mason push_space -= btrfs_token_item_size(right, item, &token); 3714cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3715db94535dSChris Mason } 3716db94535dSChris Mason 37177518a238SChris Mason left_nritems -= push_items; 37185f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 371900ec4c51SChris Mason 372034a38218SChris Mason if (left_nritems) 37215f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3722f0486c68SYan, Zheng else 37237c302b49SDavid Sterba clean_tree_block(fs_info, left); 3724f0486c68SYan, Zheng 37255f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3726a429e513SChris Mason 37275f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 37285f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3729d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 373002217ed2SChris Mason 373100ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 37327518a238SChris Mason if (path->slots[0] >= left_nritems) { 37337518a238SChris Mason path->slots[0] -= left_nritems; 3734925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 37357c302b49SDavid Sterba clean_tree_block(fs_info, path->nodes[0]); 3736925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 37375f39d397SChris Mason free_extent_buffer(path->nodes[0]); 37385f39d397SChris Mason path->nodes[0] = right; 373900ec4c51SChris Mason path->slots[1] += 1; 374000ec4c51SChris Mason } else { 3741925baeddSChris Mason btrfs_tree_unlock(right); 37425f39d397SChris Mason free_extent_buffer(right); 374300ec4c51SChris Mason } 374400ec4c51SChris Mason return 0; 3745925baeddSChris Mason 3746925baeddSChris Mason out_unlock: 3747925baeddSChris Mason btrfs_tree_unlock(right); 3748925baeddSChris Mason free_extent_buffer(right); 3749925baeddSChris Mason return 1; 375000ec4c51SChris Mason } 3751925baeddSChris Mason 375200ec4c51SChris Mason /* 375344871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 375474123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 375544871b1bSChris Mason * 375644871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 375744871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 375899d8f83cSChris Mason * 375999d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 376099d8f83cSChris Mason * push any slot lower than min_slot 376174123bd7SChris Mason */ 376244871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 376399d8f83cSChris Mason *root, struct btrfs_path *path, 376499d8f83cSChris Mason int min_data_size, int data_size, 376599d8f83cSChris Mason int empty, u32 min_slot) 3766be0e5c09SChris Mason { 37672ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 376844871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 376944871b1bSChris Mason struct extent_buffer *right; 377044871b1bSChris Mason struct extent_buffer *upper; 377144871b1bSChris Mason int slot; 377244871b1bSChris Mason int free_space; 377344871b1bSChris Mason u32 left_nritems; 377444871b1bSChris Mason int ret; 377544871b1bSChris Mason 377644871b1bSChris Mason if (!path->nodes[1]) 377744871b1bSChris Mason return 1; 377844871b1bSChris Mason 377944871b1bSChris Mason slot = path->slots[1]; 378044871b1bSChris Mason upper = path->nodes[1]; 378144871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 378244871b1bSChris Mason return 1; 378344871b1bSChris Mason 378444871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 378544871b1bSChris Mason 37862ff7e61eSJeff Mahoney right = read_node_slot(fs_info, upper, slot + 1); 3787fb770ae4SLiu Bo /* 3788fb770ae4SLiu Bo * slot + 1 is not valid or we fail to read the right node, 3789fb770ae4SLiu Bo * no big deal, just return. 3790fb770ae4SLiu Bo */ 3791fb770ae4SLiu Bo if (IS_ERR(right)) 379291ca338dSTsutomu Itoh return 1; 379391ca338dSTsutomu Itoh 379444871b1bSChris Mason btrfs_tree_lock(right); 379544871b1bSChris Mason btrfs_set_lock_blocking(right); 379644871b1bSChris Mason 37972ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, right); 379844871b1bSChris Mason if (free_space < data_size) 379944871b1bSChris Mason goto out_unlock; 380044871b1bSChris Mason 380144871b1bSChris Mason /* cow and double check */ 380244871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 380344871b1bSChris Mason slot + 1, &right); 380444871b1bSChris Mason if (ret) 380544871b1bSChris Mason goto out_unlock; 380644871b1bSChris Mason 38072ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, right); 380844871b1bSChris Mason if (free_space < data_size) 380944871b1bSChris Mason goto out_unlock; 381044871b1bSChris Mason 381144871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 381244871b1bSChris Mason if (left_nritems == 0) 381344871b1bSChris Mason goto out_unlock; 381444871b1bSChris Mason 38152ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 38162ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 38172ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 38182ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 38192ef1fed2SFilipe David Borba Manana * no need to touch/dirty our left leaft. */ 38202ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 38212ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 38222ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 38232ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 38242ef1fed2SFilipe David Borba Manana path->slots[1]++; 38252ef1fed2SFilipe David Borba Manana return 0; 38262ef1fed2SFilipe David Borba Manana } 38272ef1fed2SFilipe David Borba Manana 38281e47eef2SDavid Sterba return __push_leaf_right(fs_info, path, min_data_size, empty, 382999d8f83cSChris Mason right, free_space, left_nritems, min_slot); 383044871b1bSChris Mason out_unlock: 383144871b1bSChris Mason btrfs_tree_unlock(right); 383244871b1bSChris Mason free_extent_buffer(right); 383344871b1bSChris Mason return 1; 383444871b1bSChris Mason } 383544871b1bSChris Mason 383644871b1bSChris Mason /* 383744871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 383844871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 383999d8f83cSChris Mason * 384099d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 384199d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 384299d8f83cSChris Mason * items 384344871b1bSChris Mason */ 384466cb7ddbSDavid Sterba static noinline int __push_leaf_left(struct btrfs_fs_info *fs_info, 384544871b1bSChris Mason struct btrfs_path *path, int data_size, 384644871b1bSChris Mason int empty, struct extent_buffer *left, 384799d8f83cSChris Mason int free_space, u32 right_nritems, 384899d8f83cSChris Mason u32 max_slot) 384944871b1bSChris Mason { 38505f39d397SChris Mason struct btrfs_disk_key disk_key; 38515f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3852be0e5c09SChris Mason int i; 3853be0e5c09SChris Mason int push_space = 0; 3854be0e5c09SChris Mason int push_items = 0; 38550783fcfcSChris Mason struct btrfs_item *item; 38567518a238SChris Mason u32 old_left_nritems; 385734a38218SChris Mason u32 nr; 3858aa5d6bedSChris Mason int ret = 0; 3859db94535dSChris Mason u32 this_item_size; 3860db94535dSChris Mason u32 old_left_item_size; 3861cfed81a0SChris Mason struct btrfs_map_token token; 3862cfed81a0SChris Mason 3863cfed81a0SChris Mason btrfs_init_map_token(&token); 3864be0e5c09SChris Mason 386534a38218SChris Mason if (empty) 386699d8f83cSChris Mason nr = min(right_nritems, max_slot); 386734a38218SChris Mason else 386899d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 386934a38218SChris Mason 387034a38218SChris Mason for (i = 0; i < nr; i++) { 3871dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3872db94535dSChris Mason 387331840ae1SZheng Yan if (!empty && push_items > 0) { 387431840ae1SZheng Yan if (path->slots[0] < i) 387531840ae1SZheng Yan break; 387631840ae1SZheng Yan if (path->slots[0] == i) { 38772ff7e61eSJeff Mahoney int space = btrfs_leaf_free_space(fs_info, right); 387831840ae1SZheng Yan if (space + push_space * 2 > free_space) 387931840ae1SZheng Yan break; 388031840ae1SZheng Yan } 388131840ae1SZheng Yan } 388231840ae1SZheng Yan 3883be0e5c09SChris Mason if (path->slots[0] == i) 388487b29b20SYan Zheng push_space += data_size; 3885db94535dSChris Mason 3886db94535dSChris Mason this_item_size = btrfs_item_size(right, item); 3887db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 3888be0e5c09SChris Mason break; 3889db94535dSChris Mason 3890be0e5c09SChris Mason push_items++; 3891db94535dSChris Mason push_space += this_item_size + sizeof(*item); 3892be0e5c09SChris Mason } 3893db94535dSChris Mason 3894be0e5c09SChris Mason if (push_items == 0) { 3895925baeddSChris Mason ret = 1; 3896925baeddSChris Mason goto out; 3897be0e5c09SChris Mason } 3898fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 38995f39d397SChris Mason 3900be0e5c09SChris Mason /* push data from right to left */ 39015f39d397SChris Mason copy_extent_buffer(left, right, 39025f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 39035f39d397SChris Mason btrfs_item_nr_offset(0), 39045f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 39055f39d397SChris Mason 39060b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 39075f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1); 39085f39d397SChris Mason 39093d9ec8c4SNikolay Borisov copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET + 39102ff7e61eSJeff Mahoney leaf_data_end(fs_info, left) - push_space, 39113d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 39125f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1), 3913be0e5c09SChris Mason push_space); 39145f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 391587b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3916eb60ceacSChris Mason 3917db94535dSChris Mason old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1); 3918be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 39195f39d397SChris Mason u32 ioff; 3920db94535dSChris Mason 3921dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3922db94535dSChris Mason 3923cfed81a0SChris Mason ioff = btrfs_token_item_offset(left, item, &token); 3924cfed81a0SChris Mason btrfs_set_token_item_offset(left, item, 39250b246afaSJeff Mahoney ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size), 3926cfed81a0SChris Mason &token); 3927be0e5c09SChris Mason } 39285f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3929be0e5c09SChris Mason 3930be0e5c09SChris Mason /* fixup right node */ 393131b1a2bdSJulia Lawall if (push_items > right_nritems) 393231b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3933d397712bSChris Mason right_nritems); 393434a38218SChris Mason 393534a38218SChris Mason if (push_items < right_nritems) { 39365f39d397SChris Mason push_space = btrfs_item_offset_nr(right, push_items - 1) - 39372ff7e61eSJeff Mahoney leaf_data_end(fs_info, right); 39383d9ec8c4SNikolay Borisov memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET + 39390b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 39403d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 39412ff7e61eSJeff Mahoney leaf_data_end(fs_info, right), push_space); 39425f39d397SChris Mason 39435f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 39445f39d397SChris Mason btrfs_item_nr_offset(push_items), 39455f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 39460783fcfcSChris Mason sizeof(struct btrfs_item)); 394734a38218SChris Mason } 3948eef1c494SYan right_nritems -= push_items; 3949eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 39500b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 39515f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 3952dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3953db94535dSChris Mason 3954cfed81a0SChris Mason push_space = push_space - btrfs_token_item_size(right, 3955cfed81a0SChris Mason item, &token); 3956cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3957db94535dSChris Mason } 3958eb60ceacSChris Mason 39595f39d397SChris Mason btrfs_mark_buffer_dirty(left); 396034a38218SChris Mason if (right_nritems) 39615f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3962f0486c68SYan, Zheng else 39637c302b49SDavid Sterba clean_tree_block(fs_info, right); 3964098f59c2SChris Mason 39655f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 39660b246afaSJeff Mahoney fixup_low_keys(fs_info, path, &disk_key, 1); 3967be0e5c09SChris Mason 3968be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3969be0e5c09SChris Mason if (path->slots[0] < push_items) { 3970be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3971925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 39725f39d397SChris Mason free_extent_buffer(path->nodes[0]); 39735f39d397SChris Mason path->nodes[0] = left; 3974be0e5c09SChris Mason path->slots[1] -= 1; 3975be0e5c09SChris Mason } else { 3976925baeddSChris Mason btrfs_tree_unlock(left); 39775f39d397SChris Mason free_extent_buffer(left); 3978be0e5c09SChris Mason path->slots[0] -= push_items; 3979be0e5c09SChris Mason } 3980eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3981aa5d6bedSChris Mason return ret; 3982925baeddSChris Mason out: 3983925baeddSChris Mason btrfs_tree_unlock(left); 3984925baeddSChris Mason free_extent_buffer(left); 3985925baeddSChris Mason return ret; 3986be0e5c09SChris Mason } 3987be0e5c09SChris Mason 398874123bd7SChris Mason /* 398944871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 399044871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 399199d8f83cSChris Mason * 399299d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 399399d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 399499d8f83cSChris Mason * items 399544871b1bSChris Mason */ 399644871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 399799d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 399899d8f83cSChris Mason int data_size, int empty, u32 max_slot) 399944871b1bSChris Mason { 40002ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 400144871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 400244871b1bSChris Mason struct extent_buffer *left; 400344871b1bSChris Mason int slot; 400444871b1bSChris Mason int free_space; 400544871b1bSChris Mason u32 right_nritems; 400644871b1bSChris Mason int ret = 0; 400744871b1bSChris Mason 400844871b1bSChris Mason slot = path->slots[1]; 400944871b1bSChris Mason if (slot == 0) 401044871b1bSChris Mason return 1; 401144871b1bSChris Mason if (!path->nodes[1]) 401244871b1bSChris Mason return 1; 401344871b1bSChris Mason 401444871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 401544871b1bSChris Mason if (right_nritems == 0) 401644871b1bSChris Mason return 1; 401744871b1bSChris Mason 401844871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 401944871b1bSChris Mason 40202ff7e61eSJeff Mahoney left = read_node_slot(fs_info, path->nodes[1], slot - 1); 4021fb770ae4SLiu Bo /* 4022fb770ae4SLiu Bo * slot - 1 is not valid or we fail to read the left node, 4023fb770ae4SLiu Bo * no big deal, just return. 4024fb770ae4SLiu Bo */ 4025fb770ae4SLiu Bo if (IS_ERR(left)) 402691ca338dSTsutomu Itoh return 1; 402791ca338dSTsutomu Itoh 402844871b1bSChris Mason btrfs_tree_lock(left); 402944871b1bSChris Mason btrfs_set_lock_blocking(left); 403044871b1bSChris Mason 40312ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, left); 403244871b1bSChris Mason if (free_space < data_size) { 403344871b1bSChris Mason ret = 1; 403444871b1bSChris Mason goto out; 403544871b1bSChris Mason } 403644871b1bSChris Mason 403744871b1bSChris Mason /* cow and double check */ 403844871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 403944871b1bSChris Mason path->nodes[1], slot - 1, &left); 404044871b1bSChris Mason if (ret) { 404144871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 404279787eaaSJeff Mahoney if (ret == -ENOSPC) 404344871b1bSChris Mason ret = 1; 404444871b1bSChris Mason goto out; 404544871b1bSChris Mason } 404644871b1bSChris Mason 40472ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, left); 404844871b1bSChris Mason if (free_space < data_size) { 404944871b1bSChris Mason ret = 1; 405044871b1bSChris Mason goto out; 405144871b1bSChris Mason } 405244871b1bSChris Mason 405366cb7ddbSDavid Sterba return __push_leaf_left(fs_info, path, min_data_size, 405499d8f83cSChris Mason empty, left, free_space, right_nritems, 405599d8f83cSChris Mason max_slot); 405644871b1bSChris Mason out: 405744871b1bSChris Mason btrfs_tree_unlock(left); 405844871b1bSChris Mason free_extent_buffer(left); 405944871b1bSChris Mason return ret; 406044871b1bSChris Mason } 406144871b1bSChris Mason 406244871b1bSChris Mason /* 406374123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 406474123bd7SChris Mason * available for the resulting leaf level of the path. 406574123bd7SChris Mason */ 4066143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 40672ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 406844871b1bSChris Mason struct btrfs_path *path, 406944871b1bSChris Mason struct extent_buffer *l, 407044871b1bSChris Mason struct extent_buffer *right, 407144871b1bSChris Mason int slot, int mid, int nritems) 4072be0e5c09SChris Mason { 4073be0e5c09SChris Mason int data_copy_size; 4074be0e5c09SChris Mason int rt_data_off; 4075be0e5c09SChris Mason int i; 4076d4dbff95SChris Mason struct btrfs_disk_key disk_key; 4077cfed81a0SChris Mason struct btrfs_map_token token; 4078cfed81a0SChris Mason 4079cfed81a0SChris Mason btrfs_init_map_token(&token); 4080be0e5c09SChris Mason 40815f39d397SChris Mason nritems = nritems - mid; 40825f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 40832ff7e61eSJeff Mahoney data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(fs_info, l); 40845f39d397SChris Mason 40855f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 40865f39d397SChris Mason btrfs_item_nr_offset(mid), 40875f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 40885f39d397SChris Mason 40895f39d397SChris Mason copy_extent_buffer(right, l, 40903d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) - 40913d9ec8c4SNikolay Borisov data_copy_size, BTRFS_LEAF_DATA_OFFSET + 40922ff7e61eSJeff Mahoney leaf_data_end(fs_info, l), data_copy_size); 409374123bd7SChris Mason 40940b246afaSJeff Mahoney rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_end_nr(l, mid); 40955f39d397SChris Mason 40965f39d397SChris Mason for (i = 0; i < nritems; i++) { 4097dd3cc16bSRoss Kirk struct btrfs_item *item = btrfs_item_nr(i); 4098db94535dSChris Mason u32 ioff; 4099db94535dSChris Mason 4100cfed81a0SChris Mason ioff = btrfs_token_item_offset(right, item, &token); 4101cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, 4102cfed81a0SChris Mason ioff + rt_data_off, &token); 41030783fcfcSChris Mason } 410474123bd7SChris Mason 41055f39d397SChris Mason btrfs_set_header_nritems(l, mid); 41065f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 41072ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, right->start, 4108c3e06965SJan Schmidt path->slots[1] + 1, 1); 41095f39d397SChris Mason 41105f39d397SChris Mason btrfs_mark_buffer_dirty(right); 41115f39d397SChris Mason btrfs_mark_buffer_dirty(l); 4112eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 41135f39d397SChris Mason 4114be0e5c09SChris Mason if (mid <= slot) { 4115925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 41165f39d397SChris Mason free_extent_buffer(path->nodes[0]); 41175f39d397SChris Mason path->nodes[0] = right; 4118be0e5c09SChris Mason path->slots[0] -= mid; 4119be0e5c09SChris Mason path->slots[1] += 1; 4120925baeddSChris Mason } else { 4121925baeddSChris Mason btrfs_tree_unlock(right); 41225f39d397SChris Mason free_extent_buffer(right); 4123925baeddSChris Mason } 41245f39d397SChris Mason 4125eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 412644871b1bSChris Mason } 412744871b1bSChris Mason 412844871b1bSChris Mason /* 412999d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 413099d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 413199d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 413299d8f83cSChris Mason * A B C 413399d8f83cSChris Mason * 413499d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 413599d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 413699d8f83cSChris Mason * completely. 413799d8f83cSChris Mason */ 413899d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 413999d8f83cSChris Mason struct btrfs_root *root, 414099d8f83cSChris Mason struct btrfs_path *path, 414199d8f83cSChris Mason int data_size) 414299d8f83cSChris Mason { 41432ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 414499d8f83cSChris Mason int ret; 414599d8f83cSChris Mason int progress = 0; 414699d8f83cSChris Mason int slot; 414799d8f83cSChris Mason u32 nritems; 41485a4267caSFilipe David Borba Manana int space_needed = data_size; 414999d8f83cSChris Mason 415099d8f83cSChris Mason slot = path->slots[0]; 41515a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 41522ff7e61eSJeff Mahoney space_needed -= btrfs_leaf_free_space(fs_info, path->nodes[0]); 415399d8f83cSChris Mason 415499d8f83cSChris Mason /* 415599d8f83cSChris Mason * try to push all the items after our slot into the 415699d8f83cSChris Mason * right leaf 415799d8f83cSChris Mason */ 41585a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 415999d8f83cSChris Mason if (ret < 0) 416099d8f83cSChris Mason return ret; 416199d8f83cSChris Mason 416299d8f83cSChris Mason if (ret == 0) 416399d8f83cSChris Mason progress++; 416499d8f83cSChris Mason 416599d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 416699d8f83cSChris Mason /* 416799d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 416899d8f83cSChris Mason * we've done so we're done 416999d8f83cSChris Mason */ 417099d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 417199d8f83cSChris Mason return 0; 417299d8f83cSChris Mason 41732ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= data_size) 417499d8f83cSChris Mason return 0; 417599d8f83cSChris Mason 417699d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 417799d8f83cSChris Mason slot = path->slots[0]; 4178263d3995SFilipe Manana space_needed = data_size; 4179263d3995SFilipe Manana if (slot > 0) 4180263d3995SFilipe Manana space_needed -= btrfs_leaf_free_space(fs_info, path->nodes[0]); 41815a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 418299d8f83cSChris Mason if (ret < 0) 418399d8f83cSChris Mason return ret; 418499d8f83cSChris Mason 418599d8f83cSChris Mason if (ret == 0) 418699d8f83cSChris Mason progress++; 418799d8f83cSChris Mason 418899d8f83cSChris Mason if (progress) 418999d8f83cSChris Mason return 0; 419099d8f83cSChris Mason return 1; 419199d8f83cSChris Mason } 419299d8f83cSChris Mason 419399d8f83cSChris Mason /* 419444871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 419544871b1bSChris Mason * available for the resulting leaf level of the path. 419644871b1bSChris Mason * 419744871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 419844871b1bSChris Mason */ 419944871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 420044871b1bSChris Mason struct btrfs_root *root, 4201310712b2SOmar Sandoval const struct btrfs_key *ins_key, 420244871b1bSChris Mason struct btrfs_path *path, int data_size, 420344871b1bSChris Mason int extend) 420444871b1bSChris Mason { 42055d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 420644871b1bSChris Mason struct extent_buffer *l; 420744871b1bSChris Mason u32 nritems; 420844871b1bSChris Mason int mid; 420944871b1bSChris Mason int slot; 421044871b1bSChris Mason struct extent_buffer *right; 4211b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 421244871b1bSChris Mason int ret = 0; 421344871b1bSChris Mason int wret; 42145d4f98a2SYan Zheng int split; 421544871b1bSChris Mason int num_doubles = 0; 421699d8f83cSChris Mason int tried_avoid_double = 0; 421744871b1bSChris Mason 4218a5719521SYan, Zheng l = path->nodes[0]; 4219a5719521SYan, Zheng slot = path->slots[0]; 4220a5719521SYan, Zheng if (extend && data_size + btrfs_item_size_nr(l, slot) + 42210b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 4222a5719521SYan, Zheng return -EOVERFLOW; 4223a5719521SYan, Zheng 422444871b1bSChris Mason /* first try to make some room by pushing left and right */ 422533157e05SLiu Bo if (data_size && path->nodes[1]) { 42265a4267caSFilipe David Borba Manana int space_needed = data_size; 42275a4267caSFilipe David Borba Manana 42285a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 42292ff7e61eSJeff Mahoney space_needed -= btrfs_leaf_free_space(fs_info, l); 42305a4267caSFilipe David Borba Manana 42315a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 42325a4267caSFilipe David Borba Manana space_needed, 0, 0); 423344871b1bSChris Mason if (wret < 0) 423444871b1bSChris Mason return wret; 423544871b1bSChris Mason if (wret) { 4236263d3995SFilipe Manana space_needed = data_size; 4237263d3995SFilipe Manana if (slot > 0) 4238263d3995SFilipe Manana space_needed -= btrfs_leaf_free_space(fs_info, 4239263d3995SFilipe Manana l); 42405a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 42415a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 424244871b1bSChris Mason if (wret < 0) 424344871b1bSChris Mason return wret; 424444871b1bSChris Mason } 424544871b1bSChris Mason l = path->nodes[0]; 424644871b1bSChris Mason 424744871b1bSChris Mason /* did the pushes work? */ 42482ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, l) >= data_size) 424944871b1bSChris Mason return 0; 425044871b1bSChris Mason } 425144871b1bSChris Mason 425244871b1bSChris Mason if (!path->nodes[1]) { 4253fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 425444871b1bSChris Mason if (ret) 425544871b1bSChris Mason return ret; 425644871b1bSChris Mason } 425744871b1bSChris Mason again: 42585d4f98a2SYan Zheng split = 1; 425944871b1bSChris Mason l = path->nodes[0]; 426044871b1bSChris Mason slot = path->slots[0]; 426144871b1bSChris Mason nritems = btrfs_header_nritems(l); 426244871b1bSChris Mason mid = (nritems + 1) / 2; 426344871b1bSChris Mason 42645d4f98a2SYan Zheng if (mid <= slot) { 42655d4f98a2SYan Zheng if (nritems == 1 || 42665d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 42670b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 42685d4f98a2SYan Zheng if (slot >= nritems) { 42695d4f98a2SYan Zheng split = 0; 42705d4f98a2SYan Zheng } else { 42715d4f98a2SYan Zheng mid = slot; 42725d4f98a2SYan Zheng if (mid != nritems && 42735d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 42740b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 427599d8f83cSChris Mason if (data_size && !tried_avoid_double) 427699d8f83cSChris Mason goto push_for_double; 42775d4f98a2SYan Zheng split = 2; 42785d4f98a2SYan Zheng } 42795d4f98a2SYan Zheng } 42805d4f98a2SYan Zheng } 42815d4f98a2SYan Zheng } else { 42825d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 42830b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 42845d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 42855d4f98a2SYan Zheng split = 0; 42865d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 42875d4f98a2SYan Zheng mid = 1; 42885d4f98a2SYan Zheng } else { 42895d4f98a2SYan Zheng mid = slot; 42905d4f98a2SYan Zheng if (mid != nritems && 42915d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 42920b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 429399d8f83cSChris Mason if (data_size && !tried_avoid_double) 429499d8f83cSChris Mason goto push_for_double; 42955d4f98a2SYan Zheng split = 2; 42965d4f98a2SYan Zheng } 42975d4f98a2SYan Zheng } 42985d4f98a2SYan Zheng } 42995d4f98a2SYan Zheng } 43005d4f98a2SYan Zheng 43015d4f98a2SYan Zheng if (split == 0) 43025d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 43035d4f98a2SYan Zheng else 43045d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 43055d4f98a2SYan Zheng 43064d75f8a9SDavid Sterba right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 43075581a51aSJan Schmidt &disk_key, 0, l->start, 0); 4308f0486c68SYan, Zheng if (IS_ERR(right)) 430944871b1bSChris Mason return PTR_ERR(right); 4310f0486c68SYan, Zheng 43110b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 431244871b1bSChris Mason 4313b159fa28SDavid Sterba memzero_extent_buffer(right, 0, sizeof(struct btrfs_header)); 431444871b1bSChris Mason btrfs_set_header_bytenr(right, right->start); 431544871b1bSChris Mason btrfs_set_header_generation(right, trans->transid); 43165d4f98a2SYan Zheng btrfs_set_header_backref_rev(right, BTRFS_MIXED_BACKREF_REV); 431744871b1bSChris Mason btrfs_set_header_owner(right, root->root_key.objectid); 431844871b1bSChris Mason btrfs_set_header_level(right, 0); 4319d24ee97bSDavid Sterba write_extent_buffer_fsid(right, fs_info->fsid); 4320d24ee97bSDavid Sterba write_extent_buffer_chunk_tree_uuid(right, fs_info->chunk_tree_uuid); 432144871b1bSChris Mason 43225d4f98a2SYan Zheng if (split == 0) { 432344871b1bSChris Mason if (mid <= slot) { 432444871b1bSChris Mason btrfs_set_header_nritems(right, 0); 43252ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, 43262ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 432744871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 432844871b1bSChris Mason free_extent_buffer(path->nodes[0]); 432944871b1bSChris Mason path->nodes[0] = right; 433044871b1bSChris Mason path->slots[0] = 0; 433144871b1bSChris Mason path->slots[1] += 1; 433244871b1bSChris Mason } else { 433344871b1bSChris Mason btrfs_set_header_nritems(right, 0); 43342ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, 43352ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 433644871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 433744871b1bSChris Mason free_extent_buffer(path->nodes[0]); 433844871b1bSChris Mason path->nodes[0] = right; 433944871b1bSChris Mason path->slots[0] = 0; 4340143bede5SJeff Mahoney if (path->slots[1] == 0) 4341b7a0365eSDaniel Dressler fixup_low_keys(fs_info, path, &disk_key, 1); 43425d4f98a2SYan Zheng } 4343196e0249SLiu Bo /* 4344196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 4345196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 4346196e0249SLiu Bo * the content of ins_len to 'right'. 4347196e0249SLiu Bo */ 434844871b1bSChris Mason return ret; 434944871b1bSChris Mason } 435044871b1bSChris Mason 43512ff7e61eSJeff Mahoney copy_for_split(trans, fs_info, path, l, right, slot, mid, nritems); 435244871b1bSChris Mason 43535d4f98a2SYan Zheng if (split == 2) { 4354cc0c5538SChris Mason BUG_ON(num_doubles != 0); 4355cc0c5538SChris Mason num_doubles++; 4356cc0c5538SChris Mason goto again; 43573326d1b0SChris Mason } 435844871b1bSChris Mason 4359143bede5SJeff Mahoney return 0; 436099d8f83cSChris Mason 436199d8f83cSChris Mason push_for_double: 436299d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 436399d8f83cSChris Mason tried_avoid_double = 1; 43642ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= data_size) 436599d8f83cSChris Mason return 0; 436699d8f83cSChris Mason goto again; 4367be0e5c09SChris Mason } 4368be0e5c09SChris Mason 4369ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 4370ad48fd75SYan, Zheng struct btrfs_root *root, 4371ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 4372ad48fd75SYan, Zheng { 43732ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 4374ad48fd75SYan, Zheng struct btrfs_key key; 4375ad48fd75SYan, Zheng struct extent_buffer *leaf; 4376ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 4377ad48fd75SYan, Zheng u64 extent_len = 0; 4378ad48fd75SYan, Zheng u32 item_size; 4379ad48fd75SYan, Zheng int ret; 4380ad48fd75SYan, Zheng 4381ad48fd75SYan, Zheng leaf = path->nodes[0]; 4382ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4383ad48fd75SYan, Zheng 4384ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 4385ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 4386ad48fd75SYan, Zheng 43872ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) >= ins_len) 4388ad48fd75SYan, Zheng return 0; 4389ad48fd75SYan, Zheng 4390ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4391ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4392ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4393ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4394ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 4395ad48fd75SYan, Zheng } 4396b3b4aa74SDavid Sterba btrfs_release_path(path); 4397ad48fd75SYan, Zheng 4398ad48fd75SYan, Zheng path->keep_locks = 1; 4399ad48fd75SYan, Zheng path->search_for_split = 1; 4400ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 4401ad48fd75SYan, Zheng path->search_for_split = 0; 4402a8df6fe6SFilipe Manana if (ret > 0) 4403a8df6fe6SFilipe Manana ret = -EAGAIN; 4404ad48fd75SYan, Zheng if (ret < 0) 4405ad48fd75SYan, Zheng goto err; 4406ad48fd75SYan, Zheng 4407ad48fd75SYan, Zheng ret = -EAGAIN; 4408ad48fd75SYan, Zheng leaf = path->nodes[0]; 4409a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 4410a8df6fe6SFilipe Manana if (item_size != btrfs_item_size_nr(leaf, path->slots[0])) 4411ad48fd75SYan, Zheng goto err; 4412ad48fd75SYan, Zheng 4413109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 44142ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= ins_len) 4415109f6aefSChris Mason goto err; 4416109f6aefSChris Mason 4417ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4418ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4419ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4420ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 4421ad48fd75SYan, Zheng goto err; 4422ad48fd75SYan, Zheng } 4423ad48fd75SYan, Zheng 4424ad48fd75SYan, Zheng btrfs_set_path_blocking(path); 4425ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 4426f0486c68SYan, Zheng if (ret) 4427f0486c68SYan, Zheng goto err; 4428ad48fd75SYan, Zheng 4429ad48fd75SYan, Zheng path->keep_locks = 0; 4430ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 4431ad48fd75SYan, Zheng return 0; 4432ad48fd75SYan, Zheng err: 4433ad48fd75SYan, Zheng path->keep_locks = 0; 4434ad48fd75SYan, Zheng return ret; 4435ad48fd75SYan, Zheng } 4436ad48fd75SYan, Zheng 44374961e293SDavid Sterba static noinline int split_item(struct btrfs_fs_info *fs_info, 4438459931ecSChris Mason struct btrfs_path *path, 4439310712b2SOmar Sandoval const struct btrfs_key *new_key, 4440459931ecSChris Mason unsigned long split_offset) 4441459931ecSChris Mason { 4442459931ecSChris Mason struct extent_buffer *leaf; 4443459931ecSChris Mason struct btrfs_item *item; 4444459931ecSChris Mason struct btrfs_item *new_item; 4445459931ecSChris Mason int slot; 4446ad48fd75SYan, Zheng char *buf; 4447459931ecSChris Mason u32 nritems; 4448ad48fd75SYan, Zheng u32 item_size; 4449459931ecSChris Mason u32 orig_offset; 4450459931ecSChris Mason struct btrfs_disk_key disk_key; 4451459931ecSChris Mason 4452459931ecSChris Mason leaf = path->nodes[0]; 44532ff7e61eSJeff Mahoney BUG_ON(btrfs_leaf_free_space(fs_info, leaf) < sizeof(struct btrfs_item)); 4454b9473439SChris Mason 4455b4ce94deSChris Mason btrfs_set_path_blocking(path); 4456b4ce94deSChris Mason 4457dd3cc16bSRoss Kirk item = btrfs_item_nr(path->slots[0]); 4458459931ecSChris Mason orig_offset = btrfs_item_offset(leaf, item); 4459459931ecSChris Mason item_size = btrfs_item_size(leaf, item); 4460459931ecSChris Mason 4461459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 4462ad48fd75SYan, Zheng if (!buf) 4463ad48fd75SYan, Zheng return -ENOMEM; 4464ad48fd75SYan, Zheng 4465459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 4466459931ecSChris Mason path->slots[0]), item_size); 4467ad48fd75SYan, Zheng 4468459931ecSChris Mason slot = path->slots[0] + 1; 4469459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 4470459931ecSChris Mason if (slot != nritems) { 4471459931ecSChris Mason /* shift the items */ 4472459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 4473459931ecSChris Mason btrfs_item_nr_offset(slot), 4474459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4475459931ecSChris Mason } 4476459931ecSChris Mason 4477459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 4478459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4479459931ecSChris Mason 4480dd3cc16bSRoss Kirk new_item = btrfs_item_nr(slot); 4481459931ecSChris Mason 4482459931ecSChris Mason btrfs_set_item_offset(leaf, new_item, orig_offset); 4483459931ecSChris Mason btrfs_set_item_size(leaf, new_item, item_size - split_offset); 4484459931ecSChris Mason 4485459931ecSChris Mason btrfs_set_item_offset(leaf, item, 4486459931ecSChris Mason orig_offset + item_size - split_offset); 4487459931ecSChris Mason btrfs_set_item_size(leaf, item, split_offset); 4488459931ecSChris Mason 4489459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 4490459931ecSChris Mason 4491459931ecSChris Mason /* write the data for the start of the original item */ 4492459931ecSChris Mason write_extent_buffer(leaf, buf, 4493459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 4494459931ecSChris Mason split_offset); 4495459931ecSChris Mason 4496459931ecSChris Mason /* write the data for the new item */ 4497459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 4498459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 4499459931ecSChris Mason item_size - split_offset); 4500459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 4501459931ecSChris Mason 45022ff7e61eSJeff Mahoney BUG_ON(btrfs_leaf_free_space(fs_info, leaf) < 0); 4503459931ecSChris Mason kfree(buf); 4504ad48fd75SYan, Zheng return 0; 4505ad48fd75SYan, Zheng } 4506ad48fd75SYan, Zheng 4507ad48fd75SYan, Zheng /* 4508ad48fd75SYan, Zheng * This function splits a single item into two items, 4509ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 4510ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 4511ad48fd75SYan, Zheng * 4512ad48fd75SYan, Zheng * The path may be released by this operation. After 4513ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 4514ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 4515ad48fd75SYan, Zheng * 4516ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 4517ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 4518ad48fd75SYan, Zheng * 4519ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 4520ad48fd75SYan, Zheng * leaf the entire time. 4521ad48fd75SYan, Zheng */ 4522ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 4523ad48fd75SYan, Zheng struct btrfs_root *root, 4524ad48fd75SYan, Zheng struct btrfs_path *path, 4525310712b2SOmar Sandoval const struct btrfs_key *new_key, 4526ad48fd75SYan, Zheng unsigned long split_offset) 4527ad48fd75SYan, Zheng { 4528ad48fd75SYan, Zheng int ret; 4529ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4530ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 4531ad48fd75SYan, Zheng if (ret) 4532459931ecSChris Mason return ret; 4533ad48fd75SYan, Zheng 45344961e293SDavid Sterba ret = split_item(root->fs_info, path, new_key, split_offset); 4535ad48fd75SYan, Zheng return ret; 4536ad48fd75SYan, Zheng } 4537ad48fd75SYan, Zheng 4538ad48fd75SYan, Zheng /* 4539ad48fd75SYan, Zheng * This function duplicate a item, giving 'new_key' to the new item. 4540ad48fd75SYan, Zheng * It guarantees both items live in the same tree leaf and the new item 4541ad48fd75SYan, Zheng * is contiguous with the original item. 4542ad48fd75SYan, Zheng * 4543ad48fd75SYan, Zheng * This allows us to split file extent in place, keeping a lock on the 4544ad48fd75SYan, Zheng * leaf the entire time. 4545ad48fd75SYan, Zheng */ 4546ad48fd75SYan, Zheng int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4547ad48fd75SYan, Zheng struct btrfs_root *root, 4548ad48fd75SYan, Zheng struct btrfs_path *path, 4549310712b2SOmar Sandoval const struct btrfs_key *new_key) 4550ad48fd75SYan, Zheng { 4551ad48fd75SYan, Zheng struct extent_buffer *leaf; 4552ad48fd75SYan, Zheng int ret; 4553ad48fd75SYan, Zheng u32 item_size; 4554ad48fd75SYan, Zheng 4555ad48fd75SYan, Zheng leaf = path->nodes[0]; 4556ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4557ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4558ad48fd75SYan, Zheng item_size + sizeof(struct btrfs_item)); 4559ad48fd75SYan, Zheng if (ret) 4560ad48fd75SYan, Zheng return ret; 4561ad48fd75SYan, Zheng 4562ad48fd75SYan, Zheng path->slots[0]++; 4563afe5fea7STsutomu Itoh setup_items_for_insert(root, path, new_key, &item_size, 4564ad48fd75SYan, Zheng item_size, item_size + 4565ad48fd75SYan, Zheng sizeof(struct btrfs_item), 1); 4566ad48fd75SYan, Zheng leaf = path->nodes[0]; 4567ad48fd75SYan, Zheng memcpy_extent_buffer(leaf, 4568ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0]), 4569ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4570ad48fd75SYan, Zheng item_size); 4571ad48fd75SYan, Zheng return 0; 4572459931ecSChris Mason } 4573459931ecSChris Mason 4574459931ecSChris Mason /* 4575d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4576d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4577d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4578d352ac68SChris Mason * the front. 4579d352ac68SChris Mason */ 45802ff7e61eSJeff Mahoney void btrfs_truncate_item(struct btrfs_fs_info *fs_info, 45812ff7e61eSJeff Mahoney struct btrfs_path *path, u32 new_size, int from_end) 4582b18c6685SChris Mason { 4583b18c6685SChris Mason int slot; 45845f39d397SChris Mason struct extent_buffer *leaf; 45855f39d397SChris Mason struct btrfs_item *item; 4586b18c6685SChris Mason u32 nritems; 4587b18c6685SChris Mason unsigned int data_end; 4588b18c6685SChris Mason unsigned int old_data_start; 4589b18c6685SChris Mason unsigned int old_size; 4590b18c6685SChris Mason unsigned int size_diff; 4591b18c6685SChris Mason int i; 4592cfed81a0SChris Mason struct btrfs_map_token token; 4593cfed81a0SChris Mason 4594cfed81a0SChris Mason btrfs_init_map_token(&token); 4595b18c6685SChris Mason 45965f39d397SChris Mason leaf = path->nodes[0]; 4597179e29e4SChris Mason slot = path->slots[0]; 4598179e29e4SChris Mason 4599179e29e4SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4600179e29e4SChris Mason if (old_size == new_size) 4601143bede5SJeff Mahoney return; 4602b18c6685SChris Mason 46035f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 46042ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, leaf); 4605b18c6685SChris Mason 46065f39d397SChris Mason old_data_start = btrfs_item_offset_nr(leaf, slot); 4607179e29e4SChris Mason 4608b18c6685SChris Mason size_diff = old_size - new_size; 4609b18c6685SChris Mason 4610b18c6685SChris Mason BUG_ON(slot < 0); 4611b18c6685SChris Mason BUG_ON(slot >= nritems); 4612b18c6685SChris Mason 4613b18c6685SChris Mason /* 4614b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4615b18c6685SChris Mason */ 4616b18c6685SChris Mason /* first correct the data pointers */ 4617b18c6685SChris Mason for (i = slot; i < nritems; i++) { 46185f39d397SChris Mason u32 ioff; 4619dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4620db94535dSChris Mason 4621cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4622cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4623cfed81a0SChris Mason ioff + size_diff, &token); 4624b18c6685SChris Mason } 4625db94535dSChris Mason 4626b18c6685SChris Mason /* shift the data */ 4627179e29e4SChris Mason if (from_end) { 46283d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 46293d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 4630b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 4631179e29e4SChris Mason } else { 4632179e29e4SChris Mason struct btrfs_disk_key disk_key; 4633179e29e4SChris Mason u64 offset; 4634179e29e4SChris Mason 4635179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4636179e29e4SChris Mason 4637179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4638179e29e4SChris Mason unsigned long ptr; 4639179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4640179e29e4SChris Mason 4641179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4642179e29e4SChris Mason struct btrfs_file_extent_item); 4643179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4644179e29e4SChris Mason (unsigned long)fi - size_diff); 4645179e29e4SChris Mason 4646179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4647179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4648179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4649179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4650179e29e4SChris Mason (unsigned long)fi, 46517ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 4652179e29e4SChris Mason } 4653179e29e4SChris Mason } 4654179e29e4SChris Mason 46553d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 46563d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 4657179e29e4SChris Mason data_end, old_data_start - data_end); 4658179e29e4SChris Mason 4659179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4660179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4661179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4662179e29e4SChris Mason if (slot == 0) 46630b246afaSJeff Mahoney fixup_low_keys(fs_info, path, &disk_key, 1); 4664179e29e4SChris Mason } 46655f39d397SChris Mason 4666dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 46675f39d397SChris Mason btrfs_set_item_size(leaf, item, new_size); 46685f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4669b18c6685SChris Mason 46702ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < 0) { 4671a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4672b18c6685SChris Mason BUG(); 46735f39d397SChris Mason } 4674b18c6685SChris Mason } 4675b18c6685SChris Mason 4676d352ac68SChris Mason /* 46778f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4678d352ac68SChris Mason */ 46792ff7e61eSJeff Mahoney void btrfs_extend_item(struct btrfs_fs_info *fs_info, struct btrfs_path *path, 46805f39d397SChris Mason u32 data_size) 46816567e837SChris Mason { 46826567e837SChris Mason int slot; 46835f39d397SChris Mason struct extent_buffer *leaf; 46845f39d397SChris Mason struct btrfs_item *item; 46856567e837SChris Mason u32 nritems; 46866567e837SChris Mason unsigned int data_end; 46876567e837SChris Mason unsigned int old_data; 46886567e837SChris Mason unsigned int old_size; 46896567e837SChris Mason int i; 4690cfed81a0SChris Mason struct btrfs_map_token token; 4691cfed81a0SChris Mason 4692cfed81a0SChris Mason btrfs_init_map_token(&token); 46936567e837SChris Mason 46945f39d397SChris Mason leaf = path->nodes[0]; 46956567e837SChris Mason 46965f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 46972ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, leaf); 46986567e837SChris Mason 46992ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < data_size) { 4700a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47016567e837SChris Mason BUG(); 47025f39d397SChris Mason } 47036567e837SChris Mason slot = path->slots[0]; 47045f39d397SChris Mason old_data = btrfs_item_end_nr(leaf, slot); 47056567e837SChris Mason 47066567e837SChris Mason BUG_ON(slot < 0); 47073326d1b0SChris Mason if (slot >= nritems) { 4708a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47090b246afaSJeff Mahoney btrfs_crit(fs_info, "slot %d too large, nritems %d", 4710d397712bSChris Mason slot, nritems); 47113326d1b0SChris Mason BUG_ON(1); 47123326d1b0SChris Mason } 47136567e837SChris Mason 47146567e837SChris Mason /* 47156567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 47166567e837SChris Mason */ 47176567e837SChris Mason /* first correct the data pointers */ 47186567e837SChris Mason for (i = slot; i < nritems; i++) { 47195f39d397SChris Mason u32 ioff; 4720dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4721db94535dSChris Mason 4722cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4723cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4724cfed81a0SChris Mason ioff - data_size, &token); 47256567e837SChris Mason } 47265f39d397SChris Mason 47276567e837SChris Mason /* shift the data */ 47283d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 47293d9ec8c4SNikolay Borisov data_end - data_size, BTRFS_LEAF_DATA_OFFSET + 47306567e837SChris Mason data_end, old_data - data_end); 47315f39d397SChris Mason 47326567e837SChris Mason data_end = old_data; 47335f39d397SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4734dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 47355f39d397SChris Mason btrfs_set_item_size(leaf, item, old_size + data_size); 47365f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 47376567e837SChris Mason 47382ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < 0) { 4739a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47406567e837SChris Mason BUG(); 47415f39d397SChris Mason } 47426567e837SChris Mason } 47436567e837SChris Mason 474474123bd7SChris Mason /* 474544871b1bSChris Mason * this is a helper for btrfs_insert_empty_items, the main goal here is 474644871b1bSChris Mason * to save stack depth by doing the bulk of the work in a function 474744871b1bSChris Mason * that doesn't call btrfs_search_slot 474874123bd7SChris Mason */ 4749afe5fea7STsutomu Itoh void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4750310712b2SOmar Sandoval const struct btrfs_key *cpu_key, u32 *data_size, 475144871b1bSChris Mason u32 total_data, u32 total_size, int nr) 4752be0e5c09SChris Mason { 47530b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 47545f39d397SChris Mason struct btrfs_item *item; 47559c58309dSChris Mason int i; 47567518a238SChris Mason u32 nritems; 4757be0e5c09SChris Mason unsigned int data_end; 4758e2fa7227SChris Mason struct btrfs_disk_key disk_key; 475944871b1bSChris Mason struct extent_buffer *leaf; 476044871b1bSChris Mason int slot; 4761cfed81a0SChris Mason struct btrfs_map_token token; 4762cfed81a0SChris Mason 476324cdc847SFilipe Manana if (path->slots[0] == 0) { 476424cdc847SFilipe Manana btrfs_cpu_key_to_disk(&disk_key, cpu_key); 47650b246afaSJeff Mahoney fixup_low_keys(fs_info, path, &disk_key, 1); 476624cdc847SFilipe Manana } 476724cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 476824cdc847SFilipe Manana 4769cfed81a0SChris Mason btrfs_init_map_token(&token); 4770e2fa7227SChris Mason 47715f39d397SChris Mason leaf = path->nodes[0]; 477244871b1bSChris Mason slot = path->slots[0]; 477374123bd7SChris Mason 47745f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 47752ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, leaf); 4776eb60ceacSChris Mason 47772ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < total_size) { 4778a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47790b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 47802ff7e61eSJeff Mahoney total_size, btrfs_leaf_free_space(fs_info, leaf)); 4781be0e5c09SChris Mason BUG(); 4782d4dbff95SChris Mason } 47835f39d397SChris Mason 4784be0e5c09SChris Mason if (slot != nritems) { 47855f39d397SChris Mason unsigned int old_data = btrfs_item_end_nr(leaf, slot); 4786be0e5c09SChris Mason 47875f39d397SChris Mason if (old_data < data_end) { 4788a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47890b246afaSJeff Mahoney btrfs_crit(fs_info, "slot %d old_data %d data_end %d", 47905f39d397SChris Mason slot, old_data, data_end); 47915f39d397SChris Mason BUG_ON(1); 47925f39d397SChris Mason } 4793be0e5c09SChris Mason /* 4794be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4795be0e5c09SChris Mason */ 4796be0e5c09SChris Mason /* first correct the data pointers */ 47970783fcfcSChris Mason for (i = slot; i < nritems; i++) { 47985f39d397SChris Mason u32 ioff; 4799db94535dSChris Mason 4800dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4801cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4802cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4803cfed81a0SChris Mason ioff - total_data, &token); 48040783fcfcSChris Mason } 4805be0e5c09SChris Mason /* shift the items */ 48069c58309dSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), 48075f39d397SChris Mason btrfs_item_nr_offset(slot), 48080783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4809be0e5c09SChris Mason 4810be0e5c09SChris Mason /* shift the data */ 48113d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 48123d9ec8c4SNikolay Borisov data_end - total_data, BTRFS_LEAF_DATA_OFFSET + 4813be0e5c09SChris Mason data_end, old_data - data_end); 4814be0e5c09SChris Mason data_end = old_data; 4815be0e5c09SChris Mason } 48165f39d397SChris Mason 481762e2749eSChris Mason /* setup the item for the new data */ 48189c58309dSChris Mason for (i = 0; i < nr; i++) { 48199c58309dSChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); 48209c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4821dd3cc16bSRoss Kirk item = btrfs_item_nr(slot + i); 4822cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4823cfed81a0SChris Mason data_end - data_size[i], &token); 48249c58309dSChris Mason data_end -= data_size[i]; 4825cfed81a0SChris Mason btrfs_set_token_item_size(leaf, item, data_size[i], &token); 48269c58309dSChris Mason } 482744871b1bSChris Mason 48289c58309dSChris Mason btrfs_set_header_nritems(leaf, nritems + nr); 4829b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4830aa5d6bedSChris Mason 48312ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < 0) { 4832a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4833be0e5c09SChris Mason BUG(); 48345f39d397SChris Mason } 483544871b1bSChris Mason } 483644871b1bSChris Mason 483744871b1bSChris Mason /* 483844871b1bSChris Mason * Given a key and some data, insert items into the tree. 483944871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 484044871b1bSChris Mason */ 484144871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 484244871b1bSChris Mason struct btrfs_root *root, 484344871b1bSChris Mason struct btrfs_path *path, 4844310712b2SOmar Sandoval const struct btrfs_key *cpu_key, u32 *data_size, 484544871b1bSChris Mason int nr) 484644871b1bSChris Mason { 484744871b1bSChris Mason int ret = 0; 484844871b1bSChris Mason int slot; 484944871b1bSChris Mason int i; 485044871b1bSChris Mason u32 total_size = 0; 485144871b1bSChris Mason u32 total_data = 0; 485244871b1bSChris Mason 485344871b1bSChris Mason for (i = 0; i < nr; i++) 485444871b1bSChris Mason total_data += data_size[i]; 485544871b1bSChris Mason 485644871b1bSChris Mason total_size = total_data + (nr * sizeof(struct btrfs_item)); 485744871b1bSChris Mason ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1); 485844871b1bSChris Mason if (ret == 0) 485944871b1bSChris Mason return -EEXIST; 486044871b1bSChris Mason if (ret < 0) 4861143bede5SJeff Mahoney return ret; 486244871b1bSChris Mason 486344871b1bSChris Mason slot = path->slots[0]; 486444871b1bSChris Mason BUG_ON(slot < 0); 486544871b1bSChris Mason 4866afe5fea7STsutomu Itoh setup_items_for_insert(root, path, cpu_key, data_size, 486744871b1bSChris Mason total_data, total_size, nr); 4868143bede5SJeff Mahoney return 0; 486962e2749eSChris Mason } 487062e2749eSChris Mason 487162e2749eSChris Mason /* 487262e2749eSChris Mason * Given a key and some data, insert an item into the tree. 487362e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 487462e2749eSChris Mason */ 4875310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4876310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4877310712b2SOmar Sandoval u32 data_size) 487862e2749eSChris Mason { 487962e2749eSChris Mason int ret = 0; 48802c90e5d6SChris Mason struct btrfs_path *path; 48815f39d397SChris Mason struct extent_buffer *leaf; 48825f39d397SChris Mason unsigned long ptr; 488362e2749eSChris Mason 48842c90e5d6SChris Mason path = btrfs_alloc_path(); 4885db5b493aSTsutomu Itoh if (!path) 4886db5b493aSTsutomu Itoh return -ENOMEM; 48872c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 488862e2749eSChris Mason if (!ret) { 48895f39d397SChris Mason leaf = path->nodes[0]; 48905f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 48915f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 48925f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 489362e2749eSChris Mason } 48942c90e5d6SChris Mason btrfs_free_path(path); 4895aa5d6bedSChris Mason return ret; 4896be0e5c09SChris Mason } 4897be0e5c09SChris Mason 489874123bd7SChris Mason /* 48995de08d7dSChris Mason * delete the pointer from a given node. 490074123bd7SChris Mason * 4901d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4902d352ac68SChris Mason * empty a node. 490374123bd7SChris Mason */ 4904afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4905afe5fea7STsutomu Itoh int level, int slot) 4906be0e5c09SChris Mason { 49070b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 49085f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 49097518a238SChris Mason u32 nritems; 4910f3ea38daSJan Schmidt int ret; 4911be0e5c09SChris Mason 49125f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4913be0e5c09SChris Mason if (slot != nritems - 1) { 49140e411eceSLiu Bo if (level) 49150b246afaSJeff Mahoney tree_mod_log_eb_move(fs_info, parent, slot, 4916f3ea38daSJan Schmidt slot + 1, nritems - slot - 1); 49175f39d397SChris Mason memmove_extent_buffer(parent, 49185f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 49195f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4920d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4921d6025579SChris Mason (nritems - slot - 1)); 492257ba86c0SChris Mason } else if (level) { 49230b246afaSJeff Mahoney ret = tree_mod_log_insert_key(fs_info, parent, slot, 4924c8cc6341SJosef Bacik MOD_LOG_KEY_REMOVE, GFP_NOFS); 492557ba86c0SChris Mason BUG_ON(ret < 0); 4926be0e5c09SChris Mason } 4927f3ea38daSJan Schmidt 49287518a238SChris Mason nritems--; 49295f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 49307518a238SChris Mason if (nritems == 0 && parent == root->node) { 49315f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4932eb60ceacSChris Mason /* just turn the root into a leaf and break */ 49335f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4934bb803951SChris Mason } else if (slot == 0) { 49355f39d397SChris Mason struct btrfs_disk_key disk_key; 49365f39d397SChris Mason 49375f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 49380b246afaSJeff Mahoney fixup_low_keys(fs_info, path, &disk_key, level + 1); 4939be0e5c09SChris Mason } 4940d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4941be0e5c09SChris Mason } 4942be0e5c09SChris Mason 494374123bd7SChris Mason /* 4944323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 49455d4f98a2SYan Zheng * path->nodes[1]. 4946323ac95bSChris Mason * 4947323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4948323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4949323ac95bSChris Mason * 4950323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4951323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4952323ac95bSChris Mason */ 4953143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4954323ac95bSChris Mason struct btrfs_root *root, 49555d4f98a2SYan Zheng struct btrfs_path *path, 49565d4f98a2SYan Zheng struct extent_buffer *leaf) 4957323ac95bSChris Mason { 49585d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4959afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4960323ac95bSChris Mason 49614d081c41SChris Mason /* 49624d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 49634d081c41SChris Mason * aren't holding any locks when we call it 49644d081c41SChris Mason */ 49654d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 49664d081c41SChris Mason 4967f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4968f0486c68SYan, Zheng 49693083ee2eSJosef Bacik extent_buffer_get(leaf); 49705581a51aSJan Schmidt btrfs_free_tree_block(trans, root, leaf, 0, 1); 49713083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4972323ac95bSChris Mason } 4973323ac95bSChris Mason /* 497474123bd7SChris Mason * delete the item at the leaf level in path. If that empties 497574123bd7SChris Mason * the leaf, remove it from the tree 497674123bd7SChris Mason */ 497785e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 497885e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4979be0e5c09SChris Mason { 49800b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 49815f39d397SChris Mason struct extent_buffer *leaf; 49825f39d397SChris Mason struct btrfs_item *item; 4983ce0eac2aSAlexandru Moise u32 last_off; 4984ce0eac2aSAlexandru Moise u32 dsize = 0; 4985aa5d6bedSChris Mason int ret = 0; 4986aa5d6bedSChris Mason int wret; 498785e21bacSChris Mason int i; 49887518a238SChris Mason u32 nritems; 4989cfed81a0SChris Mason struct btrfs_map_token token; 4990cfed81a0SChris Mason 4991cfed81a0SChris Mason btrfs_init_map_token(&token); 4992be0e5c09SChris Mason 49935f39d397SChris Mason leaf = path->nodes[0]; 499485e21bacSChris Mason last_off = btrfs_item_offset_nr(leaf, slot + nr - 1); 499585e21bacSChris Mason 499685e21bacSChris Mason for (i = 0; i < nr; i++) 499785e21bacSChris Mason dsize += btrfs_item_size_nr(leaf, slot + i); 499885e21bacSChris Mason 49995f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 5000be0e5c09SChris Mason 500185e21bacSChris Mason if (slot + nr != nritems) { 50022ff7e61eSJeff Mahoney int data_end = leaf_data_end(fs_info, leaf); 50035f39d397SChris Mason 50043d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 5005d6025579SChris Mason data_end + dsize, 50063d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 500785e21bacSChris Mason last_off - data_end); 50085f39d397SChris Mason 500985e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 50105f39d397SChris Mason u32 ioff; 5011db94535dSChris Mason 5012dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 5013cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 5014cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 5015cfed81a0SChris Mason ioff + dsize, &token); 50160783fcfcSChris Mason } 5017db94535dSChris Mason 50185f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 501985e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 50200783fcfcSChris Mason sizeof(struct btrfs_item) * 502185e21bacSChris Mason (nritems - slot - nr)); 5022be0e5c09SChris Mason } 502385e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 502485e21bacSChris Mason nritems -= nr; 50255f39d397SChris Mason 502674123bd7SChris Mason /* delete the leaf if we've emptied it */ 50277518a238SChris Mason if (nritems == 0) { 50285f39d397SChris Mason if (leaf == root->node) { 50295f39d397SChris Mason btrfs_set_header_level(leaf, 0); 50309a8dd150SChris Mason } else { 5031f0486c68SYan, Zheng btrfs_set_path_blocking(path); 50327c302b49SDavid Sterba clean_tree_block(fs_info, leaf); 5033143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 50349a8dd150SChris Mason } 5035be0e5c09SChris Mason } else { 50367518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 5037aa5d6bedSChris Mason if (slot == 0) { 50385f39d397SChris Mason struct btrfs_disk_key disk_key; 50395f39d397SChris Mason 50405f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 50410b246afaSJeff Mahoney fixup_low_keys(fs_info, path, &disk_key, 1); 5042aa5d6bedSChris Mason } 5043aa5d6bedSChris Mason 504474123bd7SChris Mason /* delete the leaf if it is mostly empty */ 50450b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 5046be0e5c09SChris Mason /* push_leaf_left fixes the path. 5047be0e5c09SChris Mason * make sure the path still points to our leaf 5048be0e5c09SChris Mason * for possible call to del_ptr below 5049be0e5c09SChris Mason */ 50504920c9acSChris Mason slot = path->slots[1]; 50515f39d397SChris Mason extent_buffer_get(leaf); 50525f39d397SChris Mason 5053b9473439SChris Mason btrfs_set_path_blocking(path); 505499d8f83cSChris Mason wret = push_leaf_left(trans, root, path, 1, 1, 505599d8f83cSChris Mason 1, (u32)-1); 505654aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 5057aa5d6bedSChris Mason ret = wret; 50585f39d397SChris Mason 50595f39d397SChris Mason if (path->nodes[0] == leaf && 50605f39d397SChris Mason btrfs_header_nritems(leaf)) { 506199d8f83cSChris Mason wret = push_leaf_right(trans, root, path, 1, 506299d8f83cSChris Mason 1, 1, 0); 506354aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 5064aa5d6bedSChris Mason ret = wret; 5065aa5d6bedSChris Mason } 50665f39d397SChris Mason 50675f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 5068323ac95bSChris Mason path->slots[1] = slot; 5069143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 50705f39d397SChris Mason free_extent_buffer(leaf); 5071143bede5SJeff Mahoney ret = 0; 50725de08d7dSChris Mason } else { 5073925baeddSChris Mason /* if we're still in the path, make sure 5074925baeddSChris Mason * we're dirty. Otherwise, one of the 5075925baeddSChris Mason * push_leaf functions must have already 5076925baeddSChris Mason * dirtied this buffer 5077925baeddSChris Mason */ 5078925baeddSChris Mason if (path->nodes[0] == leaf) 50795f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 50805f39d397SChris Mason free_extent_buffer(leaf); 5081be0e5c09SChris Mason } 5082d5719762SChris Mason } else { 50835f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 5084be0e5c09SChris Mason } 50859a8dd150SChris Mason } 5086aa5d6bedSChris Mason return ret; 50879a8dd150SChris Mason } 50889a8dd150SChris Mason 508997571fd0SChris Mason /* 5090925baeddSChris Mason * search the tree again to find a leaf with lesser keys 50917bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 50927bb86316SChris Mason * returns < 0 on io errors. 5093d352ac68SChris Mason * 5094d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 5095d352ac68SChris Mason * time you call it. 50967bb86316SChris Mason */ 509716e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 50987bb86316SChris Mason { 5099925baeddSChris Mason struct btrfs_key key; 5100925baeddSChris Mason struct btrfs_disk_key found_key; 5101925baeddSChris Mason int ret; 51027bb86316SChris Mason 5103925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 5104925baeddSChris Mason 5105e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 5106925baeddSChris Mason key.offset--; 5107e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 5108925baeddSChris Mason key.type--; 5109e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5110e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 5111925baeddSChris Mason key.objectid--; 5112e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 5113e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5114e8b0d724SFilipe David Borba Manana } else { 51157bb86316SChris Mason return 1; 5116e8b0d724SFilipe David Borba Manana } 51177bb86316SChris Mason 5118b3b4aa74SDavid Sterba btrfs_release_path(path); 5119925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5120925baeddSChris Mason if (ret < 0) 5121925baeddSChris Mason return ret; 5122925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 5123925baeddSChris Mason ret = comp_keys(&found_key, &key); 5124337c6f68SFilipe Manana /* 5125337c6f68SFilipe Manana * We might have had an item with the previous key in the tree right 5126337c6f68SFilipe Manana * before we released our path. And after we released our path, that 5127337c6f68SFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 5128337c6f68SFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 5129337c6f68SFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 5130337c6f68SFilipe Manana * Therefore account for these 2 cases, so that our callers (like 5131337c6f68SFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 5132337c6f68SFilipe Manana * the previous key we computed above. 5133337c6f68SFilipe Manana */ 5134337c6f68SFilipe Manana if (ret <= 0) 51357bb86316SChris Mason return 0; 5136925baeddSChris Mason return 1; 51377bb86316SChris Mason } 51387bb86316SChris Mason 51393f157a2fSChris Mason /* 51403f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 5141de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 5142de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 51433f157a2fSChris Mason * 51443f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 51453f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 51463f157a2fSChris Mason * key and get a writable path. 51473f157a2fSChris Mason * 51483f157a2fSChris Mason * This does lock as it descends, and path->keep_locks should be set 51493f157a2fSChris Mason * to 1 by the caller. 51503f157a2fSChris Mason * 51513f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 51523f157a2fSChris Mason * of the tree. 51533f157a2fSChris Mason * 5154d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 5155d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 5156d352ac68SChris Mason * skipped over (without reading them). 5157d352ac68SChris Mason * 51583f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 51593f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 51603f157a2fSChris Mason */ 51613f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 5162de78b51aSEric Sandeen struct btrfs_path *path, 51633f157a2fSChris Mason u64 min_trans) 51643f157a2fSChris Mason { 51652ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 51663f157a2fSChris Mason struct extent_buffer *cur; 51673f157a2fSChris Mason struct btrfs_key found_key; 51683f157a2fSChris Mason int slot; 51699652480bSYan int sret; 51703f157a2fSChris Mason u32 nritems; 51713f157a2fSChris Mason int level; 51723f157a2fSChris Mason int ret = 1; 5173f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 51743f157a2fSChris Mason 5175f98de9b9SFilipe Manana path->keep_locks = 1; 51763f157a2fSChris Mason again: 5177bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 51783f157a2fSChris Mason level = btrfs_header_level(cur); 5179e02119d5SChris Mason WARN_ON(path->nodes[level]); 51803f157a2fSChris Mason path->nodes[level] = cur; 5181bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 51823f157a2fSChris Mason 51833f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 51843f157a2fSChris Mason ret = 1; 51853f157a2fSChris Mason goto out; 51863f157a2fSChris Mason } 51873f157a2fSChris Mason while (1) { 51883f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 51893f157a2fSChris Mason level = btrfs_header_level(cur); 5190a74b35ecSNikolay Borisov sret = btrfs_bin_search(cur, min_key, level, &slot); 51913f157a2fSChris Mason 5192323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 5193323ac95bSChris Mason if (level == path->lowest_level) { 5194e02119d5SChris Mason if (slot >= nritems) 5195e02119d5SChris Mason goto find_next_key; 51963f157a2fSChris Mason ret = 0; 51973f157a2fSChris Mason path->slots[level] = slot; 51983f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 51993f157a2fSChris Mason goto out; 52003f157a2fSChris Mason } 52019652480bSYan if (sret && slot > 0) 52029652480bSYan slot--; 52033f157a2fSChris Mason /* 5204de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 5205de78b51aSEric Sandeen * If it is too old, old, skip to the next one. 52063f157a2fSChris Mason */ 52073f157a2fSChris Mason while (slot < nritems) { 52083f157a2fSChris Mason u64 gen; 5209e02119d5SChris Mason 52103f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 52113f157a2fSChris Mason if (gen < min_trans) { 52123f157a2fSChris Mason slot++; 52133f157a2fSChris Mason continue; 52143f157a2fSChris Mason } 52153f157a2fSChris Mason break; 52163f157a2fSChris Mason } 5217e02119d5SChris Mason find_next_key: 52183f157a2fSChris Mason /* 52193f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 52203f157a2fSChris Mason * and find another one 52213f157a2fSChris Mason */ 52223f157a2fSChris Mason if (slot >= nritems) { 5223e02119d5SChris Mason path->slots[level] = slot; 5224b4ce94deSChris Mason btrfs_set_path_blocking(path); 5225e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 5226de78b51aSEric Sandeen min_trans); 5227e02119d5SChris Mason if (sret == 0) { 5228b3b4aa74SDavid Sterba btrfs_release_path(path); 52293f157a2fSChris Mason goto again; 52303f157a2fSChris Mason } else { 52313f157a2fSChris Mason goto out; 52323f157a2fSChris Mason } 52333f157a2fSChris Mason } 52343f157a2fSChris Mason /* save our key for returning back */ 52353f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 52363f157a2fSChris Mason path->slots[level] = slot; 52373f157a2fSChris Mason if (level == path->lowest_level) { 52383f157a2fSChris Mason ret = 0; 52393f157a2fSChris Mason goto out; 52403f157a2fSChris Mason } 5241b4ce94deSChris Mason btrfs_set_path_blocking(path); 52422ff7e61eSJeff Mahoney cur = read_node_slot(fs_info, cur, slot); 5243fb770ae4SLiu Bo if (IS_ERR(cur)) { 5244fb770ae4SLiu Bo ret = PTR_ERR(cur); 5245fb770ae4SLiu Bo goto out; 5246fb770ae4SLiu Bo } 52473f157a2fSChris Mason 5248bd681513SChris Mason btrfs_tree_read_lock(cur); 5249b4ce94deSChris Mason 5250bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 52513f157a2fSChris Mason path->nodes[level - 1] = cur; 5252f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 5253bd681513SChris Mason btrfs_clear_path_blocking(path, NULL, 0); 52543f157a2fSChris Mason } 52553f157a2fSChris Mason out: 5256f98de9b9SFilipe Manana path->keep_locks = keep_locks; 5257f98de9b9SFilipe Manana if (ret == 0) { 5258f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 5259b4ce94deSChris Mason btrfs_set_path_blocking(path); 5260f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 5261f98de9b9SFilipe Manana } 52623f157a2fSChris Mason return ret; 52633f157a2fSChris Mason } 52643f157a2fSChris Mason 52652ff7e61eSJeff Mahoney static int tree_move_down(struct btrfs_fs_info *fs_info, 52667069830aSAlexander Block struct btrfs_path *path, 5267ab6a43e1SDavid Sterba int *level) 52687069830aSAlexander Block { 5269fb770ae4SLiu Bo struct extent_buffer *eb; 5270fb770ae4SLiu Bo 527174dd17fbSChris Mason BUG_ON(*level == 0); 52722ff7e61eSJeff Mahoney eb = read_node_slot(fs_info, path->nodes[*level], path->slots[*level]); 5273fb770ae4SLiu Bo if (IS_ERR(eb)) 5274fb770ae4SLiu Bo return PTR_ERR(eb); 5275fb770ae4SLiu Bo 5276fb770ae4SLiu Bo path->nodes[*level - 1] = eb; 52777069830aSAlexander Block path->slots[*level - 1] = 0; 52787069830aSAlexander Block (*level)--; 5279fb770ae4SLiu Bo return 0; 52807069830aSAlexander Block } 52817069830aSAlexander Block 5282f1e30261SDavid Sterba static int tree_move_next_or_upnext(struct btrfs_path *path, 52837069830aSAlexander Block int *level, int root_level) 52847069830aSAlexander Block { 52857069830aSAlexander Block int ret = 0; 52867069830aSAlexander Block int nritems; 52877069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 52887069830aSAlexander Block 52897069830aSAlexander Block path->slots[*level]++; 52907069830aSAlexander Block 529174dd17fbSChris Mason while (path->slots[*level] >= nritems) { 52927069830aSAlexander Block if (*level == root_level) 52937069830aSAlexander Block return -1; 52947069830aSAlexander Block 52957069830aSAlexander Block /* move upnext */ 52967069830aSAlexander Block path->slots[*level] = 0; 52977069830aSAlexander Block free_extent_buffer(path->nodes[*level]); 52987069830aSAlexander Block path->nodes[*level] = NULL; 52997069830aSAlexander Block (*level)++; 53007069830aSAlexander Block path->slots[*level]++; 53017069830aSAlexander Block 53027069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 53037069830aSAlexander Block ret = 1; 53047069830aSAlexander Block } 53057069830aSAlexander Block return ret; 53067069830aSAlexander Block } 53077069830aSAlexander Block 53087069830aSAlexander Block /* 53097069830aSAlexander Block * Returns 1 if it had to move up and next. 0 is returned if it moved only next 53107069830aSAlexander Block * or down. 53117069830aSAlexander Block */ 53122ff7e61eSJeff Mahoney static int tree_advance(struct btrfs_fs_info *fs_info, 53137069830aSAlexander Block struct btrfs_path *path, 53147069830aSAlexander Block int *level, int root_level, 53157069830aSAlexander Block int allow_down, 53167069830aSAlexander Block struct btrfs_key *key) 53177069830aSAlexander Block { 53187069830aSAlexander Block int ret; 53197069830aSAlexander Block 53207069830aSAlexander Block if (*level == 0 || !allow_down) { 5321f1e30261SDavid Sterba ret = tree_move_next_or_upnext(path, level, root_level); 53227069830aSAlexander Block } else { 5323ab6a43e1SDavid Sterba ret = tree_move_down(fs_info, path, level); 53247069830aSAlexander Block } 53257069830aSAlexander Block if (ret >= 0) { 53267069830aSAlexander Block if (*level == 0) 53277069830aSAlexander Block btrfs_item_key_to_cpu(path->nodes[*level], key, 53287069830aSAlexander Block path->slots[*level]); 53297069830aSAlexander Block else 53307069830aSAlexander Block btrfs_node_key_to_cpu(path->nodes[*level], key, 53317069830aSAlexander Block path->slots[*level]); 53327069830aSAlexander Block } 53337069830aSAlexander Block return ret; 53347069830aSAlexander Block } 53357069830aSAlexander Block 53362ff7e61eSJeff Mahoney static int tree_compare_item(struct btrfs_path *left_path, 53377069830aSAlexander Block struct btrfs_path *right_path, 53387069830aSAlexander Block char *tmp_buf) 53397069830aSAlexander Block { 53407069830aSAlexander Block int cmp; 53417069830aSAlexander Block int len1, len2; 53427069830aSAlexander Block unsigned long off1, off2; 53437069830aSAlexander Block 53447069830aSAlexander Block len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]); 53457069830aSAlexander Block len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]); 53467069830aSAlexander Block if (len1 != len2) 53477069830aSAlexander Block return 1; 53487069830aSAlexander Block 53497069830aSAlexander Block off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]); 53507069830aSAlexander Block off2 = btrfs_item_ptr_offset(right_path->nodes[0], 53517069830aSAlexander Block right_path->slots[0]); 53527069830aSAlexander Block 53537069830aSAlexander Block read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1); 53547069830aSAlexander Block 53557069830aSAlexander Block cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1); 53567069830aSAlexander Block if (cmp) 53577069830aSAlexander Block return 1; 53587069830aSAlexander Block return 0; 53597069830aSAlexander Block } 53607069830aSAlexander Block 53617069830aSAlexander Block #define ADVANCE 1 53627069830aSAlexander Block #define ADVANCE_ONLY_NEXT -1 53637069830aSAlexander Block 53647069830aSAlexander Block /* 53657069830aSAlexander Block * This function compares two trees and calls the provided callback for 53667069830aSAlexander Block * every changed/new/deleted item it finds. 53677069830aSAlexander Block * If shared tree blocks are encountered, whole subtrees are skipped, making 53687069830aSAlexander Block * the compare pretty fast on snapshotted subvolumes. 53697069830aSAlexander Block * 53707069830aSAlexander Block * This currently works on commit roots only. As commit roots are read only, 53717069830aSAlexander Block * we don't do any locking. The commit roots are protected with transactions. 53727069830aSAlexander Block * Transactions are ended and rejoined when a commit is tried in between. 53737069830aSAlexander Block * 53747069830aSAlexander Block * This function checks for modifications done to the trees while comparing. 53757069830aSAlexander Block * If it detects a change, it aborts immediately. 53767069830aSAlexander Block */ 53777069830aSAlexander Block int btrfs_compare_trees(struct btrfs_root *left_root, 53787069830aSAlexander Block struct btrfs_root *right_root, 53797069830aSAlexander Block btrfs_changed_cb_t changed_cb, void *ctx) 53807069830aSAlexander Block { 53810b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = left_root->fs_info; 53827069830aSAlexander Block int ret; 53837069830aSAlexander Block int cmp; 53847069830aSAlexander Block struct btrfs_path *left_path = NULL; 53857069830aSAlexander Block struct btrfs_path *right_path = NULL; 53867069830aSAlexander Block struct btrfs_key left_key; 53877069830aSAlexander Block struct btrfs_key right_key; 53887069830aSAlexander Block char *tmp_buf = NULL; 53897069830aSAlexander Block int left_root_level; 53907069830aSAlexander Block int right_root_level; 53917069830aSAlexander Block int left_level; 53927069830aSAlexander Block int right_level; 53937069830aSAlexander Block int left_end_reached; 53947069830aSAlexander Block int right_end_reached; 53957069830aSAlexander Block int advance_left; 53967069830aSAlexander Block int advance_right; 53977069830aSAlexander Block u64 left_blockptr; 53987069830aSAlexander Block u64 right_blockptr; 53996baa4293SFilipe Manana u64 left_gen; 54006baa4293SFilipe Manana u64 right_gen; 54017069830aSAlexander Block 54027069830aSAlexander Block left_path = btrfs_alloc_path(); 54037069830aSAlexander Block if (!left_path) { 54047069830aSAlexander Block ret = -ENOMEM; 54057069830aSAlexander Block goto out; 54067069830aSAlexander Block } 54077069830aSAlexander Block right_path = btrfs_alloc_path(); 54087069830aSAlexander Block if (!right_path) { 54097069830aSAlexander Block ret = -ENOMEM; 54107069830aSAlexander Block goto out; 54117069830aSAlexander Block } 54127069830aSAlexander Block 5413752ade68SMichal Hocko tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL); 54147069830aSAlexander Block if (!tmp_buf) { 54157069830aSAlexander Block ret = -ENOMEM; 54167069830aSAlexander Block goto out; 54177069830aSAlexander Block } 54187069830aSAlexander Block 54197069830aSAlexander Block left_path->search_commit_root = 1; 54207069830aSAlexander Block left_path->skip_locking = 1; 54217069830aSAlexander Block right_path->search_commit_root = 1; 54227069830aSAlexander Block right_path->skip_locking = 1; 54237069830aSAlexander Block 54247069830aSAlexander Block /* 54257069830aSAlexander Block * Strategy: Go to the first items of both trees. Then do 54267069830aSAlexander Block * 54277069830aSAlexander Block * If both trees are at level 0 54287069830aSAlexander Block * Compare keys of current items 54297069830aSAlexander Block * If left < right treat left item as new, advance left tree 54307069830aSAlexander Block * and repeat 54317069830aSAlexander Block * If left > right treat right item as deleted, advance right tree 54327069830aSAlexander Block * and repeat 54337069830aSAlexander Block * If left == right do deep compare of items, treat as changed if 54347069830aSAlexander Block * needed, advance both trees and repeat 54357069830aSAlexander Block * If both trees are at the same level but not at level 0 54367069830aSAlexander Block * Compare keys of current nodes/leafs 54377069830aSAlexander Block * If left < right advance left tree and repeat 54387069830aSAlexander Block * If left > right advance right tree and repeat 54397069830aSAlexander Block * If left == right compare blockptrs of the next nodes/leafs 54407069830aSAlexander Block * If they match advance both trees but stay at the same level 54417069830aSAlexander Block * and repeat 54427069830aSAlexander Block * If they don't match advance both trees while allowing to go 54437069830aSAlexander Block * deeper and repeat 54447069830aSAlexander Block * If tree levels are different 54457069830aSAlexander Block * Advance the tree that needs it and repeat 54467069830aSAlexander Block * 54477069830aSAlexander Block * Advancing a tree means: 54487069830aSAlexander Block * If we are at level 0, try to go to the next slot. If that's not 54497069830aSAlexander Block * possible, go one level up and repeat. Stop when we found a level 54507069830aSAlexander Block * where we could go to the next slot. We may at this point be on a 54517069830aSAlexander Block * node or a leaf. 54527069830aSAlexander Block * 54537069830aSAlexander Block * If we are not at level 0 and not on shared tree blocks, go one 54547069830aSAlexander Block * level deeper. 54557069830aSAlexander Block * 54567069830aSAlexander Block * If we are not at level 0 and on shared tree blocks, go one slot to 54577069830aSAlexander Block * the right if possible or go up and right. 54587069830aSAlexander Block */ 54597069830aSAlexander Block 54600b246afaSJeff Mahoney down_read(&fs_info->commit_root_sem); 54617069830aSAlexander Block left_level = btrfs_header_level(left_root->commit_root); 54627069830aSAlexander Block left_root_level = left_level; 54637069830aSAlexander Block left_path->nodes[left_level] = left_root->commit_root; 54647069830aSAlexander Block extent_buffer_get(left_path->nodes[left_level]); 54657069830aSAlexander Block 54667069830aSAlexander Block right_level = btrfs_header_level(right_root->commit_root); 54677069830aSAlexander Block right_root_level = right_level; 54687069830aSAlexander Block right_path->nodes[right_level] = right_root->commit_root; 54697069830aSAlexander Block extent_buffer_get(right_path->nodes[right_level]); 54700b246afaSJeff Mahoney up_read(&fs_info->commit_root_sem); 54717069830aSAlexander Block 54727069830aSAlexander Block if (left_level == 0) 54737069830aSAlexander Block btrfs_item_key_to_cpu(left_path->nodes[left_level], 54747069830aSAlexander Block &left_key, left_path->slots[left_level]); 54757069830aSAlexander Block else 54767069830aSAlexander Block btrfs_node_key_to_cpu(left_path->nodes[left_level], 54777069830aSAlexander Block &left_key, left_path->slots[left_level]); 54787069830aSAlexander Block if (right_level == 0) 54797069830aSAlexander Block btrfs_item_key_to_cpu(right_path->nodes[right_level], 54807069830aSAlexander Block &right_key, right_path->slots[right_level]); 54817069830aSAlexander Block else 54827069830aSAlexander Block btrfs_node_key_to_cpu(right_path->nodes[right_level], 54837069830aSAlexander Block &right_key, right_path->slots[right_level]); 54847069830aSAlexander Block 54857069830aSAlexander Block left_end_reached = right_end_reached = 0; 54867069830aSAlexander Block advance_left = advance_right = 0; 54877069830aSAlexander Block 54887069830aSAlexander Block while (1) { 54897069830aSAlexander Block if (advance_left && !left_end_reached) { 54902ff7e61eSJeff Mahoney ret = tree_advance(fs_info, left_path, &left_level, 54917069830aSAlexander Block left_root_level, 54927069830aSAlexander Block advance_left != ADVANCE_ONLY_NEXT, 54937069830aSAlexander Block &left_key); 5494fb770ae4SLiu Bo if (ret == -1) 54957069830aSAlexander Block left_end_reached = ADVANCE; 5496fb770ae4SLiu Bo else if (ret < 0) 5497fb770ae4SLiu Bo goto out; 54987069830aSAlexander Block advance_left = 0; 54997069830aSAlexander Block } 55007069830aSAlexander Block if (advance_right && !right_end_reached) { 55012ff7e61eSJeff Mahoney ret = tree_advance(fs_info, right_path, &right_level, 55027069830aSAlexander Block right_root_level, 55037069830aSAlexander Block advance_right != ADVANCE_ONLY_NEXT, 55047069830aSAlexander Block &right_key); 5505fb770ae4SLiu Bo if (ret == -1) 55067069830aSAlexander Block right_end_reached = ADVANCE; 5507fb770ae4SLiu Bo else if (ret < 0) 5508fb770ae4SLiu Bo goto out; 55097069830aSAlexander Block advance_right = 0; 55107069830aSAlexander Block } 55117069830aSAlexander Block 55127069830aSAlexander Block if (left_end_reached && right_end_reached) { 55137069830aSAlexander Block ret = 0; 55147069830aSAlexander Block goto out; 55157069830aSAlexander Block } else if (left_end_reached) { 55167069830aSAlexander Block if (right_level == 0) { 5517ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 55187069830aSAlexander Block &right_key, 55197069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 55207069830aSAlexander Block ctx); 55217069830aSAlexander Block if (ret < 0) 55227069830aSAlexander Block goto out; 55237069830aSAlexander Block } 55247069830aSAlexander Block advance_right = ADVANCE; 55257069830aSAlexander Block continue; 55267069830aSAlexander Block } else if (right_end_reached) { 55277069830aSAlexander Block if (left_level == 0) { 5528ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 55297069830aSAlexander Block &left_key, 55307069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 55317069830aSAlexander Block ctx); 55327069830aSAlexander Block if (ret < 0) 55337069830aSAlexander Block goto out; 55347069830aSAlexander Block } 55357069830aSAlexander Block advance_left = ADVANCE; 55367069830aSAlexander Block continue; 55377069830aSAlexander Block } 55387069830aSAlexander Block 55397069830aSAlexander Block if (left_level == 0 && right_level == 0) { 55407069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 55417069830aSAlexander Block if (cmp < 0) { 5542ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 55437069830aSAlexander Block &left_key, 55447069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 55457069830aSAlexander Block ctx); 55467069830aSAlexander Block if (ret < 0) 55477069830aSAlexander Block goto out; 55487069830aSAlexander Block advance_left = ADVANCE; 55497069830aSAlexander Block } else if (cmp > 0) { 5550ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 55517069830aSAlexander Block &right_key, 55527069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 55537069830aSAlexander Block ctx); 55547069830aSAlexander Block if (ret < 0) 55557069830aSAlexander Block goto out; 55567069830aSAlexander Block advance_right = ADVANCE; 55577069830aSAlexander Block } else { 5558b99d9a6aSFabian Frederick enum btrfs_compare_tree_result result; 5559ba5e8f2eSJosef Bacik 556074dd17fbSChris Mason WARN_ON(!extent_buffer_uptodate(left_path->nodes[0])); 55612ff7e61eSJeff Mahoney ret = tree_compare_item(left_path, right_path, 55622ff7e61eSJeff Mahoney tmp_buf); 5563ba5e8f2eSJosef Bacik if (ret) 5564b99d9a6aSFabian Frederick result = BTRFS_COMPARE_TREE_CHANGED; 5565ba5e8f2eSJosef Bacik else 5566b99d9a6aSFabian Frederick result = BTRFS_COMPARE_TREE_SAME; 5567ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 5568b99d9a6aSFabian Frederick &left_key, result, ctx); 55697069830aSAlexander Block if (ret < 0) 55707069830aSAlexander Block goto out; 55717069830aSAlexander Block advance_left = ADVANCE; 55727069830aSAlexander Block advance_right = ADVANCE; 55737069830aSAlexander Block } 55747069830aSAlexander Block } else if (left_level == right_level) { 55757069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 55767069830aSAlexander Block if (cmp < 0) { 55777069830aSAlexander Block advance_left = ADVANCE; 55787069830aSAlexander Block } else if (cmp > 0) { 55797069830aSAlexander Block advance_right = ADVANCE; 55807069830aSAlexander Block } else { 55817069830aSAlexander Block left_blockptr = btrfs_node_blockptr( 55827069830aSAlexander Block left_path->nodes[left_level], 55837069830aSAlexander Block left_path->slots[left_level]); 55847069830aSAlexander Block right_blockptr = btrfs_node_blockptr( 55857069830aSAlexander Block right_path->nodes[right_level], 55867069830aSAlexander Block right_path->slots[right_level]); 55876baa4293SFilipe Manana left_gen = btrfs_node_ptr_generation( 55886baa4293SFilipe Manana left_path->nodes[left_level], 55896baa4293SFilipe Manana left_path->slots[left_level]); 55906baa4293SFilipe Manana right_gen = btrfs_node_ptr_generation( 55916baa4293SFilipe Manana right_path->nodes[right_level], 55926baa4293SFilipe Manana right_path->slots[right_level]); 55936baa4293SFilipe Manana if (left_blockptr == right_blockptr && 55946baa4293SFilipe Manana left_gen == right_gen) { 55957069830aSAlexander Block /* 55967069830aSAlexander Block * As we're on a shared block, don't 55977069830aSAlexander Block * allow to go deeper. 55987069830aSAlexander Block */ 55997069830aSAlexander Block advance_left = ADVANCE_ONLY_NEXT; 56007069830aSAlexander Block advance_right = ADVANCE_ONLY_NEXT; 56017069830aSAlexander Block } else { 56027069830aSAlexander Block advance_left = ADVANCE; 56037069830aSAlexander Block advance_right = ADVANCE; 56047069830aSAlexander Block } 56057069830aSAlexander Block } 56067069830aSAlexander Block } else if (left_level < right_level) { 56077069830aSAlexander Block advance_right = ADVANCE; 56087069830aSAlexander Block } else { 56097069830aSAlexander Block advance_left = ADVANCE; 56107069830aSAlexander Block } 56117069830aSAlexander Block } 56127069830aSAlexander Block 56137069830aSAlexander Block out: 56147069830aSAlexander Block btrfs_free_path(left_path); 56157069830aSAlexander Block btrfs_free_path(right_path); 56168f282f71SDavid Sterba kvfree(tmp_buf); 56177069830aSAlexander Block return ret; 56187069830aSAlexander Block } 56197069830aSAlexander Block 56203f157a2fSChris Mason /* 56213f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 56223f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 5623de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 56243f157a2fSChris Mason * 56253f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 56263f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 56273f157a2fSChris Mason * 56283f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 56293f157a2fSChris Mason * calling this function. 56303f157a2fSChris Mason */ 5631e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 5632de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 5633e7a84565SChris Mason { 5634e7a84565SChris Mason int slot; 5635e7a84565SChris Mason struct extent_buffer *c; 5636e7a84565SChris Mason 5637934d375bSChris Mason WARN_ON(!path->keep_locks); 5638e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 5639e7a84565SChris Mason if (!path->nodes[level]) 5640e7a84565SChris Mason return 1; 5641e7a84565SChris Mason 5642e7a84565SChris Mason slot = path->slots[level] + 1; 5643e7a84565SChris Mason c = path->nodes[level]; 56443f157a2fSChris Mason next: 5645e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 564633c66f43SYan Zheng int ret; 564733c66f43SYan Zheng int orig_lowest; 564833c66f43SYan Zheng struct btrfs_key cur_key; 564933c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 565033c66f43SYan Zheng !path->nodes[level + 1]) 5651e7a84565SChris Mason return 1; 565233c66f43SYan Zheng 565333c66f43SYan Zheng if (path->locks[level + 1]) { 565433c66f43SYan Zheng level++; 5655e7a84565SChris Mason continue; 5656e7a84565SChris Mason } 565733c66f43SYan Zheng 565833c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 565933c66f43SYan Zheng if (level == 0) 566033c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 566133c66f43SYan Zheng else 566233c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 566333c66f43SYan Zheng 566433c66f43SYan Zheng orig_lowest = path->lowest_level; 5665b3b4aa74SDavid Sterba btrfs_release_path(path); 566633c66f43SYan Zheng path->lowest_level = level; 566733c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 566833c66f43SYan Zheng 0, 0); 566933c66f43SYan Zheng path->lowest_level = orig_lowest; 567033c66f43SYan Zheng if (ret < 0) 567133c66f43SYan Zheng return ret; 567233c66f43SYan Zheng 567333c66f43SYan Zheng c = path->nodes[level]; 567433c66f43SYan Zheng slot = path->slots[level]; 567533c66f43SYan Zheng if (ret == 0) 567633c66f43SYan Zheng slot++; 567733c66f43SYan Zheng goto next; 567833c66f43SYan Zheng } 567933c66f43SYan Zheng 5680e7a84565SChris Mason if (level == 0) 5681e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 56823f157a2fSChris Mason else { 56833f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 56843f157a2fSChris Mason 56853f157a2fSChris Mason if (gen < min_trans) { 56863f157a2fSChris Mason slot++; 56873f157a2fSChris Mason goto next; 56883f157a2fSChris Mason } 5689e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 56903f157a2fSChris Mason } 5691e7a84565SChris Mason return 0; 5692e7a84565SChris Mason } 5693e7a84565SChris Mason return 1; 5694e7a84565SChris Mason } 5695e7a84565SChris Mason 56967bb86316SChris Mason /* 5697925baeddSChris Mason * search the tree again to find a leaf with greater keys 56980f70abe2SChris Mason * returns 0 if it found something or 1 if there are no greater leaves. 56990f70abe2SChris Mason * returns < 0 on io errors. 570097571fd0SChris Mason */ 5701234b63a0SChris Mason int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path) 5702d97e63b6SChris Mason { 57033d7806ecSJan Schmidt return btrfs_next_old_leaf(root, path, 0); 57043d7806ecSJan Schmidt } 57053d7806ecSJan Schmidt 57063d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 57073d7806ecSJan Schmidt u64 time_seq) 57083d7806ecSJan Schmidt { 5709d97e63b6SChris Mason int slot; 57108e73f275SChris Mason int level; 57115f39d397SChris Mason struct extent_buffer *c; 57128e73f275SChris Mason struct extent_buffer *next; 5713925baeddSChris Mason struct btrfs_key key; 5714925baeddSChris Mason u32 nritems; 5715925baeddSChris Mason int ret; 57168e73f275SChris Mason int old_spinning = path->leave_spinning; 5717bd681513SChris Mason int next_rw_lock = 0; 5718925baeddSChris Mason 5719925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5720d397712bSChris Mason if (nritems == 0) 5721925baeddSChris Mason return 1; 5722925baeddSChris Mason 57238e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 57248e73f275SChris Mason again: 57258e73f275SChris Mason level = 1; 57268e73f275SChris Mason next = NULL; 5727bd681513SChris Mason next_rw_lock = 0; 5728b3b4aa74SDavid Sterba btrfs_release_path(path); 57298e73f275SChris Mason 5730a2135011SChris Mason path->keep_locks = 1; 57318e73f275SChris Mason path->leave_spinning = 1; 57328e73f275SChris Mason 57333d7806ecSJan Schmidt if (time_seq) 57343d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 57353d7806ecSJan Schmidt else 5736925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5737925baeddSChris Mason path->keep_locks = 0; 5738925baeddSChris Mason 5739925baeddSChris Mason if (ret < 0) 5740925baeddSChris Mason return ret; 5741925baeddSChris Mason 5742a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5743168fd7d2SChris Mason /* 5744168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 5745168fd7d2SChris Mason * could have added more items next to the key that used to be 5746168fd7d2SChris Mason * at the very end of the block. So, check again here and 5747168fd7d2SChris Mason * advance the path if there are now more items available. 5748168fd7d2SChris Mason */ 5749a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 5750e457afecSYan Zheng if (ret == 0) 5751168fd7d2SChris Mason path->slots[0]++; 57528e73f275SChris Mason ret = 0; 5753925baeddSChris Mason goto done; 5754925baeddSChris Mason } 57550b43e04fSLiu Bo /* 57560b43e04fSLiu Bo * So the above check misses one case: 57570b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 57580b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 57590b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 57600b43e04fSLiu Bo * 57610b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 57620b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 57630b43e04fSLiu Bo * 57640b43e04fSLiu Bo * And a bit more explanation about this check, 57650b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 57660b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 57670b43e04fSLiu Bo * bigger one. 57680b43e04fSLiu Bo */ 57690b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 57700b43e04fSLiu Bo ret = 0; 57710b43e04fSLiu Bo goto done; 57720b43e04fSLiu Bo } 5773d97e63b6SChris Mason 5774234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 57758e73f275SChris Mason if (!path->nodes[level]) { 57768e73f275SChris Mason ret = 1; 57778e73f275SChris Mason goto done; 57788e73f275SChris Mason } 57795f39d397SChris Mason 5780d97e63b6SChris Mason slot = path->slots[level] + 1; 5781d97e63b6SChris Mason c = path->nodes[level]; 57825f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 5783d97e63b6SChris Mason level++; 57848e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 57858e73f275SChris Mason ret = 1; 57868e73f275SChris Mason goto done; 57878e73f275SChris Mason } 5788d97e63b6SChris Mason continue; 5789d97e63b6SChris Mason } 57905f39d397SChris Mason 5791925baeddSChris Mason if (next) { 5792bd681513SChris Mason btrfs_tree_unlock_rw(next, next_rw_lock); 57935f39d397SChris Mason free_extent_buffer(next); 5794925baeddSChris Mason } 57955f39d397SChris Mason 57968e73f275SChris Mason next = c; 5797bd681513SChris Mason next_rw_lock = path->locks[level]; 5798d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5799cda79c54SDavid Sterba slot, &key); 58008e73f275SChris Mason if (ret == -EAGAIN) 58018e73f275SChris Mason goto again; 58025f39d397SChris Mason 580376a05b35SChris Mason if (ret < 0) { 5804b3b4aa74SDavid Sterba btrfs_release_path(path); 580576a05b35SChris Mason goto done; 580676a05b35SChris Mason } 580776a05b35SChris Mason 58085cd57b2cSChris Mason if (!path->skip_locking) { 5809bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 5810d42244a0SJan Schmidt if (!ret && time_seq) { 5811d42244a0SJan Schmidt /* 5812d42244a0SJan Schmidt * If we don't get the lock, we may be racing 5813d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 5814d42244a0SJan Schmidt * itself waiting for the leaf we've currently 5815d42244a0SJan Schmidt * locked. To solve this situation, we give up 5816d42244a0SJan Schmidt * on our lock and cycle. 5817d42244a0SJan Schmidt */ 5818cf538830SJan Schmidt free_extent_buffer(next); 5819d42244a0SJan Schmidt btrfs_release_path(path); 5820d42244a0SJan Schmidt cond_resched(); 5821d42244a0SJan Schmidt goto again; 5822d42244a0SJan Schmidt } 58238e73f275SChris Mason if (!ret) { 58248e73f275SChris Mason btrfs_set_path_blocking(path); 5825bd681513SChris Mason btrfs_tree_read_lock(next); 5826bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5827bd681513SChris Mason BTRFS_READ_LOCK); 58288e73f275SChris Mason } 5829bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5830bd681513SChris Mason } 5831d97e63b6SChris Mason break; 5832d97e63b6SChris Mason } 5833d97e63b6SChris Mason path->slots[level] = slot; 5834d97e63b6SChris Mason while (1) { 5835d97e63b6SChris Mason level--; 5836d97e63b6SChris Mason c = path->nodes[level]; 5837925baeddSChris Mason if (path->locks[level]) 5838bd681513SChris Mason btrfs_tree_unlock_rw(c, path->locks[level]); 58398e73f275SChris Mason 58405f39d397SChris Mason free_extent_buffer(c); 5841d97e63b6SChris Mason path->nodes[level] = next; 5842d97e63b6SChris Mason path->slots[level] = 0; 5843a74a4b97SChris Mason if (!path->skip_locking) 5844bd681513SChris Mason path->locks[level] = next_rw_lock; 5845d97e63b6SChris Mason if (!level) 5846d97e63b6SChris Mason break; 5847b4ce94deSChris Mason 5848d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5849cda79c54SDavid Sterba 0, &key); 58508e73f275SChris Mason if (ret == -EAGAIN) 58518e73f275SChris Mason goto again; 58528e73f275SChris Mason 585376a05b35SChris Mason if (ret < 0) { 5854b3b4aa74SDavid Sterba btrfs_release_path(path); 585576a05b35SChris Mason goto done; 585676a05b35SChris Mason } 585776a05b35SChris Mason 58585cd57b2cSChris Mason if (!path->skip_locking) { 5859bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 58608e73f275SChris Mason if (!ret) { 58618e73f275SChris Mason btrfs_set_path_blocking(path); 5862bd681513SChris Mason btrfs_tree_read_lock(next); 5863bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5864bd681513SChris Mason BTRFS_READ_LOCK); 58658e73f275SChris Mason } 5866bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5867bd681513SChris Mason } 5868d97e63b6SChris Mason } 58698e73f275SChris Mason ret = 0; 5870925baeddSChris Mason done: 5871f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 58728e73f275SChris Mason path->leave_spinning = old_spinning; 58738e73f275SChris Mason if (!old_spinning) 58748e73f275SChris Mason btrfs_set_path_blocking(path); 58758e73f275SChris Mason 58768e73f275SChris Mason return ret; 5877d97e63b6SChris Mason } 58780b86a832SChris Mason 58793f157a2fSChris Mason /* 58803f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 58813f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 58823f157a2fSChris Mason * 58833f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 58843f157a2fSChris Mason */ 58850b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 58860b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 58870b86a832SChris Mason int type) 58880b86a832SChris Mason { 58890b86a832SChris Mason struct btrfs_key found_key; 58900b86a832SChris Mason struct extent_buffer *leaf; 5891e02119d5SChris Mason u32 nritems; 58920b86a832SChris Mason int ret; 58930b86a832SChris Mason 58940b86a832SChris Mason while (1) { 58950b86a832SChris Mason if (path->slots[0] == 0) { 5896b4ce94deSChris Mason btrfs_set_path_blocking(path); 58970b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 58980b86a832SChris Mason if (ret != 0) 58990b86a832SChris Mason return ret; 59000b86a832SChris Mason } else { 59010b86a832SChris Mason path->slots[0]--; 59020b86a832SChris Mason } 59030b86a832SChris Mason leaf = path->nodes[0]; 5904e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5905e02119d5SChris Mason if (nritems == 0) 5906e02119d5SChris Mason return 1; 5907e02119d5SChris Mason if (path->slots[0] == nritems) 5908e02119d5SChris Mason path->slots[0]--; 5909e02119d5SChris Mason 59100b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5911e02119d5SChris Mason if (found_key.objectid < min_objectid) 5912e02119d5SChris Mason break; 59130a4eefbbSYan Zheng if (found_key.type == type) 59140a4eefbbSYan Zheng return 0; 5915e02119d5SChris Mason if (found_key.objectid == min_objectid && 5916e02119d5SChris Mason found_key.type < type) 5917e02119d5SChris Mason break; 59180b86a832SChris Mason } 59190b86a832SChris Mason return 1; 59200b86a832SChris Mason } 5921ade2e0b3SWang Shilong 5922ade2e0b3SWang Shilong /* 5923ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 5924ade2e0b3SWang Shilong * min objecitd. 5925ade2e0b3SWang Shilong * 5926ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 5927ade2e0b3SWang Shilong */ 5928ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 5929ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 5930ade2e0b3SWang Shilong { 5931ade2e0b3SWang Shilong struct btrfs_key found_key; 5932ade2e0b3SWang Shilong struct extent_buffer *leaf; 5933ade2e0b3SWang Shilong u32 nritems; 5934ade2e0b3SWang Shilong int ret; 5935ade2e0b3SWang Shilong 5936ade2e0b3SWang Shilong while (1) { 5937ade2e0b3SWang Shilong if (path->slots[0] == 0) { 5938ade2e0b3SWang Shilong btrfs_set_path_blocking(path); 5939ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 5940ade2e0b3SWang Shilong if (ret != 0) 5941ade2e0b3SWang Shilong return ret; 5942ade2e0b3SWang Shilong } else { 5943ade2e0b3SWang Shilong path->slots[0]--; 5944ade2e0b3SWang Shilong } 5945ade2e0b3SWang Shilong leaf = path->nodes[0]; 5946ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5947ade2e0b3SWang Shilong if (nritems == 0) 5948ade2e0b3SWang Shilong return 1; 5949ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5950ade2e0b3SWang Shilong path->slots[0]--; 5951ade2e0b3SWang Shilong 5952ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5953ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5954ade2e0b3SWang Shilong break; 5955ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5956ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5957ade2e0b3SWang Shilong return 0; 5958ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5959ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5960ade2e0b3SWang Shilong break; 5961ade2e0b3SWang Shilong } 5962ade2e0b3SWang Shilong return 1; 5963ade2e0b3SWang Shilong } 5964