16cbd5570SChris Mason /* 2d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 36cbd5570SChris Mason * 46cbd5570SChris Mason * This program is free software; you can redistribute it and/or 56cbd5570SChris Mason * modify it under the terms of the GNU General Public 66cbd5570SChris Mason * License v2 as published by the Free Software Foundation. 76cbd5570SChris Mason * 86cbd5570SChris Mason * This program is distributed in the hope that it will be useful, 96cbd5570SChris Mason * but WITHOUT ANY WARRANTY; without even the implied warranty of 106cbd5570SChris Mason * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 116cbd5570SChris Mason * General Public License for more details. 126cbd5570SChris Mason * 136cbd5570SChris Mason * You should have received a copy of the GNU General Public 146cbd5570SChris Mason * License along with this program; if not, write to the 156cbd5570SChris Mason * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 166cbd5570SChris Mason * Boston, MA 021110-1307, USA. 176cbd5570SChris Mason */ 186cbd5570SChris Mason 19a6b6e75eSChris Mason #include <linux/sched.h> 205a0e3ad6STejun Heo #include <linux/slab.h> 21bd989ba3SJan Schmidt #include <linux/rbtree.h> 22eb60ceacSChris Mason #include "ctree.h" 23eb60ceacSChris Mason #include "disk-io.h" 247f5c1516SChris Mason #include "transaction.h" 255f39d397SChris Mason #include "print-tree.h" 26925baeddSChris Mason #include "locking.h" 279a8dd150SChris Mason 28e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 29e089f05cSChris Mason *root, struct btrfs_path *path, int level); 30e089f05cSChris Mason static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root 31d4dbff95SChris Mason *root, struct btrfs_key *ins_key, 32cc0c5538SChris Mason struct btrfs_path *path, int data_size, int extend); 335f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 345f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *dst, 35971a1f66SChris Mason struct extent_buffer *src, int empty); 365f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 375f39d397SChris Mason struct btrfs_root *root, 385f39d397SChris Mason struct extent_buffer *dst_buf, 395f39d397SChris Mason struct extent_buffer *src_buf); 40143bede5SJeff Mahoney static void del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root, 41f3ea38daSJan Schmidt struct btrfs_path *path, int level, int slot, 42f3ea38daSJan Schmidt int tree_mod_log); 43f230475eSJan Schmidt static void tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 44f230475eSJan Schmidt struct extent_buffer *eb); 45f230475eSJan Schmidt struct extent_buffer *read_old_tree_block(struct btrfs_root *root, u64 bytenr, 46f230475eSJan Schmidt u32 blocksize, u64 parent_transid, 47f230475eSJan Schmidt u64 time_seq); 48f230475eSJan Schmidt struct extent_buffer *btrfs_find_old_tree_block(struct btrfs_root *root, 49f230475eSJan Schmidt u64 bytenr, u32 blocksize, 50f230475eSJan Schmidt u64 time_seq); 51d97e63b6SChris Mason 522c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 532c90e5d6SChris Mason { 54df24a2b9SChris Mason struct btrfs_path *path; 55e00f7308SJeff Mahoney path = kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 56df24a2b9SChris Mason return path; 572c90e5d6SChris Mason } 582c90e5d6SChris Mason 59b4ce94deSChris Mason /* 60b4ce94deSChris Mason * set all locked nodes in the path to blocking locks. This should 61b4ce94deSChris Mason * be done before scheduling 62b4ce94deSChris Mason */ 63b4ce94deSChris Mason noinline void btrfs_set_path_blocking(struct btrfs_path *p) 64b4ce94deSChris Mason { 65b4ce94deSChris Mason int i; 66b4ce94deSChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 67bd681513SChris Mason if (!p->nodes[i] || !p->locks[i]) 68bd681513SChris Mason continue; 69bd681513SChris Mason btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]); 70bd681513SChris Mason if (p->locks[i] == BTRFS_READ_LOCK) 71bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK_BLOCKING; 72bd681513SChris Mason else if (p->locks[i] == BTRFS_WRITE_LOCK) 73bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING; 74b4ce94deSChris Mason } 75b4ce94deSChris Mason } 76b4ce94deSChris Mason 77b4ce94deSChris Mason /* 78b4ce94deSChris Mason * reset all the locked nodes in the patch to spinning locks. 794008c04aSChris Mason * 804008c04aSChris Mason * held is used to keep lockdep happy, when lockdep is enabled 814008c04aSChris Mason * we set held to a blocking lock before we go around and 824008c04aSChris Mason * retake all the spinlocks in the path. You can safely use NULL 834008c04aSChris Mason * for held 84b4ce94deSChris Mason */ 854008c04aSChris Mason noinline void btrfs_clear_path_blocking(struct btrfs_path *p, 86bd681513SChris Mason struct extent_buffer *held, int held_rw) 87b4ce94deSChris Mason { 88b4ce94deSChris Mason int i; 894008c04aSChris Mason 904008c04aSChris Mason #ifdef CONFIG_DEBUG_LOCK_ALLOC 914008c04aSChris Mason /* lockdep really cares that we take all of these spinlocks 924008c04aSChris Mason * in the right order. If any of the locks in the path are not 934008c04aSChris Mason * currently blocking, it is going to complain. So, make really 944008c04aSChris Mason * really sure by forcing the path to blocking before we clear 954008c04aSChris Mason * the path blocking. 964008c04aSChris Mason */ 97bd681513SChris Mason if (held) { 98bd681513SChris Mason btrfs_set_lock_blocking_rw(held, held_rw); 99bd681513SChris Mason if (held_rw == BTRFS_WRITE_LOCK) 100bd681513SChris Mason held_rw = BTRFS_WRITE_LOCK_BLOCKING; 101bd681513SChris Mason else if (held_rw == BTRFS_READ_LOCK) 102bd681513SChris Mason held_rw = BTRFS_READ_LOCK_BLOCKING; 103bd681513SChris Mason } 1044008c04aSChris Mason btrfs_set_path_blocking(p); 1054008c04aSChris Mason #endif 1064008c04aSChris Mason 1074008c04aSChris Mason for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) { 108bd681513SChris Mason if (p->nodes[i] && p->locks[i]) { 109bd681513SChris Mason btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]); 110bd681513SChris Mason if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING) 111bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK; 112bd681513SChris Mason else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING) 113bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK; 114bd681513SChris Mason } 115b4ce94deSChris Mason } 1164008c04aSChris Mason 1174008c04aSChris Mason #ifdef CONFIG_DEBUG_LOCK_ALLOC 1184008c04aSChris Mason if (held) 119bd681513SChris Mason btrfs_clear_lock_blocking_rw(held, held_rw); 1204008c04aSChris Mason #endif 121b4ce94deSChris Mason } 122b4ce94deSChris Mason 123d352ac68SChris Mason /* this also releases the path */ 1242c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 1252c90e5d6SChris Mason { 126ff175d57SJesper Juhl if (!p) 127ff175d57SJesper Juhl return; 128b3b4aa74SDavid Sterba btrfs_release_path(p); 1292c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 1302c90e5d6SChris Mason } 1312c90e5d6SChris Mason 132d352ac68SChris Mason /* 133d352ac68SChris Mason * path release drops references on the extent buffers in the path 134d352ac68SChris Mason * and it drops any locks held by this path 135d352ac68SChris Mason * 136d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 137d352ac68SChris Mason */ 138b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 139eb60ceacSChris Mason { 140eb60ceacSChris Mason int i; 141a2135011SChris Mason 142234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1433f157a2fSChris Mason p->slots[i] = 0; 144eb60ceacSChris Mason if (!p->nodes[i]) 145925baeddSChris Mason continue; 146925baeddSChris Mason if (p->locks[i]) { 147bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 148925baeddSChris Mason p->locks[i] = 0; 149925baeddSChris Mason } 1505f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1513f157a2fSChris Mason p->nodes[i] = NULL; 152eb60ceacSChris Mason } 153eb60ceacSChris Mason } 154eb60ceacSChris Mason 155d352ac68SChris Mason /* 156d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 157d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 158d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 159d352ac68SChris Mason * looping required. 160d352ac68SChris Mason * 161d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 162d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 163d352ac68SChris Mason * at any time because there are no locks held. 164d352ac68SChris Mason */ 165925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 166925baeddSChris Mason { 167925baeddSChris Mason struct extent_buffer *eb; 168240f62c8SChris Mason 1693083ee2eSJosef Bacik while (1) { 170240f62c8SChris Mason rcu_read_lock(); 171240f62c8SChris Mason eb = rcu_dereference(root->node); 1723083ee2eSJosef Bacik 1733083ee2eSJosef Bacik /* 1743083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 1753083ee2eSJosef Bacik * it was cow'ed but we may not get the new root node yet so do 1763083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1773083ee2eSJosef Bacik * synchronize_rcu and try again. 1783083ee2eSJosef Bacik */ 1793083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 180240f62c8SChris Mason rcu_read_unlock(); 1813083ee2eSJosef Bacik break; 1823083ee2eSJosef Bacik } 1833083ee2eSJosef Bacik rcu_read_unlock(); 1843083ee2eSJosef Bacik synchronize_rcu(); 1853083ee2eSJosef Bacik } 186925baeddSChris Mason return eb; 187925baeddSChris Mason } 188925baeddSChris Mason 189d352ac68SChris Mason /* loop around taking references on and locking the root node of the 190d352ac68SChris Mason * tree until you end up with a lock on the root. A locked buffer 191d352ac68SChris Mason * is returned, with a reference held. 192d352ac68SChris Mason */ 193925baeddSChris Mason struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root) 194925baeddSChris Mason { 195925baeddSChris Mason struct extent_buffer *eb; 196925baeddSChris Mason 197925baeddSChris Mason while (1) { 198925baeddSChris Mason eb = btrfs_root_node(root); 199925baeddSChris Mason btrfs_tree_lock(eb); 200240f62c8SChris Mason if (eb == root->node) 201925baeddSChris Mason break; 202925baeddSChris Mason btrfs_tree_unlock(eb); 203925baeddSChris Mason free_extent_buffer(eb); 204925baeddSChris Mason } 205925baeddSChris Mason return eb; 206925baeddSChris Mason } 207925baeddSChris Mason 208bd681513SChris Mason /* loop around taking references on and locking the root node of the 209bd681513SChris Mason * tree until you end up with a lock on the root. A locked buffer 210bd681513SChris Mason * is returned, with a reference held. 211bd681513SChris Mason */ 212bd681513SChris Mason struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) 213bd681513SChris Mason { 214bd681513SChris Mason struct extent_buffer *eb; 215bd681513SChris Mason 216bd681513SChris Mason while (1) { 217bd681513SChris Mason eb = btrfs_root_node(root); 218bd681513SChris Mason btrfs_tree_read_lock(eb); 219bd681513SChris Mason if (eb == root->node) 220bd681513SChris Mason break; 221bd681513SChris Mason btrfs_tree_read_unlock(eb); 222bd681513SChris Mason free_extent_buffer(eb); 223bd681513SChris Mason } 224bd681513SChris Mason return eb; 225bd681513SChris Mason } 226bd681513SChris Mason 227d352ac68SChris Mason /* cowonly root (everything not a reference counted cow subvolume), just get 228d352ac68SChris Mason * put onto a simple dirty list. transaction.c walks this to make sure they 229d352ac68SChris Mason * get properly updated on disk. 230d352ac68SChris Mason */ 2310b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2320b86a832SChris Mason { 233e5846fc6SChris Mason spin_lock(&root->fs_info->trans_lock); 2340b86a832SChris Mason if (root->track_dirty && list_empty(&root->dirty_list)) { 2350b86a832SChris Mason list_add(&root->dirty_list, 2360b86a832SChris Mason &root->fs_info->dirty_cowonly_roots); 2370b86a832SChris Mason } 238e5846fc6SChris Mason spin_unlock(&root->fs_info->trans_lock); 2390b86a832SChris Mason } 2400b86a832SChris Mason 241d352ac68SChris Mason /* 242d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 243d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 244d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 245d352ac68SChris Mason */ 246be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 247be20aa9dSChris Mason struct btrfs_root *root, 248be20aa9dSChris Mason struct extent_buffer *buf, 249be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 250be20aa9dSChris Mason { 251be20aa9dSChris Mason struct extent_buffer *cow; 252be20aa9dSChris Mason int ret = 0; 253be20aa9dSChris Mason int level; 2545d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 255be20aa9dSChris Mason 256be20aa9dSChris Mason WARN_ON(root->ref_cows && trans->transid != 257be20aa9dSChris Mason root->fs_info->running_transaction->transid); 258be20aa9dSChris Mason WARN_ON(root->ref_cows && trans->transid != root->last_trans); 259be20aa9dSChris Mason 260be20aa9dSChris Mason level = btrfs_header_level(buf); 2615d4f98a2SYan Zheng if (level == 0) 2625d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 2635d4f98a2SYan Zheng else 2645d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 26531840ae1SZheng Yan 2665d4f98a2SYan Zheng cow = btrfs_alloc_free_block(trans, root, buf->len, 0, 2675d4f98a2SYan Zheng new_root_objectid, &disk_key, level, 2685581a51aSJan Schmidt buf->start, 0); 2695d4f98a2SYan Zheng if (IS_ERR(cow)) 270be20aa9dSChris Mason return PTR_ERR(cow); 271be20aa9dSChris Mason 272be20aa9dSChris Mason copy_extent_buffer(cow, buf, 0, 0, cow->len); 273be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 274be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2755d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2765d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2775d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2785d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2795d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2805d4f98a2SYan Zheng else 281be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 282be20aa9dSChris Mason 2832b82032cSYan Zheng write_extent_buffer(cow, root->fs_info->fsid, 2842b82032cSYan Zheng (unsigned long)btrfs_header_fsid(cow), 2852b82032cSYan Zheng BTRFS_FSID_SIZE); 2862b82032cSYan Zheng 287be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2885d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 28966d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 2905d4f98a2SYan Zheng else 29166d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 2924aec2b52SChris Mason 293be20aa9dSChris Mason if (ret) 294be20aa9dSChris Mason return ret; 295be20aa9dSChris Mason 296be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 297be20aa9dSChris Mason *cow_ret = cow; 298be20aa9dSChris Mason return 0; 299be20aa9dSChris Mason } 300be20aa9dSChris Mason 301bd989ba3SJan Schmidt enum mod_log_op { 302bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 303bd989ba3SJan Schmidt MOD_LOG_KEY_ADD, 304bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE, 305bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_FREEING, 306bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_MOVING, 307bd989ba3SJan Schmidt MOD_LOG_MOVE_KEYS, 308bd989ba3SJan Schmidt MOD_LOG_ROOT_REPLACE, 309bd989ba3SJan Schmidt }; 310bd989ba3SJan Schmidt 311bd989ba3SJan Schmidt struct tree_mod_move { 312bd989ba3SJan Schmidt int dst_slot; 313bd989ba3SJan Schmidt int nr_items; 314bd989ba3SJan Schmidt }; 315bd989ba3SJan Schmidt 316bd989ba3SJan Schmidt struct tree_mod_root { 317bd989ba3SJan Schmidt u64 logical; 318bd989ba3SJan Schmidt u8 level; 319bd989ba3SJan Schmidt }; 320bd989ba3SJan Schmidt 321bd989ba3SJan Schmidt struct tree_mod_elem { 322bd989ba3SJan Schmidt struct rb_node node; 323bd989ba3SJan Schmidt u64 index; /* shifted logical */ 324097b8a7cSJan Schmidt u64 seq; 325bd989ba3SJan Schmidt enum mod_log_op op; 326bd989ba3SJan Schmidt 327bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */ 328bd989ba3SJan Schmidt int slot; 329bd989ba3SJan Schmidt 330bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */ 331bd989ba3SJan Schmidt u64 generation; 332bd989ba3SJan Schmidt 333bd989ba3SJan Schmidt /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */ 334bd989ba3SJan Schmidt struct btrfs_disk_key key; 335bd989ba3SJan Schmidt u64 blockptr; 336bd989ba3SJan Schmidt 337bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_MOVE_KEYS */ 338bd989ba3SJan Schmidt struct tree_mod_move move; 339bd989ba3SJan Schmidt 340bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_ROOT_REPLACE */ 341bd989ba3SJan Schmidt struct tree_mod_root old_root; 342bd989ba3SJan Schmidt }; 343bd989ba3SJan Schmidt 344097b8a7cSJan Schmidt static inline void tree_mod_log_read_lock(struct btrfs_fs_info *fs_info) 345bd989ba3SJan Schmidt { 346097b8a7cSJan Schmidt read_lock(&fs_info->tree_mod_log_lock); 347bd989ba3SJan Schmidt } 348bd989ba3SJan Schmidt 349097b8a7cSJan Schmidt static inline void tree_mod_log_read_unlock(struct btrfs_fs_info *fs_info) 350097b8a7cSJan Schmidt { 351097b8a7cSJan Schmidt read_unlock(&fs_info->tree_mod_log_lock); 352097b8a7cSJan Schmidt } 353097b8a7cSJan Schmidt 354097b8a7cSJan Schmidt static inline void tree_mod_log_write_lock(struct btrfs_fs_info *fs_info) 355097b8a7cSJan Schmidt { 356097b8a7cSJan Schmidt write_lock(&fs_info->tree_mod_log_lock); 357097b8a7cSJan Schmidt } 358097b8a7cSJan Schmidt 359097b8a7cSJan Schmidt static inline void tree_mod_log_write_unlock(struct btrfs_fs_info *fs_info) 360097b8a7cSJan Schmidt { 361097b8a7cSJan Schmidt write_unlock(&fs_info->tree_mod_log_lock); 362097b8a7cSJan Schmidt } 363097b8a7cSJan Schmidt 364097b8a7cSJan Schmidt /* 365097b8a7cSJan Schmidt * This adds a new blocker to the tree mod log's blocker list if the @elem 366097b8a7cSJan Schmidt * passed does not already have a sequence number set. So when a caller expects 367097b8a7cSJan Schmidt * to record tree modifications, it should ensure to set elem->seq to zero 368097b8a7cSJan Schmidt * before calling btrfs_get_tree_mod_seq. 369097b8a7cSJan Schmidt * Returns a fresh, unused tree log modification sequence number, even if no new 370097b8a7cSJan Schmidt * blocker was added. 371097b8a7cSJan Schmidt */ 372097b8a7cSJan Schmidt u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, 373bd989ba3SJan Schmidt struct seq_list *elem) 374bd989ba3SJan Schmidt { 375097b8a7cSJan Schmidt u64 seq; 376097b8a7cSJan Schmidt 377097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 378bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 379097b8a7cSJan Schmidt if (!elem->seq) { 380097b8a7cSJan Schmidt elem->seq = btrfs_inc_tree_mod_seq(fs_info); 381097b8a7cSJan Schmidt list_add_tail(&elem->list, &fs_info->tree_mod_seq_list); 382097b8a7cSJan Schmidt } 383097b8a7cSJan Schmidt seq = btrfs_inc_tree_mod_seq(fs_info); 384bd989ba3SJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 385097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 386097b8a7cSJan Schmidt 387097b8a7cSJan Schmidt return seq; 388bd989ba3SJan Schmidt } 389bd989ba3SJan Schmidt 390bd989ba3SJan Schmidt void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, 391bd989ba3SJan Schmidt struct seq_list *elem) 392bd989ba3SJan Schmidt { 393bd989ba3SJan Schmidt struct rb_root *tm_root; 394bd989ba3SJan Schmidt struct rb_node *node; 395bd989ba3SJan Schmidt struct rb_node *next; 396bd989ba3SJan Schmidt struct seq_list *cur_elem; 397bd989ba3SJan Schmidt struct tree_mod_elem *tm; 398bd989ba3SJan Schmidt u64 min_seq = (u64)-1; 399bd989ba3SJan Schmidt u64 seq_putting = elem->seq; 400bd989ba3SJan Schmidt 401bd989ba3SJan Schmidt if (!seq_putting) 402bd989ba3SJan Schmidt return; 403bd989ba3SJan Schmidt 404bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 405bd989ba3SJan Schmidt list_del(&elem->list); 406097b8a7cSJan Schmidt elem->seq = 0; 407bd989ba3SJan Schmidt 408bd989ba3SJan Schmidt list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) { 409097b8a7cSJan Schmidt if (cur_elem->seq < min_seq) { 410bd989ba3SJan Schmidt if (seq_putting > cur_elem->seq) { 411bd989ba3SJan Schmidt /* 412bd989ba3SJan Schmidt * blocker with lower sequence number exists, we 413bd989ba3SJan Schmidt * cannot remove anything from the log 414bd989ba3SJan Schmidt */ 415097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 416097b8a7cSJan Schmidt return; 417bd989ba3SJan Schmidt } 418bd989ba3SJan Schmidt min_seq = cur_elem->seq; 419bd989ba3SJan Schmidt } 420bd989ba3SJan Schmidt } 421097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 422097b8a7cSJan Schmidt 423097b8a7cSJan Schmidt /* 424097b8a7cSJan Schmidt * we removed the lowest blocker from the blocker list, so there may be 425097b8a7cSJan Schmidt * more processible delayed refs. 426097b8a7cSJan Schmidt */ 427097b8a7cSJan Schmidt wake_up(&fs_info->tree_mod_seq_wait); 428bd989ba3SJan Schmidt 429bd989ba3SJan Schmidt /* 430bd989ba3SJan Schmidt * anything that's lower than the lowest existing (read: blocked) 431bd989ba3SJan Schmidt * sequence number can be removed from the tree. 432bd989ba3SJan Schmidt */ 433097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 434bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 435bd989ba3SJan Schmidt for (node = rb_first(tm_root); node; node = next) { 436bd989ba3SJan Schmidt next = rb_next(node); 437bd989ba3SJan Schmidt tm = container_of(node, struct tree_mod_elem, node); 438097b8a7cSJan Schmidt if (tm->seq > min_seq) 439bd989ba3SJan Schmidt continue; 440bd989ba3SJan Schmidt rb_erase(node, tm_root); 441bd989ba3SJan Schmidt kfree(tm); 442bd989ba3SJan Schmidt } 443097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 444bd989ba3SJan Schmidt } 445bd989ba3SJan Schmidt 446bd989ba3SJan Schmidt /* 447bd989ba3SJan Schmidt * key order of the log: 448bd989ba3SJan Schmidt * index -> sequence 449bd989ba3SJan Schmidt * 450bd989ba3SJan Schmidt * the index is the shifted logical of the *new* root node for root replace 451bd989ba3SJan Schmidt * operations, or the shifted logical of the affected block for all other 452bd989ba3SJan Schmidt * operations. 453bd989ba3SJan Schmidt */ 454bd989ba3SJan Schmidt static noinline int 455bd989ba3SJan Schmidt __tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm) 456bd989ba3SJan Schmidt { 457bd989ba3SJan Schmidt struct rb_root *tm_root; 458bd989ba3SJan Schmidt struct rb_node **new; 459bd989ba3SJan Schmidt struct rb_node *parent = NULL; 460bd989ba3SJan Schmidt struct tree_mod_elem *cur; 461bd989ba3SJan Schmidt 462097b8a7cSJan Schmidt BUG_ON(!tm || !tm->seq); 463bd989ba3SJan Schmidt 464bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 465bd989ba3SJan Schmidt new = &tm_root->rb_node; 466bd989ba3SJan Schmidt while (*new) { 467bd989ba3SJan Schmidt cur = container_of(*new, struct tree_mod_elem, node); 468bd989ba3SJan Schmidt parent = *new; 469bd989ba3SJan Schmidt if (cur->index < tm->index) 470bd989ba3SJan Schmidt new = &((*new)->rb_left); 471bd989ba3SJan Schmidt else if (cur->index > tm->index) 472bd989ba3SJan Schmidt new = &((*new)->rb_right); 473097b8a7cSJan Schmidt else if (cur->seq < tm->seq) 474bd989ba3SJan Schmidt new = &((*new)->rb_left); 475097b8a7cSJan Schmidt else if (cur->seq > tm->seq) 476bd989ba3SJan Schmidt new = &((*new)->rb_right); 477bd989ba3SJan Schmidt else { 478bd989ba3SJan Schmidt kfree(tm); 479097b8a7cSJan Schmidt return -EEXIST; 480bd989ba3SJan Schmidt } 481bd989ba3SJan Schmidt } 482bd989ba3SJan Schmidt 483bd989ba3SJan Schmidt rb_link_node(&tm->node, parent, new); 484bd989ba3SJan Schmidt rb_insert_color(&tm->node, tm_root); 485097b8a7cSJan Schmidt return 0; 486bd989ba3SJan Schmidt } 487bd989ba3SJan Schmidt 488097b8a7cSJan Schmidt /* 489097b8a7cSJan Schmidt * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it 490097b8a7cSJan Schmidt * returns zero with the tree_mod_log_lock acquired. The caller must hold 491097b8a7cSJan Schmidt * this until all tree mod log insertions are recorded in the rb tree and then 492097b8a7cSJan Schmidt * call tree_mod_log_write_unlock() to release. 493097b8a7cSJan Schmidt */ 494e9b7fd4dSJan Schmidt static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info, 495e9b7fd4dSJan Schmidt struct extent_buffer *eb) { 496e9b7fd4dSJan Schmidt smp_mb(); 497e9b7fd4dSJan Schmidt if (list_empty(&(fs_info)->tree_mod_seq_list)) 498e9b7fd4dSJan Schmidt return 1; 499097b8a7cSJan Schmidt if (eb && btrfs_header_level(eb) == 0) 500e9b7fd4dSJan Schmidt return 1; 501097b8a7cSJan Schmidt 502097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 503097b8a7cSJan Schmidt if (list_empty(&fs_info->tree_mod_seq_list)) { 504097b8a7cSJan Schmidt /* 505097b8a7cSJan Schmidt * someone emptied the list while we were waiting for the lock. 506097b8a7cSJan Schmidt * we must not add to the list when no blocker exists. 507097b8a7cSJan Schmidt */ 508097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 509097b8a7cSJan Schmidt return 1; 510097b8a7cSJan Schmidt } 511097b8a7cSJan Schmidt 512e9b7fd4dSJan Schmidt return 0; 513e9b7fd4dSJan Schmidt } 514e9b7fd4dSJan Schmidt 5153310c36eSJan Schmidt /* 516097b8a7cSJan Schmidt * This allocates memory and gets a tree modification sequence number. 5173310c36eSJan Schmidt * 518097b8a7cSJan Schmidt * Returns <0 on error. 519097b8a7cSJan Schmidt * Returns >0 (the added sequence number) on success. 5203310c36eSJan Schmidt */ 521926dd8a6SJan Schmidt static inline int tree_mod_alloc(struct btrfs_fs_info *fs_info, gfp_t flags, 522bd989ba3SJan Schmidt struct tree_mod_elem **tm_ret) 523bd989ba3SJan Schmidt { 524bd989ba3SJan Schmidt struct tree_mod_elem *tm; 525bd989ba3SJan Schmidt 526097b8a7cSJan Schmidt /* 527097b8a7cSJan Schmidt * once we switch from spin locks to something different, we should 528097b8a7cSJan Schmidt * honor the flags parameter here. 529097b8a7cSJan Schmidt */ 530097b8a7cSJan Schmidt tm = *tm_ret = kzalloc(sizeof(*tm), GFP_ATOMIC); 531bd989ba3SJan Schmidt if (!tm) 532bd989ba3SJan Schmidt return -ENOMEM; 533bd989ba3SJan Schmidt 534097b8a7cSJan Schmidt tm->seq = btrfs_inc_tree_mod_seq(fs_info); 535097b8a7cSJan Schmidt return tm->seq; 536926dd8a6SJan Schmidt } 537bd989ba3SJan Schmidt 538097b8a7cSJan Schmidt static inline int 539097b8a7cSJan Schmidt __tree_mod_log_insert_key(struct btrfs_fs_info *fs_info, 540bd989ba3SJan Schmidt struct extent_buffer *eb, int slot, 541bd989ba3SJan Schmidt enum mod_log_op op, gfp_t flags) 542bd989ba3SJan Schmidt { 543bd989ba3SJan Schmidt int ret; 544097b8a7cSJan Schmidt struct tree_mod_elem *tm; 545bd989ba3SJan Schmidt 546bd989ba3SJan Schmidt ret = tree_mod_alloc(fs_info, flags, &tm); 547097b8a7cSJan Schmidt if (ret < 0) 548bd989ba3SJan Schmidt return ret; 549bd989ba3SJan Schmidt 550bd989ba3SJan Schmidt tm->index = eb->start >> PAGE_CACHE_SHIFT; 551bd989ba3SJan Schmidt if (op != MOD_LOG_KEY_ADD) { 552bd989ba3SJan Schmidt btrfs_node_key(eb, &tm->key, slot); 553bd989ba3SJan Schmidt tm->blockptr = btrfs_node_blockptr(eb, slot); 554bd989ba3SJan Schmidt } 555bd989ba3SJan Schmidt tm->op = op; 556bd989ba3SJan Schmidt tm->slot = slot; 557bd989ba3SJan Schmidt tm->generation = btrfs_node_ptr_generation(eb, slot); 558bd989ba3SJan Schmidt 559097b8a7cSJan Schmidt return __tree_mod_log_insert(fs_info, tm); 560097b8a7cSJan Schmidt } 561097b8a7cSJan Schmidt 562097b8a7cSJan Schmidt static noinline int 563097b8a7cSJan Schmidt tree_mod_log_insert_key_mask(struct btrfs_fs_info *fs_info, 564097b8a7cSJan Schmidt struct extent_buffer *eb, int slot, 565097b8a7cSJan Schmidt enum mod_log_op op, gfp_t flags) 566097b8a7cSJan Schmidt { 567097b8a7cSJan Schmidt int ret; 568097b8a7cSJan Schmidt 569097b8a7cSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 570097b8a7cSJan Schmidt return 0; 571097b8a7cSJan Schmidt 572097b8a7cSJan Schmidt ret = __tree_mod_log_insert_key(fs_info, eb, slot, op, flags); 573097b8a7cSJan Schmidt 574097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 5753310c36eSJan Schmidt return ret; 576bd989ba3SJan Schmidt } 577bd989ba3SJan Schmidt 578bd989ba3SJan Schmidt static noinline int 579bd989ba3SJan Schmidt tree_mod_log_insert_key(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 580bd989ba3SJan Schmidt int slot, enum mod_log_op op) 581bd989ba3SJan Schmidt { 582bd989ba3SJan Schmidt return tree_mod_log_insert_key_mask(fs_info, eb, slot, op, GFP_NOFS); 583bd989ba3SJan Schmidt } 584bd989ba3SJan Schmidt 585bd989ba3SJan Schmidt static noinline int 586097b8a7cSJan Schmidt tree_mod_log_insert_key_locked(struct btrfs_fs_info *fs_info, 587097b8a7cSJan Schmidt struct extent_buffer *eb, int slot, 588097b8a7cSJan Schmidt enum mod_log_op op) 589097b8a7cSJan Schmidt { 590097b8a7cSJan Schmidt return __tree_mod_log_insert_key(fs_info, eb, slot, op, GFP_NOFS); 591097b8a7cSJan Schmidt } 592097b8a7cSJan Schmidt 593097b8a7cSJan Schmidt static noinline int 594bd989ba3SJan Schmidt tree_mod_log_insert_move(struct btrfs_fs_info *fs_info, 595bd989ba3SJan Schmidt struct extent_buffer *eb, int dst_slot, int src_slot, 596bd989ba3SJan Schmidt int nr_items, gfp_t flags) 597bd989ba3SJan Schmidt { 598bd989ba3SJan Schmidt struct tree_mod_elem *tm; 599bd989ba3SJan Schmidt int ret; 600bd989ba3SJan Schmidt int i; 601bd989ba3SJan Schmidt 602f395694cSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 603f395694cSJan Schmidt return 0; 604bd989ba3SJan Schmidt 605bd989ba3SJan Schmidt for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 606097b8a7cSJan Schmidt ret = tree_mod_log_insert_key_locked(fs_info, eb, i + dst_slot, 607bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_MOVING); 608bd989ba3SJan Schmidt BUG_ON(ret < 0); 609bd989ba3SJan Schmidt } 610bd989ba3SJan Schmidt 611f395694cSJan Schmidt ret = tree_mod_alloc(fs_info, flags, &tm); 612097b8a7cSJan Schmidt if (ret < 0) 613097b8a7cSJan Schmidt goto out; 614f395694cSJan Schmidt 615bd989ba3SJan Schmidt tm->index = eb->start >> PAGE_CACHE_SHIFT; 616bd989ba3SJan Schmidt tm->slot = src_slot; 617bd989ba3SJan Schmidt tm->move.dst_slot = dst_slot; 618bd989ba3SJan Schmidt tm->move.nr_items = nr_items; 619bd989ba3SJan Schmidt tm->op = MOD_LOG_MOVE_KEYS; 620bd989ba3SJan Schmidt 6213310c36eSJan Schmidt ret = __tree_mod_log_insert(fs_info, tm); 622097b8a7cSJan Schmidt out: 623097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 6243310c36eSJan Schmidt return ret; 625bd989ba3SJan Schmidt } 626bd989ba3SJan Schmidt 627097b8a7cSJan Schmidt static inline void 628097b8a7cSJan Schmidt __tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb) 629097b8a7cSJan Schmidt { 630097b8a7cSJan Schmidt int i; 631097b8a7cSJan Schmidt u32 nritems; 632097b8a7cSJan Schmidt int ret; 633097b8a7cSJan Schmidt 634097b8a7cSJan Schmidt nritems = btrfs_header_nritems(eb); 635097b8a7cSJan Schmidt for (i = nritems - 1; i >= 0; i--) { 636097b8a7cSJan Schmidt ret = tree_mod_log_insert_key_locked(fs_info, eb, i, 637097b8a7cSJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_FREEING); 638097b8a7cSJan Schmidt BUG_ON(ret < 0); 639097b8a7cSJan Schmidt } 640097b8a7cSJan Schmidt } 641097b8a7cSJan Schmidt 642bd989ba3SJan Schmidt static noinline int 643bd989ba3SJan Schmidt tree_mod_log_insert_root(struct btrfs_fs_info *fs_info, 644bd989ba3SJan Schmidt struct extent_buffer *old_root, 645bd989ba3SJan Schmidt struct extent_buffer *new_root, gfp_t flags) 646bd989ba3SJan Schmidt { 647bd989ba3SJan Schmidt struct tree_mod_elem *tm; 648bd989ba3SJan Schmidt int ret; 649bd989ba3SJan Schmidt 650097b8a7cSJan Schmidt if (tree_mod_dont_log(fs_info, NULL)) 651097b8a7cSJan Schmidt return 0; 652097b8a7cSJan Schmidt 653097b8a7cSJan Schmidt __tree_mod_log_free_eb(fs_info, old_root); 654097b8a7cSJan Schmidt 655bd989ba3SJan Schmidt ret = tree_mod_alloc(fs_info, flags, &tm); 656097b8a7cSJan Schmidt if (ret < 0) 657097b8a7cSJan Schmidt goto out; 658bd989ba3SJan Schmidt 659bd989ba3SJan Schmidt tm->index = new_root->start >> PAGE_CACHE_SHIFT; 660bd989ba3SJan Schmidt tm->old_root.logical = old_root->start; 661bd989ba3SJan Schmidt tm->old_root.level = btrfs_header_level(old_root); 662bd989ba3SJan Schmidt tm->generation = btrfs_header_generation(old_root); 663bd989ba3SJan Schmidt tm->op = MOD_LOG_ROOT_REPLACE; 664bd989ba3SJan Schmidt 6653310c36eSJan Schmidt ret = __tree_mod_log_insert(fs_info, tm); 666097b8a7cSJan Schmidt out: 667097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 6683310c36eSJan Schmidt return ret; 669bd989ba3SJan Schmidt } 670bd989ba3SJan Schmidt 671bd989ba3SJan Schmidt static struct tree_mod_elem * 672bd989ba3SJan Schmidt __tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq, 673bd989ba3SJan Schmidt int smallest) 674bd989ba3SJan Schmidt { 675bd989ba3SJan Schmidt struct rb_root *tm_root; 676bd989ba3SJan Schmidt struct rb_node *node; 677bd989ba3SJan Schmidt struct tree_mod_elem *cur = NULL; 678bd989ba3SJan Schmidt struct tree_mod_elem *found = NULL; 679bd989ba3SJan Schmidt u64 index = start >> PAGE_CACHE_SHIFT; 680bd989ba3SJan Schmidt 681097b8a7cSJan Schmidt tree_mod_log_read_lock(fs_info); 682bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 683bd989ba3SJan Schmidt node = tm_root->rb_node; 684bd989ba3SJan Schmidt while (node) { 685bd989ba3SJan Schmidt cur = container_of(node, struct tree_mod_elem, node); 686bd989ba3SJan Schmidt if (cur->index < index) { 687bd989ba3SJan Schmidt node = node->rb_left; 688bd989ba3SJan Schmidt } else if (cur->index > index) { 689bd989ba3SJan Schmidt node = node->rb_right; 690097b8a7cSJan Schmidt } else if (cur->seq < min_seq) { 691bd989ba3SJan Schmidt node = node->rb_left; 692bd989ba3SJan Schmidt } else if (!smallest) { 693bd989ba3SJan Schmidt /* we want the node with the highest seq */ 694bd989ba3SJan Schmidt if (found) 695097b8a7cSJan Schmidt BUG_ON(found->seq > cur->seq); 696bd989ba3SJan Schmidt found = cur; 697bd989ba3SJan Schmidt node = node->rb_left; 698097b8a7cSJan Schmidt } else if (cur->seq > min_seq) { 699bd989ba3SJan Schmidt /* we want the node with the smallest seq */ 700bd989ba3SJan Schmidt if (found) 701097b8a7cSJan Schmidt BUG_ON(found->seq < cur->seq); 702bd989ba3SJan Schmidt found = cur; 703bd989ba3SJan Schmidt node = node->rb_right; 704bd989ba3SJan Schmidt } else { 705bd989ba3SJan Schmidt found = cur; 706bd989ba3SJan Schmidt break; 707bd989ba3SJan Schmidt } 708bd989ba3SJan Schmidt } 709097b8a7cSJan Schmidt tree_mod_log_read_unlock(fs_info); 710bd989ba3SJan Schmidt 711bd989ba3SJan Schmidt return found; 712bd989ba3SJan Schmidt } 713bd989ba3SJan Schmidt 714bd989ba3SJan Schmidt /* 715bd989ba3SJan Schmidt * this returns the element from the log with the smallest time sequence 716bd989ba3SJan Schmidt * value that's in the log (the oldest log item). any element with a time 717bd989ba3SJan Schmidt * sequence lower than min_seq will be ignored. 718bd989ba3SJan Schmidt */ 719bd989ba3SJan Schmidt static struct tree_mod_elem * 720bd989ba3SJan Schmidt tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start, 721bd989ba3SJan Schmidt u64 min_seq) 722bd989ba3SJan Schmidt { 723bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 1); 724bd989ba3SJan Schmidt } 725bd989ba3SJan Schmidt 726bd989ba3SJan Schmidt /* 727bd989ba3SJan Schmidt * this returns the element from the log with the largest time sequence 728bd989ba3SJan Schmidt * value that's in the log (the most recent log item). any element with 729bd989ba3SJan Schmidt * a time sequence lower than min_seq will be ignored. 730bd989ba3SJan Schmidt */ 731bd989ba3SJan Schmidt static struct tree_mod_elem * 732bd989ba3SJan Schmidt tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq) 733bd989ba3SJan Schmidt { 734bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 0); 735bd989ba3SJan Schmidt } 736bd989ba3SJan Schmidt 737097b8a7cSJan Schmidt static noinline void 738bd989ba3SJan Schmidt tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 739bd989ba3SJan Schmidt struct extent_buffer *src, unsigned long dst_offset, 740bd989ba3SJan Schmidt unsigned long src_offset, int nr_items) 741bd989ba3SJan Schmidt { 742bd989ba3SJan Schmidt int ret; 743bd989ba3SJan Schmidt int i; 744bd989ba3SJan Schmidt 745e9b7fd4dSJan Schmidt if (tree_mod_dont_log(fs_info, NULL)) 746bd989ba3SJan Schmidt return; 747bd989ba3SJan Schmidt 748097b8a7cSJan Schmidt if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0) { 749097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 750bd989ba3SJan Schmidt return; 751097b8a7cSJan Schmidt } 752bd989ba3SJan Schmidt 753bd989ba3SJan Schmidt for (i = 0; i < nr_items; i++) { 754097b8a7cSJan Schmidt ret = tree_mod_log_insert_key_locked(fs_info, src, 755097b8a7cSJan Schmidt i + src_offset, 756bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE); 757bd989ba3SJan Schmidt BUG_ON(ret < 0); 758097b8a7cSJan Schmidt ret = tree_mod_log_insert_key_locked(fs_info, dst, 759097b8a7cSJan Schmidt i + dst_offset, 760bd989ba3SJan Schmidt MOD_LOG_KEY_ADD); 761bd989ba3SJan Schmidt BUG_ON(ret < 0); 762bd989ba3SJan Schmidt } 763097b8a7cSJan Schmidt 764097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 765bd989ba3SJan Schmidt } 766bd989ba3SJan Schmidt 767bd989ba3SJan Schmidt static inline void 768bd989ba3SJan Schmidt tree_mod_log_eb_move(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 769bd989ba3SJan Schmidt int dst_offset, int src_offset, int nr_items) 770bd989ba3SJan Schmidt { 771bd989ba3SJan Schmidt int ret; 772bd989ba3SJan Schmidt ret = tree_mod_log_insert_move(fs_info, dst, dst_offset, src_offset, 773bd989ba3SJan Schmidt nr_items, GFP_NOFS); 774bd989ba3SJan Schmidt BUG_ON(ret < 0); 775bd989ba3SJan Schmidt } 776bd989ba3SJan Schmidt 777097b8a7cSJan Schmidt static noinline void 778bd989ba3SJan Schmidt tree_mod_log_set_node_key(struct btrfs_fs_info *fs_info, 779bd989ba3SJan Schmidt struct extent_buffer *eb, 780bd989ba3SJan Schmidt struct btrfs_disk_key *disk_key, int slot, int atomic) 781bd989ba3SJan Schmidt { 782bd989ba3SJan Schmidt int ret; 783bd989ba3SJan Schmidt 784bd989ba3SJan Schmidt ret = tree_mod_log_insert_key_mask(fs_info, eb, slot, 785bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 786bd989ba3SJan Schmidt atomic ? GFP_ATOMIC : GFP_NOFS); 787bd989ba3SJan Schmidt BUG_ON(ret < 0); 788bd989ba3SJan Schmidt } 789bd989ba3SJan Schmidt 790097b8a7cSJan Schmidt static noinline void 791097b8a7cSJan Schmidt tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb) 792bd989ba3SJan Schmidt { 793e9b7fd4dSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 794bd989ba3SJan Schmidt return; 795bd989ba3SJan Schmidt 796097b8a7cSJan Schmidt __tree_mod_log_free_eb(fs_info, eb); 797097b8a7cSJan Schmidt 798097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 799bd989ba3SJan Schmidt } 800bd989ba3SJan Schmidt 801097b8a7cSJan Schmidt static noinline void 802bd989ba3SJan Schmidt tree_mod_log_set_root_pointer(struct btrfs_root *root, 803bd989ba3SJan Schmidt struct extent_buffer *new_root_node) 804bd989ba3SJan Schmidt { 805bd989ba3SJan Schmidt int ret; 806bd989ba3SJan Schmidt ret = tree_mod_log_insert_root(root->fs_info, root->node, 807bd989ba3SJan Schmidt new_root_node, GFP_NOFS); 808bd989ba3SJan Schmidt BUG_ON(ret < 0); 809bd989ba3SJan Schmidt } 810bd989ba3SJan Schmidt 811d352ac68SChris Mason /* 8125d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 8135d4f98a2SYan Zheng */ 8145d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 8155d4f98a2SYan Zheng struct extent_buffer *buf) 8165d4f98a2SYan Zheng { 8175d4f98a2SYan Zheng /* 8185d4f98a2SYan Zheng * Tree blocks not in refernece counted trees and tree roots 8195d4f98a2SYan Zheng * are never shared. If a block was allocated after the last 8205d4f98a2SYan Zheng * snapshot and the block was not allocated by tree relocation, 8215d4f98a2SYan Zheng * we know the block is not shared. 8225d4f98a2SYan Zheng */ 8235d4f98a2SYan Zheng if (root->ref_cows && 8245d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 8255d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 8265d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 8275d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 8285d4f98a2SYan Zheng return 1; 8295d4f98a2SYan Zheng #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 8305d4f98a2SYan Zheng if (root->ref_cows && 8315d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 8325d4f98a2SYan Zheng return 1; 8335d4f98a2SYan Zheng #endif 8345d4f98a2SYan Zheng return 0; 8355d4f98a2SYan Zheng } 8365d4f98a2SYan Zheng 8375d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 8385d4f98a2SYan Zheng struct btrfs_root *root, 8395d4f98a2SYan Zheng struct extent_buffer *buf, 840f0486c68SYan, Zheng struct extent_buffer *cow, 841f0486c68SYan, Zheng int *last_ref) 8425d4f98a2SYan Zheng { 8435d4f98a2SYan Zheng u64 refs; 8445d4f98a2SYan Zheng u64 owner; 8455d4f98a2SYan Zheng u64 flags; 8465d4f98a2SYan Zheng u64 new_flags = 0; 8475d4f98a2SYan Zheng int ret; 8485d4f98a2SYan Zheng 8495d4f98a2SYan Zheng /* 8505d4f98a2SYan Zheng * Backrefs update rules: 8515d4f98a2SYan Zheng * 8525d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 8535d4f98a2SYan Zheng * allocated by tree relocation. 8545d4f98a2SYan Zheng * 8555d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 8565d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 8575d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8585d4f98a2SYan Zheng * 8595d4f98a2SYan Zheng * If a tree block is been relocating 8605d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 8615d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8625d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 8635d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 8645d4f98a2SYan Zheng */ 8655d4f98a2SYan Zheng 8665d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 8675d4f98a2SYan Zheng ret = btrfs_lookup_extent_info(trans, root, buf->start, 8685d4f98a2SYan Zheng buf->len, &refs, &flags); 869be1a5564SMark Fasheh if (ret) 870be1a5564SMark Fasheh return ret; 871e5df9573SMark Fasheh if (refs == 0) { 872e5df9573SMark Fasheh ret = -EROFS; 873e5df9573SMark Fasheh btrfs_std_error(root->fs_info, ret); 874e5df9573SMark Fasheh return ret; 875e5df9573SMark Fasheh } 8765d4f98a2SYan Zheng } else { 8775d4f98a2SYan Zheng refs = 1; 8785d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 8795d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 8805d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 8815d4f98a2SYan Zheng else 8825d4f98a2SYan Zheng flags = 0; 8835d4f98a2SYan Zheng } 8845d4f98a2SYan Zheng 8855d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 8865d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 8875d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 8885d4f98a2SYan Zheng 8895d4f98a2SYan Zheng if (refs > 1) { 8905d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 8915d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 8925d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 89366d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, buf, 1, 1); 89479787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 8955d4f98a2SYan Zheng 8965d4f98a2SYan Zheng if (root->root_key.objectid == 8975d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 89866d7e7f0SArne Jansen ret = btrfs_dec_ref(trans, root, buf, 0, 1); 89979787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 90066d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 90179787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 9025d4f98a2SYan Zheng } 9035d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 9045d4f98a2SYan Zheng } else { 9055d4f98a2SYan Zheng 9065d4f98a2SYan Zheng if (root->root_key.objectid == 9075d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 90866d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 9095d4f98a2SYan Zheng else 91066d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 91179787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 9125d4f98a2SYan Zheng } 9135d4f98a2SYan Zheng if (new_flags != 0) { 9145d4f98a2SYan Zheng ret = btrfs_set_disk_extent_flags(trans, root, 9155d4f98a2SYan Zheng buf->start, 9165d4f98a2SYan Zheng buf->len, 9175d4f98a2SYan Zheng new_flags, 0); 918be1a5564SMark Fasheh if (ret) 919be1a5564SMark Fasheh return ret; 9205d4f98a2SYan Zheng } 9215d4f98a2SYan Zheng } else { 9225d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 9235d4f98a2SYan Zheng if (root->root_key.objectid == 9245d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 92566d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 9265d4f98a2SYan Zheng else 92766d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 92879787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 92966d7e7f0SArne Jansen ret = btrfs_dec_ref(trans, root, buf, 1, 1); 93079787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 9315d4f98a2SYan Zheng } 932f230475eSJan Schmidt /* 933f230475eSJan Schmidt * don't log freeing in case we're freeing the root node, this 934f230475eSJan Schmidt * is done by tree_mod_log_set_root_pointer later 935f230475eSJan Schmidt */ 936f230475eSJan Schmidt if (buf != root->node && btrfs_header_level(buf) != 0) 937f230475eSJan Schmidt tree_mod_log_free_eb(root->fs_info, buf); 9385d4f98a2SYan Zheng clean_tree_block(trans, root, buf); 939f0486c68SYan, Zheng *last_ref = 1; 9405d4f98a2SYan Zheng } 9415d4f98a2SYan Zheng return 0; 9425d4f98a2SYan Zheng } 9435d4f98a2SYan Zheng 9445d4f98a2SYan Zheng /* 945d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 946d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 947d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 948d397712bSChris Mason * dirty again. 949d352ac68SChris Mason * 950d352ac68SChris Mason * search_start -- an allocation hint for the new block 951d352ac68SChris Mason * 952d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 953d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 954d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 955d352ac68SChris Mason */ 956d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 9575f39d397SChris Mason struct btrfs_root *root, 9585f39d397SChris Mason struct extent_buffer *buf, 9595f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 9605f39d397SChris Mason struct extent_buffer **cow_ret, 9619fa8cfe7SChris Mason u64 search_start, u64 empty_size) 9626702ed49SChris Mason { 9635d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 9645f39d397SChris Mason struct extent_buffer *cow; 965be1a5564SMark Fasheh int level, ret; 966f0486c68SYan, Zheng int last_ref = 0; 967925baeddSChris Mason int unlock_orig = 0; 9685d4f98a2SYan Zheng u64 parent_start; 9696702ed49SChris Mason 970925baeddSChris Mason if (*cow_ret == buf) 971925baeddSChris Mason unlock_orig = 1; 972925baeddSChris Mason 973b9447ef8SChris Mason btrfs_assert_tree_locked(buf); 974925baeddSChris Mason 9757bb86316SChris Mason WARN_ON(root->ref_cows && trans->transid != 9767bb86316SChris Mason root->fs_info->running_transaction->transid); 9776702ed49SChris Mason WARN_ON(root->ref_cows && trans->transid != root->last_trans); 9785f39d397SChris Mason 9797bb86316SChris Mason level = btrfs_header_level(buf); 98031840ae1SZheng Yan 9815d4f98a2SYan Zheng if (level == 0) 9825d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 9835d4f98a2SYan Zheng else 9845d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 9855d4f98a2SYan Zheng 9865d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) { 9875d4f98a2SYan Zheng if (parent) 9885d4f98a2SYan Zheng parent_start = parent->start; 9895d4f98a2SYan Zheng else 9905d4f98a2SYan Zheng parent_start = 0; 9915d4f98a2SYan Zheng } else 9925d4f98a2SYan Zheng parent_start = 0; 9935d4f98a2SYan Zheng 9945d4f98a2SYan Zheng cow = btrfs_alloc_free_block(trans, root, buf->len, parent_start, 9955d4f98a2SYan Zheng root->root_key.objectid, &disk_key, 9965581a51aSJan Schmidt level, search_start, empty_size); 9976702ed49SChris Mason if (IS_ERR(cow)) 9986702ed49SChris Mason return PTR_ERR(cow); 9996702ed49SChris Mason 1000b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 1001b4ce94deSChris Mason 10025f39d397SChris Mason copy_extent_buffer(cow, buf, 0, 0, cow->len); 1003db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 10045f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 10055d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 10065d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 10075d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 10085d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 10095d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 10105d4f98a2SYan Zheng else 10115f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 10126702ed49SChris Mason 10132b82032cSYan Zheng write_extent_buffer(cow, root->fs_info->fsid, 10142b82032cSYan Zheng (unsigned long)btrfs_header_fsid(cow), 10152b82032cSYan Zheng BTRFS_FSID_SIZE); 10162b82032cSYan Zheng 1017be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 1018b68dc2a9SMark Fasheh if (ret) { 101979787eaaSJeff Mahoney btrfs_abort_transaction(trans, root, ret); 1020b68dc2a9SMark Fasheh return ret; 1021b68dc2a9SMark Fasheh } 10221a40e23bSZheng Yan 10233fd0a558SYan, Zheng if (root->ref_cows) 10243fd0a558SYan, Zheng btrfs_reloc_cow_block(trans, root, buf, cow); 10253fd0a558SYan, Zheng 10266702ed49SChris Mason if (buf == root->node) { 1027925baeddSChris Mason WARN_ON(parent && parent != buf); 10285d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 10295d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 10305d4f98a2SYan Zheng parent_start = buf->start; 10315d4f98a2SYan Zheng else 10325d4f98a2SYan Zheng parent_start = 0; 1033925baeddSChris Mason 10345f39d397SChris Mason extent_buffer_get(cow); 1035f230475eSJan Schmidt tree_mod_log_set_root_pointer(root, cow); 1036240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 1037925baeddSChris Mason 1038f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 10395581a51aSJan Schmidt last_ref); 10405f39d397SChris Mason free_extent_buffer(buf); 10410b86a832SChris Mason add_root_to_dirty_list(root); 10426702ed49SChris Mason } else { 10435d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 10445d4f98a2SYan Zheng parent_start = parent->start; 10455d4f98a2SYan Zheng else 10465d4f98a2SYan Zheng parent_start = 0; 10475d4f98a2SYan Zheng 10485d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 1049f230475eSJan Schmidt tree_mod_log_insert_key(root->fs_info, parent, parent_slot, 1050f230475eSJan Schmidt MOD_LOG_KEY_REPLACE); 10515f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 1052db94535dSChris Mason cow->start); 105374493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 105474493f7aSChris Mason trans->transid); 10556702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 1056f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 10575581a51aSJan Schmidt last_ref); 10586702ed49SChris Mason } 1059925baeddSChris Mason if (unlock_orig) 1060925baeddSChris Mason btrfs_tree_unlock(buf); 10613083ee2eSJosef Bacik free_extent_buffer_stale(buf); 10626702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 10636702ed49SChris Mason *cow_ret = cow; 10646702ed49SChris Mason return 0; 10656702ed49SChris Mason } 10666702ed49SChris Mason 10675d9e75c4SJan Schmidt /* 10685d9e75c4SJan Schmidt * returns the logical address of the oldest predecessor of the given root. 10695d9e75c4SJan Schmidt * entries older than time_seq are ignored. 10705d9e75c4SJan Schmidt */ 10715d9e75c4SJan Schmidt static struct tree_mod_elem * 10725d9e75c4SJan Schmidt __tree_mod_log_oldest_root(struct btrfs_fs_info *fs_info, 10735d9e75c4SJan Schmidt struct btrfs_root *root, u64 time_seq) 10745d9e75c4SJan Schmidt { 10755d9e75c4SJan Schmidt struct tree_mod_elem *tm; 10765d9e75c4SJan Schmidt struct tree_mod_elem *found = NULL; 10775d9e75c4SJan Schmidt u64 root_logical = root->node->start; 10785d9e75c4SJan Schmidt int looped = 0; 10795d9e75c4SJan Schmidt 10805d9e75c4SJan Schmidt if (!time_seq) 10815d9e75c4SJan Schmidt return 0; 10825d9e75c4SJan Schmidt 10835d9e75c4SJan Schmidt /* 10845d9e75c4SJan Schmidt * the very last operation that's logged for a root is the replacement 10855d9e75c4SJan Schmidt * operation (if it is replaced at all). this has the index of the *new* 10865d9e75c4SJan Schmidt * root, making it the very first operation that's logged for this root. 10875d9e75c4SJan Schmidt */ 10885d9e75c4SJan Schmidt while (1) { 10895d9e75c4SJan Schmidt tm = tree_mod_log_search_oldest(fs_info, root_logical, 10905d9e75c4SJan Schmidt time_seq); 10915d9e75c4SJan Schmidt if (!looped && !tm) 10925d9e75c4SJan Schmidt return 0; 10935d9e75c4SJan Schmidt /* 109428da9fb4SJan Schmidt * if there are no tree operation for the oldest root, we simply 109528da9fb4SJan Schmidt * return it. this should only happen if that (old) root is at 109628da9fb4SJan Schmidt * level 0. 10975d9e75c4SJan Schmidt */ 109828da9fb4SJan Schmidt if (!tm) 109928da9fb4SJan Schmidt break; 11005d9e75c4SJan Schmidt 110128da9fb4SJan Schmidt /* 110228da9fb4SJan Schmidt * if there's an operation that's not a root replacement, we 110328da9fb4SJan Schmidt * found the oldest version of our root. normally, we'll find a 110428da9fb4SJan Schmidt * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here. 110528da9fb4SJan Schmidt */ 11065d9e75c4SJan Schmidt if (tm->op != MOD_LOG_ROOT_REPLACE) 11075d9e75c4SJan Schmidt break; 11085d9e75c4SJan Schmidt 11095d9e75c4SJan Schmidt found = tm; 11105d9e75c4SJan Schmidt root_logical = tm->old_root.logical; 11115d9e75c4SJan Schmidt BUG_ON(root_logical == root->node->start); 11125d9e75c4SJan Schmidt looped = 1; 11135d9e75c4SJan Schmidt } 11145d9e75c4SJan Schmidt 1115a95236d9SJan Schmidt /* if there's no old root to return, return what we found instead */ 1116a95236d9SJan Schmidt if (!found) 1117a95236d9SJan Schmidt found = tm; 1118a95236d9SJan Schmidt 11195d9e75c4SJan Schmidt return found; 11205d9e75c4SJan Schmidt } 11215d9e75c4SJan Schmidt 11225d9e75c4SJan Schmidt /* 11235d9e75c4SJan Schmidt * tm is a pointer to the first operation to rewind within eb. then, all 11245d9e75c4SJan Schmidt * previous operations will be rewinded (until we reach something older than 11255d9e75c4SJan Schmidt * time_seq). 11265d9e75c4SJan Schmidt */ 11275d9e75c4SJan Schmidt static void 11285d9e75c4SJan Schmidt __tree_mod_log_rewind(struct extent_buffer *eb, u64 time_seq, 11295d9e75c4SJan Schmidt struct tree_mod_elem *first_tm) 11305d9e75c4SJan Schmidt { 11315d9e75c4SJan Schmidt u32 n; 11325d9e75c4SJan Schmidt struct rb_node *next; 11335d9e75c4SJan Schmidt struct tree_mod_elem *tm = first_tm; 11345d9e75c4SJan Schmidt unsigned long o_dst; 11355d9e75c4SJan Schmidt unsigned long o_src; 11365d9e75c4SJan Schmidt unsigned long p_size = sizeof(struct btrfs_key_ptr); 11375d9e75c4SJan Schmidt 11385d9e75c4SJan Schmidt n = btrfs_header_nritems(eb); 1139097b8a7cSJan Schmidt while (tm && tm->seq >= time_seq) { 11405d9e75c4SJan Schmidt /* 11415d9e75c4SJan Schmidt * all the operations are recorded with the operator used for 11425d9e75c4SJan Schmidt * the modification. as we're going backwards, we do the 11435d9e75c4SJan Schmidt * opposite of each operation here. 11445d9e75c4SJan Schmidt */ 11455d9e75c4SJan Schmidt switch (tm->op) { 11465d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE_WHILE_FREEING: 11475d9e75c4SJan Schmidt BUG_ON(tm->slot < n); 11485d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE_WHILE_MOVING: 11495d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE: 11505d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 11515d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 11525d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 11535d9e75c4SJan Schmidt tm->generation); 11545d9e75c4SJan Schmidt n++; 11555d9e75c4SJan Schmidt break; 11565d9e75c4SJan Schmidt case MOD_LOG_KEY_REPLACE: 11575d9e75c4SJan Schmidt BUG_ON(tm->slot >= n); 11585d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 11595d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 11605d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 11615d9e75c4SJan Schmidt tm->generation); 11625d9e75c4SJan Schmidt break; 11635d9e75c4SJan Schmidt case MOD_LOG_KEY_ADD: 116419956c7eSJan Schmidt /* if a move operation is needed it's in the log */ 11655d9e75c4SJan Schmidt n--; 11665d9e75c4SJan Schmidt break; 11675d9e75c4SJan Schmidt case MOD_LOG_MOVE_KEYS: 1168c3193108SJan Schmidt o_dst = btrfs_node_key_ptr_offset(tm->slot); 1169c3193108SJan Schmidt o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot); 1170c3193108SJan Schmidt memmove_extent_buffer(eb, o_dst, o_src, 11715d9e75c4SJan Schmidt tm->move.nr_items * p_size); 11725d9e75c4SJan Schmidt break; 11735d9e75c4SJan Schmidt case MOD_LOG_ROOT_REPLACE: 11745d9e75c4SJan Schmidt /* 11755d9e75c4SJan Schmidt * this operation is special. for roots, this must be 11765d9e75c4SJan Schmidt * handled explicitly before rewinding. 11775d9e75c4SJan Schmidt * for non-roots, this operation may exist if the node 11785d9e75c4SJan Schmidt * was a root: root A -> child B; then A gets empty and 11795d9e75c4SJan Schmidt * B is promoted to the new root. in the mod log, we'll 11805d9e75c4SJan Schmidt * have a root-replace operation for B, a tree block 11815d9e75c4SJan Schmidt * that is no root. we simply ignore that operation. 11825d9e75c4SJan Schmidt */ 11835d9e75c4SJan Schmidt break; 11845d9e75c4SJan Schmidt } 11855d9e75c4SJan Schmidt next = rb_next(&tm->node); 11865d9e75c4SJan Schmidt if (!next) 11875d9e75c4SJan Schmidt break; 11885d9e75c4SJan Schmidt tm = container_of(next, struct tree_mod_elem, node); 11895d9e75c4SJan Schmidt if (tm->index != first_tm->index) 11905d9e75c4SJan Schmidt break; 11915d9e75c4SJan Schmidt } 11925d9e75c4SJan Schmidt btrfs_set_header_nritems(eb, n); 11935d9e75c4SJan Schmidt } 11945d9e75c4SJan Schmidt 11955d9e75c4SJan Schmidt static struct extent_buffer * 11965d9e75c4SJan Schmidt tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 11975d9e75c4SJan Schmidt u64 time_seq) 11985d9e75c4SJan Schmidt { 11995d9e75c4SJan Schmidt struct extent_buffer *eb_rewin; 12005d9e75c4SJan Schmidt struct tree_mod_elem *tm; 12015d9e75c4SJan Schmidt 12025d9e75c4SJan Schmidt if (!time_seq) 12035d9e75c4SJan Schmidt return eb; 12045d9e75c4SJan Schmidt 12055d9e75c4SJan Schmidt if (btrfs_header_level(eb) == 0) 12065d9e75c4SJan Schmidt return eb; 12075d9e75c4SJan Schmidt 12085d9e75c4SJan Schmidt tm = tree_mod_log_search(fs_info, eb->start, time_seq); 12095d9e75c4SJan Schmidt if (!tm) 12105d9e75c4SJan Schmidt return eb; 12115d9e75c4SJan Schmidt 12125d9e75c4SJan Schmidt if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 12135d9e75c4SJan Schmidt BUG_ON(tm->slot != 0); 12145d9e75c4SJan Schmidt eb_rewin = alloc_dummy_extent_buffer(eb->start, 12155d9e75c4SJan Schmidt fs_info->tree_root->nodesize); 12165d9e75c4SJan Schmidt BUG_ON(!eb_rewin); 12175d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb_rewin, eb->start); 12185d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb_rewin, 12195d9e75c4SJan Schmidt btrfs_header_backref_rev(eb)); 12205d9e75c4SJan Schmidt btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb)); 1221c3193108SJan Schmidt btrfs_set_header_level(eb_rewin, btrfs_header_level(eb)); 12225d9e75c4SJan Schmidt } else { 12235d9e75c4SJan Schmidt eb_rewin = btrfs_clone_extent_buffer(eb); 12245d9e75c4SJan Schmidt BUG_ON(!eb_rewin); 12255d9e75c4SJan Schmidt } 12265d9e75c4SJan Schmidt 12275d9e75c4SJan Schmidt extent_buffer_get(eb_rewin); 12285d9e75c4SJan Schmidt free_extent_buffer(eb); 12295d9e75c4SJan Schmidt 12305d9e75c4SJan Schmidt __tree_mod_log_rewind(eb_rewin, time_seq, tm); 12315d9e75c4SJan Schmidt 12325d9e75c4SJan Schmidt return eb_rewin; 12335d9e75c4SJan Schmidt } 12345d9e75c4SJan Schmidt 12358ba97a15SJan Schmidt /* 12368ba97a15SJan Schmidt * get_old_root() rewinds the state of @root's root node to the given @time_seq 12378ba97a15SJan Schmidt * value. If there are no changes, the current root->root_node is returned. If 12388ba97a15SJan Schmidt * anything changed in between, there's a fresh buffer allocated on which the 12398ba97a15SJan Schmidt * rewind operations are done. In any case, the returned buffer is read locked. 12408ba97a15SJan Schmidt * Returns NULL on error (with no locks held). 12418ba97a15SJan Schmidt */ 12425d9e75c4SJan Schmidt static inline struct extent_buffer * 12435d9e75c4SJan Schmidt get_old_root(struct btrfs_root *root, u64 time_seq) 12445d9e75c4SJan Schmidt { 12455d9e75c4SJan Schmidt struct tree_mod_elem *tm; 12465d9e75c4SJan Schmidt struct extent_buffer *eb; 1247a95236d9SJan Schmidt struct tree_mod_root *old_root = NULL; 12484325edd0SChris Mason u64 old_generation = 0; 1249a95236d9SJan Schmidt u64 logical; 12505d9e75c4SJan Schmidt 12518ba97a15SJan Schmidt eb = btrfs_read_lock_root_node(root); 12525d9e75c4SJan Schmidt tm = __tree_mod_log_oldest_root(root->fs_info, root, time_seq); 12535d9e75c4SJan Schmidt if (!tm) 12545d9e75c4SJan Schmidt return root->node; 12555d9e75c4SJan Schmidt 1256a95236d9SJan Schmidt if (tm->op == MOD_LOG_ROOT_REPLACE) { 12575d9e75c4SJan Schmidt old_root = &tm->old_root; 12585d9e75c4SJan Schmidt old_generation = tm->generation; 1259a95236d9SJan Schmidt logical = old_root->logical; 1260a95236d9SJan Schmidt } else { 1261a95236d9SJan Schmidt logical = root->node->start; 1262a95236d9SJan Schmidt } 12635d9e75c4SJan Schmidt 1264a95236d9SJan Schmidt tm = tree_mod_log_search(root->fs_info, logical, time_seq); 1265a95236d9SJan Schmidt if (old_root) 126628da9fb4SJan Schmidt eb = alloc_dummy_extent_buffer(logical, root->nodesize); 1267a95236d9SJan Schmidt else 1268a95236d9SJan Schmidt eb = btrfs_clone_extent_buffer(root->node); 12698ba97a15SJan Schmidt btrfs_tree_read_unlock(root->node); 12708ba97a15SJan Schmidt free_extent_buffer(root->node); 12718ba97a15SJan Schmidt if (!eb) 12728ba97a15SJan Schmidt return NULL; 12738ba97a15SJan Schmidt btrfs_tree_read_lock(eb); 1274a95236d9SJan Schmidt if (old_root) { 12755d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb, eb->start); 12765d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV); 12775d9e75c4SJan Schmidt btrfs_set_header_owner(eb, root->root_key.objectid); 12785d9e75c4SJan Schmidt btrfs_set_header_level(eb, old_root->level); 12795d9e75c4SJan Schmidt btrfs_set_header_generation(eb, old_generation); 1280a95236d9SJan Schmidt } 128128da9fb4SJan Schmidt if (tm) 12825d9e75c4SJan Schmidt __tree_mod_log_rewind(eb, time_seq, tm); 128328da9fb4SJan Schmidt else 128428da9fb4SJan Schmidt WARN_ON(btrfs_header_level(eb) != 0); 12858ba97a15SJan Schmidt extent_buffer_get(eb); 12865d9e75c4SJan Schmidt 12875d9e75c4SJan Schmidt return eb; 12885d9e75c4SJan Schmidt } 12895d9e75c4SJan Schmidt 12905d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 12915d4f98a2SYan Zheng struct btrfs_root *root, 12925d4f98a2SYan Zheng struct extent_buffer *buf) 12935d4f98a2SYan Zheng { 1294f1ebcc74SLiu Bo /* ensure we can see the force_cow */ 1295f1ebcc74SLiu Bo smp_rmb(); 1296f1ebcc74SLiu Bo 1297f1ebcc74SLiu Bo /* 1298f1ebcc74SLiu Bo * We do not need to cow a block if 1299f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 1300f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 1301f1ebcc74SLiu Bo * 3) the root is not forced COW. 1302f1ebcc74SLiu Bo * 1303f1ebcc74SLiu Bo * What is forced COW: 1304f1ebcc74SLiu Bo * when we create snapshot during commiting the transaction, 1305f1ebcc74SLiu Bo * after we've finished coping src root, we must COW the shared 1306f1ebcc74SLiu Bo * block to ensure the metadata consistency. 1307f1ebcc74SLiu Bo */ 13085d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 13095d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 13105d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 1311f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 1312f1ebcc74SLiu Bo !root->force_cow) 13135d4f98a2SYan Zheng return 0; 13145d4f98a2SYan Zheng return 1; 13155d4f98a2SYan Zheng } 13165d4f98a2SYan Zheng 1317d352ac68SChris Mason /* 1318d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 1319d352ac68SChris Mason * This version of it has extra checks so that a block isn't cow'd more than 1320d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 1321d352ac68SChris Mason */ 1322d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 13235f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 13245f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 13259fa8cfe7SChris Mason struct extent_buffer **cow_ret) 132602217ed2SChris Mason { 13276702ed49SChris Mason u64 search_start; 1328f510cfecSChris Mason int ret; 1329dc17ff8fSChris Mason 1330ccd467d6SChris Mason if (trans->transaction != root->fs_info->running_transaction) { 1331d397712bSChris Mason printk(KERN_CRIT "trans %llu running %llu\n", 1332d397712bSChris Mason (unsigned long long)trans->transid, 1333d397712bSChris Mason (unsigned long long) 1334ccd467d6SChris Mason root->fs_info->running_transaction->transid); 1335ccd467d6SChris Mason WARN_ON(1); 1336ccd467d6SChris Mason } 1337ccd467d6SChris Mason if (trans->transid != root->fs_info->generation) { 1338d397712bSChris Mason printk(KERN_CRIT "trans %llu running %llu\n", 1339d397712bSChris Mason (unsigned long long)trans->transid, 1340d397712bSChris Mason (unsigned long long)root->fs_info->generation); 1341ccd467d6SChris Mason WARN_ON(1); 1342ccd467d6SChris Mason } 1343dc17ff8fSChris Mason 13445d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 134502217ed2SChris Mason *cow_ret = buf; 134602217ed2SChris Mason return 0; 134702217ed2SChris Mason } 1348c487685dSChris Mason 13490b86a832SChris Mason search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1); 1350b4ce94deSChris Mason 1351b4ce94deSChris Mason if (parent) 1352b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1353b4ce94deSChris Mason btrfs_set_lock_blocking(buf); 1354b4ce94deSChris Mason 1355f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 13569fa8cfe7SChris Mason parent_slot, cow_ret, search_start, 0); 13571abe9b8aSliubo 13581abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 13591abe9b8aSliubo 1360f510cfecSChris Mason return ret; 13612c90e5d6SChris Mason } 13626702ed49SChris Mason 1363d352ac68SChris Mason /* 1364d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 1365d352ac68SChris Mason * node are actually close by 1366d352ac68SChris Mason */ 13676b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 13686702ed49SChris Mason { 13696b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 13706702ed49SChris Mason return 1; 13716b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 13726702ed49SChris Mason return 1; 137302217ed2SChris Mason return 0; 137402217ed2SChris Mason } 137502217ed2SChris Mason 1376081e9573SChris Mason /* 1377081e9573SChris Mason * compare two keys in a memcmp fashion 1378081e9573SChris Mason */ 1379081e9573SChris Mason static int comp_keys(struct btrfs_disk_key *disk, struct btrfs_key *k2) 1380081e9573SChris Mason { 1381081e9573SChris Mason struct btrfs_key k1; 1382081e9573SChris Mason 1383081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 1384081e9573SChris Mason 138520736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 1386081e9573SChris Mason } 1387081e9573SChris Mason 1388f3465ca4SJosef Bacik /* 1389f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 1390f3465ca4SJosef Bacik */ 13915d4f98a2SYan Zheng int btrfs_comp_cpu_keys(struct btrfs_key *k1, struct btrfs_key *k2) 1392f3465ca4SJosef Bacik { 1393f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 1394f3465ca4SJosef Bacik return 1; 1395f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 1396f3465ca4SJosef Bacik return -1; 1397f3465ca4SJosef Bacik if (k1->type > k2->type) 1398f3465ca4SJosef Bacik return 1; 1399f3465ca4SJosef Bacik if (k1->type < k2->type) 1400f3465ca4SJosef Bacik return -1; 1401f3465ca4SJosef Bacik if (k1->offset > k2->offset) 1402f3465ca4SJosef Bacik return 1; 1403f3465ca4SJosef Bacik if (k1->offset < k2->offset) 1404f3465ca4SJosef Bacik return -1; 1405f3465ca4SJosef Bacik return 0; 1406f3465ca4SJosef Bacik } 1407081e9573SChris Mason 1408d352ac68SChris Mason /* 1409d352ac68SChris Mason * this is used by the defrag code to go through all the 1410d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 1411d352ac68SChris Mason * disk order is close to key order 1412d352ac68SChris Mason */ 14136702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 14145f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 1415a6b6e75eSChris Mason int start_slot, int cache_only, u64 *last_ret, 1416a6b6e75eSChris Mason struct btrfs_key *progress) 14176702ed49SChris Mason { 14186b80053dSChris Mason struct extent_buffer *cur; 14196702ed49SChris Mason u64 blocknr; 1420ca7a79adSChris Mason u64 gen; 1421e9d0b13bSChris Mason u64 search_start = *last_ret; 1422e9d0b13bSChris Mason u64 last_block = 0; 14236702ed49SChris Mason u64 other; 14246702ed49SChris Mason u32 parent_nritems; 14256702ed49SChris Mason int end_slot; 14266702ed49SChris Mason int i; 14276702ed49SChris Mason int err = 0; 1428f2183bdeSChris Mason int parent_level; 14296b80053dSChris Mason int uptodate; 14306b80053dSChris Mason u32 blocksize; 1431081e9573SChris Mason int progress_passed = 0; 1432081e9573SChris Mason struct btrfs_disk_key disk_key; 14336702ed49SChris Mason 14345708b959SChris Mason parent_level = btrfs_header_level(parent); 14355708b959SChris Mason if (cache_only && parent_level != 1) 14365708b959SChris Mason return 0; 14375708b959SChris Mason 1438d397712bSChris Mason if (trans->transaction != root->fs_info->running_transaction) 14396702ed49SChris Mason WARN_ON(1); 1440d397712bSChris Mason if (trans->transid != root->fs_info->generation) 14416702ed49SChris Mason WARN_ON(1); 144286479a04SChris Mason 14436b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 14446b80053dSChris Mason blocksize = btrfs_level_size(root, parent_level - 1); 14456702ed49SChris Mason end_slot = parent_nritems; 14466702ed49SChris Mason 14476702ed49SChris Mason if (parent_nritems == 1) 14486702ed49SChris Mason return 0; 14496702ed49SChris Mason 1450b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1451b4ce94deSChris Mason 14526702ed49SChris Mason for (i = start_slot; i < end_slot; i++) { 14536702ed49SChris Mason int close = 1; 1454a6b6e75eSChris Mason 1455081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 1456081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 1457081e9573SChris Mason continue; 1458081e9573SChris Mason 1459081e9573SChris Mason progress_passed = 1; 14606b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 1461ca7a79adSChris Mason gen = btrfs_node_ptr_generation(parent, i); 1462e9d0b13bSChris Mason if (last_block == 0) 1463e9d0b13bSChris Mason last_block = blocknr; 14645708b959SChris Mason 14656702ed49SChris Mason if (i > 0) { 14666b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 14676b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 14686702ed49SChris Mason } 14690ef3e66bSChris Mason if (!close && i < end_slot - 2) { 14706b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 14716b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 14726702ed49SChris Mason } 1473e9d0b13bSChris Mason if (close) { 1474e9d0b13bSChris Mason last_block = blocknr; 14756702ed49SChris Mason continue; 1476e9d0b13bSChris Mason } 14776702ed49SChris Mason 14786b80053dSChris Mason cur = btrfs_find_tree_block(root, blocknr, blocksize); 14796b80053dSChris Mason if (cur) 1480b9fab919SChris Mason uptodate = btrfs_buffer_uptodate(cur, gen, 0); 14816b80053dSChris Mason else 14826b80053dSChris Mason uptodate = 0; 14835708b959SChris Mason if (!cur || !uptodate) { 14846702ed49SChris Mason if (cache_only) { 14856b80053dSChris Mason free_extent_buffer(cur); 14866702ed49SChris Mason continue; 14876702ed49SChris Mason } 14886b80053dSChris Mason if (!cur) { 14896b80053dSChris Mason cur = read_tree_block(root, blocknr, 1490ca7a79adSChris Mason blocksize, gen); 149197d9a8a4STsutomu Itoh if (!cur) 149297d9a8a4STsutomu Itoh return -EIO; 14936b80053dSChris Mason } else if (!uptodate) { 1494018642a1STsutomu Itoh err = btrfs_read_buffer(cur, gen); 1495018642a1STsutomu Itoh if (err) { 1496018642a1STsutomu Itoh free_extent_buffer(cur); 1497018642a1STsutomu Itoh return err; 1498018642a1STsutomu Itoh } 14996702ed49SChris Mason } 1500f2183bdeSChris Mason } 1501e9d0b13bSChris Mason if (search_start == 0) 15026b80053dSChris Mason search_start = last_block; 1503e9d0b13bSChris Mason 1504e7a84565SChris Mason btrfs_tree_lock(cur); 1505b4ce94deSChris Mason btrfs_set_lock_blocking(cur); 15066b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 1507e7a84565SChris Mason &cur, search_start, 15086b80053dSChris Mason min(16 * blocksize, 15099fa8cfe7SChris Mason (end_slot - i) * blocksize)); 1510252c38f0SYan if (err) { 1511e7a84565SChris Mason btrfs_tree_unlock(cur); 15126b80053dSChris Mason free_extent_buffer(cur); 15136702ed49SChris Mason break; 1514252c38f0SYan } 1515e7a84565SChris Mason search_start = cur->start; 1516e7a84565SChris Mason last_block = cur->start; 1517f2183bdeSChris Mason *last_ret = search_start; 1518e7a84565SChris Mason btrfs_tree_unlock(cur); 1519e7a84565SChris Mason free_extent_buffer(cur); 15206702ed49SChris Mason } 15216702ed49SChris Mason return err; 15226702ed49SChris Mason } 15236702ed49SChris Mason 152474123bd7SChris Mason /* 152574123bd7SChris Mason * The leaf data grows from end-to-front in the node. 152674123bd7SChris Mason * this returns the address of the start of the last item, 152774123bd7SChris Mason * which is the stop of the leaf data stack 152874123bd7SChris Mason */ 1529123abc88SChris Mason static inline unsigned int leaf_data_end(struct btrfs_root *root, 15305f39d397SChris Mason struct extent_buffer *leaf) 1531be0e5c09SChris Mason { 15325f39d397SChris Mason u32 nr = btrfs_header_nritems(leaf); 1533be0e5c09SChris Mason if (nr == 0) 1534123abc88SChris Mason return BTRFS_LEAF_DATA_SIZE(root); 15355f39d397SChris Mason return btrfs_item_offset_nr(leaf, nr - 1); 1536be0e5c09SChris Mason } 1537be0e5c09SChris Mason 1538aa5d6bedSChris Mason 153974123bd7SChris Mason /* 15405f39d397SChris Mason * search for key in the extent_buffer. The items start at offset p, 15415f39d397SChris Mason * and they are item_size apart. There are 'max' items in p. 15425f39d397SChris Mason * 154374123bd7SChris Mason * the slot in the array is returned via slot, and it points to 154474123bd7SChris Mason * the place where you would insert key if it is not found in 154574123bd7SChris Mason * the array. 154674123bd7SChris Mason * 154774123bd7SChris Mason * slot may point to max if the key is bigger than all of the keys 154874123bd7SChris Mason */ 1549e02119d5SChris Mason static noinline int generic_bin_search(struct extent_buffer *eb, 1550e02119d5SChris Mason unsigned long p, 15515f39d397SChris Mason int item_size, struct btrfs_key *key, 1552be0e5c09SChris Mason int max, int *slot) 1553be0e5c09SChris Mason { 1554be0e5c09SChris Mason int low = 0; 1555be0e5c09SChris Mason int high = max; 1556be0e5c09SChris Mason int mid; 1557be0e5c09SChris Mason int ret; 1558479965d6SChris Mason struct btrfs_disk_key *tmp = NULL; 15595f39d397SChris Mason struct btrfs_disk_key unaligned; 15605f39d397SChris Mason unsigned long offset; 15615f39d397SChris Mason char *kaddr = NULL; 15625f39d397SChris Mason unsigned long map_start = 0; 15635f39d397SChris Mason unsigned long map_len = 0; 1564479965d6SChris Mason int err; 1565be0e5c09SChris Mason 1566be0e5c09SChris Mason while (low < high) { 1567be0e5c09SChris Mason mid = (low + high) / 2; 15685f39d397SChris Mason offset = p + mid * item_size; 15695f39d397SChris Mason 1570a6591715SChris Mason if (!kaddr || offset < map_start || 15715f39d397SChris Mason (offset + sizeof(struct btrfs_disk_key)) > 15725f39d397SChris Mason map_start + map_len) { 1573934d375bSChris Mason 1574934d375bSChris Mason err = map_private_extent_buffer(eb, offset, 1575479965d6SChris Mason sizeof(struct btrfs_disk_key), 1576a6591715SChris Mason &kaddr, &map_start, &map_len); 15775f39d397SChris Mason 1578479965d6SChris Mason if (!err) { 1579479965d6SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 1580479965d6SChris Mason map_start); 1581479965d6SChris Mason } else { 15825f39d397SChris Mason read_extent_buffer(eb, &unaligned, 15835f39d397SChris Mason offset, sizeof(unaligned)); 15845f39d397SChris Mason tmp = &unaligned; 1585479965d6SChris Mason } 1586479965d6SChris Mason 15875f39d397SChris Mason } else { 15885f39d397SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 15895f39d397SChris Mason map_start); 15905f39d397SChris Mason } 1591be0e5c09SChris Mason ret = comp_keys(tmp, key); 1592be0e5c09SChris Mason 1593be0e5c09SChris Mason if (ret < 0) 1594be0e5c09SChris Mason low = mid + 1; 1595be0e5c09SChris Mason else if (ret > 0) 1596be0e5c09SChris Mason high = mid; 1597be0e5c09SChris Mason else { 1598be0e5c09SChris Mason *slot = mid; 1599be0e5c09SChris Mason return 0; 1600be0e5c09SChris Mason } 1601be0e5c09SChris Mason } 1602be0e5c09SChris Mason *slot = low; 1603be0e5c09SChris Mason return 1; 1604be0e5c09SChris Mason } 1605be0e5c09SChris Mason 160697571fd0SChris Mason /* 160797571fd0SChris Mason * simple bin_search frontend that does the right thing for 160897571fd0SChris Mason * leaves vs nodes 160997571fd0SChris Mason */ 16105f39d397SChris Mason static int bin_search(struct extent_buffer *eb, struct btrfs_key *key, 16115f39d397SChris Mason int level, int *slot) 1612be0e5c09SChris Mason { 1613f775738fSWang Sheng-Hui if (level == 0) 16145f39d397SChris Mason return generic_bin_search(eb, 16155f39d397SChris Mason offsetof(struct btrfs_leaf, items), 16160783fcfcSChris Mason sizeof(struct btrfs_item), 16175f39d397SChris Mason key, btrfs_header_nritems(eb), 16187518a238SChris Mason slot); 1619f775738fSWang Sheng-Hui else 16205f39d397SChris Mason return generic_bin_search(eb, 16215f39d397SChris Mason offsetof(struct btrfs_node, ptrs), 1622123abc88SChris Mason sizeof(struct btrfs_key_ptr), 16235f39d397SChris Mason key, btrfs_header_nritems(eb), 16247518a238SChris Mason slot); 1625be0e5c09SChris Mason } 1626be0e5c09SChris Mason 16275d4f98a2SYan Zheng int btrfs_bin_search(struct extent_buffer *eb, struct btrfs_key *key, 16285d4f98a2SYan Zheng int level, int *slot) 16295d4f98a2SYan Zheng { 16305d4f98a2SYan Zheng return bin_search(eb, key, level, slot); 16315d4f98a2SYan Zheng } 16325d4f98a2SYan Zheng 1633f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 1634f0486c68SYan, Zheng { 1635f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1636f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1637f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 1638f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1639f0486c68SYan, Zheng } 1640f0486c68SYan, Zheng 1641f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 1642f0486c68SYan, Zheng { 1643f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1644f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1645f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 1646f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1647f0486c68SYan, Zheng } 1648f0486c68SYan, Zheng 1649d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 1650d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 1651d352ac68SChris Mason * NULL is returned on error. 1652d352ac68SChris Mason */ 1653e02119d5SChris Mason static noinline struct extent_buffer *read_node_slot(struct btrfs_root *root, 16545f39d397SChris Mason struct extent_buffer *parent, int slot) 1655bb803951SChris Mason { 1656ca7a79adSChris Mason int level = btrfs_header_level(parent); 1657bb803951SChris Mason if (slot < 0) 1658bb803951SChris Mason return NULL; 16595f39d397SChris Mason if (slot >= btrfs_header_nritems(parent)) 1660bb803951SChris Mason return NULL; 1661ca7a79adSChris Mason 1662ca7a79adSChris Mason BUG_ON(level == 0); 1663ca7a79adSChris Mason 1664db94535dSChris Mason return read_tree_block(root, btrfs_node_blockptr(parent, slot), 1665ca7a79adSChris Mason btrfs_level_size(root, level - 1), 1666ca7a79adSChris Mason btrfs_node_ptr_generation(parent, slot)); 1667bb803951SChris Mason } 1668bb803951SChris Mason 1669d352ac68SChris Mason /* 1670d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 1671d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 1672d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 1673d352ac68SChris Mason */ 1674e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 167598ed5174SChris Mason struct btrfs_root *root, 167698ed5174SChris Mason struct btrfs_path *path, int level) 1677bb803951SChris Mason { 16785f39d397SChris Mason struct extent_buffer *right = NULL; 16795f39d397SChris Mason struct extent_buffer *mid; 16805f39d397SChris Mason struct extent_buffer *left = NULL; 16815f39d397SChris Mason struct extent_buffer *parent = NULL; 1682bb803951SChris Mason int ret = 0; 1683bb803951SChris Mason int wret; 1684bb803951SChris Mason int pslot; 1685bb803951SChris Mason int orig_slot = path->slots[level]; 168679f95c82SChris Mason u64 orig_ptr; 1687bb803951SChris Mason 1688bb803951SChris Mason if (level == 0) 1689bb803951SChris Mason return 0; 1690bb803951SChris Mason 16915f39d397SChris Mason mid = path->nodes[level]; 1692b4ce94deSChris Mason 1693bd681513SChris Mason WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK && 1694bd681513SChris Mason path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING); 16957bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 16967bb86316SChris Mason 16971d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 169879f95c82SChris Mason 1699a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 17005f39d397SChris Mason parent = path->nodes[level + 1]; 1701bb803951SChris Mason pslot = path->slots[level + 1]; 1702a05a9bb1SLi Zefan } 1703bb803951SChris Mason 170440689478SChris Mason /* 170540689478SChris Mason * deal with the case where there is only one pointer in the root 170640689478SChris Mason * by promoting the node below to a root 170740689478SChris Mason */ 17085f39d397SChris Mason if (!parent) { 17095f39d397SChris Mason struct extent_buffer *child; 1710bb803951SChris Mason 17115f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1712bb803951SChris Mason return 0; 1713bb803951SChris Mason 1714bb803951SChris Mason /* promote the child to a root */ 17155f39d397SChris Mason child = read_node_slot(root, mid, 0); 1716305a26afSMark Fasheh if (!child) { 1717305a26afSMark Fasheh ret = -EROFS; 1718305a26afSMark Fasheh btrfs_std_error(root->fs_info, ret); 1719305a26afSMark Fasheh goto enospc; 1720305a26afSMark Fasheh } 1721305a26afSMark Fasheh 1722925baeddSChris Mason btrfs_tree_lock(child); 1723b4ce94deSChris Mason btrfs_set_lock_blocking(child); 17249fa8cfe7SChris Mason ret = btrfs_cow_block(trans, root, child, mid, 0, &child); 1725f0486c68SYan, Zheng if (ret) { 1726f0486c68SYan, Zheng btrfs_tree_unlock(child); 1727f0486c68SYan, Zheng free_extent_buffer(child); 1728f0486c68SYan, Zheng goto enospc; 1729f0486c68SYan, Zheng } 17302f375ab9SYan 1731f230475eSJan Schmidt tree_mod_log_set_root_pointer(root, child); 1732240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1733925baeddSChris Mason 17340b86a832SChris Mason add_root_to_dirty_list(root); 1735925baeddSChris Mason btrfs_tree_unlock(child); 1736b4ce94deSChris Mason 1737925baeddSChris Mason path->locks[level] = 0; 1738bb803951SChris Mason path->nodes[level] = NULL; 17395f39d397SChris Mason clean_tree_block(trans, root, mid); 1740925baeddSChris Mason btrfs_tree_unlock(mid); 1741bb803951SChris Mason /* once for the path */ 17425f39d397SChris Mason free_extent_buffer(mid); 1743f0486c68SYan, Zheng 1744f0486c68SYan, Zheng root_sub_used(root, mid->len); 17455581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 1746bb803951SChris Mason /* once for the root ptr */ 17473083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1748f0486c68SYan, Zheng return 0; 1749bb803951SChris Mason } 17505f39d397SChris Mason if (btrfs_header_nritems(mid) > 1751123abc88SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) / 4) 1752bb803951SChris Mason return 0; 1753bb803951SChris Mason 17545f39d397SChris Mason left = read_node_slot(root, parent, pslot - 1); 17555f39d397SChris Mason if (left) { 1756925baeddSChris Mason btrfs_tree_lock(left); 1757b4ce94deSChris Mason btrfs_set_lock_blocking(left); 17585f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 17599fa8cfe7SChris Mason parent, pslot - 1, &left); 176054aa1f4dSChris Mason if (wret) { 176154aa1f4dSChris Mason ret = wret; 176254aa1f4dSChris Mason goto enospc; 176354aa1f4dSChris Mason } 17642cc58cf2SChris Mason } 17655f39d397SChris Mason right = read_node_slot(root, parent, pslot + 1); 17665f39d397SChris Mason if (right) { 1767925baeddSChris Mason btrfs_tree_lock(right); 1768b4ce94deSChris Mason btrfs_set_lock_blocking(right); 17695f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 17709fa8cfe7SChris Mason parent, pslot + 1, &right); 17712cc58cf2SChris Mason if (wret) { 17722cc58cf2SChris Mason ret = wret; 17732cc58cf2SChris Mason goto enospc; 17742cc58cf2SChris Mason } 17752cc58cf2SChris Mason } 17762cc58cf2SChris Mason 17772cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 17785f39d397SChris Mason if (left) { 17795f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1780bce4eae9SChris Mason wret = push_node_left(trans, root, left, mid, 1); 178179f95c82SChris Mason if (wret < 0) 178279f95c82SChris Mason ret = wret; 1783bb803951SChris Mason } 178479f95c82SChris Mason 178579f95c82SChris Mason /* 178679f95c82SChris Mason * then try to empty the right most buffer into the middle 178779f95c82SChris Mason */ 17885f39d397SChris Mason if (right) { 1789971a1f66SChris Mason wret = push_node_left(trans, root, mid, right, 1); 179054aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 179179f95c82SChris Mason ret = wret; 17925f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 17935f39d397SChris Mason clean_tree_block(trans, root, right); 1794925baeddSChris Mason btrfs_tree_unlock(right); 1795f3ea38daSJan Schmidt del_ptr(trans, root, path, level + 1, pslot + 1, 1); 1796f0486c68SYan, Zheng root_sub_used(root, right->len); 17975581a51aSJan Schmidt btrfs_free_tree_block(trans, root, right, 0, 1); 17983083ee2eSJosef Bacik free_extent_buffer_stale(right); 1799f0486c68SYan, Zheng right = NULL; 1800bb803951SChris Mason } else { 18015f39d397SChris Mason struct btrfs_disk_key right_key; 18025f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1803f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 1804f230475eSJan Schmidt &right_key, pslot + 1, 0); 18055f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 18065f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1807bb803951SChris Mason } 1808bb803951SChris Mason } 18095f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 181079f95c82SChris Mason /* 181179f95c82SChris Mason * we're not allowed to leave a node with one item in the 181279f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 181379f95c82SChris Mason * could try to delete the only pointer in this node. 181479f95c82SChris Mason * So, pull some keys from the left. 181579f95c82SChris Mason * There has to be a left pointer at this point because 181679f95c82SChris Mason * otherwise we would have pulled some pointers from the 181779f95c82SChris Mason * right 181879f95c82SChris Mason */ 1819305a26afSMark Fasheh if (!left) { 1820305a26afSMark Fasheh ret = -EROFS; 1821305a26afSMark Fasheh btrfs_std_error(root->fs_info, ret); 1822305a26afSMark Fasheh goto enospc; 1823305a26afSMark Fasheh } 18245f39d397SChris Mason wret = balance_node_right(trans, root, mid, left); 182554aa1f4dSChris Mason if (wret < 0) { 182679f95c82SChris Mason ret = wret; 182754aa1f4dSChris Mason goto enospc; 182854aa1f4dSChris Mason } 1829bce4eae9SChris Mason if (wret == 1) { 1830bce4eae9SChris Mason wret = push_node_left(trans, root, left, mid, 1); 1831bce4eae9SChris Mason if (wret < 0) 1832bce4eae9SChris Mason ret = wret; 1833bce4eae9SChris Mason } 183479f95c82SChris Mason BUG_ON(wret == 1); 183579f95c82SChris Mason } 18365f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 18375f39d397SChris Mason clean_tree_block(trans, root, mid); 1838925baeddSChris Mason btrfs_tree_unlock(mid); 1839f3ea38daSJan Schmidt del_ptr(trans, root, path, level + 1, pslot, 1); 1840f0486c68SYan, Zheng root_sub_used(root, mid->len); 18415581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 18423083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1843f0486c68SYan, Zheng mid = NULL; 184479f95c82SChris Mason } else { 184579f95c82SChris Mason /* update the parent key to reflect our changes */ 18465f39d397SChris Mason struct btrfs_disk_key mid_key; 18475f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1848f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, &mid_key, 1849f230475eSJan Schmidt pslot, 0); 18505f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 18515f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 185279f95c82SChris Mason } 1853bb803951SChris Mason 185479f95c82SChris Mason /* update the path */ 18555f39d397SChris Mason if (left) { 18565f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 18575f39d397SChris Mason extent_buffer_get(left); 1858925baeddSChris Mason /* left was locked after cow */ 18595f39d397SChris Mason path->nodes[level] = left; 1860bb803951SChris Mason path->slots[level + 1] -= 1; 1861bb803951SChris Mason path->slots[level] = orig_slot; 1862925baeddSChris Mason if (mid) { 1863925baeddSChris Mason btrfs_tree_unlock(mid); 18645f39d397SChris Mason free_extent_buffer(mid); 1865925baeddSChris Mason } 1866bb803951SChris Mason } else { 18675f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1868bb803951SChris Mason path->slots[level] = orig_slot; 1869bb803951SChris Mason } 1870bb803951SChris Mason } 187179f95c82SChris Mason /* double check we haven't messed things up */ 1872e20d96d6SChris Mason if (orig_ptr != 18735f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 187479f95c82SChris Mason BUG(); 187554aa1f4dSChris Mason enospc: 1876925baeddSChris Mason if (right) { 1877925baeddSChris Mason btrfs_tree_unlock(right); 18785f39d397SChris Mason free_extent_buffer(right); 1879925baeddSChris Mason } 1880925baeddSChris Mason if (left) { 1881925baeddSChris Mason if (path->nodes[level] != left) 1882925baeddSChris Mason btrfs_tree_unlock(left); 18835f39d397SChris Mason free_extent_buffer(left); 1884925baeddSChris Mason } 1885bb803951SChris Mason return ret; 1886bb803951SChris Mason } 1887bb803951SChris Mason 1888d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1889d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1890d352ac68SChris Mason * have to be pessimistic. 1891d352ac68SChris Mason */ 1892d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1893e66f709bSChris Mason struct btrfs_root *root, 1894e66f709bSChris Mason struct btrfs_path *path, int level) 1895e66f709bSChris Mason { 18965f39d397SChris Mason struct extent_buffer *right = NULL; 18975f39d397SChris Mason struct extent_buffer *mid; 18985f39d397SChris Mason struct extent_buffer *left = NULL; 18995f39d397SChris Mason struct extent_buffer *parent = NULL; 1900e66f709bSChris Mason int ret = 0; 1901e66f709bSChris Mason int wret; 1902e66f709bSChris Mason int pslot; 1903e66f709bSChris Mason int orig_slot = path->slots[level]; 1904e66f709bSChris Mason 1905e66f709bSChris Mason if (level == 0) 1906e66f709bSChris Mason return 1; 1907e66f709bSChris Mason 19085f39d397SChris Mason mid = path->nodes[level]; 19097bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1910e66f709bSChris Mason 1911a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 19125f39d397SChris Mason parent = path->nodes[level + 1]; 1913e66f709bSChris Mason pslot = path->slots[level + 1]; 1914a05a9bb1SLi Zefan } 1915e66f709bSChris Mason 19165f39d397SChris Mason if (!parent) 1917e66f709bSChris Mason return 1; 1918e66f709bSChris Mason 19195f39d397SChris Mason left = read_node_slot(root, parent, pslot - 1); 1920e66f709bSChris Mason 1921e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 19225f39d397SChris Mason if (left) { 1923e66f709bSChris Mason u32 left_nr; 1924925baeddSChris Mason 1925925baeddSChris Mason btrfs_tree_lock(left); 1926b4ce94deSChris Mason btrfs_set_lock_blocking(left); 1927b4ce94deSChris Mason 19285f39d397SChris Mason left_nr = btrfs_header_nritems(left); 192933ade1f8SChris Mason if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) { 193033ade1f8SChris Mason wret = 1; 193133ade1f8SChris Mason } else { 19325f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 19339fa8cfe7SChris Mason pslot - 1, &left); 193454aa1f4dSChris Mason if (ret) 193554aa1f4dSChris Mason wret = 1; 193654aa1f4dSChris Mason else { 193754aa1f4dSChris Mason wret = push_node_left(trans, root, 1938971a1f66SChris Mason left, mid, 0); 193954aa1f4dSChris Mason } 194033ade1f8SChris Mason } 1941e66f709bSChris Mason if (wret < 0) 1942e66f709bSChris Mason ret = wret; 1943e66f709bSChris Mason if (wret == 0) { 19445f39d397SChris Mason struct btrfs_disk_key disk_key; 1945e66f709bSChris Mason orig_slot += left_nr; 19465f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1947f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 1948f230475eSJan Schmidt &disk_key, pslot, 0); 19495f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 19505f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 19515f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 19525f39d397SChris Mason path->nodes[level] = left; 1953e66f709bSChris Mason path->slots[level + 1] -= 1; 1954e66f709bSChris Mason path->slots[level] = orig_slot; 1955925baeddSChris Mason btrfs_tree_unlock(mid); 19565f39d397SChris Mason free_extent_buffer(mid); 1957e66f709bSChris Mason } else { 1958e66f709bSChris Mason orig_slot -= 19595f39d397SChris Mason btrfs_header_nritems(left); 1960e66f709bSChris Mason path->slots[level] = orig_slot; 1961925baeddSChris Mason btrfs_tree_unlock(left); 19625f39d397SChris Mason free_extent_buffer(left); 1963e66f709bSChris Mason } 1964e66f709bSChris Mason return 0; 1965e66f709bSChris Mason } 1966925baeddSChris Mason btrfs_tree_unlock(left); 19675f39d397SChris Mason free_extent_buffer(left); 1968e66f709bSChris Mason } 19695f39d397SChris Mason right = read_node_slot(root, parent, pslot + 1); 1970e66f709bSChris Mason 1971e66f709bSChris Mason /* 1972e66f709bSChris Mason * then try to empty the right most buffer into the middle 1973e66f709bSChris Mason */ 19745f39d397SChris Mason if (right) { 197533ade1f8SChris Mason u32 right_nr; 1976b4ce94deSChris Mason 1977925baeddSChris Mason btrfs_tree_lock(right); 1978b4ce94deSChris Mason btrfs_set_lock_blocking(right); 1979b4ce94deSChris Mason 19805f39d397SChris Mason right_nr = btrfs_header_nritems(right); 198133ade1f8SChris Mason if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) { 198233ade1f8SChris Mason wret = 1; 198333ade1f8SChris Mason } else { 19845f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 19855f39d397SChris Mason parent, pslot + 1, 19869fa8cfe7SChris Mason &right); 198754aa1f4dSChris Mason if (ret) 198854aa1f4dSChris Mason wret = 1; 198954aa1f4dSChris Mason else { 199033ade1f8SChris Mason wret = balance_node_right(trans, root, 19915f39d397SChris Mason right, mid); 199233ade1f8SChris Mason } 199354aa1f4dSChris Mason } 1994e66f709bSChris Mason if (wret < 0) 1995e66f709bSChris Mason ret = wret; 1996e66f709bSChris Mason if (wret == 0) { 19975f39d397SChris Mason struct btrfs_disk_key disk_key; 19985f39d397SChris Mason 19995f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 2000f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 2001f230475eSJan Schmidt &disk_key, pslot + 1, 0); 20025f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 20035f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 20045f39d397SChris Mason 20055f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 20065f39d397SChris Mason path->nodes[level] = right; 2007e66f709bSChris Mason path->slots[level + 1] += 1; 2008e66f709bSChris Mason path->slots[level] = orig_slot - 20095f39d397SChris Mason btrfs_header_nritems(mid); 2010925baeddSChris Mason btrfs_tree_unlock(mid); 20115f39d397SChris Mason free_extent_buffer(mid); 2012e66f709bSChris Mason } else { 2013925baeddSChris Mason btrfs_tree_unlock(right); 20145f39d397SChris Mason free_extent_buffer(right); 2015e66f709bSChris Mason } 2016e66f709bSChris Mason return 0; 2017e66f709bSChris Mason } 2018925baeddSChris Mason btrfs_tree_unlock(right); 20195f39d397SChris Mason free_extent_buffer(right); 2020e66f709bSChris Mason } 2021e66f709bSChris Mason return 1; 2022e66f709bSChris Mason } 2023e66f709bSChris Mason 202474123bd7SChris Mason /* 2025d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 2026d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 20273c69faecSChris Mason */ 2028c8c42864SChris Mason static void reada_for_search(struct btrfs_root *root, 2029e02119d5SChris Mason struct btrfs_path *path, 203001f46658SChris Mason int level, int slot, u64 objectid) 20313c69faecSChris Mason { 20325f39d397SChris Mason struct extent_buffer *node; 203301f46658SChris Mason struct btrfs_disk_key disk_key; 20343c69faecSChris Mason u32 nritems; 20353c69faecSChris Mason u64 search; 2036a7175319SChris Mason u64 target; 20376b80053dSChris Mason u64 nread = 0; 2038cb25c2eaSJosef Bacik u64 gen; 20393c69faecSChris Mason int direction = path->reada; 20405f39d397SChris Mason struct extent_buffer *eb; 20416b80053dSChris Mason u32 nr; 20426b80053dSChris Mason u32 blocksize; 20436b80053dSChris Mason u32 nscan = 0; 2044db94535dSChris Mason 2045a6b6e75eSChris Mason if (level != 1) 20463c69faecSChris Mason return; 20473c69faecSChris Mason 20486702ed49SChris Mason if (!path->nodes[level]) 20496702ed49SChris Mason return; 20506702ed49SChris Mason 20515f39d397SChris Mason node = path->nodes[level]; 2052925baeddSChris Mason 20533c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 20546b80053dSChris Mason blocksize = btrfs_level_size(root, level - 1); 20556b80053dSChris Mason eb = btrfs_find_tree_block(root, search, blocksize); 20565f39d397SChris Mason if (eb) { 20575f39d397SChris Mason free_extent_buffer(eb); 20583c69faecSChris Mason return; 20593c69faecSChris Mason } 20603c69faecSChris Mason 2061a7175319SChris Mason target = search; 20626b80053dSChris Mason 20635f39d397SChris Mason nritems = btrfs_header_nritems(node); 20646b80053dSChris Mason nr = slot; 206525b8b936SJosef Bacik 20663c69faecSChris Mason while (1) { 20676b80053dSChris Mason if (direction < 0) { 20686b80053dSChris Mason if (nr == 0) 20693c69faecSChris Mason break; 20706b80053dSChris Mason nr--; 20716b80053dSChris Mason } else if (direction > 0) { 20726b80053dSChris Mason nr++; 20736b80053dSChris Mason if (nr >= nritems) 20746b80053dSChris Mason break; 20753c69faecSChris Mason } 207601f46658SChris Mason if (path->reada < 0 && objectid) { 207701f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 207801f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 207901f46658SChris Mason break; 208001f46658SChris Mason } 20816b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 2082a7175319SChris Mason if ((search <= target && target - search <= 65536) || 2083a7175319SChris Mason (search > target && search - target <= 65536)) { 2084cb25c2eaSJosef Bacik gen = btrfs_node_ptr_generation(node, nr); 2085cb25c2eaSJosef Bacik readahead_tree_block(root, search, blocksize, gen); 20866b80053dSChris Mason nread += blocksize; 20873c69faecSChris Mason } 20886b80053dSChris Mason nscan++; 2089a7175319SChris Mason if ((nread > 65536 || nscan > 32)) 20906b80053dSChris Mason break; 20913c69faecSChris Mason } 20923c69faecSChris Mason } 2093925baeddSChris Mason 2094d352ac68SChris Mason /* 2095b4ce94deSChris Mason * returns -EAGAIN if it had to drop the path, or zero if everything was in 2096b4ce94deSChris Mason * cache 2097b4ce94deSChris Mason */ 2098b4ce94deSChris Mason static noinline int reada_for_balance(struct btrfs_root *root, 2099b4ce94deSChris Mason struct btrfs_path *path, int level) 2100b4ce94deSChris Mason { 2101b4ce94deSChris Mason int slot; 2102b4ce94deSChris Mason int nritems; 2103b4ce94deSChris Mason struct extent_buffer *parent; 2104b4ce94deSChris Mason struct extent_buffer *eb; 2105b4ce94deSChris Mason u64 gen; 2106b4ce94deSChris Mason u64 block1 = 0; 2107b4ce94deSChris Mason u64 block2 = 0; 2108b4ce94deSChris Mason int ret = 0; 2109b4ce94deSChris Mason int blocksize; 2110b4ce94deSChris Mason 21118c594ea8SChris Mason parent = path->nodes[level + 1]; 2112b4ce94deSChris Mason if (!parent) 2113b4ce94deSChris Mason return 0; 2114b4ce94deSChris Mason 2115b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 21168c594ea8SChris Mason slot = path->slots[level + 1]; 2117b4ce94deSChris Mason blocksize = btrfs_level_size(root, level); 2118b4ce94deSChris Mason 2119b4ce94deSChris Mason if (slot > 0) { 2120b4ce94deSChris Mason block1 = btrfs_node_blockptr(parent, slot - 1); 2121b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot - 1); 2122b4ce94deSChris Mason eb = btrfs_find_tree_block(root, block1, blocksize); 2123b9fab919SChris Mason /* 2124b9fab919SChris Mason * if we get -eagain from btrfs_buffer_uptodate, we 2125b9fab919SChris Mason * don't want to return eagain here. That will loop 2126b9fab919SChris Mason * forever 2127b9fab919SChris Mason */ 2128b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2129b4ce94deSChris Mason block1 = 0; 2130b4ce94deSChris Mason free_extent_buffer(eb); 2131b4ce94deSChris Mason } 21328c594ea8SChris Mason if (slot + 1 < nritems) { 2133b4ce94deSChris Mason block2 = btrfs_node_blockptr(parent, slot + 1); 2134b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot + 1); 2135b4ce94deSChris Mason eb = btrfs_find_tree_block(root, block2, blocksize); 2136b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2137b4ce94deSChris Mason block2 = 0; 2138b4ce94deSChris Mason free_extent_buffer(eb); 2139b4ce94deSChris Mason } 2140b4ce94deSChris Mason if (block1 || block2) { 2141b4ce94deSChris Mason ret = -EAGAIN; 21428c594ea8SChris Mason 21438c594ea8SChris Mason /* release the whole path */ 2144b3b4aa74SDavid Sterba btrfs_release_path(path); 21458c594ea8SChris Mason 21468c594ea8SChris Mason /* read the blocks */ 2147b4ce94deSChris Mason if (block1) 2148b4ce94deSChris Mason readahead_tree_block(root, block1, blocksize, 0); 2149b4ce94deSChris Mason if (block2) 2150b4ce94deSChris Mason readahead_tree_block(root, block2, blocksize, 0); 2151b4ce94deSChris Mason 2152b4ce94deSChris Mason if (block1) { 2153b4ce94deSChris Mason eb = read_tree_block(root, block1, blocksize, 0); 2154b4ce94deSChris Mason free_extent_buffer(eb); 2155b4ce94deSChris Mason } 21568c594ea8SChris Mason if (block2) { 2157b4ce94deSChris Mason eb = read_tree_block(root, block2, blocksize, 0); 2158b4ce94deSChris Mason free_extent_buffer(eb); 2159b4ce94deSChris Mason } 2160b4ce94deSChris Mason } 2161b4ce94deSChris Mason return ret; 2162b4ce94deSChris Mason } 2163b4ce94deSChris Mason 2164b4ce94deSChris Mason 2165b4ce94deSChris Mason /* 2166d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 2167d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 2168d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 2169d397712bSChris Mason * tree. 2170d352ac68SChris Mason * 2171d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 2172d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 2173d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 2174d352ac68SChris Mason * 2175d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 2176d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 2177d352ac68SChris Mason */ 2178e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 2179f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 2180f7c79f30SChris Mason int *write_lock_level) 2181925baeddSChris Mason { 2182925baeddSChris Mason int i; 2183925baeddSChris Mason int skip_level = level; 2184051e1b9fSChris Mason int no_skips = 0; 2185925baeddSChris Mason struct extent_buffer *t; 2186925baeddSChris Mason 2187925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2188925baeddSChris Mason if (!path->nodes[i]) 2189925baeddSChris Mason break; 2190925baeddSChris Mason if (!path->locks[i]) 2191925baeddSChris Mason break; 2192051e1b9fSChris Mason if (!no_skips && path->slots[i] == 0) { 2193925baeddSChris Mason skip_level = i + 1; 2194925baeddSChris Mason continue; 2195925baeddSChris Mason } 2196051e1b9fSChris Mason if (!no_skips && path->keep_locks) { 2197925baeddSChris Mason u32 nritems; 2198925baeddSChris Mason t = path->nodes[i]; 2199925baeddSChris Mason nritems = btrfs_header_nritems(t); 2200051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 2201925baeddSChris Mason skip_level = i + 1; 2202925baeddSChris Mason continue; 2203925baeddSChris Mason } 2204925baeddSChris Mason } 2205051e1b9fSChris Mason if (skip_level < i && i >= lowest_unlock) 2206051e1b9fSChris Mason no_skips = 1; 2207051e1b9fSChris Mason 2208925baeddSChris Mason t = path->nodes[i]; 2209925baeddSChris Mason if (i >= lowest_unlock && i > skip_level && path->locks[i]) { 2210bd681513SChris Mason btrfs_tree_unlock_rw(t, path->locks[i]); 2211925baeddSChris Mason path->locks[i] = 0; 2212f7c79f30SChris Mason if (write_lock_level && 2213f7c79f30SChris Mason i > min_write_lock_level && 2214f7c79f30SChris Mason i <= *write_lock_level) { 2215f7c79f30SChris Mason *write_lock_level = i - 1; 2216f7c79f30SChris Mason } 2217925baeddSChris Mason } 2218925baeddSChris Mason } 2219925baeddSChris Mason } 2220925baeddSChris Mason 22213c69faecSChris Mason /* 2222b4ce94deSChris Mason * This releases any locks held in the path starting at level and 2223b4ce94deSChris Mason * going all the way up to the root. 2224b4ce94deSChris Mason * 2225b4ce94deSChris Mason * btrfs_search_slot will keep the lock held on higher nodes in a few 2226b4ce94deSChris Mason * corner cases, such as COW of the block at slot zero in the node. This 2227b4ce94deSChris Mason * ignores those rules, and it should only be called when there are no 2228b4ce94deSChris Mason * more updates to be done higher up in the tree. 2229b4ce94deSChris Mason */ 2230b4ce94deSChris Mason noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level) 2231b4ce94deSChris Mason { 2232b4ce94deSChris Mason int i; 2233b4ce94deSChris Mason 22345d4f98a2SYan Zheng if (path->keep_locks) 2235b4ce94deSChris Mason return; 2236b4ce94deSChris Mason 2237b4ce94deSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2238b4ce94deSChris Mason if (!path->nodes[i]) 223912f4daccSChris Mason continue; 2240b4ce94deSChris Mason if (!path->locks[i]) 224112f4daccSChris Mason continue; 2242bd681513SChris Mason btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 2243b4ce94deSChris Mason path->locks[i] = 0; 2244b4ce94deSChris Mason } 2245b4ce94deSChris Mason } 2246b4ce94deSChris Mason 2247b4ce94deSChris Mason /* 2248c8c42864SChris Mason * helper function for btrfs_search_slot. The goal is to find a block 2249c8c42864SChris Mason * in cache without setting the path to blocking. If we find the block 2250c8c42864SChris Mason * we return zero and the path is unchanged. 2251c8c42864SChris Mason * 2252c8c42864SChris Mason * If we can't find the block, we set the path blocking and do some 2253c8c42864SChris Mason * reada. -EAGAIN is returned and the search must be repeated. 2254c8c42864SChris Mason */ 2255c8c42864SChris Mason static int 2256c8c42864SChris Mason read_block_for_search(struct btrfs_trans_handle *trans, 2257c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2258c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 22595d9e75c4SJan Schmidt struct btrfs_key *key, u64 time_seq) 2260c8c42864SChris Mason { 2261c8c42864SChris Mason u64 blocknr; 2262c8c42864SChris Mason u64 gen; 2263c8c42864SChris Mason u32 blocksize; 2264c8c42864SChris Mason struct extent_buffer *b = *eb_ret; 2265c8c42864SChris Mason struct extent_buffer *tmp; 226676a05b35SChris Mason int ret; 2267c8c42864SChris Mason 2268c8c42864SChris Mason blocknr = btrfs_node_blockptr(b, slot); 2269c8c42864SChris Mason gen = btrfs_node_ptr_generation(b, slot); 2270c8c42864SChris Mason blocksize = btrfs_level_size(root, level - 1); 2271c8c42864SChris Mason 2272c8c42864SChris Mason tmp = btrfs_find_tree_block(root, blocknr, blocksize); 2273cb44921aSChris Mason if (tmp) { 2274b9fab919SChris Mason /* first we do an atomic uptodate check */ 2275b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, 0, 1) > 0) { 2276b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 227776a05b35SChris Mason /* 2278cb44921aSChris Mason * we found an up to date block without 2279cb44921aSChris Mason * sleeping, return 228076a05b35SChris Mason * right away 228176a05b35SChris Mason */ 2282c8c42864SChris Mason *eb_ret = tmp; 2283c8c42864SChris Mason return 0; 2284c8c42864SChris Mason } 2285cb44921aSChris Mason /* the pages were up to date, but we failed 2286cb44921aSChris Mason * the generation number check. Do a full 2287cb44921aSChris Mason * read for the generation number that is correct. 2288cb44921aSChris Mason * We must do this without dropping locks so 2289cb44921aSChris Mason * we can trust our generation number 2290cb44921aSChris Mason */ 2291cb44921aSChris Mason free_extent_buffer(tmp); 2292bd681513SChris Mason btrfs_set_path_blocking(p); 2293bd681513SChris Mason 2294b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 2295cb44921aSChris Mason tmp = read_tree_block(root, blocknr, blocksize, gen); 2296b9fab919SChris Mason if (tmp && btrfs_buffer_uptodate(tmp, gen, 0) > 0) { 2297cb44921aSChris Mason *eb_ret = tmp; 2298cb44921aSChris Mason return 0; 2299cb44921aSChris Mason } 2300cb44921aSChris Mason free_extent_buffer(tmp); 2301b3b4aa74SDavid Sterba btrfs_release_path(p); 2302cb44921aSChris Mason return -EIO; 2303cb44921aSChris Mason } 2304cb44921aSChris Mason } 2305c8c42864SChris Mason 2306c8c42864SChris Mason /* 2307c8c42864SChris Mason * reduce lock contention at high levels 2308c8c42864SChris Mason * of the btree by dropping locks before 230976a05b35SChris Mason * we read. Don't release the lock on the current 231076a05b35SChris Mason * level because we need to walk this node to figure 231176a05b35SChris Mason * out which blocks to read. 2312c8c42864SChris Mason */ 23138c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 23148c594ea8SChris Mason btrfs_set_path_blocking(p); 23158c594ea8SChris Mason 2316c8c42864SChris Mason free_extent_buffer(tmp); 2317c8c42864SChris Mason if (p->reada) 2318c8c42864SChris Mason reada_for_search(root, p, level, slot, key->objectid); 2319c8c42864SChris Mason 2320b3b4aa74SDavid Sterba btrfs_release_path(p); 232176a05b35SChris Mason 232276a05b35SChris Mason ret = -EAGAIN; 23235bdd3536SYan, Zheng tmp = read_tree_block(root, blocknr, blocksize, 0); 232476a05b35SChris Mason if (tmp) { 232576a05b35SChris Mason /* 232676a05b35SChris Mason * If the read above didn't mark this buffer up to date, 232776a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 232876a05b35SChris Mason * and give up so that our caller doesn't loop forever 232976a05b35SChris Mason * on our EAGAINs. 233076a05b35SChris Mason */ 2331b9fab919SChris Mason if (!btrfs_buffer_uptodate(tmp, 0, 0)) 233276a05b35SChris Mason ret = -EIO; 2333c8c42864SChris Mason free_extent_buffer(tmp); 233476a05b35SChris Mason } 233576a05b35SChris Mason return ret; 2336c8c42864SChris Mason } 2337c8c42864SChris Mason 2338c8c42864SChris Mason /* 2339c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 2340c8c42864SChris Mason * for node-level blocks and does any balancing required based on 2341c8c42864SChris Mason * the ins_len. 2342c8c42864SChris Mason * 2343c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 2344c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 2345c8c42864SChris Mason * start over 2346c8c42864SChris Mason */ 2347c8c42864SChris Mason static int 2348c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 2349c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2350bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 2351bd681513SChris Mason int *write_lock_level) 2352c8c42864SChris Mason { 2353c8c42864SChris Mason int ret; 2354c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 2355c8c42864SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) - 3) { 2356c8c42864SChris Mason int sret; 2357c8c42864SChris Mason 2358bd681513SChris Mason if (*write_lock_level < level + 1) { 2359bd681513SChris Mason *write_lock_level = level + 1; 2360bd681513SChris Mason btrfs_release_path(p); 2361bd681513SChris Mason goto again; 2362bd681513SChris Mason } 2363bd681513SChris Mason 2364c8c42864SChris Mason sret = reada_for_balance(root, p, level); 2365c8c42864SChris Mason if (sret) 2366c8c42864SChris Mason goto again; 2367c8c42864SChris Mason 2368c8c42864SChris Mason btrfs_set_path_blocking(p); 2369c8c42864SChris Mason sret = split_node(trans, root, p, level); 2370bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2371c8c42864SChris Mason 2372c8c42864SChris Mason BUG_ON(sret > 0); 2373c8c42864SChris Mason if (sret) { 2374c8c42864SChris Mason ret = sret; 2375c8c42864SChris Mason goto done; 2376c8c42864SChris Mason } 2377c8c42864SChris Mason b = p->nodes[level]; 2378c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 2379cfbb9308SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) / 2) { 2380c8c42864SChris Mason int sret; 2381c8c42864SChris Mason 2382bd681513SChris Mason if (*write_lock_level < level + 1) { 2383bd681513SChris Mason *write_lock_level = level + 1; 2384bd681513SChris Mason btrfs_release_path(p); 2385bd681513SChris Mason goto again; 2386bd681513SChris Mason } 2387bd681513SChris Mason 2388c8c42864SChris Mason sret = reada_for_balance(root, p, level); 2389c8c42864SChris Mason if (sret) 2390c8c42864SChris Mason goto again; 2391c8c42864SChris Mason 2392c8c42864SChris Mason btrfs_set_path_blocking(p); 2393c8c42864SChris Mason sret = balance_level(trans, root, p, level); 2394bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2395c8c42864SChris Mason 2396c8c42864SChris Mason if (sret) { 2397c8c42864SChris Mason ret = sret; 2398c8c42864SChris Mason goto done; 2399c8c42864SChris Mason } 2400c8c42864SChris Mason b = p->nodes[level]; 2401c8c42864SChris Mason if (!b) { 2402b3b4aa74SDavid Sterba btrfs_release_path(p); 2403c8c42864SChris Mason goto again; 2404c8c42864SChris Mason } 2405c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 2406c8c42864SChris Mason } 2407c8c42864SChris Mason return 0; 2408c8c42864SChris Mason 2409c8c42864SChris Mason again: 2410c8c42864SChris Mason ret = -EAGAIN; 2411c8c42864SChris Mason done: 2412c8c42864SChris Mason return ret; 2413c8c42864SChris Mason } 2414c8c42864SChris Mason 2415c8c42864SChris Mason /* 241674123bd7SChris Mason * look for key in the tree. path is filled in with nodes along the way 241774123bd7SChris Mason * if key is found, we return zero and you can find the item in the leaf 241874123bd7SChris Mason * level of the path (level 0) 241974123bd7SChris Mason * 242074123bd7SChris Mason * If the key isn't found, the path points to the slot where it should 2421aa5d6bedSChris Mason * be inserted, and 1 is returned. If there are other errors during the 2422aa5d6bedSChris Mason * search a negative error number is returned. 242397571fd0SChris Mason * 242497571fd0SChris Mason * if ins_len > 0, nodes and leaves will be split as we walk down the 242597571fd0SChris Mason * tree. if ins_len < 0, nodes will be merged as we walk down the tree (if 242697571fd0SChris Mason * possible) 242774123bd7SChris Mason */ 2428e089f05cSChris Mason int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root 2429e089f05cSChris Mason *root, struct btrfs_key *key, struct btrfs_path *p, int 2430e089f05cSChris Mason ins_len, int cow) 2431be0e5c09SChris Mason { 24325f39d397SChris Mason struct extent_buffer *b; 2433be0e5c09SChris Mason int slot; 2434be0e5c09SChris Mason int ret; 243533c66f43SYan Zheng int err; 2436be0e5c09SChris Mason int level; 2437925baeddSChris Mason int lowest_unlock = 1; 2438bd681513SChris Mason int root_lock; 2439bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2440bd681513SChris Mason int write_lock_level = 0; 24419f3a7427SChris Mason u8 lowest_level = 0; 2442f7c79f30SChris Mason int min_write_lock_level; 24439f3a7427SChris Mason 24446702ed49SChris Mason lowest_level = p->lowest_level; 2445323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 244622b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 244725179201SJosef Bacik 2448bd681513SChris Mason if (ins_len < 0) { 2449925baeddSChris Mason lowest_unlock = 2; 245065b51a00SChris Mason 2451bd681513SChris Mason /* when we are removing items, we might have to go up to level 2452bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2453bd681513SChris Mason * for those levels as well 2454bd681513SChris Mason */ 2455bd681513SChris Mason write_lock_level = 2; 2456bd681513SChris Mason } else if (ins_len > 0) { 2457bd681513SChris Mason /* 2458bd681513SChris Mason * for inserting items, make sure we have a write lock on 2459bd681513SChris Mason * level 1 so we can update keys 2460bd681513SChris Mason */ 2461bd681513SChris Mason write_lock_level = 1; 2462bd681513SChris Mason } 2463bd681513SChris Mason 2464bd681513SChris Mason if (!cow) 2465bd681513SChris Mason write_lock_level = -1; 2466bd681513SChris Mason 2467bd681513SChris Mason if (cow && (p->keep_locks || p->lowest_level)) 2468bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2469bd681513SChris Mason 2470f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2471f7c79f30SChris Mason 2472bb803951SChris Mason again: 2473bd681513SChris Mason /* 2474bd681513SChris Mason * we try very hard to do read locks on the root 2475bd681513SChris Mason */ 2476bd681513SChris Mason root_lock = BTRFS_READ_LOCK; 2477bd681513SChris Mason level = 0; 24785d4f98a2SYan Zheng if (p->search_commit_root) { 2479bd681513SChris Mason /* 2480bd681513SChris Mason * the commit roots are read only 2481bd681513SChris Mason * so we always do read locks 2482bd681513SChris Mason */ 24835d4f98a2SYan Zheng b = root->commit_root; 24845d4f98a2SYan Zheng extent_buffer_get(b); 2485bd681513SChris Mason level = btrfs_header_level(b); 24865d4f98a2SYan Zheng if (!p->skip_locking) 2487bd681513SChris Mason btrfs_tree_read_lock(b); 24885d4f98a2SYan Zheng } else { 2489bd681513SChris Mason if (p->skip_locking) { 24905cd57b2cSChris Mason b = btrfs_root_node(root); 2491bd681513SChris Mason level = btrfs_header_level(b); 2492bd681513SChris Mason } else { 2493bd681513SChris Mason /* we don't know the level of the root node 2494bd681513SChris Mason * until we actually have it read locked 2495bd681513SChris Mason */ 2496bd681513SChris Mason b = btrfs_read_lock_root_node(root); 2497bd681513SChris Mason level = btrfs_header_level(b); 2498bd681513SChris Mason if (level <= write_lock_level) { 2499bd681513SChris Mason /* whoops, must trade for write lock */ 2500bd681513SChris Mason btrfs_tree_read_unlock(b); 2501bd681513SChris Mason free_extent_buffer(b); 2502925baeddSChris Mason b = btrfs_lock_root_node(root); 2503bd681513SChris Mason root_lock = BTRFS_WRITE_LOCK; 2504bd681513SChris Mason 2505bd681513SChris Mason /* the level might have changed, check again */ 2506bd681513SChris Mason level = btrfs_header_level(b); 25075d4f98a2SYan Zheng } 2508bd681513SChris Mason } 2509bd681513SChris Mason } 2510bd681513SChris Mason p->nodes[level] = b; 2511bd681513SChris Mason if (!p->skip_locking) 2512bd681513SChris Mason p->locks[level] = root_lock; 2513925baeddSChris Mason 2514eb60ceacSChris Mason while (b) { 25155f39d397SChris Mason level = btrfs_header_level(b); 251665b51a00SChris Mason 251765b51a00SChris Mason /* 251865b51a00SChris Mason * setup the path here so we can release it under lock 251965b51a00SChris Mason * contention with the cow code 252065b51a00SChris Mason */ 252102217ed2SChris Mason if (cow) { 2522c8c42864SChris Mason /* 2523c8c42864SChris Mason * if we don't really need to cow this block 2524c8c42864SChris Mason * then we don't want to set the path blocking, 2525c8c42864SChris Mason * so we test it here 2526c8c42864SChris Mason */ 25275d4f98a2SYan Zheng if (!should_cow_block(trans, root, b)) 252865b51a00SChris Mason goto cow_done; 25295d4f98a2SYan Zheng 2530b4ce94deSChris Mason btrfs_set_path_blocking(p); 2531b4ce94deSChris Mason 2532bd681513SChris Mason /* 2533bd681513SChris Mason * must have write locks on this node and the 2534bd681513SChris Mason * parent 2535bd681513SChris Mason */ 2536bd681513SChris Mason if (level + 1 > write_lock_level) { 2537bd681513SChris Mason write_lock_level = level + 1; 2538bd681513SChris Mason btrfs_release_path(p); 2539bd681513SChris Mason goto again; 2540bd681513SChris Mason } 2541bd681513SChris Mason 254233c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2543e20d96d6SChris Mason p->nodes[level + 1], 25449fa8cfe7SChris Mason p->slots[level + 1], &b); 254533c66f43SYan Zheng if (err) { 254633c66f43SYan Zheng ret = err; 254765b51a00SChris Mason goto done; 254854aa1f4dSChris Mason } 254902217ed2SChris Mason } 255065b51a00SChris Mason cow_done: 255102217ed2SChris Mason BUG_ON(!cow && ins_len); 255265b51a00SChris Mason 2553eb60ceacSChris Mason p->nodes[level] = b; 2554bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2555b4ce94deSChris Mason 2556b4ce94deSChris Mason /* 2557b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2558b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2559b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2560b4ce94deSChris Mason * go through the expensive btree search on b. 2561b4ce94deSChris Mason * 2562b4ce94deSChris Mason * If cow is true, then we might be changing slot zero, 2563b4ce94deSChris Mason * which may require changing the parent. So, we can't 2564b4ce94deSChris Mason * drop the lock until after we know which slot we're 2565b4ce94deSChris Mason * operating on. 2566b4ce94deSChris Mason */ 2567b4ce94deSChris Mason if (!cow) 2568b4ce94deSChris Mason btrfs_unlock_up_safe(p, level + 1); 2569b4ce94deSChris Mason 25705f39d397SChris Mason ret = bin_search(b, key, level, &slot); 2571b4ce94deSChris Mason 25725f39d397SChris Mason if (level != 0) { 257333c66f43SYan Zheng int dec = 0; 257433c66f43SYan Zheng if (ret && slot > 0) { 257533c66f43SYan Zheng dec = 1; 2576be0e5c09SChris Mason slot -= 1; 257733c66f43SYan Zheng } 2578be0e5c09SChris Mason p->slots[level] = slot; 257933c66f43SYan Zheng err = setup_nodes_for_search(trans, root, p, b, level, 2580bd681513SChris Mason ins_len, &write_lock_level); 258133c66f43SYan Zheng if (err == -EAGAIN) 2582b4ce94deSChris Mason goto again; 258333c66f43SYan Zheng if (err) { 258433c66f43SYan Zheng ret = err; 258565b51a00SChris Mason goto done; 258633c66f43SYan Zheng } 25875c680ed6SChris Mason b = p->nodes[level]; 25885c680ed6SChris Mason slot = p->slots[level]; 2589b4ce94deSChris Mason 2590bd681513SChris Mason /* 2591bd681513SChris Mason * slot 0 is special, if we change the key 2592bd681513SChris Mason * we have to update the parent pointer 2593bd681513SChris Mason * which means we must have a write lock 2594bd681513SChris Mason * on the parent 2595bd681513SChris Mason */ 2596bd681513SChris Mason if (slot == 0 && cow && 2597bd681513SChris Mason write_lock_level < level + 1) { 2598bd681513SChris Mason write_lock_level = level + 1; 2599bd681513SChris Mason btrfs_release_path(p); 2600bd681513SChris Mason goto again; 2601bd681513SChris Mason } 2602bd681513SChris Mason 2603f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2604f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 2605f9efa9c7SChris Mason 2606925baeddSChris Mason if (level == lowest_level) { 260733c66f43SYan Zheng if (dec) 260833c66f43SYan Zheng p->slots[level]++; 26095b21f2edSZheng Yan goto done; 2610925baeddSChris Mason } 2611ca7a79adSChris Mason 261233c66f43SYan Zheng err = read_block_for_search(trans, root, p, 26135d9e75c4SJan Schmidt &b, level, slot, key, 0); 261433c66f43SYan Zheng if (err == -EAGAIN) 2615051e1b9fSChris Mason goto again; 261633c66f43SYan Zheng if (err) { 261733c66f43SYan Zheng ret = err; 261876a05b35SChris Mason goto done; 261933c66f43SYan Zheng } 262076a05b35SChris Mason 2621b4ce94deSChris Mason if (!p->skip_locking) { 2622bd681513SChris Mason level = btrfs_header_level(b); 2623bd681513SChris Mason if (level <= write_lock_level) { 2624bd681513SChris Mason err = btrfs_try_tree_write_lock(b); 262533c66f43SYan Zheng if (!err) { 2626b4ce94deSChris Mason btrfs_set_path_blocking(p); 2627925baeddSChris Mason btrfs_tree_lock(b); 2628bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2629bd681513SChris Mason BTRFS_WRITE_LOCK); 2630b4ce94deSChris Mason } 2631bd681513SChris Mason p->locks[level] = BTRFS_WRITE_LOCK; 2632bd681513SChris Mason } else { 2633bd681513SChris Mason err = btrfs_try_tree_read_lock(b); 2634bd681513SChris Mason if (!err) { 2635bd681513SChris Mason btrfs_set_path_blocking(p); 2636bd681513SChris Mason btrfs_tree_read_lock(b); 2637bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2638bd681513SChris Mason BTRFS_READ_LOCK); 2639bd681513SChris Mason } 2640bd681513SChris Mason p->locks[level] = BTRFS_READ_LOCK; 2641bd681513SChris Mason } 2642bd681513SChris Mason p->nodes[level] = b; 2643b4ce94deSChris Mason } 2644be0e5c09SChris Mason } else { 2645be0e5c09SChris Mason p->slots[level] = slot; 264687b29b20SYan Zheng if (ins_len > 0 && 264787b29b20SYan Zheng btrfs_leaf_free_space(root, b) < ins_len) { 2648bd681513SChris Mason if (write_lock_level < 1) { 2649bd681513SChris Mason write_lock_level = 1; 2650bd681513SChris Mason btrfs_release_path(p); 2651bd681513SChris Mason goto again; 2652bd681513SChris Mason } 2653bd681513SChris Mason 2654b4ce94deSChris Mason btrfs_set_path_blocking(p); 265533c66f43SYan Zheng err = split_leaf(trans, root, key, 2656cc0c5538SChris Mason p, ins_len, ret == 0); 2657bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2658b4ce94deSChris Mason 265933c66f43SYan Zheng BUG_ON(err > 0); 266033c66f43SYan Zheng if (err) { 266133c66f43SYan Zheng ret = err; 266265b51a00SChris Mason goto done; 266365b51a00SChris Mason } 26645c680ed6SChris Mason } 2665459931ecSChris Mason if (!p->search_for_split) 2666f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2667f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 266865b51a00SChris Mason goto done; 266965b51a00SChris Mason } 267065b51a00SChris Mason } 267165b51a00SChris Mason ret = 1; 267265b51a00SChris Mason done: 2673b4ce94deSChris Mason /* 2674b4ce94deSChris Mason * we don't really know what they plan on doing with the path 2675b4ce94deSChris Mason * from here on, so for now just mark it as blocking 2676b4ce94deSChris Mason */ 2677b9473439SChris Mason if (!p->leave_spinning) 2678b4ce94deSChris Mason btrfs_set_path_blocking(p); 267976a05b35SChris Mason if (ret < 0) 2680b3b4aa74SDavid Sterba btrfs_release_path(p); 2681be0e5c09SChris Mason return ret; 2682be0e5c09SChris Mason } 2683be0e5c09SChris Mason 268474123bd7SChris Mason /* 26855d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 26865d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 26875d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 26885d9e75c4SJan Schmidt * denoted by the time_seq parameter. 26895d9e75c4SJan Schmidt * 26905d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 26915d9e75c4SJan Schmidt * 26925d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 26935d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 26945d9e75c4SJan Schmidt */ 26955d9e75c4SJan Schmidt int btrfs_search_old_slot(struct btrfs_root *root, struct btrfs_key *key, 26965d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 26975d9e75c4SJan Schmidt { 26985d9e75c4SJan Schmidt struct extent_buffer *b; 26995d9e75c4SJan Schmidt int slot; 27005d9e75c4SJan Schmidt int ret; 27015d9e75c4SJan Schmidt int err; 27025d9e75c4SJan Schmidt int level; 27035d9e75c4SJan Schmidt int lowest_unlock = 1; 27045d9e75c4SJan Schmidt u8 lowest_level = 0; 27055d9e75c4SJan Schmidt 27065d9e75c4SJan Schmidt lowest_level = p->lowest_level; 27075d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 27085d9e75c4SJan Schmidt 27095d9e75c4SJan Schmidt if (p->search_commit_root) { 27105d9e75c4SJan Schmidt BUG_ON(time_seq); 27115d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 27125d9e75c4SJan Schmidt } 27135d9e75c4SJan Schmidt 27145d9e75c4SJan Schmidt again: 27155d9e75c4SJan Schmidt b = get_old_root(root, time_seq); 27165d9e75c4SJan Schmidt level = btrfs_header_level(b); 27175d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 27185d9e75c4SJan Schmidt 27195d9e75c4SJan Schmidt while (b) { 27205d9e75c4SJan Schmidt level = btrfs_header_level(b); 27215d9e75c4SJan Schmidt p->nodes[level] = b; 27225d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, NULL, 0); 27235d9e75c4SJan Schmidt 27245d9e75c4SJan Schmidt /* 27255d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 27265d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 27275d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 27285d9e75c4SJan Schmidt * go through the expensive btree search on b. 27295d9e75c4SJan Schmidt */ 27305d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 27315d9e75c4SJan Schmidt 27325d9e75c4SJan Schmidt ret = bin_search(b, key, level, &slot); 27335d9e75c4SJan Schmidt 27345d9e75c4SJan Schmidt if (level != 0) { 27355d9e75c4SJan Schmidt int dec = 0; 27365d9e75c4SJan Schmidt if (ret && slot > 0) { 27375d9e75c4SJan Schmidt dec = 1; 27385d9e75c4SJan Schmidt slot -= 1; 27395d9e75c4SJan Schmidt } 27405d9e75c4SJan Schmidt p->slots[level] = slot; 27415d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 27425d9e75c4SJan Schmidt 27435d9e75c4SJan Schmidt if (level == lowest_level) { 27445d9e75c4SJan Schmidt if (dec) 27455d9e75c4SJan Schmidt p->slots[level]++; 27465d9e75c4SJan Schmidt goto done; 27475d9e75c4SJan Schmidt } 27485d9e75c4SJan Schmidt 27495d9e75c4SJan Schmidt err = read_block_for_search(NULL, root, p, &b, level, 27505d9e75c4SJan Schmidt slot, key, time_seq); 27515d9e75c4SJan Schmidt if (err == -EAGAIN) 27525d9e75c4SJan Schmidt goto again; 27535d9e75c4SJan Schmidt if (err) { 27545d9e75c4SJan Schmidt ret = err; 27555d9e75c4SJan Schmidt goto done; 27565d9e75c4SJan Schmidt } 27575d9e75c4SJan Schmidt 27585d9e75c4SJan Schmidt level = btrfs_header_level(b); 27595d9e75c4SJan Schmidt err = btrfs_try_tree_read_lock(b); 27605d9e75c4SJan Schmidt if (!err) { 27615d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 27625d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 27635d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, b, 27645d9e75c4SJan Schmidt BTRFS_READ_LOCK); 27655d9e75c4SJan Schmidt } 27665d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 27675d9e75c4SJan Schmidt p->nodes[level] = b; 27685d9e75c4SJan Schmidt b = tree_mod_log_rewind(root->fs_info, b, time_seq); 27695d9e75c4SJan Schmidt if (b != p->nodes[level]) { 27705d9e75c4SJan Schmidt btrfs_tree_unlock_rw(p->nodes[level], 27715d9e75c4SJan Schmidt p->locks[level]); 27725d9e75c4SJan Schmidt p->locks[level] = 0; 27735d9e75c4SJan Schmidt p->nodes[level] = b; 27745d9e75c4SJan Schmidt } 27755d9e75c4SJan Schmidt } else { 27765d9e75c4SJan Schmidt p->slots[level] = slot; 27775d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 27785d9e75c4SJan Schmidt goto done; 27795d9e75c4SJan Schmidt } 27805d9e75c4SJan Schmidt } 27815d9e75c4SJan Schmidt ret = 1; 27825d9e75c4SJan Schmidt done: 27835d9e75c4SJan Schmidt if (!p->leave_spinning) 27845d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 27855d9e75c4SJan Schmidt if (ret < 0) 27865d9e75c4SJan Schmidt btrfs_release_path(p); 27875d9e75c4SJan Schmidt 27885d9e75c4SJan Schmidt return ret; 27895d9e75c4SJan Schmidt } 27905d9e75c4SJan Schmidt 27915d9e75c4SJan Schmidt /* 27922f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 27932f38b3e1SArne Jansen * instead the next or previous item should be returned. 27942f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 27952f38b3e1SArne Jansen * otherwise. 27962f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 27972f38b3e1SArne Jansen * return the next lower instead. 27982f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 27992f38b3e1SArne Jansen * return the next higher instead. 28002f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 28012f38b3e1SArne Jansen * < 0 on error 28022f38b3e1SArne Jansen */ 28032f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 28042f38b3e1SArne Jansen struct btrfs_key *key, struct btrfs_path *p, 28052f38b3e1SArne Jansen int find_higher, int return_any) 28062f38b3e1SArne Jansen { 28072f38b3e1SArne Jansen int ret; 28082f38b3e1SArne Jansen struct extent_buffer *leaf; 28092f38b3e1SArne Jansen 28102f38b3e1SArne Jansen again: 28112f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 28122f38b3e1SArne Jansen if (ret <= 0) 28132f38b3e1SArne Jansen return ret; 28142f38b3e1SArne Jansen /* 28152f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 28162f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 28172f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 28182f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 28192f38b3e1SArne Jansen * item. 28202f38b3e1SArne Jansen */ 28212f38b3e1SArne Jansen leaf = p->nodes[0]; 28222f38b3e1SArne Jansen 28232f38b3e1SArne Jansen if (find_higher) { 28242f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 28252f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 28262f38b3e1SArne Jansen if (ret <= 0) 28272f38b3e1SArne Jansen return ret; 28282f38b3e1SArne Jansen if (!return_any) 28292f38b3e1SArne Jansen return 1; 28302f38b3e1SArne Jansen /* 28312f38b3e1SArne Jansen * no higher item found, return the next 28322f38b3e1SArne Jansen * lower instead 28332f38b3e1SArne Jansen */ 28342f38b3e1SArne Jansen return_any = 0; 28352f38b3e1SArne Jansen find_higher = 0; 28362f38b3e1SArne Jansen btrfs_release_path(p); 28372f38b3e1SArne Jansen goto again; 28382f38b3e1SArne Jansen } 28392f38b3e1SArne Jansen } else { 28402f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 28412f38b3e1SArne Jansen /* we're sitting on an invalid slot */ 28422f38b3e1SArne Jansen if (p->slots[0] == 0) { 28432f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 28442f38b3e1SArne Jansen if (ret <= 0) 28452f38b3e1SArne Jansen return ret; 28462f38b3e1SArne Jansen if (!return_any) 28472f38b3e1SArne Jansen return 1; 28482f38b3e1SArne Jansen /* 28492f38b3e1SArne Jansen * no lower item found, return the next 28502f38b3e1SArne Jansen * higher instead 28512f38b3e1SArne Jansen */ 28522f38b3e1SArne Jansen return_any = 0; 28532f38b3e1SArne Jansen find_higher = 1; 28542f38b3e1SArne Jansen btrfs_release_path(p); 28552f38b3e1SArne Jansen goto again; 28562f38b3e1SArne Jansen } 28572f38b3e1SArne Jansen --p->slots[0]; 28582f38b3e1SArne Jansen } 28592f38b3e1SArne Jansen } 28602f38b3e1SArne Jansen return 0; 28612f38b3e1SArne Jansen } 28622f38b3e1SArne Jansen 28632f38b3e1SArne Jansen /* 286474123bd7SChris Mason * adjust the pointers going up the tree, starting at level 286574123bd7SChris Mason * making sure the right key of each node is points to 'key'. 286674123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 286774123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 286874123bd7SChris Mason * higher levels 2869aa5d6bedSChris Mason * 287074123bd7SChris Mason */ 2871143bede5SJeff Mahoney static void fixup_low_keys(struct btrfs_trans_handle *trans, 28725f39d397SChris Mason struct btrfs_root *root, struct btrfs_path *path, 28735f39d397SChris Mason struct btrfs_disk_key *key, int level) 2874be0e5c09SChris Mason { 2875be0e5c09SChris Mason int i; 28765f39d397SChris Mason struct extent_buffer *t; 28775f39d397SChris Mason 2878234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2879be0e5c09SChris Mason int tslot = path->slots[i]; 2880eb60ceacSChris Mason if (!path->nodes[i]) 2881be0e5c09SChris Mason break; 28825f39d397SChris Mason t = path->nodes[i]; 2883f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, t, key, tslot, 1); 28845f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2885d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2886be0e5c09SChris Mason if (tslot != 0) 2887be0e5c09SChris Mason break; 2888be0e5c09SChris Mason } 2889be0e5c09SChris Mason } 2890be0e5c09SChris Mason 289174123bd7SChris Mason /* 289231840ae1SZheng Yan * update item key. 289331840ae1SZheng Yan * 289431840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 289531840ae1SZheng Yan * that the new key won't break the order 289631840ae1SZheng Yan */ 2897143bede5SJeff Mahoney void btrfs_set_item_key_safe(struct btrfs_trans_handle *trans, 289831840ae1SZheng Yan struct btrfs_root *root, struct btrfs_path *path, 289931840ae1SZheng Yan struct btrfs_key *new_key) 290031840ae1SZheng Yan { 290131840ae1SZheng Yan struct btrfs_disk_key disk_key; 290231840ae1SZheng Yan struct extent_buffer *eb; 290331840ae1SZheng Yan int slot; 290431840ae1SZheng Yan 290531840ae1SZheng Yan eb = path->nodes[0]; 290631840ae1SZheng Yan slot = path->slots[0]; 290731840ae1SZheng Yan if (slot > 0) { 290831840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 2909143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) >= 0); 291031840ae1SZheng Yan } 291131840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 291231840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 2913143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) <= 0); 291431840ae1SZheng Yan } 291531840ae1SZheng Yan 291631840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 291731840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 291831840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 291931840ae1SZheng Yan if (slot == 0) 292031840ae1SZheng Yan fixup_low_keys(trans, root, path, &disk_key, 1); 292131840ae1SZheng Yan } 292231840ae1SZheng Yan 292331840ae1SZheng Yan /* 292474123bd7SChris Mason * try to push data from one node into the next node left in the 292579f95c82SChris Mason * tree. 2926aa5d6bedSChris Mason * 2927aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2928aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 292974123bd7SChris Mason */ 293098ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 293198ed5174SChris Mason struct btrfs_root *root, struct extent_buffer *dst, 2932971a1f66SChris Mason struct extent_buffer *src, int empty) 2933be0e5c09SChris Mason { 2934be0e5c09SChris Mason int push_items = 0; 2935bb803951SChris Mason int src_nritems; 2936bb803951SChris Mason int dst_nritems; 2937aa5d6bedSChris Mason int ret = 0; 2938be0e5c09SChris Mason 29395f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 29405f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 2941123abc88SChris Mason push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems; 29427bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 29437bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 294454aa1f4dSChris Mason 2945bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2946971a1f66SChris Mason return 1; 2947971a1f66SChris Mason 2948d397712bSChris Mason if (push_items <= 0) 2949be0e5c09SChris Mason return 1; 2950be0e5c09SChris Mason 2951bce4eae9SChris Mason if (empty) { 2952971a1f66SChris Mason push_items = min(src_nritems, push_items); 2953bce4eae9SChris Mason if (push_items < src_nritems) { 2954bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2955bce4eae9SChris Mason * we aren't going to empty it 2956bce4eae9SChris Mason */ 2957bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2958bce4eae9SChris Mason if (push_items <= 8) 2959bce4eae9SChris Mason return 1; 2960bce4eae9SChris Mason push_items -= 8; 2961bce4eae9SChris Mason } 2962bce4eae9SChris Mason } 2963bce4eae9SChris Mason } else 2964bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 296579f95c82SChris Mason 2966f230475eSJan Schmidt tree_mod_log_eb_copy(root->fs_info, dst, src, dst_nritems, 0, 2967f230475eSJan Schmidt push_items); 29685f39d397SChris Mason copy_extent_buffer(dst, src, 29695f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 29705f39d397SChris Mason btrfs_node_key_ptr_offset(0), 2971123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 29725f39d397SChris Mason 2973bb803951SChris Mason if (push_items < src_nritems) { 2974f230475eSJan Schmidt tree_mod_log_eb_move(root->fs_info, src, 0, push_items, 2975f230475eSJan Schmidt src_nritems - push_items); 29765f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 29775f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 2978e2fa7227SChris Mason (src_nritems - push_items) * 2979123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2980bb803951SChris Mason } 29815f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 29825f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 29835f39d397SChris Mason btrfs_mark_buffer_dirty(src); 29845f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 298531840ae1SZheng Yan 2986bb803951SChris Mason return ret; 2987be0e5c09SChris Mason } 2988be0e5c09SChris Mason 298997571fd0SChris Mason /* 299079f95c82SChris Mason * try to push data from one node into the next node right in the 299179f95c82SChris Mason * tree. 299279f95c82SChris Mason * 299379f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 299479f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 299579f95c82SChris Mason * 299679f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 299779f95c82SChris Mason */ 29985f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 29995f39d397SChris Mason struct btrfs_root *root, 30005f39d397SChris Mason struct extent_buffer *dst, 30015f39d397SChris Mason struct extent_buffer *src) 300279f95c82SChris Mason { 300379f95c82SChris Mason int push_items = 0; 300479f95c82SChris Mason int max_push; 300579f95c82SChris Mason int src_nritems; 300679f95c82SChris Mason int dst_nritems; 300779f95c82SChris Mason int ret = 0; 300879f95c82SChris Mason 30097bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 30107bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 30117bb86316SChris Mason 30125f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 30135f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 3014123abc88SChris Mason push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems; 3015d397712bSChris Mason if (push_items <= 0) 301679f95c82SChris Mason return 1; 3017bce4eae9SChris Mason 3018d397712bSChris Mason if (src_nritems < 4) 3019bce4eae9SChris Mason return 1; 302079f95c82SChris Mason 302179f95c82SChris Mason max_push = src_nritems / 2 + 1; 302279f95c82SChris Mason /* don't try to empty the node */ 3023d397712bSChris Mason if (max_push >= src_nritems) 302479f95c82SChris Mason return 1; 3025252c38f0SYan 302679f95c82SChris Mason if (max_push < push_items) 302779f95c82SChris Mason push_items = max_push; 302879f95c82SChris Mason 3029f230475eSJan Schmidt tree_mod_log_eb_move(root->fs_info, dst, push_items, 0, dst_nritems); 30305f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 30315f39d397SChris Mason btrfs_node_key_ptr_offset(0), 30325f39d397SChris Mason (dst_nritems) * 30335f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 3034d6025579SChris Mason 3035f230475eSJan Schmidt tree_mod_log_eb_copy(root->fs_info, dst, src, 0, 3036f230475eSJan Schmidt src_nritems - push_items, push_items); 30375f39d397SChris Mason copy_extent_buffer(dst, src, 30385f39d397SChris Mason btrfs_node_key_ptr_offset(0), 30395f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 3040123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 304179f95c82SChris Mason 30425f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 30435f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 304479f95c82SChris Mason 30455f39d397SChris Mason btrfs_mark_buffer_dirty(src); 30465f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 304731840ae1SZheng Yan 304879f95c82SChris Mason return ret; 304979f95c82SChris Mason } 305079f95c82SChris Mason 305179f95c82SChris Mason /* 305297571fd0SChris Mason * helper function to insert a new root level in the tree. 305397571fd0SChris Mason * A new node is allocated, and a single item is inserted to 305497571fd0SChris Mason * point to the existing root 3055aa5d6bedSChris Mason * 3056aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 305797571fd0SChris Mason */ 3058d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 30595f39d397SChris Mason struct btrfs_root *root, 30605f39d397SChris Mason struct btrfs_path *path, int level) 306174123bd7SChris Mason { 30627bb86316SChris Mason u64 lower_gen; 30635f39d397SChris Mason struct extent_buffer *lower; 30645f39d397SChris Mason struct extent_buffer *c; 3065925baeddSChris Mason struct extent_buffer *old; 30665f39d397SChris Mason struct btrfs_disk_key lower_key; 30675c680ed6SChris Mason 30685c680ed6SChris Mason BUG_ON(path->nodes[level]); 30695c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 30705c680ed6SChris Mason 30717bb86316SChris Mason lower = path->nodes[level-1]; 30727bb86316SChris Mason if (level == 1) 30737bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 30747bb86316SChris Mason else 30757bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 30767bb86316SChris Mason 307731840ae1SZheng Yan c = btrfs_alloc_free_block(trans, root, root->nodesize, 0, 30785d4f98a2SYan Zheng root->root_key.objectid, &lower_key, 30795581a51aSJan Schmidt level, root->node->start, 0); 30805f39d397SChris Mason if (IS_ERR(c)) 30815f39d397SChris Mason return PTR_ERR(c); 3082925baeddSChris Mason 3083f0486c68SYan, Zheng root_add_used(root, root->nodesize); 3084f0486c68SYan, Zheng 30855d4f98a2SYan Zheng memset_extent_buffer(c, 0, 0, sizeof(struct btrfs_header)); 30865f39d397SChris Mason btrfs_set_header_nritems(c, 1); 30875f39d397SChris Mason btrfs_set_header_level(c, level); 3088db94535dSChris Mason btrfs_set_header_bytenr(c, c->start); 30895f39d397SChris Mason btrfs_set_header_generation(c, trans->transid); 30905d4f98a2SYan Zheng btrfs_set_header_backref_rev(c, BTRFS_MIXED_BACKREF_REV); 30915f39d397SChris Mason btrfs_set_header_owner(c, root->root_key.objectid); 3092d5719762SChris Mason 30935f39d397SChris Mason write_extent_buffer(c, root->fs_info->fsid, 30945f39d397SChris Mason (unsigned long)btrfs_header_fsid(c), 30955f39d397SChris Mason BTRFS_FSID_SIZE); 3096e17cade2SChris Mason 3097e17cade2SChris Mason write_extent_buffer(c, root->fs_info->chunk_tree_uuid, 3098e17cade2SChris Mason (unsigned long)btrfs_header_chunk_tree_uuid(c), 3099e17cade2SChris Mason BTRFS_UUID_SIZE); 3100e17cade2SChris Mason 31015f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 3102db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 31037bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 310431840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 31057bb86316SChris Mason 31067bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 31075f39d397SChris Mason 31085f39d397SChris Mason btrfs_mark_buffer_dirty(c); 3109d5719762SChris Mason 3110925baeddSChris Mason old = root->node; 3111f230475eSJan Schmidt tree_mod_log_set_root_pointer(root, c); 3112240f62c8SChris Mason rcu_assign_pointer(root->node, c); 3113925baeddSChris Mason 3114925baeddSChris Mason /* the super has an extra ref to root->node */ 3115925baeddSChris Mason free_extent_buffer(old); 3116925baeddSChris Mason 31170b86a832SChris Mason add_root_to_dirty_list(root); 31185f39d397SChris Mason extent_buffer_get(c); 31195f39d397SChris Mason path->nodes[level] = c; 3120bd681513SChris Mason path->locks[level] = BTRFS_WRITE_LOCK; 312174123bd7SChris Mason path->slots[level] = 0; 312274123bd7SChris Mason return 0; 312374123bd7SChris Mason } 31245c680ed6SChris Mason 31255c680ed6SChris Mason /* 31265c680ed6SChris Mason * worker function to insert a single pointer in a node. 31275c680ed6SChris Mason * the node should have enough room for the pointer already 312897571fd0SChris Mason * 31295c680ed6SChris Mason * slot and level indicate where you want the key to go, and 31305c680ed6SChris Mason * blocknr is the block the key points to. 31315c680ed6SChris Mason */ 3132143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 3133143bede5SJeff Mahoney struct btrfs_root *root, struct btrfs_path *path, 3134143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 3135c3e06965SJan Schmidt int slot, int level) 31365c680ed6SChris Mason { 31375f39d397SChris Mason struct extent_buffer *lower; 31385c680ed6SChris Mason int nritems; 3139f3ea38daSJan Schmidt int ret; 31405c680ed6SChris Mason 31415c680ed6SChris Mason BUG_ON(!path->nodes[level]); 3142f0486c68SYan, Zheng btrfs_assert_tree_locked(path->nodes[level]); 31435f39d397SChris Mason lower = path->nodes[level]; 31445f39d397SChris Mason nritems = btrfs_header_nritems(lower); 3145c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 3146143bede5SJeff Mahoney BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(root)); 314774123bd7SChris Mason if (slot != nritems) { 3148c3e06965SJan Schmidt if (level) 3149f3ea38daSJan Schmidt tree_mod_log_eb_move(root->fs_info, lower, slot + 1, 3150f3ea38daSJan Schmidt slot, nritems - slot); 31515f39d397SChris Mason memmove_extent_buffer(lower, 31525f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 31535f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 3154123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 315574123bd7SChris Mason } 3156c3e06965SJan Schmidt if (level) { 3157f3ea38daSJan Schmidt ret = tree_mod_log_insert_key(root->fs_info, lower, slot, 3158f3ea38daSJan Schmidt MOD_LOG_KEY_ADD); 3159f3ea38daSJan Schmidt BUG_ON(ret < 0); 3160f3ea38daSJan Schmidt } 31615f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 3162db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 316374493f7aSChris Mason WARN_ON(trans->transid == 0); 316474493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 31655f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 31665f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 316774123bd7SChris Mason } 316874123bd7SChris Mason 316997571fd0SChris Mason /* 317097571fd0SChris Mason * split the node at the specified level in path in two. 317197571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 317297571fd0SChris Mason * 317397571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 317497571fd0SChris Mason * left and right, if either one works, it returns right away. 3175aa5d6bedSChris Mason * 3176aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 317797571fd0SChris Mason */ 3178e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3179e02119d5SChris Mason struct btrfs_root *root, 3180e02119d5SChris Mason struct btrfs_path *path, int level) 3181be0e5c09SChris Mason { 31825f39d397SChris Mason struct extent_buffer *c; 31835f39d397SChris Mason struct extent_buffer *split; 31845f39d397SChris Mason struct btrfs_disk_key disk_key; 3185be0e5c09SChris Mason int mid; 31865c680ed6SChris Mason int ret; 31877518a238SChris Mason u32 c_nritems; 3188be0e5c09SChris Mason 31895f39d397SChris Mason c = path->nodes[level]; 31907bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 31915f39d397SChris Mason if (c == root->node) { 31925c680ed6SChris Mason /* trying to split the root, lets make a new one */ 3193e089f05cSChris Mason ret = insert_new_root(trans, root, path, level + 1); 31945c680ed6SChris Mason if (ret) 31955c680ed6SChris Mason return ret; 3196b3612421SChris Mason } else { 3197e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 31985f39d397SChris Mason c = path->nodes[level]; 31995f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 3200c448acf0SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) - 3) 3201e66f709bSChris Mason return 0; 320254aa1f4dSChris Mason if (ret < 0) 320354aa1f4dSChris Mason return ret; 32045c680ed6SChris Mason } 3205e66f709bSChris Mason 32065f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 32075d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 32085d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 32097bb86316SChris Mason 32105d4f98a2SYan Zheng split = btrfs_alloc_free_block(trans, root, root->nodesize, 0, 32117bb86316SChris Mason root->root_key.objectid, 32125581a51aSJan Schmidt &disk_key, level, c->start, 0); 32135f39d397SChris Mason if (IS_ERR(split)) 32145f39d397SChris Mason return PTR_ERR(split); 321554aa1f4dSChris Mason 3216f0486c68SYan, Zheng root_add_used(root, root->nodesize); 3217f0486c68SYan, Zheng 32185d4f98a2SYan Zheng memset_extent_buffer(split, 0, 0, sizeof(struct btrfs_header)); 32195f39d397SChris Mason btrfs_set_header_level(split, btrfs_header_level(c)); 3220db94535dSChris Mason btrfs_set_header_bytenr(split, split->start); 32215f39d397SChris Mason btrfs_set_header_generation(split, trans->transid); 32225d4f98a2SYan Zheng btrfs_set_header_backref_rev(split, BTRFS_MIXED_BACKREF_REV); 32235f39d397SChris Mason btrfs_set_header_owner(split, root->root_key.objectid); 32245f39d397SChris Mason write_extent_buffer(split, root->fs_info->fsid, 32255f39d397SChris Mason (unsigned long)btrfs_header_fsid(split), 32265f39d397SChris Mason BTRFS_FSID_SIZE); 3227e17cade2SChris Mason write_extent_buffer(split, root->fs_info->chunk_tree_uuid, 3228e17cade2SChris Mason (unsigned long)btrfs_header_chunk_tree_uuid(split), 3229e17cade2SChris Mason BTRFS_UUID_SIZE); 32305f39d397SChris Mason 3231f230475eSJan Schmidt tree_mod_log_eb_copy(root->fs_info, split, c, 0, mid, c_nritems - mid); 32325f39d397SChris Mason copy_extent_buffer(split, c, 32335f39d397SChris Mason btrfs_node_key_ptr_offset(0), 32345f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 3235123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 32365f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 32375f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3238aa5d6bedSChris Mason ret = 0; 3239aa5d6bedSChris Mason 32405f39d397SChris Mason btrfs_mark_buffer_dirty(c); 32415f39d397SChris Mason btrfs_mark_buffer_dirty(split); 32425f39d397SChris Mason 3243143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, split->start, 3244c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3245aa5d6bedSChris Mason 32465de08d7dSChris Mason if (path->slots[level] >= mid) { 32475c680ed6SChris Mason path->slots[level] -= mid; 3248925baeddSChris Mason btrfs_tree_unlock(c); 32495f39d397SChris Mason free_extent_buffer(c); 32505f39d397SChris Mason path->nodes[level] = split; 32515c680ed6SChris Mason path->slots[level + 1] += 1; 3252eb60ceacSChris Mason } else { 3253925baeddSChris Mason btrfs_tree_unlock(split); 32545f39d397SChris Mason free_extent_buffer(split); 3255be0e5c09SChris Mason } 3256aa5d6bedSChris Mason return ret; 3257be0e5c09SChris Mason } 3258be0e5c09SChris Mason 325974123bd7SChris Mason /* 326074123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 326174123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 326274123bd7SChris Mason * space used both by the item structs and the item data 326374123bd7SChris Mason */ 32645f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 3265be0e5c09SChris Mason { 3266be0e5c09SChris Mason int data_len; 32675f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3268d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3269be0e5c09SChris Mason 3270be0e5c09SChris Mason if (!nr) 3271be0e5c09SChris Mason return 0; 32725f39d397SChris Mason data_len = btrfs_item_end_nr(l, start); 32735f39d397SChris Mason data_len = data_len - btrfs_item_offset_nr(l, end); 32740783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3275d4dbff95SChris Mason WARN_ON(data_len < 0); 3276be0e5c09SChris Mason return data_len; 3277be0e5c09SChris Mason } 3278be0e5c09SChris Mason 327974123bd7SChris Mason /* 3280d4dbff95SChris Mason * The space between the end of the leaf items and 3281d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3282d4dbff95SChris Mason * the leaf has left for both items and data 3283d4dbff95SChris Mason */ 3284d397712bSChris Mason noinline int btrfs_leaf_free_space(struct btrfs_root *root, 3285e02119d5SChris Mason struct extent_buffer *leaf) 3286d4dbff95SChris Mason { 32875f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 32885f39d397SChris Mason int ret; 32895f39d397SChris Mason ret = BTRFS_LEAF_DATA_SIZE(root) - leaf_space_used(leaf, 0, nritems); 32905f39d397SChris Mason if (ret < 0) { 3291d397712bSChris Mason printk(KERN_CRIT "leaf free space ret %d, leaf data size %lu, " 3292d397712bSChris Mason "used %d nritems %d\n", 3293ae2f5411SJens Axboe ret, (unsigned long) BTRFS_LEAF_DATA_SIZE(root), 32945f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 32955f39d397SChris Mason } 32965f39d397SChris Mason return ret; 3297d4dbff95SChris Mason } 3298d4dbff95SChris Mason 329999d8f83cSChris Mason /* 330099d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 330199d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 330299d8f83cSChris Mason */ 330344871b1bSChris Mason static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, 330444871b1bSChris Mason struct btrfs_root *root, 330544871b1bSChris Mason struct btrfs_path *path, 330644871b1bSChris Mason int data_size, int empty, 330744871b1bSChris Mason struct extent_buffer *right, 330899d8f83cSChris Mason int free_space, u32 left_nritems, 330999d8f83cSChris Mason u32 min_slot) 331000ec4c51SChris Mason { 33115f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 331244871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3313cfed81a0SChris Mason struct btrfs_map_token token; 33145f39d397SChris Mason struct btrfs_disk_key disk_key; 331500ec4c51SChris Mason int slot; 331634a38218SChris Mason u32 i; 331700ec4c51SChris Mason int push_space = 0; 331800ec4c51SChris Mason int push_items = 0; 33190783fcfcSChris Mason struct btrfs_item *item; 332034a38218SChris Mason u32 nr; 33217518a238SChris Mason u32 right_nritems; 33225f39d397SChris Mason u32 data_end; 3323db94535dSChris Mason u32 this_item_size; 332400ec4c51SChris Mason 3325cfed81a0SChris Mason btrfs_init_map_token(&token); 3326cfed81a0SChris Mason 332734a38218SChris Mason if (empty) 332834a38218SChris Mason nr = 0; 332934a38218SChris Mason else 333099d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 333134a38218SChris Mason 333231840ae1SZheng Yan if (path->slots[0] >= left_nritems) 333387b29b20SYan Zheng push_space += data_size; 333431840ae1SZheng Yan 333544871b1bSChris Mason slot = path->slots[1]; 333634a38218SChris Mason i = left_nritems - 1; 333734a38218SChris Mason while (i >= nr) { 33385f39d397SChris Mason item = btrfs_item_nr(left, i); 3339db94535dSChris Mason 334031840ae1SZheng Yan if (!empty && push_items > 0) { 334131840ae1SZheng Yan if (path->slots[0] > i) 334231840ae1SZheng Yan break; 334331840ae1SZheng Yan if (path->slots[0] == i) { 334431840ae1SZheng Yan int space = btrfs_leaf_free_space(root, left); 334531840ae1SZheng Yan if (space + push_space * 2 > free_space) 334631840ae1SZheng Yan break; 334731840ae1SZheng Yan } 334831840ae1SZheng Yan } 334931840ae1SZheng Yan 335000ec4c51SChris Mason if (path->slots[0] == i) 335187b29b20SYan Zheng push_space += data_size; 3352db94535dSChris Mason 3353db94535dSChris Mason this_item_size = btrfs_item_size(left, item); 3354db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 335500ec4c51SChris Mason break; 335631840ae1SZheng Yan 335700ec4c51SChris Mason push_items++; 3358db94535dSChris Mason push_space += this_item_size + sizeof(*item); 335934a38218SChris Mason if (i == 0) 336034a38218SChris Mason break; 336134a38218SChris Mason i--; 3362db94535dSChris Mason } 33635f39d397SChris Mason 3364925baeddSChris Mason if (push_items == 0) 3365925baeddSChris Mason goto out_unlock; 33665f39d397SChris Mason 336734a38218SChris Mason if (!empty && push_items == left_nritems) 3368a429e513SChris Mason WARN_ON(1); 33695f39d397SChris Mason 337000ec4c51SChris Mason /* push left to right */ 33715f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 337234a38218SChris Mason 33735f39d397SChris Mason push_space = btrfs_item_end_nr(left, left_nritems - push_items); 3374123abc88SChris Mason push_space -= leaf_data_end(root, left); 33755f39d397SChris Mason 337600ec4c51SChris Mason /* make room in the right data area */ 33775f39d397SChris Mason data_end = leaf_data_end(root, right); 33785f39d397SChris Mason memmove_extent_buffer(right, 33795f39d397SChris Mason btrfs_leaf_data(right) + data_end - push_space, 33805f39d397SChris Mason btrfs_leaf_data(right) + data_end, 33815f39d397SChris Mason BTRFS_LEAF_DATA_SIZE(root) - data_end); 33825f39d397SChris Mason 338300ec4c51SChris Mason /* copy from the left data area */ 33845f39d397SChris Mason copy_extent_buffer(right, left, btrfs_leaf_data(right) + 3385d6025579SChris Mason BTRFS_LEAF_DATA_SIZE(root) - push_space, 3386d6025579SChris Mason btrfs_leaf_data(left) + leaf_data_end(root, left), 3387d6025579SChris Mason push_space); 33885f39d397SChris Mason 33895f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 33905f39d397SChris Mason btrfs_item_nr_offset(0), 33910783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 33925f39d397SChris Mason 339300ec4c51SChris Mason /* copy the items from left to right */ 33945f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 33955f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 33960783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 339700ec4c51SChris Mason 339800ec4c51SChris Mason /* update the item pointers */ 33997518a238SChris Mason right_nritems += push_items; 34005f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 3401123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root); 34027518a238SChris Mason for (i = 0; i < right_nritems; i++) { 34035f39d397SChris Mason item = btrfs_item_nr(right, i); 3404cfed81a0SChris Mason push_space -= btrfs_token_item_size(right, item, &token); 3405cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3406db94535dSChris Mason } 3407db94535dSChris Mason 34087518a238SChris Mason left_nritems -= push_items; 34095f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 341000ec4c51SChris Mason 341134a38218SChris Mason if (left_nritems) 34125f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3413f0486c68SYan, Zheng else 3414f0486c68SYan, Zheng clean_tree_block(trans, root, left); 3415f0486c68SYan, Zheng 34165f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3417a429e513SChris Mason 34185f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 34195f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3420d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 342102217ed2SChris Mason 342200ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 34237518a238SChris Mason if (path->slots[0] >= left_nritems) { 34247518a238SChris Mason path->slots[0] -= left_nritems; 3425925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 3426925baeddSChris Mason clean_tree_block(trans, root, path->nodes[0]); 3427925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 34285f39d397SChris Mason free_extent_buffer(path->nodes[0]); 34295f39d397SChris Mason path->nodes[0] = right; 343000ec4c51SChris Mason path->slots[1] += 1; 343100ec4c51SChris Mason } else { 3432925baeddSChris Mason btrfs_tree_unlock(right); 34335f39d397SChris Mason free_extent_buffer(right); 343400ec4c51SChris Mason } 343500ec4c51SChris Mason return 0; 3436925baeddSChris Mason 3437925baeddSChris Mason out_unlock: 3438925baeddSChris Mason btrfs_tree_unlock(right); 3439925baeddSChris Mason free_extent_buffer(right); 3440925baeddSChris Mason return 1; 344100ec4c51SChris Mason } 3442925baeddSChris Mason 344300ec4c51SChris Mason /* 344444871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 344574123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 344644871b1bSChris Mason * 344744871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 344844871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 344999d8f83cSChris Mason * 345099d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 345199d8f83cSChris Mason * push any slot lower than min_slot 345274123bd7SChris Mason */ 345344871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 345499d8f83cSChris Mason *root, struct btrfs_path *path, 345599d8f83cSChris Mason int min_data_size, int data_size, 345699d8f83cSChris Mason int empty, u32 min_slot) 3457be0e5c09SChris Mason { 345844871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 345944871b1bSChris Mason struct extent_buffer *right; 346044871b1bSChris Mason struct extent_buffer *upper; 346144871b1bSChris Mason int slot; 346244871b1bSChris Mason int free_space; 346344871b1bSChris Mason u32 left_nritems; 346444871b1bSChris Mason int ret; 346544871b1bSChris Mason 346644871b1bSChris Mason if (!path->nodes[1]) 346744871b1bSChris Mason return 1; 346844871b1bSChris Mason 346944871b1bSChris Mason slot = path->slots[1]; 347044871b1bSChris Mason upper = path->nodes[1]; 347144871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 347244871b1bSChris Mason return 1; 347344871b1bSChris Mason 347444871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 347544871b1bSChris Mason 347644871b1bSChris Mason right = read_node_slot(root, upper, slot + 1); 347791ca338dSTsutomu Itoh if (right == NULL) 347891ca338dSTsutomu Itoh return 1; 347991ca338dSTsutomu Itoh 348044871b1bSChris Mason btrfs_tree_lock(right); 348144871b1bSChris Mason btrfs_set_lock_blocking(right); 348244871b1bSChris Mason 348344871b1bSChris Mason free_space = btrfs_leaf_free_space(root, right); 348444871b1bSChris Mason if (free_space < data_size) 348544871b1bSChris Mason goto out_unlock; 348644871b1bSChris Mason 348744871b1bSChris Mason /* cow and double check */ 348844871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 348944871b1bSChris Mason slot + 1, &right); 349044871b1bSChris Mason if (ret) 349144871b1bSChris Mason goto out_unlock; 349244871b1bSChris Mason 349344871b1bSChris Mason free_space = btrfs_leaf_free_space(root, right); 349444871b1bSChris Mason if (free_space < data_size) 349544871b1bSChris Mason goto out_unlock; 349644871b1bSChris Mason 349744871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 349844871b1bSChris Mason if (left_nritems == 0) 349944871b1bSChris Mason goto out_unlock; 350044871b1bSChris Mason 350199d8f83cSChris Mason return __push_leaf_right(trans, root, path, min_data_size, empty, 350299d8f83cSChris Mason right, free_space, left_nritems, min_slot); 350344871b1bSChris Mason out_unlock: 350444871b1bSChris Mason btrfs_tree_unlock(right); 350544871b1bSChris Mason free_extent_buffer(right); 350644871b1bSChris Mason return 1; 350744871b1bSChris Mason } 350844871b1bSChris Mason 350944871b1bSChris Mason /* 351044871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 351144871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 351299d8f83cSChris Mason * 351399d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 351499d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 351599d8f83cSChris Mason * items 351644871b1bSChris Mason */ 351744871b1bSChris Mason static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, 351844871b1bSChris Mason struct btrfs_root *root, 351944871b1bSChris Mason struct btrfs_path *path, int data_size, 352044871b1bSChris Mason int empty, struct extent_buffer *left, 352199d8f83cSChris Mason int free_space, u32 right_nritems, 352299d8f83cSChris Mason u32 max_slot) 352344871b1bSChris Mason { 35245f39d397SChris Mason struct btrfs_disk_key disk_key; 35255f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3526be0e5c09SChris Mason int i; 3527be0e5c09SChris Mason int push_space = 0; 3528be0e5c09SChris Mason int push_items = 0; 35290783fcfcSChris Mason struct btrfs_item *item; 35307518a238SChris Mason u32 old_left_nritems; 353134a38218SChris Mason u32 nr; 3532aa5d6bedSChris Mason int ret = 0; 3533db94535dSChris Mason u32 this_item_size; 3534db94535dSChris Mason u32 old_left_item_size; 3535cfed81a0SChris Mason struct btrfs_map_token token; 3536cfed81a0SChris Mason 3537cfed81a0SChris Mason btrfs_init_map_token(&token); 3538be0e5c09SChris Mason 353934a38218SChris Mason if (empty) 354099d8f83cSChris Mason nr = min(right_nritems, max_slot); 354134a38218SChris Mason else 354299d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 354334a38218SChris Mason 354434a38218SChris Mason for (i = 0; i < nr; i++) { 35455f39d397SChris Mason item = btrfs_item_nr(right, i); 3546db94535dSChris Mason 354731840ae1SZheng Yan if (!empty && push_items > 0) { 354831840ae1SZheng Yan if (path->slots[0] < i) 354931840ae1SZheng Yan break; 355031840ae1SZheng Yan if (path->slots[0] == i) { 355131840ae1SZheng Yan int space = btrfs_leaf_free_space(root, right); 355231840ae1SZheng Yan if (space + push_space * 2 > free_space) 355331840ae1SZheng Yan break; 355431840ae1SZheng Yan } 355531840ae1SZheng Yan } 355631840ae1SZheng Yan 3557be0e5c09SChris Mason if (path->slots[0] == i) 355887b29b20SYan Zheng push_space += data_size; 3559db94535dSChris Mason 3560db94535dSChris Mason this_item_size = btrfs_item_size(right, item); 3561db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 3562be0e5c09SChris Mason break; 3563db94535dSChris Mason 3564be0e5c09SChris Mason push_items++; 3565db94535dSChris Mason push_space += this_item_size + sizeof(*item); 3566be0e5c09SChris Mason } 3567db94535dSChris Mason 3568be0e5c09SChris Mason if (push_items == 0) { 3569925baeddSChris Mason ret = 1; 3570925baeddSChris Mason goto out; 3571be0e5c09SChris Mason } 357234a38218SChris Mason if (!empty && push_items == btrfs_header_nritems(right)) 3573a429e513SChris Mason WARN_ON(1); 35745f39d397SChris Mason 3575be0e5c09SChris Mason /* push data from right to left */ 35765f39d397SChris Mason copy_extent_buffer(left, right, 35775f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 35785f39d397SChris Mason btrfs_item_nr_offset(0), 35795f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 35805f39d397SChris Mason 3581123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root) - 35825f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1); 35835f39d397SChris Mason 35845f39d397SChris Mason copy_extent_buffer(left, right, btrfs_leaf_data(left) + 3585d6025579SChris Mason leaf_data_end(root, left) - push_space, 3586123abc88SChris Mason btrfs_leaf_data(right) + 35875f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1), 3588be0e5c09SChris Mason push_space); 35895f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 359087b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3591eb60ceacSChris Mason 3592db94535dSChris Mason old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1); 3593be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 35945f39d397SChris Mason u32 ioff; 3595db94535dSChris Mason 35965f39d397SChris Mason item = btrfs_item_nr(left, i); 3597db94535dSChris Mason 3598cfed81a0SChris Mason ioff = btrfs_token_item_offset(left, item, &token); 3599cfed81a0SChris Mason btrfs_set_token_item_offset(left, item, 3600cfed81a0SChris Mason ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size), 3601cfed81a0SChris Mason &token); 3602be0e5c09SChris Mason } 36035f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3604be0e5c09SChris Mason 3605be0e5c09SChris Mason /* fixup right node */ 360634a38218SChris Mason if (push_items > right_nritems) { 3607d397712bSChris Mason printk(KERN_CRIT "push items %d nr %u\n", push_items, 3608d397712bSChris Mason right_nritems); 360934a38218SChris Mason WARN_ON(1); 361034a38218SChris Mason } 361134a38218SChris Mason 361234a38218SChris Mason if (push_items < right_nritems) { 36135f39d397SChris Mason push_space = btrfs_item_offset_nr(right, push_items - 1) - 3614123abc88SChris Mason leaf_data_end(root, right); 36155f39d397SChris Mason memmove_extent_buffer(right, btrfs_leaf_data(right) + 3616d6025579SChris Mason BTRFS_LEAF_DATA_SIZE(root) - push_space, 3617d6025579SChris Mason btrfs_leaf_data(right) + 3618123abc88SChris Mason leaf_data_end(root, right), push_space); 36195f39d397SChris Mason 36205f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 36215f39d397SChris Mason btrfs_item_nr_offset(push_items), 36225f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 36230783fcfcSChris Mason sizeof(struct btrfs_item)); 362434a38218SChris Mason } 3625eef1c494SYan right_nritems -= push_items; 3626eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 3627123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root); 36285f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 36295f39d397SChris Mason item = btrfs_item_nr(right, i); 3630db94535dSChris Mason 3631cfed81a0SChris Mason push_space = push_space - btrfs_token_item_size(right, 3632cfed81a0SChris Mason item, &token); 3633cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3634db94535dSChris Mason } 3635eb60ceacSChris Mason 36365f39d397SChris Mason btrfs_mark_buffer_dirty(left); 363734a38218SChris Mason if (right_nritems) 36385f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3639f0486c68SYan, Zheng else 3640f0486c68SYan, Zheng clean_tree_block(trans, root, right); 3641098f59c2SChris Mason 36425f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3643143bede5SJeff Mahoney fixup_low_keys(trans, root, path, &disk_key, 1); 3644be0e5c09SChris Mason 3645be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3646be0e5c09SChris Mason if (path->slots[0] < push_items) { 3647be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3648925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 36495f39d397SChris Mason free_extent_buffer(path->nodes[0]); 36505f39d397SChris Mason path->nodes[0] = left; 3651be0e5c09SChris Mason path->slots[1] -= 1; 3652be0e5c09SChris Mason } else { 3653925baeddSChris Mason btrfs_tree_unlock(left); 36545f39d397SChris Mason free_extent_buffer(left); 3655be0e5c09SChris Mason path->slots[0] -= push_items; 3656be0e5c09SChris Mason } 3657eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3658aa5d6bedSChris Mason return ret; 3659925baeddSChris Mason out: 3660925baeddSChris Mason btrfs_tree_unlock(left); 3661925baeddSChris Mason free_extent_buffer(left); 3662925baeddSChris Mason return ret; 3663be0e5c09SChris Mason } 3664be0e5c09SChris Mason 366574123bd7SChris Mason /* 366644871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 366744871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 366899d8f83cSChris Mason * 366999d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 367099d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 367199d8f83cSChris Mason * items 367244871b1bSChris Mason */ 367344871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 367499d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 367599d8f83cSChris Mason int data_size, int empty, u32 max_slot) 367644871b1bSChris Mason { 367744871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 367844871b1bSChris Mason struct extent_buffer *left; 367944871b1bSChris Mason int slot; 368044871b1bSChris Mason int free_space; 368144871b1bSChris Mason u32 right_nritems; 368244871b1bSChris Mason int ret = 0; 368344871b1bSChris Mason 368444871b1bSChris Mason slot = path->slots[1]; 368544871b1bSChris Mason if (slot == 0) 368644871b1bSChris Mason return 1; 368744871b1bSChris Mason if (!path->nodes[1]) 368844871b1bSChris Mason return 1; 368944871b1bSChris Mason 369044871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 369144871b1bSChris Mason if (right_nritems == 0) 369244871b1bSChris Mason return 1; 369344871b1bSChris Mason 369444871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 369544871b1bSChris Mason 369644871b1bSChris Mason left = read_node_slot(root, path->nodes[1], slot - 1); 369791ca338dSTsutomu Itoh if (left == NULL) 369891ca338dSTsutomu Itoh return 1; 369991ca338dSTsutomu Itoh 370044871b1bSChris Mason btrfs_tree_lock(left); 370144871b1bSChris Mason btrfs_set_lock_blocking(left); 370244871b1bSChris Mason 370344871b1bSChris Mason free_space = btrfs_leaf_free_space(root, left); 370444871b1bSChris Mason if (free_space < data_size) { 370544871b1bSChris Mason ret = 1; 370644871b1bSChris Mason goto out; 370744871b1bSChris Mason } 370844871b1bSChris Mason 370944871b1bSChris Mason /* cow and double check */ 371044871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 371144871b1bSChris Mason path->nodes[1], slot - 1, &left); 371244871b1bSChris Mason if (ret) { 371344871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 371479787eaaSJeff Mahoney if (ret == -ENOSPC) 371544871b1bSChris Mason ret = 1; 371644871b1bSChris Mason goto out; 371744871b1bSChris Mason } 371844871b1bSChris Mason 371944871b1bSChris Mason free_space = btrfs_leaf_free_space(root, left); 372044871b1bSChris Mason if (free_space < data_size) { 372144871b1bSChris Mason ret = 1; 372244871b1bSChris Mason goto out; 372344871b1bSChris Mason } 372444871b1bSChris Mason 372599d8f83cSChris Mason return __push_leaf_left(trans, root, path, min_data_size, 372699d8f83cSChris Mason empty, left, free_space, right_nritems, 372799d8f83cSChris Mason max_slot); 372844871b1bSChris Mason out: 372944871b1bSChris Mason btrfs_tree_unlock(left); 373044871b1bSChris Mason free_extent_buffer(left); 373144871b1bSChris Mason return ret; 373244871b1bSChris Mason } 373344871b1bSChris Mason 373444871b1bSChris Mason /* 373574123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 373674123bd7SChris Mason * available for the resulting leaf level of the path. 373774123bd7SChris Mason */ 3738143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 3739e02119d5SChris Mason struct btrfs_root *root, 374044871b1bSChris Mason struct btrfs_path *path, 374144871b1bSChris Mason struct extent_buffer *l, 374244871b1bSChris Mason struct extent_buffer *right, 374344871b1bSChris Mason int slot, int mid, int nritems) 3744be0e5c09SChris Mason { 3745be0e5c09SChris Mason int data_copy_size; 3746be0e5c09SChris Mason int rt_data_off; 3747be0e5c09SChris Mason int i; 3748d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3749cfed81a0SChris Mason struct btrfs_map_token token; 3750cfed81a0SChris Mason 3751cfed81a0SChris Mason btrfs_init_map_token(&token); 3752be0e5c09SChris Mason 37535f39d397SChris Mason nritems = nritems - mid; 37545f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 37555f39d397SChris Mason data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l); 37565f39d397SChris Mason 37575f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 37585f39d397SChris Mason btrfs_item_nr_offset(mid), 37595f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 37605f39d397SChris Mason 37615f39d397SChris Mason copy_extent_buffer(right, l, 3762d6025579SChris Mason btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) - 3763123abc88SChris Mason data_copy_size, btrfs_leaf_data(l) + 3764123abc88SChris Mason leaf_data_end(root, l), data_copy_size); 376574123bd7SChris Mason 37665f39d397SChris Mason rt_data_off = BTRFS_LEAF_DATA_SIZE(root) - 37675f39d397SChris Mason btrfs_item_end_nr(l, mid); 37685f39d397SChris Mason 37695f39d397SChris Mason for (i = 0; i < nritems; i++) { 37705f39d397SChris Mason struct btrfs_item *item = btrfs_item_nr(right, i); 3771db94535dSChris Mason u32 ioff; 3772db94535dSChris Mason 3773cfed81a0SChris Mason ioff = btrfs_token_item_offset(right, item, &token); 3774cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, 3775cfed81a0SChris Mason ioff + rt_data_off, &token); 37760783fcfcSChris Mason } 377774123bd7SChris Mason 37785f39d397SChris Mason btrfs_set_header_nritems(l, mid); 37795f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3780143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 3781c3e06965SJan Schmidt path->slots[1] + 1, 1); 37825f39d397SChris Mason 37835f39d397SChris Mason btrfs_mark_buffer_dirty(right); 37845f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3785eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 37865f39d397SChris Mason 3787be0e5c09SChris Mason if (mid <= slot) { 3788925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 37895f39d397SChris Mason free_extent_buffer(path->nodes[0]); 37905f39d397SChris Mason path->nodes[0] = right; 3791be0e5c09SChris Mason path->slots[0] -= mid; 3792be0e5c09SChris Mason path->slots[1] += 1; 3793925baeddSChris Mason } else { 3794925baeddSChris Mason btrfs_tree_unlock(right); 37955f39d397SChris Mason free_extent_buffer(right); 3796925baeddSChris Mason } 37975f39d397SChris Mason 3798eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 379944871b1bSChris Mason } 380044871b1bSChris Mason 380144871b1bSChris Mason /* 380299d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 380399d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 380499d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 380599d8f83cSChris Mason * A B C 380699d8f83cSChris Mason * 380799d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 380899d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 380999d8f83cSChris Mason * completely. 381099d8f83cSChris Mason */ 381199d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 381299d8f83cSChris Mason struct btrfs_root *root, 381399d8f83cSChris Mason struct btrfs_path *path, 381499d8f83cSChris Mason int data_size) 381599d8f83cSChris Mason { 381699d8f83cSChris Mason int ret; 381799d8f83cSChris Mason int progress = 0; 381899d8f83cSChris Mason int slot; 381999d8f83cSChris Mason u32 nritems; 382099d8f83cSChris Mason 382199d8f83cSChris Mason slot = path->slots[0]; 382299d8f83cSChris Mason 382399d8f83cSChris Mason /* 382499d8f83cSChris Mason * try to push all the items after our slot into the 382599d8f83cSChris Mason * right leaf 382699d8f83cSChris Mason */ 382799d8f83cSChris Mason ret = push_leaf_right(trans, root, path, 1, data_size, 0, slot); 382899d8f83cSChris Mason if (ret < 0) 382999d8f83cSChris Mason return ret; 383099d8f83cSChris Mason 383199d8f83cSChris Mason if (ret == 0) 383299d8f83cSChris Mason progress++; 383399d8f83cSChris Mason 383499d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 383599d8f83cSChris Mason /* 383699d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 383799d8f83cSChris Mason * we've done so we're done 383899d8f83cSChris Mason */ 383999d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 384099d8f83cSChris Mason return 0; 384199d8f83cSChris Mason 384299d8f83cSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size) 384399d8f83cSChris Mason return 0; 384499d8f83cSChris Mason 384599d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 384699d8f83cSChris Mason slot = path->slots[0]; 384799d8f83cSChris Mason ret = push_leaf_left(trans, root, path, 1, data_size, 0, slot); 384899d8f83cSChris Mason if (ret < 0) 384999d8f83cSChris Mason return ret; 385099d8f83cSChris Mason 385199d8f83cSChris Mason if (ret == 0) 385299d8f83cSChris Mason progress++; 385399d8f83cSChris Mason 385499d8f83cSChris Mason if (progress) 385599d8f83cSChris Mason return 0; 385699d8f83cSChris Mason return 1; 385799d8f83cSChris Mason } 385899d8f83cSChris Mason 385999d8f83cSChris Mason /* 386044871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 386144871b1bSChris Mason * available for the resulting leaf level of the path. 386244871b1bSChris Mason * 386344871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 386444871b1bSChris Mason */ 386544871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 386644871b1bSChris Mason struct btrfs_root *root, 386744871b1bSChris Mason struct btrfs_key *ins_key, 386844871b1bSChris Mason struct btrfs_path *path, int data_size, 386944871b1bSChris Mason int extend) 387044871b1bSChris Mason { 38715d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 387244871b1bSChris Mason struct extent_buffer *l; 387344871b1bSChris Mason u32 nritems; 387444871b1bSChris Mason int mid; 387544871b1bSChris Mason int slot; 387644871b1bSChris Mason struct extent_buffer *right; 387744871b1bSChris Mason int ret = 0; 387844871b1bSChris Mason int wret; 38795d4f98a2SYan Zheng int split; 388044871b1bSChris Mason int num_doubles = 0; 388199d8f83cSChris Mason int tried_avoid_double = 0; 388244871b1bSChris Mason 3883a5719521SYan, Zheng l = path->nodes[0]; 3884a5719521SYan, Zheng slot = path->slots[0]; 3885a5719521SYan, Zheng if (extend && data_size + btrfs_item_size_nr(l, slot) + 3886a5719521SYan, Zheng sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(root)) 3887a5719521SYan, Zheng return -EOVERFLOW; 3888a5719521SYan, Zheng 388944871b1bSChris Mason /* first try to make some room by pushing left and right */ 389099d8f83cSChris Mason if (data_size) { 389199d8f83cSChris Mason wret = push_leaf_right(trans, root, path, data_size, 389299d8f83cSChris Mason data_size, 0, 0); 389344871b1bSChris Mason if (wret < 0) 389444871b1bSChris Mason return wret; 389544871b1bSChris Mason if (wret) { 389699d8f83cSChris Mason wret = push_leaf_left(trans, root, path, data_size, 389799d8f83cSChris Mason data_size, 0, (u32)-1); 389844871b1bSChris Mason if (wret < 0) 389944871b1bSChris Mason return wret; 390044871b1bSChris Mason } 390144871b1bSChris Mason l = path->nodes[0]; 390244871b1bSChris Mason 390344871b1bSChris Mason /* did the pushes work? */ 390444871b1bSChris Mason if (btrfs_leaf_free_space(root, l) >= data_size) 390544871b1bSChris Mason return 0; 390644871b1bSChris Mason } 390744871b1bSChris Mason 390844871b1bSChris Mason if (!path->nodes[1]) { 390944871b1bSChris Mason ret = insert_new_root(trans, root, path, 1); 391044871b1bSChris Mason if (ret) 391144871b1bSChris Mason return ret; 391244871b1bSChris Mason } 391344871b1bSChris Mason again: 39145d4f98a2SYan Zheng split = 1; 391544871b1bSChris Mason l = path->nodes[0]; 391644871b1bSChris Mason slot = path->slots[0]; 391744871b1bSChris Mason nritems = btrfs_header_nritems(l); 391844871b1bSChris Mason mid = (nritems + 1) / 2; 391944871b1bSChris Mason 39205d4f98a2SYan Zheng if (mid <= slot) { 39215d4f98a2SYan Zheng if (nritems == 1 || 39225d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 39235d4f98a2SYan Zheng BTRFS_LEAF_DATA_SIZE(root)) { 39245d4f98a2SYan Zheng if (slot >= nritems) { 39255d4f98a2SYan Zheng split = 0; 39265d4f98a2SYan Zheng } else { 39275d4f98a2SYan Zheng mid = slot; 39285d4f98a2SYan Zheng if (mid != nritems && 39295d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 39305d4f98a2SYan Zheng data_size > BTRFS_LEAF_DATA_SIZE(root)) { 393199d8f83cSChris Mason if (data_size && !tried_avoid_double) 393299d8f83cSChris Mason goto push_for_double; 39335d4f98a2SYan Zheng split = 2; 39345d4f98a2SYan Zheng } 39355d4f98a2SYan Zheng } 39365d4f98a2SYan Zheng } 39375d4f98a2SYan Zheng } else { 39385d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 39395d4f98a2SYan Zheng BTRFS_LEAF_DATA_SIZE(root)) { 39405d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 39415d4f98a2SYan Zheng split = 0; 39425d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 39435d4f98a2SYan Zheng mid = 1; 39445d4f98a2SYan Zheng } else { 39455d4f98a2SYan Zheng mid = slot; 39465d4f98a2SYan Zheng if (mid != nritems && 39475d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 39485d4f98a2SYan Zheng data_size > BTRFS_LEAF_DATA_SIZE(root)) { 394999d8f83cSChris Mason if (data_size && !tried_avoid_double) 395099d8f83cSChris Mason goto push_for_double; 39515d4f98a2SYan Zheng split = 2 ; 39525d4f98a2SYan Zheng } 39535d4f98a2SYan Zheng } 39545d4f98a2SYan Zheng } 39555d4f98a2SYan Zheng } 39565d4f98a2SYan Zheng 39575d4f98a2SYan Zheng if (split == 0) 39585d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 39595d4f98a2SYan Zheng else 39605d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 39615d4f98a2SYan Zheng 39625d4f98a2SYan Zheng right = btrfs_alloc_free_block(trans, root, root->leafsize, 0, 396344871b1bSChris Mason root->root_key.objectid, 39645581a51aSJan Schmidt &disk_key, 0, l->start, 0); 3965f0486c68SYan, Zheng if (IS_ERR(right)) 396644871b1bSChris Mason return PTR_ERR(right); 3967f0486c68SYan, Zheng 3968f0486c68SYan, Zheng root_add_used(root, root->leafsize); 396944871b1bSChris Mason 397044871b1bSChris Mason memset_extent_buffer(right, 0, 0, sizeof(struct btrfs_header)); 397144871b1bSChris Mason btrfs_set_header_bytenr(right, right->start); 397244871b1bSChris Mason btrfs_set_header_generation(right, trans->transid); 39735d4f98a2SYan Zheng btrfs_set_header_backref_rev(right, BTRFS_MIXED_BACKREF_REV); 397444871b1bSChris Mason btrfs_set_header_owner(right, root->root_key.objectid); 397544871b1bSChris Mason btrfs_set_header_level(right, 0); 397644871b1bSChris Mason write_extent_buffer(right, root->fs_info->fsid, 397744871b1bSChris Mason (unsigned long)btrfs_header_fsid(right), 397844871b1bSChris Mason BTRFS_FSID_SIZE); 397944871b1bSChris Mason 398044871b1bSChris Mason write_extent_buffer(right, root->fs_info->chunk_tree_uuid, 398144871b1bSChris Mason (unsigned long)btrfs_header_chunk_tree_uuid(right), 398244871b1bSChris Mason BTRFS_UUID_SIZE); 398344871b1bSChris Mason 39845d4f98a2SYan Zheng if (split == 0) { 398544871b1bSChris Mason if (mid <= slot) { 398644871b1bSChris Mason btrfs_set_header_nritems(right, 0); 3987143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 3988c3e06965SJan Schmidt path->slots[1] + 1, 1); 398944871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 399044871b1bSChris Mason free_extent_buffer(path->nodes[0]); 399144871b1bSChris Mason path->nodes[0] = right; 399244871b1bSChris Mason path->slots[0] = 0; 399344871b1bSChris Mason path->slots[1] += 1; 399444871b1bSChris Mason } else { 399544871b1bSChris Mason btrfs_set_header_nritems(right, 0); 3996143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 3997c3e06965SJan Schmidt path->slots[1], 1); 399844871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 399944871b1bSChris Mason free_extent_buffer(path->nodes[0]); 400044871b1bSChris Mason path->nodes[0] = right; 400144871b1bSChris Mason path->slots[0] = 0; 4002143bede5SJeff Mahoney if (path->slots[1] == 0) 4003143bede5SJeff Mahoney fixup_low_keys(trans, root, path, 4004143bede5SJeff Mahoney &disk_key, 1); 40055d4f98a2SYan Zheng } 400644871b1bSChris Mason btrfs_mark_buffer_dirty(right); 400744871b1bSChris Mason return ret; 400844871b1bSChris Mason } 400944871b1bSChris Mason 4010143bede5SJeff Mahoney copy_for_split(trans, root, path, l, right, slot, mid, nritems); 401144871b1bSChris Mason 40125d4f98a2SYan Zheng if (split == 2) { 4013cc0c5538SChris Mason BUG_ON(num_doubles != 0); 4014cc0c5538SChris Mason num_doubles++; 4015cc0c5538SChris Mason goto again; 40163326d1b0SChris Mason } 401744871b1bSChris Mason 4018143bede5SJeff Mahoney return 0; 401999d8f83cSChris Mason 402099d8f83cSChris Mason push_for_double: 402199d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 402299d8f83cSChris Mason tried_avoid_double = 1; 402399d8f83cSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size) 402499d8f83cSChris Mason return 0; 402599d8f83cSChris Mason goto again; 4026be0e5c09SChris Mason } 4027be0e5c09SChris Mason 4028ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 4029ad48fd75SYan, Zheng struct btrfs_root *root, 4030ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 4031ad48fd75SYan, Zheng { 4032ad48fd75SYan, Zheng struct btrfs_key key; 4033ad48fd75SYan, Zheng struct extent_buffer *leaf; 4034ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 4035ad48fd75SYan, Zheng u64 extent_len = 0; 4036ad48fd75SYan, Zheng u32 item_size; 4037ad48fd75SYan, Zheng int ret; 4038ad48fd75SYan, Zheng 4039ad48fd75SYan, Zheng leaf = path->nodes[0]; 4040ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4041ad48fd75SYan, Zheng 4042ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 4043ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 4044ad48fd75SYan, Zheng 4045ad48fd75SYan, Zheng if (btrfs_leaf_free_space(root, leaf) >= ins_len) 4046ad48fd75SYan, Zheng return 0; 4047ad48fd75SYan, Zheng 4048ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4049ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4050ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4051ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4052ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 4053ad48fd75SYan, Zheng } 4054b3b4aa74SDavid Sterba btrfs_release_path(path); 4055ad48fd75SYan, Zheng 4056ad48fd75SYan, Zheng path->keep_locks = 1; 4057ad48fd75SYan, Zheng path->search_for_split = 1; 4058ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 4059ad48fd75SYan, Zheng path->search_for_split = 0; 4060ad48fd75SYan, Zheng if (ret < 0) 4061ad48fd75SYan, Zheng goto err; 4062ad48fd75SYan, Zheng 4063ad48fd75SYan, Zheng ret = -EAGAIN; 4064ad48fd75SYan, Zheng leaf = path->nodes[0]; 4065ad48fd75SYan, Zheng /* if our item isn't there or got smaller, return now */ 4066ad48fd75SYan, Zheng if (ret > 0 || item_size != btrfs_item_size_nr(leaf, path->slots[0])) 4067ad48fd75SYan, Zheng goto err; 4068ad48fd75SYan, Zheng 4069109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 4070109f6aefSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= ins_len) 4071109f6aefSChris Mason goto err; 4072109f6aefSChris Mason 4073ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4074ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4075ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4076ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 4077ad48fd75SYan, Zheng goto err; 4078ad48fd75SYan, Zheng } 4079ad48fd75SYan, Zheng 4080ad48fd75SYan, Zheng btrfs_set_path_blocking(path); 4081ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 4082f0486c68SYan, Zheng if (ret) 4083f0486c68SYan, Zheng goto err; 4084ad48fd75SYan, Zheng 4085ad48fd75SYan, Zheng path->keep_locks = 0; 4086ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 4087ad48fd75SYan, Zheng return 0; 4088ad48fd75SYan, Zheng err: 4089ad48fd75SYan, Zheng path->keep_locks = 0; 4090ad48fd75SYan, Zheng return ret; 4091ad48fd75SYan, Zheng } 4092ad48fd75SYan, Zheng 4093ad48fd75SYan, Zheng static noinline int split_item(struct btrfs_trans_handle *trans, 4094459931ecSChris Mason struct btrfs_root *root, 4095459931ecSChris Mason struct btrfs_path *path, 4096459931ecSChris Mason struct btrfs_key *new_key, 4097459931ecSChris Mason unsigned long split_offset) 4098459931ecSChris Mason { 4099459931ecSChris Mason struct extent_buffer *leaf; 4100459931ecSChris Mason struct btrfs_item *item; 4101459931ecSChris Mason struct btrfs_item *new_item; 4102459931ecSChris Mason int slot; 4103ad48fd75SYan, Zheng char *buf; 4104459931ecSChris Mason u32 nritems; 4105ad48fd75SYan, Zheng u32 item_size; 4106459931ecSChris Mason u32 orig_offset; 4107459931ecSChris Mason struct btrfs_disk_key disk_key; 4108459931ecSChris Mason 4109459931ecSChris Mason leaf = path->nodes[0]; 4110b9473439SChris Mason BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item)); 4111b9473439SChris Mason 4112b4ce94deSChris Mason btrfs_set_path_blocking(path); 4113b4ce94deSChris Mason 4114459931ecSChris Mason item = btrfs_item_nr(leaf, path->slots[0]); 4115459931ecSChris Mason orig_offset = btrfs_item_offset(leaf, item); 4116459931ecSChris Mason item_size = btrfs_item_size(leaf, item); 4117459931ecSChris Mason 4118459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 4119ad48fd75SYan, Zheng if (!buf) 4120ad48fd75SYan, Zheng return -ENOMEM; 4121ad48fd75SYan, Zheng 4122459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 4123459931ecSChris Mason path->slots[0]), item_size); 4124ad48fd75SYan, Zheng 4125459931ecSChris Mason slot = path->slots[0] + 1; 4126459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 4127459931ecSChris Mason if (slot != nritems) { 4128459931ecSChris Mason /* shift the items */ 4129459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 4130459931ecSChris Mason btrfs_item_nr_offset(slot), 4131459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4132459931ecSChris Mason } 4133459931ecSChris Mason 4134459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 4135459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4136459931ecSChris Mason 4137459931ecSChris Mason new_item = btrfs_item_nr(leaf, slot); 4138459931ecSChris Mason 4139459931ecSChris Mason btrfs_set_item_offset(leaf, new_item, orig_offset); 4140459931ecSChris Mason btrfs_set_item_size(leaf, new_item, item_size - split_offset); 4141459931ecSChris Mason 4142459931ecSChris Mason btrfs_set_item_offset(leaf, item, 4143459931ecSChris Mason orig_offset + item_size - split_offset); 4144459931ecSChris Mason btrfs_set_item_size(leaf, item, split_offset); 4145459931ecSChris Mason 4146459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 4147459931ecSChris Mason 4148459931ecSChris Mason /* write the data for the start of the original item */ 4149459931ecSChris Mason write_extent_buffer(leaf, buf, 4150459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 4151459931ecSChris Mason split_offset); 4152459931ecSChris Mason 4153459931ecSChris Mason /* write the data for the new item */ 4154459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 4155459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 4156459931ecSChris Mason item_size - split_offset); 4157459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 4158459931ecSChris Mason 4159ad48fd75SYan, Zheng BUG_ON(btrfs_leaf_free_space(root, leaf) < 0); 4160459931ecSChris Mason kfree(buf); 4161ad48fd75SYan, Zheng return 0; 4162ad48fd75SYan, Zheng } 4163ad48fd75SYan, Zheng 4164ad48fd75SYan, Zheng /* 4165ad48fd75SYan, Zheng * This function splits a single item into two items, 4166ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 4167ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 4168ad48fd75SYan, Zheng * 4169ad48fd75SYan, Zheng * The path may be released by this operation. After 4170ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 4171ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 4172ad48fd75SYan, Zheng * 4173ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 4174ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 4175ad48fd75SYan, Zheng * 4176ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 4177ad48fd75SYan, Zheng * leaf the entire time. 4178ad48fd75SYan, Zheng */ 4179ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 4180ad48fd75SYan, Zheng struct btrfs_root *root, 4181ad48fd75SYan, Zheng struct btrfs_path *path, 4182ad48fd75SYan, Zheng struct btrfs_key *new_key, 4183ad48fd75SYan, Zheng unsigned long split_offset) 4184ad48fd75SYan, Zheng { 4185ad48fd75SYan, Zheng int ret; 4186ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4187ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 4188ad48fd75SYan, Zheng if (ret) 4189459931ecSChris Mason return ret; 4190ad48fd75SYan, Zheng 4191ad48fd75SYan, Zheng ret = split_item(trans, root, path, new_key, split_offset); 4192ad48fd75SYan, Zheng return ret; 4193ad48fd75SYan, Zheng } 4194ad48fd75SYan, Zheng 4195ad48fd75SYan, Zheng /* 4196ad48fd75SYan, Zheng * This function duplicate a item, giving 'new_key' to the new item. 4197ad48fd75SYan, Zheng * It guarantees both items live in the same tree leaf and the new item 4198ad48fd75SYan, Zheng * is contiguous with the original item. 4199ad48fd75SYan, Zheng * 4200ad48fd75SYan, Zheng * This allows us to split file extent in place, keeping a lock on the 4201ad48fd75SYan, Zheng * leaf the entire time. 4202ad48fd75SYan, Zheng */ 4203ad48fd75SYan, Zheng int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4204ad48fd75SYan, Zheng struct btrfs_root *root, 4205ad48fd75SYan, Zheng struct btrfs_path *path, 4206ad48fd75SYan, Zheng struct btrfs_key *new_key) 4207ad48fd75SYan, Zheng { 4208ad48fd75SYan, Zheng struct extent_buffer *leaf; 4209ad48fd75SYan, Zheng int ret; 4210ad48fd75SYan, Zheng u32 item_size; 4211ad48fd75SYan, Zheng 4212ad48fd75SYan, Zheng leaf = path->nodes[0]; 4213ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4214ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4215ad48fd75SYan, Zheng item_size + sizeof(struct btrfs_item)); 4216ad48fd75SYan, Zheng if (ret) 4217ad48fd75SYan, Zheng return ret; 4218ad48fd75SYan, Zheng 4219ad48fd75SYan, Zheng path->slots[0]++; 4220143bede5SJeff Mahoney setup_items_for_insert(trans, root, path, new_key, &item_size, 4221ad48fd75SYan, Zheng item_size, item_size + 4222ad48fd75SYan, Zheng sizeof(struct btrfs_item), 1); 4223ad48fd75SYan, Zheng leaf = path->nodes[0]; 4224ad48fd75SYan, Zheng memcpy_extent_buffer(leaf, 4225ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0]), 4226ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4227ad48fd75SYan, Zheng item_size); 4228ad48fd75SYan, Zheng return 0; 4229459931ecSChris Mason } 4230459931ecSChris Mason 4231459931ecSChris Mason /* 4232d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4233d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4234d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4235d352ac68SChris Mason * the front. 4236d352ac68SChris Mason */ 4237143bede5SJeff Mahoney void btrfs_truncate_item(struct btrfs_trans_handle *trans, 4238b18c6685SChris Mason struct btrfs_root *root, 4239b18c6685SChris Mason struct btrfs_path *path, 4240179e29e4SChris Mason u32 new_size, int from_end) 4241b18c6685SChris Mason { 4242b18c6685SChris Mason int slot; 42435f39d397SChris Mason struct extent_buffer *leaf; 42445f39d397SChris Mason struct btrfs_item *item; 4245b18c6685SChris Mason u32 nritems; 4246b18c6685SChris Mason unsigned int data_end; 4247b18c6685SChris Mason unsigned int old_data_start; 4248b18c6685SChris Mason unsigned int old_size; 4249b18c6685SChris Mason unsigned int size_diff; 4250b18c6685SChris Mason int i; 4251cfed81a0SChris Mason struct btrfs_map_token token; 4252cfed81a0SChris Mason 4253cfed81a0SChris Mason btrfs_init_map_token(&token); 4254b18c6685SChris Mason 42555f39d397SChris Mason leaf = path->nodes[0]; 4256179e29e4SChris Mason slot = path->slots[0]; 4257179e29e4SChris Mason 4258179e29e4SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4259179e29e4SChris Mason if (old_size == new_size) 4260143bede5SJeff Mahoney return; 4261b18c6685SChris Mason 42625f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4263b18c6685SChris Mason data_end = leaf_data_end(root, leaf); 4264b18c6685SChris Mason 42655f39d397SChris Mason old_data_start = btrfs_item_offset_nr(leaf, slot); 4266179e29e4SChris Mason 4267b18c6685SChris Mason size_diff = old_size - new_size; 4268b18c6685SChris Mason 4269b18c6685SChris Mason BUG_ON(slot < 0); 4270b18c6685SChris Mason BUG_ON(slot >= nritems); 4271b18c6685SChris Mason 4272b18c6685SChris Mason /* 4273b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4274b18c6685SChris Mason */ 4275b18c6685SChris Mason /* first correct the data pointers */ 4276b18c6685SChris Mason for (i = slot; i < nritems; i++) { 42775f39d397SChris Mason u32 ioff; 42785f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4279db94535dSChris Mason 4280cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4281cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4282cfed81a0SChris Mason ioff + size_diff, &token); 4283b18c6685SChris Mason } 4284db94535dSChris Mason 4285b18c6685SChris Mason /* shift the data */ 4286179e29e4SChris Mason if (from_end) { 42875f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4288b18c6685SChris Mason data_end + size_diff, btrfs_leaf_data(leaf) + 4289b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 4290179e29e4SChris Mason } else { 4291179e29e4SChris Mason struct btrfs_disk_key disk_key; 4292179e29e4SChris Mason u64 offset; 4293179e29e4SChris Mason 4294179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4295179e29e4SChris Mason 4296179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4297179e29e4SChris Mason unsigned long ptr; 4298179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4299179e29e4SChris Mason 4300179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4301179e29e4SChris Mason struct btrfs_file_extent_item); 4302179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4303179e29e4SChris Mason (unsigned long)fi - size_diff); 4304179e29e4SChris Mason 4305179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4306179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4307179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4308179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4309179e29e4SChris Mason (unsigned long)fi, 4310179e29e4SChris Mason offsetof(struct btrfs_file_extent_item, 4311179e29e4SChris Mason disk_bytenr)); 4312179e29e4SChris Mason } 4313179e29e4SChris Mason } 4314179e29e4SChris Mason 4315179e29e4SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4316179e29e4SChris Mason data_end + size_diff, btrfs_leaf_data(leaf) + 4317179e29e4SChris Mason data_end, old_data_start - data_end); 4318179e29e4SChris Mason 4319179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4320179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4321179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4322179e29e4SChris Mason if (slot == 0) 4323179e29e4SChris Mason fixup_low_keys(trans, root, path, &disk_key, 1); 4324179e29e4SChris Mason } 43255f39d397SChris Mason 43265f39d397SChris Mason item = btrfs_item_nr(leaf, slot); 43275f39d397SChris Mason btrfs_set_item_size(leaf, item, new_size); 43285f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4329b18c6685SChris Mason 43305f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 43315f39d397SChris Mason btrfs_print_leaf(root, leaf); 4332b18c6685SChris Mason BUG(); 43335f39d397SChris Mason } 4334b18c6685SChris Mason } 4335b18c6685SChris Mason 4336d352ac68SChris Mason /* 4337d352ac68SChris Mason * make the item pointed to by the path bigger, data_size is the new size. 4338d352ac68SChris Mason */ 4339143bede5SJeff Mahoney void btrfs_extend_item(struct btrfs_trans_handle *trans, 43405f39d397SChris Mason struct btrfs_root *root, struct btrfs_path *path, 43415f39d397SChris Mason u32 data_size) 43426567e837SChris Mason { 43436567e837SChris Mason int slot; 43445f39d397SChris Mason struct extent_buffer *leaf; 43455f39d397SChris Mason struct btrfs_item *item; 43466567e837SChris Mason u32 nritems; 43476567e837SChris Mason unsigned int data_end; 43486567e837SChris Mason unsigned int old_data; 43496567e837SChris Mason unsigned int old_size; 43506567e837SChris Mason int i; 4351cfed81a0SChris Mason struct btrfs_map_token token; 4352cfed81a0SChris Mason 4353cfed81a0SChris Mason btrfs_init_map_token(&token); 43546567e837SChris Mason 43555f39d397SChris Mason leaf = path->nodes[0]; 43566567e837SChris Mason 43575f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 43586567e837SChris Mason data_end = leaf_data_end(root, leaf); 43596567e837SChris Mason 43605f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < data_size) { 43615f39d397SChris Mason btrfs_print_leaf(root, leaf); 43626567e837SChris Mason BUG(); 43635f39d397SChris Mason } 43646567e837SChris Mason slot = path->slots[0]; 43655f39d397SChris Mason old_data = btrfs_item_end_nr(leaf, slot); 43666567e837SChris Mason 43676567e837SChris Mason BUG_ON(slot < 0); 43683326d1b0SChris Mason if (slot >= nritems) { 43693326d1b0SChris Mason btrfs_print_leaf(root, leaf); 4370d397712bSChris Mason printk(KERN_CRIT "slot %d too large, nritems %d\n", 4371d397712bSChris Mason slot, nritems); 43723326d1b0SChris Mason BUG_ON(1); 43733326d1b0SChris Mason } 43746567e837SChris Mason 43756567e837SChris Mason /* 43766567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 43776567e837SChris Mason */ 43786567e837SChris Mason /* first correct the data pointers */ 43796567e837SChris Mason for (i = slot; i < nritems; i++) { 43805f39d397SChris Mason u32 ioff; 43815f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4382db94535dSChris Mason 4383cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4384cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4385cfed81a0SChris Mason ioff - data_size, &token); 43866567e837SChris Mason } 43875f39d397SChris Mason 43886567e837SChris Mason /* shift the data */ 43895f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 43906567e837SChris Mason data_end - data_size, btrfs_leaf_data(leaf) + 43916567e837SChris Mason data_end, old_data - data_end); 43925f39d397SChris Mason 43936567e837SChris Mason data_end = old_data; 43945f39d397SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 43955f39d397SChris Mason item = btrfs_item_nr(leaf, slot); 43965f39d397SChris Mason btrfs_set_item_size(leaf, item, old_size + data_size); 43975f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 43986567e837SChris Mason 43995f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 44005f39d397SChris Mason btrfs_print_leaf(root, leaf); 44016567e837SChris Mason BUG(); 44025f39d397SChris Mason } 44036567e837SChris Mason } 44046567e837SChris Mason 440574123bd7SChris Mason /* 4406d352ac68SChris Mason * Given a key and some data, insert items into the tree. 440774123bd7SChris Mason * This does all the path init required, making room in the tree if needed. 4408f3465ca4SJosef Bacik * Returns the number of keys that were inserted. 4409f3465ca4SJosef Bacik */ 4410f3465ca4SJosef Bacik int btrfs_insert_some_items(struct btrfs_trans_handle *trans, 4411f3465ca4SJosef Bacik struct btrfs_root *root, 4412f3465ca4SJosef Bacik struct btrfs_path *path, 4413f3465ca4SJosef Bacik struct btrfs_key *cpu_key, u32 *data_size, 4414f3465ca4SJosef Bacik int nr) 4415f3465ca4SJosef Bacik { 4416f3465ca4SJosef Bacik struct extent_buffer *leaf; 4417f3465ca4SJosef Bacik struct btrfs_item *item; 4418f3465ca4SJosef Bacik int ret = 0; 4419f3465ca4SJosef Bacik int slot; 4420f3465ca4SJosef Bacik int i; 4421f3465ca4SJosef Bacik u32 nritems; 4422f3465ca4SJosef Bacik u32 total_data = 0; 4423f3465ca4SJosef Bacik u32 total_size = 0; 4424f3465ca4SJosef Bacik unsigned int data_end; 4425f3465ca4SJosef Bacik struct btrfs_disk_key disk_key; 4426f3465ca4SJosef Bacik struct btrfs_key found_key; 4427cfed81a0SChris Mason struct btrfs_map_token token; 4428cfed81a0SChris Mason 4429cfed81a0SChris Mason btrfs_init_map_token(&token); 4430f3465ca4SJosef Bacik 443187b29b20SYan Zheng for (i = 0; i < nr; i++) { 443287b29b20SYan Zheng if (total_size + data_size[i] + sizeof(struct btrfs_item) > 443387b29b20SYan Zheng BTRFS_LEAF_DATA_SIZE(root)) { 443487b29b20SYan Zheng break; 443587b29b20SYan Zheng nr = i; 443687b29b20SYan Zheng } 4437f3465ca4SJosef Bacik total_data += data_size[i]; 443887b29b20SYan Zheng total_size += data_size[i] + sizeof(struct btrfs_item); 443987b29b20SYan Zheng } 444087b29b20SYan Zheng BUG_ON(nr == 0); 4441f3465ca4SJosef Bacik 4442f3465ca4SJosef Bacik ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1); 4443f3465ca4SJosef Bacik if (ret == 0) 4444f3465ca4SJosef Bacik return -EEXIST; 4445f3465ca4SJosef Bacik if (ret < 0) 4446f3465ca4SJosef Bacik goto out; 4447f3465ca4SJosef Bacik 4448f3465ca4SJosef Bacik leaf = path->nodes[0]; 4449f3465ca4SJosef Bacik 4450f3465ca4SJosef Bacik nritems = btrfs_header_nritems(leaf); 4451f3465ca4SJosef Bacik data_end = leaf_data_end(root, leaf); 4452f3465ca4SJosef Bacik 4453f3465ca4SJosef Bacik if (btrfs_leaf_free_space(root, leaf) < total_size) { 4454f3465ca4SJosef Bacik for (i = nr; i >= 0; i--) { 4455f3465ca4SJosef Bacik total_data -= data_size[i]; 4456f3465ca4SJosef Bacik total_size -= data_size[i] + sizeof(struct btrfs_item); 4457f3465ca4SJosef Bacik if (total_size < btrfs_leaf_free_space(root, leaf)) 4458f3465ca4SJosef Bacik break; 4459f3465ca4SJosef Bacik } 4460f3465ca4SJosef Bacik nr = i; 4461f3465ca4SJosef Bacik } 4462f3465ca4SJosef Bacik 4463f3465ca4SJosef Bacik slot = path->slots[0]; 4464f3465ca4SJosef Bacik BUG_ON(slot < 0); 4465f3465ca4SJosef Bacik 4466f3465ca4SJosef Bacik if (slot != nritems) { 4467f3465ca4SJosef Bacik unsigned int old_data = btrfs_item_end_nr(leaf, slot); 4468f3465ca4SJosef Bacik 4469f3465ca4SJosef Bacik item = btrfs_item_nr(leaf, slot); 4470f3465ca4SJosef Bacik btrfs_item_key_to_cpu(leaf, &found_key, slot); 4471f3465ca4SJosef Bacik 4472f3465ca4SJosef Bacik /* figure out how many keys we can insert in here */ 4473f3465ca4SJosef Bacik total_data = data_size[0]; 4474f3465ca4SJosef Bacik for (i = 1; i < nr; i++) { 44755d4f98a2SYan Zheng if (btrfs_comp_cpu_keys(&found_key, cpu_key + i) <= 0) 4476f3465ca4SJosef Bacik break; 4477f3465ca4SJosef Bacik total_data += data_size[i]; 4478f3465ca4SJosef Bacik } 4479f3465ca4SJosef Bacik nr = i; 4480f3465ca4SJosef Bacik 4481f3465ca4SJosef Bacik if (old_data < data_end) { 4482f3465ca4SJosef Bacik btrfs_print_leaf(root, leaf); 4483d397712bSChris Mason printk(KERN_CRIT "slot %d old_data %d data_end %d\n", 4484f3465ca4SJosef Bacik slot, old_data, data_end); 4485f3465ca4SJosef Bacik BUG_ON(1); 4486f3465ca4SJosef Bacik } 4487f3465ca4SJosef Bacik /* 4488f3465ca4SJosef Bacik * item0..itemN ... dataN.offset..dataN.size .. data0.size 4489f3465ca4SJosef Bacik */ 4490f3465ca4SJosef Bacik /* first correct the data pointers */ 4491f3465ca4SJosef Bacik for (i = slot; i < nritems; i++) { 4492f3465ca4SJosef Bacik u32 ioff; 4493f3465ca4SJosef Bacik 4494f3465ca4SJosef Bacik item = btrfs_item_nr(leaf, i); 4495cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4496cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4497cfed81a0SChris Mason ioff - total_data, &token); 4498f3465ca4SJosef Bacik } 4499f3465ca4SJosef Bacik /* shift the items */ 4500f3465ca4SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), 4501f3465ca4SJosef Bacik btrfs_item_nr_offset(slot), 4502f3465ca4SJosef Bacik (nritems - slot) * sizeof(struct btrfs_item)); 4503f3465ca4SJosef Bacik 4504f3465ca4SJosef Bacik /* shift the data */ 4505f3465ca4SJosef Bacik memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4506f3465ca4SJosef Bacik data_end - total_data, btrfs_leaf_data(leaf) + 4507f3465ca4SJosef Bacik data_end, old_data - data_end); 4508f3465ca4SJosef Bacik data_end = old_data; 4509f3465ca4SJosef Bacik } else { 4510f3465ca4SJosef Bacik /* 4511f3465ca4SJosef Bacik * this sucks but it has to be done, if we are inserting at 4512f3465ca4SJosef Bacik * the end of the leaf only insert 1 of the items, since we 4513f3465ca4SJosef Bacik * have no way of knowing whats on the next leaf and we'd have 4514f3465ca4SJosef Bacik * to drop our current locks to figure it out 4515f3465ca4SJosef Bacik */ 4516f3465ca4SJosef Bacik nr = 1; 4517f3465ca4SJosef Bacik } 4518f3465ca4SJosef Bacik 4519f3465ca4SJosef Bacik /* setup the item for the new data */ 4520f3465ca4SJosef Bacik for (i = 0; i < nr; i++) { 4521f3465ca4SJosef Bacik btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); 4522f3465ca4SJosef Bacik btrfs_set_item_key(leaf, &disk_key, slot + i); 4523f3465ca4SJosef Bacik item = btrfs_item_nr(leaf, slot + i); 4524cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4525cfed81a0SChris Mason data_end - data_size[i], &token); 4526f3465ca4SJosef Bacik data_end -= data_size[i]; 4527cfed81a0SChris Mason btrfs_set_token_item_size(leaf, item, data_size[i], &token); 4528f3465ca4SJosef Bacik } 4529f3465ca4SJosef Bacik btrfs_set_header_nritems(leaf, nritems + nr); 4530f3465ca4SJosef Bacik btrfs_mark_buffer_dirty(leaf); 4531f3465ca4SJosef Bacik 4532f3465ca4SJosef Bacik ret = 0; 4533f3465ca4SJosef Bacik if (slot == 0) { 4534f3465ca4SJosef Bacik btrfs_cpu_key_to_disk(&disk_key, cpu_key); 4535143bede5SJeff Mahoney fixup_low_keys(trans, root, path, &disk_key, 1); 4536f3465ca4SJosef Bacik } 4537f3465ca4SJosef Bacik 4538f3465ca4SJosef Bacik if (btrfs_leaf_free_space(root, leaf) < 0) { 4539f3465ca4SJosef Bacik btrfs_print_leaf(root, leaf); 4540f3465ca4SJosef Bacik BUG(); 4541f3465ca4SJosef Bacik } 4542f3465ca4SJosef Bacik out: 4543f3465ca4SJosef Bacik if (!ret) 4544f3465ca4SJosef Bacik ret = nr; 4545f3465ca4SJosef Bacik return ret; 4546f3465ca4SJosef Bacik } 4547f3465ca4SJosef Bacik 4548f3465ca4SJosef Bacik /* 454944871b1bSChris Mason * this is a helper for btrfs_insert_empty_items, the main goal here is 455044871b1bSChris Mason * to save stack depth by doing the bulk of the work in a function 455144871b1bSChris Mason * that doesn't call btrfs_search_slot 455274123bd7SChris Mason */ 4553143bede5SJeff Mahoney void setup_items_for_insert(struct btrfs_trans_handle *trans, 455444871b1bSChris Mason struct btrfs_root *root, struct btrfs_path *path, 45559c58309dSChris Mason struct btrfs_key *cpu_key, u32 *data_size, 455644871b1bSChris Mason u32 total_data, u32 total_size, int nr) 4557be0e5c09SChris Mason { 45585f39d397SChris Mason struct btrfs_item *item; 45599c58309dSChris Mason int i; 45607518a238SChris Mason u32 nritems; 4561be0e5c09SChris Mason unsigned int data_end; 4562e2fa7227SChris Mason struct btrfs_disk_key disk_key; 456344871b1bSChris Mason struct extent_buffer *leaf; 456444871b1bSChris Mason int slot; 4565cfed81a0SChris Mason struct btrfs_map_token token; 4566cfed81a0SChris Mason 4567cfed81a0SChris Mason btrfs_init_map_token(&token); 4568e2fa7227SChris Mason 45695f39d397SChris Mason leaf = path->nodes[0]; 457044871b1bSChris Mason slot = path->slots[0]; 457174123bd7SChris Mason 45725f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4573123abc88SChris Mason data_end = leaf_data_end(root, leaf); 4574eb60ceacSChris Mason 4575f25956ccSChris Mason if (btrfs_leaf_free_space(root, leaf) < total_size) { 45763326d1b0SChris Mason btrfs_print_leaf(root, leaf); 4577d397712bSChris Mason printk(KERN_CRIT "not enough freespace need %u have %d\n", 45789c58309dSChris Mason total_size, btrfs_leaf_free_space(root, leaf)); 4579be0e5c09SChris Mason BUG(); 4580d4dbff95SChris Mason } 45815f39d397SChris Mason 4582be0e5c09SChris Mason if (slot != nritems) { 45835f39d397SChris Mason unsigned int old_data = btrfs_item_end_nr(leaf, slot); 4584be0e5c09SChris Mason 45855f39d397SChris Mason if (old_data < data_end) { 45865f39d397SChris Mason btrfs_print_leaf(root, leaf); 4587d397712bSChris Mason printk(KERN_CRIT "slot %d old_data %d data_end %d\n", 45885f39d397SChris Mason slot, old_data, data_end); 45895f39d397SChris Mason BUG_ON(1); 45905f39d397SChris Mason } 4591be0e5c09SChris Mason /* 4592be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4593be0e5c09SChris Mason */ 4594be0e5c09SChris Mason /* first correct the data pointers */ 45950783fcfcSChris Mason for (i = slot; i < nritems; i++) { 45965f39d397SChris Mason u32 ioff; 4597db94535dSChris Mason 45985f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4599cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4600cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4601cfed81a0SChris Mason ioff - total_data, &token); 46020783fcfcSChris Mason } 4603be0e5c09SChris Mason /* shift the items */ 46049c58309dSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), 46055f39d397SChris Mason btrfs_item_nr_offset(slot), 46060783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4607be0e5c09SChris Mason 4608be0e5c09SChris Mason /* shift the data */ 46095f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 46109c58309dSChris Mason data_end - total_data, btrfs_leaf_data(leaf) + 4611be0e5c09SChris Mason data_end, old_data - data_end); 4612be0e5c09SChris Mason data_end = old_data; 4613be0e5c09SChris Mason } 46145f39d397SChris Mason 461562e2749eSChris Mason /* setup the item for the new data */ 46169c58309dSChris Mason for (i = 0; i < nr; i++) { 46179c58309dSChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); 46189c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 46199c58309dSChris Mason item = btrfs_item_nr(leaf, slot + i); 4620cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4621cfed81a0SChris Mason data_end - data_size[i], &token); 46229c58309dSChris Mason data_end -= data_size[i]; 4623cfed81a0SChris Mason btrfs_set_token_item_size(leaf, item, data_size[i], &token); 46249c58309dSChris Mason } 462544871b1bSChris Mason 46269c58309dSChris Mason btrfs_set_header_nritems(leaf, nritems + nr); 4627aa5d6bedSChris Mason 46285a01a2e3SChris Mason if (slot == 0) { 46295a01a2e3SChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key); 4630143bede5SJeff Mahoney fixup_low_keys(trans, root, path, &disk_key, 1); 46315a01a2e3SChris Mason } 4632b9473439SChris Mason btrfs_unlock_up_safe(path, 1); 4633b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4634aa5d6bedSChris Mason 46355f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 46365f39d397SChris Mason btrfs_print_leaf(root, leaf); 4637be0e5c09SChris Mason BUG(); 46385f39d397SChris Mason } 463944871b1bSChris Mason } 464044871b1bSChris Mason 464144871b1bSChris Mason /* 464244871b1bSChris Mason * Given a key and some data, insert items into the tree. 464344871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 464444871b1bSChris Mason */ 464544871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 464644871b1bSChris Mason struct btrfs_root *root, 464744871b1bSChris Mason struct btrfs_path *path, 464844871b1bSChris Mason struct btrfs_key *cpu_key, u32 *data_size, 464944871b1bSChris Mason int nr) 465044871b1bSChris Mason { 465144871b1bSChris Mason int ret = 0; 465244871b1bSChris Mason int slot; 465344871b1bSChris Mason int i; 465444871b1bSChris Mason u32 total_size = 0; 465544871b1bSChris Mason u32 total_data = 0; 465644871b1bSChris Mason 465744871b1bSChris Mason for (i = 0; i < nr; i++) 465844871b1bSChris Mason total_data += data_size[i]; 465944871b1bSChris Mason 466044871b1bSChris Mason total_size = total_data + (nr * sizeof(struct btrfs_item)); 466144871b1bSChris Mason ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1); 466244871b1bSChris Mason if (ret == 0) 466344871b1bSChris Mason return -EEXIST; 466444871b1bSChris Mason if (ret < 0) 4665143bede5SJeff Mahoney return ret; 466644871b1bSChris Mason 466744871b1bSChris Mason slot = path->slots[0]; 466844871b1bSChris Mason BUG_ON(slot < 0); 466944871b1bSChris Mason 4670143bede5SJeff Mahoney setup_items_for_insert(trans, root, path, cpu_key, data_size, 467144871b1bSChris Mason total_data, total_size, nr); 4672143bede5SJeff Mahoney return 0; 467362e2749eSChris Mason } 467462e2749eSChris Mason 467562e2749eSChris Mason /* 467662e2749eSChris Mason * Given a key and some data, insert an item into the tree. 467762e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 467862e2749eSChris Mason */ 4679e089f05cSChris Mason int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root 4680e089f05cSChris Mason *root, struct btrfs_key *cpu_key, void *data, u32 4681e089f05cSChris Mason data_size) 468262e2749eSChris Mason { 468362e2749eSChris Mason int ret = 0; 46842c90e5d6SChris Mason struct btrfs_path *path; 46855f39d397SChris Mason struct extent_buffer *leaf; 46865f39d397SChris Mason unsigned long ptr; 468762e2749eSChris Mason 46882c90e5d6SChris Mason path = btrfs_alloc_path(); 4689db5b493aSTsutomu Itoh if (!path) 4690db5b493aSTsutomu Itoh return -ENOMEM; 46912c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 469262e2749eSChris Mason if (!ret) { 46935f39d397SChris Mason leaf = path->nodes[0]; 46945f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 46955f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 46965f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 469762e2749eSChris Mason } 46982c90e5d6SChris Mason btrfs_free_path(path); 4699aa5d6bedSChris Mason return ret; 4700be0e5c09SChris Mason } 4701be0e5c09SChris Mason 470274123bd7SChris Mason /* 47035de08d7dSChris Mason * delete the pointer from a given node. 470474123bd7SChris Mason * 4705d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4706d352ac68SChris Mason * empty a node. 470774123bd7SChris Mason */ 4708143bede5SJeff Mahoney static void del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4709f3ea38daSJan Schmidt struct btrfs_path *path, int level, int slot, 4710f3ea38daSJan Schmidt int tree_mod_log) 4711be0e5c09SChris Mason { 47125f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 47137518a238SChris Mason u32 nritems; 4714f3ea38daSJan Schmidt int ret; 4715be0e5c09SChris Mason 47165f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4717be0e5c09SChris Mason if (slot != nritems - 1) { 4718f3ea38daSJan Schmidt if (tree_mod_log && level) 4719f3ea38daSJan Schmidt tree_mod_log_eb_move(root->fs_info, parent, slot, 4720f3ea38daSJan Schmidt slot + 1, nritems - slot - 1); 47215f39d397SChris Mason memmove_extent_buffer(parent, 47225f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 47235f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4724d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4725d6025579SChris Mason (nritems - slot - 1)); 4726f395694cSJan Schmidt } else if (tree_mod_log && level) { 4727f3ea38daSJan Schmidt ret = tree_mod_log_insert_key(root->fs_info, parent, slot, 4728f3ea38daSJan Schmidt MOD_LOG_KEY_REMOVE); 4729f3ea38daSJan Schmidt BUG_ON(ret < 0); 4730be0e5c09SChris Mason } 4731f3ea38daSJan Schmidt 47327518a238SChris Mason nritems--; 47335f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 47347518a238SChris Mason if (nritems == 0 && parent == root->node) { 47355f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4736eb60ceacSChris Mason /* just turn the root into a leaf and break */ 47375f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4738bb803951SChris Mason } else if (slot == 0) { 47395f39d397SChris Mason struct btrfs_disk_key disk_key; 47405f39d397SChris Mason 47415f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4742143bede5SJeff Mahoney fixup_low_keys(trans, root, path, &disk_key, level + 1); 4743be0e5c09SChris Mason } 4744d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4745be0e5c09SChris Mason } 4746be0e5c09SChris Mason 474774123bd7SChris Mason /* 4748323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 47495d4f98a2SYan Zheng * path->nodes[1]. 4750323ac95bSChris Mason * 4751323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4752323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4753323ac95bSChris Mason * 4754323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4755323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4756323ac95bSChris Mason */ 4757143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4758323ac95bSChris Mason struct btrfs_root *root, 47595d4f98a2SYan Zheng struct btrfs_path *path, 47605d4f98a2SYan Zheng struct extent_buffer *leaf) 4761323ac95bSChris Mason { 47625d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4763f3ea38daSJan Schmidt del_ptr(trans, root, path, 1, path->slots[1], 1); 4764323ac95bSChris Mason 47654d081c41SChris Mason /* 47664d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 47674d081c41SChris Mason * aren't holding any locks when we call it 47684d081c41SChris Mason */ 47694d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 47704d081c41SChris Mason 4771f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4772f0486c68SYan, Zheng 47733083ee2eSJosef Bacik extent_buffer_get(leaf); 47745581a51aSJan Schmidt btrfs_free_tree_block(trans, root, leaf, 0, 1); 47753083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4776323ac95bSChris Mason } 4777323ac95bSChris Mason /* 477874123bd7SChris Mason * delete the item at the leaf level in path. If that empties 477974123bd7SChris Mason * the leaf, remove it from the tree 478074123bd7SChris Mason */ 478185e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 478285e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4783be0e5c09SChris Mason { 47845f39d397SChris Mason struct extent_buffer *leaf; 47855f39d397SChris Mason struct btrfs_item *item; 478685e21bacSChris Mason int last_off; 478785e21bacSChris Mason int dsize = 0; 4788aa5d6bedSChris Mason int ret = 0; 4789aa5d6bedSChris Mason int wret; 479085e21bacSChris Mason int i; 47917518a238SChris Mason u32 nritems; 4792cfed81a0SChris Mason struct btrfs_map_token token; 4793cfed81a0SChris Mason 4794cfed81a0SChris Mason btrfs_init_map_token(&token); 4795be0e5c09SChris Mason 47965f39d397SChris Mason leaf = path->nodes[0]; 479785e21bacSChris Mason last_off = btrfs_item_offset_nr(leaf, slot + nr - 1); 479885e21bacSChris Mason 479985e21bacSChris Mason for (i = 0; i < nr; i++) 480085e21bacSChris Mason dsize += btrfs_item_size_nr(leaf, slot + i); 480185e21bacSChris Mason 48025f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4803be0e5c09SChris Mason 480485e21bacSChris Mason if (slot + nr != nritems) { 4805123abc88SChris Mason int data_end = leaf_data_end(root, leaf); 48065f39d397SChris Mason 48075f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4808d6025579SChris Mason data_end + dsize, 4809123abc88SChris Mason btrfs_leaf_data(leaf) + data_end, 481085e21bacSChris Mason last_off - data_end); 48115f39d397SChris Mason 481285e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 48135f39d397SChris Mason u32 ioff; 4814db94535dSChris Mason 48155f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4816cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4817cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4818cfed81a0SChris Mason ioff + dsize, &token); 48190783fcfcSChris Mason } 4820db94535dSChris Mason 48215f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 482285e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 48230783fcfcSChris Mason sizeof(struct btrfs_item) * 482485e21bacSChris Mason (nritems - slot - nr)); 4825be0e5c09SChris Mason } 482685e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 482785e21bacSChris Mason nritems -= nr; 48285f39d397SChris Mason 482974123bd7SChris Mason /* delete the leaf if we've emptied it */ 48307518a238SChris Mason if (nritems == 0) { 48315f39d397SChris Mason if (leaf == root->node) { 48325f39d397SChris Mason btrfs_set_header_level(leaf, 0); 48339a8dd150SChris Mason } else { 4834f0486c68SYan, Zheng btrfs_set_path_blocking(path); 4835f0486c68SYan, Zheng clean_tree_block(trans, root, leaf); 4836143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 48379a8dd150SChris Mason } 4838be0e5c09SChris Mason } else { 48397518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4840aa5d6bedSChris Mason if (slot == 0) { 48415f39d397SChris Mason struct btrfs_disk_key disk_key; 48425f39d397SChris Mason 48435f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4844143bede5SJeff Mahoney fixup_low_keys(trans, root, path, &disk_key, 1); 4845aa5d6bedSChris Mason } 4846aa5d6bedSChris Mason 484774123bd7SChris Mason /* delete the leaf if it is mostly empty */ 4848d717aa1dSYan Zheng if (used < BTRFS_LEAF_DATA_SIZE(root) / 3) { 4849be0e5c09SChris Mason /* push_leaf_left fixes the path. 4850be0e5c09SChris Mason * make sure the path still points to our leaf 4851be0e5c09SChris Mason * for possible call to del_ptr below 4852be0e5c09SChris Mason */ 48534920c9acSChris Mason slot = path->slots[1]; 48545f39d397SChris Mason extent_buffer_get(leaf); 48555f39d397SChris Mason 4856b9473439SChris Mason btrfs_set_path_blocking(path); 485799d8f83cSChris Mason wret = push_leaf_left(trans, root, path, 1, 1, 485899d8f83cSChris Mason 1, (u32)-1); 485954aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4860aa5d6bedSChris Mason ret = wret; 48615f39d397SChris Mason 48625f39d397SChris Mason if (path->nodes[0] == leaf && 48635f39d397SChris Mason btrfs_header_nritems(leaf)) { 486499d8f83cSChris Mason wret = push_leaf_right(trans, root, path, 1, 486599d8f83cSChris Mason 1, 1, 0); 486654aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4867aa5d6bedSChris Mason ret = wret; 4868aa5d6bedSChris Mason } 48695f39d397SChris Mason 48705f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4871323ac95bSChris Mason path->slots[1] = slot; 4872143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 48735f39d397SChris Mason free_extent_buffer(leaf); 4874143bede5SJeff Mahoney ret = 0; 48755de08d7dSChris Mason } else { 4876925baeddSChris Mason /* if we're still in the path, make sure 4877925baeddSChris Mason * we're dirty. Otherwise, one of the 4878925baeddSChris Mason * push_leaf functions must have already 4879925baeddSChris Mason * dirtied this buffer 4880925baeddSChris Mason */ 4881925baeddSChris Mason if (path->nodes[0] == leaf) 48825f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 48835f39d397SChris Mason free_extent_buffer(leaf); 4884be0e5c09SChris Mason } 4885d5719762SChris Mason } else { 48865f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4887be0e5c09SChris Mason } 48889a8dd150SChris Mason } 4889aa5d6bedSChris Mason return ret; 48909a8dd150SChris Mason } 48919a8dd150SChris Mason 489297571fd0SChris Mason /* 4893925baeddSChris Mason * search the tree again to find a leaf with lesser keys 48947bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 48957bb86316SChris Mason * returns < 0 on io errors. 4896d352ac68SChris Mason * 4897d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 4898d352ac68SChris Mason * time you call it. 48997bb86316SChris Mason */ 49007bb86316SChris Mason int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 49017bb86316SChris Mason { 4902925baeddSChris Mason struct btrfs_key key; 4903925baeddSChris Mason struct btrfs_disk_key found_key; 4904925baeddSChris Mason int ret; 49057bb86316SChris Mason 4906925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 4907925baeddSChris Mason 4908925baeddSChris Mason if (key.offset > 0) 4909925baeddSChris Mason key.offset--; 4910925baeddSChris Mason else if (key.type > 0) 4911925baeddSChris Mason key.type--; 4912925baeddSChris Mason else if (key.objectid > 0) 4913925baeddSChris Mason key.objectid--; 4914925baeddSChris Mason else 49157bb86316SChris Mason return 1; 49167bb86316SChris Mason 4917b3b4aa74SDavid Sterba btrfs_release_path(path); 4918925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4919925baeddSChris Mason if (ret < 0) 4920925baeddSChris Mason return ret; 4921925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 4922925baeddSChris Mason ret = comp_keys(&found_key, &key); 4923925baeddSChris Mason if (ret < 0) 49247bb86316SChris Mason return 0; 4925925baeddSChris Mason return 1; 49267bb86316SChris Mason } 49277bb86316SChris Mason 49283f157a2fSChris Mason /* 49293f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 49303f157a2fSChris Mason * for nodes or leaves that are either in cache or have a minimum 4931d352ac68SChris Mason * transaction id. This is used by the btree defrag code, and tree logging 49323f157a2fSChris Mason * 49333f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 49343f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 49353f157a2fSChris Mason * key and get a writable path. 49363f157a2fSChris Mason * 49373f157a2fSChris Mason * This does lock as it descends, and path->keep_locks should be set 49383f157a2fSChris Mason * to 1 by the caller. 49393f157a2fSChris Mason * 49403f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 49413f157a2fSChris Mason * of the tree. 49423f157a2fSChris Mason * 4943d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4944d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4945d352ac68SChris Mason * skipped over (without reading them). 4946d352ac68SChris Mason * 49473f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 49483f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 49493f157a2fSChris Mason */ 49503f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4951e02119d5SChris Mason struct btrfs_key *max_key, 49523f157a2fSChris Mason struct btrfs_path *path, int cache_only, 49533f157a2fSChris Mason u64 min_trans) 49543f157a2fSChris Mason { 49553f157a2fSChris Mason struct extent_buffer *cur; 49563f157a2fSChris Mason struct btrfs_key found_key; 49573f157a2fSChris Mason int slot; 49589652480bSYan int sret; 49593f157a2fSChris Mason u32 nritems; 49603f157a2fSChris Mason int level; 49613f157a2fSChris Mason int ret = 1; 49623f157a2fSChris Mason 4963934d375bSChris Mason WARN_ON(!path->keep_locks); 49643f157a2fSChris Mason again: 4965bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 49663f157a2fSChris Mason level = btrfs_header_level(cur); 4967e02119d5SChris Mason WARN_ON(path->nodes[level]); 49683f157a2fSChris Mason path->nodes[level] = cur; 4969bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 49703f157a2fSChris Mason 49713f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 49723f157a2fSChris Mason ret = 1; 49733f157a2fSChris Mason goto out; 49743f157a2fSChris Mason } 49753f157a2fSChris Mason while (1) { 49763f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 49773f157a2fSChris Mason level = btrfs_header_level(cur); 49789652480bSYan sret = bin_search(cur, min_key, level, &slot); 49793f157a2fSChris Mason 4980323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4981323ac95bSChris Mason if (level == path->lowest_level) { 4982e02119d5SChris Mason if (slot >= nritems) 4983e02119d5SChris Mason goto find_next_key; 49843f157a2fSChris Mason ret = 0; 49853f157a2fSChris Mason path->slots[level] = slot; 49863f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 49873f157a2fSChris Mason goto out; 49883f157a2fSChris Mason } 49899652480bSYan if (sret && slot > 0) 49909652480bSYan slot--; 49913f157a2fSChris Mason /* 49923f157a2fSChris Mason * check this node pointer against the cache_only and 49933f157a2fSChris Mason * min_trans parameters. If it isn't in cache or is too 49943f157a2fSChris Mason * old, skip to the next one. 49953f157a2fSChris Mason */ 49963f157a2fSChris Mason while (slot < nritems) { 49973f157a2fSChris Mason u64 blockptr; 49983f157a2fSChris Mason u64 gen; 49993f157a2fSChris Mason struct extent_buffer *tmp; 5000e02119d5SChris Mason struct btrfs_disk_key disk_key; 5001e02119d5SChris Mason 50023f157a2fSChris Mason blockptr = btrfs_node_blockptr(cur, slot); 50033f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 50043f157a2fSChris Mason if (gen < min_trans) { 50053f157a2fSChris Mason slot++; 50063f157a2fSChris Mason continue; 50073f157a2fSChris Mason } 50083f157a2fSChris Mason if (!cache_only) 50093f157a2fSChris Mason break; 50103f157a2fSChris Mason 5011e02119d5SChris Mason if (max_key) { 5012e02119d5SChris Mason btrfs_node_key(cur, &disk_key, slot); 5013e02119d5SChris Mason if (comp_keys(&disk_key, max_key) >= 0) { 5014e02119d5SChris Mason ret = 1; 5015e02119d5SChris Mason goto out; 5016e02119d5SChris Mason } 5017e02119d5SChris Mason } 5018e02119d5SChris Mason 50193f157a2fSChris Mason tmp = btrfs_find_tree_block(root, blockptr, 50203f157a2fSChris Mason btrfs_level_size(root, level - 1)); 50213f157a2fSChris Mason 5022b9fab919SChris Mason if (tmp && btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 50233f157a2fSChris Mason free_extent_buffer(tmp); 50243f157a2fSChris Mason break; 50253f157a2fSChris Mason } 50263f157a2fSChris Mason if (tmp) 50273f157a2fSChris Mason free_extent_buffer(tmp); 50283f157a2fSChris Mason slot++; 50293f157a2fSChris Mason } 5030e02119d5SChris Mason find_next_key: 50313f157a2fSChris Mason /* 50323f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 50333f157a2fSChris Mason * and find another one 50343f157a2fSChris Mason */ 50353f157a2fSChris Mason if (slot >= nritems) { 5036e02119d5SChris Mason path->slots[level] = slot; 5037b4ce94deSChris Mason btrfs_set_path_blocking(path); 5038e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 50393f157a2fSChris Mason cache_only, min_trans); 5040e02119d5SChris Mason if (sret == 0) { 5041b3b4aa74SDavid Sterba btrfs_release_path(path); 50423f157a2fSChris Mason goto again; 50433f157a2fSChris Mason } else { 50443f157a2fSChris Mason goto out; 50453f157a2fSChris Mason } 50463f157a2fSChris Mason } 50473f157a2fSChris Mason /* save our key for returning back */ 50483f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 50493f157a2fSChris Mason path->slots[level] = slot; 50503f157a2fSChris Mason if (level == path->lowest_level) { 50513f157a2fSChris Mason ret = 0; 5052f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 50533f157a2fSChris Mason goto out; 50543f157a2fSChris Mason } 5055b4ce94deSChris Mason btrfs_set_path_blocking(path); 50563f157a2fSChris Mason cur = read_node_slot(root, cur, slot); 505779787eaaSJeff Mahoney BUG_ON(!cur); /* -ENOMEM */ 50583f157a2fSChris Mason 5059bd681513SChris Mason btrfs_tree_read_lock(cur); 5060b4ce94deSChris Mason 5061bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 50623f157a2fSChris Mason path->nodes[level - 1] = cur; 5063f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 5064bd681513SChris Mason btrfs_clear_path_blocking(path, NULL, 0); 50653f157a2fSChris Mason } 50663f157a2fSChris Mason out: 50673f157a2fSChris Mason if (ret == 0) 50683f157a2fSChris Mason memcpy(min_key, &found_key, sizeof(found_key)); 5069b4ce94deSChris Mason btrfs_set_path_blocking(path); 50703f157a2fSChris Mason return ret; 50713f157a2fSChris Mason } 50723f157a2fSChris Mason 50733f157a2fSChris Mason /* 50743f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 50753f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 50763f157a2fSChris Mason * tree based on the current path and the cache_only and min_trans 50773f157a2fSChris Mason * parameters. 50783f157a2fSChris Mason * 50793f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 50803f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 50813f157a2fSChris Mason * 50823f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 50833f157a2fSChris Mason * calling this function. 50843f157a2fSChris Mason */ 5085e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 508633c66f43SYan Zheng struct btrfs_key *key, int level, 50873f157a2fSChris Mason int cache_only, u64 min_trans) 5088e7a84565SChris Mason { 5089e7a84565SChris Mason int slot; 5090e7a84565SChris Mason struct extent_buffer *c; 5091e7a84565SChris Mason 5092934d375bSChris Mason WARN_ON(!path->keep_locks); 5093e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 5094e7a84565SChris Mason if (!path->nodes[level]) 5095e7a84565SChris Mason return 1; 5096e7a84565SChris Mason 5097e7a84565SChris Mason slot = path->slots[level] + 1; 5098e7a84565SChris Mason c = path->nodes[level]; 50993f157a2fSChris Mason next: 5100e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 510133c66f43SYan Zheng int ret; 510233c66f43SYan Zheng int orig_lowest; 510333c66f43SYan Zheng struct btrfs_key cur_key; 510433c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 510533c66f43SYan Zheng !path->nodes[level + 1]) 5106e7a84565SChris Mason return 1; 510733c66f43SYan Zheng 510833c66f43SYan Zheng if (path->locks[level + 1]) { 510933c66f43SYan Zheng level++; 5110e7a84565SChris Mason continue; 5111e7a84565SChris Mason } 511233c66f43SYan Zheng 511333c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 511433c66f43SYan Zheng if (level == 0) 511533c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 511633c66f43SYan Zheng else 511733c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 511833c66f43SYan Zheng 511933c66f43SYan Zheng orig_lowest = path->lowest_level; 5120b3b4aa74SDavid Sterba btrfs_release_path(path); 512133c66f43SYan Zheng path->lowest_level = level; 512233c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 512333c66f43SYan Zheng 0, 0); 512433c66f43SYan Zheng path->lowest_level = orig_lowest; 512533c66f43SYan Zheng if (ret < 0) 512633c66f43SYan Zheng return ret; 512733c66f43SYan Zheng 512833c66f43SYan Zheng c = path->nodes[level]; 512933c66f43SYan Zheng slot = path->slots[level]; 513033c66f43SYan Zheng if (ret == 0) 513133c66f43SYan Zheng slot++; 513233c66f43SYan Zheng goto next; 513333c66f43SYan Zheng } 513433c66f43SYan Zheng 5135e7a84565SChris Mason if (level == 0) 5136e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 51373f157a2fSChris Mason else { 51383f157a2fSChris Mason u64 blockptr = btrfs_node_blockptr(c, slot); 51393f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 51403f157a2fSChris Mason 51413f157a2fSChris Mason if (cache_only) { 51423f157a2fSChris Mason struct extent_buffer *cur; 51433f157a2fSChris Mason cur = btrfs_find_tree_block(root, blockptr, 51443f157a2fSChris Mason btrfs_level_size(root, level - 1)); 5145b9fab919SChris Mason if (!cur || 5146b9fab919SChris Mason btrfs_buffer_uptodate(cur, gen, 1) <= 0) { 51473f157a2fSChris Mason slot++; 51483f157a2fSChris Mason if (cur) 51493f157a2fSChris Mason free_extent_buffer(cur); 51503f157a2fSChris Mason goto next; 51513f157a2fSChris Mason } 51523f157a2fSChris Mason free_extent_buffer(cur); 51533f157a2fSChris Mason } 51543f157a2fSChris Mason if (gen < min_trans) { 51553f157a2fSChris Mason slot++; 51563f157a2fSChris Mason goto next; 51573f157a2fSChris Mason } 5158e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 51593f157a2fSChris Mason } 5160e7a84565SChris Mason return 0; 5161e7a84565SChris Mason } 5162e7a84565SChris Mason return 1; 5163e7a84565SChris Mason } 5164e7a84565SChris Mason 51657bb86316SChris Mason /* 5166925baeddSChris Mason * search the tree again to find a leaf with greater keys 51670f70abe2SChris Mason * returns 0 if it found something or 1 if there are no greater leaves. 51680f70abe2SChris Mason * returns < 0 on io errors. 516997571fd0SChris Mason */ 5170234b63a0SChris Mason int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path) 5171d97e63b6SChris Mason { 51723d7806ecSJan Schmidt return btrfs_next_old_leaf(root, path, 0); 51733d7806ecSJan Schmidt } 51743d7806ecSJan Schmidt 51753d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 51763d7806ecSJan Schmidt u64 time_seq) 51773d7806ecSJan Schmidt { 5178d97e63b6SChris Mason int slot; 51798e73f275SChris Mason int level; 51805f39d397SChris Mason struct extent_buffer *c; 51818e73f275SChris Mason struct extent_buffer *next; 5182925baeddSChris Mason struct btrfs_key key; 5183925baeddSChris Mason u32 nritems; 5184925baeddSChris Mason int ret; 51858e73f275SChris Mason int old_spinning = path->leave_spinning; 5186bd681513SChris Mason int next_rw_lock = 0; 5187925baeddSChris Mason 5188925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5189d397712bSChris Mason if (nritems == 0) 5190925baeddSChris Mason return 1; 5191925baeddSChris Mason 51928e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 51938e73f275SChris Mason again: 51948e73f275SChris Mason level = 1; 51958e73f275SChris Mason next = NULL; 5196bd681513SChris Mason next_rw_lock = 0; 5197b3b4aa74SDavid Sterba btrfs_release_path(path); 51988e73f275SChris Mason 5199a2135011SChris Mason path->keep_locks = 1; 52008e73f275SChris Mason path->leave_spinning = 1; 52018e73f275SChris Mason 52023d7806ecSJan Schmidt if (time_seq) 52033d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 52043d7806ecSJan Schmidt else 5205925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5206925baeddSChris Mason path->keep_locks = 0; 5207925baeddSChris Mason 5208925baeddSChris Mason if (ret < 0) 5209925baeddSChris Mason return ret; 5210925baeddSChris Mason 5211a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5212168fd7d2SChris Mason /* 5213168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 5214168fd7d2SChris Mason * could have added more items next to the key that used to be 5215168fd7d2SChris Mason * at the very end of the block. So, check again here and 5216168fd7d2SChris Mason * advance the path if there are now more items available. 5217168fd7d2SChris Mason */ 5218a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 5219e457afecSYan Zheng if (ret == 0) 5220168fd7d2SChris Mason path->slots[0]++; 52218e73f275SChris Mason ret = 0; 5222925baeddSChris Mason goto done; 5223925baeddSChris Mason } 5224d97e63b6SChris Mason 5225234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 52268e73f275SChris Mason if (!path->nodes[level]) { 52278e73f275SChris Mason ret = 1; 52288e73f275SChris Mason goto done; 52298e73f275SChris Mason } 52305f39d397SChris Mason 5231d97e63b6SChris Mason slot = path->slots[level] + 1; 5232d97e63b6SChris Mason c = path->nodes[level]; 52335f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 5234d97e63b6SChris Mason level++; 52358e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 52368e73f275SChris Mason ret = 1; 52378e73f275SChris Mason goto done; 52388e73f275SChris Mason } 5239d97e63b6SChris Mason continue; 5240d97e63b6SChris Mason } 52415f39d397SChris Mason 5242925baeddSChris Mason if (next) { 5243bd681513SChris Mason btrfs_tree_unlock_rw(next, next_rw_lock); 52445f39d397SChris Mason free_extent_buffer(next); 5245925baeddSChris Mason } 52465f39d397SChris Mason 52478e73f275SChris Mason next = c; 5248bd681513SChris Mason next_rw_lock = path->locks[level]; 52498e73f275SChris Mason ret = read_block_for_search(NULL, root, path, &next, level, 52505d9e75c4SJan Schmidt slot, &key, 0); 52518e73f275SChris Mason if (ret == -EAGAIN) 52528e73f275SChris Mason goto again; 52535f39d397SChris Mason 525476a05b35SChris Mason if (ret < 0) { 5255b3b4aa74SDavid Sterba btrfs_release_path(path); 525676a05b35SChris Mason goto done; 525776a05b35SChris Mason } 525876a05b35SChris Mason 52595cd57b2cSChris Mason if (!path->skip_locking) { 5260bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 5261d42244a0SJan Schmidt if (!ret && time_seq) { 5262d42244a0SJan Schmidt /* 5263d42244a0SJan Schmidt * If we don't get the lock, we may be racing 5264d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 5265d42244a0SJan Schmidt * itself waiting for the leaf we've currently 5266d42244a0SJan Schmidt * locked. To solve this situation, we give up 5267d42244a0SJan Schmidt * on our lock and cycle. 5268d42244a0SJan Schmidt */ 5269cf538830SJan Schmidt free_extent_buffer(next); 5270d42244a0SJan Schmidt btrfs_release_path(path); 5271d42244a0SJan Schmidt cond_resched(); 5272d42244a0SJan Schmidt goto again; 5273d42244a0SJan Schmidt } 52748e73f275SChris Mason if (!ret) { 52758e73f275SChris Mason btrfs_set_path_blocking(path); 5276bd681513SChris Mason btrfs_tree_read_lock(next); 5277bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5278bd681513SChris Mason BTRFS_READ_LOCK); 52798e73f275SChris Mason } 5280bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5281bd681513SChris Mason } 5282d97e63b6SChris Mason break; 5283d97e63b6SChris Mason } 5284d97e63b6SChris Mason path->slots[level] = slot; 5285d97e63b6SChris Mason while (1) { 5286d97e63b6SChris Mason level--; 5287d97e63b6SChris Mason c = path->nodes[level]; 5288925baeddSChris Mason if (path->locks[level]) 5289bd681513SChris Mason btrfs_tree_unlock_rw(c, path->locks[level]); 52908e73f275SChris Mason 52915f39d397SChris Mason free_extent_buffer(c); 5292d97e63b6SChris Mason path->nodes[level] = next; 5293d97e63b6SChris Mason path->slots[level] = 0; 5294a74a4b97SChris Mason if (!path->skip_locking) 5295bd681513SChris Mason path->locks[level] = next_rw_lock; 5296d97e63b6SChris Mason if (!level) 5297d97e63b6SChris Mason break; 5298b4ce94deSChris Mason 52998e73f275SChris Mason ret = read_block_for_search(NULL, root, path, &next, level, 53005d9e75c4SJan Schmidt 0, &key, 0); 53018e73f275SChris Mason if (ret == -EAGAIN) 53028e73f275SChris Mason goto again; 53038e73f275SChris Mason 530476a05b35SChris Mason if (ret < 0) { 5305b3b4aa74SDavid Sterba btrfs_release_path(path); 530676a05b35SChris Mason goto done; 530776a05b35SChris Mason } 530876a05b35SChris Mason 53095cd57b2cSChris Mason if (!path->skip_locking) { 5310bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 53118e73f275SChris Mason if (!ret) { 53128e73f275SChris Mason btrfs_set_path_blocking(path); 5313bd681513SChris Mason btrfs_tree_read_lock(next); 5314bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5315bd681513SChris Mason BTRFS_READ_LOCK); 53168e73f275SChris Mason } 5317bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5318bd681513SChris Mason } 5319d97e63b6SChris Mason } 53208e73f275SChris Mason ret = 0; 5321925baeddSChris Mason done: 5322f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 53238e73f275SChris Mason path->leave_spinning = old_spinning; 53248e73f275SChris Mason if (!old_spinning) 53258e73f275SChris Mason btrfs_set_path_blocking(path); 53268e73f275SChris Mason 53278e73f275SChris Mason return ret; 5328d97e63b6SChris Mason } 53290b86a832SChris Mason 53303f157a2fSChris Mason /* 53313f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 53323f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 53333f157a2fSChris Mason * 53343f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 53353f157a2fSChris Mason */ 53360b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 53370b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 53380b86a832SChris Mason int type) 53390b86a832SChris Mason { 53400b86a832SChris Mason struct btrfs_key found_key; 53410b86a832SChris Mason struct extent_buffer *leaf; 5342e02119d5SChris Mason u32 nritems; 53430b86a832SChris Mason int ret; 53440b86a832SChris Mason 53450b86a832SChris Mason while (1) { 53460b86a832SChris Mason if (path->slots[0] == 0) { 5347b4ce94deSChris Mason btrfs_set_path_blocking(path); 53480b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 53490b86a832SChris Mason if (ret != 0) 53500b86a832SChris Mason return ret; 53510b86a832SChris Mason } else { 53520b86a832SChris Mason path->slots[0]--; 53530b86a832SChris Mason } 53540b86a832SChris Mason leaf = path->nodes[0]; 5355e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5356e02119d5SChris Mason if (nritems == 0) 5357e02119d5SChris Mason return 1; 5358e02119d5SChris Mason if (path->slots[0] == nritems) 5359e02119d5SChris Mason path->slots[0]--; 5360e02119d5SChris Mason 53610b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5362e02119d5SChris Mason if (found_key.objectid < min_objectid) 5363e02119d5SChris Mason break; 53640a4eefbbSYan Zheng if (found_key.type == type) 53650a4eefbbSYan Zheng return 0; 5366e02119d5SChris Mason if (found_key.objectid == min_objectid && 5367e02119d5SChris Mason found_key.type < type) 5368e02119d5SChris Mason break; 53690b86a832SChris Mason } 53700b86a832SChris Mason return 1; 53710b86a832SChris Mason } 5372