1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10eb60ceacSChris Mason #include "ctree.h" 11eb60ceacSChris Mason #include "disk-io.h" 127f5c1516SChris Mason #include "transaction.h" 135f39d397SChris Mason #include "print-tree.h" 14925baeddSChris Mason #include "locking.h" 15de37aa51SNikolay Borisov #include "volumes.h" 16f616f5cdSQu Wenruo #include "qgroup.h" 179a8dd150SChris Mason 18e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 19e089f05cSChris Mason *root, struct btrfs_path *path, int level); 20310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 21310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 22310712b2SOmar Sandoval int data_size, int extend); 235f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 242ff7e61eSJeff Mahoney struct extent_buffer *dst, 25971a1f66SChris Mason struct extent_buffer *src, int empty); 265f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 275f39d397SChris Mason struct extent_buffer *dst_buf, 285f39d397SChris Mason struct extent_buffer *src_buf); 29afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 30afe5fea7STsutomu Itoh int level, int slot); 31d97e63b6SChris Mason 32af024ed2SJohannes Thumshirn static const struct btrfs_csums { 33af024ed2SJohannes Thumshirn u16 size; 3459a0fcdbSDavid Sterba const char name[10]; 3559a0fcdbSDavid Sterba const char driver[12]; 36af024ed2SJohannes Thumshirn } btrfs_csums[] = { 37af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 383951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 393831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 40352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 41352ae07bSDavid Sterba .driver = "blake2b-256" }, 42af024ed2SJohannes Thumshirn }; 43af024ed2SJohannes Thumshirn 44af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 45af024ed2SJohannes Thumshirn { 46af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 47af024ed2SJohannes Thumshirn /* 48af024ed2SJohannes Thumshirn * csum type is validated at mount time 49af024ed2SJohannes Thumshirn */ 50af024ed2SJohannes Thumshirn return btrfs_csums[t].size; 51af024ed2SJohannes Thumshirn } 52af024ed2SJohannes Thumshirn 53af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 54af024ed2SJohannes Thumshirn { 55af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 56af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 57af024ed2SJohannes Thumshirn } 58af024ed2SJohannes Thumshirn 59b4e967beSDavid Sterba /* 60b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 61b4e967beSDavid Sterba * name 62b4e967beSDavid Sterba */ 63b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 64b4e967beSDavid Sterba { 65b4e967beSDavid Sterba /* csum type is validated at mount time */ 6659a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 6759a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 68b4e967beSDavid Sterba btrfs_csums[csum_type].name; 69b4e967beSDavid Sterba } 70b4e967beSDavid Sterba 71604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 72f7cea56cSDavid Sterba { 73f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 74f7cea56cSDavid Sterba } 75f7cea56cSDavid Sterba 762c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 772c90e5d6SChris Mason { 78e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 792c90e5d6SChris Mason } 802c90e5d6SChris Mason 81d352ac68SChris Mason /* this also releases the path */ 822c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 832c90e5d6SChris Mason { 84ff175d57SJesper Juhl if (!p) 85ff175d57SJesper Juhl return; 86b3b4aa74SDavid Sterba btrfs_release_path(p); 872c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 882c90e5d6SChris Mason } 892c90e5d6SChris Mason 90d352ac68SChris Mason /* 91d352ac68SChris Mason * path release drops references on the extent buffers in the path 92d352ac68SChris Mason * and it drops any locks held by this path 93d352ac68SChris Mason * 94d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 95d352ac68SChris Mason */ 96b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 97eb60ceacSChris Mason { 98eb60ceacSChris Mason int i; 99a2135011SChris Mason 100234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1013f157a2fSChris Mason p->slots[i] = 0; 102eb60ceacSChris Mason if (!p->nodes[i]) 103925baeddSChris Mason continue; 104925baeddSChris Mason if (p->locks[i]) { 105bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 106925baeddSChris Mason p->locks[i] = 0; 107925baeddSChris Mason } 1085f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1093f157a2fSChris Mason p->nodes[i] = NULL; 110eb60ceacSChris Mason } 111eb60ceacSChris Mason } 112eb60ceacSChris Mason 113d352ac68SChris Mason /* 114d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 115d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 116d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 117d352ac68SChris Mason * looping required. 118d352ac68SChris Mason * 119d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 120d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 121d352ac68SChris Mason * at any time because there are no locks held. 122d352ac68SChris Mason */ 123925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 124925baeddSChris Mason { 125925baeddSChris Mason struct extent_buffer *eb; 126240f62c8SChris Mason 1273083ee2eSJosef Bacik while (1) { 128240f62c8SChris Mason rcu_read_lock(); 129240f62c8SChris Mason eb = rcu_dereference(root->node); 1303083ee2eSJosef Bacik 1313083ee2eSJosef Bacik /* 1323083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 13301327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 1343083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1353083ee2eSJosef Bacik * synchronize_rcu and try again. 1363083ee2eSJosef Bacik */ 1373083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 138240f62c8SChris Mason rcu_read_unlock(); 1393083ee2eSJosef Bacik break; 1403083ee2eSJosef Bacik } 1413083ee2eSJosef Bacik rcu_read_unlock(); 1423083ee2eSJosef Bacik synchronize_rcu(); 1433083ee2eSJosef Bacik } 144925baeddSChris Mason return eb; 145925baeddSChris Mason } 146925baeddSChris Mason 14792a7cc42SQu Wenruo /* 14892a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 14992a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 15092a7cc42SQu Wenruo * sure they get properly updated on disk. 151d352ac68SChris Mason */ 1520b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 1530b86a832SChris Mason { 1540b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1550b246afaSJeff Mahoney 156e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 157e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 158e7070be1SJosef Bacik return; 159e7070be1SJosef Bacik 1600b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 161e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 162e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 1634fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 164e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 1650b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 166e7070be1SJosef Bacik else 167e7070be1SJosef Bacik list_move(&root->dirty_list, 1680b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 1690b86a832SChris Mason } 1700b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 1710b86a832SChris Mason } 1720b86a832SChris Mason 173d352ac68SChris Mason /* 174d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 175d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 176d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 177d352ac68SChris Mason */ 178be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 179be20aa9dSChris Mason struct btrfs_root *root, 180be20aa9dSChris Mason struct extent_buffer *buf, 181be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 182be20aa9dSChris Mason { 1830b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 184be20aa9dSChris Mason struct extent_buffer *cow; 185be20aa9dSChris Mason int ret = 0; 186be20aa9dSChris Mason int level; 1875d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 188be20aa9dSChris Mason 18992a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 1900b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 19192a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 19227cdeb70SMiao Xie trans->transid != root->last_trans); 193be20aa9dSChris Mason 194be20aa9dSChris Mason level = btrfs_header_level(buf); 1955d4f98a2SYan Zheng if (level == 0) 1965d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 1975d4f98a2SYan Zheng else 1985d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 19931840ae1SZheng Yan 2004d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 201cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 202cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 2035d4f98a2SYan Zheng if (IS_ERR(cow)) 204be20aa9dSChris Mason return PTR_ERR(cow); 205be20aa9dSChris Mason 20658e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 207be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 208be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2095d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2105d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2115d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2125d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2135d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2145d4f98a2SYan Zheng else 215be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 216be20aa9dSChris Mason 217de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 2182b82032cSYan Zheng 219be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2205d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 221e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 2225d4f98a2SYan Zheng else 223e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 2244aec2b52SChris Mason 225be20aa9dSChris Mason if (ret) 226be20aa9dSChris Mason return ret; 227be20aa9dSChris Mason 228be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 229be20aa9dSChris Mason *cow_ret = cow; 230be20aa9dSChris Mason return 0; 231be20aa9dSChris Mason } 232be20aa9dSChris Mason 233bd989ba3SJan Schmidt enum mod_log_op { 234bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 235bd989ba3SJan Schmidt MOD_LOG_KEY_ADD, 236bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE, 237bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_FREEING, 238bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_MOVING, 239bd989ba3SJan Schmidt MOD_LOG_MOVE_KEYS, 240bd989ba3SJan Schmidt MOD_LOG_ROOT_REPLACE, 241bd989ba3SJan Schmidt }; 242bd989ba3SJan Schmidt 243bd989ba3SJan Schmidt struct tree_mod_root { 244bd989ba3SJan Schmidt u64 logical; 245bd989ba3SJan Schmidt u8 level; 246bd989ba3SJan Schmidt }; 247bd989ba3SJan Schmidt 248bd989ba3SJan Schmidt struct tree_mod_elem { 249bd989ba3SJan Schmidt struct rb_node node; 250298cfd36SChandan Rajendra u64 logical; 251097b8a7cSJan Schmidt u64 seq; 252bd989ba3SJan Schmidt enum mod_log_op op; 253bd989ba3SJan Schmidt 254bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */ 255bd989ba3SJan Schmidt int slot; 256bd989ba3SJan Schmidt 257bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */ 258bd989ba3SJan Schmidt u64 generation; 259bd989ba3SJan Schmidt 260bd989ba3SJan Schmidt /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */ 261bd989ba3SJan Schmidt struct btrfs_disk_key key; 262bd989ba3SJan Schmidt u64 blockptr; 263bd989ba3SJan Schmidt 264bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_MOVE_KEYS */ 265b6dfa35bSDavid Sterba struct { 266b6dfa35bSDavid Sterba int dst_slot; 267b6dfa35bSDavid Sterba int nr_items; 268b6dfa35bSDavid Sterba } move; 269bd989ba3SJan Schmidt 270bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_ROOT_REPLACE */ 271bd989ba3SJan Schmidt struct tree_mod_root old_root; 272bd989ba3SJan Schmidt }; 273bd989ba3SJan Schmidt 274097b8a7cSJan Schmidt /* 275fcebe456SJosef Bacik * Pull a new tree mod seq number for our operation. 276fc36ed7eSJan Schmidt */ 277fcebe456SJosef Bacik static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info) 278fc36ed7eSJan Schmidt { 279fc36ed7eSJan Schmidt return atomic64_inc_return(&fs_info->tree_mod_seq); 280fc36ed7eSJan Schmidt } 281fc36ed7eSJan Schmidt 282fc36ed7eSJan Schmidt /* 283097b8a7cSJan Schmidt * This adds a new blocker to the tree mod log's blocker list if the @elem 284097b8a7cSJan Schmidt * passed does not already have a sequence number set. So when a caller expects 285097b8a7cSJan Schmidt * to record tree modifications, it should ensure to set elem->seq to zero 286097b8a7cSJan Schmidt * before calling btrfs_get_tree_mod_seq. 287097b8a7cSJan Schmidt * Returns a fresh, unused tree log modification sequence number, even if no new 288097b8a7cSJan Schmidt * blocker was added. 289097b8a7cSJan Schmidt */ 290097b8a7cSJan Schmidt u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, 291bd989ba3SJan Schmidt struct seq_list *elem) 292bd989ba3SJan Schmidt { 293b1a09f1eSDavid Sterba write_lock(&fs_info->tree_mod_log_lock); 294097b8a7cSJan Schmidt if (!elem->seq) { 295fcebe456SJosef Bacik elem->seq = btrfs_inc_tree_mod_seq(fs_info); 296097b8a7cSJan Schmidt list_add_tail(&elem->list, &fs_info->tree_mod_seq_list); 297097b8a7cSJan Schmidt } 298b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 299097b8a7cSJan Schmidt 300fcebe456SJosef Bacik return elem->seq; 301bd989ba3SJan Schmidt } 302bd989ba3SJan Schmidt 303bd989ba3SJan Schmidt void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, 304bd989ba3SJan Schmidt struct seq_list *elem) 305bd989ba3SJan Schmidt { 306bd989ba3SJan Schmidt struct rb_root *tm_root; 307bd989ba3SJan Schmidt struct rb_node *node; 308bd989ba3SJan Schmidt struct rb_node *next; 309bd989ba3SJan Schmidt struct tree_mod_elem *tm; 310bd989ba3SJan Schmidt u64 min_seq = (u64)-1; 311bd989ba3SJan Schmidt u64 seq_putting = elem->seq; 312bd989ba3SJan Schmidt 313bd989ba3SJan Schmidt if (!seq_putting) 314bd989ba3SJan Schmidt return; 315bd989ba3SJan Schmidt 3167227ff4dSFilipe Manana write_lock(&fs_info->tree_mod_log_lock); 317bd989ba3SJan Schmidt list_del(&elem->list); 318097b8a7cSJan Schmidt elem->seq = 0; 319bd989ba3SJan Schmidt 32042836cf4SFilipe Manana if (!list_empty(&fs_info->tree_mod_seq_list)) { 32142836cf4SFilipe Manana struct seq_list *first; 32242836cf4SFilipe Manana 32342836cf4SFilipe Manana first = list_first_entry(&fs_info->tree_mod_seq_list, 32442836cf4SFilipe Manana struct seq_list, list); 32542836cf4SFilipe Manana if (seq_putting > first->seq) { 326bd989ba3SJan Schmidt /* 32742836cf4SFilipe Manana * Blocker with lower sequence number exists, we 32842836cf4SFilipe Manana * cannot remove anything from the log. 329bd989ba3SJan Schmidt */ 3307227ff4dSFilipe Manana write_unlock(&fs_info->tree_mod_log_lock); 331097b8a7cSJan Schmidt return; 332bd989ba3SJan Schmidt } 33342836cf4SFilipe Manana min_seq = first->seq; 334bd989ba3SJan Schmidt } 335097b8a7cSJan Schmidt 336097b8a7cSJan Schmidt /* 337bd989ba3SJan Schmidt * anything that's lower than the lowest existing (read: blocked) 338bd989ba3SJan Schmidt * sequence number can be removed from the tree. 339bd989ba3SJan Schmidt */ 340bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 341bd989ba3SJan Schmidt for (node = rb_first(tm_root); node; node = next) { 342bd989ba3SJan Schmidt next = rb_next(node); 3436b4df8b6SGeliang Tang tm = rb_entry(node, struct tree_mod_elem, node); 3446609fee8SFilipe Manana if (tm->seq >= min_seq) 345bd989ba3SJan Schmidt continue; 346bd989ba3SJan Schmidt rb_erase(node, tm_root); 347bd989ba3SJan Schmidt kfree(tm); 348bd989ba3SJan Schmidt } 349b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 350bd989ba3SJan Schmidt } 351bd989ba3SJan Schmidt 352bd989ba3SJan Schmidt /* 353bd989ba3SJan Schmidt * key order of the log: 354298cfd36SChandan Rajendra * node/leaf start address -> sequence 355bd989ba3SJan Schmidt * 356298cfd36SChandan Rajendra * The 'start address' is the logical address of the *new* root node 357298cfd36SChandan Rajendra * for root replace operations, or the logical address of the affected 358298cfd36SChandan Rajendra * block for all other operations. 359bd989ba3SJan Schmidt */ 360bd989ba3SJan Schmidt static noinline int 361bd989ba3SJan Schmidt __tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm) 362bd989ba3SJan Schmidt { 363bd989ba3SJan Schmidt struct rb_root *tm_root; 364bd989ba3SJan Schmidt struct rb_node **new; 365bd989ba3SJan Schmidt struct rb_node *parent = NULL; 366bd989ba3SJan Schmidt struct tree_mod_elem *cur; 367bd989ba3SJan Schmidt 36873e82fe4SDavid Sterba lockdep_assert_held_write(&fs_info->tree_mod_log_lock); 36973e82fe4SDavid Sterba 370fcebe456SJosef Bacik tm->seq = btrfs_inc_tree_mod_seq(fs_info); 371bd989ba3SJan Schmidt 372bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 373bd989ba3SJan Schmidt new = &tm_root->rb_node; 374bd989ba3SJan Schmidt while (*new) { 3756b4df8b6SGeliang Tang cur = rb_entry(*new, struct tree_mod_elem, node); 376bd989ba3SJan Schmidt parent = *new; 377298cfd36SChandan Rajendra if (cur->logical < tm->logical) 378bd989ba3SJan Schmidt new = &((*new)->rb_left); 379298cfd36SChandan Rajendra else if (cur->logical > tm->logical) 380bd989ba3SJan Schmidt new = &((*new)->rb_right); 381097b8a7cSJan Schmidt else if (cur->seq < tm->seq) 382bd989ba3SJan Schmidt new = &((*new)->rb_left); 383097b8a7cSJan Schmidt else if (cur->seq > tm->seq) 384bd989ba3SJan Schmidt new = &((*new)->rb_right); 3855de865eeSFilipe David Borba Manana else 3865de865eeSFilipe David Borba Manana return -EEXIST; 387bd989ba3SJan Schmidt } 388bd989ba3SJan Schmidt 389bd989ba3SJan Schmidt rb_link_node(&tm->node, parent, new); 390bd989ba3SJan Schmidt rb_insert_color(&tm->node, tm_root); 3915de865eeSFilipe David Borba Manana return 0; 392bd989ba3SJan Schmidt } 393bd989ba3SJan Schmidt 394097b8a7cSJan Schmidt /* 395097b8a7cSJan Schmidt * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it 396097b8a7cSJan Schmidt * returns zero with the tree_mod_log_lock acquired. The caller must hold 397097b8a7cSJan Schmidt * this until all tree mod log insertions are recorded in the rb tree and then 398b1a09f1eSDavid Sterba * write unlock fs_info::tree_mod_log_lock. 399097b8a7cSJan Schmidt */ 400e9b7fd4dSJan Schmidt static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info, 401e9b7fd4dSJan Schmidt struct extent_buffer *eb) { 402e9b7fd4dSJan Schmidt smp_mb(); 403e9b7fd4dSJan Schmidt if (list_empty(&(fs_info)->tree_mod_seq_list)) 404e9b7fd4dSJan Schmidt return 1; 405097b8a7cSJan Schmidt if (eb && btrfs_header_level(eb) == 0) 406e9b7fd4dSJan Schmidt return 1; 4075de865eeSFilipe David Borba Manana 408b1a09f1eSDavid Sterba write_lock(&fs_info->tree_mod_log_lock); 4095de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) { 410b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 4115de865eeSFilipe David Borba Manana return 1; 4125de865eeSFilipe David Borba Manana } 4135de865eeSFilipe David Borba Manana 414e9b7fd4dSJan Schmidt return 0; 415e9b7fd4dSJan Schmidt } 416e9b7fd4dSJan Schmidt 4175de865eeSFilipe David Borba Manana /* Similar to tree_mod_dont_log, but doesn't acquire any locks. */ 4185de865eeSFilipe David Borba Manana static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info, 4195de865eeSFilipe David Borba Manana struct extent_buffer *eb) 4205de865eeSFilipe David Borba Manana { 4215de865eeSFilipe David Borba Manana smp_mb(); 4225de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) 4235de865eeSFilipe David Borba Manana return 0; 4245de865eeSFilipe David Borba Manana if (eb && btrfs_header_level(eb) == 0) 4255de865eeSFilipe David Borba Manana return 0; 4265de865eeSFilipe David Borba Manana 4275de865eeSFilipe David Borba Manana return 1; 4285de865eeSFilipe David Borba Manana } 4295de865eeSFilipe David Borba Manana 4305de865eeSFilipe David Borba Manana static struct tree_mod_elem * 4315de865eeSFilipe David Borba Manana alloc_tree_mod_elem(struct extent_buffer *eb, int slot, 432bd989ba3SJan Schmidt enum mod_log_op op, gfp_t flags) 433bd989ba3SJan Schmidt { 434097b8a7cSJan Schmidt struct tree_mod_elem *tm; 435bd989ba3SJan Schmidt 436c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 437c8cc6341SJosef Bacik if (!tm) 4385de865eeSFilipe David Borba Manana return NULL; 439bd989ba3SJan Schmidt 440298cfd36SChandan Rajendra tm->logical = eb->start; 441bd989ba3SJan Schmidt if (op != MOD_LOG_KEY_ADD) { 442bd989ba3SJan Schmidt btrfs_node_key(eb, &tm->key, slot); 443bd989ba3SJan Schmidt tm->blockptr = btrfs_node_blockptr(eb, slot); 444bd989ba3SJan Schmidt } 445bd989ba3SJan Schmidt tm->op = op; 446bd989ba3SJan Schmidt tm->slot = slot; 447bd989ba3SJan Schmidt tm->generation = btrfs_node_ptr_generation(eb, slot); 4485de865eeSFilipe David Borba Manana RB_CLEAR_NODE(&tm->node); 449bd989ba3SJan Schmidt 4505de865eeSFilipe David Borba Manana return tm; 451097b8a7cSJan Schmidt } 452097b8a7cSJan Schmidt 453e09c2efeSDavid Sterba static noinline int tree_mod_log_insert_key(struct extent_buffer *eb, int slot, 454097b8a7cSJan Schmidt enum mod_log_op op, gfp_t flags) 455097b8a7cSJan Schmidt { 4565de865eeSFilipe David Borba Manana struct tree_mod_elem *tm; 4575de865eeSFilipe David Borba Manana int ret; 4585de865eeSFilipe David Borba Manana 459e09c2efeSDavid Sterba if (!tree_mod_need_log(eb->fs_info, eb)) 460097b8a7cSJan Schmidt return 0; 461097b8a7cSJan Schmidt 4625de865eeSFilipe David Borba Manana tm = alloc_tree_mod_elem(eb, slot, op, flags); 4635de865eeSFilipe David Borba Manana if (!tm) 4645de865eeSFilipe David Borba Manana return -ENOMEM; 4655de865eeSFilipe David Borba Manana 466e09c2efeSDavid Sterba if (tree_mod_dont_log(eb->fs_info, eb)) { 4675de865eeSFilipe David Borba Manana kfree(tm); 4685de865eeSFilipe David Borba Manana return 0; 4695de865eeSFilipe David Borba Manana } 4705de865eeSFilipe David Borba Manana 471e09c2efeSDavid Sterba ret = __tree_mod_log_insert(eb->fs_info, tm); 472b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 4735de865eeSFilipe David Borba Manana if (ret) 4745de865eeSFilipe David Borba Manana kfree(tm); 4755de865eeSFilipe David Borba Manana 4765de865eeSFilipe David Borba Manana return ret; 477097b8a7cSJan Schmidt } 478097b8a7cSJan Schmidt 4796074d45fSDavid Sterba static noinline int tree_mod_log_insert_move(struct extent_buffer *eb, 4806074d45fSDavid Sterba int dst_slot, int src_slot, int nr_items) 481bd989ba3SJan Schmidt { 4825de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 4835de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 4845de865eeSFilipe David Borba Manana int ret = 0; 485bd989ba3SJan Schmidt int i; 4865de865eeSFilipe David Borba Manana int locked = 0; 487bd989ba3SJan Schmidt 4886074d45fSDavid Sterba if (!tree_mod_need_log(eb->fs_info, eb)) 489f395694cSJan Schmidt return 0; 490bd989ba3SJan Schmidt 491176ef8f5SDavid Sterba tm_list = kcalloc(nr_items, sizeof(struct tree_mod_elem *), GFP_NOFS); 4925de865eeSFilipe David Borba Manana if (!tm_list) 4935de865eeSFilipe David Borba Manana return -ENOMEM; 494bd989ba3SJan Schmidt 495176ef8f5SDavid Sterba tm = kzalloc(sizeof(*tm), GFP_NOFS); 4965de865eeSFilipe David Borba Manana if (!tm) { 4975de865eeSFilipe David Borba Manana ret = -ENOMEM; 4985de865eeSFilipe David Borba Manana goto free_tms; 4995de865eeSFilipe David Borba Manana } 500f395694cSJan Schmidt 501298cfd36SChandan Rajendra tm->logical = eb->start; 502bd989ba3SJan Schmidt tm->slot = src_slot; 503bd989ba3SJan Schmidt tm->move.dst_slot = dst_slot; 504bd989ba3SJan Schmidt tm->move.nr_items = nr_items; 505bd989ba3SJan Schmidt tm->op = MOD_LOG_MOVE_KEYS; 506bd989ba3SJan Schmidt 5075de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 5085de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot, 509176ef8f5SDavid Sterba MOD_LOG_KEY_REMOVE_WHILE_MOVING, GFP_NOFS); 5105de865eeSFilipe David Borba Manana if (!tm_list[i]) { 5115de865eeSFilipe David Borba Manana ret = -ENOMEM; 5125de865eeSFilipe David Borba Manana goto free_tms; 5135de865eeSFilipe David Borba Manana } 514bd989ba3SJan Schmidt } 515bd989ba3SJan Schmidt 5166074d45fSDavid Sterba if (tree_mod_dont_log(eb->fs_info, eb)) 5175de865eeSFilipe David Borba Manana goto free_tms; 5185de865eeSFilipe David Borba Manana locked = 1; 5195de865eeSFilipe David Borba Manana 5205de865eeSFilipe David Borba Manana /* 5215de865eeSFilipe David Borba Manana * When we override something during the move, we log these removals. 5225de865eeSFilipe David Borba Manana * This can only happen when we move towards the beginning of the 5235de865eeSFilipe David Borba Manana * buffer, i.e. dst_slot < src_slot. 5245de865eeSFilipe David Borba Manana */ 5255de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 5266074d45fSDavid Sterba ret = __tree_mod_log_insert(eb->fs_info, tm_list[i]); 5275de865eeSFilipe David Borba Manana if (ret) 5285de865eeSFilipe David Borba Manana goto free_tms; 5295de865eeSFilipe David Borba Manana } 5305de865eeSFilipe David Borba Manana 5316074d45fSDavid Sterba ret = __tree_mod_log_insert(eb->fs_info, tm); 5325de865eeSFilipe David Borba Manana if (ret) 5335de865eeSFilipe David Borba Manana goto free_tms; 534b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 5355de865eeSFilipe David Borba Manana kfree(tm_list); 5365de865eeSFilipe David Borba Manana 5375de865eeSFilipe David Borba Manana return 0; 5385de865eeSFilipe David Borba Manana free_tms: 5395de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 5405de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 5416074d45fSDavid Sterba rb_erase(&tm_list[i]->node, &eb->fs_info->tree_mod_log); 5425de865eeSFilipe David Borba Manana kfree(tm_list[i]); 5435de865eeSFilipe David Borba Manana } 5445de865eeSFilipe David Borba Manana if (locked) 545b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 5465de865eeSFilipe David Borba Manana kfree(tm_list); 5475de865eeSFilipe David Borba Manana kfree(tm); 5485de865eeSFilipe David Borba Manana 5495de865eeSFilipe David Borba Manana return ret; 5505de865eeSFilipe David Borba Manana } 5515de865eeSFilipe David Borba Manana 5525de865eeSFilipe David Borba Manana static inline int 5535de865eeSFilipe David Borba Manana __tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 5545de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list, 5555de865eeSFilipe David Borba Manana int nritems) 556097b8a7cSJan Schmidt { 5575de865eeSFilipe David Borba Manana int i, j; 558097b8a7cSJan Schmidt int ret; 559097b8a7cSJan Schmidt 560097b8a7cSJan Schmidt for (i = nritems - 1; i >= 0; i--) { 5615de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list[i]); 5625de865eeSFilipe David Borba Manana if (ret) { 5635de865eeSFilipe David Borba Manana for (j = nritems - 1; j > i; j--) 5645de865eeSFilipe David Borba Manana rb_erase(&tm_list[j]->node, 5655de865eeSFilipe David Borba Manana &fs_info->tree_mod_log); 5665de865eeSFilipe David Borba Manana return ret; 567097b8a7cSJan Schmidt } 568097b8a7cSJan Schmidt } 569097b8a7cSJan Schmidt 5705de865eeSFilipe David Borba Manana return 0; 5715de865eeSFilipe David Borba Manana } 5725de865eeSFilipe David Borba Manana 57395b757c1SDavid Sterba static noinline int tree_mod_log_insert_root(struct extent_buffer *old_root, 57495b757c1SDavid Sterba struct extent_buffer *new_root, int log_removal) 575bd989ba3SJan Schmidt { 57695b757c1SDavid Sterba struct btrfs_fs_info *fs_info = old_root->fs_info; 5775de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 5785de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 5795de865eeSFilipe David Borba Manana int nritems = 0; 5805de865eeSFilipe David Borba Manana int ret = 0; 5815de865eeSFilipe David Borba Manana int i; 582bd989ba3SJan Schmidt 5835de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 584097b8a7cSJan Schmidt return 0; 585097b8a7cSJan Schmidt 5865de865eeSFilipe David Borba Manana if (log_removal && btrfs_header_level(old_root) > 0) { 5875de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(old_root); 58831e818feSDavid Sterba tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), 589bcc8e07fSDavid Sterba GFP_NOFS); 5905de865eeSFilipe David Borba Manana if (!tm_list) { 5915de865eeSFilipe David Borba Manana ret = -ENOMEM; 5925de865eeSFilipe David Borba Manana goto free_tms; 5935de865eeSFilipe David Borba Manana } 5945de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 5955de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(old_root, i, 596bcc8e07fSDavid Sterba MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 5975de865eeSFilipe David Borba Manana if (!tm_list[i]) { 5985de865eeSFilipe David Borba Manana ret = -ENOMEM; 5995de865eeSFilipe David Borba Manana goto free_tms; 6005de865eeSFilipe David Borba Manana } 6015de865eeSFilipe David Borba Manana } 6025de865eeSFilipe David Borba Manana } 603d9abbf1cSJan Schmidt 604bcc8e07fSDavid Sterba tm = kzalloc(sizeof(*tm), GFP_NOFS); 6055de865eeSFilipe David Borba Manana if (!tm) { 6065de865eeSFilipe David Borba Manana ret = -ENOMEM; 6075de865eeSFilipe David Borba Manana goto free_tms; 6085de865eeSFilipe David Borba Manana } 609bd989ba3SJan Schmidt 610298cfd36SChandan Rajendra tm->logical = new_root->start; 611bd989ba3SJan Schmidt tm->old_root.logical = old_root->start; 612bd989ba3SJan Schmidt tm->old_root.level = btrfs_header_level(old_root); 613bd989ba3SJan Schmidt tm->generation = btrfs_header_generation(old_root); 614bd989ba3SJan Schmidt tm->op = MOD_LOG_ROOT_REPLACE; 615bd989ba3SJan Schmidt 6165de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 6175de865eeSFilipe David Borba Manana goto free_tms; 6185de865eeSFilipe David Borba Manana 6195de865eeSFilipe David Borba Manana if (tm_list) 6205de865eeSFilipe David Borba Manana ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems); 6215de865eeSFilipe David Borba Manana if (!ret) 6225de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 6235de865eeSFilipe David Borba Manana 624b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 6255de865eeSFilipe David Borba Manana if (ret) 6265de865eeSFilipe David Borba Manana goto free_tms; 6275de865eeSFilipe David Borba Manana kfree(tm_list); 6285de865eeSFilipe David Borba Manana 6295de865eeSFilipe David Borba Manana return ret; 6305de865eeSFilipe David Borba Manana 6315de865eeSFilipe David Borba Manana free_tms: 6325de865eeSFilipe David Borba Manana if (tm_list) { 6335de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 6345de865eeSFilipe David Borba Manana kfree(tm_list[i]); 6355de865eeSFilipe David Borba Manana kfree(tm_list); 6365de865eeSFilipe David Borba Manana } 6375de865eeSFilipe David Borba Manana kfree(tm); 6385de865eeSFilipe David Borba Manana 6395de865eeSFilipe David Borba Manana return ret; 640bd989ba3SJan Schmidt } 641bd989ba3SJan Schmidt 642bd989ba3SJan Schmidt static struct tree_mod_elem * 643bd989ba3SJan Schmidt __tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq, 644bd989ba3SJan Schmidt int smallest) 645bd989ba3SJan Schmidt { 646bd989ba3SJan Schmidt struct rb_root *tm_root; 647bd989ba3SJan Schmidt struct rb_node *node; 648bd989ba3SJan Schmidt struct tree_mod_elem *cur = NULL; 649bd989ba3SJan Schmidt struct tree_mod_elem *found = NULL; 650bd989ba3SJan Schmidt 651b1a09f1eSDavid Sterba read_lock(&fs_info->tree_mod_log_lock); 652bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 653bd989ba3SJan Schmidt node = tm_root->rb_node; 654bd989ba3SJan Schmidt while (node) { 6556b4df8b6SGeliang Tang cur = rb_entry(node, struct tree_mod_elem, node); 656298cfd36SChandan Rajendra if (cur->logical < start) { 657bd989ba3SJan Schmidt node = node->rb_left; 658298cfd36SChandan Rajendra } else if (cur->logical > start) { 659bd989ba3SJan Schmidt node = node->rb_right; 660097b8a7cSJan Schmidt } else if (cur->seq < min_seq) { 661bd989ba3SJan Schmidt node = node->rb_left; 662bd989ba3SJan Schmidt } else if (!smallest) { 663bd989ba3SJan Schmidt /* we want the node with the highest seq */ 664bd989ba3SJan Schmidt if (found) 665097b8a7cSJan Schmidt BUG_ON(found->seq > cur->seq); 666bd989ba3SJan Schmidt found = cur; 667bd989ba3SJan Schmidt node = node->rb_left; 668097b8a7cSJan Schmidt } else if (cur->seq > min_seq) { 669bd989ba3SJan Schmidt /* we want the node with the smallest seq */ 670bd989ba3SJan Schmidt if (found) 671097b8a7cSJan Schmidt BUG_ON(found->seq < cur->seq); 672bd989ba3SJan Schmidt found = cur; 673bd989ba3SJan Schmidt node = node->rb_right; 674bd989ba3SJan Schmidt } else { 675bd989ba3SJan Schmidt found = cur; 676bd989ba3SJan Schmidt break; 677bd989ba3SJan Schmidt } 678bd989ba3SJan Schmidt } 679b1a09f1eSDavid Sterba read_unlock(&fs_info->tree_mod_log_lock); 680bd989ba3SJan Schmidt 681bd989ba3SJan Schmidt return found; 682bd989ba3SJan Schmidt } 683bd989ba3SJan Schmidt 684bd989ba3SJan Schmidt /* 685bd989ba3SJan Schmidt * this returns the element from the log with the smallest time sequence 686bd989ba3SJan Schmidt * value that's in the log (the oldest log item). any element with a time 687bd989ba3SJan Schmidt * sequence lower than min_seq will be ignored. 688bd989ba3SJan Schmidt */ 689bd989ba3SJan Schmidt static struct tree_mod_elem * 690bd989ba3SJan Schmidt tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start, 691bd989ba3SJan Schmidt u64 min_seq) 692bd989ba3SJan Schmidt { 693bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 1); 694bd989ba3SJan Schmidt } 695bd989ba3SJan Schmidt 696bd989ba3SJan Schmidt /* 697bd989ba3SJan Schmidt * this returns the element from the log with the largest time sequence 698bd989ba3SJan Schmidt * value that's in the log (the most recent log item). any element with 699bd989ba3SJan Schmidt * a time sequence lower than min_seq will be ignored. 700bd989ba3SJan Schmidt */ 701bd989ba3SJan Schmidt static struct tree_mod_elem * 702bd989ba3SJan Schmidt tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq) 703bd989ba3SJan Schmidt { 704bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 0); 705bd989ba3SJan Schmidt } 706bd989ba3SJan Schmidt 707ed874f0dSDavid Sterba static noinline int tree_mod_log_eb_copy(struct extent_buffer *dst, 708bd989ba3SJan Schmidt struct extent_buffer *src, unsigned long dst_offset, 70990f8d62eSJan Schmidt unsigned long src_offset, int nr_items) 710bd989ba3SJan Schmidt { 711ed874f0dSDavid Sterba struct btrfs_fs_info *fs_info = dst->fs_info; 7125de865eeSFilipe David Borba Manana int ret = 0; 7135de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 7145de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list_add, **tm_list_rem; 715bd989ba3SJan Schmidt int i; 7165de865eeSFilipe David Borba Manana int locked = 0; 717bd989ba3SJan Schmidt 7185de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 7195de865eeSFilipe David Borba Manana return 0; 720bd989ba3SJan Schmidt 721c8cc6341SJosef Bacik if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0) 7225de865eeSFilipe David Borba Manana return 0; 7235de865eeSFilipe David Borba Manana 72431e818feSDavid Sterba tm_list = kcalloc(nr_items * 2, sizeof(struct tree_mod_elem *), 7255de865eeSFilipe David Borba Manana GFP_NOFS); 7265de865eeSFilipe David Borba Manana if (!tm_list) 7275de865eeSFilipe David Borba Manana return -ENOMEM; 7285de865eeSFilipe David Borba Manana 7295de865eeSFilipe David Borba Manana tm_list_add = tm_list; 7305de865eeSFilipe David Borba Manana tm_list_rem = tm_list + nr_items; 7315de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 7325de865eeSFilipe David Borba Manana tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset, 7335de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE, GFP_NOFS); 7345de865eeSFilipe David Borba Manana if (!tm_list_rem[i]) { 7355de865eeSFilipe David Borba Manana ret = -ENOMEM; 7365de865eeSFilipe David Borba Manana goto free_tms; 7375de865eeSFilipe David Borba Manana } 7385de865eeSFilipe David Borba Manana 7395de865eeSFilipe David Borba Manana tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset, 7405de865eeSFilipe David Borba Manana MOD_LOG_KEY_ADD, GFP_NOFS); 7415de865eeSFilipe David Borba Manana if (!tm_list_add[i]) { 7425de865eeSFilipe David Borba Manana ret = -ENOMEM; 7435de865eeSFilipe David Borba Manana goto free_tms; 7445de865eeSFilipe David Borba Manana } 7455de865eeSFilipe David Borba Manana } 7465de865eeSFilipe David Borba Manana 7475de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 7485de865eeSFilipe David Borba Manana goto free_tms; 7495de865eeSFilipe David Borba Manana locked = 1; 750bd989ba3SJan Schmidt 751bd989ba3SJan Schmidt for (i = 0; i < nr_items; i++) { 7525de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]); 7535de865eeSFilipe David Borba Manana if (ret) 7545de865eeSFilipe David Borba Manana goto free_tms; 7555de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_add[i]); 7565de865eeSFilipe David Borba Manana if (ret) 7575de865eeSFilipe David Borba Manana goto free_tms; 758bd989ba3SJan Schmidt } 7595de865eeSFilipe David Borba Manana 760b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 7615de865eeSFilipe David Borba Manana kfree(tm_list); 7625de865eeSFilipe David Borba Manana 7635de865eeSFilipe David Borba Manana return 0; 7645de865eeSFilipe David Borba Manana 7655de865eeSFilipe David Borba Manana free_tms: 7665de865eeSFilipe David Borba Manana for (i = 0; i < nr_items * 2; i++) { 7675de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 7685de865eeSFilipe David Borba Manana rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log); 7695de865eeSFilipe David Borba Manana kfree(tm_list[i]); 7705de865eeSFilipe David Borba Manana } 7715de865eeSFilipe David Borba Manana if (locked) 772b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 7735de865eeSFilipe David Borba Manana kfree(tm_list); 7745de865eeSFilipe David Borba Manana 7755de865eeSFilipe David Borba Manana return ret; 776bd989ba3SJan Schmidt } 777bd989ba3SJan Schmidt 778db7279a2SDavid Sterba static noinline int tree_mod_log_free_eb(struct extent_buffer *eb) 779bd989ba3SJan Schmidt { 7805de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 7815de865eeSFilipe David Borba Manana int nritems = 0; 7825de865eeSFilipe David Borba Manana int i; 7835de865eeSFilipe David Borba Manana int ret = 0; 7845de865eeSFilipe David Borba Manana 7855de865eeSFilipe David Borba Manana if (btrfs_header_level(eb) == 0) 7865de865eeSFilipe David Borba Manana return 0; 7875de865eeSFilipe David Borba Manana 788db7279a2SDavid Sterba if (!tree_mod_need_log(eb->fs_info, NULL)) 7895de865eeSFilipe David Borba Manana return 0; 7905de865eeSFilipe David Borba Manana 7915de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(eb); 79231e818feSDavid Sterba tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), GFP_NOFS); 7935de865eeSFilipe David Borba Manana if (!tm_list) 7945de865eeSFilipe David Borba Manana return -ENOMEM; 7955de865eeSFilipe David Borba Manana 7965de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 7975de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i, 7985de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 7995de865eeSFilipe David Borba Manana if (!tm_list[i]) { 8005de865eeSFilipe David Borba Manana ret = -ENOMEM; 8015de865eeSFilipe David Borba Manana goto free_tms; 8025de865eeSFilipe David Borba Manana } 8035de865eeSFilipe David Borba Manana } 8045de865eeSFilipe David Borba Manana 805db7279a2SDavid Sterba if (tree_mod_dont_log(eb->fs_info, eb)) 8065de865eeSFilipe David Borba Manana goto free_tms; 8075de865eeSFilipe David Borba Manana 808db7279a2SDavid Sterba ret = __tree_mod_log_free_eb(eb->fs_info, tm_list, nritems); 809b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 8105de865eeSFilipe David Borba Manana if (ret) 8115de865eeSFilipe David Borba Manana goto free_tms; 8125de865eeSFilipe David Borba Manana kfree(tm_list); 8135de865eeSFilipe David Borba Manana 8145de865eeSFilipe David Borba Manana return 0; 8155de865eeSFilipe David Borba Manana 8165de865eeSFilipe David Borba Manana free_tms: 8175de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 8185de865eeSFilipe David Borba Manana kfree(tm_list[i]); 8195de865eeSFilipe David Borba Manana kfree(tm_list); 8205de865eeSFilipe David Borba Manana 8215de865eeSFilipe David Borba Manana return ret; 822bd989ba3SJan Schmidt } 823bd989ba3SJan Schmidt 824d352ac68SChris Mason /* 8255d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 8265d4f98a2SYan Zheng */ 8275d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 8285d4f98a2SYan Zheng struct extent_buffer *buf) 8295d4f98a2SYan Zheng { 8305d4f98a2SYan Zheng /* 83192a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 83292a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 83392a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 8345d4f98a2SYan Zheng */ 83592a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 8365d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 8375d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 8385d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 8395d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 8405d4f98a2SYan Zheng return 1; 841a79865c6SNikolay Borisov 8425d4f98a2SYan Zheng return 0; 8435d4f98a2SYan Zheng } 8445d4f98a2SYan Zheng 8455d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 8465d4f98a2SYan Zheng struct btrfs_root *root, 8475d4f98a2SYan Zheng struct extent_buffer *buf, 848f0486c68SYan, Zheng struct extent_buffer *cow, 849f0486c68SYan, Zheng int *last_ref) 8505d4f98a2SYan Zheng { 8510b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 8525d4f98a2SYan Zheng u64 refs; 8535d4f98a2SYan Zheng u64 owner; 8545d4f98a2SYan Zheng u64 flags; 8555d4f98a2SYan Zheng u64 new_flags = 0; 8565d4f98a2SYan Zheng int ret; 8575d4f98a2SYan Zheng 8585d4f98a2SYan Zheng /* 8595d4f98a2SYan Zheng * Backrefs update rules: 8605d4f98a2SYan Zheng * 8615d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 8625d4f98a2SYan Zheng * allocated by tree relocation. 8635d4f98a2SYan Zheng * 8645d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 8655d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 8665d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8675d4f98a2SYan Zheng * 8685d4f98a2SYan Zheng * If a tree block is been relocating 8695d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 8705d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8715d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 8725d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 8735d4f98a2SYan Zheng */ 8745d4f98a2SYan Zheng 8755d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 8762ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 8773173a18fSJosef Bacik btrfs_header_level(buf), 1, 8783173a18fSJosef Bacik &refs, &flags); 879be1a5564SMark Fasheh if (ret) 880be1a5564SMark Fasheh return ret; 881e5df9573SMark Fasheh if (refs == 0) { 882e5df9573SMark Fasheh ret = -EROFS; 8830b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 884e5df9573SMark Fasheh return ret; 885e5df9573SMark Fasheh } 8865d4f98a2SYan Zheng } else { 8875d4f98a2SYan Zheng refs = 1; 8885d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 8895d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 8905d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 8915d4f98a2SYan Zheng else 8925d4f98a2SYan Zheng flags = 0; 8935d4f98a2SYan Zheng } 8945d4f98a2SYan Zheng 8955d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 8965d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 8975d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 8985d4f98a2SYan Zheng 8995d4f98a2SYan Zheng if (refs > 1) { 9005d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 9015d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 9025d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 903e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 904692826b2SJeff Mahoney if (ret) 905692826b2SJeff Mahoney return ret; 9065d4f98a2SYan Zheng 9075d4f98a2SYan Zheng if (root->root_key.objectid == 9085d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 909e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 910692826b2SJeff Mahoney if (ret) 911692826b2SJeff Mahoney return ret; 912e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 913692826b2SJeff Mahoney if (ret) 914692826b2SJeff Mahoney return ret; 9155d4f98a2SYan Zheng } 9165d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 9175d4f98a2SYan Zheng } else { 9185d4f98a2SYan Zheng 9195d4f98a2SYan Zheng if (root->root_key.objectid == 9205d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 921e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 9225d4f98a2SYan Zheng else 923e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 924692826b2SJeff Mahoney if (ret) 925692826b2SJeff Mahoney return ret; 9265d4f98a2SYan Zheng } 9275d4f98a2SYan Zheng if (new_flags != 0) { 928b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 929b1c79e09SJosef Bacik 93042c9d0b5SDavid Sterba ret = btrfs_set_disk_extent_flags(trans, buf, 931b1c79e09SJosef Bacik new_flags, level, 0); 932be1a5564SMark Fasheh if (ret) 933be1a5564SMark Fasheh return ret; 9345d4f98a2SYan Zheng } 9355d4f98a2SYan Zheng } else { 9365d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 9375d4f98a2SYan Zheng if (root->root_key.objectid == 9385d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 939e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 9405d4f98a2SYan Zheng else 941e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 942692826b2SJeff Mahoney if (ret) 943692826b2SJeff Mahoney return ret; 944e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 945692826b2SJeff Mahoney if (ret) 946692826b2SJeff Mahoney return ret; 9475d4f98a2SYan Zheng } 9486a884d7dSDavid Sterba btrfs_clean_tree_block(buf); 949f0486c68SYan, Zheng *last_ref = 1; 9505d4f98a2SYan Zheng } 9515d4f98a2SYan Zheng return 0; 9525d4f98a2SYan Zheng } 9535d4f98a2SYan Zheng 954a6279470SFilipe Manana static struct extent_buffer *alloc_tree_block_no_bg_flush( 955a6279470SFilipe Manana struct btrfs_trans_handle *trans, 956a6279470SFilipe Manana struct btrfs_root *root, 957a6279470SFilipe Manana u64 parent_start, 958a6279470SFilipe Manana const struct btrfs_disk_key *disk_key, 959a6279470SFilipe Manana int level, 960a6279470SFilipe Manana u64 hint, 9619631e4ccSJosef Bacik u64 empty_size, 9629631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 963a6279470SFilipe Manana { 964a6279470SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 965a6279470SFilipe Manana struct extent_buffer *ret; 966a6279470SFilipe Manana 967a6279470SFilipe Manana /* 968a6279470SFilipe Manana * If we are COWing a node/leaf from the extent, chunk, device or free 969a6279470SFilipe Manana * space trees, make sure that we do not finish block group creation of 970a6279470SFilipe Manana * pending block groups. We do this to avoid a deadlock. 971a6279470SFilipe Manana * COWing can result in allocation of a new chunk, and flushing pending 972a6279470SFilipe Manana * block groups (btrfs_create_pending_block_groups()) can be triggered 973a6279470SFilipe Manana * when finishing allocation of a new chunk. Creation of a pending block 974a6279470SFilipe Manana * group modifies the extent, chunk, device and free space trees, 975a6279470SFilipe Manana * therefore we could deadlock with ourselves since we are holding a 976a6279470SFilipe Manana * lock on an extent buffer that btrfs_create_pending_block_groups() may 977a6279470SFilipe Manana * try to COW later. 978a6279470SFilipe Manana * For similar reasons, we also need to delay flushing pending block 979a6279470SFilipe Manana * groups when splitting a leaf or node, from one of those trees, since 980a6279470SFilipe Manana * we are holding a write lock on it and its parent or when inserting a 981a6279470SFilipe Manana * new root node for one of those trees. 982a6279470SFilipe Manana */ 983a6279470SFilipe Manana if (root == fs_info->extent_root || 984a6279470SFilipe Manana root == fs_info->chunk_root || 985a6279470SFilipe Manana root == fs_info->dev_root || 986a6279470SFilipe Manana root == fs_info->free_space_root) 987a6279470SFilipe Manana trans->can_flush_pending_bgs = false; 988a6279470SFilipe Manana 989a6279470SFilipe Manana ret = btrfs_alloc_tree_block(trans, root, parent_start, 990a6279470SFilipe Manana root->root_key.objectid, disk_key, level, 9919631e4ccSJosef Bacik hint, empty_size, nest); 992a6279470SFilipe Manana trans->can_flush_pending_bgs = true; 993a6279470SFilipe Manana 994a6279470SFilipe Manana return ret; 995a6279470SFilipe Manana } 996a6279470SFilipe Manana 9975d4f98a2SYan Zheng /* 998d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 999d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 1000d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 1001d397712bSChris Mason * dirty again. 1002d352ac68SChris Mason * 1003d352ac68SChris Mason * search_start -- an allocation hint for the new block 1004d352ac68SChris Mason * 1005d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 1006d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 1007d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 1008d352ac68SChris Mason */ 1009d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 10105f39d397SChris Mason struct btrfs_root *root, 10115f39d397SChris Mason struct extent_buffer *buf, 10125f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 10135f39d397SChris Mason struct extent_buffer **cow_ret, 10149631e4ccSJosef Bacik u64 search_start, u64 empty_size, 10159631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 10166702ed49SChris Mason { 10170b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 10185d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 10195f39d397SChris Mason struct extent_buffer *cow; 1020be1a5564SMark Fasheh int level, ret; 1021f0486c68SYan, Zheng int last_ref = 0; 1022925baeddSChris Mason int unlock_orig = 0; 10230f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 10246702ed49SChris Mason 1025925baeddSChris Mason if (*cow_ret == buf) 1026925baeddSChris Mason unlock_orig = 1; 1027925baeddSChris Mason 1028b9447ef8SChris Mason btrfs_assert_tree_locked(buf); 1029925baeddSChris Mason 103092a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 10310b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 103292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 103327cdeb70SMiao Xie trans->transid != root->last_trans); 10345f39d397SChris Mason 10357bb86316SChris Mason level = btrfs_header_level(buf); 103631840ae1SZheng Yan 10375d4f98a2SYan Zheng if (level == 0) 10385d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 10395d4f98a2SYan Zheng else 10405d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 10415d4f98a2SYan Zheng 10420f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 10435d4f98a2SYan Zheng parent_start = parent->start; 10445d4f98a2SYan Zheng 1045a6279470SFilipe Manana cow = alloc_tree_block_no_bg_flush(trans, root, parent_start, &disk_key, 10469631e4ccSJosef Bacik level, search_start, empty_size, nest); 10476702ed49SChris Mason if (IS_ERR(cow)) 10486702ed49SChris Mason return PTR_ERR(cow); 10496702ed49SChris Mason 1050b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 1051b4ce94deSChris Mason 105258e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 1053db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 10545f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 10555d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 10565d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 10575d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 10585d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 10595d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 10605d4f98a2SYan Zheng else 10615f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 10626702ed49SChris Mason 1063de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 10642b82032cSYan Zheng 1065be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 1066b68dc2a9SMark Fasheh if (ret) { 1067572c83acSJosef Bacik btrfs_tree_unlock(cow); 1068572c83acSJosef Bacik free_extent_buffer(cow); 106966642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 1070b68dc2a9SMark Fasheh return ret; 1071b68dc2a9SMark Fasheh } 10721a40e23bSZheng Yan 107392a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 107483d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 107593314e3bSZhaolei if (ret) { 1076572c83acSJosef Bacik btrfs_tree_unlock(cow); 1077572c83acSJosef Bacik free_extent_buffer(cow); 107866642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 107983d4cfd4SJosef Bacik return ret; 108083d4cfd4SJosef Bacik } 108193314e3bSZhaolei } 10823fd0a558SYan, Zheng 10836702ed49SChris Mason if (buf == root->node) { 1084925baeddSChris Mason WARN_ON(parent && parent != buf); 10855d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 10865d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 10875d4f98a2SYan Zheng parent_start = buf->start; 1088925baeddSChris Mason 108967439dadSDavid Sterba atomic_inc(&cow->refs); 1090d9d19a01SDavid Sterba ret = tree_mod_log_insert_root(root->node, cow, 1); 1091d9d19a01SDavid Sterba BUG_ON(ret < 0); 1092240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 1093925baeddSChris Mason 1094f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 10955581a51aSJan Schmidt last_ref); 10965f39d397SChris Mason free_extent_buffer(buf); 10970b86a832SChris Mason add_root_to_dirty_list(root); 10986702ed49SChris Mason } else { 10995d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 1100e09c2efeSDavid Sterba tree_mod_log_insert_key(parent, parent_slot, 1101c8cc6341SJosef Bacik MOD_LOG_KEY_REPLACE, GFP_NOFS); 11025f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 1103db94535dSChris Mason cow->start); 110474493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 110574493f7aSChris Mason trans->transid); 11066702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 11075de865eeSFilipe David Borba Manana if (last_ref) { 1108db7279a2SDavid Sterba ret = tree_mod_log_free_eb(buf); 11095de865eeSFilipe David Borba Manana if (ret) { 1110572c83acSJosef Bacik btrfs_tree_unlock(cow); 1111572c83acSJosef Bacik free_extent_buffer(cow); 111266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 11135de865eeSFilipe David Borba Manana return ret; 11145de865eeSFilipe David Borba Manana } 11155de865eeSFilipe David Borba Manana } 1116f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 11175581a51aSJan Schmidt last_ref); 11186702ed49SChris Mason } 1119925baeddSChris Mason if (unlock_orig) 1120925baeddSChris Mason btrfs_tree_unlock(buf); 11213083ee2eSJosef Bacik free_extent_buffer_stale(buf); 11226702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 11236702ed49SChris Mason *cow_ret = cow; 11246702ed49SChris Mason return 0; 11256702ed49SChris Mason } 11266702ed49SChris Mason 11275d9e75c4SJan Schmidt /* 11285d9e75c4SJan Schmidt * returns the logical address of the oldest predecessor of the given root. 11295d9e75c4SJan Schmidt * entries older than time_seq are ignored. 11305d9e75c4SJan Schmidt */ 1131bcd24dabSDavid Sterba static struct tree_mod_elem *__tree_mod_log_oldest_root( 113230b0463aSJan Schmidt struct extent_buffer *eb_root, u64 time_seq) 11335d9e75c4SJan Schmidt { 11345d9e75c4SJan Schmidt struct tree_mod_elem *tm; 11355d9e75c4SJan Schmidt struct tree_mod_elem *found = NULL; 113630b0463aSJan Schmidt u64 root_logical = eb_root->start; 11375d9e75c4SJan Schmidt int looped = 0; 11385d9e75c4SJan Schmidt 11395d9e75c4SJan Schmidt if (!time_seq) 114035a3621bSStefan Behrens return NULL; 11415d9e75c4SJan Schmidt 11425d9e75c4SJan Schmidt /* 1143298cfd36SChandan Rajendra * the very last operation that's logged for a root is the 1144298cfd36SChandan Rajendra * replacement operation (if it is replaced at all). this has 1145298cfd36SChandan Rajendra * the logical address of the *new* root, making it the very 1146298cfd36SChandan Rajendra * first operation that's logged for this root. 11475d9e75c4SJan Schmidt */ 11485d9e75c4SJan Schmidt while (1) { 1149bcd24dabSDavid Sterba tm = tree_mod_log_search_oldest(eb_root->fs_info, root_logical, 11505d9e75c4SJan Schmidt time_seq); 11515d9e75c4SJan Schmidt if (!looped && !tm) 115235a3621bSStefan Behrens return NULL; 11535d9e75c4SJan Schmidt /* 115428da9fb4SJan Schmidt * if there are no tree operation for the oldest root, we simply 115528da9fb4SJan Schmidt * return it. this should only happen if that (old) root is at 115628da9fb4SJan Schmidt * level 0. 11575d9e75c4SJan Schmidt */ 115828da9fb4SJan Schmidt if (!tm) 115928da9fb4SJan Schmidt break; 11605d9e75c4SJan Schmidt 116128da9fb4SJan Schmidt /* 116228da9fb4SJan Schmidt * if there's an operation that's not a root replacement, we 116328da9fb4SJan Schmidt * found the oldest version of our root. normally, we'll find a 116428da9fb4SJan Schmidt * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here. 116528da9fb4SJan Schmidt */ 11665d9e75c4SJan Schmidt if (tm->op != MOD_LOG_ROOT_REPLACE) 11675d9e75c4SJan Schmidt break; 11685d9e75c4SJan Schmidt 11695d9e75c4SJan Schmidt found = tm; 11705d9e75c4SJan Schmidt root_logical = tm->old_root.logical; 11715d9e75c4SJan Schmidt looped = 1; 11725d9e75c4SJan Schmidt } 11735d9e75c4SJan Schmidt 1174a95236d9SJan Schmidt /* if there's no old root to return, return what we found instead */ 1175a95236d9SJan Schmidt if (!found) 1176a95236d9SJan Schmidt found = tm; 1177a95236d9SJan Schmidt 11785d9e75c4SJan Schmidt return found; 11795d9e75c4SJan Schmidt } 11805d9e75c4SJan Schmidt 11815d9e75c4SJan Schmidt /* 11825d9e75c4SJan Schmidt * tm is a pointer to the first operation to rewind within eb. then, all 118301327610SNicholas D Steeves * previous operations will be rewound (until we reach something older than 11845d9e75c4SJan Schmidt * time_seq). 11855d9e75c4SJan Schmidt */ 11865d9e75c4SJan Schmidt static void 1187f1ca7e98SJosef Bacik __tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 1188f1ca7e98SJosef Bacik u64 time_seq, struct tree_mod_elem *first_tm) 11895d9e75c4SJan Schmidt { 11905d9e75c4SJan Schmidt u32 n; 11915d9e75c4SJan Schmidt struct rb_node *next; 11925d9e75c4SJan Schmidt struct tree_mod_elem *tm = first_tm; 11935d9e75c4SJan Schmidt unsigned long o_dst; 11945d9e75c4SJan Schmidt unsigned long o_src; 11955d9e75c4SJan Schmidt unsigned long p_size = sizeof(struct btrfs_key_ptr); 11965d9e75c4SJan Schmidt 11975d9e75c4SJan Schmidt n = btrfs_header_nritems(eb); 1198b1a09f1eSDavid Sterba read_lock(&fs_info->tree_mod_log_lock); 1199097b8a7cSJan Schmidt while (tm && tm->seq >= time_seq) { 12005d9e75c4SJan Schmidt /* 12015d9e75c4SJan Schmidt * all the operations are recorded with the operator used for 12025d9e75c4SJan Schmidt * the modification. as we're going backwards, we do the 12035d9e75c4SJan Schmidt * opposite of each operation here. 12045d9e75c4SJan Schmidt */ 12055d9e75c4SJan Schmidt switch (tm->op) { 12065d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE_WHILE_FREEING: 12075d9e75c4SJan Schmidt BUG_ON(tm->slot < n); 1208c730ae0cSMarcos Paulo de Souza fallthrough; 120995c80bb1SLiu Bo case MOD_LOG_KEY_REMOVE_WHILE_MOVING: 12104c3e6969SChris Mason case MOD_LOG_KEY_REMOVE: 12115d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 12125d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 12135d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 12145d9e75c4SJan Schmidt tm->generation); 12154c3e6969SChris Mason n++; 12165d9e75c4SJan Schmidt break; 12175d9e75c4SJan Schmidt case MOD_LOG_KEY_REPLACE: 12185d9e75c4SJan Schmidt BUG_ON(tm->slot >= n); 12195d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 12205d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 12215d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 12225d9e75c4SJan Schmidt tm->generation); 12235d9e75c4SJan Schmidt break; 12245d9e75c4SJan Schmidt case MOD_LOG_KEY_ADD: 122519956c7eSJan Schmidt /* if a move operation is needed it's in the log */ 12265d9e75c4SJan Schmidt n--; 12275d9e75c4SJan Schmidt break; 12285d9e75c4SJan Schmidt case MOD_LOG_MOVE_KEYS: 1229c3193108SJan Schmidt o_dst = btrfs_node_key_ptr_offset(tm->slot); 1230c3193108SJan Schmidt o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot); 1231c3193108SJan Schmidt memmove_extent_buffer(eb, o_dst, o_src, 12325d9e75c4SJan Schmidt tm->move.nr_items * p_size); 12335d9e75c4SJan Schmidt break; 12345d9e75c4SJan Schmidt case MOD_LOG_ROOT_REPLACE: 12355d9e75c4SJan Schmidt /* 12365d9e75c4SJan Schmidt * this operation is special. for roots, this must be 12375d9e75c4SJan Schmidt * handled explicitly before rewinding. 12385d9e75c4SJan Schmidt * for non-roots, this operation may exist if the node 12395d9e75c4SJan Schmidt * was a root: root A -> child B; then A gets empty and 12405d9e75c4SJan Schmidt * B is promoted to the new root. in the mod log, we'll 12415d9e75c4SJan Schmidt * have a root-replace operation for B, a tree block 12425d9e75c4SJan Schmidt * that is no root. we simply ignore that operation. 12435d9e75c4SJan Schmidt */ 12445d9e75c4SJan Schmidt break; 12455d9e75c4SJan Schmidt } 12465d9e75c4SJan Schmidt next = rb_next(&tm->node); 12475d9e75c4SJan Schmidt if (!next) 12485d9e75c4SJan Schmidt break; 12496b4df8b6SGeliang Tang tm = rb_entry(next, struct tree_mod_elem, node); 1250298cfd36SChandan Rajendra if (tm->logical != first_tm->logical) 12515d9e75c4SJan Schmidt break; 12525d9e75c4SJan Schmidt } 1253b1a09f1eSDavid Sterba read_unlock(&fs_info->tree_mod_log_lock); 12545d9e75c4SJan Schmidt btrfs_set_header_nritems(eb, n); 12555d9e75c4SJan Schmidt } 12565d9e75c4SJan Schmidt 125747fb091fSJan Schmidt /* 125801327610SNicholas D Steeves * Called with eb read locked. If the buffer cannot be rewound, the same buffer 125947fb091fSJan Schmidt * is returned. If rewind operations happen, a fresh buffer is returned. The 126047fb091fSJan Schmidt * returned buffer is always read-locked. If the returned buffer is not the 126147fb091fSJan Schmidt * input buffer, the lock on the input buffer is released and the input buffer 126247fb091fSJan Schmidt * is freed (its refcount is decremented). 126347fb091fSJan Schmidt */ 12645d9e75c4SJan Schmidt static struct extent_buffer * 12659ec72677SJosef Bacik tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path, 12669ec72677SJosef Bacik struct extent_buffer *eb, u64 time_seq) 12675d9e75c4SJan Schmidt { 12685d9e75c4SJan Schmidt struct extent_buffer *eb_rewin; 12695d9e75c4SJan Schmidt struct tree_mod_elem *tm; 12705d9e75c4SJan Schmidt 12715d9e75c4SJan Schmidt if (!time_seq) 12725d9e75c4SJan Schmidt return eb; 12735d9e75c4SJan Schmidt 12745d9e75c4SJan Schmidt if (btrfs_header_level(eb) == 0) 12755d9e75c4SJan Schmidt return eb; 12765d9e75c4SJan Schmidt 12775d9e75c4SJan Schmidt tm = tree_mod_log_search(fs_info, eb->start, time_seq); 12785d9e75c4SJan Schmidt if (!tm) 12795d9e75c4SJan Schmidt return eb; 12805d9e75c4SJan Schmidt 12815d9e75c4SJan Schmidt if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 12825d9e75c4SJan Schmidt BUG_ON(tm->slot != 0); 1283da17066cSJeff Mahoney eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start); 1284db7f3436SJosef Bacik if (!eb_rewin) { 1285ac5887c8SJosef Bacik btrfs_tree_read_unlock(eb); 1286db7f3436SJosef Bacik free_extent_buffer(eb); 1287db7f3436SJosef Bacik return NULL; 1288db7f3436SJosef Bacik } 12895d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb_rewin, eb->start); 12905d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb_rewin, 12915d9e75c4SJan Schmidt btrfs_header_backref_rev(eb)); 12925d9e75c4SJan Schmidt btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb)); 1293c3193108SJan Schmidt btrfs_set_header_level(eb_rewin, btrfs_header_level(eb)); 12945d9e75c4SJan Schmidt } else { 12955d9e75c4SJan Schmidt eb_rewin = btrfs_clone_extent_buffer(eb); 1296db7f3436SJosef Bacik if (!eb_rewin) { 1297ac5887c8SJosef Bacik btrfs_tree_read_unlock(eb); 1298db7f3436SJosef Bacik free_extent_buffer(eb); 1299db7f3436SJosef Bacik return NULL; 1300db7f3436SJosef Bacik } 13015d9e75c4SJan Schmidt } 13025d9e75c4SJan Schmidt 1303ac5887c8SJosef Bacik btrfs_tree_read_unlock(eb); 13045d9e75c4SJan Schmidt free_extent_buffer(eb); 13055d9e75c4SJan Schmidt 1306d3beaa25SJosef Bacik btrfs_set_buffer_lockdep_class(btrfs_header_owner(eb_rewin), 1307d3beaa25SJosef Bacik eb_rewin, btrfs_header_level(eb_rewin)); 130847fb091fSJan Schmidt btrfs_tree_read_lock(eb_rewin); 1309f1ca7e98SJosef Bacik __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm); 131057911b8bSJan Schmidt WARN_ON(btrfs_header_nritems(eb_rewin) > 1311da17066cSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 13125d9e75c4SJan Schmidt 13135d9e75c4SJan Schmidt return eb_rewin; 13145d9e75c4SJan Schmidt } 13155d9e75c4SJan Schmidt 13168ba97a15SJan Schmidt /* 13178ba97a15SJan Schmidt * get_old_root() rewinds the state of @root's root node to the given @time_seq 13188ba97a15SJan Schmidt * value. If there are no changes, the current root->root_node is returned. If 13198ba97a15SJan Schmidt * anything changed in between, there's a fresh buffer allocated on which the 13208ba97a15SJan Schmidt * rewind operations are done. In any case, the returned buffer is read locked. 13218ba97a15SJan Schmidt * Returns NULL on error (with no locks held). 13228ba97a15SJan Schmidt */ 13235d9e75c4SJan Schmidt static inline struct extent_buffer * 13245d9e75c4SJan Schmidt get_old_root(struct btrfs_root *root, u64 time_seq) 13255d9e75c4SJan Schmidt { 13260b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 13275d9e75c4SJan Schmidt struct tree_mod_elem *tm; 132830b0463aSJan Schmidt struct extent_buffer *eb = NULL; 132930b0463aSJan Schmidt struct extent_buffer *eb_root; 1330efad8a85SFilipe Manana u64 eb_root_owner = 0; 13317bfdcf7fSLiu Bo struct extent_buffer *old; 1332a95236d9SJan Schmidt struct tree_mod_root *old_root = NULL; 13334325edd0SChris Mason u64 old_generation = 0; 1334a95236d9SJan Schmidt u64 logical; 1335581c1760SQu Wenruo int level; 13365d9e75c4SJan Schmidt 133730b0463aSJan Schmidt eb_root = btrfs_read_lock_root_node(root); 1338bcd24dabSDavid Sterba tm = __tree_mod_log_oldest_root(eb_root, time_seq); 13395d9e75c4SJan Schmidt if (!tm) 134030b0463aSJan Schmidt return eb_root; 13415d9e75c4SJan Schmidt 1342a95236d9SJan Schmidt if (tm->op == MOD_LOG_ROOT_REPLACE) { 13435d9e75c4SJan Schmidt old_root = &tm->old_root; 13445d9e75c4SJan Schmidt old_generation = tm->generation; 1345a95236d9SJan Schmidt logical = old_root->logical; 1346581c1760SQu Wenruo level = old_root->level; 1347a95236d9SJan Schmidt } else { 134830b0463aSJan Schmidt logical = eb_root->start; 1349581c1760SQu Wenruo level = btrfs_header_level(eb_root); 1350a95236d9SJan Schmidt } 13515d9e75c4SJan Schmidt 13520b246afaSJeff Mahoney tm = tree_mod_log_search(fs_info, logical, time_seq); 1353834328a8SJan Schmidt if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 135430b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 135530b0463aSJan Schmidt free_extent_buffer(eb_root); 1356581c1760SQu Wenruo old = read_tree_block(fs_info, logical, 0, level, NULL); 135764c043deSLiu Bo if (WARN_ON(IS_ERR(old) || !extent_buffer_uptodate(old))) { 135864c043deSLiu Bo if (!IS_ERR(old)) 1359416bc658SJosef Bacik free_extent_buffer(old); 13600b246afaSJeff Mahoney btrfs_warn(fs_info, 13610b246afaSJeff Mahoney "failed to read tree block %llu from get_old_root", 13620b246afaSJeff Mahoney logical); 1363834328a8SJan Schmidt } else { 13647bfdcf7fSLiu Bo eb = btrfs_clone_extent_buffer(old); 13657bfdcf7fSLiu Bo free_extent_buffer(old); 1366834328a8SJan Schmidt } 1367834328a8SJan Schmidt } else if (old_root) { 1368efad8a85SFilipe Manana eb_root_owner = btrfs_header_owner(eb_root); 136930b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 137030b0463aSJan Schmidt free_extent_buffer(eb_root); 13710b246afaSJeff Mahoney eb = alloc_dummy_extent_buffer(fs_info, logical); 1372834328a8SJan Schmidt } else { 137330b0463aSJan Schmidt eb = btrfs_clone_extent_buffer(eb_root); 1374ac5887c8SJosef Bacik btrfs_tree_read_unlock(eb_root); 137530b0463aSJan Schmidt free_extent_buffer(eb_root); 1376834328a8SJan Schmidt } 1377834328a8SJan Schmidt 13788ba97a15SJan Schmidt if (!eb) 13798ba97a15SJan Schmidt return NULL; 1380a95236d9SJan Schmidt if (old_root) { 13815d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb, eb->start); 13825d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV); 1383efad8a85SFilipe Manana btrfs_set_header_owner(eb, eb_root_owner); 13845d9e75c4SJan Schmidt btrfs_set_header_level(eb, old_root->level); 13855d9e75c4SJan Schmidt btrfs_set_header_generation(eb, old_generation); 1386a95236d9SJan Schmidt } 1387d3beaa25SJosef Bacik btrfs_set_buffer_lockdep_class(btrfs_header_owner(eb), eb, 1388d3beaa25SJosef Bacik btrfs_header_level(eb)); 1389d3beaa25SJosef Bacik btrfs_tree_read_lock(eb); 139028da9fb4SJan Schmidt if (tm) 13910b246afaSJeff Mahoney __tree_mod_log_rewind(fs_info, eb, time_seq, tm); 139228da9fb4SJan Schmidt else 139328da9fb4SJan Schmidt WARN_ON(btrfs_header_level(eb) != 0); 13940b246afaSJeff Mahoney WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 13955d9e75c4SJan Schmidt 13965d9e75c4SJan Schmidt return eb; 13975d9e75c4SJan Schmidt } 13985d9e75c4SJan Schmidt 13995b6602e7SJan Schmidt int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq) 14005b6602e7SJan Schmidt { 14015b6602e7SJan Schmidt struct tree_mod_elem *tm; 14025b6602e7SJan Schmidt int level; 140330b0463aSJan Schmidt struct extent_buffer *eb_root = btrfs_root_node(root); 14045b6602e7SJan Schmidt 1405bcd24dabSDavid Sterba tm = __tree_mod_log_oldest_root(eb_root, time_seq); 14065b6602e7SJan Schmidt if (tm && tm->op == MOD_LOG_ROOT_REPLACE) { 14075b6602e7SJan Schmidt level = tm->old_root.level; 14085b6602e7SJan Schmidt } else { 140930b0463aSJan Schmidt level = btrfs_header_level(eb_root); 14105b6602e7SJan Schmidt } 141130b0463aSJan Schmidt free_extent_buffer(eb_root); 14125b6602e7SJan Schmidt 14135b6602e7SJan Schmidt return level; 14145b6602e7SJan Schmidt } 14155b6602e7SJan Schmidt 14165d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 14175d4f98a2SYan Zheng struct btrfs_root *root, 14185d4f98a2SYan Zheng struct extent_buffer *buf) 14195d4f98a2SYan Zheng { 1420f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 1421faa2dbf0SJosef Bacik return 0; 1422fccb84c9SDavid Sterba 1423d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 1424d1980131SDavid Sterba smp_mb__before_atomic(); 1425f1ebcc74SLiu Bo 1426f1ebcc74SLiu Bo /* 1427f1ebcc74SLiu Bo * We do not need to cow a block if 1428f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 1429f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 1430f1ebcc74SLiu Bo * 3) the root is not forced COW. 1431f1ebcc74SLiu Bo * 1432f1ebcc74SLiu Bo * What is forced COW: 143301327610SNicholas D Steeves * when we create snapshot during committing the transaction, 143452042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 1435f1ebcc74SLiu Bo * block to ensure the metadata consistency. 1436f1ebcc74SLiu Bo */ 14375d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 14385d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 14395d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 1440f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 144127cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 14425d4f98a2SYan Zheng return 0; 14435d4f98a2SYan Zheng return 1; 14445d4f98a2SYan Zheng } 14455d4f98a2SYan Zheng 1446d352ac68SChris Mason /* 1447d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 144801327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 1449d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 1450d352ac68SChris Mason */ 1451d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 14525f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 14535f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 14549631e4ccSJosef Bacik struct extent_buffer **cow_ret, 14559631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 145602217ed2SChris Mason { 14570b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 14586702ed49SChris Mason u64 search_start; 1459f510cfecSChris Mason int ret; 1460dc17ff8fSChris Mason 146183354f07SJosef Bacik if (test_bit(BTRFS_ROOT_DELETING, &root->state)) 146283354f07SJosef Bacik btrfs_err(fs_info, 146383354f07SJosef Bacik "COW'ing blocks on a fs root that's being dropped"); 146483354f07SJosef Bacik 14650b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 146631b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 1467c1c9ff7cSGeert Uytterhoeven trans->transid, 14680b246afaSJeff Mahoney fs_info->running_transaction->transid); 146931b1a2bdSJulia Lawall 14700b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 147131b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 14720b246afaSJeff Mahoney trans->transid, fs_info->generation); 1473dc17ff8fSChris Mason 14745d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 147564c12921SJeff Mahoney trans->dirty = true; 147602217ed2SChris Mason *cow_ret = buf; 147702217ed2SChris Mason return 0; 147802217ed2SChris Mason } 1479c487685dSChris Mason 1480ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 1481b4ce94deSChris Mason 1482f616f5cdSQu Wenruo /* 1483f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 1484f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 1485f616f5cdSQu Wenruo * 1486f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 1487f616f5cdSQu Wenruo */ 1488f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 1489f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 14909631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 14911abe9b8aSliubo 14921abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 14931abe9b8aSliubo 1494f510cfecSChris Mason return ret; 14952c90e5d6SChris Mason } 14966702ed49SChris Mason 1497d352ac68SChris Mason /* 1498d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 1499d352ac68SChris Mason * node are actually close by 1500d352ac68SChris Mason */ 15016b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 15026702ed49SChris Mason { 15036b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 15046702ed49SChris Mason return 1; 15056b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 15066702ed49SChris Mason return 1; 150702217ed2SChris Mason return 0; 150802217ed2SChris Mason } 150902217ed2SChris Mason 1510ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 1511ce6ef5abSDavid Sterba 1512ce6ef5abSDavid Sterba /* 1513ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 1514ce6ef5abSDavid Sterba * we can avoid the conversion. 1515ce6ef5abSDavid Sterba */ 1516ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 1517ce6ef5abSDavid Sterba const struct btrfs_key *k2) 1518ce6ef5abSDavid Sterba { 1519ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 1520ce6ef5abSDavid Sterba 1521ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 1522ce6ef5abSDavid Sterba } 1523ce6ef5abSDavid Sterba 1524ce6ef5abSDavid Sterba #else 1525ce6ef5abSDavid Sterba 1526081e9573SChris Mason /* 1527081e9573SChris Mason * compare two keys in a memcmp fashion 1528081e9573SChris Mason */ 1529310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 1530310712b2SOmar Sandoval const struct btrfs_key *k2) 1531081e9573SChris Mason { 1532081e9573SChris Mason struct btrfs_key k1; 1533081e9573SChris Mason 1534081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 1535081e9573SChris Mason 153620736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 1537081e9573SChris Mason } 1538ce6ef5abSDavid Sterba #endif 1539081e9573SChris Mason 1540f3465ca4SJosef Bacik /* 1541f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 1542f3465ca4SJosef Bacik */ 1543e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 1544f3465ca4SJosef Bacik { 1545f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 1546f3465ca4SJosef Bacik return 1; 1547f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 1548f3465ca4SJosef Bacik return -1; 1549f3465ca4SJosef Bacik if (k1->type > k2->type) 1550f3465ca4SJosef Bacik return 1; 1551f3465ca4SJosef Bacik if (k1->type < k2->type) 1552f3465ca4SJosef Bacik return -1; 1553f3465ca4SJosef Bacik if (k1->offset > k2->offset) 1554f3465ca4SJosef Bacik return 1; 1555f3465ca4SJosef Bacik if (k1->offset < k2->offset) 1556f3465ca4SJosef Bacik return -1; 1557f3465ca4SJosef Bacik return 0; 1558f3465ca4SJosef Bacik } 1559081e9573SChris Mason 1560d352ac68SChris Mason /* 1561d352ac68SChris Mason * this is used by the defrag code to go through all the 1562d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 1563d352ac68SChris Mason * disk order is close to key order 1564d352ac68SChris Mason */ 15656702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 15665f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 1567de78b51aSEric Sandeen int start_slot, u64 *last_ret, 1568a6b6e75eSChris Mason struct btrfs_key *progress) 15696702ed49SChris Mason { 15700b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 15716b80053dSChris Mason struct extent_buffer *cur; 15726702ed49SChris Mason u64 blocknr; 1573ca7a79adSChris Mason u64 gen; 1574e9d0b13bSChris Mason u64 search_start = *last_ret; 1575e9d0b13bSChris Mason u64 last_block = 0; 15766702ed49SChris Mason u64 other; 15776702ed49SChris Mason u32 parent_nritems; 15786702ed49SChris Mason int end_slot; 15796702ed49SChris Mason int i; 15806702ed49SChris Mason int err = 0; 1581f2183bdeSChris Mason int parent_level; 15826b80053dSChris Mason int uptodate; 15836b80053dSChris Mason u32 blocksize; 1584081e9573SChris Mason int progress_passed = 0; 1585081e9573SChris Mason struct btrfs_disk_key disk_key; 15866702ed49SChris Mason 15875708b959SChris Mason parent_level = btrfs_header_level(parent); 15885708b959SChris Mason 15890b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 15900b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 159186479a04SChris Mason 15926b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 15930b246afaSJeff Mahoney blocksize = fs_info->nodesize; 15945dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 15956702ed49SChris Mason 15965dfe2be7SFilipe Manana if (parent_nritems <= 1) 15976702ed49SChris Mason return 0; 15986702ed49SChris Mason 15995dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 1600581c1760SQu Wenruo struct btrfs_key first_key; 16016702ed49SChris Mason int close = 1; 1602a6b6e75eSChris Mason 1603081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 1604081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 1605081e9573SChris Mason continue; 1606081e9573SChris Mason 1607081e9573SChris Mason progress_passed = 1; 16086b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 1609ca7a79adSChris Mason gen = btrfs_node_ptr_generation(parent, i); 1610581c1760SQu Wenruo btrfs_node_key_to_cpu(parent, &first_key, i); 1611e9d0b13bSChris Mason if (last_block == 0) 1612e9d0b13bSChris Mason last_block = blocknr; 16135708b959SChris Mason 16146702ed49SChris Mason if (i > 0) { 16156b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 16166b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 16176702ed49SChris Mason } 16185dfe2be7SFilipe Manana if (!close && i < end_slot) { 16196b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 16206b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 16216702ed49SChris Mason } 1622e9d0b13bSChris Mason if (close) { 1623e9d0b13bSChris Mason last_block = blocknr; 16246702ed49SChris Mason continue; 1625e9d0b13bSChris Mason } 16266702ed49SChris Mason 16270b246afaSJeff Mahoney cur = find_extent_buffer(fs_info, blocknr); 16286b80053dSChris Mason if (cur) 1629b9fab919SChris Mason uptodate = btrfs_buffer_uptodate(cur, gen, 0); 16306b80053dSChris Mason else 16316b80053dSChris Mason uptodate = 0; 16325708b959SChris Mason if (!cur || !uptodate) { 16336b80053dSChris Mason if (!cur) { 1634581c1760SQu Wenruo cur = read_tree_block(fs_info, blocknr, gen, 1635581c1760SQu Wenruo parent_level - 1, 1636581c1760SQu Wenruo &first_key); 163764c043deSLiu Bo if (IS_ERR(cur)) { 163864c043deSLiu Bo return PTR_ERR(cur); 163964c043deSLiu Bo } else if (!extent_buffer_uptodate(cur)) { 1640416bc658SJosef Bacik free_extent_buffer(cur); 164197d9a8a4STsutomu Itoh return -EIO; 1642416bc658SJosef Bacik } 16436b80053dSChris Mason } else if (!uptodate) { 1644581c1760SQu Wenruo err = btrfs_read_buffer(cur, gen, 1645581c1760SQu Wenruo parent_level - 1,&first_key); 1646018642a1STsutomu Itoh if (err) { 1647018642a1STsutomu Itoh free_extent_buffer(cur); 1648018642a1STsutomu Itoh return err; 1649018642a1STsutomu Itoh } 16506702ed49SChris Mason } 1651f2183bdeSChris Mason } 1652e9d0b13bSChris Mason if (search_start == 0) 16536b80053dSChris Mason search_start = last_block; 1654e9d0b13bSChris Mason 1655e7a84565SChris Mason btrfs_tree_lock(cur); 16566b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 1657e7a84565SChris Mason &cur, search_start, 16586b80053dSChris Mason min(16 * blocksize, 16599631e4ccSJosef Bacik (end_slot - i) * blocksize), 16609631e4ccSJosef Bacik BTRFS_NESTING_COW); 1661252c38f0SYan if (err) { 1662e7a84565SChris Mason btrfs_tree_unlock(cur); 16636b80053dSChris Mason free_extent_buffer(cur); 16646702ed49SChris Mason break; 1665252c38f0SYan } 1666e7a84565SChris Mason search_start = cur->start; 1667e7a84565SChris Mason last_block = cur->start; 1668f2183bdeSChris Mason *last_ret = search_start; 1669e7a84565SChris Mason btrfs_tree_unlock(cur); 1670e7a84565SChris Mason free_extent_buffer(cur); 16716702ed49SChris Mason } 16726702ed49SChris Mason return err; 16736702ed49SChris Mason } 16746702ed49SChris Mason 167574123bd7SChris Mason /* 16765f39d397SChris Mason * search for key in the extent_buffer. The items start at offset p, 16775f39d397SChris Mason * and they are item_size apart. There are 'max' items in p. 16785f39d397SChris Mason * 167974123bd7SChris Mason * the slot in the array is returned via slot, and it points to 168074123bd7SChris Mason * the place where you would insert key if it is not found in 168174123bd7SChris Mason * the array. 168274123bd7SChris Mason * 168374123bd7SChris Mason * slot may point to max if the key is bigger than all of the keys 168474123bd7SChris Mason */ 1685e02119d5SChris Mason static noinline int generic_bin_search(struct extent_buffer *eb, 1686310712b2SOmar Sandoval unsigned long p, int item_size, 1687310712b2SOmar Sandoval const struct btrfs_key *key, 1688be0e5c09SChris Mason int max, int *slot) 1689be0e5c09SChris Mason { 1690be0e5c09SChris Mason int low = 0; 1691be0e5c09SChris Mason int high = max; 1692be0e5c09SChris Mason int ret; 16935cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 1694be0e5c09SChris Mason 16955e24e9afSLiu Bo if (low > high) { 16965e24e9afSLiu Bo btrfs_err(eb->fs_info, 16975e24e9afSLiu Bo "%s: low (%d) > high (%d) eb %llu owner %llu level %d", 16985e24e9afSLiu Bo __func__, low, high, eb->start, 16995e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 17005e24e9afSLiu Bo return -EINVAL; 17015e24e9afSLiu Bo } 17025e24e9afSLiu Bo 1703be0e5c09SChris Mason while (low < high) { 17045cd17f34SDavid Sterba unsigned long oip; 17055cd17f34SDavid Sterba unsigned long offset; 17065cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 17075cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 17085cd17f34SDavid Sterba int mid; 17095cd17f34SDavid Sterba 1710be0e5c09SChris Mason mid = (low + high) / 2; 17115f39d397SChris Mason offset = p + mid * item_size; 17125cd17f34SDavid Sterba oip = offset_in_page(offset); 17135f39d397SChris Mason 17145cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 17155cd17f34SDavid Sterba const unsigned long idx = offset >> PAGE_SHIFT; 17165cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 1717934d375bSChris Mason 17185cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 17195cd17f34SDavid Sterba } else { 17205cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 17215f39d397SChris Mason tmp = &unaligned; 1722479965d6SChris Mason } 1723479965d6SChris Mason 1724be0e5c09SChris Mason ret = comp_keys(tmp, key); 1725be0e5c09SChris Mason 1726be0e5c09SChris Mason if (ret < 0) 1727be0e5c09SChris Mason low = mid + 1; 1728be0e5c09SChris Mason else if (ret > 0) 1729be0e5c09SChris Mason high = mid; 1730be0e5c09SChris Mason else { 1731be0e5c09SChris Mason *slot = mid; 1732be0e5c09SChris Mason return 0; 1733be0e5c09SChris Mason } 1734be0e5c09SChris Mason } 1735be0e5c09SChris Mason *slot = low; 1736be0e5c09SChris Mason return 1; 1737be0e5c09SChris Mason } 1738be0e5c09SChris Mason 173997571fd0SChris Mason /* 174097571fd0SChris Mason * simple bin_search frontend that does the right thing for 174197571fd0SChris Mason * leaves vs nodes 174297571fd0SChris Mason */ 1743a74b35ecSNikolay Borisov int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, 1744e3b83361SQu Wenruo int *slot) 1745be0e5c09SChris Mason { 1746e3b83361SQu Wenruo if (btrfs_header_level(eb) == 0) 17475f39d397SChris Mason return generic_bin_search(eb, 17485f39d397SChris Mason offsetof(struct btrfs_leaf, items), 17490783fcfcSChris Mason sizeof(struct btrfs_item), 17505f39d397SChris Mason key, btrfs_header_nritems(eb), 17517518a238SChris Mason slot); 1752f775738fSWang Sheng-Hui else 17535f39d397SChris Mason return generic_bin_search(eb, 17545f39d397SChris Mason offsetof(struct btrfs_node, ptrs), 1755123abc88SChris Mason sizeof(struct btrfs_key_ptr), 17565f39d397SChris Mason key, btrfs_header_nritems(eb), 17577518a238SChris Mason slot); 1758be0e5c09SChris Mason } 1759be0e5c09SChris Mason 1760f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 1761f0486c68SYan, Zheng { 1762f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1763f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1764f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 1765f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1766f0486c68SYan, Zheng } 1767f0486c68SYan, Zheng 1768f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 1769f0486c68SYan, Zheng { 1770f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1771f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1772f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 1773f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1774f0486c68SYan, Zheng } 1775f0486c68SYan, Zheng 1776d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 1777d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 1778d352ac68SChris Mason */ 17794b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 17804b231ae4SDavid Sterba int slot) 1781bb803951SChris Mason { 1782ca7a79adSChris Mason int level = btrfs_header_level(parent); 1783416bc658SJosef Bacik struct extent_buffer *eb; 1784581c1760SQu Wenruo struct btrfs_key first_key; 1785416bc658SJosef Bacik 1786fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 1787fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 1788ca7a79adSChris Mason 1789ca7a79adSChris Mason BUG_ON(level == 0); 1790ca7a79adSChris Mason 1791581c1760SQu Wenruo btrfs_node_key_to_cpu(parent, &first_key, slot); 1792d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 1793581c1760SQu Wenruo btrfs_node_ptr_generation(parent, slot), 1794581c1760SQu Wenruo level - 1, &first_key); 1795fb770ae4SLiu Bo if (!IS_ERR(eb) && !extent_buffer_uptodate(eb)) { 1796416bc658SJosef Bacik free_extent_buffer(eb); 1797fb770ae4SLiu Bo eb = ERR_PTR(-EIO); 1798416bc658SJosef Bacik } 1799416bc658SJosef Bacik 1800416bc658SJosef Bacik return eb; 1801bb803951SChris Mason } 1802bb803951SChris Mason 1803d352ac68SChris Mason /* 1804d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 1805d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 1806d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 1807d352ac68SChris Mason */ 1808e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 180998ed5174SChris Mason struct btrfs_root *root, 181098ed5174SChris Mason struct btrfs_path *path, int level) 1811bb803951SChris Mason { 18120b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 18135f39d397SChris Mason struct extent_buffer *right = NULL; 18145f39d397SChris Mason struct extent_buffer *mid; 18155f39d397SChris Mason struct extent_buffer *left = NULL; 18165f39d397SChris Mason struct extent_buffer *parent = NULL; 1817bb803951SChris Mason int ret = 0; 1818bb803951SChris Mason int wret; 1819bb803951SChris Mason int pslot; 1820bb803951SChris Mason int orig_slot = path->slots[level]; 182179f95c82SChris Mason u64 orig_ptr; 1822bb803951SChris Mason 182398e6b1ebSLiu Bo ASSERT(level > 0); 1824bb803951SChris Mason 18255f39d397SChris Mason mid = path->nodes[level]; 1826b4ce94deSChris Mason 1827ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 18287bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 18297bb86316SChris Mason 18301d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 183179f95c82SChris Mason 1832a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 18335f39d397SChris Mason parent = path->nodes[level + 1]; 1834bb803951SChris Mason pslot = path->slots[level + 1]; 1835a05a9bb1SLi Zefan } 1836bb803951SChris Mason 183740689478SChris Mason /* 183840689478SChris Mason * deal with the case where there is only one pointer in the root 183940689478SChris Mason * by promoting the node below to a root 184040689478SChris Mason */ 18415f39d397SChris Mason if (!parent) { 18425f39d397SChris Mason struct extent_buffer *child; 1843bb803951SChris Mason 18445f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1845bb803951SChris Mason return 0; 1846bb803951SChris Mason 1847bb803951SChris Mason /* promote the child to a root */ 18484b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 1849fb770ae4SLiu Bo if (IS_ERR(child)) { 1850fb770ae4SLiu Bo ret = PTR_ERR(child); 18510b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1852305a26afSMark Fasheh goto enospc; 1853305a26afSMark Fasheh } 1854305a26afSMark Fasheh 1855925baeddSChris Mason btrfs_tree_lock(child); 18569631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 18579631e4ccSJosef Bacik BTRFS_NESTING_COW); 1858f0486c68SYan, Zheng if (ret) { 1859f0486c68SYan, Zheng btrfs_tree_unlock(child); 1860f0486c68SYan, Zheng free_extent_buffer(child); 1861f0486c68SYan, Zheng goto enospc; 1862f0486c68SYan, Zheng } 18632f375ab9SYan 1864d9d19a01SDavid Sterba ret = tree_mod_log_insert_root(root->node, child, 1); 1865d9d19a01SDavid Sterba BUG_ON(ret < 0); 1866240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1867925baeddSChris Mason 18680b86a832SChris Mason add_root_to_dirty_list(root); 1869925baeddSChris Mason btrfs_tree_unlock(child); 1870b4ce94deSChris Mason 1871925baeddSChris Mason path->locks[level] = 0; 1872bb803951SChris Mason path->nodes[level] = NULL; 18736a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 1874925baeddSChris Mason btrfs_tree_unlock(mid); 1875bb803951SChris Mason /* once for the path */ 18765f39d397SChris Mason free_extent_buffer(mid); 1877f0486c68SYan, Zheng 1878f0486c68SYan, Zheng root_sub_used(root, mid->len); 18795581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 1880bb803951SChris Mason /* once for the root ptr */ 18813083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1882f0486c68SYan, Zheng return 0; 1883bb803951SChris Mason } 18845f39d397SChris Mason if (btrfs_header_nritems(mid) > 18850b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 1886bb803951SChris Mason return 0; 1887bb803951SChris Mason 18884b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1889fb770ae4SLiu Bo if (IS_ERR(left)) 1890fb770ae4SLiu Bo left = NULL; 1891fb770ae4SLiu Bo 18925f39d397SChris Mason if (left) { 1893bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 18945f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 18959631e4ccSJosef Bacik parent, pslot - 1, &left, 1896bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 189754aa1f4dSChris Mason if (wret) { 189854aa1f4dSChris Mason ret = wret; 189954aa1f4dSChris Mason goto enospc; 190054aa1f4dSChris Mason } 19012cc58cf2SChris Mason } 1902fb770ae4SLiu Bo 19034b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 1904fb770ae4SLiu Bo if (IS_ERR(right)) 1905fb770ae4SLiu Bo right = NULL; 1906fb770ae4SLiu Bo 19075f39d397SChris Mason if (right) { 1908bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 19095f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 19109631e4ccSJosef Bacik parent, pslot + 1, &right, 1911bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 19122cc58cf2SChris Mason if (wret) { 19132cc58cf2SChris Mason ret = wret; 19142cc58cf2SChris Mason goto enospc; 19152cc58cf2SChris Mason } 19162cc58cf2SChris Mason } 19172cc58cf2SChris Mason 19182cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 19195f39d397SChris Mason if (left) { 19205f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1921d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 192279f95c82SChris Mason if (wret < 0) 192379f95c82SChris Mason ret = wret; 1924bb803951SChris Mason } 192579f95c82SChris Mason 192679f95c82SChris Mason /* 192779f95c82SChris Mason * then try to empty the right most buffer into the middle 192879f95c82SChris Mason */ 19295f39d397SChris Mason if (right) { 1930d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 193154aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 193279f95c82SChris Mason ret = wret; 19335f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 19346a884d7dSDavid Sterba btrfs_clean_tree_block(right); 1935925baeddSChris Mason btrfs_tree_unlock(right); 1936afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 1937f0486c68SYan, Zheng root_sub_used(root, right->len); 19385581a51aSJan Schmidt btrfs_free_tree_block(trans, root, right, 0, 1); 19393083ee2eSJosef Bacik free_extent_buffer_stale(right); 1940f0486c68SYan, Zheng right = NULL; 1941bb803951SChris Mason } else { 19425f39d397SChris Mason struct btrfs_disk_key right_key; 19435f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 19440e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot + 1, 19450e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 19460e82bcfeSDavid Sterba BUG_ON(ret < 0); 19475f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 19485f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1949bb803951SChris Mason } 1950bb803951SChris Mason } 19515f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 195279f95c82SChris Mason /* 195379f95c82SChris Mason * we're not allowed to leave a node with one item in the 195479f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 195579f95c82SChris Mason * could try to delete the only pointer in this node. 195679f95c82SChris Mason * So, pull some keys from the left. 195779f95c82SChris Mason * There has to be a left pointer at this point because 195879f95c82SChris Mason * otherwise we would have pulled some pointers from the 195979f95c82SChris Mason * right 196079f95c82SChris Mason */ 1961305a26afSMark Fasheh if (!left) { 1962305a26afSMark Fasheh ret = -EROFS; 19630b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1964305a26afSMark Fasheh goto enospc; 1965305a26afSMark Fasheh } 196655d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 196754aa1f4dSChris Mason if (wret < 0) { 196879f95c82SChris Mason ret = wret; 196954aa1f4dSChris Mason goto enospc; 197054aa1f4dSChris Mason } 1971bce4eae9SChris Mason if (wret == 1) { 1972d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1973bce4eae9SChris Mason if (wret < 0) 1974bce4eae9SChris Mason ret = wret; 1975bce4eae9SChris Mason } 197679f95c82SChris Mason BUG_ON(wret == 1); 197779f95c82SChris Mason } 19785f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 19796a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 1980925baeddSChris Mason btrfs_tree_unlock(mid); 1981afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1982f0486c68SYan, Zheng root_sub_used(root, mid->len); 19835581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 19843083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1985f0486c68SYan, Zheng mid = NULL; 198679f95c82SChris Mason } else { 198779f95c82SChris Mason /* update the parent key to reflect our changes */ 19885f39d397SChris Mason struct btrfs_disk_key mid_key; 19895f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 19900e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot, 19910e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 19920e82bcfeSDavid Sterba BUG_ON(ret < 0); 19935f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 19945f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 199579f95c82SChris Mason } 1996bb803951SChris Mason 199779f95c82SChris Mason /* update the path */ 19985f39d397SChris Mason if (left) { 19995f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 200067439dadSDavid Sterba atomic_inc(&left->refs); 2001925baeddSChris Mason /* left was locked after cow */ 20025f39d397SChris Mason path->nodes[level] = left; 2003bb803951SChris Mason path->slots[level + 1] -= 1; 2004bb803951SChris Mason path->slots[level] = orig_slot; 2005925baeddSChris Mason if (mid) { 2006925baeddSChris Mason btrfs_tree_unlock(mid); 20075f39d397SChris Mason free_extent_buffer(mid); 2008925baeddSChris Mason } 2009bb803951SChris Mason } else { 20105f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 2011bb803951SChris Mason path->slots[level] = orig_slot; 2012bb803951SChris Mason } 2013bb803951SChris Mason } 201479f95c82SChris Mason /* double check we haven't messed things up */ 2015e20d96d6SChris Mason if (orig_ptr != 20165f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 201779f95c82SChris Mason BUG(); 201854aa1f4dSChris Mason enospc: 2019925baeddSChris Mason if (right) { 2020925baeddSChris Mason btrfs_tree_unlock(right); 20215f39d397SChris Mason free_extent_buffer(right); 2022925baeddSChris Mason } 2023925baeddSChris Mason if (left) { 2024925baeddSChris Mason if (path->nodes[level] != left) 2025925baeddSChris Mason btrfs_tree_unlock(left); 20265f39d397SChris Mason free_extent_buffer(left); 2027925baeddSChris Mason } 2028bb803951SChris Mason return ret; 2029bb803951SChris Mason } 2030bb803951SChris Mason 2031d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 2032d352ac68SChris Mason * when they are completely full. This is also done top down, so we 2033d352ac68SChris Mason * have to be pessimistic. 2034d352ac68SChris Mason */ 2035d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 2036e66f709bSChris Mason struct btrfs_root *root, 2037e66f709bSChris Mason struct btrfs_path *path, int level) 2038e66f709bSChris Mason { 20390b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 20405f39d397SChris Mason struct extent_buffer *right = NULL; 20415f39d397SChris Mason struct extent_buffer *mid; 20425f39d397SChris Mason struct extent_buffer *left = NULL; 20435f39d397SChris Mason struct extent_buffer *parent = NULL; 2044e66f709bSChris Mason int ret = 0; 2045e66f709bSChris Mason int wret; 2046e66f709bSChris Mason int pslot; 2047e66f709bSChris Mason int orig_slot = path->slots[level]; 2048e66f709bSChris Mason 2049e66f709bSChris Mason if (level == 0) 2050e66f709bSChris Mason return 1; 2051e66f709bSChris Mason 20525f39d397SChris Mason mid = path->nodes[level]; 20537bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 2054e66f709bSChris Mason 2055a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 20565f39d397SChris Mason parent = path->nodes[level + 1]; 2057e66f709bSChris Mason pslot = path->slots[level + 1]; 2058a05a9bb1SLi Zefan } 2059e66f709bSChris Mason 20605f39d397SChris Mason if (!parent) 2061e66f709bSChris Mason return 1; 2062e66f709bSChris Mason 20634b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 2064fb770ae4SLiu Bo if (IS_ERR(left)) 2065fb770ae4SLiu Bo left = NULL; 2066e66f709bSChris Mason 2067e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 20685f39d397SChris Mason if (left) { 2069e66f709bSChris Mason u32 left_nr; 2070925baeddSChris Mason 2071bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 2072b4ce94deSChris Mason 20735f39d397SChris Mason left_nr = btrfs_header_nritems(left); 20740b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 207533ade1f8SChris Mason wret = 1; 207633ade1f8SChris Mason } else { 20775f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 20789631e4ccSJosef Bacik pslot - 1, &left, 2079bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 208054aa1f4dSChris Mason if (ret) 208154aa1f4dSChris Mason wret = 1; 208254aa1f4dSChris Mason else { 2083d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 208454aa1f4dSChris Mason } 208533ade1f8SChris Mason } 2086e66f709bSChris Mason if (wret < 0) 2087e66f709bSChris Mason ret = wret; 2088e66f709bSChris Mason if (wret == 0) { 20895f39d397SChris Mason struct btrfs_disk_key disk_key; 2090e66f709bSChris Mason orig_slot += left_nr; 20915f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 20920e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot, 20930e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 20940e82bcfeSDavid Sterba BUG_ON(ret < 0); 20955f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 20965f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 20975f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 20985f39d397SChris Mason path->nodes[level] = left; 2099e66f709bSChris Mason path->slots[level + 1] -= 1; 2100e66f709bSChris Mason path->slots[level] = orig_slot; 2101925baeddSChris Mason btrfs_tree_unlock(mid); 21025f39d397SChris Mason free_extent_buffer(mid); 2103e66f709bSChris Mason } else { 2104e66f709bSChris Mason orig_slot -= 21055f39d397SChris Mason btrfs_header_nritems(left); 2106e66f709bSChris Mason path->slots[level] = orig_slot; 2107925baeddSChris Mason btrfs_tree_unlock(left); 21085f39d397SChris Mason free_extent_buffer(left); 2109e66f709bSChris Mason } 2110e66f709bSChris Mason return 0; 2111e66f709bSChris Mason } 2112925baeddSChris Mason btrfs_tree_unlock(left); 21135f39d397SChris Mason free_extent_buffer(left); 2114e66f709bSChris Mason } 21154b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 2116fb770ae4SLiu Bo if (IS_ERR(right)) 2117fb770ae4SLiu Bo right = NULL; 2118e66f709bSChris Mason 2119e66f709bSChris Mason /* 2120e66f709bSChris Mason * then try to empty the right most buffer into the middle 2121e66f709bSChris Mason */ 21225f39d397SChris Mason if (right) { 212333ade1f8SChris Mason u32 right_nr; 2124b4ce94deSChris Mason 2125bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 2126b4ce94deSChris Mason 21275f39d397SChris Mason right_nr = btrfs_header_nritems(right); 21280b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 212933ade1f8SChris Mason wret = 1; 213033ade1f8SChris Mason } else { 21315f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 21325f39d397SChris Mason parent, pslot + 1, 2133bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 213454aa1f4dSChris Mason if (ret) 213554aa1f4dSChris Mason wret = 1; 213654aa1f4dSChris Mason else { 213755d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 213833ade1f8SChris Mason } 213954aa1f4dSChris Mason } 2140e66f709bSChris Mason if (wret < 0) 2141e66f709bSChris Mason ret = wret; 2142e66f709bSChris Mason if (wret == 0) { 21435f39d397SChris Mason struct btrfs_disk_key disk_key; 21445f39d397SChris Mason 21455f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 21460e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot + 1, 21470e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 21480e82bcfeSDavid Sterba BUG_ON(ret < 0); 21495f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 21505f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 21515f39d397SChris Mason 21525f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 21535f39d397SChris Mason path->nodes[level] = right; 2154e66f709bSChris Mason path->slots[level + 1] += 1; 2155e66f709bSChris Mason path->slots[level] = orig_slot - 21565f39d397SChris Mason btrfs_header_nritems(mid); 2157925baeddSChris Mason btrfs_tree_unlock(mid); 21585f39d397SChris Mason free_extent_buffer(mid); 2159e66f709bSChris Mason } else { 2160925baeddSChris Mason btrfs_tree_unlock(right); 21615f39d397SChris Mason free_extent_buffer(right); 2162e66f709bSChris Mason } 2163e66f709bSChris Mason return 0; 2164e66f709bSChris Mason } 2165925baeddSChris Mason btrfs_tree_unlock(right); 21665f39d397SChris Mason free_extent_buffer(right); 2167e66f709bSChris Mason } 2168e66f709bSChris Mason return 1; 2169e66f709bSChris Mason } 2170e66f709bSChris Mason 217174123bd7SChris Mason /* 2172d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 2173d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 21743c69faecSChris Mason */ 21752ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 2176e02119d5SChris Mason struct btrfs_path *path, 217701f46658SChris Mason int level, int slot, u64 objectid) 21783c69faecSChris Mason { 21795f39d397SChris Mason struct extent_buffer *node; 218001f46658SChris Mason struct btrfs_disk_key disk_key; 21813c69faecSChris Mason u32 nritems; 21823c69faecSChris Mason u64 search; 2183a7175319SChris Mason u64 target; 21846b80053dSChris Mason u64 nread = 0; 21855f39d397SChris Mason struct extent_buffer *eb; 21866b80053dSChris Mason u32 nr; 21876b80053dSChris Mason u32 blocksize; 21886b80053dSChris Mason u32 nscan = 0; 2189db94535dSChris Mason 2190a6b6e75eSChris Mason if (level != 1) 21913c69faecSChris Mason return; 21923c69faecSChris Mason 21936702ed49SChris Mason if (!path->nodes[level]) 21946702ed49SChris Mason return; 21956702ed49SChris Mason 21965f39d397SChris Mason node = path->nodes[level]; 2197925baeddSChris Mason 21983c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 21990b246afaSJeff Mahoney blocksize = fs_info->nodesize; 22000b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 22015f39d397SChris Mason if (eb) { 22025f39d397SChris Mason free_extent_buffer(eb); 22033c69faecSChris Mason return; 22043c69faecSChris Mason } 22053c69faecSChris Mason 2206a7175319SChris Mason target = search; 22076b80053dSChris Mason 22085f39d397SChris Mason nritems = btrfs_header_nritems(node); 22096b80053dSChris Mason nr = slot; 221025b8b936SJosef Bacik 22113c69faecSChris Mason while (1) { 2212e4058b54SDavid Sterba if (path->reada == READA_BACK) { 22136b80053dSChris Mason if (nr == 0) 22143c69faecSChris Mason break; 22156b80053dSChris Mason nr--; 2216e4058b54SDavid Sterba } else if (path->reada == READA_FORWARD) { 22176b80053dSChris Mason nr++; 22186b80053dSChris Mason if (nr >= nritems) 22196b80053dSChris Mason break; 22203c69faecSChris Mason } 2221e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 222201f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 222301f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 222401f46658SChris Mason break; 222501f46658SChris Mason } 22266b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 2227a7175319SChris Mason if ((search <= target && target - search <= 65536) || 2228a7175319SChris Mason (search > target && search - target <= 65536)) { 2229*bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 22306b80053dSChris Mason nread += blocksize; 22313c69faecSChris Mason } 22326b80053dSChris Mason nscan++; 2233a7175319SChris Mason if ((nread > 65536 || nscan > 32)) 22346b80053dSChris Mason break; 22353c69faecSChris Mason } 22363c69faecSChris Mason } 2237925baeddSChris Mason 2238*bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 2239b4ce94deSChris Mason { 2240*bfb484d9SJosef Bacik struct extent_buffer *parent; 2241b4ce94deSChris Mason int slot; 2242b4ce94deSChris Mason int nritems; 2243b4ce94deSChris Mason 22448c594ea8SChris Mason parent = path->nodes[level + 1]; 2245b4ce94deSChris Mason if (!parent) 22460b08851fSJosef Bacik return; 2247b4ce94deSChris Mason 2248b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 22498c594ea8SChris Mason slot = path->slots[level + 1]; 2250b4ce94deSChris Mason 2251*bfb484d9SJosef Bacik if (slot > 0) 2252*bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 2253*bfb484d9SJosef Bacik if (slot + 1 < nritems) 2254*bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 2255b4ce94deSChris Mason } 2256b4ce94deSChris Mason 2257b4ce94deSChris Mason 2258b4ce94deSChris Mason /* 2259d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 2260d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 2261d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 2262d397712bSChris Mason * tree. 2263d352ac68SChris Mason * 2264d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 2265d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 2266d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 2267d352ac68SChris Mason * 2268d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 2269d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 2270d352ac68SChris Mason */ 2271e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 2272f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 2273f7c79f30SChris Mason int *write_lock_level) 2274925baeddSChris Mason { 2275925baeddSChris Mason int i; 2276925baeddSChris Mason int skip_level = level; 2277051e1b9fSChris Mason int no_skips = 0; 2278925baeddSChris Mason struct extent_buffer *t; 2279925baeddSChris Mason 2280925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2281925baeddSChris Mason if (!path->nodes[i]) 2282925baeddSChris Mason break; 2283925baeddSChris Mason if (!path->locks[i]) 2284925baeddSChris Mason break; 2285051e1b9fSChris Mason if (!no_skips && path->slots[i] == 0) { 2286925baeddSChris Mason skip_level = i + 1; 2287925baeddSChris Mason continue; 2288925baeddSChris Mason } 2289051e1b9fSChris Mason if (!no_skips && path->keep_locks) { 2290925baeddSChris Mason u32 nritems; 2291925baeddSChris Mason t = path->nodes[i]; 2292925baeddSChris Mason nritems = btrfs_header_nritems(t); 2293051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 2294925baeddSChris Mason skip_level = i + 1; 2295925baeddSChris Mason continue; 2296925baeddSChris Mason } 2297925baeddSChris Mason } 2298051e1b9fSChris Mason if (skip_level < i && i >= lowest_unlock) 2299051e1b9fSChris Mason no_skips = 1; 2300051e1b9fSChris Mason 2301925baeddSChris Mason t = path->nodes[i]; 2302d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 2303bd681513SChris Mason btrfs_tree_unlock_rw(t, path->locks[i]); 2304925baeddSChris Mason path->locks[i] = 0; 2305f7c79f30SChris Mason if (write_lock_level && 2306f7c79f30SChris Mason i > min_write_lock_level && 2307f7c79f30SChris Mason i <= *write_lock_level) { 2308f7c79f30SChris Mason *write_lock_level = i - 1; 2309f7c79f30SChris Mason } 2310925baeddSChris Mason } 2311925baeddSChris Mason } 2312925baeddSChris Mason } 2313925baeddSChris Mason 23143c69faecSChris Mason /* 2315c8c42864SChris Mason * helper function for btrfs_search_slot. The goal is to find a block 2316c8c42864SChris Mason * in cache without setting the path to blocking. If we find the block 2317c8c42864SChris Mason * we return zero and the path is unchanged. 2318c8c42864SChris Mason * 2319c8c42864SChris Mason * If we can't find the block, we set the path blocking and do some 2320c8c42864SChris Mason * reada. -EAGAIN is returned and the search must be repeated. 2321c8c42864SChris Mason */ 2322c8c42864SChris Mason static int 2323d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 2324c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 2325cda79c54SDavid Sterba const struct btrfs_key *key) 2326c8c42864SChris Mason { 23270b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2328c8c42864SChris Mason u64 blocknr; 2329c8c42864SChris Mason u64 gen; 2330c8c42864SChris Mason struct extent_buffer *tmp; 2331581c1760SQu Wenruo struct btrfs_key first_key; 233276a05b35SChris Mason int ret; 2333581c1760SQu Wenruo int parent_level; 2334c8c42864SChris Mason 2335213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 2336213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 2337213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 2338213ff4b7SNikolay Borisov btrfs_node_key_to_cpu(*eb_ret, &first_key, slot); 2339c8c42864SChris Mason 23400b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 2341cb44921aSChris Mason if (tmp) { 2342b9fab919SChris Mason /* first we do an atomic uptodate check */ 2343b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 2344448de471SQu Wenruo /* 2345448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 2346448de471SQu Wenruo * being cached, read from scrub, or have multiple 2347448de471SQu Wenruo * parents (shared tree blocks). 2348448de471SQu Wenruo */ 2349e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 2350448de471SQu Wenruo parent_level - 1, &first_key, gen)) { 2351448de471SQu Wenruo free_extent_buffer(tmp); 2352448de471SQu Wenruo return -EUCLEAN; 2353448de471SQu Wenruo } 2354c8c42864SChris Mason *eb_ret = tmp; 2355c8c42864SChris Mason return 0; 2356c8c42864SChris Mason } 2357bdf7c00eSJosef Bacik 2358b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 2359581c1760SQu Wenruo ret = btrfs_read_buffer(tmp, gen, parent_level - 1, &first_key); 2360bdf7c00eSJosef Bacik if (!ret) { 2361cb44921aSChris Mason *eb_ret = tmp; 2362cb44921aSChris Mason return 0; 2363cb44921aSChris Mason } 2364cb44921aSChris Mason free_extent_buffer(tmp); 2365b3b4aa74SDavid Sterba btrfs_release_path(p); 2366cb44921aSChris Mason return -EIO; 2367cb44921aSChris Mason } 2368c8c42864SChris Mason 2369c8c42864SChris Mason /* 2370c8c42864SChris Mason * reduce lock contention at high levels 2371c8c42864SChris Mason * of the btree by dropping locks before 237276a05b35SChris Mason * we read. Don't release the lock on the current 237376a05b35SChris Mason * level because we need to walk this node to figure 237476a05b35SChris Mason * out which blocks to read. 2375c8c42864SChris Mason */ 23768c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 23778c594ea8SChris Mason 2378e4058b54SDavid Sterba if (p->reada != READA_NONE) 23792ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 2380c8c42864SChris Mason 238176a05b35SChris Mason ret = -EAGAIN; 238202a3307aSLiu Bo tmp = read_tree_block(fs_info, blocknr, gen, parent_level - 1, 2383581c1760SQu Wenruo &first_key); 238464c043deSLiu Bo if (!IS_ERR(tmp)) { 238576a05b35SChris Mason /* 238676a05b35SChris Mason * If the read above didn't mark this buffer up to date, 238776a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 238876a05b35SChris Mason * and give up so that our caller doesn't loop forever 238976a05b35SChris Mason * on our EAGAINs. 239076a05b35SChris Mason */ 2391e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 239276a05b35SChris Mason ret = -EIO; 2393c8c42864SChris Mason free_extent_buffer(tmp); 2394c871b0f2SLiu Bo } else { 2395c871b0f2SLiu Bo ret = PTR_ERR(tmp); 239676a05b35SChris Mason } 239702a3307aSLiu Bo 239802a3307aSLiu Bo btrfs_release_path(p); 239976a05b35SChris Mason return ret; 2400c8c42864SChris Mason } 2401c8c42864SChris Mason 2402c8c42864SChris Mason /* 2403c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 2404c8c42864SChris Mason * for node-level blocks and does any balancing required based on 2405c8c42864SChris Mason * the ins_len. 2406c8c42864SChris Mason * 2407c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 2408c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 2409c8c42864SChris Mason * start over 2410c8c42864SChris Mason */ 2411c8c42864SChris Mason static int 2412c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 2413c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2414bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 2415bd681513SChris Mason int *write_lock_level) 2416c8c42864SChris Mason { 24170b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2418c8c42864SChris Mason int ret; 24190b246afaSJeff Mahoney 2420c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 24210b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 2422c8c42864SChris Mason int sret; 2423c8c42864SChris Mason 2424bd681513SChris Mason if (*write_lock_level < level + 1) { 2425bd681513SChris Mason *write_lock_level = level + 1; 2426bd681513SChris Mason btrfs_release_path(p); 2427bd681513SChris Mason goto again; 2428bd681513SChris Mason } 2429bd681513SChris Mason 2430*bfb484d9SJosef Bacik reada_for_balance(p, level); 2431c8c42864SChris Mason sret = split_node(trans, root, p, level); 2432c8c42864SChris Mason 2433c8c42864SChris Mason BUG_ON(sret > 0); 2434c8c42864SChris Mason if (sret) { 2435c8c42864SChris Mason ret = sret; 2436c8c42864SChris Mason goto done; 2437c8c42864SChris Mason } 2438c8c42864SChris Mason b = p->nodes[level]; 2439c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 24400b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 2441c8c42864SChris Mason int sret; 2442c8c42864SChris Mason 2443bd681513SChris Mason if (*write_lock_level < level + 1) { 2444bd681513SChris Mason *write_lock_level = level + 1; 2445bd681513SChris Mason btrfs_release_path(p); 2446bd681513SChris Mason goto again; 2447bd681513SChris Mason } 2448bd681513SChris Mason 2449*bfb484d9SJosef Bacik reada_for_balance(p, level); 2450c8c42864SChris Mason sret = balance_level(trans, root, p, level); 2451c8c42864SChris Mason 2452c8c42864SChris Mason if (sret) { 2453c8c42864SChris Mason ret = sret; 2454c8c42864SChris Mason goto done; 2455c8c42864SChris Mason } 2456c8c42864SChris Mason b = p->nodes[level]; 2457c8c42864SChris Mason if (!b) { 2458b3b4aa74SDavid Sterba btrfs_release_path(p); 2459c8c42864SChris Mason goto again; 2460c8c42864SChris Mason } 2461c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 2462c8c42864SChris Mason } 2463c8c42864SChris Mason return 0; 2464c8c42864SChris Mason 2465c8c42864SChris Mason again: 2466c8c42864SChris Mason ret = -EAGAIN; 2467c8c42864SChris Mason done: 2468c8c42864SChris Mason return ret; 2469c8c42864SChris Mason } 2470c8c42864SChris Mason 2471381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 2472e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 2473e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 2474e33d5c3dSKelley Nielsen { 2475e33d5c3dSKelley Nielsen int ret; 2476e33d5c3dSKelley Nielsen struct btrfs_key key; 2477e33d5c3dSKelley Nielsen struct extent_buffer *eb; 2478381cf658SDavid Sterba 2479381cf658SDavid Sterba ASSERT(path); 24801d4c08e0SDavid Sterba ASSERT(found_key); 2481e33d5c3dSKelley Nielsen 2482e33d5c3dSKelley Nielsen key.type = key_type; 2483e33d5c3dSKelley Nielsen key.objectid = iobjectid; 2484e33d5c3dSKelley Nielsen key.offset = ioff; 2485e33d5c3dSKelley Nielsen 2486e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 24871d4c08e0SDavid Sterba if (ret < 0) 2488e33d5c3dSKelley Nielsen return ret; 2489e33d5c3dSKelley Nielsen 2490e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2491e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 2492e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 2493e33d5c3dSKelley Nielsen if (ret) 2494e33d5c3dSKelley Nielsen return ret; 2495e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2496e33d5c3dSKelley Nielsen } 2497e33d5c3dSKelley Nielsen 2498e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 2499e33d5c3dSKelley Nielsen if (found_key->type != key.type || 2500e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 2501e33d5c3dSKelley Nielsen return 1; 2502e33d5c3dSKelley Nielsen 2503e33d5c3dSKelley Nielsen return 0; 2504e33d5c3dSKelley Nielsen } 2505e33d5c3dSKelley Nielsen 25061fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 25071fc28d8eSLiu Bo struct btrfs_path *p, 25081fc28d8eSLiu Bo int write_lock_level) 25091fc28d8eSLiu Bo { 25101fc28d8eSLiu Bo struct btrfs_fs_info *fs_info = root->fs_info; 25111fc28d8eSLiu Bo struct extent_buffer *b; 25121fc28d8eSLiu Bo int root_lock; 25131fc28d8eSLiu Bo int level = 0; 25141fc28d8eSLiu Bo 25151fc28d8eSLiu Bo /* We try very hard to do read locks on the root */ 25161fc28d8eSLiu Bo root_lock = BTRFS_READ_LOCK; 25171fc28d8eSLiu Bo 25181fc28d8eSLiu Bo if (p->search_commit_root) { 2519be6821f8SFilipe Manana /* 2520be6821f8SFilipe Manana * The commit roots are read only so we always do read locks, 2521be6821f8SFilipe Manana * and we always must hold the commit_root_sem when doing 2522be6821f8SFilipe Manana * searches on them, the only exception is send where we don't 2523be6821f8SFilipe Manana * want to block transaction commits for a long time, so 2524be6821f8SFilipe Manana * we need to clone the commit root in order to avoid races 2525be6821f8SFilipe Manana * with transaction commits that create a snapshot of one of 2526be6821f8SFilipe Manana * the roots used by a send operation. 2527be6821f8SFilipe Manana */ 2528be6821f8SFilipe Manana if (p->need_commit_sem) { 25291fc28d8eSLiu Bo down_read(&fs_info->commit_root_sem); 2530be6821f8SFilipe Manana b = btrfs_clone_extent_buffer(root->commit_root); 2531be6821f8SFilipe Manana up_read(&fs_info->commit_root_sem); 2532be6821f8SFilipe Manana if (!b) 2533be6821f8SFilipe Manana return ERR_PTR(-ENOMEM); 2534be6821f8SFilipe Manana 2535be6821f8SFilipe Manana } else { 25361fc28d8eSLiu Bo b = root->commit_root; 253767439dadSDavid Sterba atomic_inc(&b->refs); 2538be6821f8SFilipe Manana } 25391fc28d8eSLiu Bo level = btrfs_header_level(b); 2540f9ddfd05SLiu Bo /* 2541f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 2542f9ddfd05SLiu Bo * p->search_commit_root = 1. 2543f9ddfd05SLiu Bo */ 2544f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 25451fc28d8eSLiu Bo 25461fc28d8eSLiu Bo goto out; 25471fc28d8eSLiu Bo } 25481fc28d8eSLiu Bo 25491fc28d8eSLiu Bo if (p->skip_locking) { 25501fc28d8eSLiu Bo b = btrfs_root_node(root); 25511fc28d8eSLiu Bo level = btrfs_header_level(b); 25521fc28d8eSLiu Bo goto out; 25531fc28d8eSLiu Bo } 25541fc28d8eSLiu Bo 25551fc28d8eSLiu Bo /* 2556662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 2557662c653bSLiu Bo * lock. 2558662c653bSLiu Bo */ 2559662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 2560662c653bSLiu Bo /* 2561662c653bSLiu Bo * We don't know the level of the root node until we actually 2562662c653bSLiu Bo * have it read locked 25631fc28d8eSLiu Bo */ 256451899412SJosef Bacik b = __btrfs_read_lock_root_node(root, p->recurse); 25651fc28d8eSLiu Bo level = btrfs_header_level(b); 25661fc28d8eSLiu Bo if (level > write_lock_level) 25671fc28d8eSLiu Bo goto out; 25681fc28d8eSLiu Bo 2569662c653bSLiu Bo /* Whoops, must trade for write lock */ 25701fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 25711fc28d8eSLiu Bo free_extent_buffer(b); 2572662c653bSLiu Bo } 2573662c653bSLiu Bo 25741fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 25751fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 25761fc28d8eSLiu Bo 25771fc28d8eSLiu Bo /* The level might have changed, check again */ 25781fc28d8eSLiu Bo level = btrfs_header_level(b); 25791fc28d8eSLiu Bo 25801fc28d8eSLiu Bo out: 25811fc28d8eSLiu Bo p->nodes[level] = b; 25821fc28d8eSLiu Bo if (!p->skip_locking) 25831fc28d8eSLiu Bo p->locks[level] = root_lock; 25841fc28d8eSLiu Bo /* 25851fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 25861fc28d8eSLiu Bo */ 25871fc28d8eSLiu Bo return b; 25881fc28d8eSLiu Bo } 25891fc28d8eSLiu Bo 25901fc28d8eSLiu Bo 2591c8c42864SChris Mason /* 25924271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 25934271eceaSNikolay Borisov * modifications to preserve tree invariants. 259474123bd7SChris Mason * 25954271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 25964271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 25974271eceaSNikolay Borisov * @root: The root node of the tree 25984271eceaSNikolay Borisov * @key: The key we are looking for 25994271eceaSNikolay Borisov * @ins_len: Indicates purpose of search, for inserts it is 1, for 26004271eceaSNikolay Borisov * deletions it's -1. 0 for plain searches 26014271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 26024271eceaSNikolay Borisov * when modifying the tree. 260397571fd0SChris Mason * 26044271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 26054271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 26064271eceaSNikolay Borisov * 26074271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 26084271eceaSNikolay Borisov * of the path (level 0) 26094271eceaSNikolay Borisov * 26104271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 26114271eceaSNikolay Borisov * points to the slot where it should be inserted 26124271eceaSNikolay Borisov * 26134271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 26144271eceaSNikolay Borisov * is returned 261574123bd7SChris Mason */ 2616310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 2617310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 2618310712b2SOmar Sandoval int ins_len, int cow) 2619be0e5c09SChris Mason { 26205f39d397SChris Mason struct extent_buffer *b; 2621be0e5c09SChris Mason int slot; 2622be0e5c09SChris Mason int ret; 262333c66f43SYan Zheng int err; 2624be0e5c09SChris Mason int level; 2625925baeddSChris Mason int lowest_unlock = 1; 2626bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2627bd681513SChris Mason int write_lock_level = 0; 26289f3a7427SChris Mason u8 lowest_level = 0; 2629f7c79f30SChris Mason int min_write_lock_level; 2630d7396f07SFilipe David Borba Manana int prev_cmp; 26319f3a7427SChris Mason 26326702ed49SChris Mason lowest_level = p->lowest_level; 2633323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 263422b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2635eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 263625179201SJosef Bacik 2637bd681513SChris Mason if (ins_len < 0) { 2638925baeddSChris Mason lowest_unlock = 2; 263965b51a00SChris Mason 2640bd681513SChris Mason /* when we are removing items, we might have to go up to level 2641bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2642bd681513SChris Mason * for those levels as well 2643bd681513SChris Mason */ 2644bd681513SChris Mason write_lock_level = 2; 2645bd681513SChris Mason } else if (ins_len > 0) { 2646bd681513SChris Mason /* 2647bd681513SChris Mason * for inserting items, make sure we have a write lock on 2648bd681513SChris Mason * level 1 so we can update keys 2649bd681513SChris Mason */ 2650bd681513SChris Mason write_lock_level = 1; 2651bd681513SChris Mason } 2652bd681513SChris Mason 2653bd681513SChris Mason if (!cow) 2654bd681513SChris Mason write_lock_level = -1; 2655bd681513SChris Mason 265609a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2657bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2658bd681513SChris Mason 2659f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2660f7c79f30SChris Mason 2661bb803951SChris Mason again: 2662d7396f07SFilipe David Borba Manana prev_cmp = -1; 26631fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 2664be6821f8SFilipe Manana if (IS_ERR(b)) { 2665be6821f8SFilipe Manana ret = PTR_ERR(b); 2666be6821f8SFilipe Manana goto done; 2667be6821f8SFilipe Manana } 2668925baeddSChris Mason 2669eb60ceacSChris Mason while (b) { 2670f624d976SQu Wenruo int dec = 0; 2671f624d976SQu Wenruo 26725f39d397SChris Mason level = btrfs_header_level(b); 267365b51a00SChris Mason 267402217ed2SChris Mason if (cow) { 26759ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 26769ea2c7c9SNikolay Borisov 2677c8c42864SChris Mason /* 2678c8c42864SChris Mason * if we don't really need to cow this block 2679c8c42864SChris Mason * then we don't want to set the path blocking, 2680c8c42864SChris Mason * so we test it here 2681c8c42864SChris Mason */ 268264c12921SJeff Mahoney if (!should_cow_block(trans, root, b)) { 268364c12921SJeff Mahoney trans->dirty = true; 268465b51a00SChris Mason goto cow_done; 268564c12921SJeff Mahoney } 26865d4f98a2SYan Zheng 2687bd681513SChris Mason /* 2688bd681513SChris Mason * must have write locks on this node and the 2689bd681513SChris Mason * parent 2690bd681513SChris Mason */ 26915124e00eSJosef Bacik if (level > write_lock_level || 26925124e00eSJosef Bacik (level + 1 > write_lock_level && 26935124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 26945124e00eSJosef Bacik p->nodes[level + 1])) { 2695bd681513SChris Mason write_lock_level = level + 1; 2696bd681513SChris Mason btrfs_release_path(p); 2697bd681513SChris Mason goto again; 2698bd681513SChris Mason } 2699bd681513SChris Mason 27009ea2c7c9SNikolay Borisov if (last_level) 27019ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 27029631e4ccSJosef Bacik &b, 27039631e4ccSJosef Bacik BTRFS_NESTING_COW); 27049ea2c7c9SNikolay Borisov else 270533c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2706e20d96d6SChris Mason p->nodes[level + 1], 27079631e4ccSJosef Bacik p->slots[level + 1], &b, 27089631e4ccSJosef Bacik BTRFS_NESTING_COW); 270933c66f43SYan Zheng if (err) { 271033c66f43SYan Zheng ret = err; 271165b51a00SChris Mason goto done; 271254aa1f4dSChris Mason } 271302217ed2SChris Mason } 271465b51a00SChris Mason cow_done: 2715eb60ceacSChris Mason p->nodes[level] = b; 271652398340SLiu Bo /* 271752398340SLiu Bo * Leave path with blocking locks to avoid massive 271852398340SLiu Bo * lock context switch, this is made on purpose. 271952398340SLiu Bo */ 2720b4ce94deSChris Mason 2721b4ce94deSChris Mason /* 2722b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2723b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2724b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2725b4ce94deSChris Mason * go through the expensive btree search on b. 2726b4ce94deSChris Mason * 2727eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2728eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2729eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2730eb653de1SFilipe David Borba Manana * we're operating on. 2731b4ce94deSChris Mason */ 2732eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2733eb653de1SFilipe David Borba Manana int u = level + 1; 2734eb653de1SFilipe David Borba Manana 2735eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2736eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2737eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2738eb653de1SFilipe David Borba Manana } 2739eb653de1SFilipe David Borba Manana } 2740b4ce94deSChris Mason 2741995e9a16SNikolay Borisov /* 2742995e9a16SNikolay Borisov * If btrfs_bin_search returns an exact match (prev_cmp == 0) 2743995e9a16SNikolay Borisov * we can safely assume the target key will always be in slot 0 2744995e9a16SNikolay Borisov * on lower levels due to the invariants BTRFS' btree provides, 2745995e9a16SNikolay Borisov * namely that a btrfs_key_ptr entry always points to the 2746995e9a16SNikolay Borisov * lowest key in the child node, thus we can skip searching 2747995e9a16SNikolay Borisov * lower levels 2748995e9a16SNikolay Borisov */ 2749995e9a16SNikolay Borisov if (prev_cmp == 0) { 2750995e9a16SNikolay Borisov slot = 0; 2751995e9a16SNikolay Borisov ret = 0; 2752995e9a16SNikolay Borisov } else { 2753995e9a16SNikolay Borisov ret = btrfs_bin_search(b, key, &slot); 2754995e9a16SNikolay Borisov prev_cmp = ret; 2755415b35a5SLiu Bo if (ret < 0) 2756415b35a5SLiu Bo goto done; 2757995e9a16SNikolay Borisov } 2758b4ce94deSChris Mason 2759f624d976SQu Wenruo if (level == 0) { 2760be0e5c09SChris Mason p->slots[level] = slot; 276187b29b20SYan Zheng if (ins_len > 0 && 2762e902baacSDavid Sterba btrfs_leaf_free_space(b) < ins_len) { 2763bd681513SChris Mason if (write_lock_level < 1) { 2764bd681513SChris Mason write_lock_level = 1; 2765bd681513SChris Mason btrfs_release_path(p); 2766bd681513SChris Mason goto again; 2767bd681513SChris Mason } 2768bd681513SChris Mason 276933c66f43SYan Zheng err = split_leaf(trans, root, key, 2770cc0c5538SChris Mason p, ins_len, ret == 0); 2771b4ce94deSChris Mason 277233c66f43SYan Zheng BUG_ON(err > 0); 277333c66f43SYan Zheng if (err) { 277433c66f43SYan Zheng ret = err; 277565b51a00SChris Mason goto done; 277665b51a00SChris Mason } 27775c680ed6SChris Mason } 2778459931ecSChris Mason if (!p->search_for_split) 2779f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 27804b6f8e96SLiu Bo min_write_lock_level, NULL); 278165b51a00SChris Mason goto done; 278265b51a00SChris Mason } 2783f624d976SQu Wenruo if (ret && slot > 0) { 2784f624d976SQu Wenruo dec = 1; 2785f624d976SQu Wenruo slot--; 2786f624d976SQu Wenruo } 2787f624d976SQu Wenruo p->slots[level] = slot; 2788f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2789f624d976SQu Wenruo &write_lock_level); 2790f624d976SQu Wenruo if (err == -EAGAIN) 2791f624d976SQu Wenruo goto again; 2792f624d976SQu Wenruo if (err) { 2793f624d976SQu Wenruo ret = err; 2794f624d976SQu Wenruo goto done; 2795f624d976SQu Wenruo } 2796f624d976SQu Wenruo b = p->nodes[level]; 2797f624d976SQu Wenruo slot = p->slots[level]; 2798f624d976SQu Wenruo 2799f624d976SQu Wenruo /* 2800f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2801f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2802f624d976SQu Wenruo * the parent 2803f624d976SQu Wenruo */ 2804f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2805f624d976SQu Wenruo write_lock_level = level + 1; 2806f624d976SQu Wenruo btrfs_release_path(p); 2807f624d976SQu Wenruo goto again; 2808f624d976SQu Wenruo } 2809f624d976SQu Wenruo 2810f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2811f624d976SQu Wenruo &write_lock_level); 2812f624d976SQu Wenruo 2813f624d976SQu Wenruo if (level == lowest_level) { 2814f624d976SQu Wenruo if (dec) 2815f624d976SQu Wenruo p->slots[level]++; 2816f624d976SQu Wenruo goto done; 2817f624d976SQu Wenruo } 2818f624d976SQu Wenruo 2819f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2820f624d976SQu Wenruo if (err == -EAGAIN) 2821f624d976SQu Wenruo goto again; 2822f624d976SQu Wenruo if (err) { 2823f624d976SQu Wenruo ret = err; 2824f624d976SQu Wenruo goto done; 2825f624d976SQu Wenruo } 2826f624d976SQu Wenruo 2827f624d976SQu Wenruo if (!p->skip_locking) { 2828f624d976SQu Wenruo level = btrfs_header_level(b); 2829f624d976SQu Wenruo if (level <= write_lock_level) { 2830f624d976SQu Wenruo btrfs_tree_lock(b); 2831f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2832f624d976SQu Wenruo } else { 2833fd7ba1c1SJosef Bacik __btrfs_tree_read_lock(b, BTRFS_NESTING_NORMAL, 2834fd7ba1c1SJosef Bacik p->recurse); 2835f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2836f624d976SQu Wenruo } 2837f624d976SQu Wenruo p->nodes[level] = b; 2838f624d976SQu Wenruo } 283965b51a00SChris Mason } 284065b51a00SChris Mason ret = 1; 284165b51a00SChris Mason done: 28425f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2843b3b4aa74SDavid Sterba btrfs_release_path(p); 2844be0e5c09SChris Mason return ret; 2845be0e5c09SChris Mason } 2846be0e5c09SChris Mason 284774123bd7SChris Mason /* 28485d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 28495d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 28505d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 28515d9e75c4SJan Schmidt * denoted by the time_seq parameter. 28525d9e75c4SJan Schmidt * 28535d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 28545d9e75c4SJan Schmidt * 28555d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 28565d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 28575d9e75c4SJan Schmidt */ 2858310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 28595d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 28605d9e75c4SJan Schmidt { 28610b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 28625d9e75c4SJan Schmidt struct extent_buffer *b; 28635d9e75c4SJan Schmidt int slot; 28645d9e75c4SJan Schmidt int ret; 28655d9e75c4SJan Schmidt int err; 28665d9e75c4SJan Schmidt int level; 28675d9e75c4SJan Schmidt int lowest_unlock = 1; 28685d9e75c4SJan Schmidt u8 lowest_level = 0; 28695d9e75c4SJan Schmidt 28705d9e75c4SJan Schmidt lowest_level = p->lowest_level; 28715d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 28725d9e75c4SJan Schmidt 28735d9e75c4SJan Schmidt if (p->search_commit_root) { 28745d9e75c4SJan Schmidt BUG_ON(time_seq); 28755d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 28765d9e75c4SJan Schmidt } 28775d9e75c4SJan Schmidt 28785d9e75c4SJan Schmidt again: 28795d9e75c4SJan Schmidt b = get_old_root(root, time_seq); 2880315bed43SNikolay Borisov if (!b) { 2881315bed43SNikolay Borisov ret = -EIO; 2882315bed43SNikolay Borisov goto done; 2883315bed43SNikolay Borisov } 28845d9e75c4SJan Schmidt level = btrfs_header_level(b); 28855d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 28865d9e75c4SJan Schmidt 28875d9e75c4SJan Schmidt while (b) { 2888abe9339dSQu Wenruo int dec = 0; 2889abe9339dSQu Wenruo 28905d9e75c4SJan Schmidt level = btrfs_header_level(b); 28915d9e75c4SJan Schmidt p->nodes[level] = b; 28925d9e75c4SJan Schmidt 28935d9e75c4SJan Schmidt /* 28945d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 28955d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 28965d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 28975d9e75c4SJan Schmidt * go through the expensive btree search on b. 28985d9e75c4SJan Schmidt */ 28995d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 29005d9e75c4SJan Schmidt 2901995e9a16SNikolay Borisov ret = btrfs_bin_search(b, key, &slot); 2902cbca7d59SFilipe Manana if (ret < 0) 2903cbca7d59SFilipe Manana goto done; 29045d9e75c4SJan Schmidt 2905abe9339dSQu Wenruo if (level == 0) { 2906abe9339dSQu Wenruo p->slots[level] = slot; 2907abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2908abe9339dSQu Wenruo goto done; 2909abe9339dSQu Wenruo } 2910abe9339dSQu Wenruo 29115d9e75c4SJan Schmidt if (ret && slot > 0) { 29125d9e75c4SJan Schmidt dec = 1; 2913abe9339dSQu Wenruo slot--; 29145d9e75c4SJan Schmidt } 29155d9e75c4SJan Schmidt p->slots[level] = slot; 29165d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 29175d9e75c4SJan Schmidt 29185d9e75c4SJan Schmidt if (level == lowest_level) { 29195d9e75c4SJan Schmidt if (dec) 29205d9e75c4SJan Schmidt p->slots[level]++; 29215d9e75c4SJan Schmidt goto done; 29225d9e75c4SJan Schmidt } 29235d9e75c4SJan Schmidt 2924abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 29255d9e75c4SJan Schmidt if (err == -EAGAIN) 29265d9e75c4SJan Schmidt goto again; 29275d9e75c4SJan Schmidt if (err) { 29285d9e75c4SJan Schmidt ret = err; 29295d9e75c4SJan Schmidt goto done; 29305d9e75c4SJan Schmidt } 29315d9e75c4SJan Schmidt 29325d9e75c4SJan Schmidt level = btrfs_header_level(b); 29335d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 29340b246afaSJeff Mahoney b = tree_mod_log_rewind(fs_info, p, b, time_seq); 2935db7f3436SJosef Bacik if (!b) { 2936db7f3436SJosef Bacik ret = -ENOMEM; 2937db7f3436SJosef Bacik goto done; 2938db7f3436SJosef Bacik } 29395d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 29405d9e75c4SJan Schmidt p->nodes[level] = b; 29415d9e75c4SJan Schmidt } 29425d9e75c4SJan Schmidt ret = 1; 29435d9e75c4SJan Schmidt done: 29445d9e75c4SJan Schmidt if (ret < 0) 29455d9e75c4SJan Schmidt btrfs_release_path(p); 29465d9e75c4SJan Schmidt 29475d9e75c4SJan Schmidt return ret; 29485d9e75c4SJan Schmidt } 29495d9e75c4SJan Schmidt 29505d9e75c4SJan Schmidt /* 29512f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 29522f38b3e1SArne Jansen * instead the next or previous item should be returned. 29532f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 29542f38b3e1SArne Jansen * otherwise. 29552f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 29562f38b3e1SArne Jansen * return the next lower instead. 29572f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 29582f38b3e1SArne Jansen * return the next higher instead. 29592f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 29602f38b3e1SArne Jansen * < 0 on error 29612f38b3e1SArne Jansen */ 29622f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2963310712b2SOmar Sandoval const struct btrfs_key *key, 2964310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2965310712b2SOmar Sandoval int return_any) 29662f38b3e1SArne Jansen { 29672f38b3e1SArne Jansen int ret; 29682f38b3e1SArne Jansen struct extent_buffer *leaf; 29692f38b3e1SArne Jansen 29702f38b3e1SArne Jansen again: 29712f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 29722f38b3e1SArne Jansen if (ret <= 0) 29732f38b3e1SArne Jansen return ret; 29742f38b3e1SArne Jansen /* 29752f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 29762f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 29772f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 29782f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 29792f38b3e1SArne Jansen * item. 29802f38b3e1SArne Jansen */ 29812f38b3e1SArne Jansen leaf = p->nodes[0]; 29822f38b3e1SArne Jansen 29832f38b3e1SArne Jansen if (find_higher) { 29842f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 29852f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 29862f38b3e1SArne Jansen if (ret <= 0) 29872f38b3e1SArne Jansen return ret; 29882f38b3e1SArne Jansen if (!return_any) 29892f38b3e1SArne Jansen return 1; 29902f38b3e1SArne Jansen /* 29912f38b3e1SArne Jansen * no higher item found, return the next 29922f38b3e1SArne Jansen * lower instead 29932f38b3e1SArne Jansen */ 29942f38b3e1SArne Jansen return_any = 0; 29952f38b3e1SArne Jansen find_higher = 0; 29962f38b3e1SArne Jansen btrfs_release_path(p); 29972f38b3e1SArne Jansen goto again; 29982f38b3e1SArne Jansen } 29992f38b3e1SArne Jansen } else { 30002f38b3e1SArne Jansen if (p->slots[0] == 0) { 30012f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 3002e6793769SArne Jansen if (ret < 0) 30032f38b3e1SArne Jansen return ret; 3004e6793769SArne Jansen if (!ret) { 300523c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 300623c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 300723c6bf6aSFilipe David Borba Manana p->slots[0]--; 3008e6793769SArne Jansen return 0; 3009e6793769SArne Jansen } 30102f38b3e1SArne Jansen if (!return_any) 30112f38b3e1SArne Jansen return 1; 30122f38b3e1SArne Jansen /* 30132f38b3e1SArne Jansen * no lower item found, return the next 30142f38b3e1SArne Jansen * higher instead 30152f38b3e1SArne Jansen */ 30162f38b3e1SArne Jansen return_any = 0; 30172f38b3e1SArne Jansen find_higher = 1; 30182f38b3e1SArne Jansen btrfs_release_path(p); 30192f38b3e1SArne Jansen goto again; 3020e6793769SArne Jansen } else { 30212f38b3e1SArne Jansen --p->slots[0]; 30222f38b3e1SArne Jansen } 30232f38b3e1SArne Jansen } 30242f38b3e1SArne Jansen return 0; 30252f38b3e1SArne Jansen } 30262f38b3e1SArne Jansen 30272f38b3e1SArne Jansen /* 302874123bd7SChris Mason * adjust the pointers going up the tree, starting at level 302974123bd7SChris Mason * making sure the right key of each node is points to 'key'. 303074123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 303174123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 303274123bd7SChris Mason * higher levels 3033aa5d6bedSChris Mason * 303474123bd7SChris Mason */ 3035b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 30365f39d397SChris Mason struct btrfs_disk_key *key, int level) 3037be0e5c09SChris Mason { 3038be0e5c09SChris Mason int i; 30395f39d397SChris Mason struct extent_buffer *t; 30400e82bcfeSDavid Sterba int ret; 30415f39d397SChris Mason 3042234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 3043be0e5c09SChris Mason int tslot = path->slots[i]; 30440e82bcfeSDavid Sterba 3045eb60ceacSChris Mason if (!path->nodes[i]) 3046be0e5c09SChris Mason break; 30475f39d397SChris Mason t = path->nodes[i]; 30480e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(t, tslot, MOD_LOG_KEY_REPLACE, 30490e82bcfeSDavid Sterba GFP_ATOMIC); 30500e82bcfeSDavid Sterba BUG_ON(ret < 0); 30515f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 3052d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 3053be0e5c09SChris Mason if (tslot != 0) 3054be0e5c09SChris Mason break; 3055be0e5c09SChris Mason } 3056be0e5c09SChris Mason } 3057be0e5c09SChris Mason 305874123bd7SChris Mason /* 305931840ae1SZheng Yan * update item key. 306031840ae1SZheng Yan * 306131840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 306231840ae1SZheng Yan * that the new key won't break the order 306331840ae1SZheng Yan */ 3064b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 3065b7a0365eSDaniel Dressler struct btrfs_path *path, 3066310712b2SOmar Sandoval const struct btrfs_key *new_key) 306731840ae1SZheng Yan { 306831840ae1SZheng Yan struct btrfs_disk_key disk_key; 306931840ae1SZheng Yan struct extent_buffer *eb; 307031840ae1SZheng Yan int slot; 307131840ae1SZheng Yan 307231840ae1SZheng Yan eb = path->nodes[0]; 307331840ae1SZheng Yan slot = path->slots[0]; 307431840ae1SZheng Yan if (slot > 0) { 307531840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 30767c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 30777c15d410SQu Wenruo btrfs_crit(fs_info, 30787c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 30797c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 30807c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 30817c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 30827c15d410SQu Wenruo new_key->objectid, new_key->type, 30837c15d410SQu Wenruo new_key->offset); 30847c15d410SQu Wenruo btrfs_print_leaf(eb); 30857c15d410SQu Wenruo BUG(); 30867c15d410SQu Wenruo } 308731840ae1SZheng Yan } 308831840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 308931840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 30907c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 30917c15d410SQu Wenruo btrfs_crit(fs_info, 30927c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 30937c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 30947c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 30957c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 30967c15d410SQu Wenruo new_key->objectid, new_key->type, 30977c15d410SQu Wenruo new_key->offset); 30987c15d410SQu Wenruo btrfs_print_leaf(eb); 30997c15d410SQu Wenruo BUG(); 31007c15d410SQu Wenruo } 310131840ae1SZheng Yan } 310231840ae1SZheng Yan 310331840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 310431840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 310531840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 310631840ae1SZheng Yan if (slot == 0) 3107b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 310831840ae1SZheng Yan } 310931840ae1SZheng Yan 311031840ae1SZheng Yan /* 3111d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 3112d16c702fSQu Wenruo * 3113d16c702fSQu Wenruo * Return true if something is wrong. 3114d16c702fSQu Wenruo * Return false if everything is fine. 3115d16c702fSQu Wenruo * 3116d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 3117d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 3118d16c702fSQu Wenruo * 3119d16c702fSQu Wenruo * Leaf @left | Leaf @right 3120d16c702fSQu Wenruo * -------------------------------------------------------------- 3121d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 3122d16c702fSQu Wenruo * 3123d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 3124d16c702fSQu Wenruo * key in leaf @right is 7. 3125d16c702fSQu Wenruo * This can only be checked at tree block merge time. 3126d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 3127d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 3128d16c702fSQu Wenruo * key of @right. 3129d16c702fSQu Wenruo */ 3130d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 3131d16c702fSQu Wenruo struct extent_buffer *right) 3132d16c702fSQu Wenruo { 3133d16c702fSQu Wenruo struct btrfs_key left_last; 3134d16c702fSQu Wenruo struct btrfs_key right_first; 3135d16c702fSQu Wenruo int level = btrfs_header_level(left); 3136d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 3137d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 3138d16c702fSQu Wenruo 3139d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 3140d16c702fSQu Wenruo if (!nr_left || !nr_right) 3141d16c702fSQu Wenruo return false; 3142d16c702fSQu Wenruo 3143d16c702fSQu Wenruo if (level) { 3144d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 3145d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 3146d16c702fSQu Wenruo } else { 3147d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 3148d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 3149d16c702fSQu Wenruo } 3150d16c702fSQu Wenruo 3151d16c702fSQu Wenruo if (btrfs_comp_cpu_keys(&left_last, &right_first) >= 0) { 3152d16c702fSQu Wenruo btrfs_crit(left->fs_info, 3153d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 3154d16c702fSQu Wenruo left_last.objectid, left_last.type, 3155d16c702fSQu Wenruo left_last.offset, right_first.objectid, 3156d16c702fSQu Wenruo right_first.type, right_first.offset); 3157d16c702fSQu Wenruo return true; 3158d16c702fSQu Wenruo } 3159d16c702fSQu Wenruo return false; 3160d16c702fSQu Wenruo } 3161d16c702fSQu Wenruo 3162d16c702fSQu Wenruo /* 316374123bd7SChris Mason * try to push data from one node into the next node left in the 316479f95c82SChris Mason * tree. 3165aa5d6bedSChris Mason * 3166aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 3167aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 316874123bd7SChris Mason */ 316998ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 31702ff7e61eSJeff Mahoney struct extent_buffer *dst, 3171971a1f66SChris Mason struct extent_buffer *src, int empty) 3172be0e5c09SChris Mason { 3173d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3174be0e5c09SChris Mason int push_items = 0; 3175bb803951SChris Mason int src_nritems; 3176bb803951SChris Mason int dst_nritems; 3177aa5d6bedSChris Mason int ret = 0; 3178be0e5c09SChris Mason 31795f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 31805f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 31810b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 31827bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 31837bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 318454aa1f4dSChris Mason 3185bce4eae9SChris Mason if (!empty && src_nritems <= 8) 3186971a1f66SChris Mason return 1; 3187971a1f66SChris Mason 3188d397712bSChris Mason if (push_items <= 0) 3189be0e5c09SChris Mason return 1; 3190be0e5c09SChris Mason 3191bce4eae9SChris Mason if (empty) { 3192971a1f66SChris Mason push_items = min(src_nritems, push_items); 3193bce4eae9SChris Mason if (push_items < src_nritems) { 3194bce4eae9SChris Mason /* leave at least 8 pointers in the node if 3195bce4eae9SChris Mason * we aren't going to empty it 3196bce4eae9SChris Mason */ 3197bce4eae9SChris Mason if (src_nritems - push_items < 8) { 3198bce4eae9SChris Mason if (push_items <= 8) 3199bce4eae9SChris Mason return 1; 3200bce4eae9SChris Mason push_items -= 8; 3201bce4eae9SChris Mason } 3202bce4eae9SChris Mason } 3203bce4eae9SChris Mason } else 3204bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 320579f95c82SChris Mason 3206d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 3207d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 3208d16c702fSQu Wenruo ret = -EUCLEAN; 3209d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 3210d16c702fSQu Wenruo return ret; 3211d16c702fSQu Wenruo } 3212ed874f0dSDavid Sterba ret = tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 32135de865eeSFilipe David Borba Manana if (ret) { 321466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 32155de865eeSFilipe David Borba Manana return ret; 32165de865eeSFilipe David Borba Manana } 32175f39d397SChris Mason copy_extent_buffer(dst, src, 32185f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 32195f39d397SChris Mason btrfs_node_key_ptr_offset(0), 3220123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 32215f39d397SChris Mason 3222bb803951SChris Mason if (push_items < src_nritems) { 322357911b8bSJan Schmidt /* 3224bf1d3425SDavid Sterba * Don't call tree_mod_log_insert_move here, key removal was 3225bf1d3425SDavid Sterba * already fully logged by tree_mod_log_eb_copy above. 322657911b8bSJan Schmidt */ 32275f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 32285f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 3229e2fa7227SChris Mason (src_nritems - push_items) * 3230123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 3231bb803951SChris Mason } 32325f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 32335f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 32345f39d397SChris Mason btrfs_mark_buffer_dirty(src); 32355f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 323631840ae1SZheng Yan 3237bb803951SChris Mason return ret; 3238be0e5c09SChris Mason } 3239be0e5c09SChris Mason 324097571fd0SChris Mason /* 324179f95c82SChris Mason * try to push data from one node into the next node right in the 324279f95c82SChris Mason * tree. 324379f95c82SChris Mason * 324479f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 324579f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 324679f95c82SChris Mason * 324779f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 324879f95c82SChris Mason */ 32495f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 32505f39d397SChris Mason struct extent_buffer *dst, 32515f39d397SChris Mason struct extent_buffer *src) 325279f95c82SChris Mason { 325355d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 325479f95c82SChris Mason int push_items = 0; 325579f95c82SChris Mason int max_push; 325679f95c82SChris Mason int src_nritems; 325779f95c82SChris Mason int dst_nritems; 325879f95c82SChris Mason int ret = 0; 325979f95c82SChris Mason 32607bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 32617bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 32627bb86316SChris Mason 32635f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 32645f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 32650b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 3266d397712bSChris Mason if (push_items <= 0) 326779f95c82SChris Mason return 1; 3268bce4eae9SChris Mason 3269d397712bSChris Mason if (src_nritems < 4) 3270bce4eae9SChris Mason return 1; 327179f95c82SChris Mason 327279f95c82SChris Mason max_push = src_nritems / 2 + 1; 327379f95c82SChris Mason /* don't try to empty the node */ 3274d397712bSChris Mason if (max_push >= src_nritems) 327579f95c82SChris Mason return 1; 3276252c38f0SYan 327779f95c82SChris Mason if (max_push < push_items) 327879f95c82SChris Mason push_items = max_push; 327979f95c82SChris Mason 3280d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 3281d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 3282d16c702fSQu Wenruo ret = -EUCLEAN; 3283d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 3284d16c702fSQu Wenruo return ret; 3285d16c702fSQu Wenruo } 3286bf1d3425SDavid Sterba ret = tree_mod_log_insert_move(dst, push_items, 0, dst_nritems); 3287bf1d3425SDavid Sterba BUG_ON(ret < 0); 32885f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 32895f39d397SChris Mason btrfs_node_key_ptr_offset(0), 32905f39d397SChris Mason (dst_nritems) * 32915f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 3292d6025579SChris Mason 3293ed874f0dSDavid Sterba ret = tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 3294ed874f0dSDavid Sterba push_items); 32955de865eeSFilipe David Borba Manana if (ret) { 329666642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 32975de865eeSFilipe David Borba Manana return ret; 32985de865eeSFilipe David Borba Manana } 32995f39d397SChris Mason copy_extent_buffer(dst, src, 33005f39d397SChris Mason btrfs_node_key_ptr_offset(0), 33015f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 3302123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 330379f95c82SChris Mason 33045f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 33055f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 330679f95c82SChris Mason 33075f39d397SChris Mason btrfs_mark_buffer_dirty(src); 33085f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 330931840ae1SZheng Yan 331079f95c82SChris Mason return ret; 331179f95c82SChris Mason } 331279f95c82SChris Mason 331379f95c82SChris Mason /* 331497571fd0SChris Mason * helper function to insert a new root level in the tree. 331597571fd0SChris Mason * A new node is allocated, and a single item is inserted to 331697571fd0SChris Mason * point to the existing root 3317aa5d6bedSChris Mason * 3318aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 331997571fd0SChris Mason */ 3320d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 33215f39d397SChris Mason struct btrfs_root *root, 3322fdd99c72SLiu Bo struct btrfs_path *path, int level) 332374123bd7SChris Mason { 33240b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 33257bb86316SChris Mason u64 lower_gen; 33265f39d397SChris Mason struct extent_buffer *lower; 33275f39d397SChris Mason struct extent_buffer *c; 3328925baeddSChris Mason struct extent_buffer *old; 33295f39d397SChris Mason struct btrfs_disk_key lower_key; 3330d9d19a01SDavid Sterba int ret; 33315c680ed6SChris Mason 33325c680ed6SChris Mason BUG_ON(path->nodes[level]); 33335c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 33345c680ed6SChris Mason 33357bb86316SChris Mason lower = path->nodes[level-1]; 33367bb86316SChris Mason if (level == 1) 33377bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 33387bb86316SChris Mason else 33397bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 33407bb86316SChris Mason 3341a6279470SFilipe Manana c = alloc_tree_block_no_bg_flush(trans, root, 0, &lower_key, level, 33429631e4ccSJosef Bacik root->node->start, 0, 3343cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 33445f39d397SChris Mason if (IS_ERR(c)) 33455f39d397SChris Mason return PTR_ERR(c); 3346925baeddSChris Mason 33470b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3348f0486c68SYan, Zheng 33495f39d397SChris Mason btrfs_set_header_nritems(c, 1); 33505f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 3351db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 33527bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 335331840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 33547bb86316SChris Mason 33557bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 33565f39d397SChris Mason 33575f39d397SChris Mason btrfs_mark_buffer_dirty(c); 3358d5719762SChris Mason 3359925baeddSChris Mason old = root->node; 3360d9d19a01SDavid Sterba ret = tree_mod_log_insert_root(root->node, c, 0); 3361d9d19a01SDavid Sterba BUG_ON(ret < 0); 3362240f62c8SChris Mason rcu_assign_pointer(root->node, c); 3363925baeddSChris Mason 3364925baeddSChris Mason /* the super has an extra ref to root->node */ 3365925baeddSChris Mason free_extent_buffer(old); 3366925baeddSChris Mason 33670b86a832SChris Mason add_root_to_dirty_list(root); 336867439dadSDavid Sterba atomic_inc(&c->refs); 33695f39d397SChris Mason path->nodes[level] = c; 3370ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 337174123bd7SChris Mason path->slots[level] = 0; 337274123bd7SChris Mason return 0; 337374123bd7SChris Mason } 33745c680ed6SChris Mason 33755c680ed6SChris Mason /* 33765c680ed6SChris Mason * worker function to insert a single pointer in a node. 33775c680ed6SChris Mason * the node should have enough room for the pointer already 337897571fd0SChris Mason * 33795c680ed6SChris Mason * slot and level indicate where you want the key to go, and 33805c680ed6SChris Mason * blocknr is the block the key points to. 33815c680ed6SChris Mason */ 3382143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 33836ad3cf6dSDavid Sterba struct btrfs_path *path, 3384143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 3385c3e06965SJan Schmidt int slot, int level) 33865c680ed6SChris Mason { 33875f39d397SChris Mason struct extent_buffer *lower; 33885c680ed6SChris Mason int nritems; 3389f3ea38daSJan Schmidt int ret; 33905c680ed6SChris Mason 33915c680ed6SChris Mason BUG_ON(!path->nodes[level]); 3392f0486c68SYan, Zheng btrfs_assert_tree_locked(path->nodes[level]); 33935f39d397SChris Mason lower = path->nodes[level]; 33945f39d397SChris Mason nritems = btrfs_header_nritems(lower); 3395c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 33966ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 339774123bd7SChris Mason if (slot != nritems) { 3398bf1d3425SDavid Sterba if (level) { 3399bf1d3425SDavid Sterba ret = tree_mod_log_insert_move(lower, slot + 1, slot, 3400a446a979SDavid Sterba nritems - slot); 3401bf1d3425SDavid Sterba BUG_ON(ret < 0); 3402bf1d3425SDavid Sterba } 34035f39d397SChris Mason memmove_extent_buffer(lower, 34045f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 34055f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 3406123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 340774123bd7SChris Mason } 3408c3e06965SJan Schmidt if (level) { 3409e09c2efeSDavid Sterba ret = tree_mod_log_insert_key(lower, slot, MOD_LOG_KEY_ADD, 3410e09c2efeSDavid Sterba GFP_NOFS); 3411f3ea38daSJan Schmidt BUG_ON(ret < 0); 3412f3ea38daSJan Schmidt } 34135f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 3414db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 341574493f7aSChris Mason WARN_ON(trans->transid == 0); 341674493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 34175f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 34185f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 341974123bd7SChris Mason } 342074123bd7SChris Mason 342197571fd0SChris Mason /* 342297571fd0SChris Mason * split the node at the specified level in path in two. 342397571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 342497571fd0SChris Mason * 342597571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 342697571fd0SChris Mason * left and right, if either one works, it returns right away. 3427aa5d6bedSChris Mason * 3428aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 342997571fd0SChris Mason */ 3430e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3431e02119d5SChris Mason struct btrfs_root *root, 3432e02119d5SChris Mason struct btrfs_path *path, int level) 3433be0e5c09SChris Mason { 34340b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 34355f39d397SChris Mason struct extent_buffer *c; 34365f39d397SChris Mason struct extent_buffer *split; 34375f39d397SChris Mason struct btrfs_disk_key disk_key; 3438be0e5c09SChris Mason int mid; 34395c680ed6SChris Mason int ret; 34407518a238SChris Mason u32 c_nritems; 3441be0e5c09SChris Mason 34425f39d397SChris Mason c = path->nodes[level]; 34437bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 34445f39d397SChris Mason if (c == root->node) { 3445d9abbf1cSJan Schmidt /* 344690f8d62eSJan Schmidt * trying to split the root, lets make a new one 344790f8d62eSJan Schmidt * 3448fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 344990f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 345090f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 345190f8d62eSJan Schmidt * elements below with tree_mod_log_eb_copy. We're holding a 345290f8d62eSJan Schmidt * tree lock on the buffer, which is why we cannot race with 345390f8d62eSJan Schmidt * other tree_mod_log users. 3454d9abbf1cSJan Schmidt */ 3455fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 34565c680ed6SChris Mason if (ret) 34575c680ed6SChris Mason return ret; 3458b3612421SChris Mason } else { 3459e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 34605f39d397SChris Mason c = path->nodes[level]; 34615f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 34620b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 3463e66f709bSChris Mason return 0; 346454aa1f4dSChris Mason if (ret < 0) 346554aa1f4dSChris Mason return ret; 34665c680ed6SChris Mason } 3467e66f709bSChris Mason 34685f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 34695d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 34705d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 34717bb86316SChris Mason 3472a6279470SFilipe Manana split = alloc_tree_block_no_bg_flush(trans, root, 0, &disk_key, level, 34734dff97e6SJosef Bacik c->start, 0, BTRFS_NESTING_SPLIT); 34745f39d397SChris Mason if (IS_ERR(split)) 34755f39d397SChris Mason return PTR_ERR(split); 347654aa1f4dSChris Mason 34770b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3478bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 34795f39d397SChris Mason 3480ed874f0dSDavid Sterba ret = tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 34815de865eeSFilipe David Borba Manana if (ret) { 348266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 34835de865eeSFilipe David Borba Manana return ret; 34845de865eeSFilipe David Borba Manana } 34855f39d397SChris Mason copy_extent_buffer(split, c, 34865f39d397SChris Mason btrfs_node_key_ptr_offset(0), 34875f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 3488123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 34895f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 34905f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3491aa5d6bedSChris Mason ret = 0; 3492aa5d6bedSChris Mason 34935f39d397SChris Mason btrfs_mark_buffer_dirty(c); 34945f39d397SChris Mason btrfs_mark_buffer_dirty(split); 34955f39d397SChris Mason 34966ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, split->start, 3497c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3498aa5d6bedSChris Mason 34995de08d7dSChris Mason if (path->slots[level] >= mid) { 35005c680ed6SChris Mason path->slots[level] -= mid; 3501925baeddSChris Mason btrfs_tree_unlock(c); 35025f39d397SChris Mason free_extent_buffer(c); 35035f39d397SChris Mason path->nodes[level] = split; 35045c680ed6SChris Mason path->slots[level + 1] += 1; 3505eb60ceacSChris Mason } else { 3506925baeddSChris Mason btrfs_tree_unlock(split); 35075f39d397SChris Mason free_extent_buffer(split); 3508be0e5c09SChris Mason } 3509aa5d6bedSChris Mason return ret; 3510be0e5c09SChris Mason } 3511be0e5c09SChris Mason 351274123bd7SChris Mason /* 351374123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 351474123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 351574123bd7SChris Mason * space used both by the item structs and the item data 351674123bd7SChris Mason */ 35175f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 3518be0e5c09SChris Mason { 351941be1f3bSJosef Bacik struct btrfs_item *start_item; 352041be1f3bSJosef Bacik struct btrfs_item *end_item; 3521be0e5c09SChris Mason int data_len; 35225f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3523d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3524be0e5c09SChris Mason 3525be0e5c09SChris Mason if (!nr) 3526be0e5c09SChris Mason return 0; 3527dd3cc16bSRoss Kirk start_item = btrfs_item_nr(start); 3528dd3cc16bSRoss Kirk end_item = btrfs_item_nr(end); 3529a31356b9SDavid Sterba data_len = btrfs_item_offset(l, start_item) + 3530a31356b9SDavid Sterba btrfs_item_size(l, start_item); 3531a31356b9SDavid Sterba data_len = data_len - btrfs_item_offset(l, end_item); 35320783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3533d4dbff95SChris Mason WARN_ON(data_len < 0); 3534be0e5c09SChris Mason return data_len; 3535be0e5c09SChris Mason } 3536be0e5c09SChris Mason 353774123bd7SChris Mason /* 3538d4dbff95SChris Mason * The space between the end of the leaf items and 3539d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3540d4dbff95SChris Mason * the leaf has left for both items and data 3541d4dbff95SChris Mason */ 3542e902baacSDavid Sterba noinline int btrfs_leaf_free_space(struct extent_buffer *leaf) 3543d4dbff95SChris Mason { 3544e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 35455f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 35465f39d397SChris Mason int ret; 35470b246afaSJeff Mahoney 35480b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 35495f39d397SChris Mason if (ret < 0) { 35500b246afaSJeff Mahoney btrfs_crit(fs_info, 3551efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3552da17066cSJeff Mahoney ret, 35530b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 35545f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 35555f39d397SChris Mason } 35565f39d397SChris Mason return ret; 3557d4dbff95SChris Mason } 3558d4dbff95SChris Mason 355999d8f83cSChris Mason /* 356099d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 356199d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 356299d8f83cSChris Mason */ 3563f72f0010SDavid Sterba static noinline int __push_leaf_right(struct btrfs_path *path, 356444871b1bSChris Mason int data_size, int empty, 356544871b1bSChris Mason struct extent_buffer *right, 356699d8f83cSChris Mason int free_space, u32 left_nritems, 356799d8f83cSChris Mason u32 min_slot) 356800ec4c51SChris Mason { 3569f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 35705f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 357144871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3572cfed81a0SChris Mason struct btrfs_map_token token; 35735f39d397SChris Mason struct btrfs_disk_key disk_key; 357400ec4c51SChris Mason int slot; 357534a38218SChris Mason u32 i; 357600ec4c51SChris Mason int push_space = 0; 357700ec4c51SChris Mason int push_items = 0; 35780783fcfcSChris Mason struct btrfs_item *item; 357934a38218SChris Mason u32 nr; 35807518a238SChris Mason u32 right_nritems; 35815f39d397SChris Mason u32 data_end; 3582db94535dSChris Mason u32 this_item_size; 358300ec4c51SChris Mason 358434a38218SChris Mason if (empty) 358534a38218SChris Mason nr = 0; 358634a38218SChris Mason else 358799d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 358834a38218SChris Mason 358931840ae1SZheng Yan if (path->slots[0] >= left_nritems) 359087b29b20SYan Zheng push_space += data_size; 359131840ae1SZheng Yan 359244871b1bSChris Mason slot = path->slots[1]; 359334a38218SChris Mason i = left_nritems - 1; 359434a38218SChris Mason while (i >= nr) { 3595dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3596db94535dSChris Mason 359731840ae1SZheng Yan if (!empty && push_items > 0) { 359831840ae1SZheng Yan if (path->slots[0] > i) 359931840ae1SZheng Yan break; 360031840ae1SZheng Yan if (path->slots[0] == i) { 3601e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 3602e902baacSDavid Sterba 360331840ae1SZheng Yan if (space + push_space * 2 > free_space) 360431840ae1SZheng Yan break; 360531840ae1SZheng Yan } 360631840ae1SZheng Yan } 360731840ae1SZheng Yan 360800ec4c51SChris Mason if (path->slots[0] == i) 360987b29b20SYan Zheng push_space += data_size; 3610db94535dSChris Mason 3611db94535dSChris Mason this_item_size = btrfs_item_size(left, item); 3612db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 361300ec4c51SChris Mason break; 361431840ae1SZheng Yan 361500ec4c51SChris Mason push_items++; 3616db94535dSChris Mason push_space += this_item_size + sizeof(*item); 361734a38218SChris Mason if (i == 0) 361834a38218SChris Mason break; 361934a38218SChris Mason i--; 3620db94535dSChris Mason } 36215f39d397SChris Mason 3622925baeddSChris Mason if (push_items == 0) 3623925baeddSChris Mason goto out_unlock; 36245f39d397SChris Mason 36256c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 36265f39d397SChris Mason 362700ec4c51SChris Mason /* push left to right */ 36285f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 362934a38218SChris Mason 36305f39d397SChris Mason push_space = btrfs_item_end_nr(left, left_nritems - push_items); 36318f881e8cSDavid Sterba push_space -= leaf_data_end(left); 36325f39d397SChris Mason 363300ec4c51SChris Mason /* make room in the right data area */ 36348f881e8cSDavid Sterba data_end = leaf_data_end(right); 36355f39d397SChris Mason memmove_extent_buffer(right, 36363d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end - push_space, 36373d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 36380b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 36395f39d397SChris Mason 364000ec4c51SChris Mason /* copy from the left data area */ 36413d9ec8c4SNikolay Borisov copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET + 36420b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 36438f881e8cSDavid Sterba BTRFS_LEAF_DATA_OFFSET + leaf_data_end(left), 3644d6025579SChris Mason push_space); 36455f39d397SChris Mason 36465f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 36475f39d397SChris Mason btrfs_item_nr_offset(0), 36480783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 36495f39d397SChris Mason 365000ec4c51SChris Mason /* copy the items from left to right */ 36515f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 36525f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 36530783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 365400ec4c51SChris Mason 365500ec4c51SChris Mason /* update the item pointers */ 3656c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 36577518a238SChris Mason right_nritems += push_items; 36585f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 36590b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 36607518a238SChris Mason for (i = 0; i < right_nritems; i++) { 3661dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3662cc4c13d5SDavid Sterba push_space -= btrfs_token_item_size(&token, item); 3663cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, push_space); 3664db94535dSChris Mason } 3665db94535dSChris Mason 36667518a238SChris Mason left_nritems -= push_items; 36675f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 366800ec4c51SChris Mason 366934a38218SChris Mason if (left_nritems) 36705f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3671f0486c68SYan, Zheng else 36726a884d7dSDavid Sterba btrfs_clean_tree_block(left); 3673f0486c68SYan, Zheng 36745f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3675a429e513SChris Mason 36765f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 36775f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3678d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 367902217ed2SChris Mason 368000ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 36817518a238SChris Mason if (path->slots[0] >= left_nritems) { 36827518a238SChris Mason path->slots[0] -= left_nritems; 3683925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 36846a884d7dSDavid Sterba btrfs_clean_tree_block(path->nodes[0]); 3685925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 36865f39d397SChris Mason free_extent_buffer(path->nodes[0]); 36875f39d397SChris Mason path->nodes[0] = right; 368800ec4c51SChris Mason path->slots[1] += 1; 368900ec4c51SChris Mason } else { 3690925baeddSChris Mason btrfs_tree_unlock(right); 36915f39d397SChris Mason free_extent_buffer(right); 369200ec4c51SChris Mason } 369300ec4c51SChris Mason return 0; 3694925baeddSChris Mason 3695925baeddSChris Mason out_unlock: 3696925baeddSChris Mason btrfs_tree_unlock(right); 3697925baeddSChris Mason free_extent_buffer(right); 3698925baeddSChris Mason return 1; 369900ec4c51SChris Mason } 3700925baeddSChris Mason 370100ec4c51SChris Mason /* 370244871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 370374123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 370444871b1bSChris Mason * 370544871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 370644871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 370799d8f83cSChris Mason * 370899d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 370999d8f83cSChris Mason * push any slot lower than min_slot 371074123bd7SChris Mason */ 371144871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 371299d8f83cSChris Mason *root, struct btrfs_path *path, 371399d8f83cSChris Mason int min_data_size, int data_size, 371499d8f83cSChris Mason int empty, u32 min_slot) 3715be0e5c09SChris Mason { 371644871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 371744871b1bSChris Mason struct extent_buffer *right; 371844871b1bSChris Mason struct extent_buffer *upper; 371944871b1bSChris Mason int slot; 372044871b1bSChris Mason int free_space; 372144871b1bSChris Mason u32 left_nritems; 372244871b1bSChris Mason int ret; 372344871b1bSChris Mason 372444871b1bSChris Mason if (!path->nodes[1]) 372544871b1bSChris Mason return 1; 372644871b1bSChris Mason 372744871b1bSChris Mason slot = path->slots[1]; 372844871b1bSChris Mason upper = path->nodes[1]; 372944871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 373044871b1bSChris Mason return 1; 373144871b1bSChris Mason 373244871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 373344871b1bSChris Mason 37344b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 3735fb770ae4SLiu Bo /* 3736fb770ae4SLiu Bo * slot + 1 is not valid or we fail to read the right node, 3737fb770ae4SLiu Bo * no big deal, just return. 3738fb770ae4SLiu Bo */ 3739fb770ae4SLiu Bo if (IS_ERR(right)) 374091ca338dSTsutomu Itoh return 1; 374191ca338dSTsutomu Itoh 3742bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 374344871b1bSChris Mason 3744e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 374544871b1bSChris Mason if (free_space < data_size) 374644871b1bSChris Mason goto out_unlock; 374744871b1bSChris Mason 374844871b1bSChris Mason /* cow and double check */ 374944871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 3750bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 375144871b1bSChris Mason if (ret) 375244871b1bSChris Mason goto out_unlock; 375344871b1bSChris Mason 3754e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 375544871b1bSChris Mason if (free_space < data_size) 375644871b1bSChris Mason goto out_unlock; 375744871b1bSChris Mason 375844871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 375944871b1bSChris Mason if (left_nritems == 0) 376044871b1bSChris Mason goto out_unlock; 376144871b1bSChris Mason 3762d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3763d16c702fSQu Wenruo ret = -EUCLEAN; 3764d16c702fSQu Wenruo btrfs_tree_unlock(right); 3765d16c702fSQu Wenruo free_extent_buffer(right); 3766d16c702fSQu Wenruo return ret; 3767d16c702fSQu Wenruo } 37682ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 37692ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 37702ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 37712ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 377252042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 37732ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 37742ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 37752ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 37762ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 37772ef1fed2SFilipe David Borba Manana path->slots[1]++; 37782ef1fed2SFilipe David Borba Manana return 0; 37792ef1fed2SFilipe David Borba Manana } 37802ef1fed2SFilipe David Borba Manana 3781f72f0010SDavid Sterba return __push_leaf_right(path, min_data_size, empty, 378299d8f83cSChris Mason right, free_space, left_nritems, min_slot); 378344871b1bSChris Mason out_unlock: 378444871b1bSChris Mason btrfs_tree_unlock(right); 378544871b1bSChris Mason free_extent_buffer(right); 378644871b1bSChris Mason return 1; 378744871b1bSChris Mason } 378844871b1bSChris Mason 378944871b1bSChris Mason /* 379044871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 379144871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 379299d8f83cSChris Mason * 379399d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 379499d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 379599d8f83cSChris Mason * items 379644871b1bSChris Mason */ 37978087c193SDavid Sterba static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, 379844871b1bSChris Mason int empty, struct extent_buffer *left, 379999d8f83cSChris Mason int free_space, u32 right_nritems, 380099d8f83cSChris Mason u32 max_slot) 380144871b1bSChris Mason { 38028087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 38035f39d397SChris Mason struct btrfs_disk_key disk_key; 38045f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3805be0e5c09SChris Mason int i; 3806be0e5c09SChris Mason int push_space = 0; 3807be0e5c09SChris Mason int push_items = 0; 38080783fcfcSChris Mason struct btrfs_item *item; 38097518a238SChris Mason u32 old_left_nritems; 381034a38218SChris Mason u32 nr; 3811aa5d6bedSChris Mason int ret = 0; 3812db94535dSChris Mason u32 this_item_size; 3813db94535dSChris Mason u32 old_left_item_size; 3814cfed81a0SChris Mason struct btrfs_map_token token; 3815cfed81a0SChris Mason 381634a38218SChris Mason if (empty) 381799d8f83cSChris Mason nr = min(right_nritems, max_slot); 381834a38218SChris Mason else 381999d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 382034a38218SChris Mason 382134a38218SChris Mason for (i = 0; i < nr; i++) { 3822dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3823db94535dSChris Mason 382431840ae1SZheng Yan if (!empty && push_items > 0) { 382531840ae1SZheng Yan if (path->slots[0] < i) 382631840ae1SZheng Yan break; 382731840ae1SZheng Yan if (path->slots[0] == i) { 3828e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3829e902baacSDavid Sterba 383031840ae1SZheng Yan if (space + push_space * 2 > free_space) 383131840ae1SZheng Yan break; 383231840ae1SZheng Yan } 383331840ae1SZheng Yan } 383431840ae1SZheng Yan 3835be0e5c09SChris Mason if (path->slots[0] == i) 383687b29b20SYan Zheng push_space += data_size; 3837db94535dSChris Mason 3838db94535dSChris Mason this_item_size = btrfs_item_size(right, item); 3839db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 3840be0e5c09SChris Mason break; 3841db94535dSChris Mason 3842be0e5c09SChris Mason push_items++; 3843db94535dSChris Mason push_space += this_item_size + sizeof(*item); 3844be0e5c09SChris Mason } 3845db94535dSChris Mason 3846be0e5c09SChris Mason if (push_items == 0) { 3847925baeddSChris Mason ret = 1; 3848925baeddSChris Mason goto out; 3849be0e5c09SChris Mason } 3850fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 38515f39d397SChris Mason 3852be0e5c09SChris Mason /* push data from right to left */ 38535f39d397SChris Mason copy_extent_buffer(left, right, 38545f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 38555f39d397SChris Mason btrfs_item_nr_offset(0), 38565f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 38575f39d397SChris Mason 38580b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 38595f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1); 38605f39d397SChris Mason 38613d9ec8c4SNikolay Borisov copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET + 38628f881e8cSDavid Sterba leaf_data_end(left) - push_space, 38633d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 38645f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1), 3865be0e5c09SChris Mason push_space); 38665f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 386787b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3868eb60ceacSChris Mason 3869c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 3870db94535dSChris Mason old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1); 3871be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 38725f39d397SChris Mason u32 ioff; 3873db94535dSChris Mason 3874dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3875db94535dSChris Mason 3876cc4c13d5SDavid Sterba ioff = btrfs_token_item_offset(&token, item); 3877cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, 3878cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3879be0e5c09SChris Mason } 38805f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3881be0e5c09SChris Mason 3882be0e5c09SChris Mason /* fixup right node */ 388331b1a2bdSJulia Lawall if (push_items > right_nritems) 388431b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3885d397712bSChris Mason right_nritems); 388634a38218SChris Mason 388734a38218SChris Mason if (push_items < right_nritems) { 38885f39d397SChris Mason push_space = btrfs_item_offset_nr(right, push_items - 1) - 38898f881e8cSDavid Sterba leaf_data_end(right); 38903d9ec8c4SNikolay Borisov memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET + 38910b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 38923d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 38938f881e8cSDavid Sterba leaf_data_end(right), push_space); 38945f39d397SChris Mason 38955f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 38965f39d397SChris Mason btrfs_item_nr_offset(push_items), 38975f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 38980783fcfcSChris Mason sizeof(struct btrfs_item)); 389934a38218SChris Mason } 3900c82f823cSDavid Sterba 3901c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3902eef1c494SYan right_nritems -= push_items; 3903eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 39040b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 39055f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 3906dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3907db94535dSChris Mason 3908cc4c13d5SDavid Sterba push_space = push_space - btrfs_token_item_size(&token, item); 3909cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, push_space); 3910db94535dSChris Mason } 3911eb60ceacSChris Mason 39125f39d397SChris Mason btrfs_mark_buffer_dirty(left); 391334a38218SChris Mason if (right_nritems) 39145f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3915f0486c68SYan, Zheng else 39166a884d7dSDavid Sterba btrfs_clean_tree_block(right); 3917098f59c2SChris Mason 39185f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3919b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3920be0e5c09SChris Mason 3921be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3922be0e5c09SChris Mason if (path->slots[0] < push_items) { 3923be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3924925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 39255f39d397SChris Mason free_extent_buffer(path->nodes[0]); 39265f39d397SChris Mason path->nodes[0] = left; 3927be0e5c09SChris Mason path->slots[1] -= 1; 3928be0e5c09SChris Mason } else { 3929925baeddSChris Mason btrfs_tree_unlock(left); 39305f39d397SChris Mason free_extent_buffer(left); 3931be0e5c09SChris Mason path->slots[0] -= push_items; 3932be0e5c09SChris Mason } 3933eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3934aa5d6bedSChris Mason return ret; 3935925baeddSChris Mason out: 3936925baeddSChris Mason btrfs_tree_unlock(left); 3937925baeddSChris Mason free_extent_buffer(left); 3938925baeddSChris Mason return ret; 3939be0e5c09SChris Mason } 3940be0e5c09SChris Mason 394174123bd7SChris Mason /* 394244871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 394344871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 394499d8f83cSChris Mason * 394599d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 394699d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 394799d8f83cSChris Mason * items 394844871b1bSChris Mason */ 394944871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 395099d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 395199d8f83cSChris Mason int data_size, int empty, u32 max_slot) 395244871b1bSChris Mason { 395344871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 395444871b1bSChris Mason struct extent_buffer *left; 395544871b1bSChris Mason int slot; 395644871b1bSChris Mason int free_space; 395744871b1bSChris Mason u32 right_nritems; 395844871b1bSChris Mason int ret = 0; 395944871b1bSChris Mason 396044871b1bSChris Mason slot = path->slots[1]; 396144871b1bSChris Mason if (slot == 0) 396244871b1bSChris Mason return 1; 396344871b1bSChris Mason if (!path->nodes[1]) 396444871b1bSChris Mason return 1; 396544871b1bSChris Mason 396644871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 396744871b1bSChris Mason if (right_nritems == 0) 396844871b1bSChris Mason return 1; 396944871b1bSChris Mason 397044871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 397144871b1bSChris Mason 39724b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3973fb770ae4SLiu Bo /* 3974fb770ae4SLiu Bo * slot - 1 is not valid or we fail to read the left node, 3975fb770ae4SLiu Bo * no big deal, just return. 3976fb770ae4SLiu Bo */ 3977fb770ae4SLiu Bo if (IS_ERR(left)) 397891ca338dSTsutomu Itoh return 1; 397991ca338dSTsutomu Itoh 3980bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 398144871b1bSChris Mason 3982e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 398344871b1bSChris Mason if (free_space < data_size) { 398444871b1bSChris Mason ret = 1; 398544871b1bSChris Mason goto out; 398644871b1bSChris Mason } 398744871b1bSChris Mason 398844871b1bSChris Mason /* cow and double check */ 398944871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 39909631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3991bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 399244871b1bSChris Mason if (ret) { 399344871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 399479787eaaSJeff Mahoney if (ret == -ENOSPC) 399544871b1bSChris Mason ret = 1; 399644871b1bSChris Mason goto out; 399744871b1bSChris Mason } 399844871b1bSChris Mason 3999e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 400044871b1bSChris Mason if (free_space < data_size) { 400144871b1bSChris Mason ret = 1; 400244871b1bSChris Mason goto out; 400344871b1bSChris Mason } 400444871b1bSChris Mason 4005d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 4006d16c702fSQu Wenruo ret = -EUCLEAN; 4007d16c702fSQu Wenruo goto out; 4008d16c702fSQu Wenruo } 40098087c193SDavid Sterba return __push_leaf_left(path, min_data_size, 401099d8f83cSChris Mason empty, left, free_space, right_nritems, 401199d8f83cSChris Mason max_slot); 401244871b1bSChris Mason out: 401344871b1bSChris Mason btrfs_tree_unlock(left); 401444871b1bSChris Mason free_extent_buffer(left); 401544871b1bSChris Mason return ret; 401644871b1bSChris Mason } 401744871b1bSChris Mason 401844871b1bSChris Mason /* 401974123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 402074123bd7SChris Mason * available for the resulting leaf level of the path. 402174123bd7SChris Mason */ 4022143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 402344871b1bSChris Mason struct btrfs_path *path, 402444871b1bSChris Mason struct extent_buffer *l, 402544871b1bSChris Mason struct extent_buffer *right, 402644871b1bSChris Mason int slot, int mid, int nritems) 4027be0e5c09SChris Mason { 402894f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 4029be0e5c09SChris Mason int data_copy_size; 4030be0e5c09SChris Mason int rt_data_off; 4031be0e5c09SChris Mason int i; 4032d4dbff95SChris Mason struct btrfs_disk_key disk_key; 4033cfed81a0SChris Mason struct btrfs_map_token token; 4034cfed81a0SChris Mason 40355f39d397SChris Mason nritems = nritems - mid; 40365f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 40378f881e8cSDavid Sterba data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(l); 40385f39d397SChris Mason 40395f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 40405f39d397SChris Mason btrfs_item_nr_offset(mid), 40415f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 40425f39d397SChris Mason 40435f39d397SChris Mason copy_extent_buffer(right, l, 40443d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) - 40453d9ec8c4SNikolay Borisov data_copy_size, BTRFS_LEAF_DATA_OFFSET + 40468f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 404774123bd7SChris Mason 40480b246afaSJeff Mahoney rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_end_nr(l, mid); 40495f39d397SChris Mason 4050c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 40515f39d397SChris Mason for (i = 0; i < nritems; i++) { 4052dd3cc16bSRoss Kirk struct btrfs_item *item = btrfs_item_nr(i); 4053db94535dSChris Mason u32 ioff; 4054db94535dSChris Mason 4055cc4c13d5SDavid Sterba ioff = btrfs_token_item_offset(&token, item); 4056cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, ioff + rt_data_off); 40570783fcfcSChris Mason } 405874123bd7SChris Mason 40595f39d397SChris Mason btrfs_set_header_nritems(l, mid); 40605f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 40616ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 40625f39d397SChris Mason 40635f39d397SChris Mason btrfs_mark_buffer_dirty(right); 40645f39d397SChris Mason btrfs_mark_buffer_dirty(l); 4065eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 40665f39d397SChris Mason 4067be0e5c09SChris Mason if (mid <= slot) { 4068925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 40695f39d397SChris Mason free_extent_buffer(path->nodes[0]); 40705f39d397SChris Mason path->nodes[0] = right; 4071be0e5c09SChris Mason path->slots[0] -= mid; 4072be0e5c09SChris Mason path->slots[1] += 1; 4073925baeddSChris Mason } else { 4074925baeddSChris Mason btrfs_tree_unlock(right); 40755f39d397SChris Mason free_extent_buffer(right); 4076925baeddSChris Mason } 40775f39d397SChris Mason 4078eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 407944871b1bSChris Mason } 408044871b1bSChris Mason 408144871b1bSChris Mason /* 408299d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 408399d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 408499d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 408599d8f83cSChris Mason * A B C 408699d8f83cSChris Mason * 408799d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 408899d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 408999d8f83cSChris Mason * completely. 409099d8f83cSChris Mason */ 409199d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 409299d8f83cSChris Mason struct btrfs_root *root, 409399d8f83cSChris Mason struct btrfs_path *path, 409499d8f83cSChris Mason int data_size) 409599d8f83cSChris Mason { 409699d8f83cSChris Mason int ret; 409799d8f83cSChris Mason int progress = 0; 409899d8f83cSChris Mason int slot; 409999d8f83cSChris Mason u32 nritems; 41005a4267caSFilipe David Borba Manana int space_needed = data_size; 410199d8f83cSChris Mason 410299d8f83cSChris Mason slot = path->slots[0]; 41035a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 4104e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 410599d8f83cSChris Mason 410699d8f83cSChris Mason /* 410799d8f83cSChris Mason * try to push all the items after our slot into the 410899d8f83cSChris Mason * right leaf 410999d8f83cSChris Mason */ 41105a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 411199d8f83cSChris Mason if (ret < 0) 411299d8f83cSChris Mason return ret; 411399d8f83cSChris Mason 411499d8f83cSChris Mason if (ret == 0) 411599d8f83cSChris Mason progress++; 411699d8f83cSChris Mason 411799d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 411899d8f83cSChris Mason /* 411999d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 412099d8f83cSChris Mason * we've done so we're done 412199d8f83cSChris Mason */ 412299d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 412399d8f83cSChris Mason return 0; 412499d8f83cSChris Mason 4125e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 412699d8f83cSChris Mason return 0; 412799d8f83cSChris Mason 412899d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 412999d8f83cSChris Mason slot = path->slots[0]; 4130263d3995SFilipe Manana space_needed = data_size; 4131263d3995SFilipe Manana if (slot > 0) 4132e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 41335a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 413499d8f83cSChris Mason if (ret < 0) 413599d8f83cSChris Mason return ret; 413699d8f83cSChris Mason 413799d8f83cSChris Mason if (ret == 0) 413899d8f83cSChris Mason progress++; 413999d8f83cSChris Mason 414099d8f83cSChris Mason if (progress) 414199d8f83cSChris Mason return 0; 414299d8f83cSChris Mason return 1; 414399d8f83cSChris Mason } 414499d8f83cSChris Mason 414599d8f83cSChris Mason /* 414644871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 414744871b1bSChris Mason * available for the resulting leaf level of the path. 414844871b1bSChris Mason * 414944871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 415044871b1bSChris Mason */ 415144871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 415244871b1bSChris Mason struct btrfs_root *root, 4153310712b2SOmar Sandoval const struct btrfs_key *ins_key, 415444871b1bSChris Mason struct btrfs_path *path, int data_size, 415544871b1bSChris Mason int extend) 415644871b1bSChris Mason { 41575d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 415844871b1bSChris Mason struct extent_buffer *l; 415944871b1bSChris Mason u32 nritems; 416044871b1bSChris Mason int mid; 416144871b1bSChris Mason int slot; 416244871b1bSChris Mason struct extent_buffer *right; 4163b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 416444871b1bSChris Mason int ret = 0; 416544871b1bSChris Mason int wret; 41665d4f98a2SYan Zheng int split; 416744871b1bSChris Mason int num_doubles = 0; 416899d8f83cSChris Mason int tried_avoid_double = 0; 416944871b1bSChris Mason 4170a5719521SYan, Zheng l = path->nodes[0]; 4171a5719521SYan, Zheng slot = path->slots[0]; 4172a5719521SYan, Zheng if (extend && data_size + btrfs_item_size_nr(l, slot) + 41730b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 4174a5719521SYan, Zheng return -EOVERFLOW; 4175a5719521SYan, Zheng 417644871b1bSChris Mason /* first try to make some room by pushing left and right */ 417733157e05SLiu Bo if (data_size && path->nodes[1]) { 41785a4267caSFilipe David Borba Manana int space_needed = data_size; 41795a4267caSFilipe David Borba Manana 41805a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 4181e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 41825a4267caSFilipe David Borba Manana 41835a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 41845a4267caSFilipe David Borba Manana space_needed, 0, 0); 418544871b1bSChris Mason if (wret < 0) 418644871b1bSChris Mason return wret; 418744871b1bSChris Mason if (wret) { 4188263d3995SFilipe Manana space_needed = data_size; 4189263d3995SFilipe Manana if (slot > 0) 4190e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 41915a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 41925a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 419344871b1bSChris Mason if (wret < 0) 419444871b1bSChris Mason return wret; 419544871b1bSChris Mason } 419644871b1bSChris Mason l = path->nodes[0]; 419744871b1bSChris Mason 419844871b1bSChris Mason /* did the pushes work? */ 4199e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 420044871b1bSChris Mason return 0; 420144871b1bSChris Mason } 420244871b1bSChris Mason 420344871b1bSChris Mason if (!path->nodes[1]) { 4204fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 420544871b1bSChris Mason if (ret) 420644871b1bSChris Mason return ret; 420744871b1bSChris Mason } 420844871b1bSChris Mason again: 42095d4f98a2SYan Zheng split = 1; 421044871b1bSChris Mason l = path->nodes[0]; 421144871b1bSChris Mason slot = path->slots[0]; 421244871b1bSChris Mason nritems = btrfs_header_nritems(l); 421344871b1bSChris Mason mid = (nritems + 1) / 2; 421444871b1bSChris Mason 42155d4f98a2SYan Zheng if (mid <= slot) { 42165d4f98a2SYan Zheng if (nritems == 1 || 42175d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 42180b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 42195d4f98a2SYan Zheng if (slot >= nritems) { 42205d4f98a2SYan Zheng split = 0; 42215d4f98a2SYan Zheng } else { 42225d4f98a2SYan Zheng mid = slot; 42235d4f98a2SYan Zheng if (mid != nritems && 42245d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 42250b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 422699d8f83cSChris Mason if (data_size && !tried_avoid_double) 422799d8f83cSChris Mason goto push_for_double; 42285d4f98a2SYan Zheng split = 2; 42295d4f98a2SYan Zheng } 42305d4f98a2SYan Zheng } 42315d4f98a2SYan Zheng } 42325d4f98a2SYan Zheng } else { 42335d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 42340b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 42355d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 42365d4f98a2SYan Zheng split = 0; 42375d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 42385d4f98a2SYan Zheng mid = 1; 42395d4f98a2SYan Zheng } else { 42405d4f98a2SYan Zheng mid = slot; 42415d4f98a2SYan Zheng if (mid != nritems && 42425d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 42430b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 424499d8f83cSChris Mason if (data_size && !tried_avoid_double) 424599d8f83cSChris Mason goto push_for_double; 42465d4f98a2SYan Zheng split = 2; 42475d4f98a2SYan Zheng } 42485d4f98a2SYan Zheng } 42495d4f98a2SYan Zheng } 42505d4f98a2SYan Zheng } 42515d4f98a2SYan Zheng 42525d4f98a2SYan Zheng if (split == 0) 42535d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 42545d4f98a2SYan Zheng else 42555d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 42565d4f98a2SYan Zheng 4257ca9d473aSJosef Bacik /* 4258ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 4259ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 4260ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 4261ca9d473aSJosef Bacik * out. In the future we could add a 4262ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 4263ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 4264ca9d473aSJosef Bacik */ 4265a6279470SFilipe Manana right = alloc_tree_block_no_bg_flush(trans, root, 0, &disk_key, 0, 4266ca9d473aSJosef Bacik l->start, 0, num_doubles ? 4267ca9d473aSJosef Bacik BTRFS_NESTING_NEW_ROOT : 4268ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 4269f0486c68SYan, Zheng if (IS_ERR(right)) 427044871b1bSChris Mason return PTR_ERR(right); 4271f0486c68SYan, Zheng 42720b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 427344871b1bSChris Mason 42745d4f98a2SYan Zheng if (split == 0) { 427544871b1bSChris Mason if (mid <= slot) { 427644871b1bSChris Mason btrfs_set_header_nritems(right, 0); 42776ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 42782ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 427944871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 428044871b1bSChris Mason free_extent_buffer(path->nodes[0]); 428144871b1bSChris Mason path->nodes[0] = right; 428244871b1bSChris Mason path->slots[0] = 0; 428344871b1bSChris Mason path->slots[1] += 1; 428444871b1bSChris Mason } else { 428544871b1bSChris Mason btrfs_set_header_nritems(right, 0); 42866ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 42872ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 428844871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 428944871b1bSChris Mason free_extent_buffer(path->nodes[0]); 429044871b1bSChris Mason path->nodes[0] = right; 429144871b1bSChris Mason path->slots[0] = 0; 4292143bede5SJeff Mahoney if (path->slots[1] == 0) 4293b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 42945d4f98a2SYan Zheng } 4295196e0249SLiu Bo /* 4296196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 4297196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 4298196e0249SLiu Bo * the content of ins_len to 'right'. 4299196e0249SLiu Bo */ 430044871b1bSChris Mason return ret; 430144871b1bSChris Mason } 430244871b1bSChris Mason 430394f94ad9SDavid Sterba copy_for_split(trans, path, l, right, slot, mid, nritems); 430444871b1bSChris Mason 43055d4f98a2SYan Zheng if (split == 2) { 4306cc0c5538SChris Mason BUG_ON(num_doubles != 0); 4307cc0c5538SChris Mason num_doubles++; 4308cc0c5538SChris Mason goto again; 43093326d1b0SChris Mason } 431044871b1bSChris Mason 4311143bede5SJeff Mahoney return 0; 431299d8f83cSChris Mason 431399d8f83cSChris Mason push_for_double: 431499d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 431599d8f83cSChris Mason tried_avoid_double = 1; 4316e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 431799d8f83cSChris Mason return 0; 431899d8f83cSChris Mason goto again; 4319be0e5c09SChris Mason } 4320be0e5c09SChris Mason 4321ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 4322ad48fd75SYan, Zheng struct btrfs_root *root, 4323ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 4324ad48fd75SYan, Zheng { 4325ad48fd75SYan, Zheng struct btrfs_key key; 4326ad48fd75SYan, Zheng struct extent_buffer *leaf; 4327ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 4328ad48fd75SYan, Zheng u64 extent_len = 0; 4329ad48fd75SYan, Zheng u32 item_size; 4330ad48fd75SYan, Zheng int ret; 4331ad48fd75SYan, Zheng 4332ad48fd75SYan, Zheng leaf = path->nodes[0]; 4333ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4334ad48fd75SYan, Zheng 4335ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 4336ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 4337ad48fd75SYan, Zheng 4338e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 4339ad48fd75SYan, Zheng return 0; 4340ad48fd75SYan, Zheng 4341ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4342ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4343ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4344ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4345ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 4346ad48fd75SYan, Zheng } 4347b3b4aa74SDavid Sterba btrfs_release_path(path); 4348ad48fd75SYan, Zheng 4349ad48fd75SYan, Zheng path->keep_locks = 1; 4350ad48fd75SYan, Zheng path->search_for_split = 1; 4351ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 4352ad48fd75SYan, Zheng path->search_for_split = 0; 4353a8df6fe6SFilipe Manana if (ret > 0) 4354a8df6fe6SFilipe Manana ret = -EAGAIN; 4355ad48fd75SYan, Zheng if (ret < 0) 4356ad48fd75SYan, Zheng goto err; 4357ad48fd75SYan, Zheng 4358ad48fd75SYan, Zheng ret = -EAGAIN; 4359ad48fd75SYan, Zheng leaf = path->nodes[0]; 4360a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 4361a8df6fe6SFilipe Manana if (item_size != btrfs_item_size_nr(leaf, path->slots[0])) 4362ad48fd75SYan, Zheng goto err; 4363ad48fd75SYan, Zheng 4364109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 4365e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 4366109f6aefSChris Mason goto err; 4367109f6aefSChris Mason 4368ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4369ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4370ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4371ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 4372ad48fd75SYan, Zheng goto err; 4373ad48fd75SYan, Zheng } 4374ad48fd75SYan, Zheng 4375ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 4376f0486c68SYan, Zheng if (ret) 4377f0486c68SYan, Zheng goto err; 4378ad48fd75SYan, Zheng 4379ad48fd75SYan, Zheng path->keep_locks = 0; 4380ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 4381ad48fd75SYan, Zheng return 0; 4382ad48fd75SYan, Zheng err: 4383ad48fd75SYan, Zheng path->keep_locks = 0; 4384ad48fd75SYan, Zheng return ret; 4385ad48fd75SYan, Zheng } 4386ad48fd75SYan, Zheng 438725263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 4388310712b2SOmar Sandoval const struct btrfs_key *new_key, 4389459931ecSChris Mason unsigned long split_offset) 4390459931ecSChris Mason { 4391459931ecSChris Mason struct extent_buffer *leaf; 4392459931ecSChris Mason struct btrfs_item *item; 4393459931ecSChris Mason struct btrfs_item *new_item; 4394459931ecSChris Mason int slot; 4395ad48fd75SYan, Zheng char *buf; 4396459931ecSChris Mason u32 nritems; 4397ad48fd75SYan, Zheng u32 item_size; 4398459931ecSChris Mason u32 orig_offset; 4399459931ecSChris Mason struct btrfs_disk_key disk_key; 4400459931ecSChris Mason 4401459931ecSChris Mason leaf = path->nodes[0]; 4402e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)); 4403b9473439SChris Mason 4404dd3cc16bSRoss Kirk item = btrfs_item_nr(path->slots[0]); 4405459931ecSChris Mason orig_offset = btrfs_item_offset(leaf, item); 4406459931ecSChris Mason item_size = btrfs_item_size(leaf, item); 4407459931ecSChris Mason 4408459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 4409ad48fd75SYan, Zheng if (!buf) 4410ad48fd75SYan, Zheng return -ENOMEM; 4411ad48fd75SYan, Zheng 4412459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 4413459931ecSChris Mason path->slots[0]), item_size); 4414ad48fd75SYan, Zheng 4415459931ecSChris Mason slot = path->slots[0] + 1; 4416459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 4417459931ecSChris Mason if (slot != nritems) { 4418459931ecSChris Mason /* shift the items */ 4419459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 4420459931ecSChris Mason btrfs_item_nr_offset(slot), 4421459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4422459931ecSChris Mason } 4423459931ecSChris Mason 4424459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 4425459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4426459931ecSChris Mason 4427dd3cc16bSRoss Kirk new_item = btrfs_item_nr(slot); 4428459931ecSChris Mason 4429459931ecSChris Mason btrfs_set_item_offset(leaf, new_item, orig_offset); 4430459931ecSChris Mason btrfs_set_item_size(leaf, new_item, item_size - split_offset); 4431459931ecSChris Mason 4432459931ecSChris Mason btrfs_set_item_offset(leaf, item, 4433459931ecSChris Mason orig_offset + item_size - split_offset); 4434459931ecSChris Mason btrfs_set_item_size(leaf, item, split_offset); 4435459931ecSChris Mason 4436459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 4437459931ecSChris Mason 4438459931ecSChris Mason /* write the data for the start of the original item */ 4439459931ecSChris Mason write_extent_buffer(leaf, buf, 4440459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 4441459931ecSChris Mason split_offset); 4442459931ecSChris Mason 4443459931ecSChris Mason /* write the data for the new item */ 4444459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 4445459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 4446459931ecSChris Mason item_size - split_offset); 4447459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 4448459931ecSChris Mason 4449e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 4450459931ecSChris Mason kfree(buf); 4451ad48fd75SYan, Zheng return 0; 4452ad48fd75SYan, Zheng } 4453ad48fd75SYan, Zheng 4454ad48fd75SYan, Zheng /* 4455ad48fd75SYan, Zheng * This function splits a single item into two items, 4456ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 4457ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 4458ad48fd75SYan, Zheng * 4459ad48fd75SYan, Zheng * The path may be released by this operation. After 4460ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 4461ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 4462ad48fd75SYan, Zheng * 4463ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 4464ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 4465ad48fd75SYan, Zheng * 4466ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 4467ad48fd75SYan, Zheng * leaf the entire time. 4468ad48fd75SYan, Zheng */ 4469ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 4470ad48fd75SYan, Zheng struct btrfs_root *root, 4471ad48fd75SYan, Zheng struct btrfs_path *path, 4472310712b2SOmar Sandoval const struct btrfs_key *new_key, 4473ad48fd75SYan, Zheng unsigned long split_offset) 4474ad48fd75SYan, Zheng { 4475ad48fd75SYan, Zheng int ret; 4476ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4477ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 4478ad48fd75SYan, Zheng if (ret) 4479459931ecSChris Mason return ret; 4480ad48fd75SYan, Zheng 448125263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 4482ad48fd75SYan, Zheng return ret; 4483ad48fd75SYan, Zheng } 4484ad48fd75SYan, Zheng 4485ad48fd75SYan, Zheng /* 4486ad48fd75SYan, Zheng * This function duplicate a item, giving 'new_key' to the new item. 4487ad48fd75SYan, Zheng * It guarantees both items live in the same tree leaf and the new item 4488ad48fd75SYan, Zheng * is contiguous with the original item. 4489ad48fd75SYan, Zheng * 4490ad48fd75SYan, Zheng * This allows us to split file extent in place, keeping a lock on the 4491ad48fd75SYan, Zheng * leaf the entire time. 4492ad48fd75SYan, Zheng */ 4493ad48fd75SYan, Zheng int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4494ad48fd75SYan, Zheng struct btrfs_root *root, 4495ad48fd75SYan, Zheng struct btrfs_path *path, 4496310712b2SOmar Sandoval const struct btrfs_key *new_key) 4497ad48fd75SYan, Zheng { 4498ad48fd75SYan, Zheng struct extent_buffer *leaf; 4499ad48fd75SYan, Zheng int ret; 4500ad48fd75SYan, Zheng u32 item_size; 4501ad48fd75SYan, Zheng 4502ad48fd75SYan, Zheng leaf = path->nodes[0]; 4503ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4504ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4505ad48fd75SYan, Zheng item_size + sizeof(struct btrfs_item)); 4506ad48fd75SYan, Zheng if (ret) 4507ad48fd75SYan, Zheng return ret; 4508ad48fd75SYan, Zheng 4509ad48fd75SYan, Zheng path->slots[0]++; 4510fc0d82e1SNikolay Borisov setup_items_for_insert(root, path, new_key, &item_size, 1); 4511ad48fd75SYan, Zheng leaf = path->nodes[0]; 4512ad48fd75SYan, Zheng memcpy_extent_buffer(leaf, 4513ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0]), 4514ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4515ad48fd75SYan, Zheng item_size); 4516ad48fd75SYan, Zheng return 0; 4517459931ecSChris Mason } 4518459931ecSChris Mason 4519459931ecSChris Mason /* 4520d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4521d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4522d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4523d352ac68SChris Mason * the front. 4524d352ac68SChris Mason */ 452578ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 4526b18c6685SChris Mason { 4527b18c6685SChris Mason int slot; 45285f39d397SChris Mason struct extent_buffer *leaf; 45295f39d397SChris Mason struct btrfs_item *item; 4530b18c6685SChris Mason u32 nritems; 4531b18c6685SChris Mason unsigned int data_end; 4532b18c6685SChris Mason unsigned int old_data_start; 4533b18c6685SChris Mason unsigned int old_size; 4534b18c6685SChris Mason unsigned int size_diff; 4535b18c6685SChris Mason int i; 4536cfed81a0SChris Mason struct btrfs_map_token token; 4537cfed81a0SChris Mason 45385f39d397SChris Mason leaf = path->nodes[0]; 4539179e29e4SChris Mason slot = path->slots[0]; 4540179e29e4SChris Mason 4541179e29e4SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4542179e29e4SChris Mason if (old_size == new_size) 4543143bede5SJeff Mahoney return; 4544b18c6685SChris Mason 45455f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 45468f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4547b18c6685SChris Mason 45485f39d397SChris Mason old_data_start = btrfs_item_offset_nr(leaf, slot); 4549179e29e4SChris Mason 4550b18c6685SChris Mason size_diff = old_size - new_size; 4551b18c6685SChris Mason 4552b18c6685SChris Mason BUG_ON(slot < 0); 4553b18c6685SChris Mason BUG_ON(slot >= nritems); 4554b18c6685SChris Mason 4555b18c6685SChris Mason /* 4556b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4557b18c6685SChris Mason */ 4558b18c6685SChris Mason /* first correct the data pointers */ 4559c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4560b18c6685SChris Mason for (i = slot; i < nritems; i++) { 45615f39d397SChris Mason u32 ioff; 4562dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4563db94535dSChris Mason 4564cc4c13d5SDavid Sterba ioff = btrfs_token_item_offset(&token, item); 4565cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, ioff + size_diff); 4566b18c6685SChris Mason } 4567db94535dSChris Mason 4568b18c6685SChris Mason /* shift the data */ 4569179e29e4SChris Mason if (from_end) { 45703d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 45713d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 4572b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 4573179e29e4SChris Mason } else { 4574179e29e4SChris Mason struct btrfs_disk_key disk_key; 4575179e29e4SChris Mason u64 offset; 4576179e29e4SChris Mason 4577179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4578179e29e4SChris Mason 4579179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4580179e29e4SChris Mason unsigned long ptr; 4581179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4582179e29e4SChris Mason 4583179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4584179e29e4SChris Mason struct btrfs_file_extent_item); 4585179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4586179e29e4SChris Mason (unsigned long)fi - size_diff); 4587179e29e4SChris Mason 4588179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4589179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4590179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4591179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4592179e29e4SChris Mason (unsigned long)fi, 45937ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 4594179e29e4SChris Mason } 4595179e29e4SChris Mason } 4596179e29e4SChris Mason 45973d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 45983d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 4599179e29e4SChris Mason data_end, old_data_start - data_end); 4600179e29e4SChris Mason 4601179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4602179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4603179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4604179e29e4SChris Mason if (slot == 0) 4605b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4606179e29e4SChris Mason } 46075f39d397SChris Mason 4608dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 46095f39d397SChris Mason btrfs_set_item_size(leaf, item, new_size); 46105f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4611b18c6685SChris Mason 4612e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4613a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4614b18c6685SChris Mason BUG(); 46155f39d397SChris Mason } 4616b18c6685SChris Mason } 4617b18c6685SChris Mason 4618d352ac68SChris Mason /* 46198f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4620d352ac68SChris Mason */ 4621c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 46226567e837SChris Mason { 46236567e837SChris Mason int slot; 46245f39d397SChris Mason struct extent_buffer *leaf; 46255f39d397SChris Mason struct btrfs_item *item; 46266567e837SChris Mason u32 nritems; 46276567e837SChris Mason unsigned int data_end; 46286567e837SChris Mason unsigned int old_data; 46296567e837SChris Mason unsigned int old_size; 46306567e837SChris Mason int i; 4631cfed81a0SChris Mason struct btrfs_map_token token; 4632cfed81a0SChris Mason 46335f39d397SChris Mason leaf = path->nodes[0]; 46346567e837SChris Mason 46355f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 46368f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 46376567e837SChris Mason 4638e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 4639a4f78750SDavid Sterba btrfs_print_leaf(leaf); 46406567e837SChris Mason BUG(); 46415f39d397SChris Mason } 46426567e837SChris Mason slot = path->slots[0]; 46435f39d397SChris Mason old_data = btrfs_item_end_nr(leaf, slot); 46446567e837SChris Mason 46456567e837SChris Mason BUG_ON(slot < 0); 46463326d1b0SChris Mason if (slot >= nritems) { 4647a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4648c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 4649d397712bSChris Mason slot, nritems); 4650290342f6SArnd Bergmann BUG(); 46513326d1b0SChris Mason } 46526567e837SChris Mason 46536567e837SChris Mason /* 46546567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 46556567e837SChris Mason */ 46566567e837SChris Mason /* first correct the data pointers */ 4657c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 46586567e837SChris Mason for (i = slot; i < nritems; i++) { 46595f39d397SChris Mason u32 ioff; 4660dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4661db94535dSChris Mason 4662cc4c13d5SDavid Sterba ioff = btrfs_token_item_offset(&token, item); 4663cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, ioff - data_size); 46646567e837SChris Mason } 46655f39d397SChris Mason 46666567e837SChris Mason /* shift the data */ 46673d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 46683d9ec8c4SNikolay Borisov data_end - data_size, BTRFS_LEAF_DATA_OFFSET + 46696567e837SChris Mason data_end, old_data - data_end); 46705f39d397SChris Mason 46716567e837SChris Mason data_end = old_data; 46725f39d397SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4673dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 46745f39d397SChris Mason btrfs_set_item_size(leaf, item, old_size + data_size); 46755f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 46766567e837SChris Mason 4677e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4678a4f78750SDavid Sterba btrfs_print_leaf(leaf); 46796567e837SChris Mason BUG(); 46805f39d397SChris Mason } 46816567e837SChris Mason } 46826567e837SChris Mason 4683da9ffb24SNikolay Borisov /** 4684da9ffb24SNikolay Borisov * setup_items_for_insert - Helper called before inserting one or more items 4685da9ffb24SNikolay Borisov * to a leaf. Main purpose is to save stack depth by doing the bulk of the work 4686da9ffb24SNikolay Borisov * in a function that doesn't call btrfs_search_slot 4687da9ffb24SNikolay Borisov * 4688da9ffb24SNikolay Borisov * @root: root we are inserting items to 4689da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 4690da9ffb24SNikolay Borisov * @cpu_key: array of keys for items to be inserted 4691da9ffb24SNikolay Borisov * @data_size: size of the body of each item we are going to insert 4692da9ffb24SNikolay Borisov * @nr: size of @cpu_key/@data_size arrays 469374123bd7SChris Mason */ 4694afe5fea7STsutomu Itoh void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4695310712b2SOmar Sandoval const struct btrfs_key *cpu_key, u32 *data_size, 4696fc0d82e1SNikolay Borisov int nr) 4697be0e5c09SChris Mason { 46980b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 46995f39d397SChris Mason struct btrfs_item *item; 47009c58309dSChris Mason int i; 47017518a238SChris Mason u32 nritems; 4702be0e5c09SChris Mason unsigned int data_end; 4703e2fa7227SChris Mason struct btrfs_disk_key disk_key; 470444871b1bSChris Mason struct extent_buffer *leaf; 470544871b1bSChris Mason int slot; 4706cfed81a0SChris Mason struct btrfs_map_token token; 4707fc0d82e1SNikolay Borisov u32 total_size; 4708fc0d82e1SNikolay Borisov u32 total_data = 0; 4709fc0d82e1SNikolay Borisov 4710fc0d82e1SNikolay Borisov for (i = 0; i < nr; i++) 4711fc0d82e1SNikolay Borisov total_data += data_size[i]; 4712fc0d82e1SNikolay Borisov total_size = total_data + (nr * sizeof(struct btrfs_item)); 4713cfed81a0SChris Mason 471424cdc847SFilipe Manana if (path->slots[0] == 0) { 471524cdc847SFilipe Manana btrfs_cpu_key_to_disk(&disk_key, cpu_key); 4716b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 471724cdc847SFilipe Manana } 471824cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 471924cdc847SFilipe Manana 47205f39d397SChris Mason leaf = path->nodes[0]; 472144871b1bSChris Mason slot = path->slots[0]; 472274123bd7SChris Mason 47235f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 47248f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4725eb60ceacSChris Mason 4726e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 4727a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47280b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 4729e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 4730be0e5c09SChris Mason BUG(); 4731d4dbff95SChris Mason } 47325f39d397SChris Mason 4733c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4734be0e5c09SChris Mason if (slot != nritems) { 47355f39d397SChris Mason unsigned int old_data = btrfs_item_end_nr(leaf, slot); 4736be0e5c09SChris Mason 47375f39d397SChris Mason if (old_data < data_end) { 4738a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47397269ddd2SNikolay Borisov btrfs_crit(fs_info, 47407269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 47415f39d397SChris Mason slot, old_data, data_end); 4742290342f6SArnd Bergmann BUG(); 47435f39d397SChris Mason } 4744be0e5c09SChris Mason /* 4745be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4746be0e5c09SChris Mason */ 4747be0e5c09SChris Mason /* first correct the data pointers */ 47480783fcfcSChris Mason for (i = slot; i < nritems; i++) { 47495f39d397SChris Mason u32 ioff; 4750db94535dSChris Mason 4751dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4752cc4c13d5SDavid Sterba ioff = btrfs_token_item_offset(&token, item); 4753cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, 4754cc4c13d5SDavid Sterba ioff - total_data); 47550783fcfcSChris Mason } 4756be0e5c09SChris Mason /* shift the items */ 47579c58309dSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), 47585f39d397SChris Mason btrfs_item_nr_offset(slot), 47590783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4760be0e5c09SChris Mason 4761be0e5c09SChris Mason /* shift the data */ 47623d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 47633d9ec8c4SNikolay Borisov data_end - total_data, BTRFS_LEAF_DATA_OFFSET + 4764be0e5c09SChris Mason data_end, old_data - data_end); 4765be0e5c09SChris Mason data_end = old_data; 4766be0e5c09SChris Mason } 47675f39d397SChris Mason 476862e2749eSChris Mason /* setup the item for the new data */ 47699c58309dSChris Mason for (i = 0; i < nr; i++) { 47709c58309dSChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); 47719c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4772dd3cc16bSRoss Kirk item = btrfs_item_nr(slot + i); 47739c58309dSChris Mason data_end -= data_size[i]; 4774fc0716c2SNikolay Borisov btrfs_set_token_item_offset(&token, item, data_end); 4775cc4c13d5SDavid Sterba btrfs_set_token_item_size(&token, item, data_size[i]); 47769c58309dSChris Mason } 477744871b1bSChris Mason 47789c58309dSChris Mason btrfs_set_header_nritems(leaf, nritems + nr); 4779b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4780aa5d6bedSChris Mason 4781e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4782a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4783be0e5c09SChris Mason BUG(); 47845f39d397SChris Mason } 478544871b1bSChris Mason } 478644871b1bSChris Mason 478744871b1bSChris Mason /* 478844871b1bSChris Mason * Given a key and some data, insert items into the tree. 478944871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 479044871b1bSChris Mason */ 479144871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 479244871b1bSChris Mason struct btrfs_root *root, 479344871b1bSChris Mason struct btrfs_path *path, 4794310712b2SOmar Sandoval const struct btrfs_key *cpu_key, u32 *data_size, 479544871b1bSChris Mason int nr) 479644871b1bSChris Mason { 479744871b1bSChris Mason int ret = 0; 479844871b1bSChris Mason int slot; 479944871b1bSChris Mason int i; 480044871b1bSChris Mason u32 total_size = 0; 480144871b1bSChris Mason u32 total_data = 0; 480244871b1bSChris Mason 480344871b1bSChris Mason for (i = 0; i < nr; i++) 480444871b1bSChris Mason total_data += data_size[i]; 480544871b1bSChris Mason 480644871b1bSChris Mason total_size = total_data + (nr * sizeof(struct btrfs_item)); 480744871b1bSChris Mason ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1); 480844871b1bSChris Mason if (ret == 0) 480944871b1bSChris Mason return -EEXIST; 481044871b1bSChris Mason if (ret < 0) 4811143bede5SJeff Mahoney return ret; 481244871b1bSChris Mason 481344871b1bSChris Mason slot = path->slots[0]; 481444871b1bSChris Mason BUG_ON(slot < 0); 481544871b1bSChris Mason 4816fc0d82e1SNikolay Borisov setup_items_for_insert(root, path, cpu_key, data_size, nr); 4817143bede5SJeff Mahoney return 0; 481862e2749eSChris Mason } 481962e2749eSChris Mason 482062e2749eSChris Mason /* 482162e2749eSChris Mason * Given a key and some data, insert an item into the tree. 482262e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 482362e2749eSChris Mason */ 4824310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4825310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4826310712b2SOmar Sandoval u32 data_size) 482762e2749eSChris Mason { 482862e2749eSChris Mason int ret = 0; 48292c90e5d6SChris Mason struct btrfs_path *path; 48305f39d397SChris Mason struct extent_buffer *leaf; 48315f39d397SChris Mason unsigned long ptr; 483262e2749eSChris Mason 48332c90e5d6SChris Mason path = btrfs_alloc_path(); 4834db5b493aSTsutomu Itoh if (!path) 4835db5b493aSTsutomu Itoh return -ENOMEM; 48362c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 483762e2749eSChris Mason if (!ret) { 48385f39d397SChris Mason leaf = path->nodes[0]; 48395f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 48405f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 48415f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 484262e2749eSChris Mason } 48432c90e5d6SChris Mason btrfs_free_path(path); 4844aa5d6bedSChris Mason return ret; 4845be0e5c09SChris Mason } 4846be0e5c09SChris Mason 484774123bd7SChris Mason /* 48485de08d7dSChris Mason * delete the pointer from a given node. 484974123bd7SChris Mason * 4850d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4851d352ac68SChris Mason * empty a node. 485274123bd7SChris Mason */ 4853afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4854afe5fea7STsutomu Itoh int level, int slot) 4855be0e5c09SChris Mason { 48565f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 48577518a238SChris Mason u32 nritems; 4858f3ea38daSJan Schmidt int ret; 4859be0e5c09SChris Mason 48605f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4861be0e5c09SChris Mason if (slot != nritems - 1) { 4862bf1d3425SDavid Sterba if (level) { 4863bf1d3425SDavid Sterba ret = tree_mod_log_insert_move(parent, slot, slot + 1, 4864a446a979SDavid Sterba nritems - slot - 1); 4865bf1d3425SDavid Sterba BUG_ON(ret < 0); 4866bf1d3425SDavid Sterba } 48675f39d397SChris Mason memmove_extent_buffer(parent, 48685f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 48695f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4870d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4871d6025579SChris Mason (nritems - slot - 1)); 487257ba86c0SChris Mason } else if (level) { 4873e09c2efeSDavid Sterba ret = tree_mod_log_insert_key(parent, slot, MOD_LOG_KEY_REMOVE, 4874e09c2efeSDavid Sterba GFP_NOFS); 487557ba86c0SChris Mason BUG_ON(ret < 0); 4876be0e5c09SChris Mason } 4877f3ea38daSJan Schmidt 48787518a238SChris Mason nritems--; 48795f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 48807518a238SChris Mason if (nritems == 0 && parent == root->node) { 48815f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4882eb60ceacSChris Mason /* just turn the root into a leaf and break */ 48835f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4884bb803951SChris Mason } else if (slot == 0) { 48855f39d397SChris Mason struct btrfs_disk_key disk_key; 48865f39d397SChris Mason 48875f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4888b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4889be0e5c09SChris Mason } 4890d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4891be0e5c09SChris Mason } 4892be0e5c09SChris Mason 489374123bd7SChris Mason /* 4894323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 48955d4f98a2SYan Zheng * path->nodes[1]. 4896323ac95bSChris Mason * 4897323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4898323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4899323ac95bSChris Mason * 4900323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4901323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4902323ac95bSChris Mason */ 4903143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4904323ac95bSChris Mason struct btrfs_root *root, 49055d4f98a2SYan Zheng struct btrfs_path *path, 49065d4f98a2SYan Zheng struct extent_buffer *leaf) 4907323ac95bSChris Mason { 49085d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4909afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4910323ac95bSChris Mason 49114d081c41SChris Mason /* 49124d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 49134d081c41SChris Mason * aren't holding any locks when we call it 49144d081c41SChris Mason */ 49154d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 49164d081c41SChris Mason 4917f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4918f0486c68SYan, Zheng 491967439dadSDavid Sterba atomic_inc(&leaf->refs); 49205581a51aSJan Schmidt btrfs_free_tree_block(trans, root, leaf, 0, 1); 49213083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4922323ac95bSChris Mason } 4923323ac95bSChris Mason /* 492474123bd7SChris Mason * delete the item at the leaf level in path. If that empties 492574123bd7SChris Mason * the leaf, remove it from the tree 492674123bd7SChris Mason */ 492785e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 492885e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4929be0e5c09SChris Mason { 49300b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 49315f39d397SChris Mason struct extent_buffer *leaf; 49325f39d397SChris Mason struct btrfs_item *item; 4933ce0eac2aSAlexandru Moise u32 last_off; 4934ce0eac2aSAlexandru Moise u32 dsize = 0; 4935aa5d6bedSChris Mason int ret = 0; 4936aa5d6bedSChris Mason int wret; 493785e21bacSChris Mason int i; 49387518a238SChris Mason u32 nritems; 4939be0e5c09SChris Mason 49405f39d397SChris Mason leaf = path->nodes[0]; 494185e21bacSChris Mason last_off = btrfs_item_offset_nr(leaf, slot + nr - 1); 494285e21bacSChris Mason 494385e21bacSChris Mason for (i = 0; i < nr; i++) 494485e21bacSChris Mason dsize += btrfs_item_size_nr(leaf, slot + i); 494585e21bacSChris Mason 49465f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4947be0e5c09SChris Mason 494885e21bacSChris Mason if (slot + nr != nritems) { 49498f881e8cSDavid Sterba int data_end = leaf_data_end(leaf); 4950c82f823cSDavid Sterba struct btrfs_map_token token; 49515f39d397SChris Mason 49523d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 4953d6025579SChris Mason data_end + dsize, 49543d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 495585e21bacSChris Mason last_off - data_end); 49565f39d397SChris Mason 4957c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 495885e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 49595f39d397SChris Mason u32 ioff; 4960db94535dSChris Mason 4961dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4962cc4c13d5SDavid Sterba ioff = btrfs_token_item_offset(&token, item); 4963cc4c13d5SDavid Sterba btrfs_set_token_item_offset(&token, item, ioff + dsize); 49640783fcfcSChris Mason } 4965db94535dSChris Mason 49665f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 496785e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 49680783fcfcSChris Mason sizeof(struct btrfs_item) * 496985e21bacSChris Mason (nritems - slot - nr)); 4970be0e5c09SChris Mason } 497185e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 497285e21bacSChris Mason nritems -= nr; 49735f39d397SChris Mason 497474123bd7SChris Mason /* delete the leaf if we've emptied it */ 49757518a238SChris Mason if (nritems == 0) { 49765f39d397SChris Mason if (leaf == root->node) { 49775f39d397SChris Mason btrfs_set_header_level(leaf, 0); 49789a8dd150SChris Mason } else { 49796a884d7dSDavid Sterba btrfs_clean_tree_block(leaf); 4980143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 49819a8dd150SChris Mason } 4982be0e5c09SChris Mason } else { 49837518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4984aa5d6bedSChris Mason if (slot == 0) { 49855f39d397SChris Mason struct btrfs_disk_key disk_key; 49865f39d397SChris Mason 49875f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4988b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4989aa5d6bedSChris Mason } 4990aa5d6bedSChris Mason 499174123bd7SChris Mason /* delete the leaf if it is mostly empty */ 49920b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 4993be0e5c09SChris Mason /* push_leaf_left fixes the path. 4994be0e5c09SChris Mason * make sure the path still points to our leaf 4995be0e5c09SChris Mason * for possible call to del_ptr below 4996be0e5c09SChris Mason */ 49974920c9acSChris Mason slot = path->slots[1]; 499867439dadSDavid Sterba atomic_inc(&leaf->refs); 49995f39d397SChris Mason 500099d8f83cSChris Mason wret = push_leaf_left(trans, root, path, 1, 1, 500199d8f83cSChris Mason 1, (u32)-1); 500254aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 5003aa5d6bedSChris Mason ret = wret; 50045f39d397SChris Mason 50055f39d397SChris Mason if (path->nodes[0] == leaf && 50065f39d397SChris Mason btrfs_header_nritems(leaf)) { 500799d8f83cSChris Mason wret = push_leaf_right(trans, root, path, 1, 500899d8f83cSChris Mason 1, 1, 0); 500954aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 5010aa5d6bedSChris Mason ret = wret; 5011aa5d6bedSChris Mason } 50125f39d397SChris Mason 50135f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 5014323ac95bSChris Mason path->slots[1] = slot; 5015143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 50165f39d397SChris Mason free_extent_buffer(leaf); 5017143bede5SJeff Mahoney ret = 0; 50185de08d7dSChris Mason } else { 5019925baeddSChris Mason /* if we're still in the path, make sure 5020925baeddSChris Mason * we're dirty. Otherwise, one of the 5021925baeddSChris Mason * push_leaf functions must have already 5022925baeddSChris Mason * dirtied this buffer 5023925baeddSChris Mason */ 5024925baeddSChris Mason if (path->nodes[0] == leaf) 50255f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 50265f39d397SChris Mason free_extent_buffer(leaf); 5027be0e5c09SChris Mason } 5028d5719762SChris Mason } else { 50295f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 5030be0e5c09SChris Mason } 50319a8dd150SChris Mason } 5032aa5d6bedSChris Mason return ret; 50339a8dd150SChris Mason } 50349a8dd150SChris Mason 503597571fd0SChris Mason /* 5036925baeddSChris Mason * search the tree again to find a leaf with lesser keys 50377bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 50387bb86316SChris Mason * returns < 0 on io errors. 5039d352ac68SChris Mason * 5040d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 5041d352ac68SChris Mason * time you call it. 50427bb86316SChris Mason */ 504316e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 50447bb86316SChris Mason { 5045925baeddSChris Mason struct btrfs_key key; 5046925baeddSChris Mason struct btrfs_disk_key found_key; 5047925baeddSChris Mason int ret; 50487bb86316SChris Mason 5049925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 5050925baeddSChris Mason 5051e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 5052925baeddSChris Mason key.offset--; 5053e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 5054925baeddSChris Mason key.type--; 5055e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5056e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 5057925baeddSChris Mason key.objectid--; 5058e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 5059e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5060e8b0d724SFilipe David Borba Manana } else { 50617bb86316SChris Mason return 1; 5062e8b0d724SFilipe David Borba Manana } 50637bb86316SChris Mason 5064b3b4aa74SDavid Sterba btrfs_release_path(path); 5065925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5066925baeddSChris Mason if (ret < 0) 5067925baeddSChris Mason return ret; 5068925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 5069925baeddSChris Mason ret = comp_keys(&found_key, &key); 5070337c6f68SFilipe Manana /* 5071337c6f68SFilipe Manana * We might have had an item with the previous key in the tree right 5072337c6f68SFilipe Manana * before we released our path. And after we released our path, that 5073337c6f68SFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 5074337c6f68SFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 5075337c6f68SFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 5076337c6f68SFilipe Manana * Therefore account for these 2 cases, so that our callers (like 5077337c6f68SFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 5078337c6f68SFilipe Manana * the previous key we computed above. 5079337c6f68SFilipe Manana */ 5080337c6f68SFilipe Manana if (ret <= 0) 50817bb86316SChris Mason return 0; 5082925baeddSChris Mason return 1; 50837bb86316SChris Mason } 50847bb86316SChris Mason 50853f157a2fSChris Mason /* 50863f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 5087de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 5088de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 50893f157a2fSChris Mason * 50903f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 50913f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 50923f157a2fSChris Mason * key and get a writable path. 50933f157a2fSChris Mason * 50943f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 50953f157a2fSChris Mason * of the tree. 50963f157a2fSChris Mason * 5097d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 5098d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 5099d352ac68SChris Mason * skipped over (without reading them). 5100d352ac68SChris Mason * 51013f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 51023f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 51033f157a2fSChris Mason */ 51043f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 5105de78b51aSEric Sandeen struct btrfs_path *path, 51063f157a2fSChris Mason u64 min_trans) 51073f157a2fSChris Mason { 51083f157a2fSChris Mason struct extent_buffer *cur; 51093f157a2fSChris Mason struct btrfs_key found_key; 51103f157a2fSChris Mason int slot; 51119652480bSYan int sret; 51123f157a2fSChris Mason u32 nritems; 51133f157a2fSChris Mason int level; 51143f157a2fSChris Mason int ret = 1; 5115f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 51163f157a2fSChris Mason 5117f98de9b9SFilipe Manana path->keep_locks = 1; 51183f157a2fSChris Mason again: 5119bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 51203f157a2fSChris Mason level = btrfs_header_level(cur); 5121e02119d5SChris Mason WARN_ON(path->nodes[level]); 51223f157a2fSChris Mason path->nodes[level] = cur; 5123bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 51243f157a2fSChris Mason 51253f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 51263f157a2fSChris Mason ret = 1; 51273f157a2fSChris Mason goto out; 51283f157a2fSChris Mason } 51293f157a2fSChris Mason while (1) { 51303f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 51313f157a2fSChris Mason level = btrfs_header_level(cur); 5132e3b83361SQu Wenruo sret = btrfs_bin_search(cur, min_key, &slot); 5133cbca7d59SFilipe Manana if (sret < 0) { 5134cbca7d59SFilipe Manana ret = sret; 5135cbca7d59SFilipe Manana goto out; 5136cbca7d59SFilipe Manana } 51373f157a2fSChris Mason 5138323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 5139323ac95bSChris Mason if (level == path->lowest_level) { 5140e02119d5SChris Mason if (slot >= nritems) 5141e02119d5SChris Mason goto find_next_key; 51423f157a2fSChris Mason ret = 0; 51433f157a2fSChris Mason path->slots[level] = slot; 51443f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 51453f157a2fSChris Mason goto out; 51463f157a2fSChris Mason } 51479652480bSYan if (sret && slot > 0) 51489652480bSYan slot--; 51493f157a2fSChris Mason /* 5150de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 5151260db43cSRandy Dunlap * If it is too old, skip to the next one. 51523f157a2fSChris Mason */ 51533f157a2fSChris Mason while (slot < nritems) { 51543f157a2fSChris Mason u64 gen; 5155e02119d5SChris Mason 51563f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 51573f157a2fSChris Mason if (gen < min_trans) { 51583f157a2fSChris Mason slot++; 51593f157a2fSChris Mason continue; 51603f157a2fSChris Mason } 51613f157a2fSChris Mason break; 51623f157a2fSChris Mason } 5163e02119d5SChris Mason find_next_key: 51643f157a2fSChris Mason /* 51653f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 51663f157a2fSChris Mason * and find another one 51673f157a2fSChris Mason */ 51683f157a2fSChris Mason if (slot >= nritems) { 5169e02119d5SChris Mason path->slots[level] = slot; 5170e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 5171de78b51aSEric Sandeen min_trans); 5172e02119d5SChris Mason if (sret == 0) { 5173b3b4aa74SDavid Sterba btrfs_release_path(path); 51743f157a2fSChris Mason goto again; 51753f157a2fSChris Mason } else { 51763f157a2fSChris Mason goto out; 51773f157a2fSChris Mason } 51783f157a2fSChris Mason } 51793f157a2fSChris Mason /* save our key for returning back */ 51803f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 51813f157a2fSChris Mason path->slots[level] = slot; 51823f157a2fSChris Mason if (level == path->lowest_level) { 51833f157a2fSChris Mason ret = 0; 51843f157a2fSChris Mason goto out; 51853f157a2fSChris Mason } 51864b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 5187fb770ae4SLiu Bo if (IS_ERR(cur)) { 5188fb770ae4SLiu Bo ret = PTR_ERR(cur); 5189fb770ae4SLiu Bo goto out; 5190fb770ae4SLiu Bo } 51913f157a2fSChris Mason 5192bd681513SChris Mason btrfs_tree_read_lock(cur); 5193b4ce94deSChris Mason 5194bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 51953f157a2fSChris Mason path->nodes[level - 1] = cur; 5196f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 51973f157a2fSChris Mason } 51983f157a2fSChris Mason out: 5199f98de9b9SFilipe Manana path->keep_locks = keep_locks; 5200f98de9b9SFilipe Manana if (ret == 0) { 5201f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 5202f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 5203f98de9b9SFilipe Manana } 52043f157a2fSChris Mason return ret; 52053f157a2fSChris Mason } 52063f157a2fSChris Mason 52073f157a2fSChris Mason /* 52083f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 52093f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 5210de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 52113f157a2fSChris Mason * 52123f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 52133f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 52143f157a2fSChris Mason * 52153f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 52163f157a2fSChris Mason * calling this function. 52173f157a2fSChris Mason */ 5218e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 5219de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 5220e7a84565SChris Mason { 5221e7a84565SChris Mason int slot; 5222e7a84565SChris Mason struct extent_buffer *c; 5223e7a84565SChris Mason 52246a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 5225e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 5226e7a84565SChris Mason if (!path->nodes[level]) 5227e7a84565SChris Mason return 1; 5228e7a84565SChris Mason 5229e7a84565SChris Mason slot = path->slots[level] + 1; 5230e7a84565SChris Mason c = path->nodes[level]; 52313f157a2fSChris Mason next: 5232e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 523333c66f43SYan Zheng int ret; 523433c66f43SYan Zheng int orig_lowest; 523533c66f43SYan Zheng struct btrfs_key cur_key; 523633c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 523733c66f43SYan Zheng !path->nodes[level + 1]) 5238e7a84565SChris Mason return 1; 523933c66f43SYan Zheng 52406a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 524133c66f43SYan Zheng level++; 5242e7a84565SChris Mason continue; 5243e7a84565SChris Mason } 524433c66f43SYan Zheng 524533c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 524633c66f43SYan Zheng if (level == 0) 524733c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 524833c66f43SYan Zheng else 524933c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 525033c66f43SYan Zheng 525133c66f43SYan Zheng orig_lowest = path->lowest_level; 5252b3b4aa74SDavid Sterba btrfs_release_path(path); 525333c66f43SYan Zheng path->lowest_level = level; 525433c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 525533c66f43SYan Zheng 0, 0); 525633c66f43SYan Zheng path->lowest_level = orig_lowest; 525733c66f43SYan Zheng if (ret < 0) 525833c66f43SYan Zheng return ret; 525933c66f43SYan Zheng 526033c66f43SYan Zheng c = path->nodes[level]; 526133c66f43SYan Zheng slot = path->slots[level]; 526233c66f43SYan Zheng if (ret == 0) 526333c66f43SYan Zheng slot++; 526433c66f43SYan Zheng goto next; 526533c66f43SYan Zheng } 526633c66f43SYan Zheng 5267e7a84565SChris Mason if (level == 0) 5268e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 52693f157a2fSChris Mason else { 52703f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 52713f157a2fSChris Mason 52723f157a2fSChris Mason if (gen < min_trans) { 52733f157a2fSChris Mason slot++; 52743f157a2fSChris Mason goto next; 52753f157a2fSChris Mason } 5276e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 52773f157a2fSChris Mason } 5278e7a84565SChris Mason return 0; 5279e7a84565SChris Mason } 5280e7a84565SChris Mason return 1; 5281e7a84565SChris Mason } 5282e7a84565SChris Mason 52837bb86316SChris Mason /* 5284925baeddSChris Mason * search the tree again to find a leaf with greater keys 52850f70abe2SChris Mason * returns 0 if it found something or 1 if there are no greater leaves. 52860f70abe2SChris Mason * returns < 0 on io errors. 528797571fd0SChris Mason */ 5288234b63a0SChris Mason int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path) 5289d97e63b6SChris Mason { 52903d7806ecSJan Schmidt return btrfs_next_old_leaf(root, path, 0); 52913d7806ecSJan Schmidt } 52923d7806ecSJan Schmidt 52933d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 52943d7806ecSJan Schmidt u64 time_seq) 52953d7806ecSJan Schmidt { 5296d97e63b6SChris Mason int slot; 52978e73f275SChris Mason int level; 52985f39d397SChris Mason struct extent_buffer *c; 52998e73f275SChris Mason struct extent_buffer *next; 5300925baeddSChris Mason struct btrfs_key key; 5301925baeddSChris Mason u32 nritems; 5302925baeddSChris Mason int ret; 5303bd681513SChris Mason int next_rw_lock = 0; 5304925baeddSChris Mason 5305925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5306d397712bSChris Mason if (nritems == 0) 5307925baeddSChris Mason return 1; 5308925baeddSChris Mason 53098e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 53108e73f275SChris Mason again: 53118e73f275SChris Mason level = 1; 53128e73f275SChris Mason next = NULL; 5313bd681513SChris Mason next_rw_lock = 0; 5314b3b4aa74SDavid Sterba btrfs_release_path(path); 53158e73f275SChris Mason 5316a2135011SChris Mason path->keep_locks = 1; 53178e73f275SChris Mason 53183d7806ecSJan Schmidt if (time_seq) 53193d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 53203d7806ecSJan Schmidt else 5321925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5322925baeddSChris Mason path->keep_locks = 0; 5323925baeddSChris Mason 5324925baeddSChris Mason if (ret < 0) 5325925baeddSChris Mason return ret; 5326925baeddSChris Mason 5327a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5328168fd7d2SChris Mason /* 5329168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 5330168fd7d2SChris Mason * could have added more items next to the key that used to be 5331168fd7d2SChris Mason * at the very end of the block. So, check again here and 5332168fd7d2SChris Mason * advance the path if there are now more items available. 5333168fd7d2SChris Mason */ 5334a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 5335e457afecSYan Zheng if (ret == 0) 5336168fd7d2SChris Mason path->slots[0]++; 53378e73f275SChris Mason ret = 0; 5338925baeddSChris Mason goto done; 5339925baeddSChris Mason } 53400b43e04fSLiu Bo /* 53410b43e04fSLiu Bo * So the above check misses one case: 53420b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 53430b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 53440b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 53450b43e04fSLiu Bo * 53460b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 53470b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 53480b43e04fSLiu Bo * 53490b43e04fSLiu Bo * And a bit more explanation about this check, 53500b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 53510b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 53520b43e04fSLiu Bo * bigger one. 53530b43e04fSLiu Bo */ 53540b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 53550b43e04fSLiu Bo ret = 0; 53560b43e04fSLiu Bo goto done; 53570b43e04fSLiu Bo } 5358d97e63b6SChris Mason 5359234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 53608e73f275SChris Mason if (!path->nodes[level]) { 53618e73f275SChris Mason ret = 1; 53628e73f275SChris Mason goto done; 53638e73f275SChris Mason } 53645f39d397SChris Mason 5365d97e63b6SChris Mason slot = path->slots[level] + 1; 5366d97e63b6SChris Mason c = path->nodes[level]; 53675f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 5368d97e63b6SChris Mason level++; 53698e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 53708e73f275SChris Mason ret = 1; 53718e73f275SChris Mason goto done; 53728e73f275SChris Mason } 5373d97e63b6SChris Mason continue; 5374d97e63b6SChris Mason } 53755f39d397SChris Mason 5376925baeddSChris Mason if (next) { 5377bd681513SChris Mason btrfs_tree_unlock_rw(next, next_rw_lock); 53785f39d397SChris Mason free_extent_buffer(next); 5379925baeddSChris Mason } 53805f39d397SChris Mason 53818e73f275SChris Mason next = c; 5382bd681513SChris Mason next_rw_lock = path->locks[level]; 5383d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5384cda79c54SDavid Sterba slot, &key); 53858e73f275SChris Mason if (ret == -EAGAIN) 53868e73f275SChris Mason goto again; 53875f39d397SChris Mason 538876a05b35SChris Mason if (ret < 0) { 5389b3b4aa74SDavid Sterba btrfs_release_path(path); 539076a05b35SChris Mason goto done; 539176a05b35SChris Mason } 539276a05b35SChris Mason 53935cd57b2cSChris Mason if (!path->skip_locking) { 5394bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 5395d42244a0SJan Schmidt if (!ret && time_seq) { 5396d42244a0SJan Schmidt /* 5397d42244a0SJan Schmidt * If we don't get the lock, we may be racing 5398d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 5399d42244a0SJan Schmidt * itself waiting for the leaf we've currently 5400d42244a0SJan Schmidt * locked. To solve this situation, we give up 5401d42244a0SJan Schmidt * on our lock and cycle. 5402d42244a0SJan Schmidt */ 5403cf538830SJan Schmidt free_extent_buffer(next); 5404d42244a0SJan Schmidt btrfs_release_path(path); 5405d42244a0SJan Schmidt cond_resched(); 5406d42244a0SJan Schmidt goto again; 5407d42244a0SJan Schmidt } 54088e73f275SChris Mason if (!ret) { 5409fd7ba1c1SJosef Bacik __btrfs_tree_read_lock(next, 5410bf77467aSJosef Bacik BTRFS_NESTING_RIGHT, 5411fd7ba1c1SJosef Bacik path->recurse); 54128e73f275SChris Mason } 5413bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5414bd681513SChris Mason } 5415d97e63b6SChris Mason break; 5416d97e63b6SChris Mason } 5417d97e63b6SChris Mason path->slots[level] = slot; 5418d97e63b6SChris Mason while (1) { 5419d97e63b6SChris Mason level--; 5420d97e63b6SChris Mason c = path->nodes[level]; 5421925baeddSChris Mason if (path->locks[level]) 5422bd681513SChris Mason btrfs_tree_unlock_rw(c, path->locks[level]); 54238e73f275SChris Mason 54245f39d397SChris Mason free_extent_buffer(c); 5425d97e63b6SChris Mason path->nodes[level] = next; 5426d97e63b6SChris Mason path->slots[level] = 0; 5427a74a4b97SChris Mason if (!path->skip_locking) 5428bd681513SChris Mason path->locks[level] = next_rw_lock; 5429d97e63b6SChris Mason if (!level) 5430d97e63b6SChris Mason break; 5431b4ce94deSChris Mason 5432d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5433cda79c54SDavid Sterba 0, &key); 54348e73f275SChris Mason if (ret == -EAGAIN) 54358e73f275SChris Mason goto again; 54368e73f275SChris Mason 543776a05b35SChris Mason if (ret < 0) { 5438b3b4aa74SDavid Sterba btrfs_release_path(path); 543976a05b35SChris Mason goto done; 544076a05b35SChris Mason } 544176a05b35SChris Mason 54425cd57b2cSChris Mason if (!path->skip_locking) { 5443ac5887c8SJosef Bacik __btrfs_tree_read_lock(next, BTRFS_NESTING_RIGHT, 5444fd7ba1c1SJosef Bacik path->recurse); 5445bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5446bd681513SChris Mason } 5447d97e63b6SChris Mason } 54488e73f275SChris Mason ret = 0; 5449925baeddSChris Mason done: 5450f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 54518e73f275SChris Mason 54528e73f275SChris Mason return ret; 5453d97e63b6SChris Mason } 54540b86a832SChris Mason 54553f157a2fSChris Mason /* 54563f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 54573f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 54583f157a2fSChris Mason * 54593f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 54603f157a2fSChris Mason */ 54610b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 54620b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 54630b86a832SChris Mason int type) 54640b86a832SChris Mason { 54650b86a832SChris Mason struct btrfs_key found_key; 54660b86a832SChris Mason struct extent_buffer *leaf; 5467e02119d5SChris Mason u32 nritems; 54680b86a832SChris Mason int ret; 54690b86a832SChris Mason 54700b86a832SChris Mason while (1) { 54710b86a832SChris Mason if (path->slots[0] == 0) { 54720b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 54730b86a832SChris Mason if (ret != 0) 54740b86a832SChris Mason return ret; 54750b86a832SChris Mason } else { 54760b86a832SChris Mason path->slots[0]--; 54770b86a832SChris Mason } 54780b86a832SChris Mason leaf = path->nodes[0]; 5479e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5480e02119d5SChris Mason if (nritems == 0) 5481e02119d5SChris Mason return 1; 5482e02119d5SChris Mason if (path->slots[0] == nritems) 5483e02119d5SChris Mason path->slots[0]--; 5484e02119d5SChris Mason 54850b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5486e02119d5SChris Mason if (found_key.objectid < min_objectid) 5487e02119d5SChris Mason break; 54880a4eefbbSYan Zheng if (found_key.type == type) 54890a4eefbbSYan Zheng return 0; 5490e02119d5SChris Mason if (found_key.objectid == min_objectid && 5491e02119d5SChris Mason found_key.type < type) 5492e02119d5SChris Mason break; 54930b86a832SChris Mason } 54940b86a832SChris Mason return 1; 54950b86a832SChris Mason } 5496ade2e0b3SWang Shilong 5497ade2e0b3SWang Shilong /* 5498ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 5499ade2e0b3SWang Shilong * min objecitd. 5500ade2e0b3SWang Shilong * 5501ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 5502ade2e0b3SWang Shilong */ 5503ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 5504ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 5505ade2e0b3SWang Shilong { 5506ade2e0b3SWang Shilong struct btrfs_key found_key; 5507ade2e0b3SWang Shilong struct extent_buffer *leaf; 5508ade2e0b3SWang Shilong u32 nritems; 5509ade2e0b3SWang Shilong int ret; 5510ade2e0b3SWang Shilong 5511ade2e0b3SWang Shilong while (1) { 5512ade2e0b3SWang Shilong if (path->slots[0] == 0) { 5513ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 5514ade2e0b3SWang Shilong if (ret != 0) 5515ade2e0b3SWang Shilong return ret; 5516ade2e0b3SWang Shilong } else { 5517ade2e0b3SWang Shilong path->slots[0]--; 5518ade2e0b3SWang Shilong } 5519ade2e0b3SWang Shilong leaf = path->nodes[0]; 5520ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5521ade2e0b3SWang Shilong if (nritems == 0) 5522ade2e0b3SWang Shilong return 1; 5523ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5524ade2e0b3SWang Shilong path->slots[0]--; 5525ade2e0b3SWang Shilong 5526ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5527ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5528ade2e0b3SWang Shilong break; 5529ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5530ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5531ade2e0b3SWang Shilong return 0; 5532ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5533ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5534ade2e0b3SWang Shilong break; 5535ade2e0b3SWang Shilong } 5536ade2e0b3SWang Shilong return 1; 5537ade2e0b3SWang Shilong } 5538