1 /* 2 * Copyright (C) 2007 Oracle. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public 6 * License v2 as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public 14 * License along with this program; if not, write to the 15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 16 * Boston, MA 021110-1307, USA. 17 */ 18 #include <linux/sched.h> 19 #include <linux/pagemap.h> 20 #include <linux/writeback.h> 21 #include <linux/blkdev.h> 22 #include <linux/sort.h> 23 #include <linux/rcupdate.h> 24 #include <linux/kthread.h> 25 #include <linux/slab.h> 26 #include <linux/ratelimit.h> 27 #include <linux/percpu_counter.h> 28 #include "hash.h" 29 #include "tree-log.h" 30 #include "disk-io.h" 31 #include "print-tree.h" 32 #include "volumes.h" 33 #include "raid56.h" 34 #include "locking.h" 35 #include "free-space-cache.h" 36 #include "math.h" 37 #include "sysfs.h" 38 #include "qgroup.h" 39 40 #undef SCRAMBLE_DELAYED_REFS 41 42 /* 43 * control flags for do_chunk_alloc's force field 44 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk 45 * if we really need one. 46 * 47 * CHUNK_ALLOC_LIMITED means to only try and allocate one 48 * if we have very few chunks already allocated. This is 49 * used as part of the clustering code to help make sure 50 * we have a good pool of storage to cluster in, without 51 * filling the FS with empty chunks 52 * 53 * CHUNK_ALLOC_FORCE means it must try to allocate one 54 * 55 */ 56 enum { 57 CHUNK_ALLOC_NO_FORCE = 0, 58 CHUNK_ALLOC_LIMITED = 1, 59 CHUNK_ALLOC_FORCE = 2, 60 }; 61 62 /* 63 * Control how reservations are dealt with. 64 * 65 * RESERVE_FREE - freeing a reservation. 66 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for 67 * ENOSPC accounting 68 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update 69 * bytes_may_use as the ENOSPC accounting is done elsewhere 70 */ 71 enum { 72 RESERVE_FREE = 0, 73 RESERVE_ALLOC = 1, 74 RESERVE_ALLOC_NO_ACCOUNT = 2, 75 }; 76 77 static int update_block_group(struct btrfs_trans_handle *trans, 78 struct btrfs_root *root, u64 bytenr, 79 u64 num_bytes, int alloc); 80 static int __btrfs_free_extent(struct btrfs_trans_handle *trans, 81 struct btrfs_root *root, 82 struct btrfs_delayed_ref_node *node, u64 parent, 83 u64 root_objectid, u64 owner_objectid, 84 u64 owner_offset, int refs_to_drop, 85 struct btrfs_delayed_extent_op *extra_op); 86 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, 87 struct extent_buffer *leaf, 88 struct btrfs_extent_item *ei); 89 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, 90 struct btrfs_root *root, 91 u64 parent, u64 root_objectid, 92 u64 flags, u64 owner, u64 offset, 93 struct btrfs_key *ins, int ref_mod); 94 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, 95 struct btrfs_root *root, 96 u64 parent, u64 root_objectid, 97 u64 flags, struct btrfs_disk_key *key, 98 int level, struct btrfs_key *ins); 99 static int do_chunk_alloc(struct btrfs_trans_handle *trans, 100 struct btrfs_root *extent_root, u64 flags, 101 int force); 102 static int find_next_key(struct btrfs_path *path, int level, 103 struct btrfs_key *key); 104 static void dump_space_info(struct btrfs_space_info *info, u64 bytes, 105 int dump_block_groups); 106 static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache, 107 u64 num_bytes, int reserve, 108 int delalloc); 109 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, 110 u64 num_bytes); 111 int btrfs_pin_extent(struct btrfs_root *root, 112 u64 bytenr, u64 num_bytes, int reserved); 113 114 static noinline int 115 block_group_cache_done(struct btrfs_block_group_cache *cache) 116 { 117 smp_mb(); 118 return cache->cached == BTRFS_CACHE_FINISHED || 119 cache->cached == BTRFS_CACHE_ERROR; 120 } 121 122 static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits) 123 { 124 return (cache->flags & bits) == bits; 125 } 126 127 void btrfs_get_block_group(struct btrfs_block_group_cache *cache) 128 { 129 atomic_inc(&cache->count); 130 } 131 132 void btrfs_put_block_group(struct btrfs_block_group_cache *cache) 133 { 134 if (atomic_dec_and_test(&cache->count)) { 135 WARN_ON(cache->pinned > 0); 136 WARN_ON(cache->reserved > 0); 137 kfree(cache->free_space_ctl); 138 kfree(cache); 139 } 140 } 141 142 /* 143 * this adds the block group to the fs_info rb tree for the block group 144 * cache 145 */ 146 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, 147 struct btrfs_block_group_cache *block_group) 148 { 149 struct rb_node **p; 150 struct rb_node *parent = NULL; 151 struct btrfs_block_group_cache *cache; 152 153 spin_lock(&info->block_group_cache_lock); 154 p = &info->block_group_cache_tree.rb_node; 155 156 while (*p) { 157 parent = *p; 158 cache = rb_entry(parent, struct btrfs_block_group_cache, 159 cache_node); 160 if (block_group->key.objectid < cache->key.objectid) { 161 p = &(*p)->rb_left; 162 } else if (block_group->key.objectid > cache->key.objectid) { 163 p = &(*p)->rb_right; 164 } else { 165 spin_unlock(&info->block_group_cache_lock); 166 return -EEXIST; 167 } 168 } 169 170 rb_link_node(&block_group->cache_node, parent, p); 171 rb_insert_color(&block_group->cache_node, 172 &info->block_group_cache_tree); 173 174 if (info->first_logical_byte > block_group->key.objectid) 175 info->first_logical_byte = block_group->key.objectid; 176 177 spin_unlock(&info->block_group_cache_lock); 178 179 return 0; 180 } 181 182 /* 183 * This will return the block group at or after bytenr if contains is 0, else 184 * it will return the block group that contains the bytenr 185 */ 186 static struct btrfs_block_group_cache * 187 block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr, 188 int contains) 189 { 190 struct btrfs_block_group_cache *cache, *ret = NULL; 191 struct rb_node *n; 192 u64 end, start; 193 194 spin_lock(&info->block_group_cache_lock); 195 n = info->block_group_cache_tree.rb_node; 196 197 while (n) { 198 cache = rb_entry(n, struct btrfs_block_group_cache, 199 cache_node); 200 end = cache->key.objectid + cache->key.offset - 1; 201 start = cache->key.objectid; 202 203 if (bytenr < start) { 204 if (!contains && (!ret || start < ret->key.objectid)) 205 ret = cache; 206 n = n->rb_left; 207 } else if (bytenr > start) { 208 if (contains && bytenr <= end) { 209 ret = cache; 210 break; 211 } 212 n = n->rb_right; 213 } else { 214 ret = cache; 215 break; 216 } 217 } 218 if (ret) { 219 btrfs_get_block_group(ret); 220 if (bytenr == 0 && info->first_logical_byte > ret->key.objectid) 221 info->first_logical_byte = ret->key.objectid; 222 } 223 spin_unlock(&info->block_group_cache_lock); 224 225 return ret; 226 } 227 228 static int add_excluded_extent(struct btrfs_root *root, 229 u64 start, u64 num_bytes) 230 { 231 u64 end = start + num_bytes - 1; 232 set_extent_bits(&root->fs_info->freed_extents[0], 233 start, end, EXTENT_UPTODATE, GFP_NOFS); 234 set_extent_bits(&root->fs_info->freed_extents[1], 235 start, end, EXTENT_UPTODATE, GFP_NOFS); 236 return 0; 237 } 238 239 static void free_excluded_extents(struct btrfs_root *root, 240 struct btrfs_block_group_cache *cache) 241 { 242 u64 start, end; 243 244 start = cache->key.objectid; 245 end = start + cache->key.offset - 1; 246 247 clear_extent_bits(&root->fs_info->freed_extents[0], 248 start, end, EXTENT_UPTODATE, GFP_NOFS); 249 clear_extent_bits(&root->fs_info->freed_extents[1], 250 start, end, EXTENT_UPTODATE, GFP_NOFS); 251 } 252 253 static int exclude_super_stripes(struct btrfs_root *root, 254 struct btrfs_block_group_cache *cache) 255 { 256 u64 bytenr; 257 u64 *logical; 258 int stripe_len; 259 int i, nr, ret; 260 261 if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) { 262 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid; 263 cache->bytes_super += stripe_len; 264 ret = add_excluded_extent(root, cache->key.objectid, 265 stripe_len); 266 if (ret) 267 return ret; 268 } 269 270 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 271 bytenr = btrfs_sb_offset(i); 272 ret = btrfs_rmap_block(&root->fs_info->mapping_tree, 273 cache->key.objectid, bytenr, 274 0, &logical, &nr, &stripe_len); 275 if (ret) 276 return ret; 277 278 while (nr--) { 279 u64 start, len; 280 281 if (logical[nr] > cache->key.objectid + 282 cache->key.offset) 283 continue; 284 285 if (logical[nr] + stripe_len <= cache->key.objectid) 286 continue; 287 288 start = logical[nr]; 289 if (start < cache->key.objectid) { 290 start = cache->key.objectid; 291 len = (logical[nr] + stripe_len) - start; 292 } else { 293 len = min_t(u64, stripe_len, 294 cache->key.objectid + 295 cache->key.offset - start); 296 } 297 298 cache->bytes_super += len; 299 ret = add_excluded_extent(root, start, len); 300 if (ret) { 301 kfree(logical); 302 return ret; 303 } 304 } 305 306 kfree(logical); 307 } 308 return 0; 309 } 310 311 static struct btrfs_caching_control * 312 get_caching_control(struct btrfs_block_group_cache *cache) 313 { 314 struct btrfs_caching_control *ctl; 315 316 spin_lock(&cache->lock); 317 if (!cache->caching_ctl) { 318 spin_unlock(&cache->lock); 319 return NULL; 320 } 321 322 ctl = cache->caching_ctl; 323 atomic_inc(&ctl->count); 324 spin_unlock(&cache->lock); 325 return ctl; 326 } 327 328 static void put_caching_control(struct btrfs_caching_control *ctl) 329 { 330 if (atomic_dec_and_test(&ctl->count)) 331 kfree(ctl); 332 } 333 334 #ifdef CONFIG_BTRFS_DEBUG 335 static void fragment_free_space(struct btrfs_root *root, 336 struct btrfs_block_group_cache *block_group) 337 { 338 u64 start = block_group->key.objectid; 339 u64 len = block_group->key.offset; 340 u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ? 341 root->nodesize : root->sectorsize; 342 u64 step = chunk << 1; 343 344 while (len > chunk) { 345 btrfs_remove_free_space(block_group, start, chunk); 346 start += step; 347 if (len < step) 348 len = 0; 349 else 350 len -= step; 351 } 352 } 353 #endif 354 355 /* 356 * this is only called by cache_block_group, since we could have freed extents 357 * we need to check the pinned_extents for any extents that can't be used yet 358 * since their free space will be released as soon as the transaction commits. 359 */ 360 static u64 add_new_free_space(struct btrfs_block_group_cache *block_group, 361 struct btrfs_fs_info *info, u64 start, u64 end) 362 { 363 u64 extent_start, extent_end, size, total_added = 0; 364 int ret; 365 366 while (start < end) { 367 ret = find_first_extent_bit(info->pinned_extents, start, 368 &extent_start, &extent_end, 369 EXTENT_DIRTY | EXTENT_UPTODATE, 370 NULL); 371 if (ret) 372 break; 373 374 if (extent_start <= start) { 375 start = extent_end + 1; 376 } else if (extent_start > start && extent_start < end) { 377 size = extent_start - start; 378 total_added += size; 379 ret = btrfs_add_free_space(block_group, start, 380 size); 381 BUG_ON(ret); /* -ENOMEM or logic error */ 382 start = extent_end + 1; 383 } else { 384 break; 385 } 386 } 387 388 if (start < end) { 389 size = end - start; 390 total_added += size; 391 ret = btrfs_add_free_space(block_group, start, size); 392 BUG_ON(ret); /* -ENOMEM or logic error */ 393 } 394 395 return total_added; 396 } 397 398 static noinline void caching_thread(struct btrfs_work *work) 399 { 400 struct btrfs_block_group_cache *block_group; 401 struct btrfs_fs_info *fs_info; 402 struct btrfs_caching_control *caching_ctl; 403 struct btrfs_root *extent_root; 404 struct btrfs_path *path; 405 struct extent_buffer *leaf; 406 struct btrfs_key key; 407 u64 total_found = 0; 408 u64 last = 0; 409 u32 nritems; 410 int ret = -ENOMEM; 411 bool wakeup = true; 412 413 caching_ctl = container_of(work, struct btrfs_caching_control, work); 414 block_group = caching_ctl->block_group; 415 fs_info = block_group->fs_info; 416 extent_root = fs_info->extent_root; 417 418 path = btrfs_alloc_path(); 419 if (!path) 420 goto out; 421 422 last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET); 423 424 #ifdef CONFIG_BTRFS_DEBUG 425 /* 426 * If we're fragmenting we don't want to make anybody think we can 427 * allocate from this block group until we've had a chance to fragment 428 * the free space. 429 */ 430 if (btrfs_should_fragment_free_space(extent_root, block_group)) 431 wakeup = false; 432 #endif 433 /* 434 * We don't want to deadlock with somebody trying to allocate a new 435 * extent for the extent root while also trying to search the extent 436 * root to add free space. So we skip locking and search the commit 437 * root, since its read-only 438 */ 439 path->skip_locking = 1; 440 path->search_commit_root = 1; 441 path->reada = 1; 442 443 key.objectid = last; 444 key.offset = 0; 445 key.type = BTRFS_EXTENT_ITEM_KEY; 446 again: 447 mutex_lock(&caching_ctl->mutex); 448 /* need to make sure the commit_root doesn't disappear */ 449 down_read(&fs_info->commit_root_sem); 450 451 next: 452 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); 453 if (ret < 0) 454 goto err; 455 456 leaf = path->nodes[0]; 457 nritems = btrfs_header_nritems(leaf); 458 459 while (1) { 460 if (btrfs_fs_closing(fs_info) > 1) { 461 last = (u64)-1; 462 break; 463 } 464 465 if (path->slots[0] < nritems) { 466 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 467 } else { 468 ret = find_next_key(path, 0, &key); 469 if (ret) 470 break; 471 472 if (need_resched() || 473 rwsem_is_contended(&fs_info->commit_root_sem)) { 474 if (wakeup) 475 caching_ctl->progress = last; 476 btrfs_release_path(path); 477 up_read(&fs_info->commit_root_sem); 478 mutex_unlock(&caching_ctl->mutex); 479 cond_resched(); 480 goto again; 481 } 482 483 ret = btrfs_next_leaf(extent_root, path); 484 if (ret < 0) 485 goto err; 486 if (ret) 487 break; 488 leaf = path->nodes[0]; 489 nritems = btrfs_header_nritems(leaf); 490 continue; 491 } 492 493 if (key.objectid < last) { 494 key.objectid = last; 495 key.offset = 0; 496 key.type = BTRFS_EXTENT_ITEM_KEY; 497 498 if (wakeup) 499 caching_ctl->progress = last; 500 btrfs_release_path(path); 501 goto next; 502 } 503 504 if (key.objectid < block_group->key.objectid) { 505 path->slots[0]++; 506 continue; 507 } 508 509 if (key.objectid >= block_group->key.objectid + 510 block_group->key.offset) 511 break; 512 513 if (key.type == BTRFS_EXTENT_ITEM_KEY || 514 key.type == BTRFS_METADATA_ITEM_KEY) { 515 total_found += add_new_free_space(block_group, 516 fs_info, last, 517 key.objectid); 518 if (key.type == BTRFS_METADATA_ITEM_KEY) 519 last = key.objectid + 520 fs_info->tree_root->nodesize; 521 else 522 last = key.objectid + key.offset; 523 524 if (total_found > (1024 * 1024 * 2)) { 525 total_found = 0; 526 if (wakeup) 527 wake_up(&caching_ctl->wait); 528 } 529 } 530 path->slots[0]++; 531 } 532 ret = 0; 533 534 total_found += add_new_free_space(block_group, fs_info, last, 535 block_group->key.objectid + 536 block_group->key.offset); 537 spin_lock(&block_group->lock); 538 block_group->caching_ctl = NULL; 539 block_group->cached = BTRFS_CACHE_FINISHED; 540 spin_unlock(&block_group->lock); 541 542 #ifdef CONFIG_BTRFS_DEBUG 543 if (btrfs_should_fragment_free_space(extent_root, block_group)) { 544 u64 bytes_used; 545 546 spin_lock(&block_group->space_info->lock); 547 spin_lock(&block_group->lock); 548 bytes_used = block_group->key.offset - 549 btrfs_block_group_used(&block_group->item); 550 block_group->space_info->bytes_used += bytes_used >> 1; 551 spin_unlock(&block_group->lock); 552 spin_unlock(&block_group->space_info->lock); 553 fragment_free_space(extent_root, block_group); 554 } 555 #endif 556 557 caching_ctl->progress = (u64)-1; 558 err: 559 btrfs_free_path(path); 560 up_read(&fs_info->commit_root_sem); 561 562 free_excluded_extents(extent_root, block_group); 563 564 mutex_unlock(&caching_ctl->mutex); 565 out: 566 if (ret) { 567 spin_lock(&block_group->lock); 568 block_group->caching_ctl = NULL; 569 block_group->cached = BTRFS_CACHE_ERROR; 570 spin_unlock(&block_group->lock); 571 } 572 wake_up(&caching_ctl->wait); 573 574 put_caching_control(caching_ctl); 575 btrfs_put_block_group(block_group); 576 } 577 578 static int cache_block_group(struct btrfs_block_group_cache *cache, 579 int load_cache_only) 580 { 581 DEFINE_WAIT(wait); 582 struct btrfs_fs_info *fs_info = cache->fs_info; 583 struct btrfs_caching_control *caching_ctl; 584 int ret = 0; 585 586 caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS); 587 if (!caching_ctl) 588 return -ENOMEM; 589 590 INIT_LIST_HEAD(&caching_ctl->list); 591 mutex_init(&caching_ctl->mutex); 592 init_waitqueue_head(&caching_ctl->wait); 593 caching_ctl->block_group = cache; 594 caching_ctl->progress = cache->key.objectid; 595 atomic_set(&caching_ctl->count, 1); 596 btrfs_init_work(&caching_ctl->work, btrfs_cache_helper, 597 caching_thread, NULL, NULL); 598 599 spin_lock(&cache->lock); 600 /* 601 * This should be a rare occasion, but this could happen I think in the 602 * case where one thread starts to load the space cache info, and then 603 * some other thread starts a transaction commit which tries to do an 604 * allocation while the other thread is still loading the space cache 605 * info. The previous loop should have kept us from choosing this block 606 * group, but if we've moved to the state where we will wait on caching 607 * block groups we need to first check if we're doing a fast load here, 608 * so we can wait for it to finish, otherwise we could end up allocating 609 * from a block group who's cache gets evicted for one reason or 610 * another. 611 */ 612 while (cache->cached == BTRFS_CACHE_FAST) { 613 struct btrfs_caching_control *ctl; 614 615 ctl = cache->caching_ctl; 616 atomic_inc(&ctl->count); 617 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE); 618 spin_unlock(&cache->lock); 619 620 schedule(); 621 622 finish_wait(&ctl->wait, &wait); 623 put_caching_control(ctl); 624 spin_lock(&cache->lock); 625 } 626 627 if (cache->cached != BTRFS_CACHE_NO) { 628 spin_unlock(&cache->lock); 629 kfree(caching_ctl); 630 return 0; 631 } 632 WARN_ON(cache->caching_ctl); 633 cache->caching_ctl = caching_ctl; 634 cache->cached = BTRFS_CACHE_FAST; 635 spin_unlock(&cache->lock); 636 637 if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) { 638 mutex_lock(&caching_ctl->mutex); 639 ret = load_free_space_cache(fs_info, cache); 640 641 spin_lock(&cache->lock); 642 if (ret == 1) { 643 cache->caching_ctl = NULL; 644 cache->cached = BTRFS_CACHE_FINISHED; 645 cache->last_byte_to_unpin = (u64)-1; 646 caching_ctl->progress = (u64)-1; 647 } else { 648 if (load_cache_only) { 649 cache->caching_ctl = NULL; 650 cache->cached = BTRFS_CACHE_NO; 651 } else { 652 cache->cached = BTRFS_CACHE_STARTED; 653 cache->has_caching_ctl = 1; 654 } 655 } 656 spin_unlock(&cache->lock); 657 #ifdef CONFIG_BTRFS_DEBUG 658 if (ret == 1 && 659 btrfs_should_fragment_free_space(fs_info->extent_root, 660 cache)) { 661 u64 bytes_used; 662 663 spin_lock(&cache->space_info->lock); 664 spin_lock(&cache->lock); 665 bytes_used = cache->key.offset - 666 btrfs_block_group_used(&cache->item); 667 cache->space_info->bytes_used += bytes_used >> 1; 668 spin_unlock(&cache->lock); 669 spin_unlock(&cache->space_info->lock); 670 fragment_free_space(fs_info->extent_root, cache); 671 } 672 #endif 673 mutex_unlock(&caching_ctl->mutex); 674 675 wake_up(&caching_ctl->wait); 676 if (ret == 1) { 677 put_caching_control(caching_ctl); 678 free_excluded_extents(fs_info->extent_root, cache); 679 return 0; 680 } 681 } else { 682 /* 683 * We are not going to do the fast caching, set cached to the 684 * appropriate value and wakeup any waiters. 685 */ 686 spin_lock(&cache->lock); 687 if (load_cache_only) { 688 cache->caching_ctl = NULL; 689 cache->cached = BTRFS_CACHE_NO; 690 } else { 691 cache->cached = BTRFS_CACHE_STARTED; 692 cache->has_caching_ctl = 1; 693 } 694 spin_unlock(&cache->lock); 695 wake_up(&caching_ctl->wait); 696 } 697 698 if (load_cache_only) { 699 put_caching_control(caching_ctl); 700 return 0; 701 } 702 703 down_write(&fs_info->commit_root_sem); 704 atomic_inc(&caching_ctl->count); 705 list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups); 706 up_write(&fs_info->commit_root_sem); 707 708 btrfs_get_block_group(cache); 709 710 btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work); 711 712 return ret; 713 } 714 715 /* 716 * return the block group that starts at or after bytenr 717 */ 718 static struct btrfs_block_group_cache * 719 btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr) 720 { 721 struct btrfs_block_group_cache *cache; 722 723 cache = block_group_cache_tree_search(info, bytenr, 0); 724 725 return cache; 726 } 727 728 /* 729 * return the block group that contains the given bytenr 730 */ 731 struct btrfs_block_group_cache *btrfs_lookup_block_group( 732 struct btrfs_fs_info *info, 733 u64 bytenr) 734 { 735 struct btrfs_block_group_cache *cache; 736 737 cache = block_group_cache_tree_search(info, bytenr, 1); 738 739 return cache; 740 } 741 742 static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info, 743 u64 flags) 744 { 745 struct list_head *head = &info->space_info; 746 struct btrfs_space_info *found; 747 748 flags &= BTRFS_BLOCK_GROUP_TYPE_MASK; 749 750 rcu_read_lock(); 751 list_for_each_entry_rcu(found, head, list) { 752 if (found->flags & flags) { 753 rcu_read_unlock(); 754 return found; 755 } 756 } 757 rcu_read_unlock(); 758 return NULL; 759 } 760 761 /* 762 * after adding space to the filesystem, we need to clear the full flags 763 * on all the space infos. 764 */ 765 void btrfs_clear_space_info_full(struct btrfs_fs_info *info) 766 { 767 struct list_head *head = &info->space_info; 768 struct btrfs_space_info *found; 769 770 rcu_read_lock(); 771 list_for_each_entry_rcu(found, head, list) 772 found->full = 0; 773 rcu_read_unlock(); 774 } 775 776 /* simple helper to search for an existing data extent at a given offset */ 777 int btrfs_lookup_data_extent(struct btrfs_root *root, u64 start, u64 len) 778 { 779 int ret; 780 struct btrfs_key key; 781 struct btrfs_path *path; 782 783 path = btrfs_alloc_path(); 784 if (!path) 785 return -ENOMEM; 786 787 key.objectid = start; 788 key.offset = len; 789 key.type = BTRFS_EXTENT_ITEM_KEY; 790 ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path, 791 0, 0); 792 btrfs_free_path(path); 793 return ret; 794 } 795 796 /* 797 * helper function to lookup reference count and flags of a tree block. 798 * 799 * the head node for delayed ref is used to store the sum of all the 800 * reference count modifications queued up in the rbtree. the head 801 * node may also store the extent flags to set. This way you can check 802 * to see what the reference count and extent flags would be if all of 803 * the delayed refs are not processed. 804 */ 805 int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, 806 struct btrfs_root *root, u64 bytenr, 807 u64 offset, int metadata, u64 *refs, u64 *flags) 808 { 809 struct btrfs_delayed_ref_head *head; 810 struct btrfs_delayed_ref_root *delayed_refs; 811 struct btrfs_path *path; 812 struct btrfs_extent_item *ei; 813 struct extent_buffer *leaf; 814 struct btrfs_key key; 815 u32 item_size; 816 u64 num_refs; 817 u64 extent_flags; 818 int ret; 819 820 /* 821 * If we don't have skinny metadata, don't bother doing anything 822 * different 823 */ 824 if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA)) { 825 offset = root->nodesize; 826 metadata = 0; 827 } 828 829 path = btrfs_alloc_path(); 830 if (!path) 831 return -ENOMEM; 832 833 if (!trans) { 834 path->skip_locking = 1; 835 path->search_commit_root = 1; 836 } 837 838 search_again: 839 key.objectid = bytenr; 840 key.offset = offset; 841 if (metadata) 842 key.type = BTRFS_METADATA_ITEM_KEY; 843 else 844 key.type = BTRFS_EXTENT_ITEM_KEY; 845 846 ret = btrfs_search_slot(trans, root->fs_info->extent_root, 847 &key, path, 0, 0); 848 if (ret < 0) 849 goto out_free; 850 851 if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) { 852 if (path->slots[0]) { 853 path->slots[0]--; 854 btrfs_item_key_to_cpu(path->nodes[0], &key, 855 path->slots[0]); 856 if (key.objectid == bytenr && 857 key.type == BTRFS_EXTENT_ITEM_KEY && 858 key.offset == root->nodesize) 859 ret = 0; 860 } 861 } 862 863 if (ret == 0) { 864 leaf = path->nodes[0]; 865 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 866 if (item_size >= sizeof(*ei)) { 867 ei = btrfs_item_ptr(leaf, path->slots[0], 868 struct btrfs_extent_item); 869 num_refs = btrfs_extent_refs(leaf, ei); 870 extent_flags = btrfs_extent_flags(leaf, ei); 871 } else { 872 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 873 struct btrfs_extent_item_v0 *ei0; 874 BUG_ON(item_size != sizeof(*ei0)); 875 ei0 = btrfs_item_ptr(leaf, path->slots[0], 876 struct btrfs_extent_item_v0); 877 num_refs = btrfs_extent_refs_v0(leaf, ei0); 878 /* FIXME: this isn't correct for data */ 879 extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 880 #else 881 BUG(); 882 #endif 883 } 884 BUG_ON(num_refs == 0); 885 } else { 886 num_refs = 0; 887 extent_flags = 0; 888 ret = 0; 889 } 890 891 if (!trans) 892 goto out; 893 894 delayed_refs = &trans->transaction->delayed_refs; 895 spin_lock(&delayed_refs->lock); 896 head = btrfs_find_delayed_ref_head(trans, bytenr); 897 if (head) { 898 if (!mutex_trylock(&head->mutex)) { 899 atomic_inc(&head->node.refs); 900 spin_unlock(&delayed_refs->lock); 901 902 btrfs_release_path(path); 903 904 /* 905 * Mutex was contended, block until it's released and try 906 * again 907 */ 908 mutex_lock(&head->mutex); 909 mutex_unlock(&head->mutex); 910 btrfs_put_delayed_ref(&head->node); 911 goto search_again; 912 } 913 spin_lock(&head->lock); 914 if (head->extent_op && head->extent_op->update_flags) 915 extent_flags |= head->extent_op->flags_to_set; 916 else 917 BUG_ON(num_refs == 0); 918 919 num_refs += head->node.ref_mod; 920 spin_unlock(&head->lock); 921 mutex_unlock(&head->mutex); 922 } 923 spin_unlock(&delayed_refs->lock); 924 out: 925 WARN_ON(num_refs == 0); 926 if (refs) 927 *refs = num_refs; 928 if (flags) 929 *flags = extent_flags; 930 out_free: 931 btrfs_free_path(path); 932 return ret; 933 } 934 935 /* 936 * Back reference rules. Back refs have three main goals: 937 * 938 * 1) differentiate between all holders of references to an extent so that 939 * when a reference is dropped we can make sure it was a valid reference 940 * before freeing the extent. 941 * 942 * 2) Provide enough information to quickly find the holders of an extent 943 * if we notice a given block is corrupted or bad. 944 * 945 * 3) Make it easy to migrate blocks for FS shrinking or storage pool 946 * maintenance. This is actually the same as #2, but with a slightly 947 * different use case. 948 * 949 * There are two kinds of back refs. The implicit back refs is optimized 950 * for pointers in non-shared tree blocks. For a given pointer in a block, 951 * back refs of this kind provide information about the block's owner tree 952 * and the pointer's key. These information allow us to find the block by 953 * b-tree searching. The full back refs is for pointers in tree blocks not 954 * referenced by their owner trees. The location of tree block is recorded 955 * in the back refs. Actually the full back refs is generic, and can be 956 * used in all cases the implicit back refs is used. The major shortcoming 957 * of the full back refs is its overhead. Every time a tree block gets 958 * COWed, we have to update back refs entry for all pointers in it. 959 * 960 * For a newly allocated tree block, we use implicit back refs for 961 * pointers in it. This means most tree related operations only involve 962 * implicit back refs. For a tree block created in old transaction, the 963 * only way to drop a reference to it is COW it. So we can detect the 964 * event that tree block loses its owner tree's reference and do the 965 * back refs conversion. 966 * 967 * When a tree block is COW'd through a tree, there are four cases: 968 * 969 * The reference count of the block is one and the tree is the block's 970 * owner tree. Nothing to do in this case. 971 * 972 * The reference count of the block is one and the tree is not the 973 * block's owner tree. In this case, full back refs is used for pointers 974 * in the block. Remove these full back refs, add implicit back refs for 975 * every pointers in the new block. 976 * 977 * The reference count of the block is greater than one and the tree is 978 * the block's owner tree. In this case, implicit back refs is used for 979 * pointers in the block. Add full back refs for every pointers in the 980 * block, increase lower level extents' reference counts. The original 981 * implicit back refs are entailed to the new block. 982 * 983 * The reference count of the block is greater than one and the tree is 984 * not the block's owner tree. Add implicit back refs for every pointer in 985 * the new block, increase lower level extents' reference count. 986 * 987 * Back Reference Key composing: 988 * 989 * The key objectid corresponds to the first byte in the extent, 990 * The key type is used to differentiate between types of back refs. 991 * There are different meanings of the key offset for different types 992 * of back refs. 993 * 994 * File extents can be referenced by: 995 * 996 * - multiple snapshots, subvolumes, or different generations in one subvol 997 * - different files inside a single subvolume 998 * - different offsets inside a file (bookend extents in file.c) 999 * 1000 * The extent ref structure for the implicit back refs has fields for: 1001 * 1002 * - Objectid of the subvolume root 1003 * - objectid of the file holding the reference 1004 * - original offset in the file 1005 * - how many bookend extents 1006 * 1007 * The key offset for the implicit back refs is hash of the first 1008 * three fields. 1009 * 1010 * The extent ref structure for the full back refs has field for: 1011 * 1012 * - number of pointers in the tree leaf 1013 * 1014 * The key offset for the implicit back refs is the first byte of 1015 * the tree leaf 1016 * 1017 * When a file extent is allocated, The implicit back refs is used. 1018 * the fields are filled in: 1019 * 1020 * (root_key.objectid, inode objectid, offset in file, 1) 1021 * 1022 * When a file extent is removed file truncation, we find the 1023 * corresponding implicit back refs and check the following fields: 1024 * 1025 * (btrfs_header_owner(leaf), inode objectid, offset in file) 1026 * 1027 * Btree extents can be referenced by: 1028 * 1029 * - Different subvolumes 1030 * 1031 * Both the implicit back refs and the full back refs for tree blocks 1032 * only consist of key. The key offset for the implicit back refs is 1033 * objectid of block's owner tree. The key offset for the full back refs 1034 * is the first byte of parent block. 1035 * 1036 * When implicit back refs is used, information about the lowest key and 1037 * level of the tree block are required. These information are stored in 1038 * tree block info structure. 1039 */ 1040 1041 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1042 static int convert_extent_item_v0(struct btrfs_trans_handle *trans, 1043 struct btrfs_root *root, 1044 struct btrfs_path *path, 1045 u64 owner, u32 extra_size) 1046 { 1047 struct btrfs_extent_item *item; 1048 struct btrfs_extent_item_v0 *ei0; 1049 struct btrfs_extent_ref_v0 *ref0; 1050 struct btrfs_tree_block_info *bi; 1051 struct extent_buffer *leaf; 1052 struct btrfs_key key; 1053 struct btrfs_key found_key; 1054 u32 new_size = sizeof(*item); 1055 u64 refs; 1056 int ret; 1057 1058 leaf = path->nodes[0]; 1059 BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0)); 1060 1061 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1062 ei0 = btrfs_item_ptr(leaf, path->slots[0], 1063 struct btrfs_extent_item_v0); 1064 refs = btrfs_extent_refs_v0(leaf, ei0); 1065 1066 if (owner == (u64)-1) { 1067 while (1) { 1068 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1069 ret = btrfs_next_leaf(root, path); 1070 if (ret < 0) 1071 return ret; 1072 BUG_ON(ret > 0); /* Corruption */ 1073 leaf = path->nodes[0]; 1074 } 1075 btrfs_item_key_to_cpu(leaf, &found_key, 1076 path->slots[0]); 1077 BUG_ON(key.objectid != found_key.objectid); 1078 if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) { 1079 path->slots[0]++; 1080 continue; 1081 } 1082 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1083 struct btrfs_extent_ref_v0); 1084 owner = btrfs_ref_objectid_v0(leaf, ref0); 1085 break; 1086 } 1087 } 1088 btrfs_release_path(path); 1089 1090 if (owner < BTRFS_FIRST_FREE_OBJECTID) 1091 new_size += sizeof(*bi); 1092 1093 new_size -= sizeof(*ei0); 1094 ret = btrfs_search_slot(trans, root, &key, path, 1095 new_size + extra_size, 1); 1096 if (ret < 0) 1097 return ret; 1098 BUG_ON(ret); /* Corruption */ 1099 1100 btrfs_extend_item(root, path, new_size); 1101 1102 leaf = path->nodes[0]; 1103 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1104 btrfs_set_extent_refs(leaf, item, refs); 1105 /* FIXME: get real generation */ 1106 btrfs_set_extent_generation(leaf, item, 0); 1107 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1108 btrfs_set_extent_flags(leaf, item, 1109 BTRFS_EXTENT_FLAG_TREE_BLOCK | 1110 BTRFS_BLOCK_FLAG_FULL_BACKREF); 1111 bi = (struct btrfs_tree_block_info *)(item + 1); 1112 /* FIXME: get first key of the block */ 1113 memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi)); 1114 btrfs_set_tree_block_level(leaf, bi, (int)owner); 1115 } else { 1116 btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA); 1117 } 1118 btrfs_mark_buffer_dirty(leaf); 1119 return 0; 1120 } 1121 #endif 1122 1123 static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset) 1124 { 1125 u32 high_crc = ~(u32)0; 1126 u32 low_crc = ~(u32)0; 1127 __le64 lenum; 1128 1129 lenum = cpu_to_le64(root_objectid); 1130 high_crc = btrfs_crc32c(high_crc, &lenum, sizeof(lenum)); 1131 lenum = cpu_to_le64(owner); 1132 low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); 1133 lenum = cpu_to_le64(offset); 1134 low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); 1135 1136 return ((u64)high_crc << 31) ^ (u64)low_crc; 1137 } 1138 1139 static u64 hash_extent_data_ref_item(struct extent_buffer *leaf, 1140 struct btrfs_extent_data_ref *ref) 1141 { 1142 return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref), 1143 btrfs_extent_data_ref_objectid(leaf, ref), 1144 btrfs_extent_data_ref_offset(leaf, ref)); 1145 } 1146 1147 static int match_extent_data_ref(struct extent_buffer *leaf, 1148 struct btrfs_extent_data_ref *ref, 1149 u64 root_objectid, u64 owner, u64 offset) 1150 { 1151 if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid || 1152 btrfs_extent_data_ref_objectid(leaf, ref) != owner || 1153 btrfs_extent_data_ref_offset(leaf, ref) != offset) 1154 return 0; 1155 return 1; 1156 } 1157 1158 static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, 1159 struct btrfs_root *root, 1160 struct btrfs_path *path, 1161 u64 bytenr, u64 parent, 1162 u64 root_objectid, 1163 u64 owner, u64 offset) 1164 { 1165 struct btrfs_key key; 1166 struct btrfs_extent_data_ref *ref; 1167 struct extent_buffer *leaf; 1168 u32 nritems; 1169 int ret; 1170 int recow; 1171 int err = -ENOENT; 1172 1173 key.objectid = bytenr; 1174 if (parent) { 1175 key.type = BTRFS_SHARED_DATA_REF_KEY; 1176 key.offset = parent; 1177 } else { 1178 key.type = BTRFS_EXTENT_DATA_REF_KEY; 1179 key.offset = hash_extent_data_ref(root_objectid, 1180 owner, offset); 1181 } 1182 again: 1183 recow = 0; 1184 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1185 if (ret < 0) { 1186 err = ret; 1187 goto fail; 1188 } 1189 1190 if (parent) { 1191 if (!ret) 1192 return 0; 1193 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1194 key.type = BTRFS_EXTENT_REF_V0_KEY; 1195 btrfs_release_path(path); 1196 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1197 if (ret < 0) { 1198 err = ret; 1199 goto fail; 1200 } 1201 if (!ret) 1202 return 0; 1203 #endif 1204 goto fail; 1205 } 1206 1207 leaf = path->nodes[0]; 1208 nritems = btrfs_header_nritems(leaf); 1209 while (1) { 1210 if (path->slots[0] >= nritems) { 1211 ret = btrfs_next_leaf(root, path); 1212 if (ret < 0) 1213 err = ret; 1214 if (ret) 1215 goto fail; 1216 1217 leaf = path->nodes[0]; 1218 nritems = btrfs_header_nritems(leaf); 1219 recow = 1; 1220 } 1221 1222 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1223 if (key.objectid != bytenr || 1224 key.type != BTRFS_EXTENT_DATA_REF_KEY) 1225 goto fail; 1226 1227 ref = btrfs_item_ptr(leaf, path->slots[0], 1228 struct btrfs_extent_data_ref); 1229 1230 if (match_extent_data_ref(leaf, ref, root_objectid, 1231 owner, offset)) { 1232 if (recow) { 1233 btrfs_release_path(path); 1234 goto again; 1235 } 1236 err = 0; 1237 break; 1238 } 1239 path->slots[0]++; 1240 } 1241 fail: 1242 return err; 1243 } 1244 1245 static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans, 1246 struct btrfs_root *root, 1247 struct btrfs_path *path, 1248 u64 bytenr, u64 parent, 1249 u64 root_objectid, u64 owner, 1250 u64 offset, int refs_to_add) 1251 { 1252 struct btrfs_key key; 1253 struct extent_buffer *leaf; 1254 u32 size; 1255 u32 num_refs; 1256 int ret; 1257 1258 key.objectid = bytenr; 1259 if (parent) { 1260 key.type = BTRFS_SHARED_DATA_REF_KEY; 1261 key.offset = parent; 1262 size = sizeof(struct btrfs_shared_data_ref); 1263 } else { 1264 key.type = BTRFS_EXTENT_DATA_REF_KEY; 1265 key.offset = hash_extent_data_ref(root_objectid, 1266 owner, offset); 1267 size = sizeof(struct btrfs_extent_data_ref); 1268 } 1269 1270 ret = btrfs_insert_empty_item(trans, root, path, &key, size); 1271 if (ret && ret != -EEXIST) 1272 goto fail; 1273 1274 leaf = path->nodes[0]; 1275 if (parent) { 1276 struct btrfs_shared_data_ref *ref; 1277 ref = btrfs_item_ptr(leaf, path->slots[0], 1278 struct btrfs_shared_data_ref); 1279 if (ret == 0) { 1280 btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add); 1281 } else { 1282 num_refs = btrfs_shared_data_ref_count(leaf, ref); 1283 num_refs += refs_to_add; 1284 btrfs_set_shared_data_ref_count(leaf, ref, num_refs); 1285 } 1286 } else { 1287 struct btrfs_extent_data_ref *ref; 1288 while (ret == -EEXIST) { 1289 ref = btrfs_item_ptr(leaf, path->slots[0], 1290 struct btrfs_extent_data_ref); 1291 if (match_extent_data_ref(leaf, ref, root_objectid, 1292 owner, offset)) 1293 break; 1294 btrfs_release_path(path); 1295 key.offset++; 1296 ret = btrfs_insert_empty_item(trans, root, path, &key, 1297 size); 1298 if (ret && ret != -EEXIST) 1299 goto fail; 1300 1301 leaf = path->nodes[0]; 1302 } 1303 ref = btrfs_item_ptr(leaf, path->slots[0], 1304 struct btrfs_extent_data_ref); 1305 if (ret == 0) { 1306 btrfs_set_extent_data_ref_root(leaf, ref, 1307 root_objectid); 1308 btrfs_set_extent_data_ref_objectid(leaf, ref, owner); 1309 btrfs_set_extent_data_ref_offset(leaf, ref, offset); 1310 btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add); 1311 } else { 1312 num_refs = btrfs_extent_data_ref_count(leaf, ref); 1313 num_refs += refs_to_add; 1314 btrfs_set_extent_data_ref_count(leaf, ref, num_refs); 1315 } 1316 } 1317 btrfs_mark_buffer_dirty(leaf); 1318 ret = 0; 1319 fail: 1320 btrfs_release_path(path); 1321 return ret; 1322 } 1323 1324 static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans, 1325 struct btrfs_root *root, 1326 struct btrfs_path *path, 1327 int refs_to_drop, int *last_ref) 1328 { 1329 struct btrfs_key key; 1330 struct btrfs_extent_data_ref *ref1 = NULL; 1331 struct btrfs_shared_data_ref *ref2 = NULL; 1332 struct extent_buffer *leaf; 1333 u32 num_refs = 0; 1334 int ret = 0; 1335 1336 leaf = path->nodes[0]; 1337 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1338 1339 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { 1340 ref1 = btrfs_item_ptr(leaf, path->slots[0], 1341 struct btrfs_extent_data_ref); 1342 num_refs = btrfs_extent_data_ref_count(leaf, ref1); 1343 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { 1344 ref2 = btrfs_item_ptr(leaf, path->slots[0], 1345 struct btrfs_shared_data_ref); 1346 num_refs = btrfs_shared_data_ref_count(leaf, ref2); 1347 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1348 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) { 1349 struct btrfs_extent_ref_v0 *ref0; 1350 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1351 struct btrfs_extent_ref_v0); 1352 num_refs = btrfs_ref_count_v0(leaf, ref0); 1353 #endif 1354 } else { 1355 BUG(); 1356 } 1357 1358 BUG_ON(num_refs < refs_to_drop); 1359 num_refs -= refs_to_drop; 1360 1361 if (num_refs == 0) { 1362 ret = btrfs_del_item(trans, root, path); 1363 *last_ref = 1; 1364 } else { 1365 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) 1366 btrfs_set_extent_data_ref_count(leaf, ref1, num_refs); 1367 else if (key.type == BTRFS_SHARED_DATA_REF_KEY) 1368 btrfs_set_shared_data_ref_count(leaf, ref2, num_refs); 1369 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1370 else { 1371 struct btrfs_extent_ref_v0 *ref0; 1372 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1373 struct btrfs_extent_ref_v0); 1374 btrfs_set_ref_count_v0(leaf, ref0, num_refs); 1375 } 1376 #endif 1377 btrfs_mark_buffer_dirty(leaf); 1378 } 1379 return ret; 1380 } 1381 1382 static noinline u32 extent_data_ref_count(struct btrfs_path *path, 1383 struct btrfs_extent_inline_ref *iref) 1384 { 1385 struct btrfs_key key; 1386 struct extent_buffer *leaf; 1387 struct btrfs_extent_data_ref *ref1; 1388 struct btrfs_shared_data_ref *ref2; 1389 u32 num_refs = 0; 1390 1391 leaf = path->nodes[0]; 1392 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1393 if (iref) { 1394 if (btrfs_extent_inline_ref_type(leaf, iref) == 1395 BTRFS_EXTENT_DATA_REF_KEY) { 1396 ref1 = (struct btrfs_extent_data_ref *)(&iref->offset); 1397 num_refs = btrfs_extent_data_ref_count(leaf, ref1); 1398 } else { 1399 ref2 = (struct btrfs_shared_data_ref *)(iref + 1); 1400 num_refs = btrfs_shared_data_ref_count(leaf, ref2); 1401 } 1402 } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { 1403 ref1 = btrfs_item_ptr(leaf, path->slots[0], 1404 struct btrfs_extent_data_ref); 1405 num_refs = btrfs_extent_data_ref_count(leaf, ref1); 1406 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { 1407 ref2 = btrfs_item_ptr(leaf, path->slots[0], 1408 struct btrfs_shared_data_ref); 1409 num_refs = btrfs_shared_data_ref_count(leaf, ref2); 1410 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1411 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) { 1412 struct btrfs_extent_ref_v0 *ref0; 1413 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1414 struct btrfs_extent_ref_v0); 1415 num_refs = btrfs_ref_count_v0(leaf, ref0); 1416 #endif 1417 } else { 1418 WARN_ON(1); 1419 } 1420 return num_refs; 1421 } 1422 1423 static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans, 1424 struct btrfs_root *root, 1425 struct btrfs_path *path, 1426 u64 bytenr, u64 parent, 1427 u64 root_objectid) 1428 { 1429 struct btrfs_key key; 1430 int ret; 1431 1432 key.objectid = bytenr; 1433 if (parent) { 1434 key.type = BTRFS_SHARED_BLOCK_REF_KEY; 1435 key.offset = parent; 1436 } else { 1437 key.type = BTRFS_TREE_BLOCK_REF_KEY; 1438 key.offset = root_objectid; 1439 } 1440 1441 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1442 if (ret > 0) 1443 ret = -ENOENT; 1444 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1445 if (ret == -ENOENT && parent) { 1446 btrfs_release_path(path); 1447 key.type = BTRFS_EXTENT_REF_V0_KEY; 1448 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1449 if (ret > 0) 1450 ret = -ENOENT; 1451 } 1452 #endif 1453 return ret; 1454 } 1455 1456 static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans, 1457 struct btrfs_root *root, 1458 struct btrfs_path *path, 1459 u64 bytenr, u64 parent, 1460 u64 root_objectid) 1461 { 1462 struct btrfs_key key; 1463 int ret; 1464 1465 key.objectid = bytenr; 1466 if (parent) { 1467 key.type = BTRFS_SHARED_BLOCK_REF_KEY; 1468 key.offset = parent; 1469 } else { 1470 key.type = BTRFS_TREE_BLOCK_REF_KEY; 1471 key.offset = root_objectid; 1472 } 1473 1474 ret = btrfs_insert_empty_item(trans, root, path, &key, 0); 1475 btrfs_release_path(path); 1476 return ret; 1477 } 1478 1479 static inline int extent_ref_type(u64 parent, u64 owner) 1480 { 1481 int type; 1482 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1483 if (parent > 0) 1484 type = BTRFS_SHARED_BLOCK_REF_KEY; 1485 else 1486 type = BTRFS_TREE_BLOCK_REF_KEY; 1487 } else { 1488 if (parent > 0) 1489 type = BTRFS_SHARED_DATA_REF_KEY; 1490 else 1491 type = BTRFS_EXTENT_DATA_REF_KEY; 1492 } 1493 return type; 1494 } 1495 1496 static int find_next_key(struct btrfs_path *path, int level, 1497 struct btrfs_key *key) 1498 1499 { 1500 for (; level < BTRFS_MAX_LEVEL; level++) { 1501 if (!path->nodes[level]) 1502 break; 1503 if (path->slots[level] + 1 >= 1504 btrfs_header_nritems(path->nodes[level])) 1505 continue; 1506 if (level == 0) 1507 btrfs_item_key_to_cpu(path->nodes[level], key, 1508 path->slots[level] + 1); 1509 else 1510 btrfs_node_key_to_cpu(path->nodes[level], key, 1511 path->slots[level] + 1); 1512 return 0; 1513 } 1514 return 1; 1515 } 1516 1517 /* 1518 * look for inline back ref. if back ref is found, *ref_ret is set 1519 * to the address of inline back ref, and 0 is returned. 1520 * 1521 * if back ref isn't found, *ref_ret is set to the address where it 1522 * should be inserted, and -ENOENT is returned. 1523 * 1524 * if insert is true and there are too many inline back refs, the path 1525 * points to the extent item, and -EAGAIN is returned. 1526 * 1527 * NOTE: inline back refs are ordered in the same way that back ref 1528 * items in the tree are ordered. 1529 */ 1530 static noinline_for_stack 1531 int lookup_inline_extent_backref(struct btrfs_trans_handle *trans, 1532 struct btrfs_root *root, 1533 struct btrfs_path *path, 1534 struct btrfs_extent_inline_ref **ref_ret, 1535 u64 bytenr, u64 num_bytes, 1536 u64 parent, u64 root_objectid, 1537 u64 owner, u64 offset, int insert) 1538 { 1539 struct btrfs_key key; 1540 struct extent_buffer *leaf; 1541 struct btrfs_extent_item *ei; 1542 struct btrfs_extent_inline_ref *iref; 1543 u64 flags; 1544 u64 item_size; 1545 unsigned long ptr; 1546 unsigned long end; 1547 int extra_size; 1548 int type; 1549 int want; 1550 int ret; 1551 int err = 0; 1552 bool skinny_metadata = btrfs_fs_incompat(root->fs_info, 1553 SKINNY_METADATA); 1554 1555 key.objectid = bytenr; 1556 key.type = BTRFS_EXTENT_ITEM_KEY; 1557 key.offset = num_bytes; 1558 1559 want = extent_ref_type(parent, owner); 1560 if (insert) { 1561 extra_size = btrfs_extent_inline_ref_size(want); 1562 path->keep_locks = 1; 1563 } else 1564 extra_size = -1; 1565 1566 /* 1567 * Owner is our parent level, so we can just add one to get the level 1568 * for the block we are interested in. 1569 */ 1570 if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) { 1571 key.type = BTRFS_METADATA_ITEM_KEY; 1572 key.offset = owner; 1573 } 1574 1575 again: 1576 ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1); 1577 if (ret < 0) { 1578 err = ret; 1579 goto out; 1580 } 1581 1582 /* 1583 * We may be a newly converted file system which still has the old fat 1584 * extent entries for metadata, so try and see if we have one of those. 1585 */ 1586 if (ret > 0 && skinny_metadata) { 1587 skinny_metadata = false; 1588 if (path->slots[0]) { 1589 path->slots[0]--; 1590 btrfs_item_key_to_cpu(path->nodes[0], &key, 1591 path->slots[0]); 1592 if (key.objectid == bytenr && 1593 key.type == BTRFS_EXTENT_ITEM_KEY && 1594 key.offset == num_bytes) 1595 ret = 0; 1596 } 1597 if (ret) { 1598 key.objectid = bytenr; 1599 key.type = BTRFS_EXTENT_ITEM_KEY; 1600 key.offset = num_bytes; 1601 btrfs_release_path(path); 1602 goto again; 1603 } 1604 } 1605 1606 if (ret && !insert) { 1607 err = -ENOENT; 1608 goto out; 1609 } else if (WARN_ON(ret)) { 1610 err = -EIO; 1611 goto out; 1612 } 1613 1614 leaf = path->nodes[0]; 1615 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 1616 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1617 if (item_size < sizeof(*ei)) { 1618 if (!insert) { 1619 err = -ENOENT; 1620 goto out; 1621 } 1622 ret = convert_extent_item_v0(trans, root, path, owner, 1623 extra_size); 1624 if (ret < 0) { 1625 err = ret; 1626 goto out; 1627 } 1628 leaf = path->nodes[0]; 1629 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 1630 } 1631 #endif 1632 BUG_ON(item_size < sizeof(*ei)); 1633 1634 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1635 flags = btrfs_extent_flags(leaf, ei); 1636 1637 ptr = (unsigned long)(ei + 1); 1638 end = (unsigned long)ei + item_size; 1639 1640 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) { 1641 ptr += sizeof(struct btrfs_tree_block_info); 1642 BUG_ON(ptr > end); 1643 } 1644 1645 err = -ENOENT; 1646 while (1) { 1647 if (ptr >= end) { 1648 WARN_ON(ptr > end); 1649 break; 1650 } 1651 iref = (struct btrfs_extent_inline_ref *)ptr; 1652 type = btrfs_extent_inline_ref_type(leaf, iref); 1653 if (want < type) 1654 break; 1655 if (want > type) { 1656 ptr += btrfs_extent_inline_ref_size(type); 1657 continue; 1658 } 1659 1660 if (type == BTRFS_EXTENT_DATA_REF_KEY) { 1661 struct btrfs_extent_data_ref *dref; 1662 dref = (struct btrfs_extent_data_ref *)(&iref->offset); 1663 if (match_extent_data_ref(leaf, dref, root_objectid, 1664 owner, offset)) { 1665 err = 0; 1666 break; 1667 } 1668 if (hash_extent_data_ref_item(leaf, dref) < 1669 hash_extent_data_ref(root_objectid, owner, offset)) 1670 break; 1671 } else { 1672 u64 ref_offset; 1673 ref_offset = btrfs_extent_inline_ref_offset(leaf, iref); 1674 if (parent > 0) { 1675 if (parent == ref_offset) { 1676 err = 0; 1677 break; 1678 } 1679 if (ref_offset < parent) 1680 break; 1681 } else { 1682 if (root_objectid == ref_offset) { 1683 err = 0; 1684 break; 1685 } 1686 if (ref_offset < root_objectid) 1687 break; 1688 } 1689 } 1690 ptr += btrfs_extent_inline_ref_size(type); 1691 } 1692 if (err == -ENOENT && insert) { 1693 if (item_size + extra_size >= 1694 BTRFS_MAX_EXTENT_ITEM_SIZE(root)) { 1695 err = -EAGAIN; 1696 goto out; 1697 } 1698 /* 1699 * To add new inline back ref, we have to make sure 1700 * there is no corresponding back ref item. 1701 * For simplicity, we just do not add new inline back 1702 * ref if there is any kind of item for this block 1703 */ 1704 if (find_next_key(path, 0, &key) == 0 && 1705 key.objectid == bytenr && 1706 key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) { 1707 err = -EAGAIN; 1708 goto out; 1709 } 1710 } 1711 *ref_ret = (struct btrfs_extent_inline_ref *)ptr; 1712 out: 1713 if (insert) { 1714 path->keep_locks = 0; 1715 btrfs_unlock_up_safe(path, 1); 1716 } 1717 return err; 1718 } 1719 1720 /* 1721 * helper to add new inline back ref 1722 */ 1723 static noinline_for_stack 1724 void setup_inline_extent_backref(struct btrfs_root *root, 1725 struct btrfs_path *path, 1726 struct btrfs_extent_inline_ref *iref, 1727 u64 parent, u64 root_objectid, 1728 u64 owner, u64 offset, int refs_to_add, 1729 struct btrfs_delayed_extent_op *extent_op) 1730 { 1731 struct extent_buffer *leaf; 1732 struct btrfs_extent_item *ei; 1733 unsigned long ptr; 1734 unsigned long end; 1735 unsigned long item_offset; 1736 u64 refs; 1737 int size; 1738 int type; 1739 1740 leaf = path->nodes[0]; 1741 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1742 item_offset = (unsigned long)iref - (unsigned long)ei; 1743 1744 type = extent_ref_type(parent, owner); 1745 size = btrfs_extent_inline_ref_size(type); 1746 1747 btrfs_extend_item(root, path, size); 1748 1749 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1750 refs = btrfs_extent_refs(leaf, ei); 1751 refs += refs_to_add; 1752 btrfs_set_extent_refs(leaf, ei, refs); 1753 if (extent_op) 1754 __run_delayed_extent_op(extent_op, leaf, ei); 1755 1756 ptr = (unsigned long)ei + item_offset; 1757 end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]); 1758 if (ptr < end - size) 1759 memmove_extent_buffer(leaf, ptr + size, ptr, 1760 end - size - ptr); 1761 1762 iref = (struct btrfs_extent_inline_ref *)ptr; 1763 btrfs_set_extent_inline_ref_type(leaf, iref, type); 1764 if (type == BTRFS_EXTENT_DATA_REF_KEY) { 1765 struct btrfs_extent_data_ref *dref; 1766 dref = (struct btrfs_extent_data_ref *)(&iref->offset); 1767 btrfs_set_extent_data_ref_root(leaf, dref, root_objectid); 1768 btrfs_set_extent_data_ref_objectid(leaf, dref, owner); 1769 btrfs_set_extent_data_ref_offset(leaf, dref, offset); 1770 btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add); 1771 } else if (type == BTRFS_SHARED_DATA_REF_KEY) { 1772 struct btrfs_shared_data_ref *sref; 1773 sref = (struct btrfs_shared_data_ref *)(iref + 1); 1774 btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add); 1775 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 1776 } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) { 1777 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 1778 } else { 1779 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid); 1780 } 1781 btrfs_mark_buffer_dirty(leaf); 1782 } 1783 1784 static int lookup_extent_backref(struct btrfs_trans_handle *trans, 1785 struct btrfs_root *root, 1786 struct btrfs_path *path, 1787 struct btrfs_extent_inline_ref **ref_ret, 1788 u64 bytenr, u64 num_bytes, u64 parent, 1789 u64 root_objectid, u64 owner, u64 offset) 1790 { 1791 int ret; 1792 1793 ret = lookup_inline_extent_backref(trans, root, path, ref_ret, 1794 bytenr, num_bytes, parent, 1795 root_objectid, owner, offset, 0); 1796 if (ret != -ENOENT) 1797 return ret; 1798 1799 btrfs_release_path(path); 1800 *ref_ret = NULL; 1801 1802 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1803 ret = lookup_tree_block_ref(trans, root, path, bytenr, parent, 1804 root_objectid); 1805 } else { 1806 ret = lookup_extent_data_ref(trans, root, path, bytenr, parent, 1807 root_objectid, owner, offset); 1808 } 1809 return ret; 1810 } 1811 1812 /* 1813 * helper to update/remove inline back ref 1814 */ 1815 static noinline_for_stack 1816 void update_inline_extent_backref(struct btrfs_root *root, 1817 struct btrfs_path *path, 1818 struct btrfs_extent_inline_ref *iref, 1819 int refs_to_mod, 1820 struct btrfs_delayed_extent_op *extent_op, 1821 int *last_ref) 1822 { 1823 struct extent_buffer *leaf; 1824 struct btrfs_extent_item *ei; 1825 struct btrfs_extent_data_ref *dref = NULL; 1826 struct btrfs_shared_data_ref *sref = NULL; 1827 unsigned long ptr; 1828 unsigned long end; 1829 u32 item_size; 1830 int size; 1831 int type; 1832 u64 refs; 1833 1834 leaf = path->nodes[0]; 1835 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1836 refs = btrfs_extent_refs(leaf, ei); 1837 WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0); 1838 refs += refs_to_mod; 1839 btrfs_set_extent_refs(leaf, ei, refs); 1840 if (extent_op) 1841 __run_delayed_extent_op(extent_op, leaf, ei); 1842 1843 type = btrfs_extent_inline_ref_type(leaf, iref); 1844 1845 if (type == BTRFS_EXTENT_DATA_REF_KEY) { 1846 dref = (struct btrfs_extent_data_ref *)(&iref->offset); 1847 refs = btrfs_extent_data_ref_count(leaf, dref); 1848 } else if (type == BTRFS_SHARED_DATA_REF_KEY) { 1849 sref = (struct btrfs_shared_data_ref *)(iref + 1); 1850 refs = btrfs_shared_data_ref_count(leaf, sref); 1851 } else { 1852 refs = 1; 1853 BUG_ON(refs_to_mod != -1); 1854 } 1855 1856 BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod); 1857 refs += refs_to_mod; 1858 1859 if (refs > 0) { 1860 if (type == BTRFS_EXTENT_DATA_REF_KEY) 1861 btrfs_set_extent_data_ref_count(leaf, dref, refs); 1862 else 1863 btrfs_set_shared_data_ref_count(leaf, sref, refs); 1864 } else { 1865 *last_ref = 1; 1866 size = btrfs_extent_inline_ref_size(type); 1867 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 1868 ptr = (unsigned long)iref; 1869 end = (unsigned long)ei + item_size; 1870 if (ptr + size < end) 1871 memmove_extent_buffer(leaf, ptr, ptr + size, 1872 end - ptr - size); 1873 item_size -= size; 1874 btrfs_truncate_item(root, path, item_size, 1); 1875 } 1876 btrfs_mark_buffer_dirty(leaf); 1877 } 1878 1879 static noinline_for_stack 1880 int insert_inline_extent_backref(struct btrfs_trans_handle *trans, 1881 struct btrfs_root *root, 1882 struct btrfs_path *path, 1883 u64 bytenr, u64 num_bytes, u64 parent, 1884 u64 root_objectid, u64 owner, 1885 u64 offset, int refs_to_add, 1886 struct btrfs_delayed_extent_op *extent_op) 1887 { 1888 struct btrfs_extent_inline_ref *iref; 1889 int ret; 1890 1891 ret = lookup_inline_extent_backref(trans, root, path, &iref, 1892 bytenr, num_bytes, parent, 1893 root_objectid, owner, offset, 1); 1894 if (ret == 0) { 1895 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID); 1896 update_inline_extent_backref(root, path, iref, 1897 refs_to_add, extent_op, NULL); 1898 } else if (ret == -ENOENT) { 1899 setup_inline_extent_backref(root, path, iref, parent, 1900 root_objectid, owner, offset, 1901 refs_to_add, extent_op); 1902 ret = 0; 1903 } 1904 return ret; 1905 } 1906 1907 static int insert_extent_backref(struct btrfs_trans_handle *trans, 1908 struct btrfs_root *root, 1909 struct btrfs_path *path, 1910 u64 bytenr, u64 parent, u64 root_objectid, 1911 u64 owner, u64 offset, int refs_to_add) 1912 { 1913 int ret; 1914 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1915 BUG_ON(refs_to_add != 1); 1916 ret = insert_tree_block_ref(trans, root, path, bytenr, 1917 parent, root_objectid); 1918 } else { 1919 ret = insert_extent_data_ref(trans, root, path, bytenr, 1920 parent, root_objectid, 1921 owner, offset, refs_to_add); 1922 } 1923 return ret; 1924 } 1925 1926 static int remove_extent_backref(struct btrfs_trans_handle *trans, 1927 struct btrfs_root *root, 1928 struct btrfs_path *path, 1929 struct btrfs_extent_inline_ref *iref, 1930 int refs_to_drop, int is_data, int *last_ref) 1931 { 1932 int ret = 0; 1933 1934 BUG_ON(!is_data && refs_to_drop != 1); 1935 if (iref) { 1936 update_inline_extent_backref(root, path, iref, 1937 -refs_to_drop, NULL, last_ref); 1938 } else if (is_data) { 1939 ret = remove_extent_data_ref(trans, root, path, refs_to_drop, 1940 last_ref); 1941 } else { 1942 *last_ref = 1; 1943 ret = btrfs_del_item(trans, root, path); 1944 } 1945 return ret; 1946 } 1947 1948 #define in_range(b, first, len) ((b) >= (first) && (b) < (first) + (len)) 1949 static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len, 1950 u64 *discarded_bytes) 1951 { 1952 int j, ret = 0; 1953 u64 bytes_left, end; 1954 u64 aligned_start = ALIGN(start, 1 << 9); 1955 1956 if (WARN_ON(start != aligned_start)) { 1957 len -= aligned_start - start; 1958 len = round_down(len, 1 << 9); 1959 start = aligned_start; 1960 } 1961 1962 *discarded_bytes = 0; 1963 1964 if (!len) 1965 return 0; 1966 1967 end = start + len; 1968 bytes_left = len; 1969 1970 /* Skip any superblocks on this device. */ 1971 for (j = 0; j < BTRFS_SUPER_MIRROR_MAX; j++) { 1972 u64 sb_start = btrfs_sb_offset(j); 1973 u64 sb_end = sb_start + BTRFS_SUPER_INFO_SIZE; 1974 u64 size = sb_start - start; 1975 1976 if (!in_range(sb_start, start, bytes_left) && 1977 !in_range(sb_end, start, bytes_left) && 1978 !in_range(start, sb_start, BTRFS_SUPER_INFO_SIZE)) 1979 continue; 1980 1981 /* 1982 * Superblock spans beginning of range. Adjust start and 1983 * try again. 1984 */ 1985 if (sb_start <= start) { 1986 start += sb_end - start; 1987 if (start > end) { 1988 bytes_left = 0; 1989 break; 1990 } 1991 bytes_left = end - start; 1992 continue; 1993 } 1994 1995 if (size) { 1996 ret = blkdev_issue_discard(bdev, start >> 9, size >> 9, 1997 GFP_NOFS, 0); 1998 if (!ret) 1999 *discarded_bytes += size; 2000 else if (ret != -EOPNOTSUPP) 2001 return ret; 2002 } 2003 2004 start = sb_end; 2005 if (start > end) { 2006 bytes_left = 0; 2007 break; 2008 } 2009 bytes_left = end - start; 2010 } 2011 2012 if (bytes_left) { 2013 ret = blkdev_issue_discard(bdev, start >> 9, bytes_left >> 9, 2014 GFP_NOFS, 0); 2015 if (!ret) 2016 *discarded_bytes += bytes_left; 2017 } 2018 return ret; 2019 } 2020 2021 int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr, 2022 u64 num_bytes, u64 *actual_bytes) 2023 { 2024 int ret; 2025 u64 discarded_bytes = 0; 2026 struct btrfs_bio *bbio = NULL; 2027 2028 2029 /* Tell the block device(s) that the sectors can be discarded */ 2030 ret = btrfs_map_block(root->fs_info, REQ_DISCARD, 2031 bytenr, &num_bytes, &bbio, 0); 2032 /* Error condition is -ENOMEM */ 2033 if (!ret) { 2034 struct btrfs_bio_stripe *stripe = bbio->stripes; 2035 int i; 2036 2037 2038 for (i = 0; i < bbio->num_stripes; i++, stripe++) { 2039 u64 bytes; 2040 if (!stripe->dev->can_discard) 2041 continue; 2042 2043 ret = btrfs_issue_discard(stripe->dev->bdev, 2044 stripe->physical, 2045 stripe->length, 2046 &bytes); 2047 if (!ret) 2048 discarded_bytes += bytes; 2049 else if (ret != -EOPNOTSUPP) 2050 break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */ 2051 2052 /* 2053 * Just in case we get back EOPNOTSUPP for some reason, 2054 * just ignore the return value so we don't screw up 2055 * people calling discard_extent. 2056 */ 2057 ret = 0; 2058 } 2059 btrfs_put_bbio(bbio); 2060 } 2061 2062 if (actual_bytes) 2063 *actual_bytes = discarded_bytes; 2064 2065 2066 if (ret == -EOPNOTSUPP) 2067 ret = 0; 2068 return ret; 2069 } 2070 2071 /* Can return -ENOMEM */ 2072 int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, 2073 struct btrfs_root *root, 2074 u64 bytenr, u64 num_bytes, u64 parent, 2075 u64 root_objectid, u64 owner, u64 offset) 2076 { 2077 int ret; 2078 struct btrfs_fs_info *fs_info = root->fs_info; 2079 2080 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID && 2081 root_objectid == BTRFS_TREE_LOG_OBJECTID); 2082 2083 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 2084 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr, 2085 num_bytes, 2086 parent, root_objectid, (int)owner, 2087 BTRFS_ADD_DELAYED_REF, NULL); 2088 } else { 2089 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr, 2090 num_bytes, parent, root_objectid, 2091 owner, offset, 0, 2092 BTRFS_ADD_DELAYED_REF, NULL); 2093 } 2094 return ret; 2095 } 2096 2097 static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, 2098 struct btrfs_root *root, 2099 struct btrfs_delayed_ref_node *node, 2100 u64 parent, u64 root_objectid, 2101 u64 owner, u64 offset, int refs_to_add, 2102 struct btrfs_delayed_extent_op *extent_op) 2103 { 2104 struct btrfs_fs_info *fs_info = root->fs_info; 2105 struct btrfs_path *path; 2106 struct extent_buffer *leaf; 2107 struct btrfs_extent_item *item; 2108 struct btrfs_key key; 2109 u64 bytenr = node->bytenr; 2110 u64 num_bytes = node->num_bytes; 2111 u64 refs; 2112 int ret; 2113 2114 path = btrfs_alloc_path(); 2115 if (!path) 2116 return -ENOMEM; 2117 2118 path->reada = 1; 2119 path->leave_spinning = 1; 2120 /* this will setup the path even if it fails to insert the back ref */ 2121 ret = insert_inline_extent_backref(trans, fs_info->extent_root, path, 2122 bytenr, num_bytes, parent, 2123 root_objectid, owner, offset, 2124 refs_to_add, extent_op); 2125 if ((ret < 0 && ret != -EAGAIN) || !ret) 2126 goto out; 2127 2128 /* 2129 * Ok we had -EAGAIN which means we didn't have space to insert and 2130 * inline extent ref, so just update the reference count and add a 2131 * normal backref. 2132 */ 2133 leaf = path->nodes[0]; 2134 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 2135 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 2136 refs = btrfs_extent_refs(leaf, item); 2137 btrfs_set_extent_refs(leaf, item, refs + refs_to_add); 2138 if (extent_op) 2139 __run_delayed_extent_op(extent_op, leaf, item); 2140 2141 btrfs_mark_buffer_dirty(leaf); 2142 btrfs_release_path(path); 2143 2144 path->reada = 1; 2145 path->leave_spinning = 1; 2146 /* now insert the actual backref */ 2147 ret = insert_extent_backref(trans, root->fs_info->extent_root, 2148 path, bytenr, parent, root_objectid, 2149 owner, offset, refs_to_add); 2150 if (ret) 2151 btrfs_abort_transaction(trans, root, ret); 2152 out: 2153 btrfs_free_path(path); 2154 return ret; 2155 } 2156 2157 static int run_delayed_data_ref(struct btrfs_trans_handle *trans, 2158 struct btrfs_root *root, 2159 struct btrfs_delayed_ref_node *node, 2160 struct btrfs_delayed_extent_op *extent_op, 2161 int insert_reserved) 2162 { 2163 int ret = 0; 2164 struct btrfs_delayed_data_ref *ref; 2165 struct btrfs_key ins; 2166 u64 parent = 0; 2167 u64 ref_root = 0; 2168 u64 flags = 0; 2169 2170 ins.objectid = node->bytenr; 2171 ins.offset = node->num_bytes; 2172 ins.type = BTRFS_EXTENT_ITEM_KEY; 2173 2174 ref = btrfs_delayed_node_to_data_ref(node); 2175 trace_run_delayed_data_ref(node, ref, node->action); 2176 2177 if (node->type == BTRFS_SHARED_DATA_REF_KEY) 2178 parent = ref->parent; 2179 ref_root = ref->root; 2180 2181 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { 2182 if (extent_op) 2183 flags |= extent_op->flags_to_set; 2184 ret = alloc_reserved_file_extent(trans, root, 2185 parent, ref_root, flags, 2186 ref->objectid, ref->offset, 2187 &ins, node->ref_mod); 2188 } else if (node->action == BTRFS_ADD_DELAYED_REF) { 2189 ret = __btrfs_inc_extent_ref(trans, root, node, parent, 2190 ref_root, ref->objectid, 2191 ref->offset, node->ref_mod, 2192 extent_op); 2193 } else if (node->action == BTRFS_DROP_DELAYED_REF) { 2194 ret = __btrfs_free_extent(trans, root, node, parent, 2195 ref_root, ref->objectid, 2196 ref->offset, node->ref_mod, 2197 extent_op); 2198 } else { 2199 BUG(); 2200 } 2201 return ret; 2202 } 2203 2204 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, 2205 struct extent_buffer *leaf, 2206 struct btrfs_extent_item *ei) 2207 { 2208 u64 flags = btrfs_extent_flags(leaf, ei); 2209 if (extent_op->update_flags) { 2210 flags |= extent_op->flags_to_set; 2211 btrfs_set_extent_flags(leaf, ei, flags); 2212 } 2213 2214 if (extent_op->update_key) { 2215 struct btrfs_tree_block_info *bi; 2216 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)); 2217 bi = (struct btrfs_tree_block_info *)(ei + 1); 2218 btrfs_set_tree_block_key(leaf, bi, &extent_op->key); 2219 } 2220 } 2221 2222 static int run_delayed_extent_op(struct btrfs_trans_handle *trans, 2223 struct btrfs_root *root, 2224 struct btrfs_delayed_ref_node *node, 2225 struct btrfs_delayed_extent_op *extent_op) 2226 { 2227 struct btrfs_key key; 2228 struct btrfs_path *path; 2229 struct btrfs_extent_item *ei; 2230 struct extent_buffer *leaf; 2231 u32 item_size; 2232 int ret; 2233 int err = 0; 2234 int metadata = !extent_op->is_data; 2235 2236 if (trans->aborted) 2237 return 0; 2238 2239 if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA)) 2240 metadata = 0; 2241 2242 path = btrfs_alloc_path(); 2243 if (!path) 2244 return -ENOMEM; 2245 2246 key.objectid = node->bytenr; 2247 2248 if (metadata) { 2249 key.type = BTRFS_METADATA_ITEM_KEY; 2250 key.offset = extent_op->level; 2251 } else { 2252 key.type = BTRFS_EXTENT_ITEM_KEY; 2253 key.offset = node->num_bytes; 2254 } 2255 2256 again: 2257 path->reada = 1; 2258 path->leave_spinning = 1; 2259 ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key, 2260 path, 0, 1); 2261 if (ret < 0) { 2262 err = ret; 2263 goto out; 2264 } 2265 if (ret > 0) { 2266 if (metadata) { 2267 if (path->slots[0] > 0) { 2268 path->slots[0]--; 2269 btrfs_item_key_to_cpu(path->nodes[0], &key, 2270 path->slots[0]); 2271 if (key.objectid == node->bytenr && 2272 key.type == BTRFS_EXTENT_ITEM_KEY && 2273 key.offset == node->num_bytes) 2274 ret = 0; 2275 } 2276 if (ret > 0) { 2277 btrfs_release_path(path); 2278 metadata = 0; 2279 2280 key.objectid = node->bytenr; 2281 key.offset = node->num_bytes; 2282 key.type = BTRFS_EXTENT_ITEM_KEY; 2283 goto again; 2284 } 2285 } else { 2286 err = -EIO; 2287 goto out; 2288 } 2289 } 2290 2291 leaf = path->nodes[0]; 2292 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 2293 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 2294 if (item_size < sizeof(*ei)) { 2295 ret = convert_extent_item_v0(trans, root->fs_info->extent_root, 2296 path, (u64)-1, 0); 2297 if (ret < 0) { 2298 err = ret; 2299 goto out; 2300 } 2301 leaf = path->nodes[0]; 2302 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 2303 } 2304 #endif 2305 BUG_ON(item_size < sizeof(*ei)); 2306 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 2307 __run_delayed_extent_op(extent_op, leaf, ei); 2308 2309 btrfs_mark_buffer_dirty(leaf); 2310 out: 2311 btrfs_free_path(path); 2312 return err; 2313 } 2314 2315 static int run_delayed_tree_ref(struct btrfs_trans_handle *trans, 2316 struct btrfs_root *root, 2317 struct btrfs_delayed_ref_node *node, 2318 struct btrfs_delayed_extent_op *extent_op, 2319 int insert_reserved) 2320 { 2321 int ret = 0; 2322 struct btrfs_delayed_tree_ref *ref; 2323 struct btrfs_key ins; 2324 u64 parent = 0; 2325 u64 ref_root = 0; 2326 bool skinny_metadata = btrfs_fs_incompat(root->fs_info, 2327 SKINNY_METADATA); 2328 2329 ref = btrfs_delayed_node_to_tree_ref(node); 2330 trace_run_delayed_tree_ref(node, ref, node->action); 2331 2332 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) 2333 parent = ref->parent; 2334 ref_root = ref->root; 2335 2336 ins.objectid = node->bytenr; 2337 if (skinny_metadata) { 2338 ins.offset = ref->level; 2339 ins.type = BTRFS_METADATA_ITEM_KEY; 2340 } else { 2341 ins.offset = node->num_bytes; 2342 ins.type = BTRFS_EXTENT_ITEM_KEY; 2343 } 2344 2345 BUG_ON(node->ref_mod != 1); 2346 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { 2347 BUG_ON(!extent_op || !extent_op->update_flags); 2348 ret = alloc_reserved_tree_block(trans, root, 2349 parent, ref_root, 2350 extent_op->flags_to_set, 2351 &extent_op->key, 2352 ref->level, &ins); 2353 } else if (node->action == BTRFS_ADD_DELAYED_REF) { 2354 ret = __btrfs_inc_extent_ref(trans, root, node, 2355 parent, ref_root, 2356 ref->level, 0, 1, 2357 extent_op); 2358 } else if (node->action == BTRFS_DROP_DELAYED_REF) { 2359 ret = __btrfs_free_extent(trans, root, node, 2360 parent, ref_root, 2361 ref->level, 0, 1, extent_op); 2362 } else { 2363 BUG(); 2364 } 2365 return ret; 2366 } 2367 2368 /* helper function to actually process a single delayed ref entry */ 2369 static int run_one_delayed_ref(struct btrfs_trans_handle *trans, 2370 struct btrfs_root *root, 2371 struct btrfs_delayed_ref_node *node, 2372 struct btrfs_delayed_extent_op *extent_op, 2373 int insert_reserved) 2374 { 2375 int ret = 0; 2376 2377 if (trans->aborted) { 2378 if (insert_reserved) 2379 btrfs_pin_extent(root, node->bytenr, 2380 node->num_bytes, 1); 2381 return 0; 2382 } 2383 2384 if (btrfs_delayed_ref_is_head(node)) { 2385 struct btrfs_delayed_ref_head *head; 2386 /* 2387 * we've hit the end of the chain and we were supposed 2388 * to insert this extent into the tree. But, it got 2389 * deleted before we ever needed to insert it, so all 2390 * we have to do is clean up the accounting 2391 */ 2392 BUG_ON(extent_op); 2393 head = btrfs_delayed_node_to_head(node); 2394 trace_run_delayed_ref_head(node, head, node->action); 2395 2396 if (insert_reserved) { 2397 btrfs_pin_extent(root, node->bytenr, 2398 node->num_bytes, 1); 2399 if (head->is_data) { 2400 ret = btrfs_del_csums(trans, root, 2401 node->bytenr, 2402 node->num_bytes); 2403 } 2404 } 2405 2406 /* Also free its reserved qgroup space */ 2407 btrfs_qgroup_free_delayed_ref(root->fs_info, 2408 head->qgroup_ref_root, 2409 head->qgroup_reserved); 2410 return ret; 2411 } 2412 2413 if (node->type == BTRFS_TREE_BLOCK_REF_KEY || 2414 node->type == BTRFS_SHARED_BLOCK_REF_KEY) 2415 ret = run_delayed_tree_ref(trans, root, node, extent_op, 2416 insert_reserved); 2417 else if (node->type == BTRFS_EXTENT_DATA_REF_KEY || 2418 node->type == BTRFS_SHARED_DATA_REF_KEY) 2419 ret = run_delayed_data_ref(trans, root, node, extent_op, 2420 insert_reserved); 2421 else 2422 BUG(); 2423 return ret; 2424 } 2425 2426 static inline struct btrfs_delayed_ref_node * 2427 select_delayed_ref(struct btrfs_delayed_ref_head *head) 2428 { 2429 struct btrfs_delayed_ref_node *ref; 2430 2431 if (list_empty(&head->ref_list)) 2432 return NULL; 2433 2434 /* 2435 * Select a delayed ref of type BTRFS_ADD_DELAYED_REF first. 2436 * This is to prevent a ref count from going down to zero, which deletes 2437 * the extent item from the extent tree, when there still are references 2438 * to add, which would fail because they would not find the extent item. 2439 */ 2440 list_for_each_entry(ref, &head->ref_list, list) { 2441 if (ref->action == BTRFS_ADD_DELAYED_REF) 2442 return ref; 2443 } 2444 2445 return list_entry(head->ref_list.next, struct btrfs_delayed_ref_node, 2446 list); 2447 } 2448 2449 /* 2450 * Returns 0 on success or if called with an already aborted transaction. 2451 * Returns -ENOMEM or -EIO on failure and will abort the transaction. 2452 */ 2453 static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, 2454 struct btrfs_root *root, 2455 unsigned long nr) 2456 { 2457 struct btrfs_delayed_ref_root *delayed_refs; 2458 struct btrfs_delayed_ref_node *ref; 2459 struct btrfs_delayed_ref_head *locked_ref = NULL; 2460 struct btrfs_delayed_extent_op *extent_op; 2461 struct btrfs_fs_info *fs_info = root->fs_info; 2462 ktime_t start = ktime_get(); 2463 int ret; 2464 unsigned long count = 0; 2465 unsigned long actual_count = 0; 2466 int must_insert_reserved = 0; 2467 2468 delayed_refs = &trans->transaction->delayed_refs; 2469 while (1) { 2470 if (!locked_ref) { 2471 if (count >= nr) 2472 break; 2473 2474 spin_lock(&delayed_refs->lock); 2475 locked_ref = btrfs_select_ref_head(trans); 2476 if (!locked_ref) { 2477 spin_unlock(&delayed_refs->lock); 2478 break; 2479 } 2480 2481 /* grab the lock that says we are going to process 2482 * all the refs for this head */ 2483 ret = btrfs_delayed_ref_lock(trans, locked_ref); 2484 spin_unlock(&delayed_refs->lock); 2485 /* 2486 * we may have dropped the spin lock to get the head 2487 * mutex lock, and that might have given someone else 2488 * time to free the head. If that's true, it has been 2489 * removed from our list and we can move on. 2490 */ 2491 if (ret == -EAGAIN) { 2492 locked_ref = NULL; 2493 count++; 2494 continue; 2495 } 2496 } 2497 2498 /* 2499 * We need to try and merge add/drops of the same ref since we 2500 * can run into issues with relocate dropping the implicit ref 2501 * and then it being added back again before the drop can 2502 * finish. If we merged anything we need to re-loop so we can 2503 * get a good ref. 2504 * Or we can get node references of the same type that weren't 2505 * merged when created due to bumps in the tree mod seq, and 2506 * we need to merge them to prevent adding an inline extent 2507 * backref before dropping it (triggering a BUG_ON at 2508 * insert_inline_extent_backref()). 2509 */ 2510 spin_lock(&locked_ref->lock); 2511 btrfs_merge_delayed_refs(trans, fs_info, delayed_refs, 2512 locked_ref); 2513 2514 /* 2515 * locked_ref is the head node, so we have to go one 2516 * node back for any delayed ref updates 2517 */ 2518 ref = select_delayed_ref(locked_ref); 2519 2520 if (ref && ref->seq && 2521 btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) { 2522 spin_unlock(&locked_ref->lock); 2523 btrfs_delayed_ref_unlock(locked_ref); 2524 spin_lock(&delayed_refs->lock); 2525 locked_ref->processing = 0; 2526 delayed_refs->num_heads_ready++; 2527 spin_unlock(&delayed_refs->lock); 2528 locked_ref = NULL; 2529 cond_resched(); 2530 count++; 2531 continue; 2532 } 2533 2534 /* 2535 * record the must insert reserved flag before we 2536 * drop the spin lock. 2537 */ 2538 must_insert_reserved = locked_ref->must_insert_reserved; 2539 locked_ref->must_insert_reserved = 0; 2540 2541 extent_op = locked_ref->extent_op; 2542 locked_ref->extent_op = NULL; 2543 2544 if (!ref) { 2545 2546 2547 /* All delayed refs have been processed, Go ahead 2548 * and send the head node to run_one_delayed_ref, 2549 * so that any accounting fixes can happen 2550 */ 2551 ref = &locked_ref->node; 2552 2553 if (extent_op && must_insert_reserved) { 2554 btrfs_free_delayed_extent_op(extent_op); 2555 extent_op = NULL; 2556 } 2557 2558 if (extent_op) { 2559 spin_unlock(&locked_ref->lock); 2560 ret = run_delayed_extent_op(trans, root, 2561 ref, extent_op); 2562 btrfs_free_delayed_extent_op(extent_op); 2563 2564 if (ret) { 2565 /* 2566 * Need to reset must_insert_reserved if 2567 * there was an error so the abort stuff 2568 * can cleanup the reserved space 2569 * properly. 2570 */ 2571 if (must_insert_reserved) 2572 locked_ref->must_insert_reserved = 1; 2573 locked_ref->processing = 0; 2574 btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret); 2575 btrfs_delayed_ref_unlock(locked_ref); 2576 return ret; 2577 } 2578 continue; 2579 } 2580 2581 /* 2582 * Need to drop our head ref lock and re-aqcuire the 2583 * delayed ref lock and then re-check to make sure 2584 * nobody got added. 2585 */ 2586 spin_unlock(&locked_ref->lock); 2587 spin_lock(&delayed_refs->lock); 2588 spin_lock(&locked_ref->lock); 2589 if (!list_empty(&locked_ref->ref_list) || 2590 locked_ref->extent_op) { 2591 spin_unlock(&locked_ref->lock); 2592 spin_unlock(&delayed_refs->lock); 2593 continue; 2594 } 2595 ref->in_tree = 0; 2596 delayed_refs->num_heads--; 2597 rb_erase(&locked_ref->href_node, 2598 &delayed_refs->href_root); 2599 spin_unlock(&delayed_refs->lock); 2600 } else { 2601 actual_count++; 2602 ref->in_tree = 0; 2603 list_del(&ref->list); 2604 } 2605 atomic_dec(&delayed_refs->num_entries); 2606 2607 if (!btrfs_delayed_ref_is_head(ref)) { 2608 /* 2609 * when we play the delayed ref, also correct the 2610 * ref_mod on head 2611 */ 2612 switch (ref->action) { 2613 case BTRFS_ADD_DELAYED_REF: 2614 case BTRFS_ADD_DELAYED_EXTENT: 2615 locked_ref->node.ref_mod -= ref->ref_mod; 2616 break; 2617 case BTRFS_DROP_DELAYED_REF: 2618 locked_ref->node.ref_mod += ref->ref_mod; 2619 break; 2620 default: 2621 WARN_ON(1); 2622 } 2623 } 2624 spin_unlock(&locked_ref->lock); 2625 2626 ret = run_one_delayed_ref(trans, root, ref, extent_op, 2627 must_insert_reserved); 2628 2629 btrfs_free_delayed_extent_op(extent_op); 2630 if (ret) { 2631 locked_ref->processing = 0; 2632 btrfs_delayed_ref_unlock(locked_ref); 2633 btrfs_put_delayed_ref(ref); 2634 btrfs_debug(fs_info, "run_one_delayed_ref returned %d", ret); 2635 return ret; 2636 } 2637 2638 /* 2639 * If this node is a head, that means all the refs in this head 2640 * have been dealt with, and we will pick the next head to deal 2641 * with, so we must unlock the head and drop it from the cluster 2642 * list before we release it. 2643 */ 2644 if (btrfs_delayed_ref_is_head(ref)) { 2645 if (locked_ref->is_data && 2646 locked_ref->total_ref_mod < 0) { 2647 spin_lock(&delayed_refs->lock); 2648 delayed_refs->pending_csums -= ref->num_bytes; 2649 spin_unlock(&delayed_refs->lock); 2650 } 2651 btrfs_delayed_ref_unlock(locked_ref); 2652 locked_ref = NULL; 2653 } 2654 btrfs_put_delayed_ref(ref); 2655 count++; 2656 cond_resched(); 2657 } 2658 2659 /* 2660 * We don't want to include ref heads since we can have empty ref heads 2661 * and those will drastically skew our runtime down since we just do 2662 * accounting, no actual extent tree updates. 2663 */ 2664 if (actual_count > 0) { 2665 u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start)); 2666 u64 avg; 2667 2668 /* 2669 * We weigh the current average higher than our current runtime 2670 * to avoid large swings in the average. 2671 */ 2672 spin_lock(&delayed_refs->lock); 2673 avg = fs_info->avg_delayed_ref_runtime * 3 + runtime; 2674 fs_info->avg_delayed_ref_runtime = avg >> 2; /* div by 4 */ 2675 spin_unlock(&delayed_refs->lock); 2676 } 2677 return 0; 2678 } 2679 2680 #ifdef SCRAMBLE_DELAYED_REFS 2681 /* 2682 * Normally delayed refs get processed in ascending bytenr order. This 2683 * correlates in most cases to the order added. To expose dependencies on this 2684 * order, we start to process the tree in the middle instead of the beginning 2685 */ 2686 static u64 find_middle(struct rb_root *root) 2687 { 2688 struct rb_node *n = root->rb_node; 2689 struct btrfs_delayed_ref_node *entry; 2690 int alt = 1; 2691 u64 middle; 2692 u64 first = 0, last = 0; 2693 2694 n = rb_first(root); 2695 if (n) { 2696 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); 2697 first = entry->bytenr; 2698 } 2699 n = rb_last(root); 2700 if (n) { 2701 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); 2702 last = entry->bytenr; 2703 } 2704 n = root->rb_node; 2705 2706 while (n) { 2707 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); 2708 WARN_ON(!entry->in_tree); 2709 2710 middle = entry->bytenr; 2711 2712 if (alt) 2713 n = n->rb_left; 2714 else 2715 n = n->rb_right; 2716 2717 alt = 1 - alt; 2718 } 2719 return middle; 2720 } 2721 #endif 2722 2723 static inline u64 heads_to_leaves(struct btrfs_root *root, u64 heads) 2724 { 2725 u64 num_bytes; 2726 2727 num_bytes = heads * (sizeof(struct btrfs_extent_item) + 2728 sizeof(struct btrfs_extent_inline_ref)); 2729 if (!btrfs_fs_incompat(root->fs_info, SKINNY_METADATA)) 2730 num_bytes += heads * sizeof(struct btrfs_tree_block_info); 2731 2732 /* 2733 * We don't ever fill up leaves all the way so multiply by 2 just to be 2734 * closer to what we're really going to want to ouse. 2735 */ 2736 return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(root)); 2737 } 2738 2739 /* 2740 * Takes the number of bytes to be csumm'ed and figures out how many leaves it 2741 * would require to store the csums for that many bytes. 2742 */ 2743 u64 btrfs_csum_bytes_to_leaves(struct btrfs_root *root, u64 csum_bytes) 2744 { 2745 u64 csum_size; 2746 u64 num_csums_per_leaf; 2747 u64 num_csums; 2748 2749 csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item); 2750 num_csums_per_leaf = div64_u64(csum_size, 2751 (u64)btrfs_super_csum_size(root->fs_info->super_copy)); 2752 num_csums = div64_u64(csum_bytes, root->sectorsize); 2753 num_csums += num_csums_per_leaf - 1; 2754 num_csums = div64_u64(num_csums, num_csums_per_leaf); 2755 return num_csums; 2756 } 2757 2758 int btrfs_check_space_for_delayed_refs(struct btrfs_trans_handle *trans, 2759 struct btrfs_root *root) 2760 { 2761 struct btrfs_block_rsv *global_rsv; 2762 u64 num_heads = trans->transaction->delayed_refs.num_heads_ready; 2763 u64 csum_bytes = trans->transaction->delayed_refs.pending_csums; 2764 u64 num_dirty_bgs = trans->transaction->num_dirty_bgs; 2765 u64 num_bytes, num_dirty_bgs_bytes; 2766 int ret = 0; 2767 2768 num_bytes = btrfs_calc_trans_metadata_size(root, 1); 2769 num_heads = heads_to_leaves(root, num_heads); 2770 if (num_heads > 1) 2771 num_bytes += (num_heads - 1) * root->nodesize; 2772 num_bytes <<= 1; 2773 num_bytes += btrfs_csum_bytes_to_leaves(root, csum_bytes) * root->nodesize; 2774 num_dirty_bgs_bytes = btrfs_calc_trans_metadata_size(root, 2775 num_dirty_bgs); 2776 global_rsv = &root->fs_info->global_block_rsv; 2777 2778 /* 2779 * If we can't allocate any more chunks lets make sure we have _lots_ of 2780 * wiggle room since running delayed refs can create more delayed refs. 2781 */ 2782 if (global_rsv->space_info->full) { 2783 num_dirty_bgs_bytes <<= 1; 2784 num_bytes <<= 1; 2785 } 2786 2787 spin_lock(&global_rsv->lock); 2788 if (global_rsv->reserved <= num_bytes + num_dirty_bgs_bytes) 2789 ret = 1; 2790 spin_unlock(&global_rsv->lock); 2791 return ret; 2792 } 2793 2794 int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans, 2795 struct btrfs_root *root) 2796 { 2797 struct btrfs_fs_info *fs_info = root->fs_info; 2798 u64 num_entries = 2799 atomic_read(&trans->transaction->delayed_refs.num_entries); 2800 u64 avg_runtime; 2801 u64 val; 2802 2803 smp_mb(); 2804 avg_runtime = fs_info->avg_delayed_ref_runtime; 2805 val = num_entries * avg_runtime; 2806 if (num_entries * avg_runtime >= NSEC_PER_SEC) 2807 return 1; 2808 if (val >= NSEC_PER_SEC / 2) 2809 return 2; 2810 2811 return btrfs_check_space_for_delayed_refs(trans, root); 2812 } 2813 2814 struct async_delayed_refs { 2815 struct btrfs_root *root; 2816 int count; 2817 int error; 2818 int sync; 2819 struct completion wait; 2820 struct btrfs_work work; 2821 }; 2822 2823 static void delayed_ref_async_start(struct btrfs_work *work) 2824 { 2825 struct async_delayed_refs *async; 2826 struct btrfs_trans_handle *trans; 2827 int ret; 2828 2829 async = container_of(work, struct async_delayed_refs, work); 2830 2831 trans = btrfs_join_transaction(async->root); 2832 if (IS_ERR(trans)) { 2833 async->error = PTR_ERR(trans); 2834 goto done; 2835 } 2836 2837 /* 2838 * trans->sync means that when we call end_transaciton, we won't 2839 * wait on delayed refs 2840 */ 2841 trans->sync = true; 2842 ret = btrfs_run_delayed_refs(trans, async->root, async->count); 2843 if (ret) 2844 async->error = ret; 2845 2846 ret = btrfs_end_transaction(trans, async->root); 2847 if (ret && !async->error) 2848 async->error = ret; 2849 done: 2850 if (async->sync) 2851 complete(&async->wait); 2852 else 2853 kfree(async); 2854 } 2855 2856 int btrfs_async_run_delayed_refs(struct btrfs_root *root, 2857 unsigned long count, int wait) 2858 { 2859 struct async_delayed_refs *async; 2860 int ret; 2861 2862 async = kmalloc(sizeof(*async), GFP_NOFS); 2863 if (!async) 2864 return -ENOMEM; 2865 2866 async->root = root->fs_info->tree_root; 2867 async->count = count; 2868 async->error = 0; 2869 if (wait) 2870 async->sync = 1; 2871 else 2872 async->sync = 0; 2873 init_completion(&async->wait); 2874 2875 btrfs_init_work(&async->work, btrfs_extent_refs_helper, 2876 delayed_ref_async_start, NULL, NULL); 2877 2878 btrfs_queue_work(root->fs_info->extent_workers, &async->work); 2879 2880 if (wait) { 2881 wait_for_completion(&async->wait); 2882 ret = async->error; 2883 kfree(async); 2884 return ret; 2885 } 2886 return 0; 2887 } 2888 2889 /* 2890 * this starts processing the delayed reference count updates and 2891 * extent insertions we have queued up so far. count can be 2892 * 0, which means to process everything in the tree at the start 2893 * of the run (but not newly added entries), or it can be some target 2894 * number you'd like to process. 2895 * 2896 * Returns 0 on success or if called with an aborted transaction 2897 * Returns <0 on error and aborts the transaction 2898 */ 2899 int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, 2900 struct btrfs_root *root, unsigned long count) 2901 { 2902 struct rb_node *node; 2903 struct btrfs_delayed_ref_root *delayed_refs; 2904 struct btrfs_delayed_ref_head *head; 2905 int ret; 2906 int run_all = count == (unsigned long)-1; 2907 bool can_flush_pending_bgs = trans->can_flush_pending_bgs; 2908 2909 /* We'll clean this up in btrfs_cleanup_transaction */ 2910 if (trans->aborted) 2911 return 0; 2912 2913 if (root == root->fs_info->extent_root) 2914 root = root->fs_info->tree_root; 2915 2916 delayed_refs = &trans->transaction->delayed_refs; 2917 if (count == 0) 2918 count = atomic_read(&delayed_refs->num_entries) * 2; 2919 2920 again: 2921 #ifdef SCRAMBLE_DELAYED_REFS 2922 delayed_refs->run_delayed_start = find_middle(&delayed_refs->root); 2923 #endif 2924 trans->can_flush_pending_bgs = false; 2925 ret = __btrfs_run_delayed_refs(trans, root, count); 2926 if (ret < 0) { 2927 btrfs_abort_transaction(trans, root, ret); 2928 return ret; 2929 } 2930 2931 if (run_all) { 2932 if (!list_empty(&trans->new_bgs)) 2933 btrfs_create_pending_block_groups(trans, root); 2934 2935 spin_lock(&delayed_refs->lock); 2936 node = rb_first(&delayed_refs->href_root); 2937 if (!node) { 2938 spin_unlock(&delayed_refs->lock); 2939 goto out; 2940 } 2941 count = (unsigned long)-1; 2942 2943 while (node) { 2944 head = rb_entry(node, struct btrfs_delayed_ref_head, 2945 href_node); 2946 if (btrfs_delayed_ref_is_head(&head->node)) { 2947 struct btrfs_delayed_ref_node *ref; 2948 2949 ref = &head->node; 2950 atomic_inc(&ref->refs); 2951 2952 spin_unlock(&delayed_refs->lock); 2953 /* 2954 * Mutex was contended, block until it's 2955 * released and try again 2956 */ 2957 mutex_lock(&head->mutex); 2958 mutex_unlock(&head->mutex); 2959 2960 btrfs_put_delayed_ref(ref); 2961 cond_resched(); 2962 goto again; 2963 } else { 2964 WARN_ON(1); 2965 } 2966 node = rb_next(node); 2967 } 2968 spin_unlock(&delayed_refs->lock); 2969 cond_resched(); 2970 goto again; 2971 } 2972 out: 2973 assert_qgroups_uptodate(trans); 2974 trans->can_flush_pending_bgs = can_flush_pending_bgs; 2975 return 0; 2976 } 2977 2978 int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans, 2979 struct btrfs_root *root, 2980 u64 bytenr, u64 num_bytes, u64 flags, 2981 int level, int is_data) 2982 { 2983 struct btrfs_delayed_extent_op *extent_op; 2984 int ret; 2985 2986 extent_op = btrfs_alloc_delayed_extent_op(); 2987 if (!extent_op) 2988 return -ENOMEM; 2989 2990 extent_op->flags_to_set = flags; 2991 extent_op->update_flags = 1; 2992 extent_op->update_key = 0; 2993 extent_op->is_data = is_data ? 1 : 0; 2994 extent_op->level = level; 2995 2996 ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr, 2997 num_bytes, extent_op); 2998 if (ret) 2999 btrfs_free_delayed_extent_op(extent_op); 3000 return ret; 3001 } 3002 3003 static noinline int check_delayed_ref(struct btrfs_trans_handle *trans, 3004 struct btrfs_root *root, 3005 struct btrfs_path *path, 3006 u64 objectid, u64 offset, u64 bytenr) 3007 { 3008 struct btrfs_delayed_ref_head *head; 3009 struct btrfs_delayed_ref_node *ref; 3010 struct btrfs_delayed_data_ref *data_ref; 3011 struct btrfs_delayed_ref_root *delayed_refs; 3012 int ret = 0; 3013 3014 delayed_refs = &trans->transaction->delayed_refs; 3015 spin_lock(&delayed_refs->lock); 3016 head = btrfs_find_delayed_ref_head(trans, bytenr); 3017 if (!head) { 3018 spin_unlock(&delayed_refs->lock); 3019 return 0; 3020 } 3021 3022 if (!mutex_trylock(&head->mutex)) { 3023 atomic_inc(&head->node.refs); 3024 spin_unlock(&delayed_refs->lock); 3025 3026 btrfs_release_path(path); 3027 3028 /* 3029 * Mutex was contended, block until it's released and let 3030 * caller try again 3031 */ 3032 mutex_lock(&head->mutex); 3033 mutex_unlock(&head->mutex); 3034 btrfs_put_delayed_ref(&head->node); 3035 return -EAGAIN; 3036 } 3037 spin_unlock(&delayed_refs->lock); 3038 3039 spin_lock(&head->lock); 3040 list_for_each_entry(ref, &head->ref_list, list) { 3041 /* If it's a shared ref we know a cross reference exists */ 3042 if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) { 3043 ret = 1; 3044 break; 3045 } 3046 3047 data_ref = btrfs_delayed_node_to_data_ref(ref); 3048 3049 /* 3050 * If our ref doesn't match the one we're currently looking at 3051 * then we have a cross reference. 3052 */ 3053 if (data_ref->root != root->root_key.objectid || 3054 data_ref->objectid != objectid || 3055 data_ref->offset != offset) { 3056 ret = 1; 3057 break; 3058 } 3059 } 3060 spin_unlock(&head->lock); 3061 mutex_unlock(&head->mutex); 3062 return ret; 3063 } 3064 3065 static noinline int check_committed_ref(struct btrfs_trans_handle *trans, 3066 struct btrfs_root *root, 3067 struct btrfs_path *path, 3068 u64 objectid, u64 offset, u64 bytenr) 3069 { 3070 struct btrfs_root *extent_root = root->fs_info->extent_root; 3071 struct extent_buffer *leaf; 3072 struct btrfs_extent_data_ref *ref; 3073 struct btrfs_extent_inline_ref *iref; 3074 struct btrfs_extent_item *ei; 3075 struct btrfs_key key; 3076 u32 item_size; 3077 int ret; 3078 3079 key.objectid = bytenr; 3080 key.offset = (u64)-1; 3081 key.type = BTRFS_EXTENT_ITEM_KEY; 3082 3083 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); 3084 if (ret < 0) 3085 goto out; 3086 BUG_ON(ret == 0); /* Corruption */ 3087 3088 ret = -ENOENT; 3089 if (path->slots[0] == 0) 3090 goto out; 3091 3092 path->slots[0]--; 3093 leaf = path->nodes[0]; 3094 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3095 3096 if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY) 3097 goto out; 3098 3099 ret = 1; 3100 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 3101 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 3102 if (item_size < sizeof(*ei)) { 3103 WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0)); 3104 goto out; 3105 } 3106 #endif 3107 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 3108 3109 if (item_size != sizeof(*ei) + 3110 btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY)) 3111 goto out; 3112 3113 if (btrfs_extent_generation(leaf, ei) <= 3114 btrfs_root_last_snapshot(&root->root_item)) 3115 goto out; 3116 3117 iref = (struct btrfs_extent_inline_ref *)(ei + 1); 3118 if (btrfs_extent_inline_ref_type(leaf, iref) != 3119 BTRFS_EXTENT_DATA_REF_KEY) 3120 goto out; 3121 3122 ref = (struct btrfs_extent_data_ref *)(&iref->offset); 3123 if (btrfs_extent_refs(leaf, ei) != 3124 btrfs_extent_data_ref_count(leaf, ref) || 3125 btrfs_extent_data_ref_root(leaf, ref) != 3126 root->root_key.objectid || 3127 btrfs_extent_data_ref_objectid(leaf, ref) != objectid || 3128 btrfs_extent_data_ref_offset(leaf, ref) != offset) 3129 goto out; 3130 3131 ret = 0; 3132 out: 3133 return ret; 3134 } 3135 3136 int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans, 3137 struct btrfs_root *root, 3138 u64 objectid, u64 offset, u64 bytenr) 3139 { 3140 struct btrfs_path *path; 3141 int ret; 3142 int ret2; 3143 3144 path = btrfs_alloc_path(); 3145 if (!path) 3146 return -ENOENT; 3147 3148 do { 3149 ret = check_committed_ref(trans, root, path, objectid, 3150 offset, bytenr); 3151 if (ret && ret != -ENOENT) 3152 goto out; 3153 3154 ret2 = check_delayed_ref(trans, root, path, objectid, 3155 offset, bytenr); 3156 } while (ret2 == -EAGAIN); 3157 3158 if (ret2 && ret2 != -ENOENT) { 3159 ret = ret2; 3160 goto out; 3161 } 3162 3163 if (ret != -ENOENT || ret2 != -ENOENT) 3164 ret = 0; 3165 out: 3166 btrfs_free_path(path); 3167 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 3168 WARN_ON(ret > 0); 3169 return ret; 3170 } 3171 3172 static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, 3173 struct btrfs_root *root, 3174 struct extent_buffer *buf, 3175 int full_backref, int inc) 3176 { 3177 u64 bytenr; 3178 u64 num_bytes; 3179 u64 parent; 3180 u64 ref_root; 3181 u32 nritems; 3182 struct btrfs_key key; 3183 struct btrfs_file_extent_item *fi; 3184 int i; 3185 int level; 3186 int ret = 0; 3187 int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *, 3188 u64, u64, u64, u64, u64, u64); 3189 3190 3191 if (btrfs_test_is_dummy_root(root)) 3192 return 0; 3193 3194 ref_root = btrfs_header_owner(buf); 3195 nritems = btrfs_header_nritems(buf); 3196 level = btrfs_header_level(buf); 3197 3198 if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0) 3199 return 0; 3200 3201 if (inc) 3202 process_func = btrfs_inc_extent_ref; 3203 else 3204 process_func = btrfs_free_extent; 3205 3206 if (full_backref) 3207 parent = buf->start; 3208 else 3209 parent = 0; 3210 3211 for (i = 0; i < nritems; i++) { 3212 if (level == 0) { 3213 btrfs_item_key_to_cpu(buf, &key, i); 3214 if (key.type != BTRFS_EXTENT_DATA_KEY) 3215 continue; 3216 fi = btrfs_item_ptr(buf, i, 3217 struct btrfs_file_extent_item); 3218 if (btrfs_file_extent_type(buf, fi) == 3219 BTRFS_FILE_EXTENT_INLINE) 3220 continue; 3221 bytenr = btrfs_file_extent_disk_bytenr(buf, fi); 3222 if (bytenr == 0) 3223 continue; 3224 3225 num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi); 3226 key.offset -= btrfs_file_extent_offset(buf, fi); 3227 ret = process_func(trans, root, bytenr, num_bytes, 3228 parent, ref_root, key.objectid, 3229 key.offset); 3230 if (ret) 3231 goto fail; 3232 } else { 3233 bytenr = btrfs_node_blockptr(buf, i); 3234 num_bytes = root->nodesize; 3235 ret = process_func(trans, root, bytenr, num_bytes, 3236 parent, ref_root, level - 1, 0); 3237 if (ret) 3238 goto fail; 3239 } 3240 } 3241 return 0; 3242 fail: 3243 return ret; 3244 } 3245 3246 int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, 3247 struct extent_buffer *buf, int full_backref) 3248 { 3249 return __btrfs_mod_ref(trans, root, buf, full_backref, 1); 3250 } 3251 3252 int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, 3253 struct extent_buffer *buf, int full_backref) 3254 { 3255 return __btrfs_mod_ref(trans, root, buf, full_backref, 0); 3256 } 3257 3258 static int write_one_cache_group(struct btrfs_trans_handle *trans, 3259 struct btrfs_root *root, 3260 struct btrfs_path *path, 3261 struct btrfs_block_group_cache *cache) 3262 { 3263 int ret; 3264 struct btrfs_root *extent_root = root->fs_info->extent_root; 3265 unsigned long bi; 3266 struct extent_buffer *leaf; 3267 3268 ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1); 3269 if (ret) { 3270 if (ret > 0) 3271 ret = -ENOENT; 3272 goto fail; 3273 } 3274 3275 leaf = path->nodes[0]; 3276 bi = btrfs_item_ptr_offset(leaf, path->slots[0]); 3277 write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item)); 3278 btrfs_mark_buffer_dirty(leaf); 3279 fail: 3280 btrfs_release_path(path); 3281 return ret; 3282 3283 } 3284 3285 static struct btrfs_block_group_cache * 3286 next_block_group(struct btrfs_root *root, 3287 struct btrfs_block_group_cache *cache) 3288 { 3289 struct rb_node *node; 3290 3291 spin_lock(&root->fs_info->block_group_cache_lock); 3292 3293 /* If our block group was removed, we need a full search. */ 3294 if (RB_EMPTY_NODE(&cache->cache_node)) { 3295 const u64 next_bytenr = cache->key.objectid + cache->key.offset; 3296 3297 spin_unlock(&root->fs_info->block_group_cache_lock); 3298 btrfs_put_block_group(cache); 3299 cache = btrfs_lookup_first_block_group(root->fs_info, 3300 next_bytenr); 3301 return cache; 3302 } 3303 node = rb_next(&cache->cache_node); 3304 btrfs_put_block_group(cache); 3305 if (node) { 3306 cache = rb_entry(node, struct btrfs_block_group_cache, 3307 cache_node); 3308 btrfs_get_block_group(cache); 3309 } else 3310 cache = NULL; 3311 spin_unlock(&root->fs_info->block_group_cache_lock); 3312 return cache; 3313 } 3314 3315 static int cache_save_setup(struct btrfs_block_group_cache *block_group, 3316 struct btrfs_trans_handle *trans, 3317 struct btrfs_path *path) 3318 { 3319 struct btrfs_root *root = block_group->fs_info->tree_root; 3320 struct inode *inode = NULL; 3321 u64 alloc_hint = 0; 3322 int dcs = BTRFS_DC_ERROR; 3323 u64 num_pages = 0; 3324 int retries = 0; 3325 int ret = 0; 3326 3327 /* 3328 * If this block group is smaller than 100 megs don't bother caching the 3329 * block group. 3330 */ 3331 if (block_group->key.offset < (100 * 1024 * 1024)) { 3332 spin_lock(&block_group->lock); 3333 block_group->disk_cache_state = BTRFS_DC_WRITTEN; 3334 spin_unlock(&block_group->lock); 3335 return 0; 3336 } 3337 3338 if (trans->aborted) 3339 return 0; 3340 again: 3341 inode = lookup_free_space_inode(root, block_group, path); 3342 if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) { 3343 ret = PTR_ERR(inode); 3344 btrfs_release_path(path); 3345 goto out; 3346 } 3347 3348 if (IS_ERR(inode)) { 3349 BUG_ON(retries); 3350 retries++; 3351 3352 if (block_group->ro) 3353 goto out_free; 3354 3355 ret = create_free_space_inode(root, trans, block_group, path); 3356 if (ret) 3357 goto out_free; 3358 goto again; 3359 } 3360 3361 /* We've already setup this transaction, go ahead and exit */ 3362 if (block_group->cache_generation == trans->transid && 3363 i_size_read(inode)) { 3364 dcs = BTRFS_DC_SETUP; 3365 goto out_put; 3366 } 3367 3368 /* 3369 * We want to set the generation to 0, that way if anything goes wrong 3370 * from here on out we know not to trust this cache when we load up next 3371 * time. 3372 */ 3373 BTRFS_I(inode)->generation = 0; 3374 ret = btrfs_update_inode(trans, root, inode); 3375 if (ret) { 3376 /* 3377 * So theoretically we could recover from this, simply set the 3378 * super cache generation to 0 so we know to invalidate the 3379 * cache, but then we'd have to keep track of the block groups 3380 * that fail this way so we know we _have_ to reset this cache 3381 * before the next commit or risk reading stale cache. So to 3382 * limit our exposure to horrible edge cases lets just abort the 3383 * transaction, this only happens in really bad situations 3384 * anyway. 3385 */ 3386 btrfs_abort_transaction(trans, root, ret); 3387 goto out_put; 3388 } 3389 WARN_ON(ret); 3390 3391 if (i_size_read(inode) > 0) { 3392 ret = btrfs_check_trunc_cache_free_space(root, 3393 &root->fs_info->global_block_rsv); 3394 if (ret) 3395 goto out_put; 3396 3397 ret = btrfs_truncate_free_space_cache(root, trans, NULL, inode); 3398 if (ret) 3399 goto out_put; 3400 } 3401 3402 spin_lock(&block_group->lock); 3403 if (block_group->cached != BTRFS_CACHE_FINISHED || 3404 !btrfs_test_opt(root, SPACE_CACHE)) { 3405 /* 3406 * don't bother trying to write stuff out _if_ 3407 * a) we're not cached, 3408 * b) we're with nospace_cache mount option. 3409 */ 3410 dcs = BTRFS_DC_WRITTEN; 3411 spin_unlock(&block_group->lock); 3412 goto out_put; 3413 } 3414 spin_unlock(&block_group->lock); 3415 3416 /* 3417 * We hit an ENOSPC when setting up the cache in this transaction, just 3418 * skip doing the setup, we've already cleared the cache so we're safe. 3419 */ 3420 if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) { 3421 ret = -ENOSPC; 3422 goto out_put; 3423 } 3424 3425 /* 3426 * Try to preallocate enough space based on how big the block group is. 3427 * Keep in mind this has to include any pinned space which could end up 3428 * taking up quite a bit since it's not folded into the other space 3429 * cache. 3430 */ 3431 num_pages = div_u64(block_group->key.offset, 256 * 1024 * 1024); 3432 if (!num_pages) 3433 num_pages = 1; 3434 3435 num_pages *= 16; 3436 num_pages *= PAGE_CACHE_SIZE; 3437 3438 ret = btrfs_check_data_free_space(inode, 0, num_pages); 3439 if (ret) 3440 goto out_put; 3441 3442 ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages, 3443 num_pages, num_pages, 3444 &alloc_hint); 3445 /* 3446 * Our cache requires contiguous chunks so that we don't modify a bunch 3447 * of metadata or split extents when writing the cache out, which means 3448 * we can enospc if we are heavily fragmented in addition to just normal 3449 * out of space conditions. So if we hit this just skip setting up any 3450 * other block groups for this transaction, maybe we'll unpin enough 3451 * space the next time around. 3452 */ 3453 if (!ret) 3454 dcs = BTRFS_DC_SETUP; 3455 else if (ret == -ENOSPC) 3456 set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags); 3457 btrfs_free_reserved_data_space(inode, 0, num_pages); 3458 3459 out_put: 3460 iput(inode); 3461 out_free: 3462 btrfs_release_path(path); 3463 out: 3464 spin_lock(&block_group->lock); 3465 if (!ret && dcs == BTRFS_DC_SETUP) 3466 block_group->cache_generation = trans->transid; 3467 block_group->disk_cache_state = dcs; 3468 spin_unlock(&block_group->lock); 3469 3470 return ret; 3471 } 3472 3473 int btrfs_setup_space_cache(struct btrfs_trans_handle *trans, 3474 struct btrfs_root *root) 3475 { 3476 struct btrfs_block_group_cache *cache, *tmp; 3477 struct btrfs_transaction *cur_trans = trans->transaction; 3478 struct btrfs_path *path; 3479 3480 if (list_empty(&cur_trans->dirty_bgs) || 3481 !btrfs_test_opt(root, SPACE_CACHE)) 3482 return 0; 3483 3484 path = btrfs_alloc_path(); 3485 if (!path) 3486 return -ENOMEM; 3487 3488 /* Could add new block groups, use _safe just in case */ 3489 list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs, 3490 dirty_list) { 3491 if (cache->disk_cache_state == BTRFS_DC_CLEAR) 3492 cache_save_setup(cache, trans, path); 3493 } 3494 3495 btrfs_free_path(path); 3496 return 0; 3497 } 3498 3499 /* 3500 * transaction commit does final block group cache writeback during a 3501 * critical section where nothing is allowed to change the FS. This is 3502 * required in order for the cache to actually match the block group, 3503 * but can introduce a lot of latency into the commit. 3504 * 3505 * So, btrfs_start_dirty_block_groups is here to kick off block group 3506 * cache IO. There's a chance we'll have to redo some of it if the 3507 * block group changes again during the commit, but it greatly reduces 3508 * the commit latency by getting rid of the easy block groups while 3509 * we're still allowing others to join the commit. 3510 */ 3511 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans, 3512 struct btrfs_root *root) 3513 { 3514 struct btrfs_block_group_cache *cache; 3515 struct btrfs_transaction *cur_trans = trans->transaction; 3516 int ret = 0; 3517 int should_put; 3518 struct btrfs_path *path = NULL; 3519 LIST_HEAD(dirty); 3520 struct list_head *io = &cur_trans->io_bgs; 3521 int num_started = 0; 3522 int loops = 0; 3523 3524 spin_lock(&cur_trans->dirty_bgs_lock); 3525 if (list_empty(&cur_trans->dirty_bgs)) { 3526 spin_unlock(&cur_trans->dirty_bgs_lock); 3527 return 0; 3528 } 3529 list_splice_init(&cur_trans->dirty_bgs, &dirty); 3530 spin_unlock(&cur_trans->dirty_bgs_lock); 3531 3532 again: 3533 /* 3534 * make sure all the block groups on our dirty list actually 3535 * exist 3536 */ 3537 btrfs_create_pending_block_groups(trans, root); 3538 3539 if (!path) { 3540 path = btrfs_alloc_path(); 3541 if (!path) 3542 return -ENOMEM; 3543 } 3544 3545 /* 3546 * cache_write_mutex is here only to save us from balance or automatic 3547 * removal of empty block groups deleting this block group while we are 3548 * writing out the cache 3549 */ 3550 mutex_lock(&trans->transaction->cache_write_mutex); 3551 while (!list_empty(&dirty)) { 3552 cache = list_first_entry(&dirty, 3553 struct btrfs_block_group_cache, 3554 dirty_list); 3555 /* 3556 * this can happen if something re-dirties a block 3557 * group that is already under IO. Just wait for it to 3558 * finish and then do it all again 3559 */ 3560 if (!list_empty(&cache->io_list)) { 3561 list_del_init(&cache->io_list); 3562 btrfs_wait_cache_io(root, trans, cache, 3563 &cache->io_ctl, path, 3564 cache->key.objectid); 3565 btrfs_put_block_group(cache); 3566 } 3567 3568 3569 /* 3570 * btrfs_wait_cache_io uses the cache->dirty_list to decide 3571 * if it should update the cache_state. Don't delete 3572 * until after we wait. 3573 * 3574 * Since we're not running in the commit critical section 3575 * we need the dirty_bgs_lock to protect from update_block_group 3576 */ 3577 spin_lock(&cur_trans->dirty_bgs_lock); 3578 list_del_init(&cache->dirty_list); 3579 spin_unlock(&cur_trans->dirty_bgs_lock); 3580 3581 should_put = 1; 3582 3583 cache_save_setup(cache, trans, path); 3584 3585 if (cache->disk_cache_state == BTRFS_DC_SETUP) { 3586 cache->io_ctl.inode = NULL; 3587 ret = btrfs_write_out_cache(root, trans, cache, path); 3588 if (ret == 0 && cache->io_ctl.inode) { 3589 num_started++; 3590 should_put = 0; 3591 3592 /* 3593 * the cache_write_mutex is protecting 3594 * the io_list 3595 */ 3596 list_add_tail(&cache->io_list, io); 3597 } else { 3598 /* 3599 * if we failed to write the cache, the 3600 * generation will be bad and life goes on 3601 */ 3602 ret = 0; 3603 } 3604 } 3605 if (!ret) { 3606 ret = write_one_cache_group(trans, root, path, cache); 3607 /* 3608 * Our block group might still be attached to the list 3609 * of new block groups in the transaction handle of some 3610 * other task (struct btrfs_trans_handle->new_bgs). This 3611 * means its block group item isn't yet in the extent 3612 * tree. If this happens ignore the error, as we will 3613 * try again later in the critical section of the 3614 * transaction commit. 3615 */ 3616 if (ret == -ENOENT) { 3617 ret = 0; 3618 spin_lock(&cur_trans->dirty_bgs_lock); 3619 if (list_empty(&cache->dirty_list)) { 3620 list_add_tail(&cache->dirty_list, 3621 &cur_trans->dirty_bgs); 3622 btrfs_get_block_group(cache); 3623 } 3624 spin_unlock(&cur_trans->dirty_bgs_lock); 3625 } else if (ret) { 3626 btrfs_abort_transaction(trans, root, ret); 3627 } 3628 } 3629 3630 /* if its not on the io list, we need to put the block group */ 3631 if (should_put) 3632 btrfs_put_block_group(cache); 3633 3634 if (ret) 3635 break; 3636 3637 /* 3638 * Avoid blocking other tasks for too long. It might even save 3639 * us from writing caches for block groups that are going to be 3640 * removed. 3641 */ 3642 mutex_unlock(&trans->transaction->cache_write_mutex); 3643 mutex_lock(&trans->transaction->cache_write_mutex); 3644 } 3645 mutex_unlock(&trans->transaction->cache_write_mutex); 3646 3647 /* 3648 * go through delayed refs for all the stuff we've just kicked off 3649 * and then loop back (just once) 3650 */ 3651 ret = btrfs_run_delayed_refs(trans, root, 0); 3652 if (!ret && loops == 0) { 3653 loops++; 3654 spin_lock(&cur_trans->dirty_bgs_lock); 3655 list_splice_init(&cur_trans->dirty_bgs, &dirty); 3656 /* 3657 * dirty_bgs_lock protects us from concurrent block group 3658 * deletes too (not just cache_write_mutex). 3659 */ 3660 if (!list_empty(&dirty)) { 3661 spin_unlock(&cur_trans->dirty_bgs_lock); 3662 goto again; 3663 } 3664 spin_unlock(&cur_trans->dirty_bgs_lock); 3665 } 3666 3667 btrfs_free_path(path); 3668 return ret; 3669 } 3670 3671 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans, 3672 struct btrfs_root *root) 3673 { 3674 struct btrfs_block_group_cache *cache; 3675 struct btrfs_transaction *cur_trans = trans->transaction; 3676 int ret = 0; 3677 int should_put; 3678 struct btrfs_path *path; 3679 struct list_head *io = &cur_trans->io_bgs; 3680 int num_started = 0; 3681 3682 path = btrfs_alloc_path(); 3683 if (!path) 3684 return -ENOMEM; 3685 3686 /* 3687 * We don't need the lock here since we are protected by the transaction 3688 * commit. We want to do the cache_save_setup first and then run the 3689 * delayed refs to make sure we have the best chance at doing this all 3690 * in one shot. 3691 */ 3692 while (!list_empty(&cur_trans->dirty_bgs)) { 3693 cache = list_first_entry(&cur_trans->dirty_bgs, 3694 struct btrfs_block_group_cache, 3695 dirty_list); 3696 3697 /* 3698 * this can happen if cache_save_setup re-dirties a block 3699 * group that is already under IO. Just wait for it to 3700 * finish and then do it all again 3701 */ 3702 if (!list_empty(&cache->io_list)) { 3703 list_del_init(&cache->io_list); 3704 btrfs_wait_cache_io(root, trans, cache, 3705 &cache->io_ctl, path, 3706 cache->key.objectid); 3707 btrfs_put_block_group(cache); 3708 } 3709 3710 /* 3711 * don't remove from the dirty list until after we've waited 3712 * on any pending IO 3713 */ 3714 list_del_init(&cache->dirty_list); 3715 should_put = 1; 3716 3717 cache_save_setup(cache, trans, path); 3718 3719 if (!ret) 3720 ret = btrfs_run_delayed_refs(trans, root, (unsigned long) -1); 3721 3722 if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) { 3723 cache->io_ctl.inode = NULL; 3724 ret = btrfs_write_out_cache(root, trans, cache, path); 3725 if (ret == 0 && cache->io_ctl.inode) { 3726 num_started++; 3727 should_put = 0; 3728 list_add_tail(&cache->io_list, io); 3729 } else { 3730 /* 3731 * if we failed to write the cache, the 3732 * generation will be bad and life goes on 3733 */ 3734 ret = 0; 3735 } 3736 } 3737 if (!ret) { 3738 ret = write_one_cache_group(trans, root, path, cache); 3739 if (ret) 3740 btrfs_abort_transaction(trans, root, ret); 3741 } 3742 3743 /* if its not on the io list, we need to put the block group */ 3744 if (should_put) 3745 btrfs_put_block_group(cache); 3746 } 3747 3748 while (!list_empty(io)) { 3749 cache = list_first_entry(io, struct btrfs_block_group_cache, 3750 io_list); 3751 list_del_init(&cache->io_list); 3752 btrfs_wait_cache_io(root, trans, cache, 3753 &cache->io_ctl, path, cache->key.objectid); 3754 btrfs_put_block_group(cache); 3755 } 3756 3757 btrfs_free_path(path); 3758 return ret; 3759 } 3760 3761 int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr) 3762 { 3763 struct btrfs_block_group_cache *block_group; 3764 int readonly = 0; 3765 3766 block_group = btrfs_lookup_block_group(root->fs_info, bytenr); 3767 if (!block_group || block_group->ro) 3768 readonly = 1; 3769 if (block_group) 3770 btrfs_put_block_group(block_group); 3771 return readonly; 3772 } 3773 3774 static const char *alloc_name(u64 flags) 3775 { 3776 switch (flags) { 3777 case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA: 3778 return "mixed"; 3779 case BTRFS_BLOCK_GROUP_METADATA: 3780 return "metadata"; 3781 case BTRFS_BLOCK_GROUP_DATA: 3782 return "data"; 3783 case BTRFS_BLOCK_GROUP_SYSTEM: 3784 return "system"; 3785 default: 3786 WARN_ON(1); 3787 return "invalid-combination"; 3788 }; 3789 } 3790 3791 static int update_space_info(struct btrfs_fs_info *info, u64 flags, 3792 u64 total_bytes, u64 bytes_used, 3793 struct btrfs_space_info **space_info) 3794 { 3795 struct btrfs_space_info *found; 3796 int i; 3797 int factor; 3798 int ret; 3799 3800 if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 | 3801 BTRFS_BLOCK_GROUP_RAID10)) 3802 factor = 2; 3803 else 3804 factor = 1; 3805 3806 found = __find_space_info(info, flags); 3807 if (found) { 3808 spin_lock(&found->lock); 3809 found->total_bytes += total_bytes; 3810 found->disk_total += total_bytes * factor; 3811 found->bytes_used += bytes_used; 3812 found->disk_used += bytes_used * factor; 3813 if (total_bytes > 0) 3814 found->full = 0; 3815 spin_unlock(&found->lock); 3816 *space_info = found; 3817 return 0; 3818 } 3819 found = kzalloc(sizeof(*found), GFP_NOFS); 3820 if (!found) 3821 return -ENOMEM; 3822 3823 ret = percpu_counter_init(&found->total_bytes_pinned, 0, GFP_KERNEL); 3824 if (ret) { 3825 kfree(found); 3826 return ret; 3827 } 3828 3829 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) 3830 INIT_LIST_HEAD(&found->block_groups[i]); 3831 init_rwsem(&found->groups_sem); 3832 spin_lock_init(&found->lock); 3833 found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK; 3834 found->total_bytes = total_bytes; 3835 found->disk_total = total_bytes * factor; 3836 found->bytes_used = bytes_used; 3837 found->disk_used = bytes_used * factor; 3838 found->bytes_pinned = 0; 3839 found->bytes_reserved = 0; 3840 found->bytes_readonly = 0; 3841 found->bytes_may_use = 0; 3842 found->full = 0; 3843 found->max_extent_size = 0; 3844 found->force_alloc = CHUNK_ALLOC_NO_FORCE; 3845 found->chunk_alloc = 0; 3846 found->flush = 0; 3847 init_waitqueue_head(&found->wait); 3848 INIT_LIST_HEAD(&found->ro_bgs); 3849 3850 ret = kobject_init_and_add(&found->kobj, &space_info_ktype, 3851 info->space_info_kobj, "%s", 3852 alloc_name(found->flags)); 3853 if (ret) { 3854 kfree(found); 3855 return ret; 3856 } 3857 3858 *space_info = found; 3859 list_add_rcu(&found->list, &info->space_info); 3860 if (flags & BTRFS_BLOCK_GROUP_DATA) 3861 info->data_sinfo = found; 3862 3863 return ret; 3864 } 3865 3866 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) 3867 { 3868 u64 extra_flags = chunk_to_extended(flags) & 3869 BTRFS_EXTENDED_PROFILE_MASK; 3870 3871 write_seqlock(&fs_info->profiles_lock); 3872 if (flags & BTRFS_BLOCK_GROUP_DATA) 3873 fs_info->avail_data_alloc_bits |= extra_flags; 3874 if (flags & BTRFS_BLOCK_GROUP_METADATA) 3875 fs_info->avail_metadata_alloc_bits |= extra_flags; 3876 if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 3877 fs_info->avail_system_alloc_bits |= extra_flags; 3878 write_sequnlock(&fs_info->profiles_lock); 3879 } 3880 3881 /* 3882 * returns target flags in extended format or 0 if restripe for this 3883 * chunk_type is not in progress 3884 * 3885 * should be called with either volume_mutex or balance_lock held 3886 */ 3887 static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags) 3888 { 3889 struct btrfs_balance_control *bctl = fs_info->balance_ctl; 3890 u64 target = 0; 3891 3892 if (!bctl) 3893 return 0; 3894 3895 if (flags & BTRFS_BLOCK_GROUP_DATA && 3896 bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) { 3897 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target; 3898 } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM && 3899 bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) { 3900 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target; 3901 } else if (flags & BTRFS_BLOCK_GROUP_METADATA && 3902 bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) { 3903 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target; 3904 } 3905 3906 return target; 3907 } 3908 3909 /* 3910 * @flags: available profiles in extended format (see ctree.h) 3911 * 3912 * Returns reduced profile in chunk format. If profile changing is in 3913 * progress (either running or paused) picks the target profile (if it's 3914 * already available), otherwise falls back to plain reducing. 3915 */ 3916 static u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags) 3917 { 3918 u64 num_devices = root->fs_info->fs_devices->rw_devices; 3919 u64 target; 3920 u64 raid_type; 3921 u64 allowed = 0; 3922 3923 /* 3924 * see if restripe for this chunk_type is in progress, if so 3925 * try to reduce to the target profile 3926 */ 3927 spin_lock(&root->fs_info->balance_lock); 3928 target = get_restripe_target(root->fs_info, flags); 3929 if (target) { 3930 /* pick target profile only if it's already available */ 3931 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) { 3932 spin_unlock(&root->fs_info->balance_lock); 3933 return extended_to_chunk(target); 3934 } 3935 } 3936 spin_unlock(&root->fs_info->balance_lock); 3937 3938 /* First, mask out the RAID levels which aren't possible */ 3939 for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) { 3940 if (num_devices >= btrfs_raid_array[raid_type].devs_min) 3941 allowed |= btrfs_raid_group[raid_type]; 3942 } 3943 allowed &= flags; 3944 3945 if (allowed & BTRFS_BLOCK_GROUP_RAID6) 3946 allowed = BTRFS_BLOCK_GROUP_RAID6; 3947 else if (allowed & BTRFS_BLOCK_GROUP_RAID5) 3948 allowed = BTRFS_BLOCK_GROUP_RAID5; 3949 else if (allowed & BTRFS_BLOCK_GROUP_RAID10) 3950 allowed = BTRFS_BLOCK_GROUP_RAID10; 3951 else if (allowed & BTRFS_BLOCK_GROUP_RAID1) 3952 allowed = BTRFS_BLOCK_GROUP_RAID1; 3953 else if (allowed & BTRFS_BLOCK_GROUP_RAID0) 3954 allowed = BTRFS_BLOCK_GROUP_RAID0; 3955 3956 flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK; 3957 3958 return extended_to_chunk(flags | allowed); 3959 } 3960 3961 static u64 get_alloc_profile(struct btrfs_root *root, u64 orig_flags) 3962 { 3963 unsigned seq; 3964 u64 flags; 3965 3966 do { 3967 flags = orig_flags; 3968 seq = read_seqbegin(&root->fs_info->profiles_lock); 3969 3970 if (flags & BTRFS_BLOCK_GROUP_DATA) 3971 flags |= root->fs_info->avail_data_alloc_bits; 3972 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 3973 flags |= root->fs_info->avail_system_alloc_bits; 3974 else if (flags & BTRFS_BLOCK_GROUP_METADATA) 3975 flags |= root->fs_info->avail_metadata_alloc_bits; 3976 } while (read_seqretry(&root->fs_info->profiles_lock, seq)); 3977 3978 return btrfs_reduce_alloc_profile(root, flags); 3979 } 3980 3981 u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data) 3982 { 3983 u64 flags; 3984 u64 ret; 3985 3986 if (data) 3987 flags = BTRFS_BLOCK_GROUP_DATA; 3988 else if (root == root->fs_info->chunk_root) 3989 flags = BTRFS_BLOCK_GROUP_SYSTEM; 3990 else 3991 flags = BTRFS_BLOCK_GROUP_METADATA; 3992 3993 ret = get_alloc_profile(root, flags); 3994 return ret; 3995 } 3996 3997 int btrfs_alloc_data_chunk_ondemand(struct inode *inode, u64 bytes) 3998 { 3999 struct btrfs_space_info *data_sinfo; 4000 struct btrfs_root *root = BTRFS_I(inode)->root; 4001 struct btrfs_fs_info *fs_info = root->fs_info; 4002 u64 used; 4003 int ret = 0; 4004 int need_commit = 2; 4005 int have_pinned_space; 4006 4007 /* make sure bytes are sectorsize aligned */ 4008 bytes = ALIGN(bytes, root->sectorsize); 4009 4010 if (btrfs_is_free_space_inode(inode)) { 4011 need_commit = 0; 4012 ASSERT(current->journal_info); 4013 } 4014 4015 data_sinfo = fs_info->data_sinfo; 4016 if (!data_sinfo) 4017 goto alloc; 4018 4019 again: 4020 /* make sure we have enough space to handle the data first */ 4021 spin_lock(&data_sinfo->lock); 4022 used = data_sinfo->bytes_used + data_sinfo->bytes_reserved + 4023 data_sinfo->bytes_pinned + data_sinfo->bytes_readonly + 4024 data_sinfo->bytes_may_use; 4025 4026 if (used + bytes > data_sinfo->total_bytes) { 4027 struct btrfs_trans_handle *trans; 4028 4029 /* 4030 * if we don't have enough free bytes in this space then we need 4031 * to alloc a new chunk. 4032 */ 4033 if (!data_sinfo->full) { 4034 u64 alloc_target; 4035 4036 data_sinfo->force_alloc = CHUNK_ALLOC_FORCE; 4037 spin_unlock(&data_sinfo->lock); 4038 alloc: 4039 alloc_target = btrfs_get_alloc_profile(root, 1); 4040 /* 4041 * It is ugly that we don't call nolock join 4042 * transaction for the free space inode case here. 4043 * But it is safe because we only do the data space 4044 * reservation for the free space cache in the 4045 * transaction context, the common join transaction 4046 * just increase the counter of the current transaction 4047 * handler, doesn't try to acquire the trans_lock of 4048 * the fs. 4049 */ 4050 trans = btrfs_join_transaction(root); 4051 if (IS_ERR(trans)) 4052 return PTR_ERR(trans); 4053 4054 ret = do_chunk_alloc(trans, root->fs_info->extent_root, 4055 alloc_target, 4056 CHUNK_ALLOC_NO_FORCE); 4057 btrfs_end_transaction(trans, root); 4058 if (ret < 0) { 4059 if (ret != -ENOSPC) 4060 return ret; 4061 else { 4062 have_pinned_space = 1; 4063 goto commit_trans; 4064 } 4065 } 4066 4067 if (!data_sinfo) 4068 data_sinfo = fs_info->data_sinfo; 4069 4070 goto again; 4071 } 4072 4073 /* 4074 * If we don't have enough pinned space to deal with this 4075 * allocation, and no removed chunk in current transaction, 4076 * don't bother committing the transaction. 4077 */ 4078 have_pinned_space = percpu_counter_compare( 4079 &data_sinfo->total_bytes_pinned, 4080 used + bytes - data_sinfo->total_bytes); 4081 spin_unlock(&data_sinfo->lock); 4082 4083 /* commit the current transaction and try again */ 4084 commit_trans: 4085 if (need_commit && 4086 !atomic_read(&root->fs_info->open_ioctl_trans)) { 4087 need_commit--; 4088 4089 if (need_commit > 0) 4090 btrfs_wait_ordered_roots(fs_info, -1); 4091 4092 trans = btrfs_join_transaction(root); 4093 if (IS_ERR(trans)) 4094 return PTR_ERR(trans); 4095 if (have_pinned_space >= 0 || 4096 test_bit(BTRFS_TRANS_HAVE_FREE_BGS, 4097 &trans->transaction->flags) || 4098 need_commit > 0) { 4099 ret = btrfs_commit_transaction(trans, root); 4100 if (ret) 4101 return ret; 4102 /* 4103 * make sure that all running delayed iput are 4104 * done 4105 */ 4106 down_write(&root->fs_info->delayed_iput_sem); 4107 up_write(&root->fs_info->delayed_iput_sem); 4108 goto again; 4109 } else { 4110 btrfs_end_transaction(trans, root); 4111 } 4112 } 4113 4114 trace_btrfs_space_reservation(root->fs_info, 4115 "space_info:enospc", 4116 data_sinfo->flags, bytes, 1); 4117 return -ENOSPC; 4118 } 4119 data_sinfo->bytes_may_use += bytes; 4120 trace_btrfs_space_reservation(root->fs_info, "space_info", 4121 data_sinfo->flags, bytes, 1); 4122 spin_unlock(&data_sinfo->lock); 4123 4124 return ret; 4125 } 4126 4127 /* 4128 * New check_data_free_space() with ability for precious data reservation 4129 * Will replace old btrfs_check_data_free_space(), but for patch split, 4130 * add a new function first and then replace it. 4131 */ 4132 int btrfs_check_data_free_space(struct inode *inode, u64 start, u64 len) 4133 { 4134 struct btrfs_root *root = BTRFS_I(inode)->root; 4135 int ret; 4136 4137 /* align the range */ 4138 len = round_up(start + len, root->sectorsize) - 4139 round_down(start, root->sectorsize); 4140 start = round_down(start, root->sectorsize); 4141 4142 ret = btrfs_alloc_data_chunk_ondemand(inode, len); 4143 if (ret < 0) 4144 return ret; 4145 4146 /* 4147 * Use new btrfs_qgroup_reserve_data to reserve precious data space 4148 * 4149 * TODO: Find a good method to avoid reserve data space for NOCOW 4150 * range, but don't impact performance on quota disable case. 4151 */ 4152 ret = btrfs_qgroup_reserve_data(inode, start, len); 4153 return ret; 4154 } 4155 4156 /* 4157 * Called if we need to clear a data reservation for this inode 4158 * Normally in a error case. 4159 * 4160 * This one will *NOT* use accurate qgroup reserved space API, just for case 4161 * which we can't sleep and is sure it won't affect qgroup reserved space. 4162 * Like clear_bit_hook(). 4163 */ 4164 void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start, 4165 u64 len) 4166 { 4167 struct btrfs_root *root = BTRFS_I(inode)->root; 4168 struct btrfs_space_info *data_sinfo; 4169 4170 /* Make sure the range is aligned to sectorsize */ 4171 len = round_up(start + len, root->sectorsize) - 4172 round_down(start, root->sectorsize); 4173 start = round_down(start, root->sectorsize); 4174 4175 data_sinfo = root->fs_info->data_sinfo; 4176 spin_lock(&data_sinfo->lock); 4177 if (WARN_ON(data_sinfo->bytes_may_use < len)) 4178 data_sinfo->bytes_may_use = 0; 4179 else 4180 data_sinfo->bytes_may_use -= len; 4181 trace_btrfs_space_reservation(root->fs_info, "space_info", 4182 data_sinfo->flags, len, 0); 4183 spin_unlock(&data_sinfo->lock); 4184 } 4185 4186 /* 4187 * Called if we need to clear a data reservation for this inode 4188 * Normally in a error case. 4189 * 4190 * This one will handle the per-indoe data rsv map for accurate reserved 4191 * space framework. 4192 */ 4193 void btrfs_free_reserved_data_space(struct inode *inode, u64 start, u64 len) 4194 { 4195 btrfs_free_reserved_data_space_noquota(inode, start, len); 4196 btrfs_qgroup_free_data(inode, start, len); 4197 } 4198 4199 static void force_metadata_allocation(struct btrfs_fs_info *info) 4200 { 4201 struct list_head *head = &info->space_info; 4202 struct btrfs_space_info *found; 4203 4204 rcu_read_lock(); 4205 list_for_each_entry_rcu(found, head, list) { 4206 if (found->flags & BTRFS_BLOCK_GROUP_METADATA) 4207 found->force_alloc = CHUNK_ALLOC_FORCE; 4208 } 4209 rcu_read_unlock(); 4210 } 4211 4212 static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global) 4213 { 4214 return (global->size << 1); 4215 } 4216 4217 static int should_alloc_chunk(struct btrfs_root *root, 4218 struct btrfs_space_info *sinfo, int force) 4219 { 4220 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv; 4221 u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly; 4222 u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved; 4223 u64 thresh; 4224 4225 if (force == CHUNK_ALLOC_FORCE) 4226 return 1; 4227 4228 /* 4229 * We need to take into account the global rsv because for all intents 4230 * and purposes it's used space. Don't worry about locking the 4231 * global_rsv, it doesn't change except when the transaction commits. 4232 */ 4233 if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA) 4234 num_allocated += calc_global_rsv_need_space(global_rsv); 4235 4236 /* 4237 * in limited mode, we want to have some free space up to 4238 * about 1% of the FS size. 4239 */ 4240 if (force == CHUNK_ALLOC_LIMITED) { 4241 thresh = btrfs_super_total_bytes(root->fs_info->super_copy); 4242 thresh = max_t(u64, 64 * 1024 * 1024, 4243 div_factor_fine(thresh, 1)); 4244 4245 if (num_bytes - num_allocated < thresh) 4246 return 1; 4247 } 4248 4249 if (num_allocated + 2 * 1024 * 1024 < div_factor(num_bytes, 8)) 4250 return 0; 4251 return 1; 4252 } 4253 4254 static u64 get_profile_num_devs(struct btrfs_root *root, u64 type) 4255 { 4256 u64 num_dev; 4257 4258 if (type & (BTRFS_BLOCK_GROUP_RAID10 | 4259 BTRFS_BLOCK_GROUP_RAID0 | 4260 BTRFS_BLOCK_GROUP_RAID5 | 4261 BTRFS_BLOCK_GROUP_RAID6)) 4262 num_dev = root->fs_info->fs_devices->rw_devices; 4263 else if (type & BTRFS_BLOCK_GROUP_RAID1) 4264 num_dev = 2; 4265 else 4266 num_dev = 1; /* DUP or single */ 4267 4268 return num_dev; 4269 } 4270 4271 /* 4272 * If @is_allocation is true, reserve space in the system space info necessary 4273 * for allocating a chunk, otherwise if it's false, reserve space necessary for 4274 * removing a chunk. 4275 */ 4276 void check_system_chunk(struct btrfs_trans_handle *trans, 4277 struct btrfs_root *root, 4278 u64 type) 4279 { 4280 struct btrfs_space_info *info; 4281 u64 left; 4282 u64 thresh; 4283 int ret = 0; 4284 u64 num_devs; 4285 4286 /* 4287 * Needed because we can end up allocating a system chunk and for an 4288 * atomic and race free space reservation in the chunk block reserve. 4289 */ 4290 ASSERT(mutex_is_locked(&root->fs_info->chunk_mutex)); 4291 4292 info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM); 4293 spin_lock(&info->lock); 4294 left = info->total_bytes - info->bytes_used - info->bytes_pinned - 4295 info->bytes_reserved - info->bytes_readonly - 4296 info->bytes_may_use; 4297 spin_unlock(&info->lock); 4298 4299 num_devs = get_profile_num_devs(root, type); 4300 4301 /* num_devs device items to update and 1 chunk item to add or remove */ 4302 thresh = btrfs_calc_trunc_metadata_size(root, num_devs) + 4303 btrfs_calc_trans_metadata_size(root, 1); 4304 4305 if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) { 4306 btrfs_info(root->fs_info, "left=%llu, need=%llu, flags=%llu", 4307 left, thresh, type); 4308 dump_space_info(info, 0, 0); 4309 } 4310 4311 if (left < thresh) { 4312 u64 flags; 4313 4314 flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0); 4315 /* 4316 * Ignore failure to create system chunk. We might end up not 4317 * needing it, as we might not need to COW all nodes/leafs from 4318 * the paths we visit in the chunk tree (they were already COWed 4319 * or created in the current transaction for example). 4320 */ 4321 ret = btrfs_alloc_chunk(trans, root, flags); 4322 } 4323 4324 if (!ret) { 4325 ret = btrfs_block_rsv_add(root->fs_info->chunk_root, 4326 &root->fs_info->chunk_block_rsv, 4327 thresh, BTRFS_RESERVE_NO_FLUSH); 4328 if (!ret) 4329 trans->chunk_bytes_reserved += thresh; 4330 } 4331 } 4332 4333 static int do_chunk_alloc(struct btrfs_trans_handle *trans, 4334 struct btrfs_root *extent_root, u64 flags, int force) 4335 { 4336 struct btrfs_space_info *space_info; 4337 struct btrfs_fs_info *fs_info = extent_root->fs_info; 4338 int wait_for_alloc = 0; 4339 int ret = 0; 4340 4341 /* Don't re-enter if we're already allocating a chunk */ 4342 if (trans->allocating_chunk) 4343 return -ENOSPC; 4344 4345 space_info = __find_space_info(extent_root->fs_info, flags); 4346 if (!space_info) { 4347 ret = update_space_info(extent_root->fs_info, flags, 4348 0, 0, &space_info); 4349 BUG_ON(ret); /* -ENOMEM */ 4350 } 4351 BUG_ON(!space_info); /* Logic error */ 4352 4353 again: 4354 spin_lock(&space_info->lock); 4355 if (force < space_info->force_alloc) 4356 force = space_info->force_alloc; 4357 if (space_info->full) { 4358 if (should_alloc_chunk(extent_root, space_info, force)) 4359 ret = -ENOSPC; 4360 else 4361 ret = 0; 4362 spin_unlock(&space_info->lock); 4363 return ret; 4364 } 4365 4366 if (!should_alloc_chunk(extent_root, space_info, force)) { 4367 spin_unlock(&space_info->lock); 4368 return 0; 4369 } else if (space_info->chunk_alloc) { 4370 wait_for_alloc = 1; 4371 } else { 4372 space_info->chunk_alloc = 1; 4373 } 4374 4375 spin_unlock(&space_info->lock); 4376 4377 mutex_lock(&fs_info->chunk_mutex); 4378 4379 /* 4380 * The chunk_mutex is held throughout the entirety of a chunk 4381 * allocation, so once we've acquired the chunk_mutex we know that the 4382 * other guy is done and we need to recheck and see if we should 4383 * allocate. 4384 */ 4385 if (wait_for_alloc) { 4386 mutex_unlock(&fs_info->chunk_mutex); 4387 wait_for_alloc = 0; 4388 goto again; 4389 } 4390 4391 trans->allocating_chunk = true; 4392 4393 /* 4394 * If we have mixed data/metadata chunks we want to make sure we keep 4395 * allocating mixed chunks instead of individual chunks. 4396 */ 4397 if (btrfs_mixed_space_info(space_info)) 4398 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA); 4399 4400 /* 4401 * if we're doing a data chunk, go ahead and make sure that 4402 * we keep a reasonable number of metadata chunks allocated in the 4403 * FS as well. 4404 */ 4405 if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) { 4406 fs_info->data_chunk_allocations++; 4407 if (!(fs_info->data_chunk_allocations % 4408 fs_info->metadata_ratio)) 4409 force_metadata_allocation(fs_info); 4410 } 4411 4412 /* 4413 * Check if we have enough space in SYSTEM chunk because we may need 4414 * to update devices. 4415 */ 4416 check_system_chunk(trans, extent_root, flags); 4417 4418 ret = btrfs_alloc_chunk(trans, extent_root, flags); 4419 trans->allocating_chunk = false; 4420 4421 spin_lock(&space_info->lock); 4422 if (ret < 0 && ret != -ENOSPC) 4423 goto out; 4424 if (ret) 4425 space_info->full = 1; 4426 else 4427 ret = 1; 4428 4429 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; 4430 out: 4431 space_info->chunk_alloc = 0; 4432 spin_unlock(&space_info->lock); 4433 mutex_unlock(&fs_info->chunk_mutex); 4434 /* 4435 * When we allocate a new chunk we reserve space in the chunk block 4436 * reserve to make sure we can COW nodes/leafs in the chunk tree or 4437 * add new nodes/leafs to it if we end up needing to do it when 4438 * inserting the chunk item and updating device items as part of the 4439 * second phase of chunk allocation, performed by 4440 * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a 4441 * large number of new block groups to create in our transaction 4442 * handle's new_bgs list to avoid exhausting the chunk block reserve 4443 * in extreme cases - like having a single transaction create many new 4444 * block groups when starting to write out the free space caches of all 4445 * the block groups that were made dirty during the lifetime of the 4446 * transaction. 4447 */ 4448 if (trans->can_flush_pending_bgs && 4449 trans->chunk_bytes_reserved >= (2 * 1024 * 1024ull)) { 4450 btrfs_create_pending_block_groups(trans, trans->root); 4451 btrfs_trans_release_chunk_metadata(trans); 4452 } 4453 return ret; 4454 } 4455 4456 static int can_overcommit(struct btrfs_root *root, 4457 struct btrfs_space_info *space_info, u64 bytes, 4458 enum btrfs_reserve_flush_enum flush) 4459 { 4460 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv; 4461 u64 profile = btrfs_get_alloc_profile(root, 0); 4462 u64 space_size; 4463 u64 avail; 4464 u64 used; 4465 4466 used = space_info->bytes_used + space_info->bytes_reserved + 4467 space_info->bytes_pinned + space_info->bytes_readonly; 4468 4469 /* 4470 * We only want to allow over committing if we have lots of actual space 4471 * free, but if we don't have enough space to handle the global reserve 4472 * space then we could end up having a real enospc problem when trying 4473 * to allocate a chunk or some other such important allocation. 4474 */ 4475 spin_lock(&global_rsv->lock); 4476 space_size = calc_global_rsv_need_space(global_rsv); 4477 spin_unlock(&global_rsv->lock); 4478 if (used + space_size >= space_info->total_bytes) 4479 return 0; 4480 4481 used += space_info->bytes_may_use; 4482 4483 spin_lock(&root->fs_info->free_chunk_lock); 4484 avail = root->fs_info->free_chunk_space; 4485 spin_unlock(&root->fs_info->free_chunk_lock); 4486 4487 /* 4488 * If we have dup, raid1 or raid10 then only half of the free 4489 * space is actually useable. For raid56, the space info used 4490 * doesn't include the parity drive, so we don't have to 4491 * change the math 4492 */ 4493 if (profile & (BTRFS_BLOCK_GROUP_DUP | 4494 BTRFS_BLOCK_GROUP_RAID1 | 4495 BTRFS_BLOCK_GROUP_RAID10)) 4496 avail >>= 1; 4497 4498 /* 4499 * If we aren't flushing all things, let us overcommit up to 4500 * 1/2th of the space. If we can flush, don't let us overcommit 4501 * too much, let it overcommit up to 1/8 of the space. 4502 */ 4503 if (flush == BTRFS_RESERVE_FLUSH_ALL) 4504 avail >>= 3; 4505 else 4506 avail >>= 1; 4507 4508 if (used + bytes < space_info->total_bytes + avail) 4509 return 1; 4510 return 0; 4511 } 4512 4513 static void btrfs_writeback_inodes_sb_nr(struct btrfs_root *root, 4514 unsigned long nr_pages, int nr_items) 4515 { 4516 struct super_block *sb = root->fs_info->sb; 4517 4518 if (down_read_trylock(&sb->s_umount)) { 4519 writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE); 4520 up_read(&sb->s_umount); 4521 } else { 4522 /* 4523 * We needn't worry the filesystem going from r/w to r/o though 4524 * we don't acquire ->s_umount mutex, because the filesystem 4525 * should guarantee the delalloc inodes list be empty after 4526 * the filesystem is readonly(all dirty pages are written to 4527 * the disk). 4528 */ 4529 btrfs_start_delalloc_roots(root->fs_info, 0, nr_items); 4530 if (!current->journal_info) 4531 btrfs_wait_ordered_roots(root->fs_info, nr_items); 4532 } 4533 } 4534 4535 static inline int calc_reclaim_items_nr(struct btrfs_root *root, u64 to_reclaim) 4536 { 4537 u64 bytes; 4538 int nr; 4539 4540 bytes = btrfs_calc_trans_metadata_size(root, 1); 4541 nr = (int)div64_u64(to_reclaim, bytes); 4542 if (!nr) 4543 nr = 1; 4544 return nr; 4545 } 4546 4547 #define EXTENT_SIZE_PER_ITEM (256 * 1024) 4548 4549 /* 4550 * shrink metadata reservation for delalloc 4551 */ 4552 static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig, 4553 bool wait_ordered) 4554 { 4555 struct btrfs_block_rsv *block_rsv; 4556 struct btrfs_space_info *space_info; 4557 struct btrfs_trans_handle *trans; 4558 u64 delalloc_bytes; 4559 u64 max_reclaim; 4560 long time_left; 4561 unsigned long nr_pages; 4562 int loops; 4563 int items; 4564 enum btrfs_reserve_flush_enum flush; 4565 4566 /* Calc the number of the pages we need flush for space reservation */ 4567 items = calc_reclaim_items_nr(root, to_reclaim); 4568 to_reclaim = items * EXTENT_SIZE_PER_ITEM; 4569 4570 trans = (struct btrfs_trans_handle *)current->journal_info; 4571 block_rsv = &root->fs_info->delalloc_block_rsv; 4572 space_info = block_rsv->space_info; 4573 4574 delalloc_bytes = percpu_counter_sum_positive( 4575 &root->fs_info->delalloc_bytes); 4576 if (delalloc_bytes == 0) { 4577 if (trans) 4578 return; 4579 if (wait_ordered) 4580 btrfs_wait_ordered_roots(root->fs_info, items); 4581 return; 4582 } 4583 4584 loops = 0; 4585 while (delalloc_bytes && loops < 3) { 4586 max_reclaim = min(delalloc_bytes, to_reclaim); 4587 nr_pages = max_reclaim >> PAGE_CACHE_SHIFT; 4588 btrfs_writeback_inodes_sb_nr(root, nr_pages, items); 4589 /* 4590 * We need to wait for the async pages to actually start before 4591 * we do anything. 4592 */ 4593 max_reclaim = atomic_read(&root->fs_info->async_delalloc_pages); 4594 if (!max_reclaim) 4595 goto skip_async; 4596 4597 if (max_reclaim <= nr_pages) 4598 max_reclaim = 0; 4599 else 4600 max_reclaim -= nr_pages; 4601 4602 wait_event(root->fs_info->async_submit_wait, 4603 atomic_read(&root->fs_info->async_delalloc_pages) <= 4604 (int)max_reclaim); 4605 skip_async: 4606 if (!trans) 4607 flush = BTRFS_RESERVE_FLUSH_ALL; 4608 else 4609 flush = BTRFS_RESERVE_NO_FLUSH; 4610 spin_lock(&space_info->lock); 4611 if (can_overcommit(root, space_info, orig, flush)) { 4612 spin_unlock(&space_info->lock); 4613 break; 4614 } 4615 spin_unlock(&space_info->lock); 4616 4617 loops++; 4618 if (wait_ordered && !trans) { 4619 btrfs_wait_ordered_roots(root->fs_info, items); 4620 } else { 4621 time_left = schedule_timeout_killable(1); 4622 if (time_left) 4623 break; 4624 } 4625 delalloc_bytes = percpu_counter_sum_positive( 4626 &root->fs_info->delalloc_bytes); 4627 } 4628 } 4629 4630 /** 4631 * maybe_commit_transaction - possibly commit the transaction if its ok to 4632 * @root - the root we're allocating for 4633 * @bytes - the number of bytes we want to reserve 4634 * @force - force the commit 4635 * 4636 * This will check to make sure that committing the transaction will actually 4637 * get us somewhere and then commit the transaction if it does. Otherwise it 4638 * will return -ENOSPC. 4639 */ 4640 static int may_commit_transaction(struct btrfs_root *root, 4641 struct btrfs_space_info *space_info, 4642 u64 bytes, int force) 4643 { 4644 struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv; 4645 struct btrfs_trans_handle *trans; 4646 4647 trans = (struct btrfs_trans_handle *)current->journal_info; 4648 if (trans) 4649 return -EAGAIN; 4650 4651 if (force) 4652 goto commit; 4653 4654 /* See if there is enough pinned space to make this reservation */ 4655 if (percpu_counter_compare(&space_info->total_bytes_pinned, 4656 bytes) >= 0) 4657 goto commit; 4658 4659 /* 4660 * See if there is some space in the delayed insertion reservation for 4661 * this reservation. 4662 */ 4663 if (space_info != delayed_rsv->space_info) 4664 return -ENOSPC; 4665 4666 spin_lock(&delayed_rsv->lock); 4667 if (percpu_counter_compare(&space_info->total_bytes_pinned, 4668 bytes - delayed_rsv->size) >= 0) { 4669 spin_unlock(&delayed_rsv->lock); 4670 return -ENOSPC; 4671 } 4672 spin_unlock(&delayed_rsv->lock); 4673 4674 commit: 4675 trans = btrfs_join_transaction(root); 4676 if (IS_ERR(trans)) 4677 return -ENOSPC; 4678 4679 return btrfs_commit_transaction(trans, root); 4680 } 4681 4682 enum flush_state { 4683 FLUSH_DELAYED_ITEMS_NR = 1, 4684 FLUSH_DELAYED_ITEMS = 2, 4685 FLUSH_DELALLOC = 3, 4686 FLUSH_DELALLOC_WAIT = 4, 4687 ALLOC_CHUNK = 5, 4688 COMMIT_TRANS = 6, 4689 }; 4690 4691 static int flush_space(struct btrfs_root *root, 4692 struct btrfs_space_info *space_info, u64 num_bytes, 4693 u64 orig_bytes, int state) 4694 { 4695 struct btrfs_trans_handle *trans; 4696 int nr; 4697 int ret = 0; 4698 4699 switch (state) { 4700 case FLUSH_DELAYED_ITEMS_NR: 4701 case FLUSH_DELAYED_ITEMS: 4702 if (state == FLUSH_DELAYED_ITEMS_NR) 4703 nr = calc_reclaim_items_nr(root, num_bytes) * 2; 4704 else 4705 nr = -1; 4706 4707 trans = btrfs_join_transaction(root); 4708 if (IS_ERR(trans)) { 4709 ret = PTR_ERR(trans); 4710 break; 4711 } 4712 ret = btrfs_run_delayed_items_nr(trans, root, nr); 4713 btrfs_end_transaction(trans, root); 4714 break; 4715 case FLUSH_DELALLOC: 4716 case FLUSH_DELALLOC_WAIT: 4717 shrink_delalloc(root, num_bytes * 2, orig_bytes, 4718 state == FLUSH_DELALLOC_WAIT); 4719 break; 4720 case ALLOC_CHUNK: 4721 trans = btrfs_join_transaction(root); 4722 if (IS_ERR(trans)) { 4723 ret = PTR_ERR(trans); 4724 break; 4725 } 4726 ret = do_chunk_alloc(trans, root->fs_info->extent_root, 4727 btrfs_get_alloc_profile(root, 0), 4728 CHUNK_ALLOC_NO_FORCE); 4729 btrfs_end_transaction(trans, root); 4730 if (ret == -ENOSPC) 4731 ret = 0; 4732 break; 4733 case COMMIT_TRANS: 4734 ret = may_commit_transaction(root, space_info, orig_bytes, 0); 4735 break; 4736 default: 4737 ret = -ENOSPC; 4738 break; 4739 } 4740 4741 return ret; 4742 } 4743 4744 static inline u64 4745 btrfs_calc_reclaim_metadata_size(struct btrfs_root *root, 4746 struct btrfs_space_info *space_info) 4747 { 4748 u64 used; 4749 u64 expected; 4750 u64 to_reclaim; 4751 4752 to_reclaim = min_t(u64, num_online_cpus() * 1024 * 1024, 4753 16 * 1024 * 1024); 4754 spin_lock(&space_info->lock); 4755 if (can_overcommit(root, space_info, to_reclaim, 4756 BTRFS_RESERVE_FLUSH_ALL)) { 4757 to_reclaim = 0; 4758 goto out; 4759 } 4760 4761 used = space_info->bytes_used + space_info->bytes_reserved + 4762 space_info->bytes_pinned + space_info->bytes_readonly + 4763 space_info->bytes_may_use; 4764 if (can_overcommit(root, space_info, 1024 * 1024, 4765 BTRFS_RESERVE_FLUSH_ALL)) 4766 expected = div_factor_fine(space_info->total_bytes, 95); 4767 else 4768 expected = div_factor_fine(space_info->total_bytes, 90); 4769 4770 if (used > expected) 4771 to_reclaim = used - expected; 4772 else 4773 to_reclaim = 0; 4774 to_reclaim = min(to_reclaim, space_info->bytes_may_use + 4775 space_info->bytes_reserved); 4776 out: 4777 spin_unlock(&space_info->lock); 4778 4779 return to_reclaim; 4780 } 4781 4782 static inline int need_do_async_reclaim(struct btrfs_space_info *space_info, 4783 struct btrfs_fs_info *fs_info, u64 used) 4784 { 4785 u64 thresh = div_factor_fine(space_info->total_bytes, 98); 4786 4787 /* If we're just plain full then async reclaim just slows us down. */ 4788 if (space_info->bytes_used >= thresh) 4789 return 0; 4790 4791 return (used >= thresh && !btrfs_fs_closing(fs_info) && 4792 !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); 4793 } 4794 4795 static int btrfs_need_do_async_reclaim(struct btrfs_space_info *space_info, 4796 struct btrfs_fs_info *fs_info, 4797 int flush_state) 4798 { 4799 u64 used; 4800 4801 spin_lock(&space_info->lock); 4802 /* 4803 * We run out of space and have not got any free space via flush_space, 4804 * so don't bother doing async reclaim. 4805 */ 4806 if (flush_state > COMMIT_TRANS && space_info->full) { 4807 spin_unlock(&space_info->lock); 4808 return 0; 4809 } 4810 4811 used = space_info->bytes_used + space_info->bytes_reserved + 4812 space_info->bytes_pinned + space_info->bytes_readonly + 4813 space_info->bytes_may_use; 4814 if (need_do_async_reclaim(space_info, fs_info, used)) { 4815 spin_unlock(&space_info->lock); 4816 return 1; 4817 } 4818 spin_unlock(&space_info->lock); 4819 4820 return 0; 4821 } 4822 4823 static void btrfs_async_reclaim_metadata_space(struct work_struct *work) 4824 { 4825 struct btrfs_fs_info *fs_info; 4826 struct btrfs_space_info *space_info; 4827 u64 to_reclaim; 4828 int flush_state; 4829 4830 fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work); 4831 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); 4832 4833 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root, 4834 space_info); 4835 if (!to_reclaim) 4836 return; 4837 4838 flush_state = FLUSH_DELAYED_ITEMS_NR; 4839 do { 4840 flush_space(fs_info->fs_root, space_info, to_reclaim, 4841 to_reclaim, flush_state); 4842 flush_state++; 4843 if (!btrfs_need_do_async_reclaim(space_info, fs_info, 4844 flush_state)) 4845 return; 4846 } while (flush_state < COMMIT_TRANS); 4847 } 4848 4849 void btrfs_init_async_reclaim_work(struct work_struct *work) 4850 { 4851 INIT_WORK(work, btrfs_async_reclaim_metadata_space); 4852 } 4853 4854 /** 4855 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space 4856 * @root - the root we're allocating for 4857 * @block_rsv - the block_rsv we're allocating for 4858 * @orig_bytes - the number of bytes we want 4859 * @flush - whether or not we can flush to make our reservation 4860 * 4861 * This will reserve orgi_bytes number of bytes from the space info associated 4862 * with the block_rsv. If there is not enough space it will make an attempt to 4863 * flush out space to make room. It will do this by flushing delalloc if 4864 * possible or committing the transaction. If flush is 0 then no attempts to 4865 * regain reservations will be made and this will fail if there is not enough 4866 * space already. 4867 */ 4868 static int reserve_metadata_bytes(struct btrfs_root *root, 4869 struct btrfs_block_rsv *block_rsv, 4870 u64 orig_bytes, 4871 enum btrfs_reserve_flush_enum flush) 4872 { 4873 struct btrfs_space_info *space_info = block_rsv->space_info; 4874 u64 used; 4875 u64 num_bytes = orig_bytes; 4876 int flush_state = FLUSH_DELAYED_ITEMS_NR; 4877 int ret = 0; 4878 bool flushing = false; 4879 4880 again: 4881 ret = 0; 4882 spin_lock(&space_info->lock); 4883 /* 4884 * We only want to wait if somebody other than us is flushing and we 4885 * are actually allowed to flush all things. 4886 */ 4887 while (flush == BTRFS_RESERVE_FLUSH_ALL && !flushing && 4888 space_info->flush) { 4889 spin_unlock(&space_info->lock); 4890 /* 4891 * If we have a trans handle we can't wait because the flusher 4892 * may have to commit the transaction, which would mean we would 4893 * deadlock since we are waiting for the flusher to finish, but 4894 * hold the current transaction open. 4895 */ 4896 if (current->journal_info) 4897 return -EAGAIN; 4898 ret = wait_event_killable(space_info->wait, !space_info->flush); 4899 /* Must have been killed, return */ 4900 if (ret) 4901 return -EINTR; 4902 4903 spin_lock(&space_info->lock); 4904 } 4905 4906 ret = -ENOSPC; 4907 used = space_info->bytes_used + space_info->bytes_reserved + 4908 space_info->bytes_pinned + space_info->bytes_readonly + 4909 space_info->bytes_may_use; 4910 4911 /* 4912 * The idea here is that we've not already over-reserved the block group 4913 * then we can go ahead and save our reservation first and then start 4914 * flushing if we need to. Otherwise if we've already overcommitted 4915 * lets start flushing stuff first and then come back and try to make 4916 * our reservation. 4917 */ 4918 if (used <= space_info->total_bytes) { 4919 if (used + orig_bytes <= space_info->total_bytes) { 4920 space_info->bytes_may_use += orig_bytes; 4921 trace_btrfs_space_reservation(root->fs_info, 4922 "space_info", space_info->flags, orig_bytes, 1); 4923 ret = 0; 4924 } else { 4925 /* 4926 * Ok set num_bytes to orig_bytes since we aren't 4927 * overocmmitted, this way we only try and reclaim what 4928 * we need. 4929 */ 4930 num_bytes = orig_bytes; 4931 } 4932 } else { 4933 /* 4934 * Ok we're over committed, set num_bytes to the overcommitted 4935 * amount plus the amount of bytes that we need for this 4936 * reservation. 4937 */ 4938 num_bytes = used - space_info->total_bytes + 4939 (orig_bytes * 2); 4940 } 4941 4942 if (ret && can_overcommit(root, space_info, orig_bytes, flush)) { 4943 space_info->bytes_may_use += orig_bytes; 4944 trace_btrfs_space_reservation(root->fs_info, "space_info", 4945 space_info->flags, orig_bytes, 4946 1); 4947 ret = 0; 4948 } 4949 4950 /* 4951 * Couldn't make our reservation, save our place so while we're trying 4952 * to reclaim space we can actually use it instead of somebody else 4953 * stealing it from us. 4954 * 4955 * We make the other tasks wait for the flush only when we can flush 4956 * all things. 4957 */ 4958 if (ret && flush != BTRFS_RESERVE_NO_FLUSH) { 4959 flushing = true; 4960 space_info->flush = 1; 4961 } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { 4962 used += orig_bytes; 4963 /* 4964 * We will do the space reservation dance during log replay, 4965 * which means we won't have fs_info->fs_root set, so don't do 4966 * the async reclaim as we will panic. 4967 */ 4968 if (!root->fs_info->log_root_recovering && 4969 need_do_async_reclaim(space_info, root->fs_info, used) && 4970 !work_busy(&root->fs_info->async_reclaim_work)) 4971 queue_work(system_unbound_wq, 4972 &root->fs_info->async_reclaim_work); 4973 } 4974 spin_unlock(&space_info->lock); 4975 4976 if (!ret || flush == BTRFS_RESERVE_NO_FLUSH) 4977 goto out; 4978 4979 ret = flush_space(root, space_info, num_bytes, orig_bytes, 4980 flush_state); 4981 flush_state++; 4982 4983 /* 4984 * If we are FLUSH_LIMIT, we can not flush delalloc, or the deadlock 4985 * would happen. So skip delalloc flush. 4986 */ 4987 if (flush == BTRFS_RESERVE_FLUSH_LIMIT && 4988 (flush_state == FLUSH_DELALLOC || 4989 flush_state == FLUSH_DELALLOC_WAIT)) 4990 flush_state = ALLOC_CHUNK; 4991 4992 if (!ret) 4993 goto again; 4994 else if (flush == BTRFS_RESERVE_FLUSH_LIMIT && 4995 flush_state < COMMIT_TRANS) 4996 goto again; 4997 else if (flush == BTRFS_RESERVE_FLUSH_ALL && 4998 flush_state <= COMMIT_TRANS) 4999 goto again; 5000 5001 out: 5002 if (ret == -ENOSPC && 5003 unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) { 5004 struct btrfs_block_rsv *global_rsv = 5005 &root->fs_info->global_block_rsv; 5006 5007 if (block_rsv != global_rsv && 5008 !block_rsv_use_bytes(global_rsv, orig_bytes)) 5009 ret = 0; 5010 } 5011 if (ret == -ENOSPC) 5012 trace_btrfs_space_reservation(root->fs_info, 5013 "space_info:enospc", 5014 space_info->flags, orig_bytes, 1); 5015 if (flushing) { 5016 spin_lock(&space_info->lock); 5017 space_info->flush = 0; 5018 wake_up_all(&space_info->wait); 5019 spin_unlock(&space_info->lock); 5020 } 5021 return ret; 5022 } 5023 5024 static struct btrfs_block_rsv *get_block_rsv( 5025 const struct btrfs_trans_handle *trans, 5026 const struct btrfs_root *root) 5027 { 5028 struct btrfs_block_rsv *block_rsv = NULL; 5029 5030 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 5031 (root == root->fs_info->csum_root && trans->adding_csums) || 5032 (root == root->fs_info->uuid_root)) 5033 block_rsv = trans->block_rsv; 5034 5035 if (!block_rsv) 5036 block_rsv = root->block_rsv; 5037 5038 if (!block_rsv) 5039 block_rsv = &root->fs_info->empty_block_rsv; 5040 5041 return block_rsv; 5042 } 5043 5044 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, 5045 u64 num_bytes) 5046 { 5047 int ret = -ENOSPC; 5048 spin_lock(&block_rsv->lock); 5049 if (block_rsv->reserved >= num_bytes) { 5050 block_rsv->reserved -= num_bytes; 5051 if (block_rsv->reserved < block_rsv->size) 5052 block_rsv->full = 0; 5053 ret = 0; 5054 } 5055 spin_unlock(&block_rsv->lock); 5056 return ret; 5057 } 5058 5059 static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, 5060 u64 num_bytes, int update_size) 5061 { 5062 spin_lock(&block_rsv->lock); 5063 block_rsv->reserved += num_bytes; 5064 if (update_size) 5065 block_rsv->size += num_bytes; 5066 else if (block_rsv->reserved >= block_rsv->size) 5067 block_rsv->full = 1; 5068 spin_unlock(&block_rsv->lock); 5069 } 5070 5071 int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info, 5072 struct btrfs_block_rsv *dest, u64 num_bytes, 5073 int min_factor) 5074 { 5075 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 5076 u64 min_bytes; 5077 5078 if (global_rsv->space_info != dest->space_info) 5079 return -ENOSPC; 5080 5081 spin_lock(&global_rsv->lock); 5082 min_bytes = div_factor(global_rsv->size, min_factor); 5083 if (global_rsv->reserved < min_bytes + num_bytes) { 5084 spin_unlock(&global_rsv->lock); 5085 return -ENOSPC; 5086 } 5087 global_rsv->reserved -= num_bytes; 5088 if (global_rsv->reserved < global_rsv->size) 5089 global_rsv->full = 0; 5090 spin_unlock(&global_rsv->lock); 5091 5092 block_rsv_add_bytes(dest, num_bytes, 1); 5093 return 0; 5094 } 5095 5096 static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info, 5097 struct btrfs_block_rsv *block_rsv, 5098 struct btrfs_block_rsv *dest, u64 num_bytes) 5099 { 5100 struct btrfs_space_info *space_info = block_rsv->space_info; 5101 5102 spin_lock(&block_rsv->lock); 5103 if (num_bytes == (u64)-1) 5104 num_bytes = block_rsv->size; 5105 block_rsv->size -= num_bytes; 5106 if (block_rsv->reserved >= block_rsv->size) { 5107 num_bytes = block_rsv->reserved - block_rsv->size; 5108 block_rsv->reserved = block_rsv->size; 5109 block_rsv->full = 1; 5110 } else { 5111 num_bytes = 0; 5112 } 5113 spin_unlock(&block_rsv->lock); 5114 5115 if (num_bytes > 0) { 5116 if (dest) { 5117 spin_lock(&dest->lock); 5118 if (!dest->full) { 5119 u64 bytes_to_add; 5120 5121 bytes_to_add = dest->size - dest->reserved; 5122 bytes_to_add = min(num_bytes, bytes_to_add); 5123 dest->reserved += bytes_to_add; 5124 if (dest->reserved >= dest->size) 5125 dest->full = 1; 5126 num_bytes -= bytes_to_add; 5127 } 5128 spin_unlock(&dest->lock); 5129 } 5130 if (num_bytes) { 5131 spin_lock(&space_info->lock); 5132 space_info->bytes_may_use -= num_bytes; 5133 trace_btrfs_space_reservation(fs_info, "space_info", 5134 space_info->flags, num_bytes, 0); 5135 spin_unlock(&space_info->lock); 5136 } 5137 } 5138 } 5139 5140 static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src, 5141 struct btrfs_block_rsv *dst, u64 num_bytes) 5142 { 5143 int ret; 5144 5145 ret = block_rsv_use_bytes(src, num_bytes); 5146 if (ret) 5147 return ret; 5148 5149 block_rsv_add_bytes(dst, num_bytes, 1); 5150 return 0; 5151 } 5152 5153 void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type) 5154 { 5155 memset(rsv, 0, sizeof(*rsv)); 5156 spin_lock_init(&rsv->lock); 5157 rsv->type = type; 5158 } 5159 5160 struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root, 5161 unsigned short type) 5162 { 5163 struct btrfs_block_rsv *block_rsv; 5164 struct btrfs_fs_info *fs_info = root->fs_info; 5165 5166 block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS); 5167 if (!block_rsv) 5168 return NULL; 5169 5170 btrfs_init_block_rsv(block_rsv, type); 5171 block_rsv->space_info = __find_space_info(fs_info, 5172 BTRFS_BLOCK_GROUP_METADATA); 5173 return block_rsv; 5174 } 5175 5176 void btrfs_free_block_rsv(struct btrfs_root *root, 5177 struct btrfs_block_rsv *rsv) 5178 { 5179 if (!rsv) 5180 return; 5181 btrfs_block_rsv_release(root, rsv, (u64)-1); 5182 kfree(rsv); 5183 } 5184 5185 void __btrfs_free_block_rsv(struct btrfs_block_rsv *rsv) 5186 { 5187 kfree(rsv); 5188 } 5189 5190 int btrfs_block_rsv_add(struct btrfs_root *root, 5191 struct btrfs_block_rsv *block_rsv, u64 num_bytes, 5192 enum btrfs_reserve_flush_enum flush) 5193 { 5194 int ret; 5195 5196 if (num_bytes == 0) 5197 return 0; 5198 5199 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush); 5200 if (!ret) { 5201 block_rsv_add_bytes(block_rsv, num_bytes, 1); 5202 return 0; 5203 } 5204 5205 return ret; 5206 } 5207 5208 int btrfs_block_rsv_check(struct btrfs_root *root, 5209 struct btrfs_block_rsv *block_rsv, int min_factor) 5210 { 5211 u64 num_bytes = 0; 5212 int ret = -ENOSPC; 5213 5214 if (!block_rsv) 5215 return 0; 5216 5217 spin_lock(&block_rsv->lock); 5218 num_bytes = div_factor(block_rsv->size, min_factor); 5219 if (block_rsv->reserved >= num_bytes) 5220 ret = 0; 5221 spin_unlock(&block_rsv->lock); 5222 5223 return ret; 5224 } 5225 5226 int btrfs_block_rsv_refill(struct btrfs_root *root, 5227 struct btrfs_block_rsv *block_rsv, u64 min_reserved, 5228 enum btrfs_reserve_flush_enum flush) 5229 { 5230 u64 num_bytes = 0; 5231 int ret = -ENOSPC; 5232 5233 if (!block_rsv) 5234 return 0; 5235 5236 spin_lock(&block_rsv->lock); 5237 num_bytes = min_reserved; 5238 if (block_rsv->reserved >= num_bytes) 5239 ret = 0; 5240 else 5241 num_bytes -= block_rsv->reserved; 5242 spin_unlock(&block_rsv->lock); 5243 5244 if (!ret) 5245 return 0; 5246 5247 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush); 5248 if (!ret) { 5249 block_rsv_add_bytes(block_rsv, num_bytes, 0); 5250 return 0; 5251 } 5252 5253 return ret; 5254 } 5255 5256 int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv, 5257 struct btrfs_block_rsv *dst_rsv, 5258 u64 num_bytes) 5259 { 5260 return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes); 5261 } 5262 5263 void btrfs_block_rsv_release(struct btrfs_root *root, 5264 struct btrfs_block_rsv *block_rsv, 5265 u64 num_bytes) 5266 { 5267 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv; 5268 if (global_rsv == block_rsv || 5269 block_rsv->space_info != global_rsv->space_info) 5270 global_rsv = NULL; 5271 block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv, 5272 num_bytes); 5273 } 5274 5275 /* 5276 * helper to calculate size of global block reservation. 5277 * the desired value is sum of space used by extent tree, 5278 * checksum tree and root tree 5279 */ 5280 static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info) 5281 { 5282 struct btrfs_space_info *sinfo; 5283 u64 num_bytes; 5284 u64 meta_used; 5285 u64 data_used; 5286 int csum_size = btrfs_super_csum_size(fs_info->super_copy); 5287 5288 sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA); 5289 spin_lock(&sinfo->lock); 5290 data_used = sinfo->bytes_used; 5291 spin_unlock(&sinfo->lock); 5292 5293 sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); 5294 spin_lock(&sinfo->lock); 5295 if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA) 5296 data_used = 0; 5297 meta_used = sinfo->bytes_used; 5298 spin_unlock(&sinfo->lock); 5299 5300 num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) * 5301 csum_size * 2; 5302 num_bytes += div_u64(data_used + meta_used, 50); 5303 5304 if (num_bytes * 3 > meta_used) 5305 num_bytes = div_u64(meta_used, 3); 5306 5307 return ALIGN(num_bytes, fs_info->extent_root->nodesize << 10); 5308 } 5309 5310 static void update_global_block_rsv(struct btrfs_fs_info *fs_info) 5311 { 5312 struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; 5313 struct btrfs_space_info *sinfo = block_rsv->space_info; 5314 u64 num_bytes; 5315 5316 num_bytes = calc_global_metadata_size(fs_info); 5317 5318 spin_lock(&sinfo->lock); 5319 spin_lock(&block_rsv->lock); 5320 5321 block_rsv->size = min_t(u64, num_bytes, 512 * 1024 * 1024); 5322 5323 num_bytes = sinfo->bytes_used + sinfo->bytes_pinned + 5324 sinfo->bytes_reserved + sinfo->bytes_readonly + 5325 sinfo->bytes_may_use; 5326 5327 if (sinfo->total_bytes > num_bytes) { 5328 num_bytes = sinfo->total_bytes - num_bytes; 5329 block_rsv->reserved += num_bytes; 5330 sinfo->bytes_may_use += num_bytes; 5331 trace_btrfs_space_reservation(fs_info, "space_info", 5332 sinfo->flags, num_bytes, 1); 5333 } 5334 5335 if (block_rsv->reserved >= block_rsv->size) { 5336 num_bytes = block_rsv->reserved - block_rsv->size; 5337 sinfo->bytes_may_use -= num_bytes; 5338 trace_btrfs_space_reservation(fs_info, "space_info", 5339 sinfo->flags, num_bytes, 0); 5340 block_rsv->reserved = block_rsv->size; 5341 block_rsv->full = 1; 5342 } 5343 5344 spin_unlock(&block_rsv->lock); 5345 spin_unlock(&sinfo->lock); 5346 } 5347 5348 static void init_global_block_rsv(struct btrfs_fs_info *fs_info) 5349 { 5350 struct btrfs_space_info *space_info; 5351 5352 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); 5353 fs_info->chunk_block_rsv.space_info = space_info; 5354 5355 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); 5356 fs_info->global_block_rsv.space_info = space_info; 5357 fs_info->delalloc_block_rsv.space_info = space_info; 5358 fs_info->trans_block_rsv.space_info = space_info; 5359 fs_info->empty_block_rsv.space_info = space_info; 5360 fs_info->delayed_block_rsv.space_info = space_info; 5361 5362 fs_info->extent_root->block_rsv = &fs_info->global_block_rsv; 5363 fs_info->csum_root->block_rsv = &fs_info->global_block_rsv; 5364 fs_info->dev_root->block_rsv = &fs_info->global_block_rsv; 5365 fs_info->tree_root->block_rsv = &fs_info->global_block_rsv; 5366 if (fs_info->quota_root) 5367 fs_info->quota_root->block_rsv = &fs_info->global_block_rsv; 5368 fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv; 5369 5370 update_global_block_rsv(fs_info); 5371 } 5372 5373 static void release_global_block_rsv(struct btrfs_fs_info *fs_info) 5374 { 5375 block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL, 5376 (u64)-1); 5377 WARN_ON(fs_info->delalloc_block_rsv.size > 0); 5378 WARN_ON(fs_info->delalloc_block_rsv.reserved > 0); 5379 WARN_ON(fs_info->trans_block_rsv.size > 0); 5380 WARN_ON(fs_info->trans_block_rsv.reserved > 0); 5381 WARN_ON(fs_info->chunk_block_rsv.size > 0); 5382 WARN_ON(fs_info->chunk_block_rsv.reserved > 0); 5383 WARN_ON(fs_info->delayed_block_rsv.size > 0); 5384 WARN_ON(fs_info->delayed_block_rsv.reserved > 0); 5385 } 5386 5387 void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans, 5388 struct btrfs_root *root) 5389 { 5390 if (!trans->block_rsv) 5391 return; 5392 5393 if (!trans->bytes_reserved) 5394 return; 5395 5396 trace_btrfs_space_reservation(root->fs_info, "transaction", 5397 trans->transid, trans->bytes_reserved, 0); 5398 btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved); 5399 trans->bytes_reserved = 0; 5400 } 5401 5402 /* 5403 * To be called after all the new block groups attached to the transaction 5404 * handle have been created (btrfs_create_pending_block_groups()). 5405 */ 5406 void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans) 5407 { 5408 struct btrfs_fs_info *fs_info = trans->root->fs_info; 5409 5410 if (!trans->chunk_bytes_reserved) 5411 return; 5412 5413 WARN_ON_ONCE(!list_empty(&trans->new_bgs)); 5414 5415 block_rsv_release_bytes(fs_info, &fs_info->chunk_block_rsv, NULL, 5416 trans->chunk_bytes_reserved); 5417 trans->chunk_bytes_reserved = 0; 5418 } 5419 5420 /* Can only return 0 or -ENOSPC */ 5421 int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans, 5422 struct inode *inode) 5423 { 5424 struct btrfs_root *root = BTRFS_I(inode)->root; 5425 struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root); 5426 struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv; 5427 5428 /* 5429 * We need to hold space in order to delete our orphan item once we've 5430 * added it, so this takes the reservation so we can release it later 5431 * when we are truly done with the orphan item. 5432 */ 5433 u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1); 5434 trace_btrfs_space_reservation(root->fs_info, "orphan", 5435 btrfs_ino(inode), num_bytes, 1); 5436 return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes); 5437 } 5438 5439 void btrfs_orphan_release_metadata(struct inode *inode) 5440 { 5441 struct btrfs_root *root = BTRFS_I(inode)->root; 5442 u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1); 5443 trace_btrfs_space_reservation(root->fs_info, "orphan", 5444 btrfs_ino(inode), num_bytes, 0); 5445 btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes); 5446 } 5447 5448 /* 5449 * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation 5450 * root: the root of the parent directory 5451 * rsv: block reservation 5452 * items: the number of items that we need do reservation 5453 * qgroup_reserved: used to return the reserved size in qgroup 5454 * 5455 * This function is used to reserve the space for snapshot/subvolume 5456 * creation and deletion. Those operations are different with the 5457 * common file/directory operations, they change two fs/file trees 5458 * and root tree, the number of items that the qgroup reserves is 5459 * different with the free space reservation. So we can not use 5460 * the space reseravtion mechanism in start_transaction(). 5461 */ 5462 int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, 5463 struct btrfs_block_rsv *rsv, 5464 int items, 5465 u64 *qgroup_reserved, 5466 bool use_global_rsv) 5467 { 5468 u64 num_bytes; 5469 int ret; 5470 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv; 5471 5472 if (root->fs_info->quota_enabled) { 5473 /* One for parent inode, two for dir entries */ 5474 num_bytes = 3 * root->nodesize; 5475 ret = btrfs_qgroup_reserve_meta(root, num_bytes); 5476 if (ret) 5477 return ret; 5478 } else { 5479 num_bytes = 0; 5480 } 5481 5482 *qgroup_reserved = num_bytes; 5483 5484 num_bytes = btrfs_calc_trans_metadata_size(root, items); 5485 rsv->space_info = __find_space_info(root->fs_info, 5486 BTRFS_BLOCK_GROUP_METADATA); 5487 ret = btrfs_block_rsv_add(root, rsv, num_bytes, 5488 BTRFS_RESERVE_FLUSH_ALL); 5489 5490 if (ret == -ENOSPC && use_global_rsv) 5491 ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes); 5492 5493 if (ret && *qgroup_reserved) 5494 btrfs_qgroup_free_meta(root, *qgroup_reserved); 5495 5496 return ret; 5497 } 5498 5499 void btrfs_subvolume_release_metadata(struct btrfs_root *root, 5500 struct btrfs_block_rsv *rsv, 5501 u64 qgroup_reserved) 5502 { 5503 btrfs_block_rsv_release(root, rsv, (u64)-1); 5504 } 5505 5506 /** 5507 * drop_outstanding_extent - drop an outstanding extent 5508 * @inode: the inode we're dropping the extent for 5509 * @num_bytes: the number of bytes we're relaseing. 5510 * 5511 * This is called when we are freeing up an outstanding extent, either called 5512 * after an error or after an extent is written. This will return the number of 5513 * reserved extents that need to be freed. This must be called with 5514 * BTRFS_I(inode)->lock held. 5515 */ 5516 static unsigned drop_outstanding_extent(struct inode *inode, u64 num_bytes) 5517 { 5518 unsigned drop_inode_space = 0; 5519 unsigned dropped_extents = 0; 5520 unsigned num_extents = 0; 5521 5522 num_extents = (unsigned)div64_u64(num_bytes + 5523 BTRFS_MAX_EXTENT_SIZE - 1, 5524 BTRFS_MAX_EXTENT_SIZE); 5525 ASSERT(num_extents); 5526 ASSERT(BTRFS_I(inode)->outstanding_extents >= num_extents); 5527 BTRFS_I(inode)->outstanding_extents -= num_extents; 5528 5529 if (BTRFS_I(inode)->outstanding_extents == 0 && 5530 test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED, 5531 &BTRFS_I(inode)->runtime_flags)) 5532 drop_inode_space = 1; 5533 5534 /* 5535 * If we have more or the same amount of outsanding extents than we have 5536 * reserved then we need to leave the reserved extents count alone. 5537 */ 5538 if (BTRFS_I(inode)->outstanding_extents >= 5539 BTRFS_I(inode)->reserved_extents) 5540 return drop_inode_space; 5541 5542 dropped_extents = BTRFS_I(inode)->reserved_extents - 5543 BTRFS_I(inode)->outstanding_extents; 5544 BTRFS_I(inode)->reserved_extents -= dropped_extents; 5545 return dropped_extents + drop_inode_space; 5546 } 5547 5548 /** 5549 * calc_csum_metadata_size - return the amount of metada space that must be 5550 * reserved/free'd for the given bytes. 5551 * @inode: the inode we're manipulating 5552 * @num_bytes: the number of bytes in question 5553 * @reserve: 1 if we are reserving space, 0 if we are freeing space 5554 * 5555 * This adjusts the number of csum_bytes in the inode and then returns the 5556 * correct amount of metadata that must either be reserved or freed. We 5557 * calculate how many checksums we can fit into one leaf and then divide the 5558 * number of bytes that will need to be checksumed by this value to figure out 5559 * how many checksums will be required. If we are adding bytes then the number 5560 * may go up and we will return the number of additional bytes that must be 5561 * reserved. If it is going down we will return the number of bytes that must 5562 * be freed. 5563 * 5564 * This must be called with BTRFS_I(inode)->lock held. 5565 */ 5566 static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes, 5567 int reserve) 5568 { 5569 struct btrfs_root *root = BTRFS_I(inode)->root; 5570 u64 old_csums, num_csums; 5571 5572 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM && 5573 BTRFS_I(inode)->csum_bytes == 0) 5574 return 0; 5575 5576 old_csums = btrfs_csum_bytes_to_leaves(root, BTRFS_I(inode)->csum_bytes); 5577 if (reserve) 5578 BTRFS_I(inode)->csum_bytes += num_bytes; 5579 else 5580 BTRFS_I(inode)->csum_bytes -= num_bytes; 5581 num_csums = btrfs_csum_bytes_to_leaves(root, BTRFS_I(inode)->csum_bytes); 5582 5583 /* No change, no need to reserve more */ 5584 if (old_csums == num_csums) 5585 return 0; 5586 5587 if (reserve) 5588 return btrfs_calc_trans_metadata_size(root, 5589 num_csums - old_csums); 5590 5591 return btrfs_calc_trans_metadata_size(root, old_csums - num_csums); 5592 } 5593 5594 int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes) 5595 { 5596 struct btrfs_root *root = BTRFS_I(inode)->root; 5597 struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv; 5598 u64 to_reserve = 0; 5599 u64 csum_bytes; 5600 unsigned nr_extents = 0; 5601 int extra_reserve = 0; 5602 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL; 5603 int ret = 0; 5604 bool delalloc_lock = true; 5605 u64 to_free = 0; 5606 unsigned dropped; 5607 5608 /* If we are a free space inode we need to not flush since we will be in 5609 * the middle of a transaction commit. We also don't need the delalloc 5610 * mutex since we won't race with anybody. We need this mostly to make 5611 * lockdep shut its filthy mouth. 5612 */ 5613 if (btrfs_is_free_space_inode(inode)) { 5614 flush = BTRFS_RESERVE_NO_FLUSH; 5615 delalloc_lock = false; 5616 } 5617 5618 if (flush != BTRFS_RESERVE_NO_FLUSH && 5619 btrfs_transaction_in_commit(root->fs_info)) 5620 schedule_timeout(1); 5621 5622 if (delalloc_lock) 5623 mutex_lock(&BTRFS_I(inode)->delalloc_mutex); 5624 5625 num_bytes = ALIGN(num_bytes, root->sectorsize); 5626 5627 spin_lock(&BTRFS_I(inode)->lock); 5628 nr_extents = (unsigned)div64_u64(num_bytes + 5629 BTRFS_MAX_EXTENT_SIZE - 1, 5630 BTRFS_MAX_EXTENT_SIZE); 5631 BTRFS_I(inode)->outstanding_extents += nr_extents; 5632 nr_extents = 0; 5633 5634 if (BTRFS_I(inode)->outstanding_extents > 5635 BTRFS_I(inode)->reserved_extents) 5636 nr_extents = BTRFS_I(inode)->outstanding_extents - 5637 BTRFS_I(inode)->reserved_extents; 5638 5639 /* 5640 * Add an item to reserve for updating the inode when we complete the 5641 * delalloc io. 5642 */ 5643 if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED, 5644 &BTRFS_I(inode)->runtime_flags)) { 5645 nr_extents++; 5646 extra_reserve = 1; 5647 } 5648 5649 to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents); 5650 to_reserve += calc_csum_metadata_size(inode, num_bytes, 1); 5651 csum_bytes = BTRFS_I(inode)->csum_bytes; 5652 spin_unlock(&BTRFS_I(inode)->lock); 5653 5654 if (root->fs_info->quota_enabled) { 5655 ret = btrfs_qgroup_reserve_meta(root, 5656 nr_extents * root->nodesize); 5657 if (ret) 5658 goto out_fail; 5659 } 5660 5661 ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush); 5662 if (unlikely(ret)) { 5663 btrfs_qgroup_free_meta(root, nr_extents * root->nodesize); 5664 goto out_fail; 5665 } 5666 5667 spin_lock(&BTRFS_I(inode)->lock); 5668 if (extra_reserve) { 5669 set_bit(BTRFS_INODE_DELALLOC_META_RESERVED, 5670 &BTRFS_I(inode)->runtime_flags); 5671 nr_extents--; 5672 } 5673 BTRFS_I(inode)->reserved_extents += nr_extents; 5674 spin_unlock(&BTRFS_I(inode)->lock); 5675 5676 if (delalloc_lock) 5677 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex); 5678 5679 if (to_reserve) 5680 trace_btrfs_space_reservation(root->fs_info, "delalloc", 5681 btrfs_ino(inode), to_reserve, 1); 5682 block_rsv_add_bytes(block_rsv, to_reserve, 1); 5683 5684 return 0; 5685 5686 out_fail: 5687 spin_lock(&BTRFS_I(inode)->lock); 5688 dropped = drop_outstanding_extent(inode, num_bytes); 5689 /* 5690 * If the inodes csum_bytes is the same as the original 5691 * csum_bytes then we know we haven't raced with any free()ers 5692 * so we can just reduce our inodes csum bytes and carry on. 5693 */ 5694 if (BTRFS_I(inode)->csum_bytes == csum_bytes) { 5695 calc_csum_metadata_size(inode, num_bytes, 0); 5696 } else { 5697 u64 orig_csum_bytes = BTRFS_I(inode)->csum_bytes; 5698 u64 bytes; 5699 5700 /* 5701 * This is tricky, but first we need to figure out how much we 5702 * free'd from any free-ers that occured during this 5703 * reservation, so we reset ->csum_bytes to the csum_bytes 5704 * before we dropped our lock, and then call the free for the 5705 * number of bytes that were freed while we were trying our 5706 * reservation. 5707 */ 5708 bytes = csum_bytes - BTRFS_I(inode)->csum_bytes; 5709 BTRFS_I(inode)->csum_bytes = csum_bytes; 5710 to_free = calc_csum_metadata_size(inode, bytes, 0); 5711 5712 5713 /* 5714 * Now we need to see how much we would have freed had we not 5715 * been making this reservation and our ->csum_bytes were not 5716 * artificially inflated. 5717 */ 5718 BTRFS_I(inode)->csum_bytes = csum_bytes - num_bytes; 5719 bytes = csum_bytes - orig_csum_bytes; 5720 bytes = calc_csum_metadata_size(inode, bytes, 0); 5721 5722 /* 5723 * Now reset ->csum_bytes to what it should be. If bytes is 5724 * more than to_free then we would have free'd more space had we 5725 * not had an artificially high ->csum_bytes, so we need to free 5726 * the remainder. If bytes is the same or less then we don't 5727 * need to do anything, the other free-ers did the correct 5728 * thing. 5729 */ 5730 BTRFS_I(inode)->csum_bytes = orig_csum_bytes - num_bytes; 5731 if (bytes > to_free) 5732 to_free = bytes - to_free; 5733 else 5734 to_free = 0; 5735 } 5736 spin_unlock(&BTRFS_I(inode)->lock); 5737 if (dropped) 5738 to_free += btrfs_calc_trans_metadata_size(root, dropped); 5739 5740 if (to_free) { 5741 btrfs_block_rsv_release(root, block_rsv, to_free); 5742 trace_btrfs_space_reservation(root->fs_info, "delalloc", 5743 btrfs_ino(inode), to_free, 0); 5744 } 5745 if (delalloc_lock) 5746 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex); 5747 return ret; 5748 } 5749 5750 /** 5751 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode 5752 * @inode: the inode to release the reservation for 5753 * @num_bytes: the number of bytes we're releasing 5754 * 5755 * This will release the metadata reservation for an inode. This can be called 5756 * once we complete IO for a given set of bytes to release their metadata 5757 * reservations. 5758 */ 5759 void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes) 5760 { 5761 struct btrfs_root *root = BTRFS_I(inode)->root; 5762 u64 to_free = 0; 5763 unsigned dropped; 5764 5765 num_bytes = ALIGN(num_bytes, root->sectorsize); 5766 spin_lock(&BTRFS_I(inode)->lock); 5767 dropped = drop_outstanding_extent(inode, num_bytes); 5768 5769 if (num_bytes) 5770 to_free = calc_csum_metadata_size(inode, num_bytes, 0); 5771 spin_unlock(&BTRFS_I(inode)->lock); 5772 if (dropped > 0) 5773 to_free += btrfs_calc_trans_metadata_size(root, dropped); 5774 5775 if (btrfs_test_is_dummy_root(root)) 5776 return; 5777 5778 trace_btrfs_space_reservation(root->fs_info, "delalloc", 5779 btrfs_ino(inode), to_free, 0); 5780 5781 btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv, 5782 to_free); 5783 } 5784 5785 /** 5786 * btrfs_delalloc_reserve_space - reserve data and metadata space for 5787 * delalloc 5788 * @inode: inode we're writing to 5789 * @start: start range we are writing to 5790 * @len: how long the range we are writing to 5791 * 5792 * TODO: This function will finally replace old btrfs_delalloc_reserve_space() 5793 * 5794 * This will do the following things 5795 * 5796 * o reserve space in data space info for num bytes 5797 * and reserve precious corresponding qgroup space 5798 * (Done in check_data_free_space) 5799 * 5800 * o reserve space for metadata space, based on the number of outstanding 5801 * extents and how much csums will be needed 5802 * also reserve metadata space in a per root over-reserve method. 5803 * o add to the inodes->delalloc_bytes 5804 * o add it to the fs_info's delalloc inodes list. 5805 * (Above 3 all done in delalloc_reserve_metadata) 5806 * 5807 * Return 0 for success 5808 * Return <0 for error(-ENOSPC or -EQUOT) 5809 */ 5810 int btrfs_delalloc_reserve_space(struct inode *inode, u64 start, u64 len) 5811 { 5812 int ret; 5813 5814 ret = btrfs_check_data_free_space(inode, start, len); 5815 if (ret < 0) 5816 return ret; 5817 ret = btrfs_delalloc_reserve_metadata(inode, len); 5818 if (ret < 0) 5819 btrfs_free_reserved_data_space(inode, start, len); 5820 return ret; 5821 } 5822 5823 /** 5824 * btrfs_delalloc_release_space - release data and metadata space for delalloc 5825 * @inode: inode we're releasing space for 5826 * @start: start position of the space already reserved 5827 * @len: the len of the space already reserved 5828 * 5829 * This must be matched with a call to btrfs_delalloc_reserve_space. This is 5830 * called in the case that we don't need the metadata AND data reservations 5831 * anymore. So if there is an error or we insert an inline extent. 5832 * 5833 * This function will release the metadata space that was not used and will 5834 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes 5835 * list if there are no delalloc bytes left. 5836 * Also it will handle the qgroup reserved space. 5837 */ 5838 void btrfs_delalloc_release_space(struct inode *inode, u64 start, u64 len) 5839 { 5840 btrfs_delalloc_release_metadata(inode, len); 5841 btrfs_free_reserved_data_space(inode, start, len); 5842 } 5843 5844 static int update_block_group(struct btrfs_trans_handle *trans, 5845 struct btrfs_root *root, u64 bytenr, 5846 u64 num_bytes, int alloc) 5847 { 5848 struct btrfs_block_group_cache *cache = NULL; 5849 struct btrfs_fs_info *info = root->fs_info; 5850 u64 total = num_bytes; 5851 u64 old_val; 5852 u64 byte_in_group; 5853 int factor; 5854 5855 /* block accounting for super block */ 5856 spin_lock(&info->delalloc_root_lock); 5857 old_val = btrfs_super_bytes_used(info->super_copy); 5858 if (alloc) 5859 old_val += num_bytes; 5860 else 5861 old_val -= num_bytes; 5862 btrfs_set_super_bytes_used(info->super_copy, old_val); 5863 spin_unlock(&info->delalloc_root_lock); 5864 5865 while (total) { 5866 cache = btrfs_lookup_block_group(info, bytenr); 5867 if (!cache) 5868 return -ENOENT; 5869 if (cache->flags & (BTRFS_BLOCK_GROUP_DUP | 5870 BTRFS_BLOCK_GROUP_RAID1 | 5871 BTRFS_BLOCK_GROUP_RAID10)) 5872 factor = 2; 5873 else 5874 factor = 1; 5875 /* 5876 * If this block group has free space cache written out, we 5877 * need to make sure to load it if we are removing space. This 5878 * is because we need the unpinning stage to actually add the 5879 * space back to the block group, otherwise we will leak space. 5880 */ 5881 if (!alloc && cache->cached == BTRFS_CACHE_NO) 5882 cache_block_group(cache, 1); 5883 5884 byte_in_group = bytenr - cache->key.objectid; 5885 WARN_ON(byte_in_group > cache->key.offset); 5886 5887 spin_lock(&cache->space_info->lock); 5888 spin_lock(&cache->lock); 5889 5890 if (btrfs_test_opt(root, SPACE_CACHE) && 5891 cache->disk_cache_state < BTRFS_DC_CLEAR) 5892 cache->disk_cache_state = BTRFS_DC_CLEAR; 5893 5894 old_val = btrfs_block_group_used(&cache->item); 5895 num_bytes = min(total, cache->key.offset - byte_in_group); 5896 if (alloc) { 5897 old_val += num_bytes; 5898 btrfs_set_block_group_used(&cache->item, old_val); 5899 cache->reserved -= num_bytes; 5900 cache->space_info->bytes_reserved -= num_bytes; 5901 cache->space_info->bytes_used += num_bytes; 5902 cache->space_info->disk_used += num_bytes * factor; 5903 spin_unlock(&cache->lock); 5904 spin_unlock(&cache->space_info->lock); 5905 } else { 5906 old_val -= num_bytes; 5907 btrfs_set_block_group_used(&cache->item, old_val); 5908 cache->pinned += num_bytes; 5909 cache->space_info->bytes_pinned += num_bytes; 5910 cache->space_info->bytes_used -= num_bytes; 5911 cache->space_info->disk_used -= num_bytes * factor; 5912 spin_unlock(&cache->lock); 5913 spin_unlock(&cache->space_info->lock); 5914 5915 set_extent_dirty(info->pinned_extents, 5916 bytenr, bytenr + num_bytes - 1, 5917 GFP_NOFS | __GFP_NOFAIL); 5918 } 5919 5920 spin_lock(&trans->transaction->dirty_bgs_lock); 5921 if (list_empty(&cache->dirty_list)) { 5922 list_add_tail(&cache->dirty_list, 5923 &trans->transaction->dirty_bgs); 5924 trans->transaction->num_dirty_bgs++; 5925 btrfs_get_block_group(cache); 5926 } 5927 spin_unlock(&trans->transaction->dirty_bgs_lock); 5928 5929 /* 5930 * No longer have used bytes in this block group, queue it for 5931 * deletion. We do this after adding the block group to the 5932 * dirty list to avoid races between cleaner kthread and space 5933 * cache writeout. 5934 */ 5935 if (!alloc && old_val == 0) { 5936 spin_lock(&info->unused_bgs_lock); 5937 if (list_empty(&cache->bg_list)) { 5938 btrfs_get_block_group(cache); 5939 list_add_tail(&cache->bg_list, 5940 &info->unused_bgs); 5941 } 5942 spin_unlock(&info->unused_bgs_lock); 5943 } 5944 5945 btrfs_put_block_group(cache); 5946 total -= num_bytes; 5947 bytenr += num_bytes; 5948 } 5949 return 0; 5950 } 5951 5952 static u64 first_logical_byte(struct btrfs_root *root, u64 search_start) 5953 { 5954 struct btrfs_block_group_cache *cache; 5955 u64 bytenr; 5956 5957 spin_lock(&root->fs_info->block_group_cache_lock); 5958 bytenr = root->fs_info->first_logical_byte; 5959 spin_unlock(&root->fs_info->block_group_cache_lock); 5960 5961 if (bytenr < (u64)-1) 5962 return bytenr; 5963 5964 cache = btrfs_lookup_first_block_group(root->fs_info, search_start); 5965 if (!cache) 5966 return 0; 5967 5968 bytenr = cache->key.objectid; 5969 btrfs_put_block_group(cache); 5970 5971 return bytenr; 5972 } 5973 5974 static int pin_down_extent(struct btrfs_root *root, 5975 struct btrfs_block_group_cache *cache, 5976 u64 bytenr, u64 num_bytes, int reserved) 5977 { 5978 spin_lock(&cache->space_info->lock); 5979 spin_lock(&cache->lock); 5980 cache->pinned += num_bytes; 5981 cache->space_info->bytes_pinned += num_bytes; 5982 if (reserved) { 5983 cache->reserved -= num_bytes; 5984 cache->space_info->bytes_reserved -= num_bytes; 5985 } 5986 spin_unlock(&cache->lock); 5987 spin_unlock(&cache->space_info->lock); 5988 5989 set_extent_dirty(root->fs_info->pinned_extents, bytenr, 5990 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL); 5991 if (reserved) 5992 trace_btrfs_reserved_extent_free(root, bytenr, num_bytes); 5993 return 0; 5994 } 5995 5996 /* 5997 * this function must be called within transaction 5998 */ 5999 int btrfs_pin_extent(struct btrfs_root *root, 6000 u64 bytenr, u64 num_bytes, int reserved) 6001 { 6002 struct btrfs_block_group_cache *cache; 6003 6004 cache = btrfs_lookup_block_group(root->fs_info, bytenr); 6005 BUG_ON(!cache); /* Logic error */ 6006 6007 pin_down_extent(root, cache, bytenr, num_bytes, reserved); 6008 6009 btrfs_put_block_group(cache); 6010 return 0; 6011 } 6012 6013 /* 6014 * this function must be called within transaction 6015 */ 6016 int btrfs_pin_extent_for_log_replay(struct btrfs_root *root, 6017 u64 bytenr, u64 num_bytes) 6018 { 6019 struct btrfs_block_group_cache *cache; 6020 int ret; 6021 6022 cache = btrfs_lookup_block_group(root->fs_info, bytenr); 6023 if (!cache) 6024 return -EINVAL; 6025 6026 /* 6027 * pull in the free space cache (if any) so that our pin 6028 * removes the free space from the cache. We have load_only set 6029 * to one because the slow code to read in the free extents does check 6030 * the pinned extents. 6031 */ 6032 cache_block_group(cache, 1); 6033 6034 pin_down_extent(root, cache, bytenr, num_bytes, 0); 6035 6036 /* remove us from the free space cache (if we're there at all) */ 6037 ret = btrfs_remove_free_space(cache, bytenr, num_bytes); 6038 btrfs_put_block_group(cache); 6039 return ret; 6040 } 6041 6042 static int __exclude_logged_extent(struct btrfs_root *root, u64 start, u64 num_bytes) 6043 { 6044 int ret; 6045 struct btrfs_block_group_cache *block_group; 6046 struct btrfs_caching_control *caching_ctl; 6047 6048 block_group = btrfs_lookup_block_group(root->fs_info, start); 6049 if (!block_group) 6050 return -EINVAL; 6051 6052 cache_block_group(block_group, 0); 6053 caching_ctl = get_caching_control(block_group); 6054 6055 if (!caching_ctl) { 6056 /* Logic error */ 6057 BUG_ON(!block_group_cache_done(block_group)); 6058 ret = btrfs_remove_free_space(block_group, start, num_bytes); 6059 } else { 6060 mutex_lock(&caching_ctl->mutex); 6061 6062 if (start >= caching_ctl->progress) { 6063 ret = add_excluded_extent(root, start, num_bytes); 6064 } else if (start + num_bytes <= caching_ctl->progress) { 6065 ret = btrfs_remove_free_space(block_group, 6066 start, num_bytes); 6067 } else { 6068 num_bytes = caching_ctl->progress - start; 6069 ret = btrfs_remove_free_space(block_group, 6070 start, num_bytes); 6071 if (ret) 6072 goto out_lock; 6073 6074 num_bytes = (start + num_bytes) - 6075 caching_ctl->progress; 6076 start = caching_ctl->progress; 6077 ret = add_excluded_extent(root, start, num_bytes); 6078 } 6079 out_lock: 6080 mutex_unlock(&caching_ctl->mutex); 6081 put_caching_control(caching_ctl); 6082 } 6083 btrfs_put_block_group(block_group); 6084 return ret; 6085 } 6086 6087 int btrfs_exclude_logged_extents(struct btrfs_root *log, 6088 struct extent_buffer *eb) 6089 { 6090 struct btrfs_file_extent_item *item; 6091 struct btrfs_key key; 6092 int found_type; 6093 int i; 6094 6095 if (!btrfs_fs_incompat(log->fs_info, MIXED_GROUPS)) 6096 return 0; 6097 6098 for (i = 0; i < btrfs_header_nritems(eb); i++) { 6099 btrfs_item_key_to_cpu(eb, &key, i); 6100 if (key.type != BTRFS_EXTENT_DATA_KEY) 6101 continue; 6102 item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item); 6103 found_type = btrfs_file_extent_type(eb, item); 6104 if (found_type == BTRFS_FILE_EXTENT_INLINE) 6105 continue; 6106 if (btrfs_file_extent_disk_bytenr(eb, item) == 0) 6107 continue; 6108 key.objectid = btrfs_file_extent_disk_bytenr(eb, item); 6109 key.offset = btrfs_file_extent_disk_num_bytes(eb, item); 6110 __exclude_logged_extent(log, key.objectid, key.offset); 6111 } 6112 6113 return 0; 6114 } 6115 6116 /** 6117 * btrfs_update_reserved_bytes - update the block_group and space info counters 6118 * @cache: The cache we are manipulating 6119 * @num_bytes: The number of bytes in question 6120 * @reserve: One of the reservation enums 6121 * @delalloc: The blocks are allocated for the delalloc write 6122 * 6123 * This is called by the allocator when it reserves space, or by somebody who is 6124 * freeing space that was never actually used on disk. For example if you 6125 * reserve some space for a new leaf in transaction A and before transaction A 6126 * commits you free that leaf, you call this with reserve set to 0 in order to 6127 * clear the reservation. 6128 * 6129 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper 6130 * ENOSPC accounting. For data we handle the reservation through clearing the 6131 * delalloc bits in the io_tree. We have to do this since we could end up 6132 * allocating less disk space for the amount of data we have reserved in the 6133 * case of compression. 6134 * 6135 * If this is a reservation and the block group has become read only we cannot 6136 * make the reservation and return -EAGAIN, otherwise this function always 6137 * succeeds. 6138 */ 6139 static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache, 6140 u64 num_bytes, int reserve, int delalloc) 6141 { 6142 struct btrfs_space_info *space_info = cache->space_info; 6143 int ret = 0; 6144 6145 spin_lock(&space_info->lock); 6146 spin_lock(&cache->lock); 6147 if (reserve != RESERVE_FREE) { 6148 if (cache->ro) { 6149 ret = -EAGAIN; 6150 } else { 6151 cache->reserved += num_bytes; 6152 space_info->bytes_reserved += num_bytes; 6153 if (reserve == RESERVE_ALLOC) { 6154 trace_btrfs_space_reservation(cache->fs_info, 6155 "space_info", space_info->flags, 6156 num_bytes, 0); 6157 space_info->bytes_may_use -= num_bytes; 6158 } 6159 6160 if (delalloc) 6161 cache->delalloc_bytes += num_bytes; 6162 } 6163 } else { 6164 if (cache->ro) 6165 space_info->bytes_readonly += num_bytes; 6166 cache->reserved -= num_bytes; 6167 space_info->bytes_reserved -= num_bytes; 6168 6169 if (delalloc) 6170 cache->delalloc_bytes -= num_bytes; 6171 } 6172 spin_unlock(&cache->lock); 6173 spin_unlock(&space_info->lock); 6174 return ret; 6175 } 6176 6177 void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans, 6178 struct btrfs_root *root) 6179 { 6180 struct btrfs_fs_info *fs_info = root->fs_info; 6181 struct btrfs_caching_control *next; 6182 struct btrfs_caching_control *caching_ctl; 6183 struct btrfs_block_group_cache *cache; 6184 6185 down_write(&fs_info->commit_root_sem); 6186 6187 list_for_each_entry_safe(caching_ctl, next, 6188 &fs_info->caching_block_groups, list) { 6189 cache = caching_ctl->block_group; 6190 if (block_group_cache_done(cache)) { 6191 cache->last_byte_to_unpin = (u64)-1; 6192 list_del_init(&caching_ctl->list); 6193 put_caching_control(caching_ctl); 6194 } else { 6195 cache->last_byte_to_unpin = caching_ctl->progress; 6196 } 6197 } 6198 6199 if (fs_info->pinned_extents == &fs_info->freed_extents[0]) 6200 fs_info->pinned_extents = &fs_info->freed_extents[1]; 6201 else 6202 fs_info->pinned_extents = &fs_info->freed_extents[0]; 6203 6204 up_write(&fs_info->commit_root_sem); 6205 6206 update_global_block_rsv(fs_info); 6207 } 6208 6209 /* 6210 * Returns the free cluster for the given space info and sets empty_cluster to 6211 * what it should be based on the mount options. 6212 */ 6213 static struct btrfs_free_cluster * 6214 fetch_cluster_info(struct btrfs_root *root, struct btrfs_space_info *space_info, 6215 u64 *empty_cluster) 6216 { 6217 struct btrfs_free_cluster *ret = NULL; 6218 bool ssd = btrfs_test_opt(root, SSD); 6219 6220 *empty_cluster = 0; 6221 if (btrfs_mixed_space_info(space_info)) 6222 return ret; 6223 6224 if (ssd) 6225 *empty_cluster = 2 * 1024 * 1024; 6226 if (space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { 6227 ret = &root->fs_info->meta_alloc_cluster; 6228 if (!ssd) 6229 *empty_cluster = 64 * 1024; 6230 } else if ((space_info->flags & BTRFS_BLOCK_GROUP_DATA) && ssd) { 6231 ret = &root->fs_info->data_alloc_cluster; 6232 } 6233 6234 return ret; 6235 } 6236 6237 static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end, 6238 const bool return_free_space) 6239 { 6240 struct btrfs_fs_info *fs_info = root->fs_info; 6241 struct btrfs_block_group_cache *cache = NULL; 6242 struct btrfs_space_info *space_info; 6243 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 6244 struct btrfs_free_cluster *cluster = NULL; 6245 u64 len; 6246 u64 total_unpinned = 0; 6247 u64 empty_cluster = 0; 6248 bool readonly; 6249 6250 while (start <= end) { 6251 readonly = false; 6252 if (!cache || 6253 start >= cache->key.objectid + cache->key.offset) { 6254 if (cache) 6255 btrfs_put_block_group(cache); 6256 total_unpinned = 0; 6257 cache = btrfs_lookup_block_group(fs_info, start); 6258 BUG_ON(!cache); /* Logic error */ 6259 6260 cluster = fetch_cluster_info(root, 6261 cache->space_info, 6262 &empty_cluster); 6263 empty_cluster <<= 1; 6264 } 6265 6266 len = cache->key.objectid + cache->key.offset - start; 6267 len = min(len, end + 1 - start); 6268 6269 if (start < cache->last_byte_to_unpin) { 6270 len = min(len, cache->last_byte_to_unpin - start); 6271 if (return_free_space) 6272 btrfs_add_free_space(cache, start, len); 6273 } 6274 6275 start += len; 6276 total_unpinned += len; 6277 space_info = cache->space_info; 6278 6279 /* 6280 * If this space cluster has been marked as fragmented and we've 6281 * unpinned enough in this block group to potentially allow a 6282 * cluster to be created inside of it go ahead and clear the 6283 * fragmented check. 6284 */ 6285 if (cluster && cluster->fragmented && 6286 total_unpinned > empty_cluster) { 6287 spin_lock(&cluster->lock); 6288 cluster->fragmented = 0; 6289 spin_unlock(&cluster->lock); 6290 } 6291 6292 spin_lock(&space_info->lock); 6293 spin_lock(&cache->lock); 6294 cache->pinned -= len; 6295 space_info->bytes_pinned -= len; 6296 space_info->max_extent_size = 0; 6297 percpu_counter_add(&space_info->total_bytes_pinned, -len); 6298 if (cache->ro) { 6299 space_info->bytes_readonly += len; 6300 readonly = true; 6301 } 6302 spin_unlock(&cache->lock); 6303 if (!readonly && global_rsv->space_info == space_info) { 6304 spin_lock(&global_rsv->lock); 6305 if (!global_rsv->full) { 6306 len = min(len, global_rsv->size - 6307 global_rsv->reserved); 6308 global_rsv->reserved += len; 6309 space_info->bytes_may_use += len; 6310 if (global_rsv->reserved >= global_rsv->size) 6311 global_rsv->full = 1; 6312 } 6313 spin_unlock(&global_rsv->lock); 6314 } 6315 spin_unlock(&space_info->lock); 6316 } 6317 6318 if (cache) 6319 btrfs_put_block_group(cache); 6320 return 0; 6321 } 6322 6323 int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans, 6324 struct btrfs_root *root) 6325 { 6326 struct btrfs_fs_info *fs_info = root->fs_info; 6327 struct btrfs_block_group_cache *block_group, *tmp; 6328 struct list_head *deleted_bgs; 6329 struct extent_io_tree *unpin; 6330 u64 start; 6331 u64 end; 6332 int ret; 6333 6334 if (fs_info->pinned_extents == &fs_info->freed_extents[0]) 6335 unpin = &fs_info->freed_extents[1]; 6336 else 6337 unpin = &fs_info->freed_extents[0]; 6338 6339 while (!trans->aborted) { 6340 mutex_lock(&fs_info->unused_bg_unpin_mutex); 6341 ret = find_first_extent_bit(unpin, 0, &start, &end, 6342 EXTENT_DIRTY, NULL); 6343 if (ret) { 6344 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 6345 break; 6346 } 6347 6348 if (btrfs_test_opt(root, DISCARD)) 6349 ret = btrfs_discard_extent(root, start, 6350 end + 1 - start, NULL); 6351 6352 clear_extent_dirty(unpin, start, end, GFP_NOFS); 6353 unpin_extent_range(root, start, end, true); 6354 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 6355 cond_resched(); 6356 } 6357 6358 /* 6359 * Transaction is finished. We don't need the lock anymore. We 6360 * do need to clean up the block groups in case of a transaction 6361 * abort. 6362 */ 6363 deleted_bgs = &trans->transaction->deleted_bgs; 6364 list_for_each_entry_safe(block_group, tmp, deleted_bgs, bg_list) { 6365 u64 trimmed = 0; 6366 6367 ret = -EROFS; 6368 if (!trans->aborted) 6369 ret = btrfs_discard_extent(root, 6370 block_group->key.objectid, 6371 block_group->key.offset, 6372 &trimmed); 6373 6374 list_del_init(&block_group->bg_list); 6375 btrfs_put_block_group_trimming(block_group); 6376 btrfs_put_block_group(block_group); 6377 6378 if (ret) { 6379 const char *errstr = btrfs_decode_error(ret); 6380 btrfs_warn(fs_info, 6381 "Discard failed while removing blockgroup: errno=%d %s\n", 6382 ret, errstr); 6383 } 6384 } 6385 6386 return 0; 6387 } 6388 6389 static void add_pinned_bytes(struct btrfs_fs_info *fs_info, u64 num_bytes, 6390 u64 owner, u64 root_objectid) 6391 { 6392 struct btrfs_space_info *space_info; 6393 u64 flags; 6394 6395 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 6396 if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID) 6397 flags = BTRFS_BLOCK_GROUP_SYSTEM; 6398 else 6399 flags = BTRFS_BLOCK_GROUP_METADATA; 6400 } else { 6401 flags = BTRFS_BLOCK_GROUP_DATA; 6402 } 6403 6404 space_info = __find_space_info(fs_info, flags); 6405 BUG_ON(!space_info); /* Logic bug */ 6406 percpu_counter_add(&space_info->total_bytes_pinned, num_bytes); 6407 } 6408 6409 6410 static int __btrfs_free_extent(struct btrfs_trans_handle *trans, 6411 struct btrfs_root *root, 6412 struct btrfs_delayed_ref_node *node, u64 parent, 6413 u64 root_objectid, u64 owner_objectid, 6414 u64 owner_offset, int refs_to_drop, 6415 struct btrfs_delayed_extent_op *extent_op) 6416 { 6417 struct btrfs_key key; 6418 struct btrfs_path *path; 6419 struct btrfs_fs_info *info = root->fs_info; 6420 struct btrfs_root *extent_root = info->extent_root; 6421 struct extent_buffer *leaf; 6422 struct btrfs_extent_item *ei; 6423 struct btrfs_extent_inline_ref *iref; 6424 int ret; 6425 int is_data; 6426 int extent_slot = 0; 6427 int found_extent = 0; 6428 int num_to_del = 1; 6429 u32 item_size; 6430 u64 refs; 6431 u64 bytenr = node->bytenr; 6432 u64 num_bytes = node->num_bytes; 6433 int last_ref = 0; 6434 bool skinny_metadata = btrfs_fs_incompat(root->fs_info, 6435 SKINNY_METADATA); 6436 6437 path = btrfs_alloc_path(); 6438 if (!path) 6439 return -ENOMEM; 6440 6441 path->reada = 1; 6442 path->leave_spinning = 1; 6443 6444 is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID; 6445 BUG_ON(!is_data && refs_to_drop != 1); 6446 6447 if (is_data) 6448 skinny_metadata = 0; 6449 6450 ret = lookup_extent_backref(trans, extent_root, path, &iref, 6451 bytenr, num_bytes, parent, 6452 root_objectid, owner_objectid, 6453 owner_offset); 6454 if (ret == 0) { 6455 extent_slot = path->slots[0]; 6456 while (extent_slot >= 0) { 6457 btrfs_item_key_to_cpu(path->nodes[0], &key, 6458 extent_slot); 6459 if (key.objectid != bytenr) 6460 break; 6461 if (key.type == BTRFS_EXTENT_ITEM_KEY && 6462 key.offset == num_bytes) { 6463 found_extent = 1; 6464 break; 6465 } 6466 if (key.type == BTRFS_METADATA_ITEM_KEY && 6467 key.offset == owner_objectid) { 6468 found_extent = 1; 6469 break; 6470 } 6471 if (path->slots[0] - extent_slot > 5) 6472 break; 6473 extent_slot--; 6474 } 6475 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 6476 item_size = btrfs_item_size_nr(path->nodes[0], extent_slot); 6477 if (found_extent && item_size < sizeof(*ei)) 6478 found_extent = 0; 6479 #endif 6480 if (!found_extent) { 6481 BUG_ON(iref); 6482 ret = remove_extent_backref(trans, extent_root, path, 6483 NULL, refs_to_drop, 6484 is_data, &last_ref); 6485 if (ret) { 6486 btrfs_abort_transaction(trans, extent_root, ret); 6487 goto out; 6488 } 6489 btrfs_release_path(path); 6490 path->leave_spinning = 1; 6491 6492 key.objectid = bytenr; 6493 key.type = BTRFS_EXTENT_ITEM_KEY; 6494 key.offset = num_bytes; 6495 6496 if (!is_data && skinny_metadata) { 6497 key.type = BTRFS_METADATA_ITEM_KEY; 6498 key.offset = owner_objectid; 6499 } 6500 6501 ret = btrfs_search_slot(trans, extent_root, 6502 &key, path, -1, 1); 6503 if (ret > 0 && skinny_metadata && path->slots[0]) { 6504 /* 6505 * Couldn't find our skinny metadata item, 6506 * see if we have ye olde extent item. 6507 */ 6508 path->slots[0]--; 6509 btrfs_item_key_to_cpu(path->nodes[0], &key, 6510 path->slots[0]); 6511 if (key.objectid == bytenr && 6512 key.type == BTRFS_EXTENT_ITEM_KEY && 6513 key.offset == num_bytes) 6514 ret = 0; 6515 } 6516 6517 if (ret > 0 && skinny_metadata) { 6518 skinny_metadata = false; 6519 key.objectid = bytenr; 6520 key.type = BTRFS_EXTENT_ITEM_KEY; 6521 key.offset = num_bytes; 6522 btrfs_release_path(path); 6523 ret = btrfs_search_slot(trans, extent_root, 6524 &key, path, -1, 1); 6525 } 6526 6527 if (ret) { 6528 btrfs_err(info, "umm, got %d back from search, was looking for %llu", 6529 ret, bytenr); 6530 if (ret > 0) 6531 btrfs_print_leaf(extent_root, 6532 path->nodes[0]); 6533 } 6534 if (ret < 0) { 6535 btrfs_abort_transaction(trans, extent_root, ret); 6536 goto out; 6537 } 6538 extent_slot = path->slots[0]; 6539 } 6540 } else if (WARN_ON(ret == -ENOENT)) { 6541 btrfs_print_leaf(extent_root, path->nodes[0]); 6542 btrfs_err(info, 6543 "unable to find ref byte nr %llu parent %llu root %llu owner %llu offset %llu", 6544 bytenr, parent, root_objectid, owner_objectid, 6545 owner_offset); 6546 btrfs_abort_transaction(trans, extent_root, ret); 6547 goto out; 6548 } else { 6549 btrfs_abort_transaction(trans, extent_root, ret); 6550 goto out; 6551 } 6552 6553 leaf = path->nodes[0]; 6554 item_size = btrfs_item_size_nr(leaf, extent_slot); 6555 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 6556 if (item_size < sizeof(*ei)) { 6557 BUG_ON(found_extent || extent_slot != path->slots[0]); 6558 ret = convert_extent_item_v0(trans, extent_root, path, 6559 owner_objectid, 0); 6560 if (ret < 0) { 6561 btrfs_abort_transaction(trans, extent_root, ret); 6562 goto out; 6563 } 6564 6565 btrfs_release_path(path); 6566 path->leave_spinning = 1; 6567 6568 key.objectid = bytenr; 6569 key.type = BTRFS_EXTENT_ITEM_KEY; 6570 key.offset = num_bytes; 6571 6572 ret = btrfs_search_slot(trans, extent_root, &key, path, 6573 -1, 1); 6574 if (ret) { 6575 btrfs_err(info, "umm, got %d back from search, was looking for %llu", 6576 ret, bytenr); 6577 btrfs_print_leaf(extent_root, path->nodes[0]); 6578 } 6579 if (ret < 0) { 6580 btrfs_abort_transaction(trans, extent_root, ret); 6581 goto out; 6582 } 6583 6584 extent_slot = path->slots[0]; 6585 leaf = path->nodes[0]; 6586 item_size = btrfs_item_size_nr(leaf, extent_slot); 6587 } 6588 #endif 6589 BUG_ON(item_size < sizeof(*ei)); 6590 ei = btrfs_item_ptr(leaf, extent_slot, 6591 struct btrfs_extent_item); 6592 if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID && 6593 key.type == BTRFS_EXTENT_ITEM_KEY) { 6594 struct btrfs_tree_block_info *bi; 6595 BUG_ON(item_size < sizeof(*ei) + sizeof(*bi)); 6596 bi = (struct btrfs_tree_block_info *)(ei + 1); 6597 WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi)); 6598 } 6599 6600 refs = btrfs_extent_refs(leaf, ei); 6601 if (refs < refs_to_drop) { 6602 btrfs_err(info, "trying to drop %d refs but we only have %Lu " 6603 "for bytenr %Lu", refs_to_drop, refs, bytenr); 6604 ret = -EINVAL; 6605 btrfs_abort_transaction(trans, extent_root, ret); 6606 goto out; 6607 } 6608 refs -= refs_to_drop; 6609 6610 if (refs > 0) { 6611 if (extent_op) 6612 __run_delayed_extent_op(extent_op, leaf, ei); 6613 /* 6614 * In the case of inline back ref, reference count will 6615 * be updated by remove_extent_backref 6616 */ 6617 if (iref) { 6618 BUG_ON(!found_extent); 6619 } else { 6620 btrfs_set_extent_refs(leaf, ei, refs); 6621 btrfs_mark_buffer_dirty(leaf); 6622 } 6623 if (found_extent) { 6624 ret = remove_extent_backref(trans, extent_root, path, 6625 iref, refs_to_drop, 6626 is_data, &last_ref); 6627 if (ret) { 6628 btrfs_abort_transaction(trans, extent_root, ret); 6629 goto out; 6630 } 6631 } 6632 add_pinned_bytes(root->fs_info, -num_bytes, owner_objectid, 6633 root_objectid); 6634 } else { 6635 if (found_extent) { 6636 BUG_ON(is_data && refs_to_drop != 6637 extent_data_ref_count(path, iref)); 6638 if (iref) { 6639 BUG_ON(path->slots[0] != extent_slot); 6640 } else { 6641 BUG_ON(path->slots[0] != extent_slot + 1); 6642 path->slots[0] = extent_slot; 6643 num_to_del = 2; 6644 } 6645 } 6646 6647 last_ref = 1; 6648 ret = btrfs_del_items(trans, extent_root, path, path->slots[0], 6649 num_to_del); 6650 if (ret) { 6651 btrfs_abort_transaction(trans, extent_root, ret); 6652 goto out; 6653 } 6654 btrfs_release_path(path); 6655 6656 if (is_data) { 6657 ret = btrfs_del_csums(trans, root, bytenr, num_bytes); 6658 if (ret) { 6659 btrfs_abort_transaction(trans, extent_root, ret); 6660 goto out; 6661 } 6662 } 6663 6664 ret = update_block_group(trans, root, bytenr, num_bytes, 0); 6665 if (ret) { 6666 btrfs_abort_transaction(trans, extent_root, ret); 6667 goto out; 6668 } 6669 } 6670 btrfs_release_path(path); 6671 6672 out: 6673 btrfs_free_path(path); 6674 return ret; 6675 } 6676 6677 /* 6678 * when we free an block, it is possible (and likely) that we free the last 6679 * delayed ref for that extent as well. This searches the delayed ref tree for 6680 * a given extent, and if there are no other delayed refs to be processed, it 6681 * removes it from the tree. 6682 */ 6683 static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans, 6684 struct btrfs_root *root, u64 bytenr) 6685 { 6686 struct btrfs_delayed_ref_head *head; 6687 struct btrfs_delayed_ref_root *delayed_refs; 6688 int ret = 0; 6689 6690 delayed_refs = &trans->transaction->delayed_refs; 6691 spin_lock(&delayed_refs->lock); 6692 head = btrfs_find_delayed_ref_head(trans, bytenr); 6693 if (!head) 6694 goto out_delayed_unlock; 6695 6696 spin_lock(&head->lock); 6697 if (!list_empty(&head->ref_list)) 6698 goto out; 6699 6700 if (head->extent_op) { 6701 if (!head->must_insert_reserved) 6702 goto out; 6703 btrfs_free_delayed_extent_op(head->extent_op); 6704 head->extent_op = NULL; 6705 } 6706 6707 /* 6708 * waiting for the lock here would deadlock. If someone else has it 6709 * locked they are already in the process of dropping it anyway 6710 */ 6711 if (!mutex_trylock(&head->mutex)) 6712 goto out; 6713 6714 /* 6715 * at this point we have a head with no other entries. Go 6716 * ahead and process it. 6717 */ 6718 head->node.in_tree = 0; 6719 rb_erase(&head->href_node, &delayed_refs->href_root); 6720 6721 atomic_dec(&delayed_refs->num_entries); 6722 6723 /* 6724 * we don't take a ref on the node because we're removing it from the 6725 * tree, so we just steal the ref the tree was holding. 6726 */ 6727 delayed_refs->num_heads--; 6728 if (head->processing == 0) 6729 delayed_refs->num_heads_ready--; 6730 head->processing = 0; 6731 spin_unlock(&head->lock); 6732 spin_unlock(&delayed_refs->lock); 6733 6734 BUG_ON(head->extent_op); 6735 if (head->must_insert_reserved) 6736 ret = 1; 6737 6738 mutex_unlock(&head->mutex); 6739 btrfs_put_delayed_ref(&head->node); 6740 return ret; 6741 out: 6742 spin_unlock(&head->lock); 6743 6744 out_delayed_unlock: 6745 spin_unlock(&delayed_refs->lock); 6746 return 0; 6747 } 6748 6749 void btrfs_free_tree_block(struct btrfs_trans_handle *trans, 6750 struct btrfs_root *root, 6751 struct extent_buffer *buf, 6752 u64 parent, int last_ref) 6753 { 6754 int pin = 1; 6755 int ret; 6756 6757 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 6758 ret = btrfs_add_delayed_tree_ref(root->fs_info, trans, 6759 buf->start, buf->len, 6760 parent, root->root_key.objectid, 6761 btrfs_header_level(buf), 6762 BTRFS_DROP_DELAYED_REF, NULL); 6763 BUG_ON(ret); /* -ENOMEM */ 6764 } 6765 6766 if (!last_ref) 6767 return; 6768 6769 if (btrfs_header_generation(buf) == trans->transid) { 6770 struct btrfs_block_group_cache *cache; 6771 6772 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 6773 ret = check_ref_cleanup(trans, root, buf->start); 6774 if (!ret) 6775 goto out; 6776 } 6777 6778 cache = btrfs_lookup_block_group(root->fs_info, buf->start); 6779 6780 if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) { 6781 pin_down_extent(root, cache, buf->start, buf->len, 1); 6782 btrfs_put_block_group(cache); 6783 goto out; 6784 } 6785 6786 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)); 6787 6788 btrfs_add_free_space(cache, buf->start, buf->len); 6789 btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE, 0); 6790 btrfs_put_block_group(cache); 6791 trace_btrfs_reserved_extent_free(root, buf->start, buf->len); 6792 pin = 0; 6793 } 6794 out: 6795 if (pin) 6796 add_pinned_bytes(root->fs_info, buf->len, 6797 btrfs_header_level(buf), 6798 root->root_key.objectid); 6799 6800 /* 6801 * Deleting the buffer, clear the corrupt flag since it doesn't matter 6802 * anymore. 6803 */ 6804 clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags); 6805 } 6806 6807 /* Can return -ENOMEM */ 6808 int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root, 6809 u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid, 6810 u64 owner, u64 offset) 6811 { 6812 int ret; 6813 struct btrfs_fs_info *fs_info = root->fs_info; 6814 6815 if (btrfs_test_is_dummy_root(root)) 6816 return 0; 6817 6818 add_pinned_bytes(root->fs_info, num_bytes, owner, root_objectid); 6819 6820 /* 6821 * tree log blocks never actually go into the extent allocation 6822 * tree, just update pinning info and exit early. 6823 */ 6824 if (root_objectid == BTRFS_TREE_LOG_OBJECTID) { 6825 WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID); 6826 /* unlocks the pinned mutex */ 6827 btrfs_pin_extent(root, bytenr, num_bytes, 1); 6828 ret = 0; 6829 } else if (owner < BTRFS_FIRST_FREE_OBJECTID) { 6830 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr, 6831 num_bytes, 6832 parent, root_objectid, (int)owner, 6833 BTRFS_DROP_DELAYED_REF, NULL); 6834 } else { 6835 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr, 6836 num_bytes, 6837 parent, root_objectid, owner, 6838 offset, 0, 6839 BTRFS_DROP_DELAYED_REF, NULL); 6840 } 6841 return ret; 6842 } 6843 6844 /* 6845 * when we wait for progress in the block group caching, its because 6846 * our allocation attempt failed at least once. So, we must sleep 6847 * and let some progress happen before we try again. 6848 * 6849 * This function will sleep at least once waiting for new free space to 6850 * show up, and then it will check the block group free space numbers 6851 * for our min num_bytes. Another option is to have it go ahead 6852 * and look in the rbtree for a free extent of a given size, but this 6853 * is a good start. 6854 * 6855 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using 6856 * any of the information in this block group. 6857 */ 6858 static noinline void 6859 wait_block_group_cache_progress(struct btrfs_block_group_cache *cache, 6860 u64 num_bytes) 6861 { 6862 struct btrfs_caching_control *caching_ctl; 6863 6864 caching_ctl = get_caching_control(cache); 6865 if (!caching_ctl) 6866 return; 6867 6868 wait_event(caching_ctl->wait, block_group_cache_done(cache) || 6869 (cache->free_space_ctl->free_space >= num_bytes)); 6870 6871 put_caching_control(caching_ctl); 6872 } 6873 6874 static noinline int 6875 wait_block_group_cache_done(struct btrfs_block_group_cache *cache) 6876 { 6877 struct btrfs_caching_control *caching_ctl; 6878 int ret = 0; 6879 6880 caching_ctl = get_caching_control(cache); 6881 if (!caching_ctl) 6882 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0; 6883 6884 wait_event(caching_ctl->wait, block_group_cache_done(cache)); 6885 if (cache->cached == BTRFS_CACHE_ERROR) 6886 ret = -EIO; 6887 put_caching_control(caching_ctl); 6888 return ret; 6889 } 6890 6891 int __get_raid_index(u64 flags) 6892 { 6893 if (flags & BTRFS_BLOCK_GROUP_RAID10) 6894 return BTRFS_RAID_RAID10; 6895 else if (flags & BTRFS_BLOCK_GROUP_RAID1) 6896 return BTRFS_RAID_RAID1; 6897 else if (flags & BTRFS_BLOCK_GROUP_DUP) 6898 return BTRFS_RAID_DUP; 6899 else if (flags & BTRFS_BLOCK_GROUP_RAID0) 6900 return BTRFS_RAID_RAID0; 6901 else if (flags & BTRFS_BLOCK_GROUP_RAID5) 6902 return BTRFS_RAID_RAID5; 6903 else if (flags & BTRFS_BLOCK_GROUP_RAID6) 6904 return BTRFS_RAID_RAID6; 6905 6906 return BTRFS_RAID_SINGLE; /* BTRFS_BLOCK_GROUP_SINGLE */ 6907 } 6908 6909 int get_block_group_index(struct btrfs_block_group_cache *cache) 6910 { 6911 return __get_raid_index(cache->flags); 6912 } 6913 6914 static const char *btrfs_raid_type_names[BTRFS_NR_RAID_TYPES] = { 6915 [BTRFS_RAID_RAID10] = "raid10", 6916 [BTRFS_RAID_RAID1] = "raid1", 6917 [BTRFS_RAID_DUP] = "dup", 6918 [BTRFS_RAID_RAID0] = "raid0", 6919 [BTRFS_RAID_SINGLE] = "single", 6920 [BTRFS_RAID_RAID5] = "raid5", 6921 [BTRFS_RAID_RAID6] = "raid6", 6922 }; 6923 6924 static const char *get_raid_name(enum btrfs_raid_types type) 6925 { 6926 if (type >= BTRFS_NR_RAID_TYPES) 6927 return NULL; 6928 6929 return btrfs_raid_type_names[type]; 6930 } 6931 6932 enum btrfs_loop_type { 6933 LOOP_CACHING_NOWAIT = 0, 6934 LOOP_CACHING_WAIT = 1, 6935 LOOP_ALLOC_CHUNK = 2, 6936 LOOP_NO_EMPTY_SIZE = 3, 6937 }; 6938 6939 static inline void 6940 btrfs_lock_block_group(struct btrfs_block_group_cache *cache, 6941 int delalloc) 6942 { 6943 if (delalloc) 6944 down_read(&cache->data_rwsem); 6945 } 6946 6947 static inline void 6948 btrfs_grab_block_group(struct btrfs_block_group_cache *cache, 6949 int delalloc) 6950 { 6951 btrfs_get_block_group(cache); 6952 if (delalloc) 6953 down_read(&cache->data_rwsem); 6954 } 6955 6956 static struct btrfs_block_group_cache * 6957 btrfs_lock_cluster(struct btrfs_block_group_cache *block_group, 6958 struct btrfs_free_cluster *cluster, 6959 int delalloc) 6960 { 6961 struct btrfs_block_group_cache *used_bg; 6962 bool locked = false; 6963 again: 6964 spin_lock(&cluster->refill_lock); 6965 if (locked) { 6966 if (used_bg == cluster->block_group) 6967 return used_bg; 6968 6969 up_read(&used_bg->data_rwsem); 6970 btrfs_put_block_group(used_bg); 6971 } 6972 6973 used_bg = cluster->block_group; 6974 if (!used_bg) 6975 return NULL; 6976 6977 if (used_bg == block_group) 6978 return used_bg; 6979 6980 btrfs_get_block_group(used_bg); 6981 6982 if (!delalloc) 6983 return used_bg; 6984 6985 if (down_read_trylock(&used_bg->data_rwsem)) 6986 return used_bg; 6987 6988 spin_unlock(&cluster->refill_lock); 6989 down_read(&used_bg->data_rwsem); 6990 locked = true; 6991 goto again; 6992 } 6993 6994 static inline void 6995 btrfs_release_block_group(struct btrfs_block_group_cache *cache, 6996 int delalloc) 6997 { 6998 if (delalloc) 6999 up_read(&cache->data_rwsem); 7000 btrfs_put_block_group(cache); 7001 } 7002 7003 /* 7004 * walks the btree of allocated extents and find a hole of a given size. 7005 * The key ins is changed to record the hole: 7006 * ins->objectid == start position 7007 * ins->flags = BTRFS_EXTENT_ITEM_KEY 7008 * ins->offset == the size of the hole. 7009 * Any available blocks before search_start are skipped. 7010 * 7011 * If there is no suitable free space, we will record the max size of 7012 * the free space extent currently. 7013 */ 7014 static noinline int find_free_extent(struct btrfs_root *orig_root, 7015 u64 num_bytes, u64 empty_size, 7016 u64 hint_byte, struct btrfs_key *ins, 7017 u64 flags, int delalloc) 7018 { 7019 int ret = 0; 7020 struct btrfs_root *root = orig_root->fs_info->extent_root; 7021 struct btrfs_free_cluster *last_ptr = NULL; 7022 struct btrfs_block_group_cache *block_group = NULL; 7023 u64 search_start = 0; 7024 u64 max_extent_size = 0; 7025 u64 empty_cluster = 0; 7026 struct btrfs_space_info *space_info; 7027 int loop = 0; 7028 int index = __get_raid_index(flags); 7029 int alloc_type = (flags & BTRFS_BLOCK_GROUP_DATA) ? 7030 RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC; 7031 bool failed_cluster_refill = false; 7032 bool failed_alloc = false; 7033 bool use_cluster = true; 7034 bool have_caching_bg = false; 7035 bool orig_have_caching_bg = false; 7036 bool full_search = false; 7037 7038 WARN_ON(num_bytes < root->sectorsize); 7039 ins->type = BTRFS_EXTENT_ITEM_KEY; 7040 ins->objectid = 0; 7041 ins->offset = 0; 7042 7043 trace_find_free_extent(orig_root, num_bytes, empty_size, flags); 7044 7045 space_info = __find_space_info(root->fs_info, flags); 7046 if (!space_info) { 7047 btrfs_err(root->fs_info, "No space info for %llu", flags); 7048 return -ENOSPC; 7049 } 7050 7051 /* 7052 * If our free space is heavily fragmented we may not be able to make 7053 * big contiguous allocations, so instead of doing the expensive search 7054 * for free space, simply return ENOSPC with our max_extent_size so we 7055 * can go ahead and search for a more manageable chunk. 7056 * 7057 * If our max_extent_size is large enough for our allocation simply 7058 * disable clustering since we will likely not be able to find enough 7059 * space to create a cluster and induce latency trying. 7060 */ 7061 if (unlikely(space_info->max_extent_size)) { 7062 spin_lock(&space_info->lock); 7063 if (space_info->max_extent_size && 7064 num_bytes > space_info->max_extent_size) { 7065 ins->offset = space_info->max_extent_size; 7066 spin_unlock(&space_info->lock); 7067 return -ENOSPC; 7068 } else if (space_info->max_extent_size) { 7069 use_cluster = false; 7070 } 7071 spin_unlock(&space_info->lock); 7072 } 7073 7074 last_ptr = fetch_cluster_info(orig_root, space_info, &empty_cluster); 7075 if (last_ptr) { 7076 spin_lock(&last_ptr->lock); 7077 if (last_ptr->block_group) 7078 hint_byte = last_ptr->window_start; 7079 if (last_ptr->fragmented) { 7080 /* 7081 * We still set window_start so we can keep track of the 7082 * last place we found an allocation to try and save 7083 * some time. 7084 */ 7085 hint_byte = last_ptr->window_start; 7086 use_cluster = false; 7087 } 7088 spin_unlock(&last_ptr->lock); 7089 } 7090 7091 search_start = max(search_start, first_logical_byte(root, 0)); 7092 search_start = max(search_start, hint_byte); 7093 if (search_start == hint_byte) { 7094 block_group = btrfs_lookup_block_group(root->fs_info, 7095 search_start); 7096 /* 7097 * we don't want to use the block group if it doesn't match our 7098 * allocation bits, or if its not cached. 7099 * 7100 * However if we are re-searching with an ideal block group 7101 * picked out then we don't care that the block group is cached. 7102 */ 7103 if (block_group && block_group_bits(block_group, flags) && 7104 block_group->cached != BTRFS_CACHE_NO) { 7105 down_read(&space_info->groups_sem); 7106 if (list_empty(&block_group->list) || 7107 block_group->ro) { 7108 /* 7109 * someone is removing this block group, 7110 * we can't jump into the have_block_group 7111 * target because our list pointers are not 7112 * valid 7113 */ 7114 btrfs_put_block_group(block_group); 7115 up_read(&space_info->groups_sem); 7116 } else { 7117 index = get_block_group_index(block_group); 7118 btrfs_lock_block_group(block_group, delalloc); 7119 goto have_block_group; 7120 } 7121 } else if (block_group) { 7122 btrfs_put_block_group(block_group); 7123 } 7124 } 7125 search: 7126 have_caching_bg = false; 7127 if (index == 0 || index == __get_raid_index(flags)) 7128 full_search = true; 7129 down_read(&space_info->groups_sem); 7130 list_for_each_entry(block_group, &space_info->block_groups[index], 7131 list) { 7132 u64 offset; 7133 int cached; 7134 7135 btrfs_grab_block_group(block_group, delalloc); 7136 search_start = block_group->key.objectid; 7137 7138 /* 7139 * this can happen if we end up cycling through all the 7140 * raid types, but we want to make sure we only allocate 7141 * for the proper type. 7142 */ 7143 if (!block_group_bits(block_group, flags)) { 7144 u64 extra = BTRFS_BLOCK_GROUP_DUP | 7145 BTRFS_BLOCK_GROUP_RAID1 | 7146 BTRFS_BLOCK_GROUP_RAID5 | 7147 BTRFS_BLOCK_GROUP_RAID6 | 7148 BTRFS_BLOCK_GROUP_RAID10; 7149 7150 /* 7151 * if they asked for extra copies and this block group 7152 * doesn't provide them, bail. This does allow us to 7153 * fill raid0 from raid1. 7154 */ 7155 if ((flags & extra) && !(block_group->flags & extra)) 7156 goto loop; 7157 } 7158 7159 have_block_group: 7160 cached = block_group_cache_done(block_group); 7161 if (unlikely(!cached)) { 7162 have_caching_bg = true; 7163 ret = cache_block_group(block_group, 0); 7164 BUG_ON(ret < 0); 7165 ret = 0; 7166 } 7167 7168 if (unlikely(block_group->cached == BTRFS_CACHE_ERROR)) 7169 goto loop; 7170 if (unlikely(block_group->ro)) 7171 goto loop; 7172 7173 /* 7174 * Ok we want to try and use the cluster allocator, so 7175 * lets look there 7176 */ 7177 if (last_ptr && use_cluster) { 7178 struct btrfs_block_group_cache *used_block_group; 7179 unsigned long aligned_cluster; 7180 /* 7181 * the refill lock keeps out other 7182 * people trying to start a new cluster 7183 */ 7184 used_block_group = btrfs_lock_cluster(block_group, 7185 last_ptr, 7186 delalloc); 7187 if (!used_block_group) 7188 goto refill_cluster; 7189 7190 if (used_block_group != block_group && 7191 (used_block_group->ro || 7192 !block_group_bits(used_block_group, flags))) 7193 goto release_cluster; 7194 7195 offset = btrfs_alloc_from_cluster(used_block_group, 7196 last_ptr, 7197 num_bytes, 7198 used_block_group->key.objectid, 7199 &max_extent_size); 7200 if (offset) { 7201 /* we have a block, we're done */ 7202 spin_unlock(&last_ptr->refill_lock); 7203 trace_btrfs_reserve_extent_cluster(root, 7204 used_block_group, 7205 search_start, num_bytes); 7206 if (used_block_group != block_group) { 7207 btrfs_release_block_group(block_group, 7208 delalloc); 7209 block_group = used_block_group; 7210 } 7211 goto checks; 7212 } 7213 7214 WARN_ON(last_ptr->block_group != used_block_group); 7215 release_cluster: 7216 /* If we are on LOOP_NO_EMPTY_SIZE, we can't 7217 * set up a new clusters, so lets just skip it 7218 * and let the allocator find whatever block 7219 * it can find. If we reach this point, we 7220 * will have tried the cluster allocator 7221 * plenty of times and not have found 7222 * anything, so we are likely way too 7223 * fragmented for the clustering stuff to find 7224 * anything. 7225 * 7226 * However, if the cluster is taken from the 7227 * current block group, release the cluster 7228 * first, so that we stand a better chance of 7229 * succeeding in the unclustered 7230 * allocation. */ 7231 if (loop >= LOOP_NO_EMPTY_SIZE && 7232 used_block_group != block_group) { 7233 spin_unlock(&last_ptr->refill_lock); 7234 btrfs_release_block_group(used_block_group, 7235 delalloc); 7236 goto unclustered_alloc; 7237 } 7238 7239 /* 7240 * this cluster didn't work out, free it and 7241 * start over 7242 */ 7243 btrfs_return_cluster_to_free_space(NULL, last_ptr); 7244 7245 if (used_block_group != block_group) 7246 btrfs_release_block_group(used_block_group, 7247 delalloc); 7248 refill_cluster: 7249 if (loop >= LOOP_NO_EMPTY_SIZE) { 7250 spin_unlock(&last_ptr->refill_lock); 7251 goto unclustered_alloc; 7252 } 7253 7254 aligned_cluster = max_t(unsigned long, 7255 empty_cluster + empty_size, 7256 block_group->full_stripe_len); 7257 7258 /* allocate a cluster in this block group */ 7259 ret = btrfs_find_space_cluster(root, block_group, 7260 last_ptr, search_start, 7261 num_bytes, 7262 aligned_cluster); 7263 if (ret == 0) { 7264 /* 7265 * now pull our allocation out of this 7266 * cluster 7267 */ 7268 offset = btrfs_alloc_from_cluster(block_group, 7269 last_ptr, 7270 num_bytes, 7271 search_start, 7272 &max_extent_size); 7273 if (offset) { 7274 /* we found one, proceed */ 7275 spin_unlock(&last_ptr->refill_lock); 7276 trace_btrfs_reserve_extent_cluster(root, 7277 block_group, search_start, 7278 num_bytes); 7279 goto checks; 7280 } 7281 } else if (!cached && loop > LOOP_CACHING_NOWAIT 7282 && !failed_cluster_refill) { 7283 spin_unlock(&last_ptr->refill_lock); 7284 7285 failed_cluster_refill = true; 7286 wait_block_group_cache_progress(block_group, 7287 num_bytes + empty_cluster + empty_size); 7288 goto have_block_group; 7289 } 7290 7291 /* 7292 * at this point we either didn't find a cluster 7293 * or we weren't able to allocate a block from our 7294 * cluster. Free the cluster we've been trying 7295 * to use, and go to the next block group 7296 */ 7297 btrfs_return_cluster_to_free_space(NULL, last_ptr); 7298 spin_unlock(&last_ptr->refill_lock); 7299 goto loop; 7300 } 7301 7302 unclustered_alloc: 7303 /* 7304 * We are doing an unclustered alloc, set the fragmented flag so 7305 * we don't bother trying to setup a cluster again until we get 7306 * more space. 7307 */ 7308 if (unlikely(last_ptr)) { 7309 spin_lock(&last_ptr->lock); 7310 last_ptr->fragmented = 1; 7311 spin_unlock(&last_ptr->lock); 7312 } 7313 spin_lock(&block_group->free_space_ctl->tree_lock); 7314 if (cached && 7315 block_group->free_space_ctl->free_space < 7316 num_bytes + empty_cluster + empty_size) { 7317 if (block_group->free_space_ctl->free_space > 7318 max_extent_size) 7319 max_extent_size = 7320 block_group->free_space_ctl->free_space; 7321 spin_unlock(&block_group->free_space_ctl->tree_lock); 7322 goto loop; 7323 } 7324 spin_unlock(&block_group->free_space_ctl->tree_lock); 7325 7326 offset = btrfs_find_space_for_alloc(block_group, search_start, 7327 num_bytes, empty_size, 7328 &max_extent_size); 7329 /* 7330 * If we didn't find a chunk, and we haven't failed on this 7331 * block group before, and this block group is in the middle of 7332 * caching and we are ok with waiting, then go ahead and wait 7333 * for progress to be made, and set failed_alloc to true. 7334 * 7335 * If failed_alloc is true then we've already waited on this 7336 * block group once and should move on to the next block group. 7337 */ 7338 if (!offset && !failed_alloc && !cached && 7339 loop > LOOP_CACHING_NOWAIT) { 7340 wait_block_group_cache_progress(block_group, 7341 num_bytes + empty_size); 7342 failed_alloc = true; 7343 goto have_block_group; 7344 } else if (!offset) { 7345 goto loop; 7346 } 7347 checks: 7348 search_start = ALIGN(offset, root->stripesize); 7349 7350 /* move on to the next group */ 7351 if (search_start + num_bytes > 7352 block_group->key.objectid + block_group->key.offset) { 7353 btrfs_add_free_space(block_group, offset, num_bytes); 7354 goto loop; 7355 } 7356 7357 if (offset < search_start) 7358 btrfs_add_free_space(block_group, offset, 7359 search_start - offset); 7360 BUG_ON(offset > search_start); 7361 7362 ret = btrfs_update_reserved_bytes(block_group, num_bytes, 7363 alloc_type, delalloc); 7364 if (ret == -EAGAIN) { 7365 btrfs_add_free_space(block_group, offset, num_bytes); 7366 goto loop; 7367 } 7368 7369 /* we are all good, lets return */ 7370 ins->objectid = search_start; 7371 ins->offset = num_bytes; 7372 7373 trace_btrfs_reserve_extent(orig_root, block_group, 7374 search_start, num_bytes); 7375 btrfs_release_block_group(block_group, delalloc); 7376 break; 7377 loop: 7378 failed_cluster_refill = false; 7379 failed_alloc = false; 7380 BUG_ON(index != get_block_group_index(block_group)); 7381 btrfs_release_block_group(block_group, delalloc); 7382 } 7383 up_read(&space_info->groups_sem); 7384 7385 if ((loop == LOOP_CACHING_NOWAIT) && have_caching_bg 7386 && !orig_have_caching_bg) 7387 orig_have_caching_bg = true; 7388 7389 if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg) 7390 goto search; 7391 7392 if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES) 7393 goto search; 7394 7395 /* 7396 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking 7397 * caching kthreads as we move along 7398 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching 7399 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again 7400 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try 7401 * again 7402 */ 7403 if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) { 7404 index = 0; 7405 if (loop == LOOP_CACHING_NOWAIT) { 7406 /* 7407 * We want to skip the LOOP_CACHING_WAIT step if we 7408 * don't have any unached bgs and we've alrelady done a 7409 * full search through. 7410 */ 7411 if (orig_have_caching_bg || !full_search) 7412 loop = LOOP_CACHING_WAIT; 7413 else 7414 loop = LOOP_ALLOC_CHUNK; 7415 } else { 7416 loop++; 7417 } 7418 7419 if (loop == LOOP_ALLOC_CHUNK) { 7420 struct btrfs_trans_handle *trans; 7421 int exist = 0; 7422 7423 trans = current->journal_info; 7424 if (trans) 7425 exist = 1; 7426 else 7427 trans = btrfs_join_transaction(root); 7428 7429 if (IS_ERR(trans)) { 7430 ret = PTR_ERR(trans); 7431 goto out; 7432 } 7433 7434 ret = do_chunk_alloc(trans, root, flags, 7435 CHUNK_ALLOC_FORCE); 7436 7437 /* 7438 * If we can't allocate a new chunk we've already looped 7439 * through at least once, move on to the NO_EMPTY_SIZE 7440 * case. 7441 */ 7442 if (ret == -ENOSPC) 7443 loop = LOOP_NO_EMPTY_SIZE; 7444 7445 /* 7446 * Do not bail out on ENOSPC since we 7447 * can do more things. 7448 */ 7449 if (ret < 0 && ret != -ENOSPC) 7450 btrfs_abort_transaction(trans, 7451 root, ret); 7452 else 7453 ret = 0; 7454 if (!exist) 7455 btrfs_end_transaction(trans, root); 7456 if (ret) 7457 goto out; 7458 } 7459 7460 if (loop == LOOP_NO_EMPTY_SIZE) { 7461 /* 7462 * Don't loop again if we already have no empty_size and 7463 * no empty_cluster. 7464 */ 7465 if (empty_size == 0 && 7466 empty_cluster == 0) { 7467 ret = -ENOSPC; 7468 goto out; 7469 } 7470 empty_size = 0; 7471 empty_cluster = 0; 7472 } 7473 7474 goto search; 7475 } else if (!ins->objectid) { 7476 ret = -ENOSPC; 7477 } else if (ins->objectid) { 7478 if (!use_cluster && last_ptr) { 7479 spin_lock(&last_ptr->lock); 7480 last_ptr->window_start = ins->objectid; 7481 spin_unlock(&last_ptr->lock); 7482 } 7483 ret = 0; 7484 } 7485 out: 7486 if (ret == -ENOSPC) { 7487 spin_lock(&space_info->lock); 7488 space_info->max_extent_size = max_extent_size; 7489 spin_unlock(&space_info->lock); 7490 ins->offset = max_extent_size; 7491 } 7492 return ret; 7493 } 7494 7495 static void dump_space_info(struct btrfs_space_info *info, u64 bytes, 7496 int dump_block_groups) 7497 { 7498 struct btrfs_block_group_cache *cache; 7499 int index = 0; 7500 7501 spin_lock(&info->lock); 7502 printk(KERN_INFO "BTRFS: space_info %llu has %llu free, is %sfull\n", 7503 info->flags, 7504 info->total_bytes - info->bytes_used - info->bytes_pinned - 7505 info->bytes_reserved - info->bytes_readonly, 7506 (info->full) ? "" : "not "); 7507 printk(KERN_INFO "BTRFS: space_info total=%llu, used=%llu, pinned=%llu, " 7508 "reserved=%llu, may_use=%llu, readonly=%llu\n", 7509 info->total_bytes, info->bytes_used, info->bytes_pinned, 7510 info->bytes_reserved, info->bytes_may_use, 7511 info->bytes_readonly); 7512 spin_unlock(&info->lock); 7513 7514 if (!dump_block_groups) 7515 return; 7516 7517 down_read(&info->groups_sem); 7518 again: 7519 list_for_each_entry(cache, &info->block_groups[index], list) { 7520 spin_lock(&cache->lock); 7521 printk(KERN_INFO "BTRFS: " 7522 "block group %llu has %llu bytes, " 7523 "%llu used %llu pinned %llu reserved %s\n", 7524 cache->key.objectid, cache->key.offset, 7525 btrfs_block_group_used(&cache->item), cache->pinned, 7526 cache->reserved, cache->ro ? "[readonly]" : ""); 7527 btrfs_dump_free_space(cache, bytes); 7528 spin_unlock(&cache->lock); 7529 } 7530 if (++index < BTRFS_NR_RAID_TYPES) 7531 goto again; 7532 up_read(&info->groups_sem); 7533 } 7534 7535 int btrfs_reserve_extent(struct btrfs_root *root, 7536 u64 num_bytes, u64 min_alloc_size, 7537 u64 empty_size, u64 hint_byte, 7538 struct btrfs_key *ins, int is_data, int delalloc) 7539 { 7540 bool final_tried = num_bytes == min_alloc_size; 7541 u64 flags; 7542 int ret; 7543 7544 flags = btrfs_get_alloc_profile(root, is_data); 7545 again: 7546 WARN_ON(num_bytes < root->sectorsize); 7547 ret = find_free_extent(root, num_bytes, empty_size, hint_byte, ins, 7548 flags, delalloc); 7549 7550 if (ret == -ENOSPC) { 7551 if (!final_tried && ins->offset) { 7552 num_bytes = min(num_bytes >> 1, ins->offset); 7553 num_bytes = round_down(num_bytes, root->sectorsize); 7554 num_bytes = max(num_bytes, min_alloc_size); 7555 if (num_bytes == min_alloc_size) 7556 final_tried = true; 7557 goto again; 7558 } else if (btrfs_test_opt(root, ENOSPC_DEBUG)) { 7559 struct btrfs_space_info *sinfo; 7560 7561 sinfo = __find_space_info(root->fs_info, flags); 7562 btrfs_err(root->fs_info, "allocation failed flags %llu, wanted %llu", 7563 flags, num_bytes); 7564 if (sinfo) 7565 dump_space_info(sinfo, num_bytes, 1); 7566 } 7567 } 7568 7569 return ret; 7570 } 7571 7572 static int __btrfs_free_reserved_extent(struct btrfs_root *root, 7573 u64 start, u64 len, 7574 int pin, int delalloc) 7575 { 7576 struct btrfs_block_group_cache *cache; 7577 int ret = 0; 7578 7579 cache = btrfs_lookup_block_group(root->fs_info, start); 7580 if (!cache) { 7581 btrfs_err(root->fs_info, "Unable to find block group for %llu", 7582 start); 7583 return -ENOSPC; 7584 } 7585 7586 if (pin) 7587 pin_down_extent(root, cache, start, len, 1); 7588 else { 7589 if (btrfs_test_opt(root, DISCARD)) 7590 ret = btrfs_discard_extent(root, start, len, NULL); 7591 btrfs_add_free_space(cache, start, len); 7592 btrfs_update_reserved_bytes(cache, len, RESERVE_FREE, delalloc); 7593 } 7594 7595 btrfs_put_block_group(cache); 7596 7597 trace_btrfs_reserved_extent_free(root, start, len); 7598 7599 return ret; 7600 } 7601 7602 int btrfs_free_reserved_extent(struct btrfs_root *root, 7603 u64 start, u64 len, int delalloc) 7604 { 7605 return __btrfs_free_reserved_extent(root, start, len, 0, delalloc); 7606 } 7607 7608 int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root, 7609 u64 start, u64 len) 7610 { 7611 return __btrfs_free_reserved_extent(root, start, len, 1, 0); 7612 } 7613 7614 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, 7615 struct btrfs_root *root, 7616 u64 parent, u64 root_objectid, 7617 u64 flags, u64 owner, u64 offset, 7618 struct btrfs_key *ins, int ref_mod) 7619 { 7620 int ret; 7621 struct btrfs_fs_info *fs_info = root->fs_info; 7622 struct btrfs_extent_item *extent_item; 7623 struct btrfs_extent_inline_ref *iref; 7624 struct btrfs_path *path; 7625 struct extent_buffer *leaf; 7626 int type; 7627 u32 size; 7628 7629 if (parent > 0) 7630 type = BTRFS_SHARED_DATA_REF_KEY; 7631 else 7632 type = BTRFS_EXTENT_DATA_REF_KEY; 7633 7634 size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type); 7635 7636 path = btrfs_alloc_path(); 7637 if (!path) 7638 return -ENOMEM; 7639 7640 path->leave_spinning = 1; 7641 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path, 7642 ins, size); 7643 if (ret) { 7644 btrfs_free_path(path); 7645 return ret; 7646 } 7647 7648 leaf = path->nodes[0]; 7649 extent_item = btrfs_item_ptr(leaf, path->slots[0], 7650 struct btrfs_extent_item); 7651 btrfs_set_extent_refs(leaf, extent_item, ref_mod); 7652 btrfs_set_extent_generation(leaf, extent_item, trans->transid); 7653 btrfs_set_extent_flags(leaf, extent_item, 7654 flags | BTRFS_EXTENT_FLAG_DATA); 7655 7656 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1); 7657 btrfs_set_extent_inline_ref_type(leaf, iref, type); 7658 if (parent > 0) { 7659 struct btrfs_shared_data_ref *ref; 7660 ref = (struct btrfs_shared_data_ref *)(iref + 1); 7661 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 7662 btrfs_set_shared_data_ref_count(leaf, ref, ref_mod); 7663 } else { 7664 struct btrfs_extent_data_ref *ref; 7665 ref = (struct btrfs_extent_data_ref *)(&iref->offset); 7666 btrfs_set_extent_data_ref_root(leaf, ref, root_objectid); 7667 btrfs_set_extent_data_ref_objectid(leaf, ref, owner); 7668 btrfs_set_extent_data_ref_offset(leaf, ref, offset); 7669 btrfs_set_extent_data_ref_count(leaf, ref, ref_mod); 7670 } 7671 7672 btrfs_mark_buffer_dirty(path->nodes[0]); 7673 btrfs_free_path(path); 7674 7675 ret = update_block_group(trans, root, ins->objectid, ins->offset, 1); 7676 if (ret) { /* -ENOENT, logic error */ 7677 btrfs_err(fs_info, "update block group failed for %llu %llu", 7678 ins->objectid, ins->offset); 7679 BUG(); 7680 } 7681 trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset); 7682 return ret; 7683 } 7684 7685 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, 7686 struct btrfs_root *root, 7687 u64 parent, u64 root_objectid, 7688 u64 flags, struct btrfs_disk_key *key, 7689 int level, struct btrfs_key *ins) 7690 { 7691 int ret; 7692 struct btrfs_fs_info *fs_info = root->fs_info; 7693 struct btrfs_extent_item *extent_item; 7694 struct btrfs_tree_block_info *block_info; 7695 struct btrfs_extent_inline_ref *iref; 7696 struct btrfs_path *path; 7697 struct extent_buffer *leaf; 7698 u32 size = sizeof(*extent_item) + sizeof(*iref); 7699 u64 num_bytes = ins->offset; 7700 bool skinny_metadata = btrfs_fs_incompat(root->fs_info, 7701 SKINNY_METADATA); 7702 7703 if (!skinny_metadata) 7704 size += sizeof(*block_info); 7705 7706 path = btrfs_alloc_path(); 7707 if (!path) { 7708 btrfs_free_and_pin_reserved_extent(root, ins->objectid, 7709 root->nodesize); 7710 return -ENOMEM; 7711 } 7712 7713 path->leave_spinning = 1; 7714 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path, 7715 ins, size); 7716 if (ret) { 7717 btrfs_free_path(path); 7718 btrfs_free_and_pin_reserved_extent(root, ins->objectid, 7719 root->nodesize); 7720 return ret; 7721 } 7722 7723 leaf = path->nodes[0]; 7724 extent_item = btrfs_item_ptr(leaf, path->slots[0], 7725 struct btrfs_extent_item); 7726 btrfs_set_extent_refs(leaf, extent_item, 1); 7727 btrfs_set_extent_generation(leaf, extent_item, trans->transid); 7728 btrfs_set_extent_flags(leaf, extent_item, 7729 flags | BTRFS_EXTENT_FLAG_TREE_BLOCK); 7730 7731 if (skinny_metadata) { 7732 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1); 7733 num_bytes = root->nodesize; 7734 } else { 7735 block_info = (struct btrfs_tree_block_info *)(extent_item + 1); 7736 btrfs_set_tree_block_key(leaf, block_info, key); 7737 btrfs_set_tree_block_level(leaf, block_info, level); 7738 iref = (struct btrfs_extent_inline_ref *)(block_info + 1); 7739 } 7740 7741 if (parent > 0) { 7742 BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 7743 btrfs_set_extent_inline_ref_type(leaf, iref, 7744 BTRFS_SHARED_BLOCK_REF_KEY); 7745 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 7746 } else { 7747 btrfs_set_extent_inline_ref_type(leaf, iref, 7748 BTRFS_TREE_BLOCK_REF_KEY); 7749 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid); 7750 } 7751 7752 btrfs_mark_buffer_dirty(leaf); 7753 btrfs_free_path(path); 7754 7755 ret = update_block_group(trans, root, ins->objectid, root->nodesize, 7756 1); 7757 if (ret) { /* -ENOENT, logic error */ 7758 btrfs_err(fs_info, "update block group failed for %llu %llu", 7759 ins->objectid, ins->offset); 7760 BUG(); 7761 } 7762 7763 trace_btrfs_reserved_extent_alloc(root, ins->objectid, root->nodesize); 7764 return ret; 7765 } 7766 7767 int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans, 7768 struct btrfs_root *root, 7769 u64 root_objectid, u64 owner, 7770 u64 offset, u64 ram_bytes, 7771 struct btrfs_key *ins) 7772 { 7773 int ret; 7774 7775 BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID); 7776 7777 ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid, 7778 ins->offset, 0, 7779 root_objectid, owner, offset, 7780 ram_bytes, BTRFS_ADD_DELAYED_EXTENT, 7781 NULL); 7782 return ret; 7783 } 7784 7785 /* 7786 * this is used by the tree logging recovery code. It records that 7787 * an extent has been allocated and makes sure to clear the free 7788 * space cache bits as well 7789 */ 7790 int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans, 7791 struct btrfs_root *root, 7792 u64 root_objectid, u64 owner, u64 offset, 7793 struct btrfs_key *ins) 7794 { 7795 int ret; 7796 struct btrfs_block_group_cache *block_group; 7797 7798 /* 7799 * Mixed block groups will exclude before processing the log so we only 7800 * need to do the exlude dance if this fs isn't mixed. 7801 */ 7802 if (!btrfs_fs_incompat(root->fs_info, MIXED_GROUPS)) { 7803 ret = __exclude_logged_extent(root, ins->objectid, ins->offset); 7804 if (ret) 7805 return ret; 7806 } 7807 7808 block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid); 7809 if (!block_group) 7810 return -EINVAL; 7811 7812 ret = btrfs_update_reserved_bytes(block_group, ins->offset, 7813 RESERVE_ALLOC_NO_ACCOUNT, 0); 7814 BUG_ON(ret); /* logic error */ 7815 ret = alloc_reserved_file_extent(trans, root, 0, root_objectid, 7816 0, owner, offset, ins, 1); 7817 btrfs_put_block_group(block_group); 7818 return ret; 7819 } 7820 7821 static struct extent_buffer * 7822 btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, 7823 u64 bytenr, int level) 7824 { 7825 struct extent_buffer *buf; 7826 7827 buf = btrfs_find_create_tree_block(root, bytenr); 7828 if (!buf) 7829 return ERR_PTR(-ENOMEM); 7830 btrfs_set_header_generation(buf, trans->transid); 7831 btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level); 7832 btrfs_tree_lock(buf); 7833 clean_tree_block(trans, root->fs_info, buf); 7834 clear_bit(EXTENT_BUFFER_STALE, &buf->bflags); 7835 7836 btrfs_set_lock_blocking(buf); 7837 btrfs_set_buffer_uptodate(buf); 7838 7839 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) { 7840 buf->log_index = root->log_transid % 2; 7841 /* 7842 * we allow two log transactions at a time, use different 7843 * EXENT bit to differentiate dirty pages. 7844 */ 7845 if (buf->log_index == 0) 7846 set_extent_dirty(&root->dirty_log_pages, buf->start, 7847 buf->start + buf->len - 1, GFP_NOFS); 7848 else 7849 set_extent_new(&root->dirty_log_pages, buf->start, 7850 buf->start + buf->len - 1, GFP_NOFS); 7851 } else { 7852 buf->log_index = -1; 7853 set_extent_dirty(&trans->transaction->dirty_pages, buf->start, 7854 buf->start + buf->len - 1, GFP_NOFS); 7855 } 7856 trans->blocks_used++; 7857 /* this returns a buffer locked for blocking */ 7858 return buf; 7859 } 7860 7861 static struct btrfs_block_rsv * 7862 use_block_rsv(struct btrfs_trans_handle *trans, 7863 struct btrfs_root *root, u32 blocksize) 7864 { 7865 struct btrfs_block_rsv *block_rsv; 7866 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv; 7867 int ret; 7868 bool global_updated = false; 7869 7870 block_rsv = get_block_rsv(trans, root); 7871 7872 if (unlikely(block_rsv->size == 0)) 7873 goto try_reserve; 7874 again: 7875 ret = block_rsv_use_bytes(block_rsv, blocksize); 7876 if (!ret) 7877 return block_rsv; 7878 7879 if (block_rsv->failfast) 7880 return ERR_PTR(ret); 7881 7882 if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) { 7883 global_updated = true; 7884 update_global_block_rsv(root->fs_info); 7885 goto again; 7886 } 7887 7888 if (btrfs_test_opt(root, ENOSPC_DEBUG)) { 7889 static DEFINE_RATELIMIT_STATE(_rs, 7890 DEFAULT_RATELIMIT_INTERVAL * 10, 7891 /*DEFAULT_RATELIMIT_BURST*/ 1); 7892 if (__ratelimit(&_rs)) 7893 WARN(1, KERN_DEBUG 7894 "BTRFS: block rsv returned %d\n", ret); 7895 } 7896 try_reserve: 7897 ret = reserve_metadata_bytes(root, block_rsv, blocksize, 7898 BTRFS_RESERVE_NO_FLUSH); 7899 if (!ret) 7900 return block_rsv; 7901 /* 7902 * If we couldn't reserve metadata bytes try and use some from 7903 * the global reserve if its space type is the same as the global 7904 * reservation. 7905 */ 7906 if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL && 7907 block_rsv->space_info == global_rsv->space_info) { 7908 ret = block_rsv_use_bytes(global_rsv, blocksize); 7909 if (!ret) 7910 return global_rsv; 7911 } 7912 return ERR_PTR(ret); 7913 } 7914 7915 static void unuse_block_rsv(struct btrfs_fs_info *fs_info, 7916 struct btrfs_block_rsv *block_rsv, u32 blocksize) 7917 { 7918 block_rsv_add_bytes(block_rsv, blocksize, 0); 7919 block_rsv_release_bytes(fs_info, block_rsv, NULL, 0); 7920 } 7921 7922 /* 7923 * finds a free extent and does all the dirty work required for allocation 7924 * returns the tree buffer or an ERR_PTR on error. 7925 */ 7926 struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans, 7927 struct btrfs_root *root, 7928 u64 parent, u64 root_objectid, 7929 struct btrfs_disk_key *key, int level, 7930 u64 hint, u64 empty_size) 7931 { 7932 struct btrfs_key ins; 7933 struct btrfs_block_rsv *block_rsv; 7934 struct extent_buffer *buf; 7935 struct btrfs_delayed_extent_op *extent_op; 7936 u64 flags = 0; 7937 int ret; 7938 u32 blocksize = root->nodesize; 7939 bool skinny_metadata = btrfs_fs_incompat(root->fs_info, 7940 SKINNY_METADATA); 7941 7942 if (btrfs_test_is_dummy_root(root)) { 7943 buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr, 7944 level); 7945 if (!IS_ERR(buf)) 7946 root->alloc_bytenr += blocksize; 7947 return buf; 7948 } 7949 7950 block_rsv = use_block_rsv(trans, root, blocksize); 7951 if (IS_ERR(block_rsv)) 7952 return ERR_CAST(block_rsv); 7953 7954 ret = btrfs_reserve_extent(root, blocksize, blocksize, 7955 empty_size, hint, &ins, 0, 0); 7956 if (ret) 7957 goto out_unuse; 7958 7959 buf = btrfs_init_new_buffer(trans, root, ins.objectid, level); 7960 if (IS_ERR(buf)) { 7961 ret = PTR_ERR(buf); 7962 goto out_free_reserved; 7963 } 7964 7965 if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) { 7966 if (parent == 0) 7967 parent = ins.objectid; 7968 flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 7969 } else 7970 BUG_ON(parent > 0); 7971 7972 if (root_objectid != BTRFS_TREE_LOG_OBJECTID) { 7973 extent_op = btrfs_alloc_delayed_extent_op(); 7974 if (!extent_op) { 7975 ret = -ENOMEM; 7976 goto out_free_buf; 7977 } 7978 if (key) 7979 memcpy(&extent_op->key, key, sizeof(extent_op->key)); 7980 else 7981 memset(&extent_op->key, 0, sizeof(extent_op->key)); 7982 extent_op->flags_to_set = flags; 7983 if (skinny_metadata) 7984 extent_op->update_key = 0; 7985 else 7986 extent_op->update_key = 1; 7987 extent_op->update_flags = 1; 7988 extent_op->is_data = 0; 7989 extent_op->level = level; 7990 7991 ret = btrfs_add_delayed_tree_ref(root->fs_info, trans, 7992 ins.objectid, ins.offset, 7993 parent, root_objectid, level, 7994 BTRFS_ADD_DELAYED_EXTENT, 7995 extent_op); 7996 if (ret) 7997 goto out_free_delayed; 7998 } 7999 return buf; 8000 8001 out_free_delayed: 8002 btrfs_free_delayed_extent_op(extent_op); 8003 out_free_buf: 8004 free_extent_buffer(buf); 8005 out_free_reserved: 8006 btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 0); 8007 out_unuse: 8008 unuse_block_rsv(root->fs_info, block_rsv, blocksize); 8009 return ERR_PTR(ret); 8010 } 8011 8012 struct walk_control { 8013 u64 refs[BTRFS_MAX_LEVEL]; 8014 u64 flags[BTRFS_MAX_LEVEL]; 8015 struct btrfs_key update_progress; 8016 int stage; 8017 int level; 8018 int shared_level; 8019 int update_ref; 8020 int keep_locks; 8021 int reada_slot; 8022 int reada_count; 8023 int for_reloc; 8024 }; 8025 8026 #define DROP_REFERENCE 1 8027 #define UPDATE_BACKREF 2 8028 8029 static noinline void reada_walk_down(struct btrfs_trans_handle *trans, 8030 struct btrfs_root *root, 8031 struct walk_control *wc, 8032 struct btrfs_path *path) 8033 { 8034 u64 bytenr; 8035 u64 generation; 8036 u64 refs; 8037 u64 flags; 8038 u32 nritems; 8039 u32 blocksize; 8040 struct btrfs_key key; 8041 struct extent_buffer *eb; 8042 int ret; 8043 int slot; 8044 int nread = 0; 8045 8046 if (path->slots[wc->level] < wc->reada_slot) { 8047 wc->reada_count = wc->reada_count * 2 / 3; 8048 wc->reada_count = max(wc->reada_count, 2); 8049 } else { 8050 wc->reada_count = wc->reada_count * 3 / 2; 8051 wc->reada_count = min_t(int, wc->reada_count, 8052 BTRFS_NODEPTRS_PER_BLOCK(root)); 8053 } 8054 8055 eb = path->nodes[wc->level]; 8056 nritems = btrfs_header_nritems(eb); 8057 blocksize = root->nodesize; 8058 8059 for (slot = path->slots[wc->level]; slot < nritems; slot++) { 8060 if (nread >= wc->reada_count) 8061 break; 8062 8063 cond_resched(); 8064 bytenr = btrfs_node_blockptr(eb, slot); 8065 generation = btrfs_node_ptr_generation(eb, slot); 8066 8067 if (slot == path->slots[wc->level]) 8068 goto reada; 8069 8070 if (wc->stage == UPDATE_BACKREF && 8071 generation <= root->root_key.offset) 8072 continue; 8073 8074 /* We don't lock the tree block, it's OK to be racy here */ 8075 ret = btrfs_lookup_extent_info(trans, root, bytenr, 8076 wc->level - 1, 1, &refs, 8077 &flags); 8078 /* We don't care about errors in readahead. */ 8079 if (ret < 0) 8080 continue; 8081 BUG_ON(refs == 0); 8082 8083 if (wc->stage == DROP_REFERENCE) { 8084 if (refs == 1) 8085 goto reada; 8086 8087 if (wc->level == 1 && 8088 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8089 continue; 8090 if (!wc->update_ref || 8091 generation <= root->root_key.offset) 8092 continue; 8093 btrfs_node_key_to_cpu(eb, &key, slot); 8094 ret = btrfs_comp_cpu_keys(&key, 8095 &wc->update_progress); 8096 if (ret < 0) 8097 continue; 8098 } else { 8099 if (wc->level == 1 && 8100 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8101 continue; 8102 } 8103 reada: 8104 readahead_tree_block(root, bytenr); 8105 nread++; 8106 } 8107 wc->reada_slot = slot; 8108 } 8109 8110 /* 8111 * These may not be seen by the usual inc/dec ref code so we have to 8112 * add them here. 8113 */ 8114 static int record_one_subtree_extent(struct btrfs_trans_handle *trans, 8115 struct btrfs_root *root, u64 bytenr, 8116 u64 num_bytes) 8117 { 8118 struct btrfs_qgroup_extent_record *qrecord; 8119 struct btrfs_delayed_ref_root *delayed_refs; 8120 8121 qrecord = kmalloc(sizeof(*qrecord), GFP_NOFS); 8122 if (!qrecord) 8123 return -ENOMEM; 8124 8125 qrecord->bytenr = bytenr; 8126 qrecord->num_bytes = num_bytes; 8127 qrecord->old_roots = NULL; 8128 8129 delayed_refs = &trans->transaction->delayed_refs; 8130 spin_lock(&delayed_refs->lock); 8131 if (btrfs_qgroup_insert_dirty_extent(delayed_refs, qrecord)) 8132 kfree(qrecord); 8133 spin_unlock(&delayed_refs->lock); 8134 8135 return 0; 8136 } 8137 8138 static int account_leaf_items(struct btrfs_trans_handle *trans, 8139 struct btrfs_root *root, 8140 struct extent_buffer *eb) 8141 { 8142 int nr = btrfs_header_nritems(eb); 8143 int i, extent_type, ret; 8144 struct btrfs_key key; 8145 struct btrfs_file_extent_item *fi; 8146 u64 bytenr, num_bytes; 8147 8148 /* We can be called directly from walk_up_proc() */ 8149 if (!root->fs_info->quota_enabled) 8150 return 0; 8151 8152 for (i = 0; i < nr; i++) { 8153 btrfs_item_key_to_cpu(eb, &key, i); 8154 8155 if (key.type != BTRFS_EXTENT_DATA_KEY) 8156 continue; 8157 8158 fi = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item); 8159 /* filter out non qgroup-accountable extents */ 8160 extent_type = btrfs_file_extent_type(eb, fi); 8161 8162 if (extent_type == BTRFS_FILE_EXTENT_INLINE) 8163 continue; 8164 8165 bytenr = btrfs_file_extent_disk_bytenr(eb, fi); 8166 if (!bytenr) 8167 continue; 8168 8169 num_bytes = btrfs_file_extent_disk_num_bytes(eb, fi); 8170 8171 ret = record_one_subtree_extent(trans, root, bytenr, num_bytes); 8172 if (ret) 8173 return ret; 8174 } 8175 return 0; 8176 } 8177 8178 /* 8179 * Walk up the tree from the bottom, freeing leaves and any interior 8180 * nodes which have had all slots visited. If a node (leaf or 8181 * interior) is freed, the node above it will have it's slot 8182 * incremented. The root node will never be freed. 8183 * 8184 * At the end of this function, we should have a path which has all 8185 * slots incremented to the next position for a search. If we need to 8186 * read a new node it will be NULL and the node above it will have the 8187 * correct slot selected for a later read. 8188 * 8189 * If we increment the root nodes slot counter past the number of 8190 * elements, 1 is returned to signal completion of the search. 8191 */ 8192 static int adjust_slots_upwards(struct btrfs_root *root, 8193 struct btrfs_path *path, int root_level) 8194 { 8195 int level = 0; 8196 int nr, slot; 8197 struct extent_buffer *eb; 8198 8199 if (root_level == 0) 8200 return 1; 8201 8202 while (level <= root_level) { 8203 eb = path->nodes[level]; 8204 nr = btrfs_header_nritems(eb); 8205 path->slots[level]++; 8206 slot = path->slots[level]; 8207 if (slot >= nr || level == 0) { 8208 /* 8209 * Don't free the root - we will detect this 8210 * condition after our loop and return a 8211 * positive value for caller to stop walking the tree. 8212 */ 8213 if (level != root_level) { 8214 btrfs_tree_unlock_rw(eb, path->locks[level]); 8215 path->locks[level] = 0; 8216 8217 free_extent_buffer(eb); 8218 path->nodes[level] = NULL; 8219 path->slots[level] = 0; 8220 } 8221 } else { 8222 /* 8223 * We have a valid slot to walk back down 8224 * from. Stop here so caller can process these 8225 * new nodes. 8226 */ 8227 break; 8228 } 8229 8230 level++; 8231 } 8232 8233 eb = path->nodes[root_level]; 8234 if (path->slots[root_level] >= btrfs_header_nritems(eb)) 8235 return 1; 8236 8237 return 0; 8238 } 8239 8240 /* 8241 * root_eb is the subtree root and is locked before this function is called. 8242 */ 8243 static int account_shared_subtree(struct btrfs_trans_handle *trans, 8244 struct btrfs_root *root, 8245 struct extent_buffer *root_eb, 8246 u64 root_gen, 8247 int root_level) 8248 { 8249 int ret = 0; 8250 int level; 8251 struct extent_buffer *eb = root_eb; 8252 struct btrfs_path *path = NULL; 8253 8254 BUG_ON(root_level < 0 || root_level > BTRFS_MAX_LEVEL); 8255 BUG_ON(root_eb == NULL); 8256 8257 if (!root->fs_info->quota_enabled) 8258 return 0; 8259 8260 if (!extent_buffer_uptodate(root_eb)) { 8261 ret = btrfs_read_buffer(root_eb, root_gen); 8262 if (ret) 8263 goto out; 8264 } 8265 8266 if (root_level == 0) { 8267 ret = account_leaf_items(trans, root, root_eb); 8268 goto out; 8269 } 8270 8271 path = btrfs_alloc_path(); 8272 if (!path) 8273 return -ENOMEM; 8274 8275 /* 8276 * Walk down the tree. Missing extent blocks are filled in as 8277 * we go. Metadata is accounted every time we read a new 8278 * extent block. 8279 * 8280 * When we reach a leaf, we account for file extent items in it, 8281 * walk back up the tree (adjusting slot pointers as we go) 8282 * and restart the search process. 8283 */ 8284 extent_buffer_get(root_eb); /* For path */ 8285 path->nodes[root_level] = root_eb; 8286 path->slots[root_level] = 0; 8287 path->locks[root_level] = 0; /* so release_path doesn't try to unlock */ 8288 walk_down: 8289 level = root_level; 8290 while (level >= 0) { 8291 if (path->nodes[level] == NULL) { 8292 int parent_slot; 8293 u64 child_gen; 8294 u64 child_bytenr; 8295 8296 /* We need to get child blockptr/gen from 8297 * parent before we can read it. */ 8298 eb = path->nodes[level + 1]; 8299 parent_slot = path->slots[level + 1]; 8300 child_bytenr = btrfs_node_blockptr(eb, parent_slot); 8301 child_gen = btrfs_node_ptr_generation(eb, parent_slot); 8302 8303 eb = read_tree_block(root, child_bytenr, child_gen); 8304 if (IS_ERR(eb)) { 8305 ret = PTR_ERR(eb); 8306 goto out; 8307 } else if (!extent_buffer_uptodate(eb)) { 8308 free_extent_buffer(eb); 8309 ret = -EIO; 8310 goto out; 8311 } 8312 8313 path->nodes[level] = eb; 8314 path->slots[level] = 0; 8315 8316 btrfs_tree_read_lock(eb); 8317 btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK); 8318 path->locks[level] = BTRFS_READ_LOCK_BLOCKING; 8319 8320 ret = record_one_subtree_extent(trans, root, child_bytenr, 8321 root->nodesize); 8322 if (ret) 8323 goto out; 8324 } 8325 8326 if (level == 0) { 8327 ret = account_leaf_items(trans, root, path->nodes[level]); 8328 if (ret) 8329 goto out; 8330 8331 /* Nonzero return here means we completed our search */ 8332 ret = adjust_slots_upwards(root, path, root_level); 8333 if (ret) 8334 break; 8335 8336 /* Restart search with new slots */ 8337 goto walk_down; 8338 } 8339 8340 level--; 8341 } 8342 8343 ret = 0; 8344 out: 8345 btrfs_free_path(path); 8346 8347 return ret; 8348 } 8349 8350 /* 8351 * helper to process tree block while walking down the tree. 8352 * 8353 * when wc->stage == UPDATE_BACKREF, this function updates 8354 * back refs for pointers in the block. 8355 * 8356 * NOTE: return value 1 means we should stop walking down. 8357 */ 8358 static noinline int walk_down_proc(struct btrfs_trans_handle *trans, 8359 struct btrfs_root *root, 8360 struct btrfs_path *path, 8361 struct walk_control *wc, int lookup_info) 8362 { 8363 int level = wc->level; 8364 struct extent_buffer *eb = path->nodes[level]; 8365 u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF; 8366 int ret; 8367 8368 if (wc->stage == UPDATE_BACKREF && 8369 btrfs_header_owner(eb) != root->root_key.objectid) 8370 return 1; 8371 8372 /* 8373 * when reference count of tree block is 1, it won't increase 8374 * again. once full backref flag is set, we never clear it. 8375 */ 8376 if (lookup_info && 8377 ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) || 8378 (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) { 8379 BUG_ON(!path->locks[level]); 8380 ret = btrfs_lookup_extent_info(trans, root, 8381 eb->start, level, 1, 8382 &wc->refs[level], 8383 &wc->flags[level]); 8384 BUG_ON(ret == -ENOMEM); 8385 if (ret) 8386 return ret; 8387 BUG_ON(wc->refs[level] == 0); 8388 } 8389 8390 if (wc->stage == DROP_REFERENCE) { 8391 if (wc->refs[level] > 1) 8392 return 1; 8393 8394 if (path->locks[level] && !wc->keep_locks) { 8395 btrfs_tree_unlock_rw(eb, path->locks[level]); 8396 path->locks[level] = 0; 8397 } 8398 return 0; 8399 } 8400 8401 /* wc->stage == UPDATE_BACKREF */ 8402 if (!(wc->flags[level] & flag)) { 8403 BUG_ON(!path->locks[level]); 8404 ret = btrfs_inc_ref(trans, root, eb, 1); 8405 BUG_ON(ret); /* -ENOMEM */ 8406 ret = btrfs_dec_ref(trans, root, eb, 0); 8407 BUG_ON(ret); /* -ENOMEM */ 8408 ret = btrfs_set_disk_extent_flags(trans, root, eb->start, 8409 eb->len, flag, 8410 btrfs_header_level(eb), 0); 8411 BUG_ON(ret); /* -ENOMEM */ 8412 wc->flags[level] |= flag; 8413 } 8414 8415 /* 8416 * the block is shared by multiple trees, so it's not good to 8417 * keep the tree lock 8418 */ 8419 if (path->locks[level] && level > 0) { 8420 btrfs_tree_unlock_rw(eb, path->locks[level]); 8421 path->locks[level] = 0; 8422 } 8423 return 0; 8424 } 8425 8426 /* 8427 * helper to process tree block pointer. 8428 * 8429 * when wc->stage == DROP_REFERENCE, this function checks 8430 * reference count of the block pointed to. if the block 8431 * is shared and we need update back refs for the subtree 8432 * rooted at the block, this function changes wc->stage to 8433 * UPDATE_BACKREF. if the block is shared and there is no 8434 * need to update back, this function drops the reference 8435 * to the block. 8436 * 8437 * NOTE: return value 1 means we should stop walking down. 8438 */ 8439 static noinline int do_walk_down(struct btrfs_trans_handle *trans, 8440 struct btrfs_root *root, 8441 struct btrfs_path *path, 8442 struct walk_control *wc, int *lookup_info) 8443 { 8444 u64 bytenr; 8445 u64 generation; 8446 u64 parent; 8447 u32 blocksize; 8448 struct btrfs_key key; 8449 struct extent_buffer *next; 8450 int level = wc->level; 8451 int reada = 0; 8452 int ret = 0; 8453 bool need_account = false; 8454 8455 generation = btrfs_node_ptr_generation(path->nodes[level], 8456 path->slots[level]); 8457 /* 8458 * if the lower level block was created before the snapshot 8459 * was created, we know there is no need to update back refs 8460 * for the subtree 8461 */ 8462 if (wc->stage == UPDATE_BACKREF && 8463 generation <= root->root_key.offset) { 8464 *lookup_info = 1; 8465 return 1; 8466 } 8467 8468 bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]); 8469 blocksize = root->nodesize; 8470 8471 next = btrfs_find_tree_block(root->fs_info, bytenr); 8472 if (!next) { 8473 next = btrfs_find_create_tree_block(root, bytenr); 8474 if (!next) 8475 return -ENOMEM; 8476 btrfs_set_buffer_lockdep_class(root->root_key.objectid, next, 8477 level - 1); 8478 reada = 1; 8479 } 8480 btrfs_tree_lock(next); 8481 btrfs_set_lock_blocking(next); 8482 8483 ret = btrfs_lookup_extent_info(trans, root, bytenr, level - 1, 1, 8484 &wc->refs[level - 1], 8485 &wc->flags[level - 1]); 8486 if (ret < 0) { 8487 btrfs_tree_unlock(next); 8488 return ret; 8489 } 8490 8491 if (unlikely(wc->refs[level - 1] == 0)) { 8492 btrfs_err(root->fs_info, "Missing references."); 8493 BUG(); 8494 } 8495 *lookup_info = 0; 8496 8497 if (wc->stage == DROP_REFERENCE) { 8498 if (wc->refs[level - 1] > 1) { 8499 need_account = true; 8500 if (level == 1 && 8501 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8502 goto skip; 8503 8504 if (!wc->update_ref || 8505 generation <= root->root_key.offset) 8506 goto skip; 8507 8508 btrfs_node_key_to_cpu(path->nodes[level], &key, 8509 path->slots[level]); 8510 ret = btrfs_comp_cpu_keys(&key, &wc->update_progress); 8511 if (ret < 0) 8512 goto skip; 8513 8514 wc->stage = UPDATE_BACKREF; 8515 wc->shared_level = level - 1; 8516 } 8517 } else { 8518 if (level == 1 && 8519 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8520 goto skip; 8521 } 8522 8523 if (!btrfs_buffer_uptodate(next, generation, 0)) { 8524 btrfs_tree_unlock(next); 8525 free_extent_buffer(next); 8526 next = NULL; 8527 *lookup_info = 1; 8528 } 8529 8530 if (!next) { 8531 if (reada && level == 1) 8532 reada_walk_down(trans, root, wc, path); 8533 next = read_tree_block(root, bytenr, generation); 8534 if (IS_ERR(next)) { 8535 return PTR_ERR(next); 8536 } else if (!extent_buffer_uptodate(next)) { 8537 free_extent_buffer(next); 8538 return -EIO; 8539 } 8540 btrfs_tree_lock(next); 8541 btrfs_set_lock_blocking(next); 8542 } 8543 8544 level--; 8545 BUG_ON(level != btrfs_header_level(next)); 8546 path->nodes[level] = next; 8547 path->slots[level] = 0; 8548 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8549 wc->level = level; 8550 if (wc->level == 1) 8551 wc->reada_slot = 0; 8552 return 0; 8553 skip: 8554 wc->refs[level - 1] = 0; 8555 wc->flags[level - 1] = 0; 8556 if (wc->stage == DROP_REFERENCE) { 8557 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 8558 parent = path->nodes[level]->start; 8559 } else { 8560 BUG_ON(root->root_key.objectid != 8561 btrfs_header_owner(path->nodes[level])); 8562 parent = 0; 8563 } 8564 8565 if (need_account) { 8566 ret = account_shared_subtree(trans, root, next, 8567 generation, level - 1); 8568 if (ret) { 8569 btrfs_err_rl(root->fs_info, 8570 "Error " 8571 "%d accounting shared subtree. Quota " 8572 "is out of sync, rescan required.", 8573 ret); 8574 } 8575 } 8576 ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent, 8577 root->root_key.objectid, level - 1, 0); 8578 BUG_ON(ret); /* -ENOMEM */ 8579 } 8580 btrfs_tree_unlock(next); 8581 free_extent_buffer(next); 8582 *lookup_info = 1; 8583 return 1; 8584 } 8585 8586 /* 8587 * helper to process tree block while walking up the tree. 8588 * 8589 * when wc->stage == DROP_REFERENCE, this function drops 8590 * reference count on the block. 8591 * 8592 * when wc->stage == UPDATE_BACKREF, this function changes 8593 * wc->stage back to DROP_REFERENCE if we changed wc->stage 8594 * to UPDATE_BACKREF previously while processing the block. 8595 * 8596 * NOTE: return value 1 means we should stop walking up. 8597 */ 8598 static noinline int walk_up_proc(struct btrfs_trans_handle *trans, 8599 struct btrfs_root *root, 8600 struct btrfs_path *path, 8601 struct walk_control *wc) 8602 { 8603 int ret; 8604 int level = wc->level; 8605 struct extent_buffer *eb = path->nodes[level]; 8606 u64 parent = 0; 8607 8608 if (wc->stage == UPDATE_BACKREF) { 8609 BUG_ON(wc->shared_level < level); 8610 if (level < wc->shared_level) 8611 goto out; 8612 8613 ret = find_next_key(path, level + 1, &wc->update_progress); 8614 if (ret > 0) 8615 wc->update_ref = 0; 8616 8617 wc->stage = DROP_REFERENCE; 8618 wc->shared_level = -1; 8619 path->slots[level] = 0; 8620 8621 /* 8622 * check reference count again if the block isn't locked. 8623 * we should start walking down the tree again if reference 8624 * count is one. 8625 */ 8626 if (!path->locks[level]) { 8627 BUG_ON(level == 0); 8628 btrfs_tree_lock(eb); 8629 btrfs_set_lock_blocking(eb); 8630 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8631 8632 ret = btrfs_lookup_extent_info(trans, root, 8633 eb->start, level, 1, 8634 &wc->refs[level], 8635 &wc->flags[level]); 8636 if (ret < 0) { 8637 btrfs_tree_unlock_rw(eb, path->locks[level]); 8638 path->locks[level] = 0; 8639 return ret; 8640 } 8641 BUG_ON(wc->refs[level] == 0); 8642 if (wc->refs[level] == 1) { 8643 btrfs_tree_unlock_rw(eb, path->locks[level]); 8644 path->locks[level] = 0; 8645 return 1; 8646 } 8647 } 8648 } 8649 8650 /* wc->stage == DROP_REFERENCE */ 8651 BUG_ON(wc->refs[level] > 1 && !path->locks[level]); 8652 8653 if (wc->refs[level] == 1) { 8654 if (level == 0) { 8655 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) 8656 ret = btrfs_dec_ref(trans, root, eb, 1); 8657 else 8658 ret = btrfs_dec_ref(trans, root, eb, 0); 8659 BUG_ON(ret); /* -ENOMEM */ 8660 ret = account_leaf_items(trans, root, eb); 8661 if (ret) { 8662 btrfs_err_rl(root->fs_info, 8663 "error " 8664 "%d accounting leaf items. Quota " 8665 "is out of sync, rescan required.", 8666 ret); 8667 } 8668 } 8669 /* make block locked assertion in clean_tree_block happy */ 8670 if (!path->locks[level] && 8671 btrfs_header_generation(eb) == trans->transid) { 8672 btrfs_tree_lock(eb); 8673 btrfs_set_lock_blocking(eb); 8674 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8675 } 8676 clean_tree_block(trans, root->fs_info, eb); 8677 } 8678 8679 if (eb == root->node) { 8680 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) 8681 parent = eb->start; 8682 else 8683 BUG_ON(root->root_key.objectid != 8684 btrfs_header_owner(eb)); 8685 } else { 8686 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF) 8687 parent = path->nodes[level + 1]->start; 8688 else 8689 BUG_ON(root->root_key.objectid != 8690 btrfs_header_owner(path->nodes[level + 1])); 8691 } 8692 8693 btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1); 8694 out: 8695 wc->refs[level] = 0; 8696 wc->flags[level] = 0; 8697 return 0; 8698 } 8699 8700 static noinline int walk_down_tree(struct btrfs_trans_handle *trans, 8701 struct btrfs_root *root, 8702 struct btrfs_path *path, 8703 struct walk_control *wc) 8704 { 8705 int level = wc->level; 8706 int lookup_info = 1; 8707 int ret; 8708 8709 while (level >= 0) { 8710 ret = walk_down_proc(trans, root, path, wc, lookup_info); 8711 if (ret > 0) 8712 break; 8713 8714 if (level == 0) 8715 break; 8716 8717 if (path->slots[level] >= 8718 btrfs_header_nritems(path->nodes[level])) 8719 break; 8720 8721 ret = do_walk_down(trans, root, path, wc, &lookup_info); 8722 if (ret > 0) { 8723 path->slots[level]++; 8724 continue; 8725 } else if (ret < 0) 8726 return ret; 8727 level = wc->level; 8728 } 8729 return 0; 8730 } 8731 8732 static noinline int walk_up_tree(struct btrfs_trans_handle *trans, 8733 struct btrfs_root *root, 8734 struct btrfs_path *path, 8735 struct walk_control *wc, int max_level) 8736 { 8737 int level = wc->level; 8738 int ret; 8739 8740 path->slots[level] = btrfs_header_nritems(path->nodes[level]); 8741 while (level < max_level && path->nodes[level]) { 8742 wc->level = level; 8743 if (path->slots[level] + 1 < 8744 btrfs_header_nritems(path->nodes[level])) { 8745 path->slots[level]++; 8746 return 0; 8747 } else { 8748 ret = walk_up_proc(trans, root, path, wc); 8749 if (ret > 0) 8750 return 0; 8751 8752 if (path->locks[level]) { 8753 btrfs_tree_unlock_rw(path->nodes[level], 8754 path->locks[level]); 8755 path->locks[level] = 0; 8756 } 8757 free_extent_buffer(path->nodes[level]); 8758 path->nodes[level] = NULL; 8759 level++; 8760 } 8761 } 8762 return 1; 8763 } 8764 8765 /* 8766 * drop a subvolume tree. 8767 * 8768 * this function traverses the tree freeing any blocks that only 8769 * referenced by the tree. 8770 * 8771 * when a shared tree block is found. this function decreases its 8772 * reference count by one. if update_ref is true, this function 8773 * also make sure backrefs for the shared block and all lower level 8774 * blocks are properly updated. 8775 * 8776 * If called with for_reloc == 0, may exit early with -EAGAIN 8777 */ 8778 int btrfs_drop_snapshot(struct btrfs_root *root, 8779 struct btrfs_block_rsv *block_rsv, int update_ref, 8780 int for_reloc) 8781 { 8782 struct btrfs_path *path; 8783 struct btrfs_trans_handle *trans; 8784 struct btrfs_root *tree_root = root->fs_info->tree_root; 8785 struct btrfs_root_item *root_item = &root->root_item; 8786 struct walk_control *wc; 8787 struct btrfs_key key; 8788 int err = 0; 8789 int ret; 8790 int level; 8791 bool root_dropped = false; 8792 8793 btrfs_debug(root->fs_info, "Drop subvolume %llu", root->objectid); 8794 8795 path = btrfs_alloc_path(); 8796 if (!path) { 8797 err = -ENOMEM; 8798 goto out; 8799 } 8800 8801 wc = kzalloc(sizeof(*wc), GFP_NOFS); 8802 if (!wc) { 8803 btrfs_free_path(path); 8804 err = -ENOMEM; 8805 goto out; 8806 } 8807 8808 trans = btrfs_start_transaction(tree_root, 0); 8809 if (IS_ERR(trans)) { 8810 err = PTR_ERR(trans); 8811 goto out_free; 8812 } 8813 8814 if (block_rsv) 8815 trans->block_rsv = block_rsv; 8816 8817 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) { 8818 level = btrfs_header_level(root->node); 8819 path->nodes[level] = btrfs_lock_root_node(root); 8820 btrfs_set_lock_blocking(path->nodes[level]); 8821 path->slots[level] = 0; 8822 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8823 memset(&wc->update_progress, 0, 8824 sizeof(wc->update_progress)); 8825 } else { 8826 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress); 8827 memcpy(&wc->update_progress, &key, 8828 sizeof(wc->update_progress)); 8829 8830 level = root_item->drop_level; 8831 BUG_ON(level == 0); 8832 path->lowest_level = level; 8833 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 8834 path->lowest_level = 0; 8835 if (ret < 0) { 8836 err = ret; 8837 goto out_end_trans; 8838 } 8839 WARN_ON(ret > 0); 8840 8841 /* 8842 * unlock our path, this is safe because only this 8843 * function is allowed to delete this snapshot 8844 */ 8845 btrfs_unlock_up_safe(path, 0); 8846 8847 level = btrfs_header_level(root->node); 8848 while (1) { 8849 btrfs_tree_lock(path->nodes[level]); 8850 btrfs_set_lock_blocking(path->nodes[level]); 8851 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8852 8853 ret = btrfs_lookup_extent_info(trans, root, 8854 path->nodes[level]->start, 8855 level, 1, &wc->refs[level], 8856 &wc->flags[level]); 8857 if (ret < 0) { 8858 err = ret; 8859 goto out_end_trans; 8860 } 8861 BUG_ON(wc->refs[level] == 0); 8862 8863 if (level == root_item->drop_level) 8864 break; 8865 8866 btrfs_tree_unlock(path->nodes[level]); 8867 path->locks[level] = 0; 8868 WARN_ON(wc->refs[level] != 1); 8869 level--; 8870 } 8871 } 8872 8873 wc->level = level; 8874 wc->shared_level = -1; 8875 wc->stage = DROP_REFERENCE; 8876 wc->update_ref = update_ref; 8877 wc->keep_locks = 0; 8878 wc->for_reloc = for_reloc; 8879 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root); 8880 8881 while (1) { 8882 8883 ret = walk_down_tree(trans, root, path, wc); 8884 if (ret < 0) { 8885 err = ret; 8886 break; 8887 } 8888 8889 ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL); 8890 if (ret < 0) { 8891 err = ret; 8892 break; 8893 } 8894 8895 if (ret > 0) { 8896 BUG_ON(wc->stage != DROP_REFERENCE); 8897 break; 8898 } 8899 8900 if (wc->stage == DROP_REFERENCE) { 8901 level = wc->level; 8902 btrfs_node_key(path->nodes[level], 8903 &root_item->drop_progress, 8904 path->slots[level]); 8905 root_item->drop_level = level; 8906 } 8907 8908 BUG_ON(wc->level == 0); 8909 if (btrfs_should_end_transaction(trans, tree_root) || 8910 (!for_reloc && btrfs_need_cleaner_sleep(root))) { 8911 ret = btrfs_update_root(trans, tree_root, 8912 &root->root_key, 8913 root_item); 8914 if (ret) { 8915 btrfs_abort_transaction(trans, tree_root, ret); 8916 err = ret; 8917 goto out_end_trans; 8918 } 8919 8920 btrfs_end_transaction_throttle(trans, tree_root); 8921 if (!for_reloc && btrfs_need_cleaner_sleep(root)) { 8922 pr_debug("BTRFS: drop snapshot early exit\n"); 8923 err = -EAGAIN; 8924 goto out_free; 8925 } 8926 8927 trans = btrfs_start_transaction(tree_root, 0); 8928 if (IS_ERR(trans)) { 8929 err = PTR_ERR(trans); 8930 goto out_free; 8931 } 8932 if (block_rsv) 8933 trans->block_rsv = block_rsv; 8934 } 8935 } 8936 btrfs_release_path(path); 8937 if (err) 8938 goto out_end_trans; 8939 8940 ret = btrfs_del_root(trans, tree_root, &root->root_key); 8941 if (ret) { 8942 btrfs_abort_transaction(trans, tree_root, ret); 8943 goto out_end_trans; 8944 } 8945 8946 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) { 8947 ret = btrfs_find_root(tree_root, &root->root_key, path, 8948 NULL, NULL); 8949 if (ret < 0) { 8950 btrfs_abort_transaction(trans, tree_root, ret); 8951 err = ret; 8952 goto out_end_trans; 8953 } else if (ret > 0) { 8954 /* if we fail to delete the orphan item this time 8955 * around, it'll get picked up the next time. 8956 * 8957 * The most common failure here is just -ENOENT. 8958 */ 8959 btrfs_del_orphan_item(trans, tree_root, 8960 root->root_key.objectid); 8961 } 8962 } 8963 8964 if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) { 8965 btrfs_add_dropped_root(trans, root); 8966 } else { 8967 free_extent_buffer(root->node); 8968 free_extent_buffer(root->commit_root); 8969 btrfs_put_fs_root(root); 8970 } 8971 root_dropped = true; 8972 out_end_trans: 8973 btrfs_end_transaction_throttle(trans, tree_root); 8974 out_free: 8975 kfree(wc); 8976 btrfs_free_path(path); 8977 out: 8978 /* 8979 * So if we need to stop dropping the snapshot for whatever reason we 8980 * need to make sure to add it back to the dead root list so that we 8981 * keep trying to do the work later. This also cleans up roots if we 8982 * don't have it in the radix (like when we recover after a power fail 8983 * or unmount) so we don't leak memory. 8984 */ 8985 if (!for_reloc && root_dropped == false) 8986 btrfs_add_dead_root(root); 8987 if (err && err != -EAGAIN) 8988 btrfs_std_error(root->fs_info, err, NULL); 8989 return err; 8990 } 8991 8992 /* 8993 * drop subtree rooted at tree block 'node'. 8994 * 8995 * NOTE: this function will unlock and release tree block 'node' 8996 * only used by relocation code 8997 */ 8998 int btrfs_drop_subtree(struct btrfs_trans_handle *trans, 8999 struct btrfs_root *root, 9000 struct extent_buffer *node, 9001 struct extent_buffer *parent) 9002 { 9003 struct btrfs_path *path; 9004 struct walk_control *wc; 9005 int level; 9006 int parent_level; 9007 int ret = 0; 9008 int wret; 9009 9010 BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID); 9011 9012 path = btrfs_alloc_path(); 9013 if (!path) 9014 return -ENOMEM; 9015 9016 wc = kzalloc(sizeof(*wc), GFP_NOFS); 9017 if (!wc) { 9018 btrfs_free_path(path); 9019 return -ENOMEM; 9020 } 9021 9022 btrfs_assert_tree_locked(parent); 9023 parent_level = btrfs_header_level(parent); 9024 extent_buffer_get(parent); 9025 path->nodes[parent_level] = parent; 9026 path->slots[parent_level] = btrfs_header_nritems(parent); 9027 9028 btrfs_assert_tree_locked(node); 9029 level = btrfs_header_level(node); 9030 path->nodes[level] = node; 9031 path->slots[level] = 0; 9032 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 9033 9034 wc->refs[parent_level] = 1; 9035 wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF; 9036 wc->level = level; 9037 wc->shared_level = -1; 9038 wc->stage = DROP_REFERENCE; 9039 wc->update_ref = 0; 9040 wc->keep_locks = 1; 9041 wc->for_reloc = 1; 9042 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root); 9043 9044 while (1) { 9045 wret = walk_down_tree(trans, root, path, wc); 9046 if (wret < 0) { 9047 ret = wret; 9048 break; 9049 } 9050 9051 wret = walk_up_tree(trans, root, path, wc, parent_level); 9052 if (wret < 0) 9053 ret = wret; 9054 if (wret != 0) 9055 break; 9056 } 9057 9058 kfree(wc); 9059 btrfs_free_path(path); 9060 return ret; 9061 } 9062 9063 static u64 update_block_group_flags(struct btrfs_root *root, u64 flags) 9064 { 9065 u64 num_devices; 9066 u64 stripped; 9067 9068 /* 9069 * if restripe for this chunk_type is on pick target profile and 9070 * return, otherwise do the usual balance 9071 */ 9072 stripped = get_restripe_target(root->fs_info, flags); 9073 if (stripped) 9074 return extended_to_chunk(stripped); 9075 9076 num_devices = root->fs_info->fs_devices->rw_devices; 9077 9078 stripped = BTRFS_BLOCK_GROUP_RAID0 | 9079 BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 | 9080 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10; 9081 9082 if (num_devices == 1) { 9083 stripped |= BTRFS_BLOCK_GROUP_DUP; 9084 stripped = flags & ~stripped; 9085 9086 /* turn raid0 into single device chunks */ 9087 if (flags & BTRFS_BLOCK_GROUP_RAID0) 9088 return stripped; 9089 9090 /* turn mirroring into duplication */ 9091 if (flags & (BTRFS_BLOCK_GROUP_RAID1 | 9092 BTRFS_BLOCK_GROUP_RAID10)) 9093 return stripped | BTRFS_BLOCK_GROUP_DUP; 9094 } else { 9095 /* they already had raid on here, just return */ 9096 if (flags & stripped) 9097 return flags; 9098 9099 stripped |= BTRFS_BLOCK_GROUP_DUP; 9100 stripped = flags & ~stripped; 9101 9102 /* switch duplicated blocks with raid1 */ 9103 if (flags & BTRFS_BLOCK_GROUP_DUP) 9104 return stripped | BTRFS_BLOCK_GROUP_RAID1; 9105 9106 /* this is drive concat, leave it alone */ 9107 } 9108 9109 return flags; 9110 } 9111 9112 static int inc_block_group_ro(struct btrfs_block_group_cache *cache, int force) 9113 { 9114 struct btrfs_space_info *sinfo = cache->space_info; 9115 u64 num_bytes; 9116 u64 min_allocable_bytes; 9117 int ret = -ENOSPC; 9118 9119 /* 9120 * We need some metadata space and system metadata space for 9121 * allocating chunks in some corner cases until we force to set 9122 * it to be readonly. 9123 */ 9124 if ((sinfo->flags & 9125 (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) && 9126 !force) 9127 min_allocable_bytes = 1 * 1024 * 1024; 9128 else 9129 min_allocable_bytes = 0; 9130 9131 spin_lock(&sinfo->lock); 9132 spin_lock(&cache->lock); 9133 9134 if (cache->ro) { 9135 cache->ro++; 9136 ret = 0; 9137 goto out; 9138 } 9139 9140 num_bytes = cache->key.offset - cache->reserved - cache->pinned - 9141 cache->bytes_super - btrfs_block_group_used(&cache->item); 9142 9143 if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned + 9144 sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes + 9145 min_allocable_bytes <= sinfo->total_bytes) { 9146 sinfo->bytes_readonly += num_bytes; 9147 cache->ro++; 9148 list_add_tail(&cache->ro_list, &sinfo->ro_bgs); 9149 ret = 0; 9150 } 9151 out: 9152 spin_unlock(&cache->lock); 9153 spin_unlock(&sinfo->lock); 9154 return ret; 9155 } 9156 9157 int btrfs_inc_block_group_ro(struct btrfs_root *root, 9158 struct btrfs_block_group_cache *cache) 9159 9160 { 9161 struct btrfs_trans_handle *trans; 9162 u64 alloc_flags; 9163 int ret; 9164 9165 again: 9166 trans = btrfs_join_transaction(root); 9167 if (IS_ERR(trans)) 9168 return PTR_ERR(trans); 9169 9170 /* 9171 * we're not allowed to set block groups readonly after the dirty 9172 * block groups cache has started writing. If it already started, 9173 * back off and let this transaction commit 9174 */ 9175 mutex_lock(&root->fs_info->ro_block_group_mutex); 9176 if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) { 9177 u64 transid = trans->transid; 9178 9179 mutex_unlock(&root->fs_info->ro_block_group_mutex); 9180 btrfs_end_transaction(trans, root); 9181 9182 ret = btrfs_wait_for_commit(root, transid); 9183 if (ret) 9184 return ret; 9185 goto again; 9186 } 9187 9188 /* 9189 * if we are changing raid levels, try to allocate a corresponding 9190 * block group with the new raid level. 9191 */ 9192 alloc_flags = update_block_group_flags(root, cache->flags); 9193 if (alloc_flags != cache->flags) { 9194 ret = do_chunk_alloc(trans, root, alloc_flags, 9195 CHUNK_ALLOC_FORCE); 9196 /* 9197 * ENOSPC is allowed here, we may have enough space 9198 * already allocated at the new raid level to 9199 * carry on 9200 */ 9201 if (ret == -ENOSPC) 9202 ret = 0; 9203 if (ret < 0) 9204 goto out; 9205 } 9206 9207 ret = inc_block_group_ro(cache, 0); 9208 if (!ret) 9209 goto out; 9210 alloc_flags = get_alloc_profile(root, cache->space_info->flags); 9211 ret = do_chunk_alloc(trans, root, alloc_flags, 9212 CHUNK_ALLOC_FORCE); 9213 if (ret < 0) 9214 goto out; 9215 ret = inc_block_group_ro(cache, 0); 9216 out: 9217 if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) { 9218 alloc_flags = update_block_group_flags(root, cache->flags); 9219 lock_chunks(root->fs_info->chunk_root); 9220 check_system_chunk(trans, root, alloc_flags); 9221 unlock_chunks(root->fs_info->chunk_root); 9222 } 9223 mutex_unlock(&root->fs_info->ro_block_group_mutex); 9224 9225 btrfs_end_transaction(trans, root); 9226 return ret; 9227 } 9228 9229 int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, 9230 struct btrfs_root *root, u64 type) 9231 { 9232 u64 alloc_flags = get_alloc_profile(root, type); 9233 return do_chunk_alloc(trans, root, alloc_flags, 9234 CHUNK_ALLOC_FORCE); 9235 } 9236 9237 /* 9238 * helper to account the unused space of all the readonly block group in the 9239 * space_info. takes mirrors into account. 9240 */ 9241 u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo) 9242 { 9243 struct btrfs_block_group_cache *block_group; 9244 u64 free_bytes = 0; 9245 int factor; 9246 9247 /* It's df, we don't care if it's racey */ 9248 if (list_empty(&sinfo->ro_bgs)) 9249 return 0; 9250 9251 spin_lock(&sinfo->lock); 9252 list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) { 9253 spin_lock(&block_group->lock); 9254 9255 if (!block_group->ro) { 9256 spin_unlock(&block_group->lock); 9257 continue; 9258 } 9259 9260 if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 | 9261 BTRFS_BLOCK_GROUP_RAID10 | 9262 BTRFS_BLOCK_GROUP_DUP)) 9263 factor = 2; 9264 else 9265 factor = 1; 9266 9267 free_bytes += (block_group->key.offset - 9268 btrfs_block_group_used(&block_group->item)) * 9269 factor; 9270 9271 spin_unlock(&block_group->lock); 9272 } 9273 spin_unlock(&sinfo->lock); 9274 9275 return free_bytes; 9276 } 9277 9278 void btrfs_dec_block_group_ro(struct btrfs_root *root, 9279 struct btrfs_block_group_cache *cache) 9280 { 9281 struct btrfs_space_info *sinfo = cache->space_info; 9282 u64 num_bytes; 9283 9284 BUG_ON(!cache->ro); 9285 9286 spin_lock(&sinfo->lock); 9287 spin_lock(&cache->lock); 9288 if (!--cache->ro) { 9289 num_bytes = cache->key.offset - cache->reserved - 9290 cache->pinned - cache->bytes_super - 9291 btrfs_block_group_used(&cache->item); 9292 sinfo->bytes_readonly -= num_bytes; 9293 list_del_init(&cache->ro_list); 9294 } 9295 spin_unlock(&cache->lock); 9296 spin_unlock(&sinfo->lock); 9297 } 9298 9299 /* 9300 * checks to see if its even possible to relocate this block group. 9301 * 9302 * @return - -1 if it's not a good idea to relocate this block group, 0 if its 9303 * ok to go ahead and try. 9304 */ 9305 int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr) 9306 { 9307 struct btrfs_block_group_cache *block_group; 9308 struct btrfs_space_info *space_info; 9309 struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices; 9310 struct btrfs_device *device; 9311 struct btrfs_trans_handle *trans; 9312 u64 min_free; 9313 u64 dev_min = 1; 9314 u64 dev_nr = 0; 9315 u64 target; 9316 int index; 9317 int full = 0; 9318 int ret = 0; 9319 9320 block_group = btrfs_lookup_block_group(root->fs_info, bytenr); 9321 9322 /* odd, couldn't find the block group, leave it alone */ 9323 if (!block_group) 9324 return -1; 9325 9326 min_free = btrfs_block_group_used(&block_group->item); 9327 9328 /* no bytes used, we're good */ 9329 if (!min_free) 9330 goto out; 9331 9332 space_info = block_group->space_info; 9333 spin_lock(&space_info->lock); 9334 9335 full = space_info->full; 9336 9337 /* 9338 * if this is the last block group we have in this space, we can't 9339 * relocate it unless we're able to allocate a new chunk below. 9340 * 9341 * Otherwise, we need to make sure we have room in the space to handle 9342 * all of the extents from this block group. If we can, we're good 9343 */ 9344 if ((space_info->total_bytes != block_group->key.offset) && 9345 (space_info->bytes_used + space_info->bytes_reserved + 9346 space_info->bytes_pinned + space_info->bytes_readonly + 9347 min_free < space_info->total_bytes)) { 9348 spin_unlock(&space_info->lock); 9349 goto out; 9350 } 9351 spin_unlock(&space_info->lock); 9352 9353 /* 9354 * ok we don't have enough space, but maybe we have free space on our 9355 * devices to allocate new chunks for relocation, so loop through our 9356 * alloc devices and guess if we have enough space. if this block 9357 * group is going to be restriped, run checks against the target 9358 * profile instead of the current one. 9359 */ 9360 ret = -1; 9361 9362 /* 9363 * index: 9364 * 0: raid10 9365 * 1: raid1 9366 * 2: dup 9367 * 3: raid0 9368 * 4: single 9369 */ 9370 target = get_restripe_target(root->fs_info, block_group->flags); 9371 if (target) { 9372 index = __get_raid_index(extended_to_chunk(target)); 9373 } else { 9374 /* 9375 * this is just a balance, so if we were marked as full 9376 * we know there is no space for a new chunk 9377 */ 9378 if (full) 9379 goto out; 9380 9381 index = get_block_group_index(block_group); 9382 } 9383 9384 if (index == BTRFS_RAID_RAID10) { 9385 dev_min = 4; 9386 /* Divide by 2 */ 9387 min_free >>= 1; 9388 } else if (index == BTRFS_RAID_RAID1) { 9389 dev_min = 2; 9390 } else if (index == BTRFS_RAID_DUP) { 9391 /* Multiply by 2 */ 9392 min_free <<= 1; 9393 } else if (index == BTRFS_RAID_RAID0) { 9394 dev_min = fs_devices->rw_devices; 9395 min_free = div64_u64(min_free, dev_min); 9396 } 9397 9398 /* We need to do this so that we can look at pending chunks */ 9399 trans = btrfs_join_transaction(root); 9400 if (IS_ERR(trans)) { 9401 ret = PTR_ERR(trans); 9402 goto out; 9403 } 9404 9405 mutex_lock(&root->fs_info->chunk_mutex); 9406 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { 9407 u64 dev_offset; 9408 9409 /* 9410 * check to make sure we can actually find a chunk with enough 9411 * space to fit our block group in. 9412 */ 9413 if (device->total_bytes > device->bytes_used + min_free && 9414 !device->is_tgtdev_for_dev_replace) { 9415 ret = find_free_dev_extent(trans, device, min_free, 9416 &dev_offset, NULL); 9417 if (!ret) 9418 dev_nr++; 9419 9420 if (dev_nr >= dev_min) 9421 break; 9422 9423 ret = -1; 9424 } 9425 } 9426 mutex_unlock(&root->fs_info->chunk_mutex); 9427 btrfs_end_transaction(trans, root); 9428 out: 9429 btrfs_put_block_group(block_group); 9430 return ret; 9431 } 9432 9433 static int find_first_block_group(struct btrfs_root *root, 9434 struct btrfs_path *path, struct btrfs_key *key) 9435 { 9436 int ret = 0; 9437 struct btrfs_key found_key; 9438 struct extent_buffer *leaf; 9439 int slot; 9440 9441 ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 9442 if (ret < 0) 9443 goto out; 9444 9445 while (1) { 9446 slot = path->slots[0]; 9447 leaf = path->nodes[0]; 9448 if (slot >= btrfs_header_nritems(leaf)) { 9449 ret = btrfs_next_leaf(root, path); 9450 if (ret == 0) 9451 continue; 9452 if (ret < 0) 9453 goto out; 9454 break; 9455 } 9456 btrfs_item_key_to_cpu(leaf, &found_key, slot); 9457 9458 if (found_key.objectid >= key->objectid && 9459 found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) { 9460 ret = 0; 9461 goto out; 9462 } 9463 path->slots[0]++; 9464 } 9465 out: 9466 return ret; 9467 } 9468 9469 void btrfs_put_block_group_cache(struct btrfs_fs_info *info) 9470 { 9471 struct btrfs_block_group_cache *block_group; 9472 u64 last = 0; 9473 9474 while (1) { 9475 struct inode *inode; 9476 9477 block_group = btrfs_lookup_first_block_group(info, last); 9478 while (block_group) { 9479 spin_lock(&block_group->lock); 9480 if (block_group->iref) 9481 break; 9482 spin_unlock(&block_group->lock); 9483 block_group = next_block_group(info->tree_root, 9484 block_group); 9485 } 9486 if (!block_group) { 9487 if (last == 0) 9488 break; 9489 last = 0; 9490 continue; 9491 } 9492 9493 inode = block_group->inode; 9494 block_group->iref = 0; 9495 block_group->inode = NULL; 9496 spin_unlock(&block_group->lock); 9497 iput(inode); 9498 last = block_group->key.objectid + block_group->key.offset; 9499 btrfs_put_block_group(block_group); 9500 } 9501 } 9502 9503 int btrfs_free_block_groups(struct btrfs_fs_info *info) 9504 { 9505 struct btrfs_block_group_cache *block_group; 9506 struct btrfs_space_info *space_info; 9507 struct btrfs_caching_control *caching_ctl; 9508 struct rb_node *n; 9509 9510 down_write(&info->commit_root_sem); 9511 while (!list_empty(&info->caching_block_groups)) { 9512 caching_ctl = list_entry(info->caching_block_groups.next, 9513 struct btrfs_caching_control, list); 9514 list_del(&caching_ctl->list); 9515 put_caching_control(caching_ctl); 9516 } 9517 up_write(&info->commit_root_sem); 9518 9519 spin_lock(&info->unused_bgs_lock); 9520 while (!list_empty(&info->unused_bgs)) { 9521 block_group = list_first_entry(&info->unused_bgs, 9522 struct btrfs_block_group_cache, 9523 bg_list); 9524 list_del_init(&block_group->bg_list); 9525 btrfs_put_block_group(block_group); 9526 } 9527 spin_unlock(&info->unused_bgs_lock); 9528 9529 spin_lock(&info->block_group_cache_lock); 9530 while ((n = rb_last(&info->block_group_cache_tree)) != NULL) { 9531 block_group = rb_entry(n, struct btrfs_block_group_cache, 9532 cache_node); 9533 rb_erase(&block_group->cache_node, 9534 &info->block_group_cache_tree); 9535 RB_CLEAR_NODE(&block_group->cache_node); 9536 spin_unlock(&info->block_group_cache_lock); 9537 9538 down_write(&block_group->space_info->groups_sem); 9539 list_del(&block_group->list); 9540 up_write(&block_group->space_info->groups_sem); 9541 9542 if (block_group->cached == BTRFS_CACHE_STARTED) 9543 wait_block_group_cache_done(block_group); 9544 9545 /* 9546 * We haven't cached this block group, which means we could 9547 * possibly have excluded extents on this block group. 9548 */ 9549 if (block_group->cached == BTRFS_CACHE_NO || 9550 block_group->cached == BTRFS_CACHE_ERROR) 9551 free_excluded_extents(info->extent_root, block_group); 9552 9553 btrfs_remove_free_space_cache(block_group); 9554 btrfs_put_block_group(block_group); 9555 9556 spin_lock(&info->block_group_cache_lock); 9557 } 9558 spin_unlock(&info->block_group_cache_lock); 9559 9560 /* now that all the block groups are freed, go through and 9561 * free all the space_info structs. This is only called during 9562 * the final stages of unmount, and so we know nobody is 9563 * using them. We call synchronize_rcu() once before we start, 9564 * just to be on the safe side. 9565 */ 9566 synchronize_rcu(); 9567 9568 release_global_block_rsv(info); 9569 9570 while (!list_empty(&info->space_info)) { 9571 int i; 9572 9573 space_info = list_entry(info->space_info.next, 9574 struct btrfs_space_info, 9575 list); 9576 if (btrfs_test_opt(info->tree_root, ENOSPC_DEBUG)) { 9577 if (WARN_ON(space_info->bytes_pinned > 0 || 9578 space_info->bytes_reserved > 0 || 9579 space_info->bytes_may_use > 0)) { 9580 dump_space_info(space_info, 0, 0); 9581 } 9582 } 9583 list_del(&space_info->list); 9584 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) { 9585 struct kobject *kobj; 9586 kobj = space_info->block_group_kobjs[i]; 9587 space_info->block_group_kobjs[i] = NULL; 9588 if (kobj) { 9589 kobject_del(kobj); 9590 kobject_put(kobj); 9591 } 9592 } 9593 kobject_del(&space_info->kobj); 9594 kobject_put(&space_info->kobj); 9595 } 9596 return 0; 9597 } 9598 9599 static void __link_block_group(struct btrfs_space_info *space_info, 9600 struct btrfs_block_group_cache *cache) 9601 { 9602 int index = get_block_group_index(cache); 9603 bool first = false; 9604 9605 down_write(&space_info->groups_sem); 9606 if (list_empty(&space_info->block_groups[index])) 9607 first = true; 9608 list_add_tail(&cache->list, &space_info->block_groups[index]); 9609 up_write(&space_info->groups_sem); 9610 9611 if (first) { 9612 struct raid_kobject *rkobj; 9613 int ret; 9614 9615 rkobj = kzalloc(sizeof(*rkobj), GFP_NOFS); 9616 if (!rkobj) 9617 goto out_err; 9618 rkobj->raid_type = index; 9619 kobject_init(&rkobj->kobj, &btrfs_raid_ktype); 9620 ret = kobject_add(&rkobj->kobj, &space_info->kobj, 9621 "%s", get_raid_name(index)); 9622 if (ret) { 9623 kobject_put(&rkobj->kobj); 9624 goto out_err; 9625 } 9626 space_info->block_group_kobjs[index] = &rkobj->kobj; 9627 } 9628 9629 return; 9630 out_err: 9631 pr_warn("BTRFS: failed to add kobject for block cache. ignoring.\n"); 9632 } 9633 9634 static struct btrfs_block_group_cache * 9635 btrfs_create_block_group_cache(struct btrfs_root *root, u64 start, u64 size) 9636 { 9637 struct btrfs_block_group_cache *cache; 9638 9639 cache = kzalloc(sizeof(*cache), GFP_NOFS); 9640 if (!cache) 9641 return NULL; 9642 9643 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl), 9644 GFP_NOFS); 9645 if (!cache->free_space_ctl) { 9646 kfree(cache); 9647 return NULL; 9648 } 9649 9650 cache->key.objectid = start; 9651 cache->key.offset = size; 9652 cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 9653 9654 cache->sectorsize = root->sectorsize; 9655 cache->fs_info = root->fs_info; 9656 cache->full_stripe_len = btrfs_full_stripe_len(root, 9657 &root->fs_info->mapping_tree, 9658 start); 9659 atomic_set(&cache->count, 1); 9660 spin_lock_init(&cache->lock); 9661 init_rwsem(&cache->data_rwsem); 9662 INIT_LIST_HEAD(&cache->list); 9663 INIT_LIST_HEAD(&cache->cluster_list); 9664 INIT_LIST_HEAD(&cache->bg_list); 9665 INIT_LIST_HEAD(&cache->ro_list); 9666 INIT_LIST_HEAD(&cache->dirty_list); 9667 INIT_LIST_HEAD(&cache->io_list); 9668 btrfs_init_free_space_ctl(cache); 9669 atomic_set(&cache->trimming, 0); 9670 9671 return cache; 9672 } 9673 9674 int btrfs_read_block_groups(struct btrfs_root *root) 9675 { 9676 struct btrfs_path *path; 9677 int ret; 9678 struct btrfs_block_group_cache *cache; 9679 struct btrfs_fs_info *info = root->fs_info; 9680 struct btrfs_space_info *space_info; 9681 struct btrfs_key key; 9682 struct btrfs_key found_key; 9683 struct extent_buffer *leaf; 9684 int need_clear = 0; 9685 u64 cache_gen; 9686 9687 root = info->extent_root; 9688 key.objectid = 0; 9689 key.offset = 0; 9690 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 9691 path = btrfs_alloc_path(); 9692 if (!path) 9693 return -ENOMEM; 9694 path->reada = 1; 9695 9696 cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy); 9697 if (btrfs_test_opt(root, SPACE_CACHE) && 9698 btrfs_super_generation(root->fs_info->super_copy) != cache_gen) 9699 need_clear = 1; 9700 if (btrfs_test_opt(root, CLEAR_CACHE)) 9701 need_clear = 1; 9702 9703 while (1) { 9704 ret = find_first_block_group(root, path, &key); 9705 if (ret > 0) 9706 break; 9707 if (ret != 0) 9708 goto error; 9709 9710 leaf = path->nodes[0]; 9711 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 9712 9713 cache = btrfs_create_block_group_cache(root, found_key.objectid, 9714 found_key.offset); 9715 if (!cache) { 9716 ret = -ENOMEM; 9717 goto error; 9718 } 9719 9720 if (need_clear) { 9721 /* 9722 * When we mount with old space cache, we need to 9723 * set BTRFS_DC_CLEAR and set dirty flag. 9724 * 9725 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we 9726 * truncate the old free space cache inode and 9727 * setup a new one. 9728 * b) Setting 'dirty flag' makes sure that we flush 9729 * the new space cache info onto disk. 9730 */ 9731 if (btrfs_test_opt(root, SPACE_CACHE)) 9732 cache->disk_cache_state = BTRFS_DC_CLEAR; 9733 } 9734 9735 read_extent_buffer(leaf, &cache->item, 9736 btrfs_item_ptr_offset(leaf, path->slots[0]), 9737 sizeof(cache->item)); 9738 cache->flags = btrfs_block_group_flags(&cache->item); 9739 9740 key.objectid = found_key.objectid + found_key.offset; 9741 btrfs_release_path(path); 9742 9743 /* 9744 * We need to exclude the super stripes now so that the space 9745 * info has super bytes accounted for, otherwise we'll think 9746 * we have more space than we actually do. 9747 */ 9748 ret = exclude_super_stripes(root, cache); 9749 if (ret) { 9750 /* 9751 * We may have excluded something, so call this just in 9752 * case. 9753 */ 9754 free_excluded_extents(root, cache); 9755 btrfs_put_block_group(cache); 9756 goto error; 9757 } 9758 9759 /* 9760 * check for two cases, either we are full, and therefore 9761 * don't need to bother with the caching work since we won't 9762 * find any space, or we are empty, and we can just add all 9763 * the space in and be done with it. This saves us _alot_ of 9764 * time, particularly in the full case. 9765 */ 9766 if (found_key.offset == btrfs_block_group_used(&cache->item)) { 9767 cache->last_byte_to_unpin = (u64)-1; 9768 cache->cached = BTRFS_CACHE_FINISHED; 9769 free_excluded_extents(root, cache); 9770 } else if (btrfs_block_group_used(&cache->item) == 0) { 9771 cache->last_byte_to_unpin = (u64)-1; 9772 cache->cached = BTRFS_CACHE_FINISHED; 9773 add_new_free_space(cache, root->fs_info, 9774 found_key.objectid, 9775 found_key.objectid + 9776 found_key.offset); 9777 free_excluded_extents(root, cache); 9778 } 9779 9780 ret = btrfs_add_block_group_cache(root->fs_info, cache); 9781 if (ret) { 9782 btrfs_remove_free_space_cache(cache); 9783 btrfs_put_block_group(cache); 9784 goto error; 9785 } 9786 9787 ret = update_space_info(info, cache->flags, found_key.offset, 9788 btrfs_block_group_used(&cache->item), 9789 &space_info); 9790 if (ret) { 9791 btrfs_remove_free_space_cache(cache); 9792 spin_lock(&info->block_group_cache_lock); 9793 rb_erase(&cache->cache_node, 9794 &info->block_group_cache_tree); 9795 RB_CLEAR_NODE(&cache->cache_node); 9796 spin_unlock(&info->block_group_cache_lock); 9797 btrfs_put_block_group(cache); 9798 goto error; 9799 } 9800 9801 cache->space_info = space_info; 9802 spin_lock(&cache->space_info->lock); 9803 cache->space_info->bytes_readonly += cache->bytes_super; 9804 spin_unlock(&cache->space_info->lock); 9805 9806 __link_block_group(space_info, cache); 9807 9808 set_avail_alloc_bits(root->fs_info, cache->flags); 9809 if (btrfs_chunk_readonly(root, cache->key.objectid)) { 9810 inc_block_group_ro(cache, 1); 9811 } else if (btrfs_block_group_used(&cache->item) == 0) { 9812 spin_lock(&info->unused_bgs_lock); 9813 /* Should always be true but just in case. */ 9814 if (list_empty(&cache->bg_list)) { 9815 btrfs_get_block_group(cache); 9816 list_add_tail(&cache->bg_list, 9817 &info->unused_bgs); 9818 } 9819 spin_unlock(&info->unused_bgs_lock); 9820 } 9821 } 9822 9823 list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) { 9824 if (!(get_alloc_profile(root, space_info->flags) & 9825 (BTRFS_BLOCK_GROUP_RAID10 | 9826 BTRFS_BLOCK_GROUP_RAID1 | 9827 BTRFS_BLOCK_GROUP_RAID5 | 9828 BTRFS_BLOCK_GROUP_RAID6 | 9829 BTRFS_BLOCK_GROUP_DUP))) 9830 continue; 9831 /* 9832 * avoid allocating from un-mirrored block group if there are 9833 * mirrored block groups. 9834 */ 9835 list_for_each_entry(cache, 9836 &space_info->block_groups[BTRFS_RAID_RAID0], 9837 list) 9838 inc_block_group_ro(cache, 1); 9839 list_for_each_entry(cache, 9840 &space_info->block_groups[BTRFS_RAID_SINGLE], 9841 list) 9842 inc_block_group_ro(cache, 1); 9843 } 9844 9845 init_global_block_rsv(info); 9846 ret = 0; 9847 error: 9848 btrfs_free_path(path); 9849 return ret; 9850 } 9851 9852 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans, 9853 struct btrfs_root *root) 9854 { 9855 struct btrfs_block_group_cache *block_group, *tmp; 9856 struct btrfs_root *extent_root = root->fs_info->extent_root; 9857 struct btrfs_block_group_item item; 9858 struct btrfs_key key; 9859 int ret = 0; 9860 bool can_flush_pending_bgs = trans->can_flush_pending_bgs; 9861 9862 trans->can_flush_pending_bgs = false; 9863 list_for_each_entry_safe(block_group, tmp, &trans->new_bgs, bg_list) { 9864 if (ret) 9865 goto next; 9866 9867 spin_lock(&block_group->lock); 9868 memcpy(&item, &block_group->item, sizeof(item)); 9869 memcpy(&key, &block_group->key, sizeof(key)); 9870 spin_unlock(&block_group->lock); 9871 9872 ret = btrfs_insert_item(trans, extent_root, &key, &item, 9873 sizeof(item)); 9874 if (ret) 9875 btrfs_abort_transaction(trans, extent_root, ret); 9876 ret = btrfs_finish_chunk_alloc(trans, extent_root, 9877 key.objectid, key.offset); 9878 if (ret) 9879 btrfs_abort_transaction(trans, extent_root, ret); 9880 next: 9881 list_del_init(&block_group->bg_list); 9882 } 9883 trans->can_flush_pending_bgs = can_flush_pending_bgs; 9884 } 9885 9886 int btrfs_make_block_group(struct btrfs_trans_handle *trans, 9887 struct btrfs_root *root, u64 bytes_used, 9888 u64 type, u64 chunk_objectid, u64 chunk_offset, 9889 u64 size) 9890 { 9891 int ret; 9892 struct btrfs_root *extent_root; 9893 struct btrfs_block_group_cache *cache; 9894 9895 extent_root = root->fs_info->extent_root; 9896 9897 btrfs_set_log_full_commit(root->fs_info, trans); 9898 9899 cache = btrfs_create_block_group_cache(root, chunk_offset, size); 9900 if (!cache) 9901 return -ENOMEM; 9902 9903 btrfs_set_block_group_used(&cache->item, bytes_used); 9904 btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid); 9905 btrfs_set_block_group_flags(&cache->item, type); 9906 9907 cache->flags = type; 9908 cache->last_byte_to_unpin = (u64)-1; 9909 cache->cached = BTRFS_CACHE_FINISHED; 9910 ret = exclude_super_stripes(root, cache); 9911 if (ret) { 9912 /* 9913 * We may have excluded something, so call this just in 9914 * case. 9915 */ 9916 free_excluded_extents(root, cache); 9917 btrfs_put_block_group(cache); 9918 return ret; 9919 } 9920 9921 add_new_free_space(cache, root->fs_info, chunk_offset, 9922 chunk_offset + size); 9923 9924 free_excluded_extents(root, cache); 9925 9926 #ifdef CONFIG_BTRFS_DEBUG 9927 if (btrfs_should_fragment_free_space(root, cache)) { 9928 u64 new_bytes_used = size - bytes_used; 9929 9930 bytes_used += new_bytes_used >> 1; 9931 fragment_free_space(root, cache); 9932 } 9933 #endif 9934 /* 9935 * Call to ensure the corresponding space_info object is created and 9936 * assigned to our block group, but don't update its counters just yet. 9937 * We want our bg to be added to the rbtree with its ->space_info set. 9938 */ 9939 ret = update_space_info(root->fs_info, cache->flags, 0, 0, 9940 &cache->space_info); 9941 if (ret) { 9942 btrfs_remove_free_space_cache(cache); 9943 btrfs_put_block_group(cache); 9944 return ret; 9945 } 9946 9947 ret = btrfs_add_block_group_cache(root->fs_info, cache); 9948 if (ret) { 9949 btrfs_remove_free_space_cache(cache); 9950 btrfs_put_block_group(cache); 9951 return ret; 9952 } 9953 9954 /* 9955 * Now that our block group has its ->space_info set and is inserted in 9956 * the rbtree, update the space info's counters. 9957 */ 9958 ret = update_space_info(root->fs_info, cache->flags, size, bytes_used, 9959 &cache->space_info); 9960 if (ret) { 9961 btrfs_remove_free_space_cache(cache); 9962 spin_lock(&root->fs_info->block_group_cache_lock); 9963 rb_erase(&cache->cache_node, 9964 &root->fs_info->block_group_cache_tree); 9965 RB_CLEAR_NODE(&cache->cache_node); 9966 spin_unlock(&root->fs_info->block_group_cache_lock); 9967 btrfs_put_block_group(cache); 9968 return ret; 9969 } 9970 update_global_block_rsv(root->fs_info); 9971 9972 spin_lock(&cache->space_info->lock); 9973 cache->space_info->bytes_readonly += cache->bytes_super; 9974 spin_unlock(&cache->space_info->lock); 9975 9976 __link_block_group(cache->space_info, cache); 9977 9978 list_add_tail(&cache->bg_list, &trans->new_bgs); 9979 9980 set_avail_alloc_bits(extent_root->fs_info, type); 9981 9982 return 0; 9983 } 9984 9985 static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) 9986 { 9987 u64 extra_flags = chunk_to_extended(flags) & 9988 BTRFS_EXTENDED_PROFILE_MASK; 9989 9990 write_seqlock(&fs_info->profiles_lock); 9991 if (flags & BTRFS_BLOCK_GROUP_DATA) 9992 fs_info->avail_data_alloc_bits &= ~extra_flags; 9993 if (flags & BTRFS_BLOCK_GROUP_METADATA) 9994 fs_info->avail_metadata_alloc_bits &= ~extra_flags; 9995 if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 9996 fs_info->avail_system_alloc_bits &= ~extra_flags; 9997 write_sequnlock(&fs_info->profiles_lock); 9998 } 9999 10000 int btrfs_remove_block_group(struct btrfs_trans_handle *trans, 10001 struct btrfs_root *root, u64 group_start, 10002 struct extent_map *em) 10003 { 10004 struct btrfs_path *path; 10005 struct btrfs_block_group_cache *block_group; 10006 struct btrfs_free_cluster *cluster; 10007 struct btrfs_root *tree_root = root->fs_info->tree_root; 10008 struct btrfs_key key; 10009 struct inode *inode; 10010 struct kobject *kobj = NULL; 10011 int ret; 10012 int index; 10013 int factor; 10014 struct btrfs_caching_control *caching_ctl = NULL; 10015 bool remove_em; 10016 10017 root = root->fs_info->extent_root; 10018 10019 block_group = btrfs_lookup_block_group(root->fs_info, group_start); 10020 BUG_ON(!block_group); 10021 BUG_ON(!block_group->ro); 10022 10023 /* 10024 * Free the reserved super bytes from this block group before 10025 * remove it. 10026 */ 10027 free_excluded_extents(root, block_group); 10028 10029 memcpy(&key, &block_group->key, sizeof(key)); 10030 index = get_block_group_index(block_group); 10031 if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP | 10032 BTRFS_BLOCK_GROUP_RAID1 | 10033 BTRFS_BLOCK_GROUP_RAID10)) 10034 factor = 2; 10035 else 10036 factor = 1; 10037 10038 /* make sure this block group isn't part of an allocation cluster */ 10039 cluster = &root->fs_info->data_alloc_cluster; 10040 spin_lock(&cluster->refill_lock); 10041 btrfs_return_cluster_to_free_space(block_group, cluster); 10042 spin_unlock(&cluster->refill_lock); 10043 10044 /* 10045 * make sure this block group isn't part of a metadata 10046 * allocation cluster 10047 */ 10048 cluster = &root->fs_info->meta_alloc_cluster; 10049 spin_lock(&cluster->refill_lock); 10050 btrfs_return_cluster_to_free_space(block_group, cluster); 10051 spin_unlock(&cluster->refill_lock); 10052 10053 path = btrfs_alloc_path(); 10054 if (!path) { 10055 ret = -ENOMEM; 10056 goto out; 10057 } 10058 10059 /* 10060 * get the inode first so any iput calls done for the io_list 10061 * aren't the final iput (no unlinks allowed now) 10062 */ 10063 inode = lookup_free_space_inode(tree_root, block_group, path); 10064 10065 mutex_lock(&trans->transaction->cache_write_mutex); 10066 /* 10067 * make sure our free spache cache IO is done before remove the 10068 * free space inode 10069 */ 10070 spin_lock(&trans->transaction->dirty_bgs_lock); 10071 if (!list_empty(&block_group->io_list)) { 10072 list_del_init(&block_group->io_list); 10073 10074 WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode); 10075 10076 spin_unlock(&trans->transaction->dirty_bgs_lock); 10077 btrfs_wait_cache_io(root, trans, block_group, 10078 &block_group->io_ctl, path, 10079 block_group->key.objectid); 10080 btrfs_put_block_group(block_group); 10081 spin_lock(&trans->transaction->dirty_bgs_lock); 10082 } 10083 10084 if (!list_empty(&block_group->dirty_list)) { 10085 list_del_init(&block_group->dirty_list); 10086 btrfs_put_block_group(block_group); 10087 } 10088 spin_unlock(&trans->transaction->dirty_bgs_lock); 10089 mutex_unlock(&trans->transaction->cache_write_mutex); 10090 10091 if (!IS_ERR(inode)) { 10092 ret = btrfs_orphan_add(trans, inode); 10093 if (ret) { 10094 btrfs_add_delayed_iput(inode); 10095 goto out; 10096 } 10097 clear_nlink(inode); 10098 /* One for the block groups ref */ 10099 spin_lock(&block_group->lock); 10100 if (block_group->iref) { 10101 block_group->iref = 0; 10102 block_group->inode = NULL; 10103 spin_unlock(&block_group->lock); 10104 iput(inode); 10105 } else { 10106 spin_unlock(&block_group->lock); 10107 } 10108 /* One for our lookup ref */ 10109 btrfs_add_delayed_iput(inode); 10110 } 10111 10112 key.objectid = BTRFS_FREE_SPACE_OBJECTID; 10113 key.offset = block_group->key.objectid; 10114 key.type = 0; 10115 10116 ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1); 10117 if (ret < 0) 10118 goto out; 10119 if (ret > 0) 10120 btrfs_release_path(path); 10121 if (ret == 0) { 10122 ret = btrfs_del_item(trans, tree_root, path); 10123 if (ret) 10124 goto out; 10125 btrfs_release_path(path); 10126 } 10127 10128 spin_lock(&root->fs_info->block_group_cache_lock); 10129 rb_erase(&block_group->cache_node, 10130 &root->fs_info->block_group_cache_tree); 10131 RB_CLEAR_NODE(&block_group->cache_node); 10132 10133 if (root->fs_info->first_logical_byte == block_group->key.objectid) 10134 root->fs_info->first_logical_byte = (u64)-1; 10135 spin_unlock(&root->fs_info->block_group_cache_lock); 10136 10137 down_write(&block_group->space_info->groups_sem); 10138 /* 10139 * we must use list_del_init so people can check to see if they 10140 * are still on the list after taking the semaphore 10141 */ 10142 list_del_init(&block_group->list); 10143 if (list_empty(&block_group->space_info->block_groups[index])) { 10144 kobj = block_group->space_info->block_group_kobjs[index]; 10145 block_group->space_info->block_group_kobjs[index] = NULL; 10146 clear_avail_alloc_bits(root->fs_info, block_group->flags); 10147 } 10148 up_write(&block_group->space_info->groups_sem); 10149 if (kobj) { 10150 kobject_del(kobj); 10151 kobject_put(kobj); 10152 } 10153 10154 if (block_group->has_caching_ctl) 10155 caching_ctl = get_caching_control(block_group); 10156 if (block_group->cached == BTRFS_CACHE_STARTED) 10157 wait_block_group_cache_done(block_group); 10158 if (block_group->has_caching_ctl) { 10159 down_write(&root->fs_info->commit_root_sem); 10160 if (!caching_ctl) { 10161 struct btrfs_caching_control *ctl; 10162 10163 list_for_each_entry(ctl, 10164 &root->fs_info->caching_block_groups, list) 10165 if (ctl->block_group == block_group) { 10166 caching_ctl = ctl; 10167 atomic_inc(&caching_ctl->count); 10168 break; 10169 } 10170 } 10171 if (caching_ctl) 10172 list_del_init(&caching_ctl->list); 10173 up_write(&root->fs_info->commit_root_sem); 10174 if (caching_ctl) { 10175 /* Once for the caching bgs list and once for us. */ 10176 put_caching_control(caching_ctl); 10177 put_caching_control(caching_ctl); 10178 } 10179 } 10180 10181 spin_lock(&trans->transaction->dirty_bgs_lock); 10182 if (!list_empty(&block_group->dirty_list)) { 10183 WARN_ON(1); 10184 } 10185 if (!list_empty(&block_group->io_list)) { 10186 WARN_ON(1); 10187 } 10188 spin_unlock(&trans->transaction->dirty_bgs_lock); 10189 btrfs_remove_free_space_cache(block_group); 10190 10191 spin_lock(&block_group->space_info->lock); 10192 list_del_init(&block_group->ro_list); 10193 10194 if (btrfs_test_opt(root, ENOSPC_DEBUG)) { 10195 WARN_ON(block_group->space_info->total_bytes 10196 < block_group->key.offset); 10197 WARN_ON(block_group->space_info->bytes_readonly 10198 < block_group->key.offset); 10199 WARN_ON(block_group->space_info->disk_total 10200 < block_group->key.offset * factor); 10201 } 10202 block_group->space_info->total_bytes -= block_group->key.offset; 10203 block_group->space_info->bytes_readonly -= block_group->key.offset; 10204 block_group->space_info->disk_total -= block_group->key.offset * factor; 10205 10206 spin_unlock(&block_group->space_info->lock); 10207 10208 memcpy(&key, &block_group->key, sizeof(key)); 10209 10210 lock_chunks(root); 10211 if (!list_empty(&em->list)) { 10212 /* We're in the transaction->pending_chunks list. */ 10213 free_extent_map(em); 10214 } 10215 spin_lock(&block_group->lock); 10216 block_group->removed = 1; 10217 /* 10218 * At this point trimming can't start on this block group, because we 10219 * removed the block group from the tree fs_info->block_group_cache_tree 10220 * so no one can't find it anymore and even if someone already got this 10221 * block group before we removed it from the rbtree, they have already 10222 * incremented block_group->trimming - if they didn't, they won't find 10223 * any free space entries because we already removed them all when we 10224 * called btrfs_remove_free_space_cache(). 10225 * 10226 * And we must not remove the extent map from the fs_info->mapping_tree 10227 * to prevent the same logical address range and physical device space 10228 * ranges from being reused for a new block group. This is because our 10229 * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is 10230 * completely transactionless, so while it is trimming a range the 10231 * currently running transaction might finish and a new one start, 10232 * allowing for new block groups to be created that can reuse the same 10233 * physical device locations unless we take this special care. 10234 * 10235 * There may also be an implicit trim operation if the file system 10236 * is mounted with -odiscard. The same protections must remain 10237 * in place until the extents have been discarded completely when 10238 * the transaction commit has completed. 10239 */ 10240 remove_em = (atomic_read(&block_group->trimming) == 0); 10241 /* 10242 * Make sure a trimmer task always sees the em in the pinned_chunks list 10243 * if it sees block_group->removed == 1 (needs to lock block_group->lock 10244 * before checking block_group->removed). 10245 */ 10246 if (!remove_em) { 10247 /* 10248 * Our em might be in trans->transaction->pending_chunks which 10249 * is protected by fs_info->chunk_mutex ([lock|unlock]_chunks), 10250 * and so is the fs_info->pinned_chunks list. 10251 * 10252 * So at this point we must be holding the chunk_mutex to avoid 10253 * any races with chunk allocation (more specifically at 10254 * volumes.c:contains_pending_extent()), to ensure it always 10255 * sees the em, either in the pending_chunks list or in the 10256 * pinned_chunks list. 10257 */ 10258 list_move_tail(&em->list, &root->fs_info->pinned_chunks); 10259 } 10260 spin_unlock(&block_group->lock); 10261 10262 if (remove_em) { 10263 struct extent_map_tree *em_tree; 10264 10265 em_tree = &root->fs_info->mapping_tree.map_tree; 10266 write_lock(&em_tree->lock); 10267 /* 10268 * The em might be in the pending_chunks list, so make sure the 10269 * chunk mutex is locked, since remove_extent_mapping() will 10270 * delete us from that list. 10271 */ 10272 remove_extent_mapping(em_tree, em); 10273 write_unlock(&em_tree->lock); 10274 /* once for the tree */ 10275 free_extent_map(em); 10276 } 10277 10278 unlock_chunks(root); 10279 10280 btrfs_put_block_group(block_group); 10281 btrfs_put_block_group(block_group); 10282 10283 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 10284 if (ret > 0) 10285 ret = -EIO; 10286 if (ret < 0) 10287 goto out; 10288 10289 ret = btrfs_del_item(trans, root, path); 10290 out: 10291 btrfs_free_path(path); 10292 return ret; 10293 } 10294 10295 struct btrfs_trans_handle * 10296 btrfs_start_trans_remove_block_group(struct btrfs_fs_info *fs_info, 10297 const u64 chunk_offset) 10298 { 10299 struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree; 10300 struct extent_map *em; 10301 struct map_lookup *map; 10302 unsigned int num_items; 10303 10304 read_lock(&em_tree->lock); 10305 em = lookup_extent_mapping(em_tree, chunk_offset, 1); 10306 read_unlock(&em_tree->lock); 10307 ASSERT(em && em->start == chunk_offset); 10308 10309 /* 10310 * We need to reserve 3 + N units from the metadata space info in order 10311 * to remove a block group (done at btrfs_remove_chunk() and at 10312 * btrfs_remove_block_group()), which are used for: 10313 * 10314 * 1 unit for adding the free space inode's orphan (located in the tree 10315 * of tree roots). 10316 * 1 unit for deleting the block group item (located in the extent 10317 * tree). 10318 * 1 unit for deleting the free space item (located in tree of tree 10319 * roots). 10320 * N units for deleting N device extent items corresponding to each 10321 * stripe (located in the device tree). 10322 * 10323 * In order to remove a block group we also need to reserve units in the 10324 * system space info in order to update the chunk tree (update one or 10325 * more device items and remove one chunk item), but this is done at 10326 * btrfs_remove_chunk() through a call to check_system_chunk(). 10327 */ 10328 map = (struct map_lookup *)em->bdev; 10329 num_items = 3 + map->num_stripes; 10330 free_extent_map(em); 10331 10332 return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root, 10333 num_items, 1); 10334 } 10335 10336 /* 10337 * Process the unused_bgs list and remove any that don't have any allocated 10338 * space inside of them. 10339 */ 10340 void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) 10341 { 10342 struct btrfs_block_group_cache *block_group; 10343 struct btrfs_space_info *space_info; 10344 struct btrfs_root *root = fs_info->extent_root; 10345 struct btrfs_trans_handle *trans; 10346 int ret = 0; 10347 10348 if (!fs_info->open) 10349 return; 10350 10351 spin_lock(&fs_info->unused_bgs_lock); 10352 while (!list_empty(&fs_info->unused_bgs)) { 10353 u64 start, end; 10354 int trimming; 10355 10356 block_group = list_first_entry(&fs_info->unused_bgs, 10357 struct btrfs_block_group_cache, 10358 bg_list); 10359 list_del_init(&block_group->bg_list); 10360 10361 space_info = block_group->space_info; 10362 10363 if (ret || btrfs_mixed_space_info(space_info)) { 10364 btrfs_put_block_group(block_group); 10365 continue; 10366 } 10367 spin_unlock(&fs_info->unused_bgs_lock); 10368 10369 mutex_lock(&fs_info->delete_unused_bgs_mutex); 10370 10371 /* Don't want to race with allocators so take the groups_sem */ 10372 down_write(&space_info->groups_sem); 10373 spin_lock(&block_group->lock); 10374 if (block_group->reserved || 10375 btrfs_block_group_used(&block_group->item) || 10376 block_group->ro || 10377 list_is_singular(&block_group->list)) { 10378 /* 10379 * We want to bail if we made new allocations or have 10380 * outstanding allocations in this block group. We do 10381 * the ro check in case balance is currently acting on 10382 * this block group. 10383 */ 10384 spin_unlock(&block_group->lock); 10385 up_write(&space_info->groups_sem); 10386 goto next; 10387 } 10388 spin_unlock(&block_group->lock); 10389 10390 /* We don't want to force the issue, only flip if it's ok. */ 10391 ret = inc_block_group_ro(block_group, 0); 10392 up_write(&space_info->groups_sem); 10393 if (ret < 0) { 10394 ret = 0; 10395 goto next; 10396 } 10397 10398 /* 10399 * Want to do this before we do anything else so we can recover 10400 * properly if we fail to join the transaction. 10401 */ 10402 trans = btrfs_start_trans_remove_block_group(fs_info, 10403 block_group->key.objectid); 10404 if (IS_ERR(trans)) { 10405 btrfs_dec_block_group_ro(root, block_group); 10406 ret = PTR_ERR(trans); 10407 goto next; 10408 } 10409 10410 /* 10411 * We could have pending pinned extents for this block group, 10412 * just delete them, we don't care about them anymore. 10413 */ 10414 start = block_group->key.objectid; 10415 end = start + block_group->key.offset - 1; 10416 /* 10417 * Hold the unused_bg_unpin_mutex lock to avoid racing with 10418 * btrfs_finish_extent_commit(). If we are at transaction N, 10419 * another task might be running finish_extent_commit() for the 10420 * previous transaction N - 1, and have seen a range belonging 10421 * to the block group in freed_extents[] before we were able to 10422 * clear the whole block group range from freed_extents[]. This 10423 * means that task can lookup for the block group after we 10424 * unpinned it from freed_extents[] and removed it, leading to 10425 * a BUG_ON() at btrfs_unpin_extent_range(). 10426 */ 10427 mutex_lock(&fs_info->unused_bg_unpin_mutex); 10428 ret = clear_extent_bits(&fs_info->freed_extents[0], start, end, 10429 EXTENT_DIRTY, GFP_NOFS); 10430 if (ret) { 10431 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 10432 btrfs_dec_block_group_ro(root, block_group); 10433 goto end_trans; 10434 } 10435 ret = clear_extent_bits(&fs_info->freed_extents[1], start, end, 10436 EXTENT_DIRTY, GFP_NOFS); 10437 if (ret) { 10438 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 10439 btrfs_dec_block_group_ro(root, block_group); 10440 goto end_trans; 10441 } 10442 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 10443 10444 /* Reset pinned so btrfs_put_block_group doesn't complain */ 10445 spin_lock(&space_info->lock); 10446 spin_lock(&block_group->lock); 10447 10448 space_info->bytes_pinned -= block_group->pinned; 10449 space_info->bytes_readonly += block_group->pinned; 10450 percpu_counter_add(&space_info->total_bytes_pinned, 10451 -block_group->pinned); 10452 block_group->pinned = 0; 10453 10454 spin_unlock(&block_group->lock); 10455 spin_unlock(&space_info->lock); 10456 10457 /* DISCARD can flip during remount */ 10458 trimming = btrfs_test_opt(root, DISCARD); 10459 10460 /* Implicit trim during transaction commit. */ 10461 if (trimming) 10462 btrfs_get_block_group_trimming(block_group); 10463 10464 /* 10465 * Btrfs_remove_chunk will abort the transaction if things go 10466 * horribly wrong. 10467 */ 10468 ret = btrfs_remove_chunk(trans, root, 10469 block_group->key.objectid); 10470 10471 if (ret) { 10472 if (trimming) 10473 btrfs_put_block_group_trimming(block_group); 10474 goto end_trans; 10475 } 10476 10477 /* 10478 * If we're not mounted with -odiscard, we can just forget 10479 * about this block group. Otherwise we'll need to wait 10480 * until transaction commit to do the actual discard. 10481 */ 10482 if (trimming) { 10483 spin_lock(&fs_info->unused_bgs_lock); 10484 /* 10485 * A concurrent scrub might have added us to the list 10486 * fs_info->unused_bgs, so use a list_move operation 10487 * to add the block group to the deleted_bgs list. 10488 */ 10489 list_move(&block_group->bg_list, 10490 &trans->transaction->deleted_bgs); 10491 spin_unlock(&fs_info->unused_bgs_lock); 10492 btrfs_get_block_group(block_group); 10493 } 10494 end_trans: 10495 btrfs_end_transaction(trans, root); 10496 next: 10497 mutex_unlock(&fs_info->delete_unused_bgs_mutex); 10498 btrfs_put_block_group(block_group); 10499 spin_lock(&fs_info->unused_bgs_lock); 10500 } 10501 spin_unlock(&fs_info->unused_bgs_lock); 10502 } 10503 10504 int btrfs_init_space_info(struct btrfs_fs_info *fs_info) 10505 { 10506 struct btrfs_space_info *space_info; 10507 struct btrfs_super_block *disk_super; 10508 u64 features; 10509 u64 flags; 10510 int mixed = 0; 10511 int ret; 10512 10513 disk_super = fs_info->super_copy; 10514 if (!btrfs_super_root(disk_super)) 10515 return 1; 10516 10517 features = btrfs_super_incompat_flags(disk_super); 10518 if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) 10519 mixed = 1; 10520 10521 flags = BTRFS_BLOCK_GROUP_SYSTEM; 10522 ret = update_space_info(fs_info, flags, 0, 0, &space_info); 10523 if (ret) 10524 goto out; 10525 10526 if (mixed) { 10527 flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA; 10528 ret = update_space_info(fs_info, flags, 0, 0, &space_info); 10529 } else { 10530 flags = BTRFS_BLOCK_GROUP_METADATA; 10531 ret = update_space_info(fs_info, flags, 0, 0, &space_info); 10532 if (ret) 10533 goto out; 10534 10535 flags = BTRFS_BLOCK_GROUP_DATA; 10536 ret = update_space_info(fs_info, flags, 0, 0, &space_info); 10537 } 10538 out: 10539 return ret; 10540 } 10541 10542 int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end) 10543 { 10544 return unpin_extent_range(root, start, end, false); 10545 } 10546 10547 /* 10548 * It used to be that old block groups would be left around forever. 10549 * Iterating over them would be enough to trim unused space. Since we 10550 * now automatically remove them, we also need to iterate over unallocated 10551 * space. 10552 * 10553 * We don't want a transaction for this since the discard may take a 10554 * substantial amount of time. We don't require that a transaction be 10555 * running, but we do need to take a running transaction into account 10556 * to ensure that we're not discarding chunks that were released in 10557 * the current transaction. 10558 * 10559 * Holding the chunks lock will prevent other threads from allocating 10560 * or releasing chunks, but it won't prevent a running transaction 10561 * from committing and releasing the memory that the pending chunks 10562 * list head uses. For that, we need to take a reference to the 10563 * transaction. 10564 */ 10565 static int btrfs_trim_free_extents(struct btrfs_device *device, 10566 u64 minlen, u64 *trimmed) 10567 { 10568 u64 start = 0, len = 0; 10569 int ret; 10570 10571 *trimmed = 0; 10572 10573 /* Not writeable = nothing to do. */ 10574 if (!device->writeable) 10575 return 0; 10576 10577 /* No free space = nothing to do. */ 10578 if (device->total_bytes <= device->bytes_used) 10579 return 0; 10580 10581 ret = 0; 10582 10583 while (1) { 10584 struct btrfs_fs_info *fs_info = device->dev_root->fs_info; 10585 struct btrfs_transaction *trans; 10586 u64 bytes; 10587 10588 ret = mutex_lock_interruptible(&fs_info->chunk_mutex); 10589 if (ret) 10590 return ret; 10591 10592 down_read(&fs_info->commit_root_sem); 10593 10594 spin_lock(&fs_info->trans_lock); 10595 trans = fs_info->running_transaction; 10596 if (trans) 10597 atomic_inc(&trans->use_count); 10598 spin_unlock(&fs_info->trans_lock); 10599 10600 ret = find_free_dev_extent_start(trans, device, minlen, start, 10601 &start, &len); 10602 if (trans) 10603 btrfs_put_transaction(trans); 10604 10605 if (ret) { 10606 up_read(&fs_info->commit_root_sem); 10607 mutex_unlock(&fs_info->chunk_mutex); 10608 if (ret == -ENOSPC) 10609 ret = 0; 10610 break; 10611 } 10612 10613 ret = btrfs_issue_discard(device->bdev, start, len, &bytes); 10614 up_read(&fs_info->commit_root_sem); 10615 mutex_unlock(&fs_info->chunk_mutex); 10616 10617 if (ret) 10618 break; 10619 10620 start += len; 10621 *trimmed += bytes; 10622 10623 if (fatal_signal_pending(current)) { 10624 ret = -ERESTARTSYS; 10625 break; 10626 } 10627 10628 cond_resched(); 10629 } 10630 10631 return ret; 10632 } 10633 10634 int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range) 10635 { 10636 struct btrfs_fs_info *fs_info = root->fs_info; 10637 struct btrfs_block_group_cache *cache = NULL; 10638 struct btrfs_device *device; 10639 struct list_head *devices; 10640 u64 group_trimmed; 10641 u64 start; 10642 u64 end; 10643 u64 trimmed = 0; 10644 u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy); 10645 int ret = 0; 10646 10647 /* 10648 * try to trim all FS space, our block group may start from non-zero. 10649 */ 10650 if (range->len == total_bytes) 10651 cache = btrfs_lookup_first_block_group(fs_info, range->start); 10652 else 10653 cache = btrfs_lookup_block_group(fs_info, range->start); 10654 10655 while (cache) { 10656 if (cache->key.objectid >= (range->start + range->len)) { 10657 btrfs_put_block_group(cache); 10658 break; 10659 } 10660 10661 start = max(range->start, cache->key.objectid); 10662 end = min(range->start + range->len, 10663 cache->key.objectid + cache->key.offset); 10664 10665 if (end - start >= range->minlen) { 10666 if (!block_group_cache_done(cache)) { 10667 ret = cache_block_group(cache, 0); 10668 if (ret) { 10669 btrfs_put_block_group(cache); 10670 break; 10671 } 10672 ret = wait_block_group_cache_done(cache); 10673 if (ret) { 10674 btrfs_put_block_group(cache); 10675 break; 10676 } 10677 } 10678 ret = btrfs_trim_block_group(cache, 10679 &group_trimmed, 10680 start, 10681 end, 10682 range->minlen); 10683 10684 trimmed += group_trimmed; 10685 if (ret) { 10686 btrfs_put_block_group(cache); 10687 break; 10688 } 10689 } 10690 10691 cache = next_block_group(fs_info->tree_root, cache); 10692 } 10693 10694 mutex_lock(&root->fs_info->fs_devices->device_list_mutex); 10695 devices = &root->fs_info->fs_devices->alloc_list; 10696 list_for_each_entry(device, devices, dev_alloc_list) { 10697 ret = btrfs_trim_free_extents(device, range->minlen, 10698 &group_trimmed); 10699 if (ret) 10700 break; 10701 10702 trimmed += group_trimmed; 10703 } 10704 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex); 10705 10706 range->len = trimmed; 10707 return ret; 10708 } 10709 10710 /* 10711 * btrfs_{start,end}_write_no_snapshoting() are similar to 10712 * mnt_{want,drop}_write(), they are used to prevent some tasks from writing 10713 * data into the page cache through nocow before the subvolume is snapshoted, 10714 * but flush the data into disk after the snapshot creation, or to prevent 10715 * operations while snapshoting is ongoing and that cause the snapshot to be 10716 * inconsistent (writes followed by expanding truncates for example). 10717 */ 10718 void btrfs_end_write_no_snapshoting(struct btrfs_root *root) 10719 { 10720 percpu_counter_dec(&root->subv_writers->counter); 10721 /* 10722 * Make sure counter is updated before we wake up waiters. 10723 */ 10724 smp_mb(); 10725 if (waitqueue_active(&root->subv_writers->wait)) 10726 wake_up(&root->subv_writers->wait); 10727 } 10728 10729 int btrfs_start_write_no_snapshoting(struct btrfs_root *root) 10730 { 10731 if (atomic_read(&root->will_be_snapshoted)) 10732 return 0; 10733 10734 percpu_counter_inc(&root->subv_writers->counter); 10735 /* 10736 * Make sure counter is updated before we check for snapshot creation. 10737 */ 10738 smp_mb(); 10739 if (atomic_read(&root->will_be_snapshoted)) { 10740 btrfs_end_write_no_snapshoting(root); 10741 return 0; 10742 } 10743 return 1; 10744 } 10745