1 /* 2 * Copyright (C) 2007 Oracle. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public 6 * License v2 as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public 14 * License along with this program; if not, write to the 15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 16 * Boston, MA 021110-1307, USA. 17 */ 18 #include <linux/sched.h> 19 #include <linux/pagemap.h> 20 #include <linux/writeback.h> 21 #include <linux/blkdev.h> 22 #include <linux/sort.h> 23 #include <linux/rcupdate.h> 24 #include <linux/kthread.h> 25 #include <linux/slab.h> 26 #include <linux/ratelimit.h> 27 #include <linux/percpu_counter.h> 28 #include "hash.h" 29 #include "tree-log.h" 30 #include "disk-io.h" 31 #include "print-tree.h" 32 #include "volumes.h" 33 #include "raid56.h" 34 #include "locking.h" 35 #include "free-space-cache.h" 36 #include "free-space-tree.h" 37 #include "math.h" 38 #include "sysfs.h" 39 #include "qgroup.h" 40 41 #undef SCRAMBLE_DELAYED_REFS 42 43 /* 44 * control flags for do_chunk_alloc's force field 45 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk 46 * if we really need one. 47 * 48 * CHUNK_ALLOC_LIMITED means to only try and allocate one 49 * if we have very few chunks already allocated. This is 50 * used as part of the clustering code to help make sure 51 * we have a good pool of storage to cluster in, without 52 * filling the FS with empty chunks 53 * 54 * CHUNK_ALLOC_FORCE means it must try to allocate one 55 * 56 */ 57 enum { 58 CHUNK_ALLOC_NO_FORCE = 0, 59 CHUNK_ALLOC_LIMITED = 1, 60 CHUNK_ALLOC_FORCE = 2, 61 }; 62 63 static int update_block_group(struct btrfs_trans_handle *trans, 64 struct btrfs_fs_info *fs_info, u64 bytenr, 65 u64 num_bytes, int alloc); 66 static int __btrfs_free_extent(struct btrfs_trans_handle *trans, 67 struct btrfs_fs_info *fs_info, 68 struct btrfs_delayed_ref_node *node, u64 parent, 69 u64 root_objectid, u64 owner_objectid, 70 u64 owner_offset, int refs_to_drop, 71 struct btrfs_delayed_extent_op *extra_op); 72 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, 73 struct extent_buffer *leaf, 74 struct btrfs_extent_item *ei); 75 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, 76 struct btrfs_fs_info *fs_info, 77 u64 parent, u64 root_objectid, 78 u64 flags, u64 owner, u64 offset, 79 struct btrfs_key *ins, int ref_mod); 80 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, 81 struct btrfs_fs_info *fs_info, 82 u64 parent, u64 root_objectid, 83 u64 flags, struct btrfs_disk_key *key, 84 int level, struct btrfs_key *ins); 85 static int do_chunk_alloc(struct btrfs_trans_handle *trans, 86 struct btrfs_fs_info *fs_info, u64 flags, 87 int force); 88 static int find_next_key(struct btrfs_path *path, int level, 89 struct btrfs_key *key); 90 static void dump_space_info(struct btrfs_fs_info *fs_info, 91 struct btrfs_space_info *info, u64 bytes, 92 int dump_block_groups); 93 static int btrfs_add_reserved_bytes(struct btrfs_block_group_cache *cache, 94 u64 ram_bytes, u64 num_bytes, int delalloc); 95 static int btrfs_free_reserved_bytes(struct btrfs_block_group_cache *cache, 96 u64 num_bytes, int delalloc); 97 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, 98 u64 num_bytes); 99 static int __reserve_metadata_bytes(struct btrfs_root *root, 100 struct btrfs_space_info *space_info, 101 u64 orig_bytes, 102 enum btrfs_reserve_flush_enum flush); 103 static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info, 104 struct btrfs_space_info *space_info, 105 u64 num_bytes); 106 static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info, 107 struct btrfs_space_info *space_info, 108 u64 num_bytes); 109 110 static noinline int 111 block_group_cache_done(struct btrfs_block_group_cache *cache) 112 { 113 smp_mb(); 114 return cache->cached == BTRFS_CACHE_FINISHED || 115 cache->cached == BTRFS_CACHE_ERROR; 116 } 117 118 static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits) 119 { 120 return (cache->flags & bits) == bits; 121 } 122 123 void btrfs_get_block_group(struct btrfs_block_group_cache *cache) 124 { 125 atomic_inc(&cache->count); 126 } 127 128 void btrfs_put_block_group(struct btrfs_block_group_cache *cache) 129 { 130 if (atomic_dec_and_test(&cache->count)) { 131 WARN_ON(cache->pinned > 0); 132 WARN_ON(cache->reserved > 0); 133 kfree(cache->free_space_ctl); 134 kfree(cache); 135 } 136 } 137 138 /* 139 * this adds the block group to the fs_info rb tree for the block group 140 * cache 141 */ 142 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, 143 struct btrfs_block_group_cache *block_group) 144 { 145 struct rb_node **p; 146 struct rb_node *parent = NULL; 147 struct btrfs_block_group_cache *cache; 148 149 spin_lock(&info->block_group_cache_lock); 150 p = &info->block_group_cache_tree.rb_node; 151 152 while (*p) { 153 parent = *p; 154 cache = rb_entry(parent, struct btrfs_block_group_cache, 155 cache_node); 156 if (block_group->key.objectid < cache->key.objectid) { 157 p = &(*p)->rb_left; 158 } else if (block_group->key.objectid > cache->key.objectid) { 159 p = &(*p)->rb_right; 160 } else { 161 spin_unlock(&info->block_group_cache_lock); 162 return -EEXIST; 163 } 164 } 165 166 rb_link_node(&block_group->cache_node, parent, p); 167 rb_insert_color(&block_group->cache_node, 168 &info->block_group_cache_tree); 169 170 if (info->first_logical_byte > block_group->key.objectid) 171 info->first_logical_byte = block_group->key.objectid; 172 173 spin_unlock(&info->block_group_cache_lock); 174 175 return 0; 176 } 177 178 /* 179 * This will return the block group at or after bytenr if contains is 0, else 180 * it will return the block group that contains the bytenr 181 */ 182 static struct btrfs_block_group_cache * 183 block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr, 184 int contains) 185 { 186 struct btrfs_block_group_cache *cache, *ret = NULL; 187 struct rb_node *n; 188 u64 end, start; 189 190 spin_lock(&info->block_group_cache_lock); 191 n = info->block_group_cache_tree.rb_node; 192 193 while (n) { 194 cache = rb_entry(n, struct btrfs_block_group_cache, 195 cache_node); 196 end = cache->key.objectid + cache->key.offset - 1; 197 start = cache->key.objectid; 198 199 if (bytenr < start) { 200 if (!contains && (!ret || start < ret->key.objectid)) 201 ret = cache; 202 n = n->rb_left; 203 } else if (bytenr > start) { 204 if (contains && bytenr <= end) { 205 ret = cache; 206 break; 207 } 208 n = n->rb_right; 209 } else { 210 ret = cache; 211 break; 212 } 213 } 214 if (ret) { 215 btrfs_get_block_group(ret); 216 if (bytenr == 0 && info->first_logical_byte > ret->key.objectid) 217 info->first_logical_byte = ret->key.objectid; 218 } 219 spin_unlock(&info->block_group_cache_lock); 220 221 return ret; 222 } 223 224 static int add_excluded_extent(struct btrfs_fs_info *fs_info, 225 u64 start, u64 num_bytes) 226 { 227 u64 end = start + num_bytes - 1; 228 set_extent_bits(&fs_info->freed_extents[0], 229 start, end, EXTENT_UPTODATE); 230 set_extent_bits(&fs_info->freed_extents[1], 231 start, end, EXTENT_UPTODATE); 232 return 0; 233 } 234 235 static void free_excluded_extents(struct btrfs_fs_info *fs_info, 236 struct btrfs_block_group_cache *cache) 237 { 238 u64 start, end; 239 240 start = cache->key.objectid; 241 end = start + cache->key.offset - 1; 242 243 clear_extent_bits(&fs_info->freed_extents[0], 244 start, end, EXTENT_UPTODATE); 245 clear_extent_bits(&fs_info->freed_extents[1], 246 start, end, EXTENT_UPTODATE); 247 } 248 249 static int exclude_super_stripes(struct btrfs_fs_info *fs_info, 250 struct btrfs_block_group_cache *cache) 251 { 252 u64 bytenr; 253 u64 *logical; 254 int stripe_len; 255 int i, nr, ret; 256 257 if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) { 258 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid; 259 cache->bytes_super += stripe_len; 260 ret = add_excluded_extent(fs_info, cache->key.objectid, 261 stripe_len); 262 if (ret) 263 return ret; 264 } 265 266 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 267 bytenr = btrfs_sb_offset(i); 268 ret = btrfs_rmap_block(fs_info, cache->key.objectid, 269 bytenr, 0, &logical, &nr, &stripe_len); 270 if (ret) 271 return ret; 272 273 while (nr--) { 274 u64 start, len; 275 276 if (logical[nr] > cache->key.objectid + 277 cache->key.offset) 278 continue; 279 280 if (logical[nr] + stripe_len <= cache->key.objectid) 281 continue; 282 283 start = logical[nr]; 284 if (start < cache->key.objectid) { 285 start = cache->key.objectid; 286 len = (logical[nr] + stripe_len) - start; 287 } else { 288 len = min_t(u64, stripe_len, 289 cache->key.objectid + 290 cache->key.offset - start); 291 } 292 293 cache->bytes_super += len; 294 ret = add_excluded_extent(fs_info, start, len); 295 if (ret) { 296 kfree(logical); 297 return ret; 298 } 299 } 300 301 kfree(logical); 302 } 303 return 0; 304 } 305 306 static struct btrfs_caching_control * 307 get_caching_control(struct btrfs_block_group_cache *cache) 308 { 309 struct btrfs_caching_control *ctl; 310 311 spin_lock(&cache->lock); 312 if (!cache->caching_ctl) { 313 spin_unlock(&cache->lock); 314 return NULL; 315 } 316 317 ctl = cache->caching_ctl; 318 atomic_inc(&ctl->count); 319 spin_unlock(&cache->lock); 320 return ctl; 321 } 322 323 static void put_caching_control(struct btrfs_caching_control *ctl) 324 { 325 if (atomic_dec_and_test(&ctl->count)) 326 kfree(ctl); 327 } 328 329 #ifdef CONFIG_BTRFS_DEBUG 330 static void fragment_free_space(struct btrfs_block_group_cache *block_group) 331 { 332 struct btrfs_fs_info *fs_info = block_group->fs_info; 333 u64 start = block_group->key.objectid; 334 u64 len = block_group->key.offset; 335 u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ? 336 fs_info->nodesize : fs_info->sectorsize; 337 u64 step = chunk << 1; 338 339 while (len > chunk) { 340 btrfs_remove_free_space(block_group, start, chunk); 341 start += step; 342 if (len < step) 343 len = 0; 344 else 345 len -= step; 346 } 347 } 348 #endif 349 350 /* 351 * this is only called by cache_block_group, since we could have freed extents 352 * we need to check the pinned_extents for any extents that can't be used yet 353 * since their free space will be released as soon as the transaction commits. 354 */ 355 u64 add_new_free_space(struct btrfs_block_group_cache *block_group, 356 struct btrfs_fs_info *info, u64 start, u64 end) 357 { 358 u64 extent_start, extent_end, size, total_added = 0; 359 int ret; 360 361 while (start < end) { 362 ret = find_first_extent_bit(info->pinned_extents, start, 363 &extent_start, &extent_end, 364 EXTENT_DIRTY | EXTENT_UPTODATE, 365 NULL); 366 if (ret) 367 break; 368 369 if (extent_start <= start) { 370 start = extent_end + 1; 371 } else if (extent_start > start && extent_start < end) { 372 size = extent_start - start; 373 total_added += size; 374 ret = btrfs_add_free_space(block_group, start, 375 size); 376 BUG_ON(ret); /* -ENOMEM or logic error */ 377 start = extent_end + 1; 378 } else { 379 break; 380 } 381 } 382 383 if (start < end) { 384 size = end - start; 385 total_added += size; 386 ret = btrfs_add_free_space(block_group, start, size); 387 BUG_ON(ret); /* -ENOMEM or logic error */ 388 } 389 390 return total_added; 391 } 392 393 static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl) 394 { 395 struct btrfs_block_group_cache *block_group = caching_ctl->block_group; 396 struct btrfs_fs_info *fs_info = block_group->fs_info; 397 struct btrfs_root *extent_root = fs_info->extent_root; 398 struct btrfs_path *path; 399 struct extent_buffer *leaf; 400 struct btrfs_key key; 401 u64 total_found = 0; 402 u64 last = 0; 403 u32 nritems; 404 int ret; 405 bool wakeup = true; 406 407 path = btrfs_alloc_path(); 408 if (!path) 409 return -ENOMEM; 410 411 last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET); 412 413 #ifdef CONFIG_BTRFS_DEBUG 414 /* 415 * If we're fragmenting we don't want to make anybody think we can 416 * allocate from this block group until we've had a chance to fragment 417 * the free space. 418 */ 419 if (btrfs_should_fragment_free_space(block_group)) 420 wakeup = false; 421 #endif 422 /* 423 * We don't want to deadlock with somebody trying to allocate a new 424 * extent for the extent root while also trying to search the extent 425 * root to add free space. So we skip locking and search the commit 426 * root, since its read-only 427 */ 428 path->skip_locking = 1; 429 path->search_commit_root = 1; 430 path->reada = READA_FORWARD; 431 432 key.objectid = last; 433 key.offset = 0; 434 key.type = BTRFS_EXTENT_ITEM_KEY; 435 436 next: 437 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); 438 if (ret < 0) 439 goto out; 440 441 leaf = path->nodes[0]; 442 nritems = btrfs_header_nritems(leaf); 443 444 while (1) { 445 if (btrfs_fs_closing(fs_info) > 1) { 446 last = (u64)-1; 447 break; 448 } 449 450 if (path->slots[0] < nritems) { 451 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 452 } else { 453 ret = find_next_key(path, 0, &key); 454 if (ret) 455 break; 456 457 if (need_resched() || 458 rwsem_is_contended(&fs_info->commit_root_sem)) { 459 if (wakeup) 460 caching_ctl->progress = last; 461 btrfs_release_path(path); 462 up_read(&fs_info->commit_root_sem); 463 mutex_unlock(&caching_ctl->mutex); 464 cond_resched(); 465 mutex_lock(&caching_ctl->mutex); 466 down_read(&fs_info->commit_root_sem); 467 goto next; 468 } 469 470 ret = btrfs_next_leaf(extent_root, path); 471 if (ret < 0) 472 goto out; 473 if (ret) 474 break; 475 leaf = path->nodes[0]; 476 nritems = btrfs_header_nritems(leaf); 477 continue; 478 } 479 480 if (key.objectid < last) { 481 key.objectid = last; 482 key.offset = 0; 483 key.type = BTRFS_EXTENT_ITEM_KEY; 484 485 if (wakeup) 486 caching_ctl->progress = last; 487 btrfs_release_path(path); 488 goto next; 489 } 490 491 if (key.objectid < block_group->key.objectid) { 492 path->slots[0]++; 493 continue; 494 } 495 496 if (key.objectid >= block_group->key.objectid + 497 block_group->key.offset) 498 break; 499 500 if (key.type == BTRFS_EXTENT_ITEM_KEY || 501 key.type == BTRFS_METADATA_ITEM_KEY) { 502 total_found += add_new_free_space(block_group, 503 fs_info, last, 504 key.objectid); 505 if (key.type == BTRFS_METADATA_ITEM_KEY) 506 last = key.objectid + 507 fs_info->nodesize; 508 else 509 last = key.objectid + key.offset; 510 511 if (total_found > CACHING_CTL_WAKE_UP) { 512 total_found = 0; 513 if (wakeup) 514 wake_up(&caching_ctl->wait); 515 } 516 } 517 path->slots[0]++; 518 } 519 ret = 0; 520 521 total_found += add_new_free_space(block_group, fs_info, last, 522 block_group->key.objectid + 523 block_group->key.offset); 524 caching_ctl->progress = (u64)-1; 525 526 out: 527 btrfs_free_path(path); 528 return ret; 529 } 530 531 static noinline void caching_thread(struct btrfs_work *work) 532 { 533 struct btrfs_block_group_cache *block_group; 534 struct btrfs_fs_info *fs_info; 535 struct btrfs_caching_control *caching_ctl; 536 struct btrfs_root *extent_root; 537 int ret; 538 539 caching_ctl = container_of(work, struct btrfs_caching_control, work); 540 block_group = caching_ctl->block_group; 541 fs_info = block_group->fs_info; 542 extent_root = fs_info->extent_root; 543 544 mutex_lock(&caching_ctl->mutex); 545 down_read(&fs_info->commit_root_sem); 546 547 if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) 548 ret = load_free_space_tree(caching_ctl); 549 else 550 ret = load_extent_tree_free(caching_ctl); 551 552 spin_lock(&block_group->lock); 553 block_group->caching_ctl = NULL; 554 block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED; 555 spin_unlock(&block_group->lock); 556 557 #ifdef CONFIG_BTRFS_DEBUG 558 if (btrfs_should_fragment_free_space(block_group)) { 559 u64 bytes_used; 560 561 spin_lock(&block_group->space_info->lock); 562 spin_lock(&block_group->lock); 563 bytes_used = block_group->key.offset - 564 btrfs_block_group_used(&block_group->item); 565 block_group->space_info->bytes_used += bytes_used >> 1; 566 spin_unlock(&block_group->lock); 567 spin_unlock(&block_group->space_info->lock); 568 fragment_free_space(block_group); 569 } 570 #endif 571 572 caching_ctl->progress = (u64)-1; 573 574 up_read(&fs_info->commit_root_sem); 575 free_excluded_extents(fs_info, block_group); 576 mutex_unlock(&caching_ctl->mutex); 577 578 wake_up(&caching_ctl->wait); 579 580 put_caching_control(caching_ctl); 581 btrfs_put_block_group(block_group); 582 } 583 584 static int cache_block_group(struct btrfs_block_group_cache *cache, 585 int load_cache_only) 586 { 587 DEFINE_WAIT(wait); 588 struct btrfs_fs_info *fs_info = cache->fs_info; 589 struct btrfs_caching_control *caching_ctl; 590 int ret = 0; 591 592 caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS); 593 if (!caching_ctl) 594 return -ENOMEM; 595 596 INIT_LIST_HEAD(&caching_ctl->list); 597 mutex_init(&caching_ctl->mutex); 598 init_waitqueue_head(&caching_ctl->wait); 599 caching_ctl->block_group = cache; 600 caching_ctl->progress = cache->key.objectid; 601 atomic_set(&caching_ctl->count, 1); 602 btrfs_init_work(&caching_ctl->work, btrfs_cache_helper, 603 caching_thread, NULL, NULL); 604 605 spin_lock(&cache->lock); 606 /* 607 * This should be a rare occasion, but this could happen I think in the 608 * case where one thread starts to load the space cache info, and then 609 * some other thread starts a transaction commit which tries to do an 610 * allocation while the other thread is still loading the space cache 611 * info. The previous loop should have kept us from choosing this block 612 * group, but if we've moved to the state where we will wait on caching 613 * block groups we need to first check if we're doing a fast load here, 614 * so we can wait for it to finish, otherwise we could end up allocating 615 * from a block group who's cache gets evicted for one reason or 616 * another. 617 */ 618 while (cache->cached == BTRFS_CACHE_FAST) { 619 struct btrfs_caching_control *ctl; 620 621 ctl = cache->caching_ctl; 622 atomic_inc(&ctl->count); 623 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE); 624 spin_unlock(&cache->lock); 625 626 schedule(); 627 628 finish_wait(&ctl->wait, &wait); 629 put_caching_control(ctl); 630 spin_lock(&cache->lock); 631 } 632 633 if (cache->cached != BTRFS_CACHE_NO) { 634 spin_unlock(&cache->lock); 635 kfree(caching_ctl); 636 return 0; 637 } 638 WARN_ON(cache->caching_ctl); 639 cache->caching_ctl = caching_ctl; 640 cache->cached = BTRFS_CACHE_FAST; 641 spin_unlock(&cache->lock); 642 643 if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) { 644 mutex_lock(&caching_ctl->mutex); 645 ret = load_free_space_cache(fs_info, cache); 646 647 spin_lock(&cache->lock); 648 if (ret == 1) { 649 cache->caching_ctl = NULL; 650 cache->cached = BTRFS_CACHE_FINISHED; 651 cache->last_byte_to_unpin = (u64)-1; 652 caching_ctl->progress = (u64)-1; 653 } else { 654 if (load_cache_only) { 655 cache->caching_ctl = NULL; 656 cache->cached = BTRFS_CACHE_NO; 657 } else { 658 cache->cached = BTRFS_CACHE_STARTED; 659 cache->has_caching_ctl = 1; 660 } 661 } 662 spin_unlock(&cache->lock); 663 #ifdef CONFIG_BTRFS_DEBUG 664 if (ret == 1 && 665 btrfs_should_fragment_free_space(cache)) { 666 u64 bytes_used; 667 668 spin_lock(&cache->space_info->lock); 669 spin_lock(&cache->lock); 670 bytes_used = cache->key.offset - 671 btrfs_block_group_used(&cache->item); 672 cache->space_info->bytes_used += bytes_used >> 1; 673 spin_unlock(&cache->lock); 674 spin_unlock(&cache->space_info->lock); 675 fragment_free_space(cache); 676 } 677 #endif 678 mutex_unlock(&caching_ctl->mutex); 679 680 wake_up(&caching_ctl->wait); 681 if (ret == 1) { 682 put_caching_control(caching_ctl); 683 free_excluded_extents(fs_info, cache); 684 return 0; 685 } 686 } else { 687 /* 688 * We're either using the free space tree or no caching at all. 689 * Set cached to the appropriate value and wakeup any waiters. 690 */ 691 spin_lock(&cache->lock); 692 if (load_cache_only) { 693 cache->caching_ctl = NULL; 694 cache->cached = BTRFS_CACHE_NO; 695 } else { 696 cache->cached = BTRFS_CACHE_STARTED; 697 cache->has_caching_ctl = 1; 698 } 699 spin_unlock(&cache->lock); 700 wake_up(&caching_ctl->wait); 701 } 702 703 if (load_cache_only) { 704 put_caching_control(caching_ctl); 705 return 0; 706 } 707 708 down_write(&fs_info->commit_root_sem); 709 atomic_inc(&caching_ctl->count); 710 list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups); 711 up_write(&fs_info->commit_root_sem); 712 713 btrfs_get_block_group(cache); 714 715 btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work); 716 717 return ret; 718 } 719 720 /* 721 * return the block group that starts at or after bytenr 722 */ 723 static struct btrfs_block_group_cache * 724 btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr) 725 { 726 return block_group_cache_tree_search(info, bytenr, 0); 727 } 728 729 /* 730 * return the block group that contains the given bytenr 731 */ 732 struct btrfs_block_group_cache *btrfs_lookup_block_group( 733 struct btrfs_fs_info *info, 734 u64 bytenr) 735 { 736 return block_group_cache_tree_search(info, bytenr, 1); 737 } 738 739 static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info, 740 u64 flags) 741 { 742 struct list_head *head = &info->space_info; 743 struct btrfs_space_info *found; 744 745 flags &= BTRFS_BLOCK_GROUP_TYPE_MASK; 746 747 rcu_read_lock(); 748 list_for_each_entry_rcu(found, head, list) { 749 if (found->flags & flags) { 750 rcu_read_unlock(); 751 return found; 752 } 753 } 754 rcu_read_unlock(); 755 return NULL; 756 } 757 758 /* 759 * after adding space to the filesystem, we need to clear the full flags 760 * on all the space infos. 761 */ 762 void btrfs_clear_space_info_full(struct btrfs_fs_info *info) 763 { 764 struct list_head *head = &info->space_info; 765 struct btrfs_space_info *found; 766 767 rcu_read_lock(); 768 list_for_each_entry_rcu(found, head, list) 769 found->full = 0; 770 rcu_read_unlock(); 771 } 772 773 /* simple helper to search for an existing data extent at a given offset */ 774 int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len) 775 { 776 int ret; 777 struct btrfs_key key; 778 struct btrfs_path *path; 779 780 path = btrfs_alloc_path(); 781 if (!path) 782 return -ENOMEM; 783 784 key.objectid = start; 785 key.offset = len; 786 key.type = BTRFS_EXTENT_ITEM_KEY; 787 ret = btrfs_search_slot(NULL, fs_info->extent_root, &key, path, 0, 0); 788 btrfs_free_path(path); 789 return ret; 790 } 791 792 /* 793 * helper function to lookup reference count and flags of a tree block. 794 * 795 * the head node for delayed ref is used to store the sum of all the 796 * reference count modifications queued up in the rbtree. the head 797 * node may also store the extent flags to set. This way you can check 798 * to see what the reference count and extent flags would be if all of 799 * the delayed refs are not processed. 800 */ 801 int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, 802 struct btrfs_fs_info *fs_info, u64 bytenr, 803 u64 offset, int metadata, u64 *refs, u64 *flags) 804 { 805 struct btrfs_delayed_ref_head *head; 806 struct btrfs_delayed_ref_root *delayed_refs; 807 struct btrfs_path *path; 808 struct btrfs_extent_item *ei; 809 struct extent_buffer *leaf; 810 struct btrfs_key key; 811 u32 item_size; 812 u64 num_refs; 813 u64 extent_flags; 814 int ret; 815 816 /* 817 * If we don't have skinny metadata, don't bother doing anything 818 * different 819 */ 820 if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) { 821 offset = fs_info->nodesize; 822 metadata = 0; 823 } 824 825 path = btrfs_alloc_path(); 826 if (!path) 827 return -ENOMEM; 828 829 if (!trans) { 830 path->skip_locking = 1; 831 path->search_commit_root = 1; 832 } 833 834 search_again: 835 key.objectid = bytenr; 836 key.offset = offset; 837 if (metadata) 838 key.type = BTRFS_METADATA_ITEM_KEY; 839 else 840 key.type = BTRFS_EXTENT_ITEM_KEY; 841 842 ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 0); 843 if (ret < 0) 844 goto out_free; 845 846 if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) { 847 if (path->slots[0]) { 848 path->slots[0]--; 849 btrfs_item_key_to_cpu(path->nodes[0], &key, 850 path->slots[0]); 851 if (key.objectid == bytenr && 852 key.type == BTRFS_EXTENT_ITEM_KEY && 853 key.offset == fs_info->nodesize) 854 ret = 0; 855 } 856 } 857 858 if (ret == 0) { 859 leaf = path->nodes[0]; 860 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 861 if (item_size >= sizeof(*ei)) { 862 ei = btrfs_item_ptr(leaf, path->slots[0], 863 struct btrfs_extent_item); 864 num_refs = btrfs_extent_refs(leaf, ei); 865 extent_flags = btrfs_extent_flags(leaf, ei); 866 } else { 867 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 868 struct btrfs_extent_item_v0 *ei0; 869 BUG_ON(item_size != sizeof(*ei0)); 870 ei0 = btrfs_item_ptr(leaf, path->slots[0], 871 struct btrfs_extent_item_v0); 872 num_refs = btrfs_extent_refs_v0(leaf, ei0); 873 /* FIXME: this isn't correct for data */ 874 extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 875 #else 876 BUG(); 877 #endif 878 } 879 BUG_ON(num_refs == 0); 880 } else { 881 num_refs = 0; 882 extent_flags = 0; 883 ret = 0; 884 } 885 886 if (!trans) 887 goto out; 888 889 delayed_refs = &trans->transaction->delayed_refs; 890 spin_lock(&delayed_refs->lock); 891 head = btrfs_find_delayed_ref_head(trans, bytenr); 892 if (head) { 893 if (!mutex_trylock(&head->mutex)) { 894 atomic_inc(&head->node.refs); 895 spin_unlock(&delayed_refs->lock); 896 897 btrfs_release_path(path); 898 899 /* 900 * Mutex was contended, block until it's released and try 901 * again 902 */ 903 mutex_lock(&head->mutex); 904 mutex_unlock(&head->mutex); 905 btrfs_put_delayed_ref(&head->node); 906 goto search_again; 907 } 908 spin_lock(&head->lock); 909 if (head->extent_op && head->extent_op->update_flags) 910 extent_flags |= head->extent_op->flags_to_set; 911 else 912 BUG_ON(num_refs == 0); 913 914 num_refs += head->node.ref_mod; 915 spin_unlock(&head->lock); 916 mutex_unlock(&head->mutex); 917 } 918 spin_unlock(&delayed_refs->lock); 919 out: 920 WARN_ON(num_refs == 0); 921 if (refs) 922 *refs = num_refs; 923 if (flags) 924 *flags = extent_flags; 925 out_free: 926 btrfs_free_path(path); 927 return ret; 928 } 929 930 /* 931 * Back reference rules. Back refs have three main goals: 932 * 933 * 1) differentiate between all holders of references to an extent so that 934 * when a reference is dropped we can make sure it was a valid reference 935 * before freeing the extent. 936 * 937 * 2) Provide enough information to quickly find the holders of an extent 938 * if we notice a given block is corrupted or bad. 939 * 940 * 3) Make it easy to migrate blocks for FS shrinking or storage pool 941 * maintenance. This is actually the same as #2, but with a slightly 942 * different use case. 943 * 944 * There are two kinds of back refs. The implicit back refs is optimized 945 * for pointers in non-shared tree blocks. For a given pointer in a block, 946 * back refs of this kind provide information about the block's owner tree 947 * and the pointer's key. These information allow us to find the block by 948 * b-tree searching. The full back refs is for pointers in tree blocks not 949 * referenced by their owner trees. The location of tree block is recorded 950 * in the back refs. Actually the full back refs is generic, and can be 951 * used in all cases the implicit back refs is used. The major shortcoming 952 * of the full back refs is its overhead. Every time a tree block gets 953 * COWed, we have to update back refs entry for all pointers in it. 954 * 955 * For a newly allocated tree block, we use implicit back refs for 956 * pointers in it. This means most tree related operations only involve 957 * implicit back refs. For a tree block created in old transaction, the 958 * only way to drop a reference to it is COW it. So we can detect the 959 * event that tree block loses its owner tree's reference and do the 960 * back refs conversion. 961 * 962 * When a tree block is COWed through a tree, there are four cases: 963 * 964 * The reference count of the block is one and the tree is the block's 965 * owner tree. Nothing to do in this case. 966 * 967 * The reference count of the block is one and the tree is not the 968 * block's owner tree. In this case, full back refs is used for pointers 969 * in the block. Remove these full back refs, add implicit back refs for 970 * every pointers in the new block. 971 * 972 * The reference count of the block is greater than one and the tree is 973 * the block's owner tree. In this case, implicit back refs is used for 974 * pointers in the block. Add full back refs for every pointers in the 975 * block, increase lower level extents' reference counts. The original 976 * implicit back refs are entailed to the new block. 977 * 978 * The reference count of the block is greater than one and the tree is 979 * not the block's owner tree. Add implicit back refs for every pointer in 980 * the new block, increase lower level extents' reference count. 981 * 982 * Back Reference Key composing: 983 * 984 * The key objectid corresponds to the first byte in the extent, 985 * The key type is used to differentiate between types of back refs. 986 * There are different meanings of the key offset for different types 987 * of back refs. 988 * 989 * File extents can be referenced by: 990 * 991 * - multiple snapshots, subvolumes, or different generations in one subvol 992 * - different files inside a single subvolume 993 * - different offsets inside a file (bookend extents in file.c) 994 * 995 * The extent ref structure for the implicit back refs has fields for: 996 * 997 * - Objectid of the subvolume root 998 * - objectid of the file holding the reference 999 * - original offset in the file 1000 * - how many bookend extents 1001 * 1002 * The key offset for the implicit back refs is hash of the first 1003 * three fields. 1004 * 1005 * The extent ref structure for the full back refs has field for: 1006 * 1007 * - number of pointers in the tree leaf 1008 * 1009 * The key offset for the implicit back refs is the first byte of 1010 * the tree leaf 1011 * 1012 * When a file extent is allocated, The implicit back refs is used. 1013 * the fields are filled in: 1014 * 1015 * (root_key.objectid, inode objectid, offset in file, 1) 1016 * 1017 * When a file extent is removed file truncation, we find the 1018 * corresponding implicit back refs and check the following fields: 1019 * 1020 * (btrfs_header_owner(leaf), inode objectid, offset in file) 1021 * 1022 * Btree extents can be referenced by: 1023 * 1024 * - Different subvolumes 1025 * 1026 * Both the implicit back refs and the full back refs for tree blocks 1027 * only consist of key. The key offset for the implicit back refs is 1028 * objectid of block's owner tree. The key offset for the full back refs 1029 * is the first byte of parent block. 1030 * 1031 * When implicit back refs is used, information about the lowest key and 1032 * level of the tree block are required. These information are stored in 1033 * tree block info structure. 1034 */ 1035 1036 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1037 static int convert_extent_item_v0(struct btrfs_trans_handle *trans, 1038 struct btrfs_root *root, 1039 struct btrfs_path *path, 1040 u64 owner, u32 extra_size) 1041 { 1042 struct btrfs_extent_item *item; 1043 struct btrfs_extent_item_v0 *ei0; 1044 struct btrfs_extent_ref_v0 *ref0; 1045 struct btrfs_tree_block_info *bi; 1046 struct extent_buffer *leaf; 1047 struct btrfs_key key; 1048 struct btrfs_key found_key; 1049 u32 new_size = sizeof(*item); 1050 u64 refs; 1051 int ret; 1052 1053 leaf = path->nodes[0]; 1054 BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0)); 1055 1056 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1057 ei0 = btrfs_item_ptr(leaf, path->slots[0], 1058 struct btrfs_extent_item_v0); 1059 refs = btrfs_extent_refs_v0(leaf, ei0); 1060 1061 if (owner == (u64)-1) { 1062 while (1) { 1063 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1064 ret = btrfs_next_leaf(root, path); 1065 if (ret < 0) 1066 return ret; 1067 BUG_ON(ret > 0); /* Corruption */ 1068 leaf = path->nodes[0]; 1069 } 1070 btrfs_item_key_to_cpu(leaf, &found_key, 1071 path->slots[0]); 1072 BUG_ON(key.objectid != found_key.objectid); 1073 if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) { 1074 path->slots[0]++; 1075 continue; 1076 } 1077 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1078 struct btrfs_extent_ref_v0); 1079 owner = btrfs_ref_objectid_v0(leaf, ref0); 1080 break; 1081 } 1082 } 1083 btrfs_release_path(path); 1084 1085 if (owner < BTRFS_FIRST_FREE_OBJECTID) 1086 new_size += sizeof(*bi); 1087 1088 new_size -= sizeof(*ei0); 1089 ret = btrfs_search_slot(trans, root, &key, path, 1090 new_size + extra_size, 1); 1091 if (ret < 0) 1092 return ret; 1093 BUG_ON(ret); /* Corruption */ 1094 1095 btrfs_extend_item(root->fs_info, path, new_size); 1096 1097 leaf = path->nodes[0]; 1098 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1099 btrfs_set_extent_refs(leaf, item, refs); 1100 /* FIXME: get real generation */ 1101 btrfs_set_extent_generation(leaf, item, 0); 1102 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1103 btrfs_set_extent_flags(leaf, item, 1104 BTRFS_EXTENT_FLAG_TREE_BLOCK | 1105 BTRFS_BLOCK_FLAG_FULL_BACKREF); 1106 bi = (struct btrfs_tree_block_info *)(item + 1); 1107 /* FIXME: get first key of the block */ 1108 memzero_extent_buffer(leaf, (unsigned long)bi, sizeof(*bi)); 1109 btrfs_set_tree_block_level(leaf, bi, (int)owner); 1110 } else { 1111 btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA); 1112 } 1113 btrfs_mark_buffer_dirty(leaf); 1114 return 0; 1115 } 1116 #endif 1117 1118 static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset) 1119 { 1120 u32 high_crc = ~(u32)0; 1121 u32 low_crc = ~(u32)0; 1122 __le64 lenum; 1123 1124 lenum = cpu_to_le64(root_objectid); 1125 high_crc = btrfs_crc32c(high_crc, &lenum, sizeof(lenum)); 1126 lenum = cpu_to_le64(owner); 1127 low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); 1128 lenum = cpu_to_le64(offset); 1129 low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); 1130 1131 return ((u64)high_crc << 31) ^ (u64)low_crc; 1132 } 1133 1134 static u64 hash_extent_data_ref_item(struct extent_buffer *leaf, 1135 struct btrfs_extent_data_ref *ref) 1136 { 1137 return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref), 1138 btrfs_extent_data_ref_objectid(leaf, ref), 1139 btrfs_extent_data_ref_offset(leaf, ref)); 1140 } 1141 1142 static int match_extent_data_ref(struct extent_buffer *leaf, 1143 struct btrfs_extent_data_ref *ref, 1144 u64 root_objectid, u64 owner, u64 offset) 1145 { 1146 if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid || 1147 btrfs_extent_data_ref_objectid(leaf, ref) != owner || 1148 btrfs_extent_data_ref_offset(leaf, ref) != offset) 1149 return 0; 1150 return 1; 1151 } 1152 1153 static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, 1154 struct btrfs_root *root, 1155 struct btrfs_path *path, 1156 u64 bytenr, u64 parent, 1157 u64 root_objectid, 1158 u64 owner, u64 offset) 1159 { 1160 struct btrfs_key key; 1161 struct btrfs_extent_data_ref *ref; 1162 struct extent_buffer *leaf; 1163 u32 nritems; 1164 int ret; 1165 int recow; 1166 int err = -ENOENT; 1167 1168 key.objectid = bytenr; 1169 if (parent) { 1170 key.type = BTRFS_SHARED_DATA_REF_KEY; 1171 key.offset = parent; 1172 } else { 1173 key.type = BTRFS_EXTENT_DATA_REF_KEY; 1174 key.offset = hash_extent_data_ref(root_objectid, 1175 owner, offset); 1176 } 1177 again: 1178 recow = 0; 1179 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1180 if (ret < 0) { 1181 err = ret; 1182 goto fail; 1183 } 1184 1185 if (parent) { 1186 if (!ret) 1187 return 0; 1188 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1189 key.type = BTRFS_EXTENT_REF_V0_KEY; 1190 btrfs_release_path(path); 1191 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1192 if (ret < 0) { 1193 err = ret; 1194 goto fail; 1195 } 1196 if (!ret) 1197 return 0; 1198 #endif 1199 goto fail; 1200 } 1201 1202 leaf = path->nodes[0]; 1203 nritems = btrfs_header_nritems(leaf); 1204 while (1) { 1205 if (path->slots[0] >= nritems) { 1206 ret = btrfs_next_leaf(root, path); 1207 if (ret < 0) 1208 err = ret; 1209 if (ret) 1210 goto fail; 1211 1212 leaf = path->nodes[0]; 1213 nritems = btrfs_header_nritems(leaf); 1214 recow = 1; 1215 } 1216 1217 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1218 if (key.objectid != bytenr || 1219 key.type != BTRFS_EXTENT_DATA_REF_KEY) 1220 goto fail; 1221 1222 ref = btrfs_item_ptr(leaf, path->slots[0], 1223 struct btrfs_extent_data_ref); 1224 1225 if (match_extent_data_ref(leaf, ref, root_objectid, 1226 owner, offset)) { 1227 if (recow) { 1228 btrfs_release_path(path); 1229 goto again; 1230 } 1231 err = 0; 1232 break; 1233 } 1234 path->slots[0]++; 1235 } 1236 fail: 1237 return err; 1238 } 1239 1240 static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans, 1241 struct btrfs_root *root, 1242 struct btrfs_path *path, 1243 u64 bytenr, u64 parent, 1244 u64 root_objectid, u64 owner, 1245 u64 offset, int refs_to_add) 1246 { 1247 struct btrfs_key key; 1248 struct extent_buffer *leaf; 1249 u32 size; 1250 u32 num_refs; 1251 int ret; 1252 1253 key.objectid = bytenr; 1254 if (parent) { 1255 key.type = BTRFS_SHARED_DATA_REF_KEY; 1256 key.offset = parent; 1257 size = sizeof(struct btrfs_shared_data_ref); 1258 } else { 1259 key.type = BTRFS_EXTENT_DATA_REF_KEY; 1260 key.offset = hash_extent_data_ref(root_objectid, 1261 owner, offset); 1262 size = sizeof(struct btrfs_extent_data_ref); 1263 } 1264 1265 ret = btrfs_insert_empty_item(trans, root, path, &key, size); 1266 if (ret && ret != -EEXIST) 1267 goto fail; 1268 1269 leaf = path->nodes[0]; 1270 if (parent) { 1271 struct btrfs_shared_data_ref *ref; 1272 ref = btrfs_item_ptr(leaf, path->slots[0], 1273 struct btrfs_shared_data_ref); 1274 if (ret == 0) { 1275 btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add); 1276 } else { 1277 num_refs = btrfs_shared_data_ref_count(leaf, ref); 1278 num_refs += refs_to_add; 1279 btrfs_set_shared_data_ref_count(leaf, ref, num_refs); 1280 } 1281 } else { 1282 struct btrfs_extent_data_ref *ref; 1283 while (ret == -EEXIST) { 1284 ref = btrfs_item_ptr(leaf, path->slots[0], 1285 struct btrfs_extent_data_ref); 1286 if (match_extent_data_ref(leaf, ref, root_objectid, 1287 owner, offset)) 1288 break; 1289 btrfs_release_path(path); 1290 key.offset++; 1291 ret = btrfs_insert_empty_item(trans, root, path, &key, 1292 size); 1293 if (ret && ret != -EEXIST) 1294 goto fail; 1295 1296 leaf = path->nodes[0]; 1297 } 1298 ref = btrfs_item_ptr(leaf, path->slots[0], 1299 struct btrfs_extent_data_ref); 1300 if (ret == 0) { 1301 btrfs_set_extent_data_ref_root(leaf, ref, 1302 root_objectid); 1303 btrfs_set_extent_data_ref_objectid(leaf, ref, owner); 1304 btrfs_set_extent_data_ref_offset(leaf, ref, offset); 1305 btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add); 1306 } else { 1307 num_refs = btrfs_extent_data_ref_count(leaf, ref); 1308 num_refs += refs_to_add; 1309 btrfs_set_extent_data_ref_count(leaf, ref, num_refs); 1310 } 1311 } 1312 btrfs_mark_buffer_dirty(leaf); 1313 ret = 0; 1314 fail: 1315 btrfs_release_path(path); 1316 return ret; 1317 } 1318 1319 static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans, 1320 struct btrfs_root *root, 1321 struct btrfs_path *path, 1322 int refs_to_drop, int *last_ref) 1323 { 1324 struct btrfs_key key; 1325 struct btrfs_extent_data_ref *ref1 = NULL; 1326 struct btrfs_shared_data_ref *ref2 = NULL; 1327 struct extent_buffer *leaf; 1328 u32 num_refs = 0; 1329 int ret = 0; 1330 1331 leaf = path->nodes[0]; 1332 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1333 1334 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { 1335 ref1 = btrfs_item_ptr(leaf, path->slots[0], 1336 struct btrfs_extent_data_ref); 1337 num_refs = btrfs_extent_data_ref_count(leaf, ref1); 1338 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { 1339 ref2 = btrfs_item_ptr(leaf, path->slots[0], 1340 struct btrfs_shared_data_ref); 1341 num_refs = btrfs_shared_data_ref_count(leaf, ref2); 1342 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1343 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) { 1344 struct btrfs_extent_ref_v0 *ref0; 1345 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1346 struct btrfs_extent_ref_v0); 1347 num_refs = btrfs_ref_count_v0(leaf, ref0); 1348 #endif 1349 } else { 1350 BUG(); 1351 } 1352 1353 BUG_ON(num_refs < refs_to_drop); 1354 num_refs -= refs_to_drop; 1355 1356 if (num_refs == 0) { 1357 ret = btrfs_del_item(trans, root, path); 1358 *last_ref = 1; 1359 } else { 1360 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) 1361 btrfs_set_extent_data_ref_count(leaf, ref1, num_refs); 1362 else if (key.type == BTRFS_SHARED_DATA_REF_KEY) 1363 btrfs_set_shared_data_ref_count(leaf, ref2, num_refs); 1364 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1365 else { 1366 struct btrfs_extent_ref_v0 *ref0; 1367 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1368 struct btrfs_extent_ref_v0); 1369 btrfs_set_ref_count_v0(leaf, ref0, num_refs); 1370 } 1371 #endif 1372 btrfs_mark_buffer_dirty(leaf); 1373 } 1374 return ret; 1375 } 1376 1377 static noinline u32 extent_data_ref_count(struct btrfs_path *path, 1378 struct btrfs_extent_inline_ref *iref) 1379 { 1380 struct btrfs_key key; 1381 struct extent_buffer *leaf; 1382 struct btrfs_extent_data_ref *ref1; 1383 struct btrfs_shared_data_ref *ref2; 1384 u32 num_refs = 0; 1385 1386 leaf = path->nodes[0]; 1387 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 1388 if (iref) { 1389 if (btrfs_extent_inline_ref_type(leaf, iref) == 1390 BTRFS_EXTENT_DATA_REF_KEY) { 1391 ref1 = (struct btrfs_extent_data_ref *)(&iref->offset); 1392 num_refs = btrfs_extent_data_ref_count(leaf, ref1); 1393 } else { 1394 ref2 = (struct btrfs_shared_data_ref *)(iref + 1); 1395 num_refs = btrfs_shared_data_ref_count(leaf, ref2); 1396 } 1397 } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { 1398 ref1 = btrfs_item_ptr(leaf, path->slots[0], 1399 struct btrfs_extent_data_ref); 1400 num_refs = btrfs_extent_data_ref_count(leaf, ref1); 1401 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { 1402 ref2 = btrfs_item_ptr(leaf, path->slots[0], 1403 struct btrfs_shared_data_ref); 1404 num_refs = btrfs_shared_data_ref_count(leaf, ref2); 1405 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1406 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) { 1407 struct btrfs_extent_ref_v0 *ref0; 1408 ref0 = btrfs_item_ptr(leaf, path->slots[0], 1409 struct btrfs_extent_ref_v0); 1410 num_refs = btrfs_ref_count_v0(leaf, ref0); 1411 #endif 1412 } else { 1413 WARN_ON(1); 1414 } 1415 return num_refs; 1416 } 1417 1418 static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans, 1419 struct btrfs_root *root, 1420 struct btrfs_path *path, 1421 u64 bytenr, u64 parent, 1422 u64 root_objectid) 1423 { 1424 struct btrfs_key key; 1425 int ret; 1426 1427 key.objectid = bytenr; 1428 if (parent) { 1429 key.type = BTRFS_SHARED_BLOCK_REF_KEY; 1430 key.offset = parent; 1431 } else { 1432 key.type = BTRFS_TREE_BLOCK_REF_KEY; 1433 key.offset = root_objectid; 1434 } 1435 1436 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1437 if (ret > 0) 1438 ret = -ENOENT; 1439 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1440 if (ret == -ENOENT && parent) { 1441 btrfs_release_path(path); 1442 key.type = BTRFS_EXTENT_REF_V0_KEY; 1443 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 1444 if (ret > 0) 1445 ret = -ENOENT; 1446 } 1447 #endif 1448 return ret; 1449 } 1450 1451 static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans, 1452 struct btrfs_root *root, 1453 struct btrfs_path *path, 1454 u64 bytenr, u64 parent, 1455 u64 root_objectid) 1456 { 1457 struct btrfs_key key; 1458 int ret; 1459 1460 key.objectid = bytenr; 1461 if (parent) { 1462 key.type = BTRFS_SHARED_BLOCK_REF_KEY; 1463 key.offset = parent; 1464 } else { 1465 key.type = BTRFS_TREE_BLOCK_REF_KEY; 1466 key.offset = root_objectid; 1467 } 1468 1469 ret = btrfs_insert_empty_item(trans, root, path, &key, 0); 1470 btrfs_release_path(path); 1471 return ret; 1472 } 1473 1474 static inline int extent_ref_type(u64 parent, u64 owner) 1475 { 1476 int type; 1477 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1478 if (parent > 0) 1479 type = BTRFS_SHARED_BLOCK_REF_KEY; 1480 else 1481 type = BTRFS_TREE_BLOCK_REF_KEY; 1482 } else { 1483 if (parent > 0) 1484 type = BTRFS_SHARED_DATA_REF_KEY; 1485 else 1486 type = BTRFS_EXTENT_DATA_REF_KEY; 1487 } 1488 return type; 1489 } 1490 1491 static int find_next_key(struct btrfs_path *path, int level, 1492 struct btrfs_key *key) 1493 1494 { 1495 for (; level < BTRFS_MAX_LEVEL; level++) { 1496 if (!path->nodes[level]) 1497 break; 1498 if (path->slots[level] + 1 >= 1499 btrfs_header_nritems(path->nodes[level])) 1500 continue; 1501 if (level == 0) 1502 btrfs_item_key_to_cpu(path->nodes[level], key, 1503 path->slots[level] + 1); 1504 else 1505 btrfs_node_key_to_cpu(path->nodes[level], key, 1506 path->slots[level] + 1); 1507 return 0; 1508 } 1509 return 1; 1510 } 1511 1512 /* 1513 * look for inline back ref. if back ref is found, *ref_ret is set 1514 * to the address of inline back ref, and 0 is returned. 1515 * 1516 * if back ref isn't found, *ref_ret is set to the address where it 1517 * should be inserted, and -ENOENT is returned. 1518 * 1519 * if insert is true and there are too many inline back refs, the path 1520 * points to the extent item, and -EAGAIN is returned. 1521 * 1522 * NOTE: inline back refs are ordered in the same way that back ref 1523 * items in the tree are ordered. 1524 */ 1525 static noinline_for_stack 1526 int lookup_inline_extent_backref(struct btrfs_trans_handle *trans, 1527 struct btrfs_root *root, 1528 struct btrfs_path *path, 1529 struct btrfs_extent_inline_ref **ref_ret, 1530 u64 bytenr, u64 num_bytes, 1531 u64 parent, u64 root_objectid, 1532 u64 owner, u64 offset, int insert) 1533 { 1534 struct btrfs_fs_info *fs_info = root->fs_info; 1535 struct btrfs_key key; 1536 struct extent_buffer *leaf; 1537 struct btrfs_extent_item *ei; 1538 struct btrfs_extent_inline_ref *iref; 1539 u64 flags; 1540 u64 item_size; 1541 unsigned long ptr; 1542 unsigned long end; 1543 int extra_size; 1544 int type; 1545 int want; 1546 int ret; 1547 int err = 0; 1548 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); 1549 1550 key.objectid = bytenr; 1551 key.type = BTRFS_EXTENT_ITEM_KEY; 1552 key.offset = num_bytes; 1553 1554 want = extent_ref_type(parent, owner); 1555 if (insert) { 1556 extra_size = btrfs_extent_inline_ref_size(want); 1557 path->keep_locks = 1; 1558 } else 1559 extra_size = -1; 1560 1561 /* 1562 * Owner is our parent level, so we can just add one to get the level 1563 * for the block we are interested in. 1564 */ 1565 if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) { 1566 key.type = BTRFS_METADATA_ITEM_KEY; 1567 key.offset = owner; 1568 } 1569 1570 again: 1571 ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1); 1572 if (ret < 0) { 1573 err = ret; 1574 goto out; 1575 } 1576 1577 /* 1578 * We may be a newly converted file system which still has the old fat 1579 * extent entries for metadata, so try and see if we have one of those. 1580 */ 1581 if (ret > 0 && skinny_metadata) { 1582 skinny_metadata = false; 1583 if (path->slots[0]) { 1584 path->slots[0]--; 1585 btrfs_item_key_to_cpu(path->nodes[0], &key, 1586 path->slots[0]); 1587 if (key.objectid == bytenr && 1588 key.type == BTRFS_EXTENT_ITEM_KEY && 1589 key.offset == num_bytes) 1590 ret = 0; 1591 } 1592 if (ret) { 1593 key.objectid = bytenr; 1594 key.type = BTRFS_EXTENT_ITEM_KEY; 1595 key.offset = num_bytes; 1596 btrfs_release_path(path); 1597 goto again; 1598 } 1599 } 1600 1601 if (ret && !insert) { 1602 err = -ENOENT; 1603 goto out; 1604 } else if (WARN_ON(ret)) { 1605 err = -EIO; 1606 goto out; 1607 } 1608 1609 leaf = path->nodes[0]; 1610 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 1611 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 1612 if (item_size < sizeof(*ei)) { 1613 if (!insert) { 1614 err = -ENOENT; 1615 goto out; 1616 } 1617 ret = convert_extent_item_v0(trans, root, path, owner, 1618 extra_size); 1619 if (ret < 0) { 1620 err = ret; 1621 goto out; 1622 } 1623 leaf = path->nodes[0]; 1624 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 1625 } 1626 #endif 1627 BUG_ON(item_size < sizeof(*ei)); 1628 1629 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1630 flags = btrfs_extent_flags(leaf, ei); 1631 1632 ptr = (unsigned long)(ei + 1); 1633 end = (unsigned long)ei + item_size; 1634 1635 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) { 1636 ptr += sizeof(struct btrfs_tree_block_info); 1637 BUG_ON(ptr > end); 1638 } 1639 1640 err = -ENOENT; 1641 while (1) { 1642 if (ptr >= end) { 1643 WARN_ON(ptr > end); 1644 break; 1645 } 1646 iref = (struct btrfs_extent_inline_ref *)ptr; 1647 type = btrfs_extent_inline_ref_type(leaf, iref); 1648 if (want < type) 1649 break; 1650 if (want > type) { 1651 ptr += btrfs_extent_inline_ref_size(type); 1652 continue; 1653 } 1654 1655 if (type == BTRFS_EXTENT_DATA_REF_KEY) { 1656 struct btrfs_extent_data_ref *dref; 1657 dref = (struct btrfs_extent_data_ref *)(&iref->offset); 1658 if (match_extent_data_ref(leaf, dref, root_objectid, 1659 owner, offset)) { 1660 err = 0; 1661 break; 1662 } 1663 if (hash_extent_data_ref_item(leaf, dref) < 1664 hash_extent_data_ref(root_objectid, owner, offset)) 1665 break; 1666 } else { 1667 u64 ref_offset; 1668 ref_offset = btrfs_extent_inline_ref_offset(leaf, iref); 1669 if (parent > 0) { 1670 if (parent == ref_offset) { 1671 err = 0; 1672 break; 1673 } 1674 if (ref_offset < parent) 1675 break; 1676 } else { 1677 if (root_objectid == ref_offset) { 1678 err = 0; 1679 break; 1680 } 1681 if (ref_offset < root_objectid) 1682 break; 1683 } 1684 } 1685 ptr += btrfs_extent_inline_ref_size(type); 1686 } 1687 if (err == -ENOENT && insert) { 1688 if (item_size + extra_size >= 1689 BTRFS_MAX_EXTENT_ITEM_SIZE(root)) { 1690 err = -EAGAIN; 1691 goto out; 1692 } 1693 /* 1694 * To add new inline back ref, we have to make sure 1695 * there is no corresponding back ref item. 1696 * For simplicity, we just do not add new inline back 1697 * ref if there is any kind of item for this block 1698 */ 1699 if (find_next_key(path, 0, &key) == 0 && 1700 key.objectid == bytenr && 1701 key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) { 1702 err = -EAGAIN; 1703 goto out; 1704 } 1705 } 1706 *ref_ret = (struct btrfs_extent_inline_ref *)ptr; 1707 out: 1708 if (insert) { 1709 path->keep_locks = 0; 1710 btrfs_unlock_up_safe(path, 1); 1711 } 1712 return err; 1713 } 1714 1715 /* 1716 * helper to add new inline back ref 1717 */ 1718 static noinline_for_stack 1719 void setup_inline_extent_backref(struct btrfs_root *root, 1720 struct btrfs_path *path, 1721 struct btrfs_extent_inline_ref *iref, 1722 u64 parent, u64 root_objectid, 1723 u64 owner, u64 offset, int refs_to_add, 1724 struct btrfs_delayed_extent_op *extent_op) 1725 { 1726 struct extent_buffer *leaf; 1727 struct btrfs_extent_item *ei; 1728 unsigned long ptr; 1729 unsigned long end; 1730 unsigned long item_offset; 1731 u64 refs; 1732 int size; 1733 int type; 1734 1735 leaf = path->nodes[0]; 1736 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1737 item_offset = (unsigned long)iref - (unsigned long)ei; 1738 1739 type = extent_ref_type(parent, owner); 1740 size = btrfs_extent_inline_ref_size(type); 1741 1742 btrfs_extend_item(root->fs_info, path, size); 1743 1744 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1745 refs = btrfs_extent_refs(leaf, ei); 1746 refs += refs_to_add; 1747 btrfs_set_extent_refs(leaf, ei, refs); 1748 if (extent_op) 1749 __run_delayed_extent_op(extent_op, leaf, ei); 1750 1751 ptr = (unsigned long)ei + item_offset; 1752 end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]); 1753 if (ptr < end - size) 1754 memmove_extent_buffer(leaf, ptr + size, ptr, 1755 end - size - ptr); 1756 1757 iref = (struct btrfs_extent_inline_ref *)ptr; 1758 btrfs_set_extent_inline_ref_type(leaf, iref, type); 1759 if (type == BTRFS_EXTENT_DATA_REF_KEY) { 1760 struct btrfs_extent_data_ref *dref; 1761 dref = (struct btrfs_extent_data_ref *)(&iref->offset); 1762 btrfs_set_extent_data_ref_root(leaf, dref, root_objectid); 1763 btrfs_set_extent_data_ref_objectid(leaf, dref, owner); 1764 btrfs_set_extent_data_ref_offset(leaf, dref, offset); 1765 btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add); 1766 } else if (type == BTRFS_SHARED_DATA_REF_KEY) { 1767 struct btrfs_shared_data_ref *sref; 1768 sref = (struct btrfs_shared_data_ref *)(iref + 1); 1769 btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add); 1770 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 1771 } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) { 1772 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 1773 } else { 1774 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid); 1775 } 1776 btrfs_mark_buffer_dirty(leaf); 1777 } 1778 1779 static int lookup_extent_backref(struct btrfs_trans_handle *trans, 1780 struct btrfs_root *root, 1781 struct btrfs_path *path, 1782 struct btrfs_extent_inline_ref **ref_ret, 1783 u64 bytenr, u64 num_bytes, u64 parent, 1784 u64 root_objectid, u64 owner, u64 offset) 1785 { 1786 int ret; 1787 1788 ret = lookup_inline_extent_backref(trans, root, path, ref_ret, 1789 bytenr, num_bytes, parent, 1790 root_objectid, owner, offset, 0); 1791 if (ret != -ENOENT) 1792 return ret; 1793 1794 btrfs_release_path(path); 1795 *ref_ret = NULL; 1796 1797 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1798 ret = lookup_tree_block_ref(trans, root, path, bytenr, parent, 1799 root_objectid); 1800 } else { 1801 ret = lookup_extent_data_ref(trans, root, path, bytenr, parent, 1802 root_objectid, owner, offset); 1803 } 1804 return ret; 1805 } 1806 1807 /* 1808 * helper to update/remove inline back ref 1809 */ 1810 static noinline_for_stack 1811 void update_inline_extent_backref(struct btrfs_root *root, 1812 struct btrfs_path *path, 1813 struct btrfs_extent_inline_ref *iref, 1814 int refs_to_mod, 1815 struct btrfs_delayed_extent_op *extent_op, 1816 int *last_ref) 1817 { 1818 struct extent_buffer *leaf; 1819 struct btrfs_extent_item *ei; 1820 struct btrfs_extent_data_ref *dref = NULL; 1821 struct btrfs_shared_data_ref *sref = NULL; 1822 unsigned long ptr; 1823 unsigned long end; 1824 u32 item_size; 1825 int size; 1826 int type; 1827 u64 refs; 1828 1829 leaf = path->nodes[0]; 1830 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 1831 refs = btrfs_extent_refs(leaf, ei); 1832 WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0); 1833 refs += refs_to_mod; 1834 btrfs_set_extent_refs(leaf, ei, refs); 1835 if (extent_op) 1836 __run_delayed_extent_op(extent_op, leaf, ei); 1837 1838 type = btrfs_extent_inline_ref_type(leaf, iref); 1839 1840 if (type == BTRFS_EXTENT_DATA_REF_KEY) { 1841 dref = (struct btrfs_extent_data_ref *)(&iref->offset); 1842 refs = btrfs_extent_data_ref_count(leaf, dref); 1843 } else if (type == BTRFS_SHARED_DATA_REF_KEY) { 1844 sref = (struct btrfs_shared_data_ref *)(iref + 1); 1845 refs = btrfs_shared_data_ref_count(leaf, sref); 1846 } else { 1847 refs = 1; 1848 BUG_ON(refs_to_mod != -1); 1849 } 1850 1851 BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod); 1852 refs += refs_to_mod; 1853 1854 if (refs > 0) { 1855 if (type == BTRFS_EXTENT_DATA_REF_KEY) 1856 btrfs_set_extent_data_ref_count(leaf, dref, refs); 1857 else 1858 btrfs_set_shared_data_ref_count(leaf, sref, refs); 1859 } else { 1860 *last_ref = 1; 1861 size = btrfs_extent_inline_ref_size(type); 1862 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 1863 ptr = (unsigned long)iref; 1864 end = (unsigned long)ei + item_size; 1865 if (ptr + size < end) 1866 memmove_extent_buffer(leaf, ptr, ptr + size, 1867 end - ptr - size); 1868 item_size -= size; 1869 btrfs_truncate_item(root->fs_info, path, item_size, 1); 1870 } 1871 btrfs_mark_buffer_dirty(leaf); 1872 } 1873 1874 static noinline_for_stack 1875 int insert_inline_extent_backref(struct btrfs_trans_handle *trans, 1876 struct btrfs_root *root, 1877 struct btrfs_path *path, 1878 u64 bytenr, u64 num_bytes, u64 parent, 1879 u64 root_objectid, u64 owner, 1880 u64 offset, int refs_to_add, 1881 struct btrfs_delayed_extent_op *extent_op) 1882 { 1883 struct btrfs_extent_inline_ref *iref; 1884 int ret; 1885 1886 ret = lookup_inline_extent_backref(trans, root, path, &iref, 1887 bytenr, num_bytes, parent, 1888 root_objectid, owner, offset, 1); 1889 if (ret == 0) { 1890 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID); 1891 update_inline_extent_backref(root, path, iref, 1892 refs_to_add, extent_op, NULL); 1893 } else if (ret == -ENOENT) { 1894 setup_inline_extent_backref(root, path, iref, parent, 1895 root_objectid, owner, offset, 1896 refs_to_add, extent_op); 1897 ret = 0; 1898 } 1899 return ret; 1900 } 1901 1902 static int insert_extent_backref(struct btrfs_trans_handle *trans, 1903 struct btrfs_root *root, 1904 struct btrfs_path *path, 1905 u64 bytenr, u64 parent, u64 root_objectid, 1906 u64 owner, u64 offset, int refs_to_add) 1907 { 1908 int ret; 1909 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 1910 BUG_ON(refs_to_add != 1); 1911 ret = insert_tree_block_ref(trans, root, path, bytenr, 1912 parent, root_objectid); 1913 } else { 1914 ret = insert_extent_data_ref(trans, root, path, bytenr, 1915 parent, root_objectid, 1916 owner, offset, refs_to_add); 1917 } 1918 return ret; 1919 } 1920 1921 static int remove_extent_backref(struct btrfs_trans_handle *trans, 1922 struct btrfs_root *root, 1923 struct btrfs_path *path, 1924 struct btrfs_extent_inline_ref *iref, 1925 int refs_to_drop, int is_data, int *last_ref) 1926 { 1927 int ret = 0; 1928 1929 BUG_ON(!is_data && refs_to_drop != 1); 1930 if (iref) { 1931 update_inline_extent_backref(root, path, iref, 1932 -refs_to_drop, NULL, last_ref); 1933 } else if (is_data) { 1934 ret = remove_extent_data_ref(trans, root, path, refs_to_drop, 1935 last_ref); 1936 } else { 1937 *last_ref = 1; 1938 ret = btrfs_del_item(trans, root, path); 1939 } 1940 return ret; 1941 } 1942 1943 #define in_range(b, first, len) ((b) >= (first) && (b) < (first) + (len)) 1944 static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len, 1945 u64 *discarded_bytes) 1946 { 1947 int j, ret = 0; 1948 u64 bytes_left, end; 1949 u64 aligned_start = ALIGN(start, 1 << 9); 1950 1951 if (WARN_ON(start != aligned_start)) { 1952 len -= aligned_start - start; 1953 len = round_down(len, 1 << 9); 1954 start = aligned_start; 1955 } 1956 1957 *discarded_bytes = 0; 1958 1959 if (!len) 1960 return 0; 1961 1962 end = start + len; 1963 bytes_left = len; 1964 1965 /* Skip any superblocks on this device. */ 1966 for (j = 0; j < BTRFS_SUPER_MIRROR_MAX; j++) { 1967 u64 sb_start = btrfs_sb_offset(j); 1968 u64 sb_end = sb_start + BTRFS_SUPER_INFO_SIZE; 1969 u64 size = sb_start - start; 1970 1971 if (!in_range(sb_start, start, bytes_left) && 1972 !in_range(sb_end, start, bytes_left) && 1973 !in_range(start, sb_start, BTRFS_SUPER_INFO_SIZE)) 1974 continue; 1975 1976 /* 1977 * Superblock spans beginning of range. Adjust start and 1978 * try again. 1979 */ 1980 if (sb_start <= start) { 1981 start += sb_end - start; 1982 if (start > end) { 1983 bytes_left = 0; 1984 break; 1985 } 1986 bytes_left = end - start; 1987 continue; 1988 } 1989 1990 if (size) { 1991 ret = blkdev_issue_discard(bdev, start >> 9, size >> 9, 1992 GFP_NOFS, 0); 1993 if (!ret) 1994 *discarded_bytes += size; 1995 else if (ret != -EOPNOTSUPP) 1996 return ret; 1997 } 1998 1999 start = sb_end; 2000 if (start > end) { 2001 bytes_left = 0; 2002 break; 2003 } 2004 bytes_left = end - start; 2005 } 2006 2007 if (bytes_left) { 2008 ret = blkdev_issue_discard(bdev, start >> 9, bytes_left >> 9, 2009 GFP_NOFS, 0); 2010 if (!ret) 2011 *discarded_bytes += bytes_left; 2012 } 2013 return ret; 2014 } 2015 2016 int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, 2017 u64 num_bytes, u64 *actual_bytes) 2018 { 2019 int ret; 2020 u64 discarded_bytes = 0; 2021 struct btrfs_bio *bbio = NULL; 2022 2023 2024 /* 2025 * Avoid races with device replace and make sure our bbio has devices 2026 * associated to its stripes that don't go away while we are discarding. 2027 */ 2028 btrfs_bio_counter_inc_blocked(fs_info); 2029 /* Tell the block device(s) that the sectors can be discarded */ 2030 ret = btrfs_map_block(fs_info, BTRFS_MAP_DISCARD, bytenr, &num_bytes, 2031 &bbio, 0); 2032 /* Error condition is -ENOMEM */ 2033 if (!ret) { 2034 struct btrfs_bio_stripe *stripe = bbio->stripes; 2035 int i; 2036 2037 2038 for (i = 0; i < bbio->num_stripes; i++, stripe++) { 2039 u64 bytes; 2040 if (!stripe->dev->can_discard) 2041 continue; 2042 2043 ret = btrfs_issue_discard(stripe->dev->bdev, 2044 stripe->physical, 2045 stripe->length, 2046 &bytes); 2047 if (!ret) 2048 discarded_bytes += bytes; 2049 else if (ret != -EOPNOTSUPP) 2050 break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */ 2051 2052 /* 2053 * Just in case we get back EOPNOTSUPP for some reason, 2054 * just ignore the return value so we don't screw up 2055 * people calling discard_extent. 2056 */ 2057 ret = 0; 2058 } 2059 btrfs_put_bbio(bbio); 2060 } 2061 btrfs_bio_counter_dec(fs_info); 2062 2063 if (actual_bytes) 2064 *actual_bytes = discarded_bytes; 2065 2066 2067 if (ret == -EOPNOTSUPP) 2068 ret = 0; 2069 return ret; 2070 } 2071 2072 /* Can return -ENOMEM */ 2073 int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, 2074 struct btrfs_fs_info *fs_info, 2075 u64 bytenr, u64 num_bytes, u64 parent, 2076 u64 root_objectid, u64 owner, u64 offset) 2077 { 2078 int ret; 2079 2080 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID && 2081 root_objectid == BTRFS_TREE_LOG_OBJECTID); 2082 2083 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 2084 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr, 2085 num_bytes, 2086 parent, root_objectid, (int)owner, 2087 BTRFS_ADD_DELAYED_REF, NULL); 2088 } else { 2089 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr, 2090 num_bytes, parent, root_objectid, 2091 owner, offset, 0, 2092 BTRFS_ADD_DELAYED_REF, NULL); 2093 } 2094 return ret; 2095 } 2096 2097 static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, 2098 struct btrfs_fs_info *fs_info, 2099 struct btrfs_delayed_ref_node *node, 2100 u64 parent, u64 root_objectid, 2101 u64 owner, u64 offset, int refs_to_add, 2102 struct btrfs_delayed_extent_op *extent_op) 2103 { 2104 struct btrfs_path *path; 2105 struct extent_buffer *leaf; 2106 struct btrfs_extent_item *item; 2107 struct btrfs_key key; 2108 u64 bytenr = node->bytenr; 2109 u64 num_bytes = node->num_bytes; 2110 u64 refs; 2111 int ret; 2112 2113 path = btrfs_alloc_path(); 2114 if (!path) 2115 return -ENOMEM; 2116 2117 path->reada = READA_FORWARD; 2118 path->leave_spinning = 1; 2119 /* this will setup the path even if it fails to insert the back ref */ 2120 ret = insert_inline_extent_backref(trans, fs_info->extent_root, path, 2121 bytenr, num_bytes, parent, 2122 root_objectid, owner, offset, 2123 refs_to_add, extent_op); 2124 if ((ret < 0 && ret != -EAGAIN) || !ret) 2125 goto out; 2126 2127 /* 2128 * Ok we had -EAGAIN which means we didn't have space to insert and 2129 * inline extent ref, so just update the reference count and add a 2130 * normal backref. 2131 */ 2132 leaf = path->nodes[0]; 2133 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 2134 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 2135 refs = btrfs_extent_refs(leaf, item); 2136 btrfs_set_extent_refs(leaf, item, refs + refs_to_add); 2137 if (extent_op) 2138 __run_delayed_extent_op(extent_op, leaf, item); 2139 2140 btrfs_mark_buffer_dirty(leaf); 2141 btrfs_release_path(path); 2142 2143 path->reada = READA_FORWARD; 2144 path->leave_spinning = 1; 2145 /* now insert the actual backref */ 2146 ret = insert_extent_backref(trans, fs_info->extent_root, 2147 path, bytenr, parent, root_objectid, 2148 owner, offset, refs_to_add); 2149 if (ret) 2150 btrfs_abort_transaction(trans, ret); 2151 out: 2152 btrfs_free_path(path); 2153 return ret; 2154 } 2155 2156 static int run_delayed_data_ref(struct btrfs_trans_handle *trans, 2157 struct btrfs_fs_info *fs_info, 2158 struct btrfs_delayed_ref_node *node, 2159 struct btrfs_delayed_extent_op *extent_op, 2160 int insert_reserved) 2161 { 2162 int ret = 0; 2163 struct btrfs_delayed_data_ref *ref; 2164 struct btrfs_key ins; 2165 u64 parent = 0; 2166 u64 ref_root = 0; 2167 u64 flags = 0; 2168 2169 ins.objectid = node->bytenr; 2170 ins.offset = node->num_bytes; 2171 ins.type = BTRFS_EXTENT_ITEM_KEY; 2172 2173 ref = btrfs_delayed_node_to_data_ref(node); 2174 trace_run_delayed_data_ref(fs_info, node, ref, node->action); 2175 2176 if (node->type == BTRFS_SHARED_DATA_REF_KEY) 2177 parent = ref->parent; 2178 ref_root = ref->root; 2179 2180 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { 2181 if (extent_op) 2182 flags |= extent_op->flags_to_set; 2183 ret = alloc_reserved_file_extent(trans, fs_info, 2184 parent, ref_root, flags, 2185 ref->objectid, ref->offset, 2186 &ins, node->ref_mod); 2187 } else if (node->action == BTRFS_ADD_DELAYED_REF) { 2188 ret = __btrfs_inc_extent_ref(trans, fs_info, node, parent, 2189 ref_root, ref->objectid, 2190 ref->offset, node->ref_mod, 2191 extent_op); 2192 } else if (node->action == BTRFS_DROP_DELAYED_REF) { 2193 ret = __btrfs_free_extent(trans, fs_info, node, parent, 2194 ref_root, ref->objectid, 2195 ref->offset, node->ref_mod, 2196 extent_op); 2197 } else { 2198 BUG(); 2199 } 2200 return ret; 2201 } 2202 2203 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, 2204 struct extent_buffer *leaf, 2205 struct btrfs_extent_item *ei) 2206 { 2207 u64 flags = btrfs_extent_flags(leaf, ei); 2208 if (extent_op->update_flags) { 2209 flags |= extent_op->flags_to_set; 2210 btrfs_set_extent_flags(leaf, ei, flags); 2211 } 2212 2213 if (extent_op->update_key) { 2214 struct btrfs_tree_block_info *bi; 2215 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)); 2216 bi = (struct btrfs_tree_block_info *)(ei + 1); 2217 btrfs_set_tree_block_key(leaf, bi, &extent_op->key); 2218 } 2219 } 2220 2221 static int run_delayed_extent_op(struct btrfs_trans_handle *trans, 2222 struct btrfs_fs_info *fs_info, 2223 struct btrfs_delayed_ref_node *node, 2224 struct btrfs_delayed_extent_op *extent_op) 2225 { 2226 struct btrfs_key key; 2227 struct btrfs_path *path; 2228 struct btrfs_extent_item *ei; 2229 struct extent_buffer *leaf; 2230 u32 item_size; 2231 int ret; 2232 int err = 0; 2233 int metadata = !extent_op->is_data; 2234 2235 if (trans->aborted) 2236 return 0; 2237 2238 if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) 2239 metadata = 0; 2240 2241 path = btrfs_alloc_path(); 2242 if (!path) 2243 return -ENOMEM; 2244 2245 key.objectid = node->bytenr; 2246 2247 if (metadata) { 2248 key.type = BTRFS_METADATA_ITEM_KEY; 2249 key.offset = extent_op->level; 2250 } else { 2251 key.type = BTRFS_EXTENT_ITEM_KEY; 2252 key.offset = node->num_bytes; 2253 } 2254 2255 again: 2256 path->reada = READA_FORWARD; 2257 path->leave_spinning = 1; 2258 ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 1); 2259 if (ret < 0) { 2260 err = ret; 2261 goto out; 2262 } 2263 if (ret > 0) { 2264 if (metadata) { 2265 if (path->slots[0] > 0) { 2266 path->slots[0]--; 2267 btrfs_item_key_to_cpu(path->nodes[0], &key, 2268 path->slots[0]); 2269 if (key.objectid == node->bytenr && 2270 key.type == BTRFS_EXTENT_ITEM_KEY && 2271 key.offset == node->num_bytes) 2272 ret = 0; 2273 } 2274 if (ret > 0) { 2275 btrfs_release_path(path); 2276 metadata = 0; 2277 2278 key.objectid = node->bytenr; 2279 key.offset = node->num_bytes; 2280 key.type = BTRFS_EXTENT_ITEM_KEY; 2281 goto again; 2282 } 2283 } else { 2284 err = -EIO; 2285 goto out; 2286 } 2287 } 2288 2289 leaf = path->nodes[0]; 2290 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 2291 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 2292 if (item_size < sizeof(*ei)) { 2293 ret = convert_extent_item_v0(trans, fs_info->extent_root, 2294 path, (u64)-1, 0); 2295 if (ret < 0) { 2296 err = ret; 2297 goto out; 2298 } 2299 leaf = path->nodes[0]; 2300 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 2301 } 2302 #endif 2303 BUG_ON(item_size < sizeof(*ei)); 2304 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 2305 __run_delayed_extent_op(extent_op, leaf, ei); 2306 2307 btrfs_mark_buffer_dirty(leaf); 2308 out: 2309 btrfs_free_path(path); 2310 return err; 2311 } 2312 2313 static int run_delayed_tree_ref(struct btrfs_trans_handle *trans, 2314 struct btrfs_fs_info *fs_info, 2315 struct btrfs_delayed_ref_node *node, 2316 struct btrfs_delayed_extent_op *extent_op, 2317 int insert_reserved) 2318 { 2319 int ret = 0; 2320 struct btrfs_delayed_tree_ref *ref; 2321 struct btrfs_key ins; 2322 u64 parent = 0; 2323 u64 ref_root = 0; 2324 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); 2325 2326 ref = btrfs_delayed_node_to_tree_ref(node); 2327 trace_run_delayed_tree_ref(fs_info, node, ref, node->action); 2328 2329 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) 2330 parent = ref->parent; 2331 ref_root = ref->root; 2332 2333 ins.objectid = node->bytenr; 2334 if (skinny_metadata) { 2335 ins.offset = ref->level; 2336 ins.type = BTRFS_METADATA_ITEM_KEY; 2337 } else { 2338 ins.offset = node->num_bytes; 2339 ins.type = BTRFS_EXTENT_ITEM_KEY; 2340 } 2341 2342 if (node->ref_mod != 1) { 2343 btrfs_err(fs_info, 2344 "btree block(%llu) has %d references rather than 1: action %d ref_root %llu parent %llu", 2345 node->bytenr, node->ref_mod, node->action, ref_root, 2346 parent); 2347 return -EIO; 2348 } 2349 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { 2350 BUG_ON(!extent_op || !extent_op->update_flags); 2351 ret = alloc_reserved_tree_block(trans, fs_info, 2352 parent, ref_root, 2353 extent_op->flags_to_set, 2354 &extent_op->key, 2355 ref->level, &ins); 2356 } else if (node->action == BTRFS_ADD_DELAYED_REF) { 2357 ret = __btrfs_inc_extent_ref(trans, fs_info, node, 2358 parent, ref_root, 2359 ref->level, 0, 1, 2360 extent_op); 2361 } else if (node->action == BTRFS_DROP_DELAYED_REF) { 2362 ret = __btrfs_free_extent(trans, fs_info, node, 2363 parent, ref_root, 2364 ref->level, 0, 1, extent_op); 2365 } else { 2366 BUG(); 2367 } 2368 return ret; 2369 } 2370 2371 /* helper function to actually process a single delayed ref entry */ 2372 static int run_one_delayed_ref(struct btrfs_trans_handle *trans, 2373 struct btrfs_fs_info *fs_info, 2374 struct btrfs_delayed_ref_node *node, 2375 struct btrfs_delayed_extent_op *extent_op, 2376 int insert_reserved) 2377 { 2378 int ret = 0; 2379 2380 if (trans->aborted) { 2381 if (insert_reserved) 2382 btrfs_pin_extent(fs_info, node->bytenr, 2383 node->num_bytes, 1); 2384 return 0; 2385 } 2386 2387 if (btrfs_delayed_ref_is_head(node)) { 2388 struct btrfs_delayed_ref_head *head; 2389 /* 2390 * we've hit the end of the chain and we were supposed 2391 * to insert this extent into the tree. But, it got 2392 * deleted before we ever needed to insert it, so all 2393 * we have to do is clean up the accounting 2394 */ 2395 BUG_ON(extent_op); 2396 head = btrfs_delayed_node_to_head(node); 2397 trace_run_delayed_ref_head(fs_info, node, head, node->action); 2398 2399 if (insert_reserved) { 2400 btrfs_pin_extent(fs_info, node->bytenr, 2401 node->num_bytes, 1); 2402 if (head->is_data) { 2403 ret = btrfs_del_csums(trans, fs_info, 2404 node->bytenr, 2405 node->num_bytes); 2406 } 2407 } 2408 2409 /* Also free its reserved qgroup space */ 2410 btrfs_qgroup_free_delayed_ref(fs_info, head->qgroup_ref_root, 2411 head->qgroup_reserved); 2412 return ret; 2413 } 2414 2415 if (node->type == BTRFS_TREE_BLOCK_REF_KEY || 2416 node->type == BTRFS_SHARED_BLOCK_REF_KEY) 2417 ret = run_delayed_tree_ref(trans, fs_info, node, extent_op, 2418 insert_reserved); 2419 else if (node->type == BTRFS_EXTENT_DATA_REF_KEY || 2420 node->type == BTRFS_SHARED_DATA_REF_KEY) 2421 ret = run_delayed_data_ref(trans, fs_info, node, extent_op, 2422 insert_reserved); 2423 else 2424 BUG(); 2425 return ret; 2426 } 2427 2428 static inline struct btrfs_delayed_ref_node * 2429 select_delayed_ref(struct btrfs_delayed_ref_head *head) 2430 { 2431 struct btrfs_delayed_ref_node *ref; 2432 2433 if (list_empty(&head->ref_list)) 2434 return NULL; 2435 2436 /* 2437 * Select a delayed ref of type BTRFS_ADD_DELAYED_REF first. 2438 * This is to prevent a ref count from going down to zero, which deletes 2439 * the extent item from the extent tree, when there still are references 2440 * to add, which would fail because they would not find the extent item. 2441 */ 2442 if (!list_empty(&head->ref_add_list)) 2443 return list_first_entry(&head->ref_add_list, 2444 struct btrfs_delayed_ref_node, add_list); 2445 2446 ref = list_first_entry(&head->ref_list, struct btrfs_delayed_ref_node, 2447 list); 2448 ASSERT(list_empty(&ref->add_list)); 2449 return ref; 2450 } 2451 2452 /* 2453 * Returns 0 on success or if called with an already aborted transaction. 2454 * Returns -ENOMEM or -EIO on failure and will abort the transaction. 2455 */ 2456 static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, 2457 struct btrfs_fs_info *fs_info, 2458 unsigned long nr) 2459 { 2460 struct btrfs_delayed_ref_root *delayed_refs; 2461 struct btrfs_delayed_ref_node *ref; 2462 struct btrfs_delayed_ref_head *locked_ref = NULL; 2463 struct btrfs_delayed_extent_op *extent_op; 2464 ktime_t start = ktime_get(); 2465 int ret; 2466 unsigned long count = 0; 2467 unsigned long actual_count = 0; 2468 int must_insert_reserved = 0; 2469 2470 delayed_refs = &trans->transaction->delayed_refs; 2471 while (1) { 2472 if (!locked_ref) { 2473 if (count >= nr) 2474 break; 2475 2476 spin_lock(&delayed_refs->lock); 2477 locked_ref = btrfs_select_ref_head(trans); 2478 if (!locked_ref) { 2479 spin_unlock(&delayed_refs->lock); 2480 break; 2481 } 2482 2483 /* grab the lock that says we are going to process 2484 * all the refs for this head */ 2485 ret = btrfs_delayed_ref_lock(trans, locked_ref); 2486 spin_unlock(&delayed_refs->lock); 2487 /* 2488 * we may have dropped the spin lock to get the head 2489 * mutex lock, and that might have given someone else 2490 * time to free the head. If that's true, it has been 2491 * removed from our list and we can move on. 2492 */ 2493 if (ret == -EAGAIN) { 2494 locked_ref = NULL; 2495 count++; 2496 continue; 2497 } 2498 } 2499 2500 /* 2501 * We need to try and merge add/drops of the same ref since we 2502 * can run into issues with relocate dropping the implicit ref 2503 * and then it being added back again before the drop can 2504 * finish. If we merged anything we need to re-loop so we can 2505 * get a good ref. 2506 * Or we can get node references of the same type that weren't 2507 * merged when created due to bumps in the tree mod seq, and 2508 * we need to merge them to prevent adding an inline extent 2509 * backref before dropping it (triggering a BUG_ON at 2510 * insert_inline_extent_backref()). 2511 */ 2512 spin_lock(&locked_ref->lock); 2513 btrfs_merge_delayed_refs(trans, fs_info, delayed_refs, 2514 locked_ref); 2515 2516 /* 2517 * locked_ref is the head node, so we have to go one 2518 * node back for any delayed ref updates 2519 */ 2520 ref = select_delayed_ref(locked_ref); 2521 2522 if (ref && ref->seq && 2523 btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) { 2524 spin_unlock(&locked_ref->lock); 2525 btrfs_delayed_ref_unlock(locked_ref); 2526 spin_lock(&delayed_refs->lock); 2527 locked_ref->processing = 0; 2528 delayed_refs->num_heads_ready++; 2529 spin_unlock(&delayed_refs->lock); 2530 locked_ref = NULL; 2531 cond_resched(); 2532 count++; 2533 continue; 2534 } 2535 2536 /* 2537 * record the must insert reserved flag before we 2538 * drop the spin lock. 2539 */ 2540 must_insert_reserved = locked_ref->must_insert_reserved; 2541 locked_ref->must_insert_reserved = 0; 2542 2543 extent_op = locked_ref->extent_op; 2544 locked_ref->extent_op = NULL; 2545 2546 if (!ref) { 2547 2548 2549 /* All delayed refs have been processed, Go ahead 2550 * and send the head node to run_one_delayed_ref, 2551 * so that any accounting fixes can happen 2552 */ 2553 ref = &locked_ref->node; 2554 2555 if (extent_op && must_insert_reserved) { 2556 btrfs_free_delayed_extent_op(extent_op); 2557 extent_op = NULL; 2558 } 2559 2560 if (extent_op) { 2561 spin_unlock(&locked_ref->lock); 2562 ret = run_delayed_extent_op(trans, fs_info, 2563 ref, extent_op); 2564 btrfs_free_delayed_extent_op(extent_op); 2565 2566 if (ret) { 2567 /* 2568 * Need to reset must_insert_reserved if 2569 * there was an error so the abort stuff 2570 * can cleanup the reserved space 2571 * properly. 2572 */ 2573 if (must_insert_reserved) 2574 locked_ref->must_insert_reserved = 1; 2575 locked_ref->processing = 0; 2576 btrfs_debug(fs_info, 2577 "run_delayed_extent_op returned %d", 2578 ret); 2579 btrfs_delayed_ref_unlock(locked_ref); 2580 return ret; 2581 } 2582 continue; 2583 } 2584 2585 /* 2586 * Need to drop our head ref lock and re-acquire the 2587 * delayed ref lock and then re-check to make sure 2588 * nobody got added. 2589 */ 2590 spin_unlock(&locked_ref->lock); 2591 spin_lock(&delayed_refs->lock); 2592 spin_lock(&locked_ref->lock); 2593 if (!list_empty(&locked_ref->ref_list) || 2594 locked_ref->extent_op) { 2595 spin_unlock(&locked_ref->lock); 2596 spin_unlock(&delayed_refs->lock); 2597 continue; 2598 } 2599 ref->in_tree = 0; 2600 delayed_refs->num_heads--; 2601 rb_erase(&locked_ref->href_node, 2602 &delayed_refs->href_root); 2603 spin_unlock(&delayed_refs->lock); 2604 } else { 2605 actual_count++; 2606 ref->in_tree = 0; 2607 list_del(&ref->list); 2608 if (!list_empty(&ref->add_list)) 2609 list_del(&ref->add_list); 2610 } 2611 atomic_dec(&delayed_refs->num_entries); 2612 2613 if (!btrfs_delayed_ref_is_head(ref)) { 2614 /* 2615 * when we play the delayed ref, also correct the 2616 * ref_mod on head 2617 */ 2618 switch (ref->action) { 2619 case BTRFS_ADD_DELAYED_REF: 2620 case BTRFS_ADD_DELAYED_EXTENT: 2621 locked_ref->node.ref_mod -= ref->ref_mod; 2622 break; 2623 case BTRFS_DROP_DELAYED_REF: 2624 locked_ref->node.ref_mod += ref->ref_mod; 2625 break; 2626 default: 2627 WARN_ON(1); 2628 } 2629 } 2630 spin_unlock(&locked_ref->lock); 2631 2632 ret = run_one_delayed_ref(trans, fs_info, ref, extent_op, 2633 must_insert_reserved); 2634 2635 btrfs_free_delayed_extent_op(extent_op); 2636 if (ret) { 2637 spin_lock(&delayed_refs->lock); 2638 locked_ref->processing = 0; 2639 delayed_refs->num_heads_ready++; 2640 spin_unlock(&delayed_refs->lock); 2641 btrfs_delayed_ref_unlock(locked_ref); 2642 btrfs_put_delayed_ref(ref); 2643 btrfs_debug(fs_info, "run_one_delayed_ref returned %d", 2644 ret); 2645 return ret; 2646 } 2647 2648 /* 2649 * If this node is a head, that means all the refs in this head 2650 * have been dealt with, and we will pick the next head to deal 2651 * with, so we must unlock the head and drop it from the cluster 2652 * list before we release it. 2653 */ 2654 if (btrfs_delayed_ref_is_head(ref)) { 2655 if (locked_ref->is_data && 2656 locked_ref->total_ref_mod < 0) { 2657 spin_lock(&delayed_refs->lock); 2658 delayed_refs->pending_csums -= ref->num_bytes; 2659 spin_unlock(&delayed_refs->lock); 2660 } 2661 btrfs_delayed_ref_unlock(locked_ref); 2662 locked_ref = NULL; 2663 } 2664 btrfs_put_delayed_ref(ref); 2665 count++; 2666 cond_resched(); 2667 } 2668 2669 /* 2670 * We don't want to include ref heads since we can have empty ref heads 2671 * and those will drastically skew our runtime down since we just do 2672 * accounting, no actual extent tree updates. 2673 */ 2674 if (actual_count > 0) { 2675 u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start)); 2676 u64 avg; 2677 2678 /* 2679 * We weigh the current average higher than our current runtime 2680 * to avoid large swings in the average. 2681 */ 2682 spin_lock(&delayed_refs->lock); 2683 avg = fs_info->avg_delayed_ref_runtime * 3 + runtime; 2684 fs_info->avg_delayed_ref_runtime = avg >> 2; /* div by 4 */ 2685 spin_unlock(&delayed_refs->lock); 2686 } 2687 return 0; 2688 } 2689 2690 #ifdef SCRAMBLE_DELAYED_REFS 2691 /* 2692 * Normally delayed refs get processed in ascending bytenr order. This 2693 * correlates in most cases to the order added. To expose dependencies on this 2694 * order, we start to process the tree in the middle instead of the beginning 2695 */ 2696 static u64 find_middle(struct rb_root *root) 2697 { 2698 struct rb_node *n = root->rb_node; 2699 struct btrfs_delayed_ref_node *entry; 2700 int alt = 1; 2701 u64 middle; 2702 u64 first = 0, last = 0; 2703 2704 n = rb_first(root); 2705 if (n) { 2706 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); 2707 first = entry->bytenr; 2708 } 2709 n = rb_last(root); 2710 if (n) { 2711 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); 2712 last = entry->bytenr; 2713 } 2714 n = root->rb_node; 2715 2716 while (n) { 2717 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); 2718 WARN_ON(!entry->in_tree); 2719 2720 middle = entry->bytenr; 2721 2722 if (alt) 2723 n = n->rb_left; 2724 else 2725 n = n->rb_right; 2726 2727 alt = 1 - alt; 2728 } 2729 return middle; 2730 } 2731 #endif 2732 2733 static inline u64 heads_to_leaves(struct btrfs_fs_info *fs_info, u64 heads) 2734 { 2735 u64 num_bytes; 2736 2737 num_bytes = heads * (sizeof(struct btrfs_extent_item) + 2738 sizeof(struct btrfs_extent_inline_ref)); 2739 if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA)) 2740 num_bytes += heads * sizeof(struct btrfs_tree_block_info); 2741 2742 /* 2743 * We don't ever fill up leaves all the way so multiply by 2 just to be 2744 * closer to what we're really going to want to use. 2745 */ 2746 return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(fs_info)); 2747 } 2748 2749 /* 2750 * Takes the number of bytes to be csumm'ed and figures out how many leaves it 2751 * would require to store the csums for that many bytes. 2752 */ 2753 u64 btrfs_csum_bytes_to_leaves(struct btrfs_fs_info *fs_info, u64 csum_bytes) 2754 { 2755 u64 csum_size; 2756 u64 num_csums_per_leaf; 2757 u64 num_csums; 2758 2759 csum_size = BTRFS_MAX_ITEM_SIZE(fs_info); 2760 num_csums_per_leaf = div64_u64(csum_size, 2761 (u64)btrfs_super_csum_size(fs_info->super_copy)); 2762 num_csums = div64_u64(csum_bytes, fs_info->sectorsize); 2763 num_csums += num_csums_per_leaf - 1; 2764 num_csums = div64_u64(num_csums, num_csums_per_leaf); 2765 return num_csums; 2766 } 2767 2768 int btrfs_check_space_for_delayed_refs(struct btrfs_trans_handle *trans, 2769 struct btrfs_fs_info *fs_info) 2770 { 2771 struct btrfs_block_rsv *global_rsv; 2772 u64 num_heads = trans->transaction->delayed_refs.num_heads_ready; 2773 u64 csum_bytes = trans->transaction->delayed_refs.pending_csums; 2774 u64 num_dirty_bgs = trans->transaction->num_dirty_bgs; 2775 u64 num_bytes, num_dirty_bgs_bytes; 2776 int ret = 0; 2777 2778 num_bytes = btrfs_calc_trans_metadata_size(fs_info, 1); 2779 num_heads = heads_to_leaves(fs_info, num_heads); 2780 if (num_heads > 1) 2781 num_bytes += (num_heads - 1) * fs_info->nodesize; 2782 num_bytes <<= 1; 2783 num_bytes += btrfs_csum_bytes_to_leaves(fs_info, csum_bytes) * 2784 fs_info->nodesize; 2785 num_dirty_bgs_bytes = btrfs_calc_trans_metadata_size(fs_info, 2786 num_dirty_bgs); 2787 global_rsv = &fs_info->global_block_rsv; 2788 2789 /* 2790 * If we can't allocate any more chunks lets make sure we have _lots_ of 2791 * wiggle room since running delayed refs can create more delayed refs. 2792 */ 2793 if (global_rsv->space_info->full) { 2794 num_dirty_bgs_bytes <<= 1; 2795 num_bytes <<= 1; 2796 } 2797 2798 spin_lock(&global_rsv->lock); 2799 if (global_rsv->reserved <= num_bytes + num_dirty_bgs_bytes) 2800 ret = 1; 2801 spin_unlock(&global_rsv->lock); 2802 return ret; 2803 } 2804 2805 int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans, 2806 struct btrfs_fs_info *fs_info) 2807 { 2808 u64 num_entries = 2809 atomic_read(&trans->transaction->delayed_refs.num_entries); 2810 u64 avg_runtime; 2811 u64 val; 2812 2813 smp_mb(); 2814 avg_runtime = fs_info->avg_delayed_ref_runtime; 2815 val = num_entries * avg_runtime; 2816 if (val >= NSEC_PER_SEC) 2817 return 1; 2818 if (val >= NSEC_PER_SEC / 2) 2819 return 2; 2820 2821 return btrfs_check_space_for_delayed_refs(trans, fs_info); 2822 } 2823 2824 struct async_delayed_refs { 2825 struct btrfs_root *root; 2826 u64 transid; 2827 int count; 2828 int error; 2829 int sync; 2830 struct completion wait; 2831 struct btrfs_work work; 2832 }; 2833 2834 static inline struct async_delayed_refs * 2835 to_async_delayed_refs(struct btrfs_work *work) 2836 { 2837 return container_of(work, struct async_delayed_refs, work); 2838 } 2839 2840 static void delayed_ref_async_start(struct btrfs_work *work) 2841 { 2842 struct async_delayed_refs *async = to_async_delayed_refs(work); 2843 struct btrfs_trans_handle *trans; 2844 struct btrfs_fs_info *fs_info = async->root->fs_info; 2845 int ret; 2846 2847 /* if the commit is already started, we don't need to wait here */ 2848 if (btrfs_transaction_blocked(fs_info)) 2849 goto done; 2850 2851 trans = btrfs_join_transaction(async->root); 2852 if (IS_ERR(trans)) { 2853 async->error = PTR_ERR(trans); 2854 goto done; 2855 } 2856 2857 /* 2858 * trans->sync means that when we call end_transaction, we won't 2859 * wait on delayed refs 2860 */ 2861 trans->sync = true; 2862 2863 /* Don't bother flushing if we got into a different transaction */ 2864 if (trans->transid > async->transid) 2865 goto end; 2866 2867 ret = btrfs_run_delayed_refs(trans, fs_info, async->count); 2868 if (ret) 2869 async->error = ret; 2870 end: 2871 ret = btrfs_end_transaction(trans); 2872 if (ret && !async->error) 2873 async->error = ret; 2874 done: 2875 if (async->sync) 2876 complete(&async->wait); 2877 else 2878 kfree(async); 2879 } 2880 2881 int btrfs_async_run_delayed_refs(struct btrfs_fs_info *fs_info, 2882 unsigned long count, u64 transid, int wait) 2883 { 2884 struct async_delayed_refs *async; 2885 int ret; 2886 2887 async = kmalloc(sizeof(*async), GFP_NOFS); 2888 if (!async) 2889 return -ENOMEM; 2890 2891 async->root = fs_info->tree_root; 2892 async->count = count; 2893 async->error = 0; 2894 async->transid = transid; 2895 if (wait) 2896 async->sync = 1; 2897 else 2898 async->sync = 0; 2899 init_completion(&async->wait); 2900 2901 btrfs_init_work(&async->work, btrfs_extent_refs_helper, 2902 delayed_ref_async_start, NULL, NULL); 2903 2904 btrfs_queue_work(fs_info->extent_workers, &async->work); 2905 2906 if (wait) { 2907 wait_for_completion(&async->wait); 2908 ret = async->error; 2909 kfree(async); 2910 return ret; 2911 } 2912 return 0; 2913 } 2914 2915 /* 2916 * this starts processing the delayed reference count updates and 2917 * extent insertions we have queued up so far. count can be 2918 * 0, which means to process everything in the tree at the start 2919 * of the run (but not newly added entries), or it can be some target 2920 * number you'd like to process. 2921 * 2922 * Returns 0 on success or if called with an aborted transaction 2923 * Returns <0 on error and aborts the transaction 2924 */ 2925 int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, 2926 struct btrfs_fs_info *fs_info, unsigned long count) 2927 { 2928 struct rb_node *node; 2929 struct btrfs_delayed_ref_root *delayed_refs; 2930 struct btrfs_delayed_ref_head *head; 2931 int ret; 2932 int run_all = count == (unsigned long)-1; 2933 bool can_flush_pending_bgs = trans->can_flush_pending_bgs; 2934 2935 /* We'll clean this up in btrfs_cleanup_transaction */ 2936 if (trans->aborted) 2937 return 0; 2938 2939 if (test_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags)) 2940 return 0; 2941 2942 delayed_refs = &trans->transaction->delayed_refs; 2943 if (count == 0) 2944 count = atomic_read(&delayed_refs->num_entries) * 2; 2945 2946 again: 2947 #ifdef SCRAMBLE_DELAYED_REFS 2948 delayed_refs->run_delayed_start = find_middle(&delayed_refs->root); 2949 #endif 2950 trans->can_flush_pending_bgs = false; 2951 ret = __btrfs_run_delayed_refs(trans, fs_info, count); 2952 if (ret < 0) { 2953 btrfs_abort_transaction(trans, ret); 2954 return ret; 2955 } 2956 2957 if (run_all) { 2958 if (!list_empty(&trans->new_bgs)) 2959 btrfs_create_pending_block_groups(trans, fs_info); 2960 2961 spin_lock(&delayed_refs->lock); 2962 node = rb_first(&delayed_refs->href_root); 2963 if (!node) { 2964 spin_unlock(&delayed_refs->lock); 2965 goto out; 2966 } 2967 2968 while (node) { 2969 head = rb_entry(node, struct btrfs_delayed_ref_head, 2970 href_node); 2971 if (btrfs_delayed_ref_is_head(&head->node)) { 2972 struct btrfs_delayed_ref_node *ref; 2973 2974 ref = &head->node; 2975 atomic_inc(&ref->refs); 2976 2977 spin_unlock(&delayed_refs->lock); 2978 /* 2979 * Mutex was contended, block until it's 2980 * released and try again 2981 */ 2982 mutex_lock(&head->mutex); 2983 mutex_unlock(&head->mutex); 2984 2985 btrfs_put_delayed_ref(ref); 2986 cond_resched(); 2987 goto again; 2988 } else { 2989 WARN_ON(1); 2990 } 2991 node = rb_next(node); 2992 } 2993 spin_unlock(&delayed_refs->lock); 2994 cond_resched(); 2995 goto again; 2996 } 2997 out: 2998 assert_qgroups_uptodate(trans); 2999 trans->can_flush_pending_bgs = can_flush_pending_bgs; 3000 return 0; 3001 } 3002 3003 int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans, 3004 struct btrfs_fs_info *fs_info, 3005 u64 bytenr, u64 num_bytes, u64 flags, 3006 int level, int is_data) 3007 { 3008 struct btrfs_delayed_extent_op *extent_op; 3009 int ret; 3010 3011 extent_op = btrfs_alloc_delayed_extent_op(); 3012 if (!extent_op) 3013 return -ENOMEM; 3014 3015 extent_op->flags_to_set = flags; 3016 extent_op->update_flags = true; 3017 extent_op->update_key = false; 3018 extent_op->is_data = is_data ? true : false; 3019 extent_op->level = level; 3020 3021 ret = btrfs_add_delayed_extent_op(fs_info, trans, bytenr, 3022 num_bytes, extent_op); 3023 if (ret) 3024 btrfs_free_delayed_extent_op(extent_op); 3025 return ret; 3026 } 3027 3028 static noinline int check_delayed_ref(struct btrfs_trans_handle *trans, 3029 struct btrfs_root *root, 3030 struct btrfs_path *path, 3031 u64 objectid, u64 offset, u64 bytenr) 3032 { 3033 struct btrfs_delayed_ref_head *head; 3034 struct btrfs_delayed_ref_node *ref; 3035 struct btrfs_delayed_data_ref *data_ref; 3036 struct btrfs_delayed_ref_root *delayed_refs; 3037 int ret = 0; 3038 3039 delayed_refs = &trans->transaction->delayed_refs; 3040 spin_lock(&delayed_refs->lock); 3041 head = btrfs_find_delayed_ref_head(trans, bytenr); 3042 if (!head) { 3043 spin_unlock(&delayed_refs->lock); 3044 return 0; 3045 } 3046 3047 if (!mutex_trylock(&head->mutex)) { 3048 atomic_inc(&head->node.refs); 3049 spin_unlock(&delayed_refs->lock); 3050 3051 btrfs_release_path(path); 3052 3053 /* 3054 * Mutex was contended, block until it's released and let 3055 * caller try again 3056 */ 3057 mutex_lock(&head->mutex); 3058 mutex_unlock(&head->mutex); 3059 btrfs_put_delayed_ref(&head->node); 3060 return -EAGAIN; 3061 } 3062 spin_unlock(&delayed_refs->lock); 3063 3064 spin_lock(&head->lock); 3065 list_for_each_entry(ref, &head->ref_list, list) { 3066 /* If it's a shared ref we know a cross reference exists */ 3067 if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) { 3068 ret = 1; 3069 break; 3070 } 3071 3072 data_ref = btrfs_delayed_node_to_data_ref(ref); 3073 3074 /* 3075 * If our ref doesn't match the one we're currently looking at 3076 * then we have a cross reference. 3077 */ 3078 if (data_ref->root != root->root_key.objectid || 3079 data_ref->objectid != objectid || 3080 data_ref->offset != offset) { 3081 ret = 1; 3082 break; 3083 } 3084 } 3085 spin_unlock(&head->lock); 3086 mutex_unlock(&head->mutex); 3087 return ret; 3088 } 3089 3090 static noinline int check_committed_ref(struct btrfs_trans_handle *trans, 3091 struct btrfs_root *root, 3092 struct btrfs_path *path, 3093 u64 objectid, u64 offset, u64 bytenr) 3094 { 3095 struct btrfs_fs_info *fs_info = root->fs_info; 3096 struct btrfs_root *extent_root = fs_info->extent_root; 3097 struct extent_buffer *leaf; 3098 struct btrfs_extent_data_ref *ref; 3099 struct btrfs_extent_inline_ref *iref; 3100 struct btrfs_extent_item *ei; 3101 struct btrfs_key key; 3102 u32 item_size; 3103 int ret; 3104 3105 key.objectid = bytenr; 3106 key.offset = (u64)-1; 3107 key.type = BTRFS_EXTENT_ITEM_KEY; 3108 3109 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); 3110 if (ret < 0) 3111 goto out; 3112 BUG_ON(ret == 0); /* Corruption */ 3113 3114 ret = -ENOENT; 3115 if (path->slots[0] == 0) 3116 goto out; 3117 3118 path->slots[0]--; 3119 leaf = path->nodes[0]; 3120 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3121 3122 if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY) 3123 goto out; 3124 3125 ret = 1; 3126 item_size = btrfs_item_size_nr(leaf, path->slots[0]); 3127 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 3128 if (item_size < sizeof(*ei)) { 3129 WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0)); 3130 goto out; 3131 } 3132 #endif 3133 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); 3134 3135 if (item_size != sizeof(*ei) + 3136 btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY)) 3137 goto out; 3138 3139 if (btrfs_extent_generation(leaf, ei) <= 3140 btrfs_root_last_snapshot(&root->root_item)) 3141 goto out; 3142 3143 iref = (struct btrfs_extent_inline_ref *)(ei + 1); 3144 if (btrfs_extent_inline_ref_type(leaf, iref) != 3145 BTRFS_EXTENT_DATA_REF_KEY) 3146 goto out; 3147 3148 ref = (struct btrfs_extent_data_ref *)(&iref->offset); 3149 if (btrfs_extent_refs(leaf, ei) != 3150 btrfs_extent_data_ref_count(leaf, ref) || 3151 btrfs_extent_data_ref_root(leaf, ref) != 3152 root->root_key.objectid || 3153 btrfs_extent_data_ref_objectid(leaf, ref) != objectid || 3154 btrfs_extent_data_ref_offset(leaf, ref) != offset) 3155 goto out; 3156 3157 ret = 0; 3158 out: 3159 return ret; 3160 } 3161 3162 int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans, 3163 struct btrfs_root *root, 3164 u64 objectid, u64 offset, u64 bytenr) 3165 { 3166 struct btrfs_path *path; 3167 int ret; 3168 int ret2; 3169 3170 path = btrfs_alloc_path(); 3171 if (!path) 3172 return -ENOENT; 3173 3174 do { 3175 ret = check_committed_ref(trans, root, path, objectid, 3176 offset, bytenr); 3177 if (ret && ret != -ENOENT) 3178 goto out; 3179 3180 ret2 = check_delayed_ref(trans, root, path, objectid, 3181 offset, bytenr); 3182 } while (ret2 == -EAGAIN); 3183 3184 if (ret2 && ret2 != -ENOENT) { 3185 ret = ret2; 3186 goto out; 3187 } 3188 3189 if (ret != -ENOENT || ret2 != -ENOENT) 3190 ret = 0; 3191 out: 3192 btrfs_free_path(path); 3193 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 3194 WARN_ON(ret > 0); 3195 return ret; 3196 } 3197 3198 static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, 3199 struct btrfs_root *root, 3200 struct extent_buffer *buf, 3201 int full_backref, int inc) 3202 { 3203 struct btrfs_fs_info *fs_info = root->fs_info; 3204 u64 bytenr; 3205 u64 num_bytes; 3206 u64 parent; 3207 u64 ref_root; 3208 u32 nritems; 3209 struct btrfs_key key; 3210 struct btrfs_file_extent_item *fi; 3211 int i; 3212 int level; 3213 int ret = 0; 3214 int (*process_func)(struct btrfs_trans_handle *, 3215 struct btrfs_fs_info *, 3216 u64, u64, u64, u64, u64, u64); 3217 3218 3219 if (btrfs_is_testing(fs_info)) 3220 return 0; 3221 3222 ref_root = btrfs_header_owner(buf); 3223 nritems = btrfs_header_nritems(buf); 3224 level = btrfs_header_level(buf); 3225 3226 if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0) 3227 return 0; 3228 3229 if (inc) 3230 process_func = btrfs_inc_extent_ref; 3231 else 3232 process_func = btrfs_free_extent; 3233 3234 if (full_backref) 3235 parent = buf->start; 3236 else 3237 parent = 0; 3238 3239 for (i = 0; i < nritems; i++) { 3240 if (level == 0) { 3241 btrfs_item_key_to_cpu(buf, &key, i); 3242 if (key.type != BTRFS_EXTENT_DATA_KEY) 3243 continue; 3244 fi = btrfs_item_ptr(buf, i, 3245 struct btrfs_file_extent_item); 3246 if (btrfs_file_extent_type(buf, fi) == 3247 BTRFS_FILE_EXTENT_INLINE) 3248 continue; 3249 bytenr = btrfs_file_extent_disk_bytenr(buf, fi); 3250 if (bytenr == 0) 3251 continue; 3252 3253 num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi); 3254 key.offset -= btrfs_file_extent_offset(buf, fi); 3255 ret = process_func(trans, fs_info, bytenr, num_bytes, 3256 parent, ref_root, key.objectid, 3257 key.offset); 3258 if (ret) 3259 goto fail; 3260 } else { 3261 bytenr = btrfs_node_blockptr(buf, i); 3262 num_bytes = fs_info->nodesize; 3263 ret = process_func(trans, fs_info, bytenr, num_bytes, 3264 parent, ref_root, level - 1, 0); 3265 if (ret) 3266 goto fail; 3267 } 3268 } 3269 return 0; 3270 fail: 3271 return ret; 3272 } 3273 3274 int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, 3275 struct extent_buffer *buf, int full_backref) 3276 { 3277 return __btrfs_mod_ref(trans, root, buf, full_backref, 1); 3278 } 3279 3280 int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, 3281 struct extent_buffer *buf, int full_backref) 3282 { 3283 return __btrfs_mod_ref(trans, root, buf, full_backref, 0); 3284 } 3285 3286 static int write_one_cache_group(struct btrfs_trans_handle *trans, 3287 struct btrfs_fs_info *fs_info, 3288 struct btrfs_path *path, 3289 struct btrfs_block_group_cache *cache) 3290 { 3291 int ret; 3292 struct btrfs_root *extent_root = fs_info->extent_root; 3293 unsigned long bi; 3294 struct extent_buffer *leaf; 3295 3296 ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1); 3297 if (ret) { 3298 if (ret > 0) 3299 ret = -ENOENT; 3300 goto fail; 3301 } 3302 3303 leaf = path->nodes[0]; 3304 bi = btrfs_item_ptr_offset(leaf, path->slots[0]); 3305 write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item)); 3306 btrfs_mark_buffer_dirty(leaf); 3307 fail: 3308 btrfs_release_path(path); 3309 return ret; 3310 3311 } 3312 3313 static struct btrfs_block_group_cache * 3314 next_block_group(struct btrfs_fs_info *fs_info, 3315 struct btrfs_block_group_cache *cache) 3316 { 3317 struct rb_node *node; 3318 3319 spin_lock(&fs_info->block_group_cache_lock); 3320 3321 /* If our block group was removed, we need a full search. */ 3322 if (RB_EMPTY_NODE(&cache->cache_node)) { 3323 const u64 next_bytenr = cache->key.objectid + cache->key.offset; 3324 3325 spin_unlock(&fs_info->block_group_cache_lock); 3326 btrfs_put_block_group(cache); 3327 cache = btrfs_lookup_first_block_group(fs_info, next_bytenr); return cache; 3328 } 3329 node = rb_next(&cache->cache_node); 3330 btrfs_put_block_group(cache); 3331 if (node) { 3332 cache = rb_entry(node, struct btrfs_block_group_cache, 3333 cache_node); 3334 btrfs_get_block_group(cache); 3335 } else 3336 cache = NULL; 3337 spin_unlock(&fs_info->block_group_cache_lock); 3338 return cache; 3339 } 3340 3341 static int cache_save_setup(struct btrfs_block_group_cache *block_group, 3342 struct btrfs_trans_handle *trans, 3343 struct btrfs_path *path) 3344 { 3345 struct btrfs_fs_info *fs_info = block_group->fs_info; 3346 struct btrfs_root *root = fs_info->tree_root; 3347 struct inode *inode = NULL; 3348 u64 alloc_hint = 0; 3349 int dcs = BTRFS_DC_ERROR; 3350 u64 num_pages = 0; 3351 int retries = 0; 3352 int ret = 0; 3353 3354 /* 3355 * If this block group is smaller than 100 megs don't bother caching the 3356 * block group. 3357 */ 3358 if (block_group->key.offset < (100 * SZ_1M)) { 3359 spin_lock(&block_group->lock); 3360 block_group->disk_cache_state = BTRFS_DC_WRITTEN; 3361 spin_unlock(&block_group->lock); 3362 return 0; 3363 } 3364 3365 if (trans->aborted) 3366 return 0; 3367 again: 3368 inode = lookup_free_space_inode(root, block_group, path); 3369 if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) { 3370 ret = PTR_ERR(inode); 3371 btrfs_release_path(path); 3372 goto out; 3373 } 3374 3375 if (IS_ERR(inode)) { 3376 BUG_ON(retries); 3377 retries++; 3378 3379 if (block_group->ro) 3380 goto out_free; 3381 3382 ret = create_free_space_inode(root, trans, block_group, path); 3383 if (ret) 3384 goto out_free; 3385 goto again; 3386 } 3387 3388 /* We've already setup this transaction, go ahead and exit */ 3389 if (block_group->cache_generation == trans->transid && 3390 i_size_read(inode)) { 3391 dcs = BTRFS_DC_SETUP; 3392 goto out_put; 3393 } 3394 3395 /* 3396 * We want to set the generation to 0, that way if anything goes wrong 3397 * from here on out we know not to trust this cache when we load up next 3398 * time. 3399 */ 3400 BTRFS_I(inode)->generation = 0; 3401 ret = btrfs_update_inode(trans, root, inode); 3402 if (ret) { 3403 /* 3404 * So theoretically we could recover from this, simply set the 3405 * super cache generation to 0 so we know to invalidate the 3406 * cache, but then we'd have to keep track of the block groups 3407 * that fail this way so we know we _have_ to reset this cache 3408 * before the next commit or risk reading stale cache. So to 3409 * limit our exposure to horrible edge cases lets just abort the 3410 * transaction, this only happens in really bad situations 3411 * anyway. 3412 */ 3413 btrfs_abort_transaction(trans, ret); 3414 goto out_put; 3415 } 3416 WARN_ON(ret); 3417 3418 if (i_size_read(inode) > 0) { 3419 ret = btrfs_check_trunc_cache_free_space(fs_info, 3420 &fs_info->global_block_rsv); 3421 if (ret) 3422 goto out_put; 3423 3424 ret = btrfs_truncate_free_space_cache(root, trans, NULL, inode); 3425 if (ret) 3426 goto out_put; 3427 } 3428 3429 spin_lock(&block_group->lock); 3430 if (block_group->cached != BTRFS_CACHE_FINISHED || 3431 !btrfs_test_opt(fs_info, SPACE_CACHE)) { 3432 /* 3433 * don't bother trying to write stuff out _if_ 3434 * a) we're not cached, 3435 * b) we're with nospace_cache mount option. 3436 */ 3437 dcs = BTRFS_DC_WRITTEN; 3438 spin_unlock(&block_group->lock); 3439 goto out_put; 3440 } 3441 spin_unlock(&block_group->lock); 3442 3443 /* 3444 * We hit an ENOSPC when setting up the cache in this transaction, just 3445 * skip doing the setup, we've already cleared the cache so we're safe. 3446 */ 3447 if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) { 3448 ret = -ENOSPC; 3449 goto out_put; 3450 } 3451 3452 /* 3453 * Try to preallocate enough space based on how big the block group is. 3454 * Keep in mind this has to include any pinned space which could end up 3455 * taking up quite a bit since it's not folded into the other space 3456 * cache. 3457 */ 3458 num_pages = div_u64(block_group->key.offset, SZ_256M); 3459 if (!num_pages) 3460 num_pages = 1; 3461 3462 num_pages *= 16; 3463 num_pages *= PAGE_SIZE; 3464 3465 ret = btrfs_check_data_free_space(inode, 0, num_pages); 3466 if (ret) 3467 goto out_put; 3468 3469 ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages, 3470 num_pages, num_pages, 3471 &alloc_hint); 3472 /* 3473 * Our cache requires contiguous chunks so that we don't modify a bunch 3474 * of metadata or split extents when writing the cache out, which means 3475 * we can enospc if we are heavily fragmented in addition to just normal 3476 * out of space conditions. So if we hit this just skip setting up any 3477 * other block groups for this transaction, maybe we'll unpin enough 3478 * space the next time around. 3479 */ 3480 if (!ret) 3481 dcs = BTRFS_DC_SETUP; 3482 else if (ret == -ENOSPC) 3483 set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags); 3484 3485 out_put: 3486 iput(inode); 3487 out_free: 3488 btrfs_release_path(path); 3489 out: 3490 spin_lock(&block_group->lock); 3491 if (!ret && dcs == BTRFS_DC_SETUP) 3492 block_group->cache_generation = trans->transid; 3493 block_group->disk_cache_state = dcs; 3494 spin_unlock(&block_group->lock); 3495 3496 return ret; 3497 } 3498 3499 int btrfs_setup_space_cache(struct btrfs_trans_handle *trans, 3500 struct btrfs_fs_info *fs_info) 3501 { 3502 struct btrfs_block_group_cache *cache, *tmp; 3503 struct btrfs_transaction *cur_trans = trans->transaction; 3504 struct btrfs_path *path; 3505 3506 if (list_empty(&cur_trans->dirty_bgs) || 3507 !btrfs_test_opt(fs_info, SPACE_CACHE)) 3508 return 0; 3509 3510 path = btrfs_alloc_path(); 3511 if (!path) 3512 return -ENOMEM; 3513 3514 /* Could add new block groups, use _safe just in case */ 3515 list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs, 3516 dirty_list) { 3517 if (cache->disk_cache_state == BTRFS_DC_CLEAR) 3518 cache_save_setup(cache, trans, path); 3519 } 3520 3521 btrfs_free_path(path); 3522 return 0; 3523 } 3524 3525 /* 3526 * transaction commit does final block group cache writeback during a 3527 * critical section where nothing is allowed to change the FS. This is 3528 * required in order for the cache to actually match the block group, 3529 * but can introduce a lot of latency into the commit. 3530 * 3531 * So, btrfs_start_dirty_block_groups is here to kick off block group 3532 * cache IO. There's a chance we'll have to redo some of it if the 3533 * block group changes again during the commit, but it greatly reduces 3534 * the commit latency by getting rid of the easy block groups while 3535 * we're still allowing others to join the commit. 3536 */ 3537 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans, 3538 struct btrfs_fs_info *fs_info) 3539 { 3540 struct btrfs_block_group_cache *cache; 3541 struct btrfs_transaction *cur_trans = trans->transaction; 3542 int ret = 0; 3543 int should_put; 3544 struct btrfs_path *path = NULL; 3545 LIST_HEAD(dirty); 3546 struct list_head *io = &cur_trans->io_bgs; 3547 int num_started = 0; 3548 int loops = 0; 3549 3550 spin_lock(&cur_trans->dirty_bgs_lock); 3551 if (list_empty(&cur_trans->dirty_bgs)) { 3552 spin_unlock(&cur_trans->dirty_bgs_lock); 3553 return 0; 3554 } 3555 list_splice_init(&cur_trans->dirty_bgs, &dirty); 3556 spin_unlock(&cur_trans->dirty_bgs_lock); 3557 3558 again: 3559 /* 3560 * make sure all the block groups on our dirty list actually 3561 * exist 3562 */ 3563 btrfs_create_pending_block_groups(trans, fs_info); 3564 3565 if (!path) { 3566 path = btrfs_alloc_path(); 3567 if (!path) 3568 return -ENOMEM; 3569 } 3570 3571 /* 3572 * cache_write_mutex is here only to save us from balance or automatic 3573 * removal of empty block groups deleting this block group while we are 3574 * writing out the cache 3575 */ 3576 mutex_lock(&trans->transaction->cache_write_mutex); 3577 while (!list_empty(&dirty)) { 3578 cache = list_first_entry(&dirty, 3579 struct btrfs_block_group_cache, 3580 dirty_list); 3581 /* 3582 * this can happen if something re-dirties a block 3583 * group that is already under IO. Just wait for it to 3584 * finish and then do it all again 3585 */ 3586 if (!list_empty(&cache->io_list)) { 3587 list_del_init(&cache->io_list); 3588 btrfs_wait_cache_io(trans, cache, path); 3589 btrfs_put_block_group(cache); 3590 } 3591 3592 3593 /* 3594 * btrfs_wait_cache_io uses the cache->dirty_list to decide 3595 * if it should update the cache_state. Don't delete 3596 * until after we wait. 3597 * 3598 * Since we're not running in the commit critical section 3599 * we need the dirty_bgs_lock to protect from update_block_group 3600 */ 3601 spin_lock(&cur_trans->dirty_bgs_lock); 3602 list_del_init(&cache->dirty_list); 3603 spin_unlock(&cur_trans->dirty_bgs_lock); 3604 3605 should_put = 1; 3606 3607 cache_save_setup(cache, trans, path); 3608 3609 if (cache->disk_cache_state == BTRFS_DC_SETUP) { 3610 cache->io_ctl.inode = NULL; 3611 ret = btrfs_write_out_cache(fs_info, trans, 3612 cache, path); 3613 if (ret == 0 && cache->io_ctl.inode) { 3614 num_started++; 3615 should_put = 0; 3616 3617 /* 3618 * the cache_write_mutex is protecting 3619 * the io_list 3620 */ 3621 list_add_tail(&cache->io_list, io); 3622 } else { 3623 /* 3624 * if we failed to write the cache, the 3625 * generation will be bad and life goes on 3626 */ 3627 ret = 0; 3628 } 3629 } 3630 if (!ret) { 3631 ret = write_one_cache_group(trans, fs_info, 3632 path, cache); 3633 /* 3634 * Our block group might still be attached to the list 3635 * of new block groups in the transaction handle of some 3636 * other task (struct btrfs_trans_handle->new_bgs). This 3637 * means its block group item isn't yet in the extent 3638 * tree. If this happens ignore the error, as we will 3639 * try again later in the critical section of the 3640 * transaction commit. 3641 */ 3642 if (ret == -ENOENT) { 3643 ret = 0; 3644 spin_lock(&cur_trans->dirty_bgs_lock); 3645 if (list_empty(&cache->dirty_list)) { 3646 list_add_tail(&cache->dirty_list, 3647 &cur_trans->dirty_bgs); 3648 btrfs_get_block_group(cache); 3649 } 3650 spin_unlock(&cur_trans->dirty_bgs_lock); 3651 } else if (ret) { 3652 btrfs_abort_transaction(trans, ret); 3653 } 3654 } 3655 3656 /* if its not on the io list, we need to put the block group */ 3657 if (should_put) 3658 btrfs_put_block_group(cache); 3659 3660 if (ret) 3661 break; 3662 3663 /* 3664 * Avoid blocking other tasks for too long. It might even save 3665 * us from writing caches for block groups that are going to be 3666 * removed. 3667 */ 3668 mutex_unlock(&trans->transaction->cache_write_mutex); 3669 mutex_lock(&trans->transaction->cache_write_mutex); 3670 } 3671 mutex_unlock(&trans->transaction->cache_write_mutex); 3672 3673 /* 3674 * go through delayed refs for all the stuff we've just kicked off 3675 * and then loop back (just once) 3676 */ 3677 ret = btrfs_run_delayed_refs(trans, fs_info, 0); 3678 if (!ret && loops == 0) { 3679 loops++; 3680 spin_lock(&cur_trans->dirty_bgs_lock); 3681 list_splice_init(&cur_trans->dirty_bgs, &dirty); 3682 /* 3683 * dirty_bgs_lock protects us from concurrent block group 3684 * deletes too (not just cache_write_mutex). 3685 */ 3686 if (!list_empty(&dirty)) { 3687 spin_unlock(&cur_trans->dirty_bgs_lock); 3688 goto again; 3689 } 3690 spin_unlock(&cur_trans->dirty_bgs_lock); 3691 } else if (ret < 0) { 3692 btrfs_cleanup_dirty_bgs(cur_trans, fs_info); 3693 } 3694 3695 btrfs_free_path(path); 3696 return ret; 3697 } 3698 3699 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans, 3700 struct btrfs_fs_info *fs_info) 3701 { 3702 struct btrfs_block_group_cache *cache; 3703 struct btrfs_transaction *cur_trans = trans->transaction; 3704 int ret = 0; 3705 int should_put; 3706 struct btrfs_path *path; 3707 struct list_head *io = &cur_trans->io_bgs; 3708 int num_started = 0; 3709 3710 path = btrfs_alloc_path(); 3711 if (!path) 3712 return -ENOMEM; 3713 3714 /* 3715 * Even though we are in the critical section of the transaction commit, 3716 * we can still have concurrent tasks adding elements to this 3717 * transaction's list of dirty block groups. These tasks correspond to 3718 * endio free space workers started when writeback finishes for a 3719 * space cache, which run inode.c:btrfs_finish_ordered_io(), and can 3720 * allocate new block groups as a result of COWing nodes of the root 3721 * tree when updating the free space inode. The writeback for the space 3722 * caches is triggered by an earlier call to 3723 * btrfs_start_dirty_block_groups() and iterations of the following 3724 * loop. 3725 * Also we want to do the cache_save_setup first and then run the 3726 * delayed refs to make sure we have the best chance at doing this all 3727 * in one shot. 3728 */ 3729 spin_lock(&cur_trans->dirty_bgs_lock); 3730 while (!list_empty(&cur_trans->dirty_bgs)) { 3731 cache = list_first_entry(&cur_trans->dirty_bgs, 3732 struct btrfs_block_group_cache, 3733 dirty_list); 3734 3735 /* 3736 * this can happen if cache_save_setup re-dirties a block 3737 * group that is already under IO. Just wait for it to 3738 * finish and then do it all again 3739 */ 3740 if (!list_empty(&cache->io_list)) { 3741 spin_unlock(&cur_trans->dirty_bgs_lock); 3742 list_del_init(&cache->io_list); 3743 btrfs_wait_cache_io(trans, cache, path); 3744 btrfs_put_block_group(cache); 3745 spin_lock(&cur_trans->dirty_bgs_lock); 3746 } 3747 3748 /* 3749 * don't remove from the dirty list until after we've waited 3750 * on any pending IO 3751 */ 3752 list_del_init(&cache->dirty_list); 3753 spin_unlock(&cur_trans->dirty_bgs_lock); 3754 should_put = 1; 3755 3756 cache_save_setup(cache, trans, path); 3757 3758 if (!ret) 3759 ret = btrfs_run_delayed_refs(trans, fs_info, 3760 (unsigned long) -1); 3761 3762 if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) { 3763 cache->io_ctl.inode = NULL; 3764 ret = btrfs_write_out_cache(fs_info, trans, 3765 cache, path); 3766 if (ret == 0 && cache->io_ctl.inode) { 3767 num_started++; 3768 should_put = 0; 3769 list_add_tail(&cache->io_list, io); 3770 } else { 3771 /* 3772 * if we failed to write the cache, the 3773 * generation will be bad and life goes on 3774 */ 3775 ret = 0; 3776 } 3777 } 3778 if (!ret) { 3779 ret = write_one_cache_group(trans, fs_info, 3780 path, cache); 3781 /* 3782 * One of the free space endio workers might have 3783 * created a new block group while updating a free space 3784 * cache's inode (at inode.c:btrfs_finish_ordered_io()) 3785 * and hasn't released its transaction handle yet, in 3786 * which case the new block group is still attached to 3787 * its transaction handle and its creation has not 3788 * finished yet (no block group item in the extent tree 3789 * yet, etc). If this is the case, wait for all free 3790 * space endio workers to finish and retry. This is a 3791 * a very rare case so no need for a more efficient and 3792 * complex approach. 3793 */ 3794 if (ret == -ENOENT) { 3795 wait_event(cur_trans->writer_wait, 3796 atomic_read(&cur_trans->num_writers) == 1); 3797 ret = write_one_cache_group(trans, fs_info, 3798 path, cache); 3799 } 3800 if (ret) 3801 btrfs_abort_transaction(trans, ret); 3802 } 3803 3804 /* if its not on the io list, we need to put the block group */ 3805 if (should_put) 3806 btrfs_put_block_group(cache); 3807 spin_lock(&cur_trans->dirty_bgs_lock); 3808 } 3809 spin_unlock(&cur_trans->dirty_bgs_lock); 3810 3811 while (!list_empty(io)) { 3812 cache = list_first_entry(io, struct btrfs_block_group_cache, 3813 io_list); 3814 list_del_init(&cache->io_list); 3815 btrfs_wait_cache_io(trans, cache, path); 3816 btrfs_put_block_group(cache); 3817 } 3818 3819 btrfs_free_path(path); 3820 return ret; 3821 } 3822 3823 int btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr) 3824 { 3825 struct btrfs_block_group_cache *block_group; 3826 int readonly = 0; 3827 3828 block_group = btrfs_lookup_block_group(fs_info, bytenr); 3829 if (!block_group || block_group->ro) 3830 readonly = 1; 3831 if (block_group) 3832 btrfs_put_block_group(block_group); 3833 return readonly; 3834 } 3835 3836 bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr) 3837 { 3838 struct btrfs_block_group_cache *bg; 3839 bool ret = true; 3840 3841 bg = btrfs_lookup_block_group(fs_info, bytenr); 3842 if (!bg) 3843 return false; 3844 3845 spin_lock(&bg->lock); 3846 if (bg->ro) 3847 ret = false; 3848 else 3849 atomic_inc(&bg->nocow_writers); 3850 spin_unlock(&bg->lock); 3851 3852 /* no put on block group, done by btrfs_dec_nocow_writers */ 3853 if (!ret) 3854 btrfs_put_block_group(bg); 3855 3856 return ret; 3857 3858 } 3859 3860 void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr) 3861 { 3862 struct btrfs_block_group_cache *bg; 3863 3864 bg = btrfs_lookup_block_group(fs_info, bytenr); 3865 ASSERT(bg); 3866 if (atomic_dec_and_test(&bg->nocow_writers)) 3867 wake_up_atomic_t(&bg->nocow_writers); 3868 /* 3869 * Once for our lookup and once for the lookup done by a previous call 3870 * to btrfs_inc_nocow_writers() 3871 */ 3872 btrfs_put_block_group(bg); 3873 btrfs_put_block_group(bg); 3874 } 3875 3876 static int btrfs_wait_nocow_writers_atomic_t(atomic_t *a) 3877 { 3878 schedule(); 3879 return 0; 3880 } 3881 3882 void btrfs_wait_nocow_writers(struct btrfs_block_group_cache *bg) 3883 { 3884 wait_on_atomic_t(&bg->nocow_writers, 3885 btrfs_wait_nocow_writers_atomic_t, 3886 TASK_UNINTERRUPTIBLE); 3887 } 3888 3889 static const char *alloc_name(u64 flags) 3890 { 3891 switch (flags) { 3892 case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA: 3893 return "mixed"; 3894 case BTRFS_BLOCK_GROUP_METADATA: 3895 return "metadata"; 3896 case BTRFS_BLOCK_GROUP_DATA: 3897 return "data"; 3898 case BTRFS_BLOCK_GROUP_SYSTEM: 3899 return "system"; 3900 default: 3901 WARN_ON(1); 3902 return "invalid-combination"; 3903 }; 3904 } 3905 3906 static int update_space_info(struct btrfs_fs_info *info, u64 flags, 3907 u64 total_bytes, u64 bytes_used, 3908 u64 bytes_readonly, 3909 struct btrfs_space_info **space_info) 3910 { 3911 struct btrfs_space_info *found; 3912 int i; 3913 int factor; 3914 int ret; 3915 3916 if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 | 3917 BTRFS_BLOCK_GROUP_RAID10)) 3918 factor = 2; 3919 else 3920 factor = 1; 3921 3922 found = __find_space_info(info, flags); 3923 if (found) { 3924 spin_lock(&found->lock); 3925 found->total_bytes += total_bytes; 3926 found->disk_total += total_bytes * factor; 3927 found->bytes_used += bytes_used; 3928 found->disk_used += bytes_used * factor; 3929 found->bytes_readonly += bytes_readonly; 3930 if (total_bytes > 0) 3931 found->full = 0; 3932 space_info_add_new_bytes(info, found, total_bytes - 3933 bytes_used - bytes_readonly); 3934 spin_unlock(&found->lock); 3935 *space_info = found; 3936 return 0; 3937 } 3938 found = kzalloc(sizeof(*found), GFP_NOFS); 3939 if (!found) 3940 return -ENOMEM; 3941 3942 ret = percpu_counter_init(&found->total_bytes_pinned, 0, GFP_KERNEL); 3943 if (ret) { 3944 kfree(found); 3945 return ret; 3946 } 3947 3948 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) 3949 INIT_LIST_HEAD(&found->block_groups[i]); 3950 init_rwsem(&found->groups_sem); 3951 spin_lock_init(&found->lock); 3952 found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK; 3953 found->total_bytes = total_bytes; 3954 found->disk_total = total_bytes * factor; 3955 found->bytes_used = bytes_used; 3956 found->disk_used = bytes_used * factor; 3957 found->bytes_pinned = 0; 3958 found->bytes_reserved = 0; 3959 found->bytes_readonly = bytes_readonly; 3960 found->bytes_may_use = 0; 3961 found->full = 0; 3962 found->max_extent_size = 0; 3963 found->force_alloc = CHUNK_ALLOC_NO_FORCE; 3964 found->chunk_alloc = 0; 3965 found->flush = 0; 3966 init_waitqueue_head(&found->wait); 3967 INIT_LIST_HEAD(&found->ro_bgs); 3968 INIT_LIST_HEAD(&found->tickets); 3969 INIT_LIST_HEAD(&found->priority_tickets); 3970 3971 ret = kobject_init_and_add(&found->kobj, &space_info_ktype, 3972 info->space_info_kobj, "%s", 3973 alloc_name(found->flags)); 3974 if (ret) { 3975 kfree(found); 3976 return ret; 3977 } 3978 3979 *space_info = found; 3980 list_add_rcu(&found->list, &info->space_info); 3981 if (flags & BTRFS_BLOCK_GROUP_DATA) 3982 info->data_sinfo = found; 3983 3984 return ret; 3985 } 3986 3987 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) 3988 { 3989 u64 extra_flags = chunk_to_extended(flags) & 3990 BTRFS_EXTENDED_PROFILE_MASK; 3991 3992 write_seqlock(&fs_info->profiles_lock); 3993 if (flags & BTRFS_BLOCK_GROUP_DATA) 3994 fs_info->avail_data_alloc_bits |= extra_flags; 3995 if (flags & BTRFS_BLOCK_GROUP_METADATA) 3996 fs_info->avail_metadata_alloc_bits |= extra_flags; 3997 if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 3998 fs_info->avail_system_alloc_bits |= extra_flags; 3999 write_sequnlock(&fs_info->profiles_lock); 4000 } 4001 4002 /* 4003 * returns target flags in extended format or 0 if restripe for this 4004 * chunk_type is not in progress 4005 * 4006 * should be called with either volume_mutex or balance_lock held 4007 */ 4008 static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags) 4009 { 4010 struct btrfs_balance_control *bctl = fs_info->balance_ctl; 4011 u64 target = 0; 4012 4013 if (!bctl) 4014 return 0; 4015 4016 if (flags & BTRFS_BLOCK_GROUP_DATA && 4017 bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) { 4018 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target; 4019 } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM && 4020 bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) { 4021 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target; 4022 } else if (flags & BTRFS_BLOCK_GROUP_METADATA && 4023 bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) { 4024 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target; 4025 } 4026 4027 return target; 4028 } 4029 4030 /* 4031 * @flags: available profiles in extended format (see ctree.h) 4032 * 4033 * Returns reduced profile in chunk format. If profile changing is in 4034 * progress (either running or paused) picks the target profile (if it's 4035 * already available), otherwise falls back to plain reducing. 4036 */ 4037 static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags) 4038 { 4039 u64 num_devices = fs_info->fs_devices->rw_devices; 4040 u64 target; 4041 u64 raid_type; 4042 u64 allowed = 0; 4043 4044 /* 4045 * see if restripe for this chunk_type is in progress, if so 4046 * try to reduce to the target profile 4047 */ 4048 spin_lock(&fs_info->balance_lock); 4049 target = get_restripe_target(fs_info, flags); 4050 if (target) { 4051 /* pick target profile only if it's already available */ 4052 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) { 4053 spin_unlock(&fs_info->balance_lock); 4054 return extended_to_chunk(target); 4055 } 4056 } 4057 spin_unlock(&fs_info->balance_lock); 4058 4059 /* First, mask out the RAID levels which aren't possible */ 4060 for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) { 4061 if (num_devices >= btrfs_raid_array[raid_type].devs_min) 4062 allowed |= btrfs_raid_group[raid_type]; 4063 } 4064 allowed &= flags; 4065 4066 if (allowed & BTRFS_BLOCK_GROUP_RAID6) 4067 allowed = BTRFS_BLOCK_GROUP_RAID6; 4068 else if (allowed & BTRFS_BLOCK_GROUP_RAID5) 4069 allowed = BTRFS_BLOCK_GROUP_RAID5; 4070 else if (allowed & BTRFS_BLOCK_GROUP_RAID10) 4071 allowed = BTRFS_BLOCK_GROUP_RAID10; 4072 else if (allowed & BTRFS_BLOCK_GROUP_RAID1) 4073 allowed = BTRFS_BLOCK_GROUP_RAID1; 4074 else if (allowed & BTRFS_BLOCK_GROUP_RAID0) 4075 allowed = BTRFS_BLOCK_GROUP_RAID0; 4076 4077 flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK; 4078 4079 return extended_to_chunk(flags | allowed); 4080 } 4081 4082 static u64 get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags) 4083 { 4084 unsigned seq; 4085 u64 flags; 4086 4087 do { 4088 flags = orig_flags; 4089 seq = read_seqbegin(&fs_info->profiles_lock); 4090 4091 if (flags & BTRFS_BLOCK_GROUP_DATA) 4092 flags |= fs_info->avail_data_alloc_bits; 4093 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 4094 flags |= fs_info->avail_system_alloc_bits; 4095 else if (flags & BTRFS_BLOCK_GROUP_METADATA) 4096 flags |= fs_info->avail_metadata_alloc_bits; 4097 } while (read_seqretry(&fs_info->profiles_lock, seq)); 4098 4099 return btrfs_reduce_alloc_profile(fs_info, flags); 4100 } 4101 4102 u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data) 4103 { 4104 struct btrfs_fs_info *fs_info = root->fs_info; 4105 u64 flags; 4106 u64 ret; 4107 4108 if (data) 4109 flags = BTRFS_BLOCK_GROUP_DATA; 4110 else if (root == fs_info->chunk_root) 4111 flags = BTRFS_BLOCK_GROUP_SYSTEM; 4112 else 4113 flags = BTRFS_BLOCK_GROUP_METADATA; 4114 4115 ret = get_alloc_profile(fs_info, flags); 4116 return ret; 4117 } 4118 4119 int btrfs_alloc_data_chunk_ondemand(struct inode *inode, u64 bytes) 4120 { 4121 struct btrfs_space_info *data_sinfo; 4122 struct btrfs_root *root = BTRFS_I(inode)->root; 4123 struct btrfs_fs_info *fs_info = root->fs_info; 4124 u64 used; 4125 int ret = 0; 4126 int need_commit = 2; 4127 int have_pinned_space; 4128 4129 /* make sure bytes are sectorsize aligned */ 4130 bytes = ALIGN(bytes, fs_info->sectorsize); 4131 4132 if (btrfs_is_free_space_inode(inode)) { 4133 need_commit = 0; 4134 ASSERT(current->journal_info); 4135 } 4136 4137 data_sinfo = fs_info->data_sinfo; 4138 if (!data_sinfo) 4139 goto alloc; 4140 4141 again: 4142 /* make sure we have enough space to handle the data first */ 4143 spin_lock(&data_sinfo->lock); 4144 used = data_sinfo->bytes_used + data_sinfo->bytes_reserved + 4145 data_sinfo->bytes_pinned + data_sinfo->bytes_readonly + 4146 data_sinfo->bytes_may_use; 4147 4148 if (used + bytes > data_sinfo->total_bytes) { 4149 struct btrfs_trans_handle *trans; 4150 4151 /* 4152 * if we don't have enough free bytes in this space then we need 4153 * to alloc a new chunk. 4154 */ 4155 if (!data_sinfo->full) { 4156 u64 alloc_target; 4157 4158 data_sinfo->force_alloc = CHUNK_ALLOC_FORCE; 4159 spin_unlock(&data_sinfo->lock); 4160 alloc: 4161 alloc_target = btrfs_get_alloc_profile(root, 1); 4162 /* 4163 * It is ugly that we don't call nolock join 4164 * transaction for the free space inode case here. 4165 * But it is safe because we only do the data space 4166 * reservation for the free space cache in the 4167 * transaction context, the common join transaction 4168 * just increase the counter of the current transaction 4169 * handler, doesn't try to acquire the trans_lock of 4170 * the fs. 4171 */ 4172 trans = btrfs_join_transaction(root); 4173 if (IS_ERR(trans)) 4174 return PTR_ERR(trans); 4175 4176 ret = do_chunk_alloc(trans, fs_info, alloc_target, 4177 CHUNK_ALLOC_NO_FORCE); 4178 btrfs_end_transaction(trans); 4179 if (ret < 0) { 4180 if (ret != -ENOSPC) 4181 return ret; 4182 else { 4183 have_pinned_space = 1; 4184 goto commit_trans; 4185 } 4186 } 4187 4188 if (!data_sinfo) 4189 data_sinfo = fs_info->data_sinfo; 4190 4191 goto again; 4192 } 4193 4194 /* 4195 * If we don't have enough pinned space to deal with this 4196 * allocation, and no removed chunk in current transaction, 4197 * don't bother committing the transaction. 4198 */ 4199 have_pinned_space = percpu_counter_compare( 4200 &data_sinfo->total_bytes_pinned, 4201 used + bytes - data_sinfo->total_bytes); 4202 spin_unlock(&data_sinfo->lock); 4203 4204 /* commit the current transaction and try again */ 4205 commit_trans: 4206 if (need_commit && 4207 !atomic_read(&fs_info->open_ioctl_trans)) { 4208 need_commit--; 4209 4210 if (need_commit > 0) { 4211 btrfs_start_delalloc_roots(fs_info, 0, -1); 4212 btrfs_wait_ordered_roots(fs_info, -1, 0, 4213 (u64)-1); 4214 } 4215 4216 trans = btrfs_join_transaction(root); 4217 if (IS_ERR(trans)) 4218 return PTR_ERR(trans); 4219 if (have_pinned_space >= 0 || 4220 test_bit(BTRFS_TRANS_HAVE_FREE_BGS, 4221 &trans->transaction->flags) || 4222 need_commit > 0) { 4223 ret = btrfs_commit_transaction(trans); 4224 if (ret) 4225 return ret; 4226 /* 4227 * The cleaner kthread might still be doing iput 4228 * operations. Wait for it to finish so that 4229 * more space is released. 4230 */ 4231 mutex_lock(&fs_info->cleaner_delayed_iput_mutex); 4232 mutex_unlock(&fs_info->cleaner_delayed_iput_mutex); 4233 goto again; 4234 } else { 4235 btrfs_end_transaction(trans); 4236 } 4237 } 4238 4239 trace_btrfs_space_reservation(fs_info, 4240 "space_info:enospc", 4241 data_sinfo->flags, bytes, 1); 4242 return -ENOSPC; 4243 } 4244 data_sinfo->bytes_may_use += bytes; 4245 trace_btrfs_space_reservation(fs_info, "space_info", 4246 data_sinfo->flags, bytes, 1); 4247 spin_unlock(&data_sinfo->lock); 4248 4249 return ret; 4250 } 4251 4252 /* 4253 * New check_data_free_space() with ability for precious data reservation 4254 * Will replace old btrfs_check_data_free_space(), but for patch split, 4255 * add a new function first and then replace it. 4256 */ 4257 int btrfs_check_data_free_space(struct inode *inode, u64 start, u64 len) 4258 { 4259 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4260 int ret; 4261 4262 /* align the range */ 4263 len = round_up(start + len, fs_info->sectorsize) - 4264 round_down(start, fs_info->sectorsize); 4265 start = round_down(start, fs_info->sectorsize); 4266 4267 ret = btrfs_alloc_data_chunk_ondemand(inode, len); 4268 if (ret < 0) 4269 return ret; 4270 4271 /* Use new btrfs_qgroup_reserve_data to reserve precious data space. */ 4272 ret = btrfs_qgroup_reserve_data(inode, start, len); 4273 if (ret) 4274 btrfs_free_reserved_data_space_noquota(inode, start, len); 4275 return ret; 4276 } 4277 4278 /* 4279 * Called if we need to clear a data reservation for this inode 4280 * Normally in a error case. 4281 * 4282 * This one will *NOT* use accurate qgroup reserved space API, just for case 4283 * which we can't sleep and is sure it won't affect qgroup reserved space. 4284 * Like clear_bit_hook(). 4285 */ 4286 void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start, 4287 u64 len) 4288 { 4289 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4290 struct btrfs_space_info *data_sinfo; 4291 4292 /* Make sure the range is aligned to sectorsize */ 4293 len = round_up(start + len, fs_info->sectorsize) - 4294 round_down(start, fs_info->sectorsize); 4295 start = round_down(start, fs_info->sectorsize); 4296 4297 data_sinfo = fs_info->data_sinfo; 4298 spin_lock(&data_sinfo->lock); 4299 if (WARN_ON(data_sinfo->bytes_may_use < len)) 4300 data_sinfo->bytes_may_use = 0; 4301 else 4302 data_sinfo->bytes_may_use -= len; 4303 trace_btrfs_space_reservation(fs_info, "space_info", 4304 data_sinfo->flags, len, 0); 4305 spin_unlock(&data_sinfo->lock); 4306 } 4307 4308 /* 4309 * Called if we need to clear a data reservation for this inode 4310 * Normally in a error case. 4311 * 4312 * This one will handle the per-inode data rsv map for accurate reserved 4313 * space framework. 4314 */ 4315 void btrfs_free_reserved_data_space(struct inode *inode, u64 start, u64 len) 4316 { 4317 struct btrfs_root *root = BTRFS_I(inode)->root; 4318 4319 /* Make sure the range is aligned to sectorsize */ 4320 len = round_up(start + len, root->fs_info->sectorsize) - 4321 round_down(start, root->fs_info->sectorsize); 4322 start = round_down(start, root->fs_info->sectorsize); 4323 4324 btrfs_free_reserved_data_space_noquota(inode, start, len); 4325 btrfs_qgroup_free_data(inode, start, len); 4326 } 4327 4328 static void force_metadata_allocation(struct btrfs_fs_info *info) 4329 { 4330 struct list_head *head = &info->space_info; 4331 struct btrfs_space_info *found; 4332 4333 rcu_read_lock(); 4334 list_for_each_entry_rcu(found, head, list) { 4335 if (found->flags & BTRFS_BLOCK_GROUP_METADATA) 4336 found->force_alloc = CHUNK_ALLOC_FORCE; 4337 } 4338 rcu_read_unlock(); 4339 } 4340 4341 static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global) 4342 { 4343 return (global->size << 1); 4344 } 4345 4346 static int should_alloc_chunk(struct btrfs_fs_info *fs_info, 4347 struct btrfs_space_info *sinfo, int force) 4348 { 4349 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 4350 u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly; 4351 u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved; 4352 u64 thresh; 4353 4354 if (force == CHUNK_ALLOC_FORCE) 4355 return 1; 4356 4357 /* 4358 * We need to take into account the global rsv because for all intents 4359 * and purposes it's used space. Don't worry about locking the 4360 * global_rsv, it doesn't change except when the transaction commits. 4361 */ 4362 if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA) 4363 num_allocated += calc_global_rsv_need_space(global_rsv); 4364 4365 /* 4366 * in limited mode, we want to have some free space up to 4367 * about 1% of the FS size. 4368 */ 4369 if (force == CHUNK_ALLOC_LIMITED) { 4370 thresh = btrfs_super_total_bytes(fs_info->super_copy); 4371 thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1)); 4372 4373 if (num_bytes - num_allocated < thresh) 4374 return 1; 4375 } 4376 4377 if (num_allocated + SZ_2M < div_factor(num_bytes, 8)) 4378 return 0; 4379 return 1; 4380 } 4381 4382 static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type) 4383 { 4384 u64 num_dev; 4385 4386 if (type & (BTRFS_BLOCK_GROUP_RAID10 | 4387 BTRFS_BLOCK_GROUP_RAID0 | 4388 BTRFS_BLOCK_GROUP_RAID5 | 4389 BTRFS_BLOCK_GROUP_RAID6)) 4390 num_dev = fs_info->fs_devices->rw_devices; 4391 else if (type & BTRFS_BLOCK_GROUP_RAID1) 4392 num_dev = 2; 4393 else 4394 num_dev = 1; /* DUP or single */ 4395 4396 return num_dev; 4397 } 4398 4399 /* 4400 * If @is_allocation is true, reserve space in the system space info necessary 4401 * for allocating a chunk, otherwise if it's false, reserve space necessary for 4402 * removing a chunk. 4403 */ 4404 void check_system_chunk(struct btrfs_trans_handle *trans, 4405 struct btrfs_fs_info *fs_info, u64 type) 4406 { 4407 struct btrfs_space_info *info; 4408 u64 left; 4409 u64 thresh; 4410 int ret = 0; 4411 u64 num_devs; 4412 4413 /* 4414 * Needed because we can end up allocating a system chunk and for an 4415 * atomic and race free space reservation in the chunk block reserve. 4416 */ 4417 ASSERT(mutex_is_locked(&fs_info->chunk_mutex)); 4418 4419 info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); 4420 spin_lock(&info->lock); 4421 left = info->total_bytes - info->bytes_used - info->bytes_pinned - 4422 info->bytes_reserved - info->bytes_readonly - 4423 info->bytes_may_use; 4424 spin_unlock(&info->lock); 4425 4426 num_devs = get_profile_num_devs(fs_info, type); 4427 4428 /* num_devs device items to update and 1 chunk item to add or remove */ 4429 thresh = btrfs_calc_trunc_metadata_size(fs_info, num_devs) + 4430 btrfs_calc_trans_metadata_size(fs_info, 1); 4431 4432 if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { 4433 btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu", 4434 left, thresh, type); 4435 dump_space_info(fs_info, info, 0, 0); 4436 } 4437 4438 if (left < thresh) { 4439 u64 flags; 4440 4441 flags = btrfs_get_alloc_profile(fs_info->chunk_root, 0); 4442 /* 4443 * Ignore failure to create system chunk. We might end up not 4444 * needing it, as we might not need to COW all nodes/leafs from 4445 * the paths we visit in the chunk tree (they were already COWed 4446 * or created in the current transaction for example). 4447 */ 4448 ret = btrfs_alloc_chunk(trans, fs_info, flags); 4449 } 4450 4451 if (!ret) { 4452 ret = btrfs_block_rsv_add(fs_info->chunk_root, 4453 &fs_info->chunk_block_rsv, 4454 thresh, BTRFS_RESERVE_NO_FLUSH); 4455 if (!ret) 4456 trans->chunk_bytes_reserved += thresh; 4457 } 4458 } 4459 4460 /* 4461 * If force is CHUNK_ALLOC_FORCE: 4462 * - return 1 if it successfully allocates a chunk, 4463 * - return errors including -ENOSPC otherwise. 4464 * If force is NOT CHUNK_ALLOC_FORCE: 4465 * - return 0 if it doesn't need to allocate a new chunk, 4466 * - return 1 if it successfully allocates a chunk, 4467 * - return errors including -ENOSPC otherwise. 4468 */ 4469 static int do_chunk_alloc(struct btrfs_trans_handle *trans, 4470 struct btrfs_fs_info *fs_info, u64 flags, int force) 4471 { 4472 struct btrfs_space_info *space_info; 4473 int wait_for_alloc = 0; 4474 int ret = 0; 4475 4476 /* Don't re-enter if we're already allocating a chunk */ 4477 if (trans->allocating_chunk) 4478 return -ENOSPC; 4479 4480 space_info = __find_space_info(fs_info, flags); 4481 if (!space_info) { 4482 ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info); 4483 BUG_ON(ret); /* -ENOMEM */ 4484 } 4485 BUG_ON(!space_info); /* Logic error */ 4486 4487 again: 4488 spin_lock(&space_info->lock); 4489 if (force < space_info->force_alloc) 4490 force = space_info->force_alloc; 4491 if (space_info->full) { 4492 if (should_alloc_chunk(fs_info, space_info, force)) 4493 ret = -ENOSPC; 4494 else 4495 ret = 0; 4496 spin_unlock(&space_info->lock); 4497 return ret; 4498 } 4499 4500 if (!should_alloc_chunk(fs_info, space_info, force)) { 4501 spin_unlock(&space_info->lock); 4502 return 0; 4503 } else if (space_info->chunk_alloc) { 4504 wait_for_alloc = 1; 4505 } else { 4506 space_info->chunk_alloc = 1; 4507 } 4508 4509 spin_unlock(&space_info->lock); 4510 4511 mutex_lock(&fs_info->chunk_mutex); 4512 4513 /* 4514 * The chunk_mutex is held throughout the entirety of a chunk 4515 * allocation, so once we've acquired the chunk_mutex we know that the 4516 * other guy is done and we need to recheck and see if we should 4517 * allocate. 4518 */ 4519 if (wait_for_alloc) { 4520 mutex_unlock(&fs_info->chunk_mutex); 4521 wait_for_alloc = 0; 4522 goto again; 4523 } 4524 4525 trans->allocating_chunk = true; 4526 4527 /* 4528 * If we have mixed data/metadata chunks we want to make sure we keep 4529 * allocating mixed chunks instead of individual chunks. 4530 */ 4531 if (btrfs_mixed_space_info(space_info)) 4532 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA); 4533 4534 /* 4535 * if we're doing a data chunk, go ahead and make sure that 4536 * we keep a reasonable number of metadata chunks allocated in the 4537 * FS as well. 4538 */ 4539 if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) { 4540 fs_info->data_chunk_allocations++; 4541 if (!(fs_info->data_chunk_allocations % 4542 fs_info->metadata_ratio)) 4543 force_metadata_allocation(fs_info); 4544 } 4545 4546 /* 4547 * Check if we have enough space in SYSTEM chunk because we may need 4548 * to update devices. 4549 */ 4550 check_system_chunk(trans, fs_info, flags); 4551 4552 ret = btrfs_alloc_chunk(trans, fs_info, flags); 4553 trans->allocating_chunk = false; 4554 4555 spin_lock(&space_info->lock); 4556 if (ret < 0 && ret != -ENOSPC) 4557 goto out; 4558 if (ret) 4559 space_info->full = 1; 4560 else 4561 ret = 1; 4562 4563 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; 4564 out: 4565 space_info->chunk_alloc = 0; 4566 spin_unlock(&space_info->lock); 4567 mutex_unlock(&fs_info->chunk_mutex); 4568 /* 4569 * When we allocate a new chunk we reserve space in the chunk block 4570 * reserve to make sure we can COW nodes/leafs in the chunk tree or 4571 * add new nodes/leafs to it if we end up needing to do it when 4572 * inserting the chunk item and updating device items as part of the 4573 * second phase of chunk allocation, performed by 4574 * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a 4575 * large number of new block groups to create in our transaction 4576 * handle's new_bgs list to avoid exhausting the chunk block reserve 4577 * in extreme cases - like having a single transaction create many new 4578 * block groups when starting to write out the free space caches of all 4579 * the block groups that were made dirty during the lifetime of the 4580 * transaction. 4581 */ 4582 if (trans->can_flush_pending_bgs && 4583 trans->chunk_bytes_reserved >= (u64)SZ_2M) { 4584 btrfs_create_pending_block_groups(trans, fs_info); 4585 btrfs_trans_release_chunk_metadata(trans); 4586 } 4587 return ret; 4588 } 4589 4590 static int can_overcommit(struct btrfs_root *root, 4591 struct btrfs_space_info *space_info, u64 bytes, 4592 enum btrfs_reserve_flush_enum flush) 4593 { 4594 struct btrfs_fs_info *fs_info = root->fs_info; 4595 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 4596 u64 profile; 4597 u64 space_size; 4598 u64 avail; 4599 u64 used; 4600 4601 /* Don't overcommit when in mixed mode. */ 4602 if (space_info->flags & BTRFS_BLOCK_GROUP_DATA) 4603 return 0; 4604 4605 profile = btrfs_get_alloc_profile(root, 0); 4606 used = space_info->bytes_used + space_info->bytes_reserved + 4607 space_info->bytes_pinned + space_info->bytes_readonly; 4608 4609 /* 4610 * We only want to allow over committing if we have lots of actual space 4611 * free, but if we don't have enough space to handle the global reserve 4612 * space then we could end up having a real enospc problem when trying 4613 * to allocate a chunk or some other such important allocation. 4614 */ 4615 spin_lock(&global_rsv->lock); 4616 space_size = calc_global_rsv_need_space(global_rsv); 4617 spin_unlock(&global_rsv->lock); 4618 if (used + space_size >= space_info->total_bytes) 4619 return 0; 4620 4621 used += space_info->bytes_may_use; 4622 4623 spin_lock(&fs_info->free_chunk_lock); 4624 avail = fs_info->free_chunk_space; 4625 spin_unlock(&fs_info->free_chunk_lock); 4626 4627 /* 4628 * If we have dup, raid1 or raid10 then only half of the free 4629 * space is actually useable. For raid56, the space info used 4630 * doesn't include the parity drive, so we don't have to 4631 * change the math 4632 */ 4633 if (profile & (BTRFS_BLOCK_GROUP_DUP | 4634 BTRFS_BLOCK_GROUP_RAID1 | 4635 BTRFS_BLOCK_GROUP_RAID10)) 4636 avail >>= 1; 4637 4638 /* 4639 * If we aren't flushing all things, let us overcommit up to 4640 * 1/2th of the space. If we can flush, don't let us overcommit 4641 * too much, let it overcommit up to 1/8 of the space. 4642 */ 4643 if (flush == BTRFS_RESERVE_FLUSH_ALL) 4644 avail >>= 3; 4645 else 4646 avail >>= 1; 4647 4648 if (used + bytes < space_info->total_bytes + avail) 4649 return 1; 4650 return 0; 4651 } 4652 4653 static void btrfs_writeback_inodes_sb_nr(struct btrfs_fs_info *fs_info, 4654 unsigned long nr_pages, int nr_items) 4655 { 4656 struct super_block *sb = fs_info->sb; 4657 4658 if (down_read_trylock(&sb->s_umount)) { 4659 writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE); 4660 up_read(&sb->s_umount); 4661 } else { 4662 /* 4663 * We needn't worry the filesystem going from r/w to r/o though 4664 * we don't acquire ->s_umount mutex, because the filesystem 4665 * should guarantee the delalloc inodes list be empty after 4666 * the filesystem is readonly(all dirty pages are written to 4667 * the disk). 4668 */ 4669 btrfs_start_delalloc_roots(fs_info, 0, nr_items); 4670 if (!current->journal_info) 4671 btrfs_wait_ordered_roots(fs_info, nr_items, 0, (u64)-1); 4672 } 4673 } 4674 4675 static inline int calc_reclaim_items_nr(struct btrfs_fs_info *fs_info, 4676 u64 to_reclaim) 4677 { 4678 u64 bytes; 4679 int nr; 4680 4681 bytes = btrfs_calc_trans_metadata_size(fs_info, 1); 4682 nr = (int)div64_u64(to_reclaim, bytes); 4683 if (!nr) 4684 nr = 1; 4685 return nr; 4686 } 4687 4688 #define EXTENT_SIZE_PER_ITEM SZ_256K 4689 4690 /* 4691 * shrink metadata reservation for delalloc 4692 */ 4693 static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig, 4694 bool wait_ordered) 4695 { 4696 struct btrfs_fs_info *fs_info = root->fs_info; 4697 struct btrfs_block_rsv *block_rsv; 4698 struct btrfs_space_info *space_info; 4699 struct btrfs_trans_handle *trans; 4700 u64 delalloc_bytes; 4701 u64 max_reclaim; 4702 long time_left; 4703 unsigned long nr_pages; 4704 int loops; 4705 int items; 4706 enum btrfs_reserve_flush_enum flush; 4707 4708 /* Calc the number of the pages we need flush for space reservation */ 4709 items = calc_reclaim_items_nr(fs_info, to_reclaim); 4710 to_reclaim = (u64)items * EXTENT_SIZE_PER_ITEM; 4711 4712 trans = (struct btrfs_trans_handle *)current->journal_info; 4713 block_rsv = &fs_info->delalloc_block_rsv; 4714 space_info = block_rsv->space_info; 4715 4716 delalloc_bytes = percpu_counter_sum_positive( 4717 &fs_info->delalloc_bytes); 4718 if (delalloc_bytes == 0) { 4719 if (trans) 4720 return; 4721 if (wait_ordered) 4722 btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1); 4723 return; 4724 } 4725 4726 loops = 0; 4727 while (delalloc_bytes && loops < 3) { 4728 max_reclaim = min(delalloc_bytes, to_reclaim); 4729 nr_pages = max_reclaim >> PAGE_SHIFT; 4730 btrfs_writeback_inodes_sb_nr(fs_info, nr_pages, items); 4731 /* 4732 * We need to wait for the async pages to actually start before 4733 * we do anything. 4734 */ 4735 max_reclaim = atomic_read(&fs_info->async_delalloc_pages); 4736 if (!max_reclaim) 4737 goto skip_async; 4738 4739 if (max_reclaim <= nr_pages) 4740 max_reclaim = 0; 4741 else 4742 max_reclaim -= nr_pages; 4743 4744 wait_event(fs_info->async_submit_wait, 4745 atomic_read(&fs_info->async_delalloc_pages) <= 4746 (int)max_reclaim); 4747 skip_async: 4748 if (!trans) 4749 flush = BTRFS_RESERVE_FLUSH_ALL; 4750 else 4751 flush = BTRFS_RESERVE_NO_FLUSH; 4752 spin_lock(&space_info->lock); 4753 if (can_overcommit(root, space_info, orig, flush)) { 4754 spin_unlock(&space_info->lock); 4755 break; 4756 } 4757 if (list_empty(&space_info->tickets) && 4758 list_empty(&space_info->priority_tickets)) { 4759 spin_unlock(&space_info->lock); 4760 break; 4761 } 4762 spin_unlock(&space_info->lock); 4763 4764 loops++; 4765 if (wait_ordered && !trans) { 4766 btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1); 4767 } else { 4768 time_left = schedule_timeout_killable(1); 4769 if (time_left) 4770 break; 4771 } 4772 delalloc_bytes = percpu_counter_sum_positive( 4773 &fs_info->delalloc_bytes); 4774 } 4775 } 4776 4777 /** 4778 * maybe_commit_transaction - possibly commit the transaction if its ok to 4779 * @root - the root we're allocating for 4780 * @bytes - the number of bytes we want to reserve 4781 * @force - force the commit 4782 * 4783 * This will check to make sure that committing the transaction will actually 4784 * get us somewhere and then commit the transaction if it does. Otherwise it 4785 * will return -ENOSPC. 4786 */ 4787 static int may_commit_transaction(struct btrfs_root *root, 4788 struct btrfs_space_info *space_info, 4789 u64 bytes, int force) 4790 { 4791 struct btrfs_fs_info *fs_info = root->fs_info; 4792 struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv; 4793 struct btrfs_trans_handle *trans; 4794 4795 trans = (struct btrfs_trans_handle *)current->journal_info; 4796 if (trans) 4797 return -EAGAIN; 4798 4799 if (force) 4800 goto commit; 4801 4802 /* See if there is enough pinned space to make this reservation */ 4803 if (percpu_counter_compare(&space_info->total_bytes_pinned, 4804 bytes) >= 0) 4805 goto commit; 4806 4807 /* 4808 * See if there is some space in the delayed insertion reservation for 4809 * this reservation. 4810 */ 4811 if (space_info != delayed_rsv->space_info) 4812 return -ENOSPC; 4813 4814 spin_lock(&delayed_rsv->lock); 4815 if (percpu_counter_compare(&space_info->total_bytes_pinned, 4816 bytes - delayed_rsv->size) >= 0) { 4817 spin_unlock(&delayed_rsv->lock); 4818 return -ENOSPC; 4819 } 4820 spin_unlock(&delayed_rsv->lock); 4821 4822 commit: 4823 trans = btrfs_join_transaction(root); 4824 if (IS_ERR(trans)) 4825 return -ENOSPC; 4826 4827 return btrfs_commit_transaction(trans); 4828 } 4829 4830 struct reserve_ticket { 4831 u64 bytes; 4832 int error; 4833 struct list_head list; 4834 wait_queue_head_t wait; 4835 }; 4836 4837 static int flush_space(struct btrfs_root *root, 4838 struct btrfs_space_info *space_info, u64 num_bytes, 4839 u64 orig_bytes, int state) 4840 { 4841 struct btrfs_fs_info *fs_info = root->fs_info; 4842 struct btrfs_trans_handle *trans; 4843 int nr; 4844 int ret = 0; 4845 4846 switch (state) { 4847 case FLUSH_DELAYED_ITEMS_NR: 4848 case FLUSH_DELAYED_ITEMS: 4849 if (state == FLUSH_DELAYED_ITEMS_NR) 4850 nr = calc_reclaim_items_nr(fs_info, num_bytes) * 2; 4851 else 4852 nr = -1; 4853 4854 trans = btrfs_join_transaction(root); 4855 if (IS_ERR(trans)) { 4856 ret = PTR_ERR(trans); 4857 break; 4858 } 4859 ret = btrfs_run_delayed_items_nr(trans, fs_info, nr); 4860 btrfs_end_transaction(trans); 4861 break; 4862 case FLUSH_DELALLOC: 4863 case FLUSH_DELALLOC_WAIT: 4864 shrink_delalloc(root, num_bytes * 2, orig_bytes, 4865 state == FLUSH_DELALLOC_WAIT); 4866 break; 4867 case ALLOC_CHUNK: 4868 trans = btrfs_join_transaction(root); 4869 if (IS_ERR(trans)) { 4870 ret = PTR_ERR(trans); 4871 break; 4872 } 4873 ret = do_chunk_alloc(trans, fs_info, 4874 btrfs_get_alloc_profile(root, 0), 4875 CHUNK_ALLOC_NO_FORCE); 4876 btrfs_end_transaction(trans); 4877 if (ret > 0 || ret == -ENOSPC) 4878 ret = 0; 4879 break; 4880 case COMMIT_TRANS: 4881 ret = may_commit_transaction(root, space_info, orig_bytes, 0); 4882 break; 4883 default: 4884 ret = -ENOSPC; 4885 break; 4886 } 4887 4888 trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, 4889 orig_bytes, state, ret); 4890 return ret; 4891 } 4892 4893 static inline u64 4894 btrfs_calc_reclaim_metadata_size(struct btrfs_root *root, 4895 struct btrfs_space_info *space_info) 4896 { 4897 struct reserve_ticket *ticket; 4898 u64 used; 4899 u64 expected; 4900 u64 to_reclaim = 0; 4901 4902 list_for_each_entry(ticket, &space_info->tickets, list) 4903 to_reclaim += ticket->bytes; 4904 list_for_each_entry(ticket, &space_info->priority_tickets, list) 4905 to_reclaim += ticket->bytes; 4906 if (to_reclaim) 4907 return to_reclaim; 4908 4909 to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M); 4910 if (can_overcommit(root, space_info, to_reclaim, 4911 BTRFS_RESERVE_FLUSH_ALL)) 4912 return 0; 4913 4914 used = space_info->bytes_used + space_info->bytes_reserved + 4915 space_info->bytes_pinned + space_info->bytes_readonly + 4916 space_info->bytes_may_use; 4917 if (can_overcommit(root, space_info, SZ_1M, BTRFS_RESERVE_FLUSH_ALL)) 4918 expected = div_factor_fine(space_info->total_bytes, 95); 4919 else 4920 expected = div_factor_fine(space_info->total_bytes, 90); 4921 4922 if (used > expected) 4923 to_reclaim = used - expected; 4924 else 4925 to_reclaim = 0; 4926 to_reclaim = min(to_reclaim, space_info->bytes_may_use + 4927 space_info->bytes_reserved); 4928 return to_reclaim; 4929 } 4930 4931 static inline int need_do_async_reclaim(struct btrfs_space_info *space_info, 4932 struct btrfs_root *root, u64 used) 4933 { 4934 struct btrfs_fs_info *fs_info = root->fs_info; 4935 u64 thresh = div_factor_fine(space_info->total_bytes, 98); 4936 4937 /* If we're just plain full then async reclaim just slows us down. */ 4938 if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) 4939 return 0; 4940 4941 if (!btrfs_calc_reclaim_metadata_size(root, space_info)) 4942 return 0; 4943 4944 return (used >= thresh && !btrfs_fs_closing(fs_info) && 4945 !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); 4946 } 4947 4948 static void wake_all_tickets(struct list_head *head) 4949 { 4950 struct reserve_ticket *ticket; 4951 4952 while (!list_empty(head)) { 4953 ticket = list_first_entry(head, struct reserve_ticket, list); 4954 list_del_init(&ticket->list); 4955 ticket->error = -ENOSPC; 4956 wake_up(&ticket->wait); 4957 } 4958 } 4959 4960 /* 4961 * This is for normal flushers, we can wait all goddamned day if we want to. We 4962 * will loop and continuously try to flush as long as we are making progress. 4963 * We count progress as clearing off tickets each time we have to loop. 4964 */ 4965 static void btrfs_async_reclaim_metadata_space(struct work_struct *work) 4966 { 4967 struct btrfs_fs_info *fs_info; 4968 struct btrfs_space_info *space_info; 4969 u64 to_reclaim; 4970 int flush_state; 4971 int commit_cycles = 0; 4972 u64 last_tickets_id; 4973 4974 fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work); 4975 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); 4976 4977 spin_lock(&space_info->lock); 4978 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root, 4979 space_info); 4980 if (!to_reclaim) { 4981 space_info->flush = 0; 4982 spin_unlock(&space_info->lock); 4983 return; 4984 } 4985 last_tickets_id = space_info->tickets_id; 4986 spin_unlock(&space_info->lock); 4987 4988 flush_state = FLUSH_DELAYED_ITEMS_NR; 4989 do { 4990 struct reserve_ticket *ticket; 4991 int ret; 4992 4993 ret = flush_space(fs_info->fs_root, space_info, to_reclaim, 4994 to_reclaim, flush_state); 4995 spin_lock(&space_info->lock); 4996 if (list_empty(&space_info->tickets)) { 4997 space_info->flush = 0; 4998 spin_unlock(&space_info->lock); 4999 return; 5000 } 5001 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root, 5002 space_info); 5003 ticket = list_first_entry(&space_info->tickets, 5004 struct reserve_ticket, list); 5005 if (last_tickets_id == space_info->tickets_id) { 5006 flush_state++; 5007 } else { 5008 last_tickets_id = space_info->tickets_id; 5009 flush_state = FLUSH_DELAYED_ITEMS_NR; 5010 if (commit_cycles) 5011 commit_cycles--; 5012 } 5013 5014 if (flush_state > COMMIT_TRANS) { 5015 commit_cycles++; 5016 if (commit_cycles > 2) { 5017 wake_all_tickets(&space_info->tickets); 5018 space_info->flush = 0; 5019 } else { 5020 flush_state = FLUSH_DELAYED_ITEMS_NR; 5021 } 5022 } 5023 spin_unlock(&space_info->lock); 5024 } while (flush_state <= COMMIT_TRANS); 5025 } 5026 5027 void btrfs_init_async_reclaim_work(struct work_struct *work) 5028 { 5029 INIT_WORK(work, btrfs_async_reclaim_metadata_space); 5030 } 5031 5032 static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info, 5033 struct btrfs_space_info *space_info, 5034 struct reserve_ticket *ticket) 5035 { 5036 u64 to_reclaim; 5037 int flush_state = FLUSH_DELAYED_ITEMS_NR; 5038 5039 spin_lock(&space_info->lock); 5040 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root, 5041 space_info); 5042 if (!to_reclaim) { 5043 spin_unlock(&space_info->lock); 5044 return; 5045 } 5046 spin_unlock(&space_info->lock); 5047 5048 do { 5049 flush_space(fs_info->fs_root, space_info, to_reclaim, 5050 to_reclaim, flush_state); 5051 flush_state++; 5052 spin_lock(&space_info->lock); 5053 if (ticket->bytes == 0) { 5054 spin_unlock(&space_info->lock); 5055 return; 5056 } 5057 spin_unlock(&space_info->lock); 5058 5059 /* 5060 * Priority flushers can't wait on delalloc without 5061 * deadlocking. 5062 */ 5063 if (flush_state == FLUSH_DELALLOC || 5064 flush_state == FLUSH_DELALLOC_WAIT) 5065 flush_state = ALLOC_CHUNK; 5066 } while (flush_state < COMMIT_TRANS); 5067 } 5068 5069 static int wait_reserve_ticket(struct btrfs_fs_info *fs_info, 5070 struct btrfs_space_info *space_info, 5071 struct reserve_ticket *ticket, u64 orig_bytes) 5072 5073 { 5074 DEFINE_WAIT(wait); 5075 int ret = 0; 5076 5077 spin_lock(&space_info->lock); 5078 while (ticket->bytes > 0 && ticket->error == 0) { 5079 ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE); 5080 if (ret) { 5081 ret = -EINTR; 5082 break; 5083 } 5084 spin_unlock(&space_info->lock); 5085 5086 schedule(); 5087 5088 finish_wait(&ticket->wait, &wait); 5089 spin_lock(&space_info->lock); 5090 } 5091 if (!ret) 5092 ret = ticket->error; 5093 if (!list_empty(&ticket->list)) 5094 list_del_init(&ticket->list); 5095 if (ticket->bytes && ticket->bytes < orig_bytes) { 5096 u64 num_bytes = orig_bytes - ticket->bytes; 5097 space_info->bytes_may_use -= num_bytes; 5098 trace_btrfs_space_reservation(fs_info, "space_info", 5099 space_info->flags, num_bytes, 0); 5100 } 5101 spin_unlock(&space_info->lock); 5102 5103 return ret; 5104 } 5105 5106 /** 5107 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space 5108 * @root - the root we're allocating for 5109 * @space_info - the space info we want to allocate from 5110 * @orig_bytes - the number of bytes we want 5111 * @flush - whether or not we can flush to make our reservation 5112 * 5113 * This will reserve orig_bytes number of bytes from the space info associated 5114 * with the block_rsv. If there is not enough space it will make an attempt to 5115 * flush out space to make room. It will do this by flushing delalloc if 5116 * possible or committing the transaction. If flush is 0 then no attempts to 5117 * regain reservations will be made and this will fail if there is not enough 5118 * space already. 5119 */ 5120 static int __reserve_metadata_bytes(struct btrfs_root *root, 5121 struct btrfs_space_info *space_info, 5122 u64 orig_bytes, 5123 enum btrfs_reserve_flush_enum flush) 5124 { 5125 struct btrfs_fs_info *fs_info = root->fs_info; 5126 struct reserve_ticket ticket; 5127 u64 used; 5128 int ret = 0; 5129 5130 ASSERT(orig_bytes); 5131 ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_ALL); 5132 5133 spin_lock(&space_info->lock); 5134 ret = -ENOSPC; 5135 used = space_info->bytes_used + space_info->bytes_reserved + 5136 space_info->bytes_pinned + space_info->bytes_readonly + 5137 space_info->bytes_may_use; 5138 5139 /* 5140 * If we have enough space then hooray, make our reservation and carry 5141 * on. If not see if we can overcommit, and if we can, hooray carry on. 5142 * If not things get more complicated. 5143 */ 5144 if (used + orig_bytes <= space_info->total_bytes) { 5145 space_info->bytes_may_use += orig_bytes; 5146 trace_btrfs_space_reservation(fs_info, "space_info", 5147 space_info->flags, orig_bytes, 1); 5148 ret = 0; 5149 } else if (can_overcommit(root, space_info, orig_bytes, flush)) { 5150 space_info->bytes_may_use += orig_bytes; 5151 trace_btrfs_space_reservation(fs_info, "space_info", 5152 space_info->flags, orig_bytes, 1); 5153 ret = 0; 5154 } 5155 5156 /* 5157 * If we couldn't make a reservation then setup our reservation ticket 5158 * and kick the async worker if it's not already running. 5159 * 5160 * If we are a priority flusher then we just need to add our ticket to 5161 * the list and we will do our own flushing further down. 5162 */ 5163 if (ret && flush != BTRFS_RESERVE_NO_FLUSH) { 5164 ticket.bytes = orig_bytes; 5165 ticket.error = 0; 5166 init_waitqueue_head(&ticket.wait); 5167 if (flush == BTRFS_RESERVE_FLUSH_ALL) { 5168 list_add_tail(&ticket.list, &space_info->tickets); 5169 if (!space_info->flush) { 5170 space_info->flush = 1; 5171 trace_btrfs_trigger_flush(fs_info, 5172 space_info->flags, 5173 orig_bytes, flush, 5174 "enospc"); 5175 queue_work(system_unbound_wq, 5176 &root->fs_info->async_reclaim_work); 5177 } 5178 } else { 5179 list_add_tail(&ticket.list, 5180 &space_info->priority_tickets); 5181 } 5182 } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { 5183 used += orig_bytes; 5184 /* 5185 * We will do the space reservation dance during log replay, 5186 * which means we won't have fs_info->fs_root set, so don't do 5187 * the async reclaim as we will panic. 5188 */ 5189 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && 5190 need_do_async_reclaim(space_info, root, used) && 5191 !work_busy(&fs_info->async_reclaim_work)) { 5192 trace_btrfs_trigger_flush(fs_info, space_info->flags, 5193 orig_bytes, flush, "preempt"); 5194 queue_work(system_unbound_wq, 5195 &fs_info->async_reclaim_work); 5196 } 5197 } 5198 spin_unlock(&space_info->lock); 5199 if (!ret || flush == BTRFS_RESERVE_NO_FLUSH) 5200 return ret; 5201 5202 if (flush == BTRFS_RESERVE_FLUSH_ALL) 5203 return wait_reserve_ticket(fs_info, space_info, &ticket, 5204 orig_bytes); 5205 5206 ret = 0; 5207 priority_reclaim_metadata_space(fs_info, space_info, &ticket); 5208 spin_lock(&space_info->lock); 5209 if (ticket.bytes) { 5210 if (ticket.bytes < orig_bytes) { 5211 u64 num_bytes = orig_bytes - ticket.bytes; 5212 space_info->bytes_may_use -= num_bytes; 5213 trace_btrfs_space_reservation(fs_info, "space_info", 5214 space_info->flags, 5215 num_bytes, 0); 5216 5217 } 5218 list_del_init(&ticket.list); 5219 ret = -ENOSPC; 5220 } 5221 spin_unlock(&space_info->lock); 5222 ASSERT(list_empty(&ticket.list)); 5223 return ret; 5224 } 5225 5226 /** 5227 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space 5228 * @root - the root we're allocating for 5229 * @block_rsv - the block_rsv we're allocating for 5230 * @orig_bytes - the number of bytes we want 5231 * @flush - whether or not we can flush to make our reservation 5232 * 5233 * This will reserve orgi_bytes number of bytes from the space info associated 5234 * with the block_rsv. If there is not enough space it will make an attempt to 5235 * flush out space to make room. It will do this by flushing delalloc if 5236 * possible or committing the transaction. If flush is 0 then no attempts to 5237 * regain reservations will be made and this will fail if there is not enough 5238 * space already. 5239 */ 5240 static int reserve_metadata_bytes(struct btrfs_root *root, 5241 struct btrfs_block_rsv *block_rsv, 5242 u64 orig_bytes, 5243 enum btrfs_reserve_flush_enum flush) 5244 { 5245 struct btrfs_fs_info *fs_info = root->fs_info; 5246 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 5247 int ret; 5248 5249 ret = __reserve_metadata_bytes(root, block_rsv->space_info, orig_bytes, 5250 flush); 5251 if (ret == -ENOSPC && 5252 unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) { 5253 if (block_rsv != global_rsv && 5254 !block_rsv_use_bytes(global_rsv, orig_bytes)) 5255 ret = 0; 5256 } 5257 if (ret == -ENOSPC) 5258 trace_btrfs_space_reservation(fs_info, "space_info:enospc", 5259 block_rsv->space_info->flags, 5260 orig_bytes, 1); 5261 return ret; 5262 } 5263 5264 static struct btrfs_block_rsv *get_block_rsv( 5265 const struct btrfs_trans_handle *trans, 5266 const struct btrfs_root *root) 5267 { 5268 struct btrfs_fs_info *fs_info = root->fs_info; 5269 struct btrfs_block_rsv *block_rsv = NULL; 5270 5271 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 5272 (root == fs_info->csum_root && trans->adding_csums) || 5273 (root == fs_info->uuid_root)) 5274 block_rsv = trans->block_rsv; 5275 5276 if (!block_rsv) 5277 block_rsv = root->block_rsv; 5278 5279 if (!block_rsv) 5280 block_rsv = &fs_info->empty_block_rsv; 5281 5282 return block_rsv; 5283 } 5284 5285 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, 5286 u64 num_bytes) 5287 { 5288 int ret = -ENOSPC; 5289 spin_lock(&block_rsv->lock); 5290 if (block_rsv->reserved >= num_bytes) { 5291 block_rsv->reserved -= num_bytes; 5292 if (block_rsv->reserved < block_rsv->size) 5293 block_rsv->full = 0; 5294 ret = 0; 5295 } 5296 spin_unlock(&block_rsv->lock); 5297 return ret; 5298 } 5299 5300 static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, 5301 u64 num_bytes, int update_size) 5302 { 5303 spin_lock(&block_rsv->lock); 5304 block_rsv->reserved += num_bytes; 5305 if (update_size) 5306 block_rsv->size += num_bytes; 5307 else if (block_rsv->reserved >= block_rsv->size) 5308 block_rsv->full = 1; 5309 spin_unlock(&block_rsv->lock); 5310 } 5311 5312 int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info, 5313 struct btrfs_block_rsv *dest, u64 num_bytes, 5314 int min_factor) 5315 { 5316 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 5317 u64 min_bytes; 5318 5319 if (global_rsv->space_info != dest->space_info) 5320 return -ENOSPC; 5321 5322 spin_lock(&global_rsv->lock); 5323 min_bytes = div_factor(global_rsv->size, min_factor); 5324 if (global_rsv->reserved < min_bytes + num_bytes) { 5325 spin_unlock(&global_rsv->lock); 5326 return -ENOSPC; 5327 } 5328 global_rsv->reserved -= num_bytes; 5329 if (global_rsv->reserved < global_rsv->size) 5330 global_rsv->full = 0; 5331 spin_unlock(&global_rsv->lock); 5332 5333 block_rsv_add_bytes(dest, num_bytes, 1); 5334 return 0; 5335 } 5336 5337 /* 5338 * This is for space we already have accounted in space_info->bytes_may_use, so 5339 * basically when we're returning space from block_rsv's. 5340 */ 5341 static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info, 5342 struct btrfs_space_info *space_info, 5343 u64 num_bytes) 5344 { 5345 struct reserve_ticket *ticket; 5346 struct list_head *head; 5347 u64 used; 5348 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH; 5349 bool check_overcommit = false; 5350 5351 spin_lock(&space_info->lock); 5352 head = &space_info->priority_tickets; 5353 5354 /* 5355 * If we are over our limit then we need to check and see if we can 5356 * overcommit, and if we can't then we just need to free up our space 5357 * and not satisfy any requests. 5358 */ 5359 used = space_info->bytes_used + space_info->bytes_reserved + 5360 space_info->bytes_pinned + space_info->bytes_readonly + 5361 space_info->bytes_may_use; 5362 if (used - num_bytes >= space_info->total_bytes) 5363 check_overcommit = true; 5364 again: 5365 while (!list_empty(head) && num_bytes) { 5366 ticket = list_first_entry(head, struct reserve_ticket, 5367 list); 5368 /* 5369 * We use 0 bytes because this space is already reserved, so 5370 * adding the ticket space would be a double count. 5371 */ 5372 if (check_overcommit && 5373 !can_overcommit(fs_info->extent_root, space_info, 0, 5374 flush)) 5375 break; 5376 if (num_bytes >= ticket->bytes) { 5377 list_del_init(&ticket->list); 5378 num_bytes -= ticket->bytes; 5379 ticket->bytes = 0; 5380 space_info->tickets_id++; 5381 wake_up(&ticket->wait); 5382 } else { 5383 ticket->bytes -= num_bytes; 5384 num_bytes = 0; 5385 } 5386 } 5387 5388 if (num_bytes && head == &space_info->priority_tickets) { 5389 head = &space_info->tickets; 5390 flush = BTRFS_RESERVE_FLUSH_ALL; 5391 goto again; 5392 } 5393 space_info->bytes_may_use -= num_bytes; 5394 trace_btrfs_space_reservation(fs_info, "space_info", 5395 space_info->flags, num_bytes, 0); 5396 spin_unlock(&space_info->lock); 5397 } 5398 5399 /* 5400 * This is for newly allocated space that isn't accounted in 5401 * space_info->bytes_may_use yet. So if we allocate a chunk or unpin an extent 5402 * we use this helper. 5403 */ 5404 static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info, 5405 struct btrfs_space_info *space_info, 5406 u64 num_bytes) 5407 { 5408 struct reserve_ticket *ticket; 5409 struct list_head *head = &space_info->priority_tickets; 5410 5411 again: 5412 while (!list_empty(head) && num_bytes) { 5413 ticket = list_first_entry(head, struct reserve_ticket, 5414 list); 5415 if (num_bytes >= ticket->bytes) { 5416 trace_btrfs_space_reservation(fs_info, "space_info", 5417 space_info->flags, 5418 ticket->bytes, 1); 5419 list_del_init(&ticket->list); 5420 num_bytes -= ticket->bytes; 5421 space_info->bytes_may_use += ticket->bytes; 5422 ticket->bytes = 0; 5423 space_info->tickets_id++; 5424 wake_up(&ticket->wait); 5425 } else { 5426 trace_btrfs_space_reservation(fs_info, "space_info", 5427 space_info->flags, 5428 num_bytes, 1); 5429 space_info->bytes_may_use += num_bytes; 5430 ticket->bytes -= num_bytes; 5431 num_bytes = 0; 5432 } 5433 } 5434 5435 if (num_bytes && head == &space_info->priority_tickets) { 5436 head = &space_info->tickets; 5437 goto again; 5438 } 5439 } 5440 5441 static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info, 5442 struct btrfs_block_rsv *block_rsv, 5443 struct btrfs_block_rsv *dest, u64 num_bytes) 5444 { 5445 struct btrfs_space_info *space_info = block_rsv->space_info; 5446 5447 spin_lock(&block_rsv->lock); 5448 if (num_bytes == (u64)-1) 5449 num_bytes = block_rsv->size; 5450 block_rsv->size -= num_bytes; 5451 if (block_rsv->reserved >= block_rsv->size) { 5452 num_bytes = block_rsv->reserved - block_rsv->size; 5453 block_rsv->reserved = block_rsv->size; 5454 block_rsv->full = 1; 5455 } else { 5456 num_bytes = 0; 5457 } 5458 spin_unlock(&block_rsv->lock); 5459 5460 if (num_bytes > 0) { 5461 if (dest) { 5462 spin_lock(&dest->lock); 5463 if (!dest->full) { 5464 u64 bytes_to_add; 5465 5466 bytes_to_add = dest->size - dest->reserved; 5467 bytes_to_add = min(num_bytes, bytes_to_add); 5468 dest->reserved += bytes_to_add; 5469 if (dest->reserved >= dest->size) 5470 dest->full = 1; 5471 num_bytes -= bytes_to_add; 5472 } 5473 spin_unlock(&dest->lock); 5474 } 5475 if (num_bytes) 5476 space_info_add_old_bytes(fs_info, space_info, 5477 num_bytes); 5478 } 5479 } 5480 5481 int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src, 5482 struct btrfs_block_rsv *dst, u64 num_bytes, 5483 int update_size) 5484 { 5485 int ret; 5486 5487 ret = block_rsv_use_bytes(src, num_bytes); 5488 if (ret) 5489 return ret; 5490 5491 block_rsv_add_bytes(dst, num_bytes, update_size); 5492 return 0; 5493 } 5494 5495 void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type) 5496 { 5497 memset(rsv, 0, sizeof(*rsv)); 5498 spin_lock_init(&rsv->lock); 5499 rsv->type = type; 5500 } 5501 5502 struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, 5503 unsigned short type) 5504 { 5505 struct btrfs_block_rsv *block_rsv; 5506 5507 block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS); 5508 if (!block_rsv) 5509 return NULL; 5510 5511 btrfs_init_block_rsv(block_rsv, type); 5512 block_rsv->space_info = __find_space_info(fs_info, 5513 BTRFS_BLOCK_GROUP_METADATA); 5514 return block_rsv; 5515 } 5516 5517 void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, 5518 struct btrfs_block_rsv *rsv) 5519 { 5520 if (!rsv) 5521 return; 5522 btrfs_block_rsv_release(fs_info, rsv, (u64)-1); 5523 kfree(rsv); 5524 } 5525 5526 void __btrfs_free_block_rsv(struct btrfs_block_rsv *rsv) 5527 { 5528 kfree(rsv); 5529 } 5530 5531 int btrfs_block_rsv_add(struct btrfs_root *root, 5532 struct btrfs_block_rsv *block_rsv, u64 num_bytes, 5533 enum btrfs_reserve_flush_enum flush) 5534 { 5535 int ret; 5536 5537 if (num_bytes == 0) 5538 return 0; 5539 5540 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush); 5541 if (!ret) { 5542 block_rsv_add_bytes(block_rsv, num_bytes, 1); 5543 return 0; 5544 } 5545 5546 return ret; 5547 } 5548 5549 int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor) 5550 { 5551 u64 num_bytes = 0; 5552 int ret = -ENOSPC; 5553 5554 if (!block_rsv) 5555 return 0; 5556 5557 spin_lock(&block_rsv->lock); 5558 num_bytes = div_factor(block_rsv->size, min_factor); 5559 if (block_rsv->reserved >= num_bytes) 5560 ret = 0; 5561 spin_unlock(&block_rsv->lock); 5562 5563 return ret; 5564 } 5565 5566 int btrfs_block_rsv_refill(struct btrfs_root *root, 5567 struct btrfs_block_rsv *block_rsv, u64 min_reserved, 5568 enum btrfs_reserve_flush_enum flush) 5569 { 5570 u64 num_bytes = 0; 5571 int ret = -ENOSPC; 5572 5573 if (!block_rsv) 5574 return 0; 5575 5576 spin_lock(&block_rsv->lock); 5577 num_bytes = min_reserved; 5578 if (block_rsv->reserved >= num_bytes) 5579 ret = 0; 5580 else 5581 num_bytes -= block_rsv->reserved; 5582 spin_unlock(&block_rsv->lock); 5583 5584 if (!ret) 5585 return 0; 5586 5587 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush); 5588 if (!ret) { 5589 block_rsv_add_bytes(block_rsv, num_bytes, 0); 5590 return 0; 5591 } 5592 5593 return ret; 5594 } 5595 5596 void btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, 5597 struct btrfs_block_rsv *block_rsv, 5598 u64 num_bytes) 5599 { 5600 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 5601 5602 if (global_rsv == block_rsv || 5603 block_rsv->space_info != global_rsv->space_info) 5604 global_rsv = NULL; 5605 block_rsv_release_bytes(fs_info, block_rsv, global_rsv, num_bytes); 5606 } 5607 5608 static void update_global_block_rsv(struct btrfs_fs_info *fs_info) 5609 { 5610 struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; 5611 struct btrfs_space_info *sinfo = block_rsv->space_info; 5612 u64 num_bytes; 5613 5614 /* 5615 * The global block rsv is based on the size of the extent tree, the 5616 * checksum tree and the root tree. If the fs is empty we want to set 5617 * it to a minimal amount for safety. 5618 */ 5619 num_bytes = btrfs_root_used(&fs_info->extent_root->root_item) + 5620 btrfs_root_used(&fs_info->csum_root->root_item) + 5621 btrfs_root_used(&fs_info->tree_root->root_item); 5622 num_bytes = max_t(u64, num_bytes, SZ_16M); 5623 5624 spin_lock(&sinfo->lock); 5625 spin_lock(&block_rsv->lock); 5626 5627 block_rsv->size = min_t(u64, num_bytes, SZ_512M); 5628 5629 if (block_rsv->reserved < block_rsv->size) { 5630 num_bytes = sinfo->bytes_used + sinfo->bytes_pinned + 5631 sinfo->bytes_reserved + sinfo->bytes_readonly + 5632 sinfo->bytes_may_use; 5633 if (sinfo->total_bytes > num_bytes) { 5634 num_bytes = sinfo->total_bytes - num_bytes; 5635 num_bytes = min(num_bytes, 5636 block_rsv->size - block_rsv->reserved); 5637 block_rsv->reserved += num_bytes; 5638 sinfo->bytes_may_use += num_bytes; 5639 trace_btrfs_space_reservation(fs_info, "space_info", 5640 sinfo->flags, num_bytes, 5641 1); 5642 } 5643 } else if (block_rsv->reserved > block_rsv->size) { 5644 num_bytes = block_rsv->reserved - block_rsv->size; 5645 sinfo->bytes_may_use -= num_bytes; 5646 trace_btrfs_space_reservation(fs_info, "space_info", 5647 sinfo->flags, num_bytes, 0); 5648 block_rsv->reserved = block_rsv->size; 5649 } 5650 5651 if (block_rsv->reserved == block_rsv->size) 5652 block_rsv->full = 1; 5653 else 5654 block_rsv->full = 0; 5655 5656 spin_unlock(&block_rsv->lock); 5657 spin_unlock(&sinfo->lock); 5658 } 5659 5660 static void init_global_block_rsv(struct btrfs_fs_info *fs_info) 5661 { 5662 struct btrfs_space_info *space_info; 5663 5664 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); 5665 fs_info->chunk_block_rsv.space_info = space_info; 5666 5667 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); 5668 fs_info->global_block_rsv.space_info = space_info; 5669 fs_info->delalloc_block_rsv.space_info = space_info; 5670 fs_info->trans_block_rsv.space_info = space_info; 5671 fs_info->empty_block_rsv.space_info = space_info; 5672 fs_info->delayed_block_rsv.space_info = space_info; 5673 5674 fs_info->extent_root->block_rsv = &fs_info->global_block_rsv; 5675 fs_info->csum_root->block_rsv = &fs_info->global_block_rsv; 5676 fs_info->dev_root->block_rsv = &fs_info->global_block_rsv; 5677 fs_info->tree_root->block_rsv = &fs_info->global_block_rsv; 5678 if (fs_info->quota_root) 5679 fs_info->quota_root->block_rsv = &fs_info->global_block_rsv; 5680 fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv; 5681 5682 update_global_block_rsv(fs_info); 5683 } 5684 5685 static void release_global_block_rsv(struct btrfs_fs_info *fs_info) 5686 { 5687 block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL, 5688 (u64)-1); 5689 WARN_ON(fs_info->delalloc_block_rsv.size > 0); 5690 WARN_ON(fs_info->delalloc_block_rsv.reserved > 0); 5691 WARN_ON(fs_info->trans_block_rsv.size > 0); 5692 WARN_ON(fs_info->trans_block_rsv.reserved > 0); 5693 WARN_ON(fs_info->chunk_block_rsv.size > 0); 5694 WARN_ON(fs_info->chunk_block_rsv.reserved > 0); 5695 WARN_ON(fs_info->delayed_block_rsv.size > 0); 5696 WARN_ON(fs_info->delayed_block_rsv.reserved > 0); 5697 } 5698 5699 void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans, 5700 struct btrfs_fs_info *fs_info) 5701 { 5702 if (!trans->block_rsv) 5703 return; 5704 5705 if (!trans->bytes_reserved) 5706 return; 5707 5708 trace_btrfs_space_reservation(fs_info, "transaction", 5709 trans->transid, trans->bytes_reserved, 0); 5710 btrfs_block_rsv_release(fs_info, trans->block_rsv, 5711 trans->bytes_reserved); 5712 trans->bytes_reserved = 0; 5713 } 5714 5715 /* 5716 * To be called after all the new block groups attached to the transaction 5717 * handle have been created (btrfs_create_pending_block_groups()). 5718 */ 5719 void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans) 5720 { 5721 struct btrfs_fs_info *fs_info = trans->fs_info; 5722 5723 if (!trans->chunk_bytes_reserved) 5724 return; 5725 5726 WARN_ON_ONCE(!list_empty(&trans->new_bgs)); 5727 5728 block_rsv_release_bytes(fs_info, &fs_info->chunk_block_rsv, NULL, 5729 trans->chunk_bytes_reserved); 5730 trans->chunk_bytes_reserved = 0; 5731 } 5732 5733 /* Can only return 0 or -ENOSPC */ 5734 int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans, 5735 struct inode *inode) 5736 { 5737 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5738 struct btrfs_root *root = BTRFS_I(inode)->root; 5739 /* 5740 * We always use trans->block_rsv here as we will have reserved space 5741 * for our orphan when starting the transaction, using get_block_rsv() 5742 * here will sometimes make us choose the wrong block rsv as we could be 5743 * doing a reloc inode for a non refcounted root. 5744 */ 5745 struct btrfs_block_rsv *src_rsv = trans->block_rsv; 5746 struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv; 5747 5748 /* 5749 * We need to hold space in order to delete our orphan item once we've 5750 * added it, so this takes the reservation so we can release it later 5751 * when we are truly done with the orphan item. 5752 */ 5753 u64 num_bytes = btrfs_calc_trans_metadata_size(fs_info, 1); 5754 5755 trace_btrfs_space_reservation(fs_info, "orphan", 5756 btrfs_ino(inode), num_bytes, 1); 5757 return btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes, 1); 5758 } 5759 5760 void btrfs_orphan_release_metadata(struct inode *inode) 5761 { 5762 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5763 struct btrfs_root *root = BTRFS_I(inode)->root; 5764 u64 num_bytes = btrfs_calc_trans_metadata_size(fs_info, 1); 5765 5766 trace_btrfs_space_reservation(fs_info, "orphan", 5767 btrfs_ino(inode), num_bytes, 0); 5768 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv, num_bytes); 5769 } 5770 5771 /* 5772 * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation 5773 * root: the root of the parent directory 5774 * rsv: block reservation 5775 * items: the number of items that we need do reservation 5776 * qgroup_reserved: used to return the reserved size in qgroup 5777 * 5778 * This function is used to reserve the space for snapshot/subvolume 5779 * creation and deletion. Those operations are different with the 5780 * common file/directory operations, they change two fs/file trees 5781 * and root tree, the number of items that the qgroup reserves is 5782 * different with the free space reservation. So we can not use 5783 * the space reservation mechanism in start_transaction(). 5784 */ 5785 int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, 5786 struct btrfs_block_rsv *rsv, 5787 int items, 5788 u64 *qgroup_reserved, 5789 bool use_global_rsv) 5790 { 5791 u64 num_bytes; 5792 int ret; 5793 struct btrfs_fs_info *fs_info = root->fs_info; 5794 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 5795 5796 if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) { 5797 /* One for parent inode, two for dir entries */ 5798 num_bytes = 3 * fs_info->nodesize; 5799 ret = btrfs_qgroup_reserve_meta(root, num_bytes); 5800 if (ret) 5801 return ret; 5802 } else { 5803 num_bytes = 0; 5804 } 5805 5806 *qgroup_reserved = num_bytes; 5807 5808 num_bytes = btrfs_calc_trans_metadata_size(fs_info, items); 5809 rsv->space_info = __find_space_info(fs_info, 5810 BTRFS_BLOCK_GROUP_METADATA); 5811 ret = btrfs_block_rsv_add(root, rsv, num_bytes, 5812 BTRFS_RESERVE_FLUSH_ALL); 5813 5814 if (ret == -ENOSPC && use_global_rsv) 5815 ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes, 1); 5816 5817 if (ret && *qgroup_reserved) 5818 btrfs_qgroup_free_meta(root, *qgroup_reserved); 5819 5820 return ret; 5821 } 5822 5823 void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info, 5824 struct btrfs_block_rsv *rsv, 5825 u64 qgroup_reserved) 5826 { 5827 btrfs_block_rsv_release(fs_info, rsv, (u64)-1); 5828 } 5829 5830 /** 5831 * drop_outstanding_extent - drop an outstanding extent 5832 * @inode: the inode we're dropping the extent for 5833 * @num_bytes: the number of bytes we're releasing. 5834 * 5835 * This is called when we are freeing up an outstanding extent, either called 5836 * after an error or after an extent is written. This will return the number of 5837 * reserved extents that need to be freed. This must be called with 5838 * BTRFS_I(inode)->lock held. 5839 */ 5840 static unsigned drop_outstanding_extent(struct inode *inode, u64 num_bytes) 5841 { 5842 unsigned drop_inode_space = 0; 5843 unsigned dropped_extents = 0; 5844 unsigned num_extents = 0; 5845 5846 num_extents = (unsigned)div64_u64(num_bytes + 5847 BTRFS_MAX_EXTENT_SIZE - 1, 5848 BTRFS_MAX_EXTENT_SIZE); 5849 ASSERT(num_extents); 5850 ASSERT(BTRFS_I(inode)->outstanding_extents >= num_extents); 5851 BTRFS_I(inode)->outstanding_extents -= num_extents; 5852 5853 if (BTRFS_I(inode)->outstanding_extents == 0 && 5854 test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED, 5855 &BTRFS_I(inode)->runtime_flags)) 5856 drop_inode_space = 1; 5857 5858 /* 5859 * If we have more or the same amount of outstanding extents than we have 5860 * reserved then we need to leave the reserved extents count alone. 5861 */ 5862 if (BTRFS_I(inode)->outstanding_extents >= 5863 BTRFS_I(inode)->reserved_extents) 5864 return drop_inode_space; 5865 5866 dropped_extents = BTRFS_I(inode)->reserved_extents - 5867 BTRFS_I(inode)->outstanding_extents; 5868 BTRFS_I(inode)->reserved_extents -= dropped_extents; 5869 return dropped_extents + drop_inode_space; 5870 } 5871 5872 /** 5873 * calc_csum_metadata_size - return the amount of metadata space that must be 5874 * reserved/freed for the given bytes. 5875 * @inode: the inode we're manipulating 5876 * @num_bytes: the number of bytes in question 5877 * @reserve: 1 if we are reserving space, 0 if we are freeing space 5878 * 5879 * This adjusts the number of csum_bytes in the inode and then returns the 5880 * correct amount of metadata that must either be reserved or freed. We 5881 * calculate how many checksums we can fit into one leaf and then divide the 5882 * number of bytes that will need to be checksumed by this value to figure out 5883 * how many checksums will be required. If we are adding bytes then the number 5884 * may go up and we will return the number of additional bytes that must be 5885 * reserved. If it is going down we will return the number of bytes that must 5886 * be freed. 5887 * 5888 * This must be called with BTRFS_I(inode)->lock held. 5889 */ 5890 static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes, 5891 int reserve) 5892 { 5893 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5894 u64 old_csums, num_csums; 5895 5896 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM && 5897 BTRFS_I(inode)->csum_bytes == 0) 5898 return 0; 5899 5900 old_csums = btrfs_csum_bytes_to_leaves(fs_info, 5901 BTRFS_I(inode)->csum_bytes); 5902 if (reserve) 5903 BTRFS_I(inode)->csum_bytes += num_bytes; 5904 else 5905 BTRFS_I(inode)->csum_bytes -= num_bytes; 5906 num_csums = btrfs_csum_bytes_to_leaves(fs_info, 5907 BTRFS_I(inode)->csum_bytes); 5908 5909 /* No change, no need to reserve more */ 5910 if (old_csums == num_csums) 5911 return 0; 5912 5913 if (reserve) 5914 return btrfs_calc_trans_metadata_size(fs_info, 5915 num_csums - old_csums); 5916 5917 return btrfs_calc_trans_metadata_size(fs_info, old_csums - num_csums); 5918 } 5919 5920 int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes) 5921 { 5922 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5923 struct btrfs_root *root = BTRFS_I(inode)->root; 5924 struct btrfs_block_rsv *block_rsv = &fs_info->delalloc_block_rsv; 5925 u64 to_reserve = 0; 5926 u64 csum_bytes; 5927 unsigned nr_extents = 0; 5928 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL; 5929 int ret = 0; 5930 bool delalloc_lock = true; 5931 u64 to_free = 0; 5932 unsigned dropped; 5933 bool release_extra = false; 5934 5935 /* If we are a free space inode we need to not flush since we will be in 5936 * the middle of a transaction commit. We also don't need the delalloc 5937 * mutex since we won't race with anybody. We need this mostly to make 5938 * lockdep shut its filthy mouth. 5939 * 5940 * If we have a transaction open (can happen if we call truncate_block 5941 * from truncate), then we need FLUSH_LIMIT so we don't deadlock. 5942 */ 5943 if (btrfs_is_free_space_inode(inode)) { 5944 flush = BTRFS_RESERVE_NO_FLUSH; 5945 delalloc_lock = false; 5946 } else if (current->journal_info) { 5947 flush = BTRFS_RESERVE_FLUSH_LIMIT; 5948 } 5949 5950 if (flush != BTRFS_RESERVE_NO_FLUSH && 5951 btrfs_transaction_in_commit(fs_info)) 5952 schedule_timeout(1); 5953 5954 if (delalloc_lock) 5955 mutex_lock(&BTRFS_I(inode)->delalloc_mutex); 5956 5957 num_bytes = ALIGN(num_bytes, fs_info->sectorsize); 5958 5959 spin_lock(&BTRFS_I(inode)->lock); 5960 nr_extents = (unsigned)div64_u64(num_bytes + 5961 BTRFS_MAX_EXTENT_SIZE - 1, 5962 BTRFS_MAX_EXTENT_SIZE); 5963 BTRFS_I(inode)->outstanding_extents += nr_extents; 5964 5965 nr_extents = 0; 5966 if (BTRFS_I(inode)->outstanding_extents > 5967 BTRFS_I(inode)->reserved_extents) 5968 nr_extents += BTRFS_I(inode)->outstanding_extents - 5969 BTRFS_I(inode)->reserved_extents; 5970 5971 /* We always want to reserve a slot for updating the inode. */ 5972 to_reserve = btrfs_calc_trans_metadata_size(fs_info, nr_extents + 1); 5973 to_reserve += calc_csum_metadata_size(inode, num_bytes, 1); 5974 csum_bytes = BTRFS_I(inode)->csum_bytes; 5975 spin_unlock(&BTRFS_I(inode)->lock); 5976 5977 if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) { 5978 ret = btrfs_qgroup_reserve_meta(root, 5979 nr_extents * fs_info->nodesize); 5980 if (ret) 5981 goto out_fail; 5982 } 5983 5984 ret = btrfs_block_rsv_add(root, block_rsv, to_reserve, flush); 5985 if (unlikely(ret)) { 5986 btrfs_qgroup_free_meta(root, 5987 nr_extents * fs_info->nodesize); 5988 goto out_fail; 5989 } 5990 5991 spin_lock(&BTRFS_I(inode)->lock); 5992 if (test_and_set_bit(BTRFS_INODE_DELALLOC_META_RESERVED, 5993 &BTRFS_I(inode)->runtime_flags)) { 5994 to_reserve -= btrfs_calc_trans_metadata_size(fs_info, 1); 5995 release_extra = true; 5996 } 5997 BTRFS_I(inode)->reserved_extents += nr_extents; 5998 spin_unlock(&BTRFS_I(inode)->lock); 5999 6000 if (delalloc_lock) 6001 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex); 6002 6003 if (to_reserve) 6004 trace_btrfs_space_reservation(fs_info, "delalloc", 6005 btrfs_ino(inode), to_reserve, 1); 6006 if (release_extra) 6007 btrfs_block_rsv_release(fs_info, block_rsv, 6008 btrfs_calc_trans_metadata_size(fs_info, 1)); 6009 return 0; 6010 6011 out_fail: 6012 spin_lock(&BTRFS_I(inode)->lock); 6013 dropped = drop_outstanding_extent(inode, num_bytes); 6014 /* 6015 * If the inodes csum_bytes is the same as the original 6016 * csum_bytes then we know we haven't raced with any free()ers 6017 * so we can just reduce our inodes csum bytes and carry on. 6018 */ 6019 if (BTRFS_I(inode)->csum_bytes == csum_bytes) { 6020 calc_csum_metadata_size(inode, num_bytes, 0); 6021 } else { 6022 u64 orig_csum_bytes = BTRFS_I(inode)->csum_bytes; 6023 u64 bytes; 6024 6025 /* 6026 * This is tricky, but first we need to figure out how much we 6027 * freed from any free-ers that occurred during this 6028 * reservation, so we reset ->csum_bytes to the csum_bytes 6029 * before we dropped our lock, and then call the free for the 6030 * number of bytes that were freed while we were trying our 6031 * reservation. 6032 */ 6033 bytes = csum_bytes - BTRFS_I(inode)->csum_bytes; 6034 BTRFS_I(inode)->csum_bytes = csum_bytes; 6035 to_free = calc_csum_metadata_size(inode, bytes, 0); 6036 6037 6038 /* 6039 * Now we need to see how much we would have freed had we not 6040 * been making this reservation and our ->csum_bytes were not 6041 * artificially inflated. 6042 */ 6043 BTRFS_I(inode)->csum_bytes = csum_bytes - num_bytes; 6044 bytes = csum_bytes - orig_csum_bytes; 6045 bytes = calc_csum_metadata_size(inode, bytes, 0); 6046 6047 /* 6048 * Now reset ->csum_bytes to what it should be. If bytes is 6049 * more than to_free then we would have freed more space had we 6050 * not had an artificially high ->csum_bytes, so we need to free 6051 * the remainder. If bytes is the same or less then we don't 6052 * need to do anything, the other free-ers did the correct 6053 * thing. 6054 */ 6055 BTRFS_I(inode)->csum_bytes = orig_csum_bytes - num_bytes; 6056 if (bytes > to_free) 6057 to_free = bytes - to_free; 6058 else 6059 to_free = 0; 6060 } 6061 spin_unlock(&BTRFS_I(inode)->lock); 6062 if (dropped) 6063 to_free += btrfs_calc_trans_metadata_size(fs_info, dropped); 6064 6065 if (to_free) { 6066 btrfs_block_rsv_release(fs_info, block_rsv, to_free); 6067 trace_btrfs_space_reservation(fs_info, "delalloc", 6068 btrfs_ino(inode), to_free, 0); 6069 } 6070 if (delalloc_lock) 6071 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex); 6072 return ret; 6073 } 6074 6075 /** 6076 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode 6077 * @inode: the inode to release the reservation for 6078 * @num_bytes: the number of bytes we're releasing 6079 * 6080 * This will release the metadata reservation for an inode. This can be called 6081 * once we complete IO for a given set of bytes to release their metadata 6082 * reservations. 6083 */ 6084 void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes) 6085 { 6086 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6087 u64 to_free = 0; 6088 unsigned dropped; 6089 6090 num_bytes = ALIGN(num_bytes, fs_info->sectorsize); 6091 spin_lock(&BTRFS_I(inode)->lock); 6092 dropped = drop_outstanding_extent(inode, num_bytes); 6093 6094 if (num_bytes) 6095 to_free = calc_csum_metadata_size(inode, num_bytes, 0); 6096 spin_unlock(&BTRFS_I(inode)->lock); 6097 if (dropped > 0) 6098 to_free += btrfs_calc_trans_metadata_size(fs_info, dropped); 6099 6100 if (btrfs_is_testing(fs_info)) 6101 return; 6102 6103 trace_btrfs_space_reservation(fs_info, "delalloc", 6104 btrfs_ino(inode), to_free, 0); 6105 6106 btrfs_block_rsv_release(fs_info, &fs_info->delalloc_block_rsv, to_free); 6107 } 6108 6109 /** 6110 * btrfs_delalloc_reserve_space - reserve data and metadata space for 6111 * delalloc 6112 * @inode: inode we're writing to 6113 * @start: start range we are writing to 6114 * @len: how long the range we are writing to 6115 * 6116 * This will do the following things 6117 * 6118 * o reserve space in data space info for num bytes 6119 * and reserve precious corresponding qgroup space 6120 * (Done in check_data_free_space) 6121 * 6122 * o reserve space for metadata space, based on the number of outstanding 6123 * extents and how much csums will be needed 6124 * also reserve metadata space in a per root over-reserve method. 6125 * o add to the inodes->delalloc_bytes 6126 * o add it to the fs_info's delalloc inodes list. 6127 * (Above 3 all done in delalloc_reserve_metadata) 6128 * 6129 * Return 0 for success 6130 * Return <0 for error(-ENOSPC or -EQUOT) 6131 */ 6132 int btrfs_delalloc_reserve_space(struct inode *inode, u64 start, u64 len) 6133 { 6134 int ret; 6135 6136 ret = btrfs_check_data_free_space(inode, start, len); 6137 if (ret < 0) 6138 return ret; 6139 ret = btrfs_delalloc_reserve_metadata(inode, len); 6140 if (ret < 0) 6141 btrfs_free_reserved_data_space(inode, start, len); 6142 return ret; 6143 } 6144 6145 /** 6146 * btrfs_delalloc_release_space - release data and metadata space for delalloc 6147 * @inode: inode we're releasing space for 6148 * @start: start position of the space already reserved 6149 * @len: the len of the space already reserved 6150 * 6151 * This must be matched with a call to btrfs_delalloc_reserve_space. This is 6152 * called in the case that we don't need the metadata AND data reservations 6153 * anymore. So if there is an error or we insert an inline extent. 6154 * 6155 * This function will release the metadata space that was not used and will 6156 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes 6157 * list if there are no delalloc bytes left. 6158 * Also it will handle the qgroup reserved space. 6159 */ 6160 void btrfs_delalloc_release_space(struct inode *inode, u64 start, u64 len) 6161 { 6162 btrfs_delalloc_release_metadata(inode, len); 6163 btrfs_free_reserved_data_space(inode, start, len); 6164 } 6165 6166 static int update_block_group(struct btrfs_trans_handle *trans, 6167 struct btrfs_fs_info *info, u64 bytenr, 6168 u64 num_bytes, int alloc) 6169 { 6170 struct btrfs_block_group_cache *cache = NULL; 6171 u64 total = num_bytes; 6172 u64 old_val; 6173 u64 byte_in_group; 6174 int factor; 6175 6176 /* block accounting for super block */ 6177 spin_lock(&info->delalloc_root_lock); 6178 old_val = btrfs_super_bytes_used(info->super_copy); 6179 if (alloc) 6180 old_val += num_bytes; 6181 else 6182 old_val -= num_bytes; 6183 btrfs_set_super_bytes_used(info->super_copy, old_val); 6184 spin_unlock(&info->delalloc_root_lock); 6185 6186 while (total) { 6187 cache = btrfs_lookup_block_group(info, bytenr); 6188 if (!cache) 6189 return -ENOENT; 6190 if (cache->flags & (BTRFS_BLOCK_GROUP_DUP | 6191 BTRFS_BLOCK_GROUP_RAID1 | 6192 BTRFS_BLOCK_GROUP_RAID10)) 6193 factor = 2; 6194 else 6195 factor = 1; 6196 /* 6197 * If this block group has free space cache written out, we 6198 * need to make sure to load it if we are removing space. This 6199 * is because we need the unpinning stage to actually add the 6200 * space back to the block group, otherwise we will leak space. 6201 */ 6202 if (!alloc && cache->cached == BTRFS_CACHE_NO) 6203 cache_block_group(cache, 1); 6204 6205 byte_in_group = bytenr - cache->key.objectid; 6206 WARN_ON(byte_in_group > cache->key.offset); 6207 6208 spin_lock(&cache->space_info->lock); 6209 spin_lock(&cache->lock); 6210 6211 if (btrfs_test_opt(info, SPACE_CACHE) && 6212 cache->disk_cache_state < BTRFS_DC_CLEAR) 6213 cache->disk_cache_state = BTRFS_DC_CLEAR; 6214 6215 old_val = btrfs_block_group_used(&cache->item); 6216 num_bytes = min(total, cache->key.offset - byte_in_group); 6217 if (alloc) { 6218 old_val += num_bytes; 6219 btrfs_set_block_group_used(&cache->item, old_val); 6220 cache->reserved -= num_bytes; 6221 cache->space_info->bytes_reserved -= num_bytes; 6222 cache->space_info->bytes_used += num_bytes; 6223 cache->space_info->disk_used += num_bytes * factor; 6224 spin_unlock(&cache->lock); 6225 spin_unlock(&cache->space_info->lock); 6226 } else { 6227 old_val -= num_bytes; 6228 btrfs_set_block_group_used(&cache->item, old_val); 6229 cache->pinned += num_bytes; 6230 cache->space_info->bytes_pinned += num_bytes; 6231 cache->space_info->bytes_used -= num_bytes; 6232 cache->space_info->disk_used -= num_bytes * factor; 6233 spin_unlock(&cache->lock); 6234 spin_unlock(&cache->space_info->lock); 6235 6236 trace_btrfs_space_reservation(info, "pinned", 6237 cache->space_info->flags, 6238 num_bytes, 1); 6239 set_extent_dirty(info->pinned_extents, 6240 bytenr, bytenr + num_bytes - 1, 6241 GFP_NOFS | __GFP_NOFAIL); 6242 } 6243 6244 spin_lock(&trans->transaction->dirty_bgs_lock); 6245 if (list_empty(&cache->dirty_list)) { 6246 list_add_tail(&cache->dirty_list, 6247 &trans->transaction->dirty_bgs); 6248 trans->transaction->num_dirty_bgs++; 6249 btrfs_get_block_group(cache); 6250 } 6251 spin_unlock(&trans->transaction->dirty_bgs_lock); 6252 6253 /* 6254 * No longer have used bytes in this block group, queue it for 6255 * deletion. We do this after adding the block group to the 6256 * dirty list to avoid races between cleaner kthread and space 6257 * cache writeout. 6258 */ 6259 if (!alloc && old_val == 0) { 6260 spin_lock(&info->unused_bgs_lock); 6261 if (list_empty(&cache->bg_list)) { 6262 btrfs_get_block_group(cache); 6263 list_add_tail(&cache->bg_list, 6264 &info->unused_bgs); 6265 } 6266 spin_unlock(&info->unused_bgs_lock); 6267 } 6268 6269 btrfs_put_block_group(cache); 6270 total -= num_bytes; 6271 bytenr += num_bytes; 6272 } 6273 return 0; 6274 } 6275 6276 static u64 first_logical_byte(struct btrfs_fs_info *fs_info, u64 search_start) 6277 { 6278 struct btrfs_block_group_cache *cache; 6279 u64 bytenr; 6280 6281 spin_lock(&fs_info->block_group_cache_lock); 6282 bytenr = fs_info->first_logical_byte; 6283 spin_unlock(&fs_info->block_group_cache_lock); 6284 6285 if (bytenr < (u64)-1) 6286 return bytenr; 6287 6288 cache = btrfs_lookup_first_block_group(fs_info, search_start); 6289 if (!cache) 6290 return 0; 6291 6292 bytenr = cache->key.objectid; 6293 btrfs_put_block_group(cache); 6294 6295 return bytenr; 6296 } 6297 6298 static int pin_down_extent(struct btrfs_fs_info *fs_info, 6299 struct btrfs_block_group_cache *cache, 6300 u64 bytenr, u64 num_bytes, int reserved) 6301 { 6302 spin_lock(&cache->space_info->lock); 6303 spin_lock(&cache->lock); 6304 cache->pinned += num_bytes; 6305 cache->space_info->bytes_pinned += num_bytes; 6306 if (reserved) { 6307 cache->reserved -= num_bytes; 6308 cache->space_info->bytes_reserved -= num_bytes; 6309 } 6310 spin_unlock(&cache->lock); 6311 spin_unlock(&cache->space_info->lock); 6312 6313 trace_btrfs_space_reservation(fs_info, "pinned", 6314 cache->space_info->flags, num_bytes, 1); 6315 set_extent_dirty(fs_info->pinned_extents, bytenr, 6316 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL); 6317 return 0; 6318 } 6319 6320 /* 6321 * this function must be called within transaction 6322 */ 6323 int btrfs_pin_extent(struct btrfs_fs_info *fs_info, 6324 u64 bytenr, u64 num_bytes, int reserved) 6325 { 6326 struct btrfs_block_group_cache *cache; 6327 6328 cache = btrfs_lookup_block_group(fs_info, bytenr); 6329 BUG_ON(!cache); /* Logic error */ 6330 6331 pin_down_extent(fs_info, cache, bytenr, num_bytes, reserved); 6332 6333 btrfs_put_block_group(cache); 6334 return 0; 6335 } 6336 6337 /* 6338 * this function must be called within transaction 6339 */ 6340 int btrfs_pin_extent_for_log_replay(struct btrfs_fs_info *fs_info, 6341 u64 bytenr, u64 num_bytes) 6342 { 6343 struct btrfs_block_group_cache *cache; 6344 int ret; 6345 6346 cache = btrfs_lookup_block_group(fs_info, bytenr); 6347 if (!cache) 6348 return -EINVAL; 6349 6350 /* 6351 * pull in the free space cache (if any) so that our pin 6352 * removes the free space from the cache. We have load_only set 6353 * to one because the slow code to read in the free extents does check 6354 * the pinned extents. 6355 */ 6356 cache_block_group(cache, 1); 6357 6358 pin_down_extent(fs_info, cache, bytenr, num_bytes, 0); 6359 6360 /* remove us from the free space cache (if we're there at all) */ 6361 ret = btrfs_remove_free_space(cache, bytenr, num_bytes); 6362 btrfs_put_block_group(cache); 6363 return ret; 6364 } 6365 6366 static int __exclude_logged_extent(struct btrfs_fs_info *fs_info, 6367 u64 start, u64 num_bytes) 6368 { 6369 int ret; 6370 struct btrfs_block_group_cache *block_group; 6371 struct btrfs_caching_control *caching_ctl; 6372 6373 block_group = btrfs_lookup_block_group(fs_info, start); 6374 if (!block_group) 6375 return -EINVAL; 6376 6377 cache_block_group(block_group, 0); 6378 caching_ctl = get_caching_control(block_group); 6379 6380 if (!caching_ctl) { 6381 /* Logic error */ 6382 BUG_ON(!block_group_cache_done(block_group)); 6383 ret = btrfs_remove_free_space(block_group, start, num_bytes); 6384 } else { 6385 mutex_lock(&caching_ctl->mutex); 6386 6387 if (start >= caching_ctl->progress) { 6388 ret = add_excluded_extent(fs_info, start, num_bytes); 6389 } else if (start + num_bytes <= caching_ctl->progress) { 6390 ret = btrfs_remove_free_space(block_group, 6391 start, num_bytes); 6392 } else { 6393 num_bytes = caching_ctl->progress - start; 6394 ret = btrfs_remove_free_space(block_group, 6395 start, num_bytes); 6396 if (ret) 6397 goto out_lock; 6398 6399 num_bytes = (start + num_bytes) - 6400 caching_ctl->progress; 6401 start = caching_ctl->progress; 6402 ret = add_excluded_extent(fs_info, start, num_bytes); 6403 } 6404 out_lock: 6405 mutex_unlock(&caching_ctl->mutex); 6406 put_caching_control(caching_ctl); 6407 } 6408 btrfs_put_block_group(block_group); 6409 return ret; 6410 } 6411 6412 int btrfs_exclude_logged_extents(struct btrfs_fs_info *fs_info, 6413 struct extent_buffer *eb) 6414 { 6415 struct btrfs_file_extent_item *item; 6416 struct btrfs_key key; 6417 int found_type; 6418 int i; 6419 6420 if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS)) 6421 return 0; 6422 6423 for (i = 0; i < btrfs_header_nritems(eb); i++) { 6424 btrfs_item_key_to_cpu(eb, &key, i); 6425 if (key.type != BTRFS_EXTENT_DATA_KEY) 6426 continue; 6427 item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item); 6428 found_type = btrfs_file_extent_type(eb, item); 6429 if (found_type == BTRFS_FILE_EXTENT_INLINE) 6430 continue; 6431 if (btrfs_file_extent_disk_bytenr(eb, item) == 0) 6432 continue; 6433 key.objectid = btrfs_file_extent_disk_bytenr(eb, item); 6434 key.offset = btrfs_file_extent_disk_num_bytes(eb, item); 6435 __exclude_logged_extent(fs_info, key.objectid, key.offset); 6436 } 6437 6438 return 0; 6439 } 6440 6441 static void 6442 btrfs_inc_block_group_reservations(struct btrfs_block_group_cache *bg) 6443 { 6444 atomic_inc(&bg->reservations); 6445 } 6446 6447 void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info, 6448 const u64 start) 6449 { 6450 struct btrfs_block_group_cache *bg; 6451 6452 bg = btrfs_lookup_block_group(fs_info, start); 6453 ASSERT(bg); 6454 if (atomic_dec_and_test(&bg->reservations)) 6455 wake_up_atomic_t(&bg->reservations); 6456 btrfs_put_block_group(bg); 6457 } 6458 6459 static int btrfs_wait_bg_reservations_atomic_t(atomic_t *a) 6460 { 6461 schedule(); 6462 return 0; 6463 } 6464 6465 void btrfs_wait_block_group_reservations(struct btrfs_block_group_cache *bg) 6466 { 6467 struct btrfs_space_info *space_info = bg->space_info; 6468 6469 ASSERT(bg->ro); 6470 6471 if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA)) 6472 return; 6473 6474 /* 6475 * Our block group is read only but before we set it to read only, 6476 * some task might have had allocated an extent from it already, but it 6477 * has not yet created a respective ordered extent (and added it to a 6478 * root's list of ordered extents). 6479 * Therefore wait for any task currently allocating extents, since the 6480 * block group's reservations counter is incremented while a read lock 6481 * on the groups' semaphore is held and decremented after releasing 6482 * the read access on that semaphore and creating the ordered extent. 6483 */ 6484 down_write(&space_info->groups_sem); 6485 up_write(&space_info->groups_sem); 6486 6487 wait_on_atomic_t(&bg->reservations, 6488 btrfs_wait_bg_reservations_atomic_t, 6489 TASK_UNINTERRUPTIBLE); 6490 } 6491 6492 /** 6493 * btrfs_add_reserved_bytes - update the block_group and space info counters 6494 * @cache: The cache we are manipulating 6495 * @ram_bytes: The number of bytes of file content, and will be same to 6496 * @num_bytes except for the compress path. 6497 * @num_bytes: The number of bytes in question 6498 * @delalloc: The blocks are allocated for the delalloc write 6499 * 6500 * This is called by the allocator when it reserves space. If this is a 6501 * reservation and the block group has become read only we cannot make the 6502 * reservation and return -EAGAIN, otherwise this function always succeeds. 6503 */ 6504 static int btrfs_add_reserved_bytes(struct btrfs_block_group_cache *cache, 6505 u64 ram_bytes, u64 num_bytes, int delalloc) 6506 { 6507 struct btrfs_space_info *space_info = cache->space_info; 6508 int ret = 0; 6509 6510 spin_lock(&space_info->lock); 6511 spin_lock(&cache->lock); 6512 if (cache->ro) { 6513 ret = -EAGAIN; 6514 } else { 6515 cache->reserved += num_bytes; 6516 space_info->bytes_reserved += num_bytes; 6517 6518 trace_btrfs_space_reservation(cache->fs_info, 6519 "space_info", space_info->flags, 6520 ram_bytes, 0); 6521 space_info->bytes_may_use -= ram_bytes; 6522 if (delalloc) 6523 cache->delalloc_bytes += num_bytes; 6524 } 6525 spin_unlock(&cache->lock); 6526 spin_unlock(&space_info->lock); 6527 return ret; 6528 } 6529 6530 /** 6531 * btrfs_free_reserved_bytes - update the block_group and space info counters 6532 * @cache: The cache we are manipulating 6533 * @num_bytes: The number of bytes in question 6534 * @delalloc: The blocks are allocated for the delalloc write 6535 * 6536 * This is called by somebody who is freeing space that was never actually used 6537 * on disk. For example if you reserve some space for a new leaf in transaction 6538 * A and before transaction A commits you free that leaf, you call this with 6539 * reserve set to 0 in order to clear the reservation. 6540 */ 6541 6542 static int btrfs_free_reserved_bytes(struct btrfs_block_group_cache *cache, 6543 u64 num_bytes, int delalloc) 6544 { 6545 struct btrfs_space_info *space_info = cache->space_info; 6546 int ret = 0; 6547 6548 spin_lock(&space_info->lock); 6549 spin_lock(&cache->lock); 6550 if (cache->ro) 6551 space_info->bytes_readonly += num_bytes; 6552 cache->reserved -= num_bytes; 6553 space_info->bytes_reserved -= num_bytes; 6554 6555 if (delalloc) 6556 cache->delalloc_bytes -= num_bytes; 6557 spin_unlock(&cache->lock); 6558 spin_unlock(&space_info->lock); 6559 return ret; 6560 } 6561 void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans, 6562 struct btrfs_fs_info *fs_info) 6563 { 6564 struct btrfs_caching_control *next; 6565 struct btrfs_caching_control *caching_ctl; 6566 struct btrfs_block_group_cache *cache; 6567 6568 down_write(&fs_info->commit_root_sem); 6569 6570 list_for_each_entry_safe(caching_ctl, next, 6571 &fs_info->caching_block_groups, list) { 6572 cache = caching_ctl->block_group; 6573 if (block_group_cache_done(cache)) { 6574 cache->last_byte_to_unpin = (u64)-1; 6575 list_del_init(&caching_ctl->list); 6576 put_caching_control(caching_ctl); 6577 } else { 6578 cache->last_byte_to_unpin = caching_ctl->progress; 6579 } 6580 } 6581 6582 if (fs_info->pinned_extents == &fs_info->freed_extents[0]) 6583 fs_info->pinned_extents = &fs_info->freed_extents[1]; 6584 else 6585 fs_info->pinned_extents = &fs_info->freed_extents[0]; 6586 6587 up_write(&fs_info->commit_root_sem); 6588 6589 update_global_block_rsv(fs_info); 6590 } 6591 6592 /* 6593 * Returns the free cluster for the given space info and sets empty_cluster to 6594 * what it should be based on the mount options. 6595 */ 6596 static struct btrfs_free_cluster * 6597 fetch_cluster_info(struct btrfs_fs_info *fs_info, 6598 struct btrfs_space_info *space_info, u64 *empty_cluster) 6599 { 6600 struct btrfs_free_cluster *ret = NULL; 6601 bool ssd = btrfs_test_opt(fs_info, SSD); 6602 6603 *empty_cluster = 0; 6604 if (btrfs_mixed_space_info(space_info)) 6605 return ret; 6606 6607 if (ssd) 6608 *empty_cluster = SZ_2M; 6609 if (space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { 6610 ret = &fs_info->meta_alloc_cluster; 6611 if (!ssd) 6612 *empty_cluster = SZ_64K; 6613 } else if ((space_info->flags & BTRFS_BLOCK_GROUP_DATA) && ssd) { 6614 ret = &fs_info->data_alloc_cluster; 6615 } 6616 6617 return ret; 6618 } 6619 6620 static int unpin_extent_range(struct btrfs_fs_info *fs_info, 6621 u64 start, u64 end, 6622 const bool return_free_space) 6623 { 6624 struct btrfs_block_group_cache *cache = NULL; 6625 struct btrfs_space_info *space_info; 6626 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 6627 struct btrfs_free_cluster *cluster = NULL; 6628 u64 len; 6629 u64 total_unpinned = 0; 6630 u64 empty_cluster = 0; 6631 bool readonly; 6632 6633 while (start <= end) { 6634 readonly = false; 6635 if (!cache || 6636 start >= cache->key.objectid + cache->key.offset) { 6637 if (cache) 6638 btrfs_put_block_group(cache); 6639 total_unpinned = 0; 6640 cache = btrfs_lookup_block_group(fs_info, start); 6641 BUG_ON(!cache); /* Logic error */ 6642 6643 cluster = fetch_cluster_info(fs_info, 6644 cache->space_info, 6645 &empty_cluster); 6646 empty_cluster <<= 1; 6647 } 6648 6649 len = cache->key.objectid + cache->key.offset - start; 6650 len = min(len, end + 1 - start); 6651 6652 if (start < cache->last_byte_to_unpin) { 6653 len = min(len, cache->last_byte_to_unpin - start); 6654 if (return_free_space) 6655 btrfs_add_free_space(cache, start, len); 6656 } 6657 6658 start += len; 6659 total_unpinned += len; 6660 space_info = cache->space_info; 6661 6662 /* 6663 * If this space cluster has been marked as fragmented and we've 6664 * unpinned enough in this block group to potentially allow a 6665 * cluster to be created inside of it go ahead and clear the 6666 * fragmented check. 6667 */ 6668 if (cluster && cluster->fragmented && 6669 total_unpinned > empty_cluster) { 6670 spin_lock(&cluster->lock); 6671 cluster->fragmented = 0; 6672 spin_unlock(&cluster->lock); 6673 } 6674 6675 spin_lock(&space_info->lock); 6676 spin_lock(&cache->lock); 6677 cache->pinned -= len; 6678 space_info->bytes_pinned -= len; 6679 6680 trace_btrfs_space_reservation(fs_info, "pinned", 6681 space_info->flags, len, 0); 6682 space_info->max_extent_size = 0; 6683 percpu_counter_add(&space_info->total_bytes_pinned, -len); 6684 if (cache->ro) { 6685 space_info->bytes_readonly += len; 6686 readonly = true; 6687 } 6688 spin_unlock(&cache->lock); 6689 if (!readonly && return_free_space && 6690 global_rsv->space_info == space_info) { 6691 u64 to_add = len; 6692 WARN_ON(!return_free_space); 6693 spin_lock(&global_rsv->lock); 6694 if (!global_rsv->full) { 6695 to_add = min(len, global_rsv->size - 6696 global_rsv->reserved); 6697 global_rsv->reserved += to_add; 6698 space_info->bytes_may_use += to_add; 6699 if (global_rsv->reserved >= global_rsv->size) 6700 global_rsv->full = 1; 6701 trace_btrfs_space_reservation(fs_info, 6702 "space_info", 6703 space_info->flags, 6704 to_add, 1); 6705 len -= to_add; 6706 } 6707 spin_unlock(&global_rsv->lock); 6708 /* Add to any tickets we may have */ 6709 if (len) 6710 space_info_add_new_bytes(fs_info, space_info, 6711 len); 6712 } 6713 spin_unlock(&space_info->lock); 6714 } 6715 6716 if (cache) 6717 btrfs_put_block_group(cache); 6718 return 0; 6719 } 6720 6721 int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans, 6722 struct btrfs_fs_info *fs_info) 6723 { 6724 struct btrfs_block_group_cache *block_group, *tmp; 6725 struct list_head *deleted_bgs; 6726 struct extent_io_tree *unpin; 6727 u64 start; 6728 u64 end; 6729 int ret; 6730 6731 if (fs_info->pinned_extents == &fs_info->freed_extents[0]) 6732 unpin = &fs_info->freed_extents[1]; 6733 else 6734 unpin = &fs_info->freed_extents[0]; 6735 6736 while (!trans->aborted) { 6737 mutex_lock(&fs_info->unused_bg_unpin_mutex); 6738 ret = find_first_extent_bit(unpin, 0, &start, &end, 6739 EXTENT_DIRTY, NULL); 6740 if (ret) { 6741 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 6742 break; 6743 } 6744 6745 if (btrfs_test_opt(fs_info, DISCARD)) 6746 ret = btrfs_discard_extent(fs_info, start, 6747 end + 1 - start, NULL); 6748 6749 clear_extent_dirty(unpin, start, end); 6750 unpin_extent_range(fs_info, start, end, true); 6751 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 6752 cond_resched(); 6753 } 6754 6755 /* 6756 * Transaction is finished. We don't need the lock anymore. We 6757 * do need to clean up the block groups in case of a transaction 6758 * abort. 6759 */ 6760 deleted_bgs = &trans->transaction->deleted_bgs; 6761 list_for_each_entry_safe(block_group, tmp, deleted_bgs, bg_list) { 6762 u64 trimmed = 0; 6763 6764 ret = -EROFS; 6765 if (!trans->aborted) 6766 ret = btrfs_discard_extent(fs_info, 6767 block_group->key.objectid, 6768 block_group->key.offset, 6769 &trimmed); 6770 6771 list_del_init(&block_group->bg_list); 6772 btrfs_put_block_group_trimming(block_group); 6773 btrfs_put_block_group(block_group); 6774 6775 if (ret) { 6776 const char *errstr = btrfs_decode_error(ret); 6777 btrfs_warn(fs_info, 6778 "Discard failed while removing blockgroup: errno=%d %s\n", 6779 ret, errstr); 6780 } 6781 } 6782 6783 return 0; 6784 } 6785 6786 static void add_pinned_bytes(struct btrfs_fs_info *fs_info, u64 num_bytes, 6787 u64 owner, u64 root_objectid) 6788 { 6789 struct btrfs_space_info *space_info; 6790 u64 flags; 6791 6792 if (owner < BTRFS_FIRST_FREE_OBJECTID) { 6793 if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID) 6794 flags = BTRFS_BLOCK_GROUP_SYSTEM; 6795 else 6796 flags = BTRFS_BLOCK_GROUP_METADATA; 6797 } else { 6798 flags = BTRFS_BLOCK_GROUP_DATA; 6799 } 6800 6801 space_info = __find_space_info(fs_info, flags); 6802 BUG_ON(!space_info); /* Logic bug */ 6803 percpu_counter_add(&space_info->total_bytes_pinned, num_bytes); 6804 } 6805 6806 6807 static int __btrfs_free_extent(struct btrfs_trans_handle *trans, 6808 struct btrfs_fs_info *info, 6809 struct btrfs_delayed_ref_node *node, u64 parent, 6810 u64 root_objectid, u64 owner_objectid, 6811 u64 owner_offset, int refs_to_drop, 6812 struct btrfs_delayed_extent_op *extent_op) 6813 { 6814 struct btrfs_key key; 6815 struct btrfs_path *path; 6816 struct btrfs_root *extent_root = info->extent_root; 6817 struct extent_buffer *leaf; 6818 struct btrfs_extent_item *ei; 6819 struct btrfs_extent_inline_ref *iref; 6820 int ret; 6821 int is_data; 6822 int extent_slot = 0; 6823 int found_extent = 0; 6824 int num_to_del = 1; 6825 u32 item_size; 6826 u64 refs; 6827 u64 bytenr = node->bytenr; 6828 u64 num_bytes = node->num_bytes; 6829 int last_ref = 0; 6830 bool skinny_metadata = btrfs_fs_incompat(info, SKINNY_METADATA); 6831 6832 path = btrfs_alloc_path(); 6833 if (!path) 6834 return -ENOMEM; 6835 6836 path->reada = READA_FORWARD; 6837 path->leave_spinning = 1; 6838 6839 is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID; 6840 BUG_ON(!is_data && refs_to_drop != 1); 6841 6842 if (is_data) 6843 skinny_metadata = 0; 6844 6845 ret = lookup_extent_backref(trans, extent_root, path, &iref, 6846 bytenr, num_bytes, parent, 6847 root_objectid, owner_objectid, 6848 owner_offset); 6849 if (ret == 0) { 6850 extent_slot = path->slots[0]; 6851 while (extent_slot >= 0) { 6852 btrfs_item_key_to_cpu(path->nodes[0], &key, 6853 extent_slot); 6854 if (key.objectid != bytenr) 6855 break; 6856 if (key.type == BTRFS_EXTENT_ITEM_KEY && 6857 key.offset == num_bytes) { 6858 found_extent = 1; 6859 break; 6860 } 6861 if (key.type == BTRFS_METADATA_ITEM_KEY && 6862 key.offset == owner_objectid) { 6863 found_extent = 1; 6864 break; 6865 } 6866 if (path->slots[0] - extent_slot > 5) 6867 break; 6868 extent_slot--; 6869 } 6870 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 6871 item_size = btrfs_item_size_nr(path->nodes[0], extent_slot); 6872 if (found_extent && item_size < sizeof(*ei)) 6873 found_extent = 0; 6874 #endif 6875 if (!found_extent) { 6876 BUG_ON(iref); 6877 ret = remove_extent_backref(trans, extent_root, path, 6878 NULL, refs_to_drop, 6879 is_data, &last_ref); 6880 if (ret) { 6881 btrfs_abort_transaction(trans, ret); 6882 goto out; 6883 } 6884 btrfs_release_path(path); 6885 path->leave_spinning = 1; 6886 6887 key.objectid = bytenr; 6888 key.type = BTRFS_EXTENT_ITEM_KEY; 6889 key.offset = num_bytes; 6890 6891 if (!is_data && skinny_metadata) { 6892 key.type = BTRFS_METADATA_ITEM_KEY; 6893 key.offset = owner_objectid; 6894 } 6895 6896 ret = btrfs_search_slot(trans, extent_root, 6897 &key, path, -1, 1); 6898 if (ret > 0 && skinny_metadata && path->slots[0]) { 6899 /* 6900 * Couldn't find our skinny metadata item, 6901 * see if we have ye olde extent item. 6902 */ 6903 path->slots[0]--; 6904 btrfs_item_key_to_cpu(path->nodes[0], &key, 6905 path->slots[0]); 6906 if (key.objectid == bytenr && 6907 key.type == BTRFS_EXTENT_ITEM_KEY && 6908 key.offset == num_bytes) 6909 ret = 0; 6910 } 6911 6912 if (ret > 0 && skinny_metadata) { 6913 skinny_metadata = false; 6914 key.objectid = bytenr; 6915 key.type = BTRFS_EXTENT_ITEM_KEY; 6916 key.offset = num_bytes; 6917 btrfs_release_path(path); 6918 ret = btrfs_search_slot(trans, extent_root, 6919 &key, path, -1, 1); 6920 } 6921 6922 if (ret) { 6923 btrfs_err(info, 6924 "umm, got %d back from search, was looking for %llu", 6925 ret, bytenr); 6926 if (ret > 0) 6927 btrfs_print_leaf(info, path->nodes[0]); 6928 } 6929 if (ret < 0) { 6930 btrfs_abort_transaction(trans, ret); 6931 goto out; 6932 } 6933 extent_slot = path->slots[0]; 6934 } 6935 } else if (WARN_ON(ret == -ENOENT)) { 6936 btrfs_print_leaf(info, path->nodes[0]); 6937 btrfs_err(info, 6938 "unable to find ref byte nr %llu parent %llu root %llu owner %llu offset %llu", 6939 bytenr, parent, root_objectid, owner_objectid, 6940 owner_offset); 6941 btrfs_abort_transaction(trans, ret); 6942 goto out; 6943 } else { 6944 btrfs_abort_transaction(trans, ret); 6945 goto out; 6946 } 6947 6948 leaf = path->nodes[0]; 6949 item_size = btrfs_item_size_nr(leaf, extent_slot); 6950 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 6951 if (item_size < sizeof(*ei)) { 6952 BUG_ON(found_extent || extent_slot != path->slots[0]); 6953 ret = convert_extent_item_v0(trans, extent_root, path, 6954 owner_objectid, 0); 6955 if (ret < 0) { 6956 btrfs_abort_transaction(trans, ret); 6957 goto out; 6958 } 6959 6960 btrfs_release_path(path); 6961 path->leave_spinning = 1; 6962 6963 key.objectid = bytenr; 6964 key.type = BTRFS_EXTENT_ITEM_KEY; 6965 key.offset = num_bytes; 6966 6967 ret = btrfs_search_slot(trans, extent_root, &key, path, 6968 -1, 1); 6969 if (ret) { 6970 btrfs_err(info, 6971 "umm, got %d back from search, was looking for %llu", 6972 ret, bytenr); 6973 btrfs_print_leaf(info, path->nodes[0]); 6974 } 6975 if (ret < 0) { 6976 btrfs_abort_transaction(trans, ret); 6977 goto out; 6978 } 6979 6980 extent_slot = path->slots[0]; 6981 leaf = path->nodes[0]; 6982 item_size = btrfs_item_size_nr(leaf, extent_slot); 6983 } 6984 #endif 6985 BUG_ON(item_size < sizeof(*ei)); 6986 ei = btrfs_item_ptr(leaf, extent_slot, 6987 struct btrfs_extent_item); 6988 if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID && 6989 key.type == BTRFS_EXTENT_ITEM_KEY) { 6990 struct btrfs_tree_block_info *bi; 6991 BUG_ON(item_size < sizeof(*ei) + sizeof(*bi)); 6992 bi = (struct btrfs_tree_block_info *)(ei + 1); 6993 WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi)); 6994 } 6995 6996 refs = btrfs_extent_refs(leaf, ei); 6997 if (refs < refs_to_drop) { 6998 btrfs_err(info, 6999 "trying to drop %d refs but we only have %Lu for bytenr %Lu", 7000 refs_to_drop, refs, bytenr); 7001 ret = -EINVAL; 7002 btrfs_abort_transaction(trans, ret); 7003 goto out; 7004 } 7005 refs -= refs_to_drop; 7006 7007 if (refs > 0) { 7008 if (extent_op) 7009 __run_delayed_extent_op(extent_op, leaf, ei); 7010 /* 7011 * In the case of inline back ref, reference count will 7012 * be updated by remove_extent_backref 7013 */ 7014 if (iref) { 7015 BUG_ON(!found_extent); 7016 } else { 7017 btrfs_set_extent_refs(leaf, ei, refs); 7018 btrfs_mark_buffer_dirty(leaf); 7019 } 7020 if (found_extent) { 7021 ret = remove_extent_backref(trans, extent_root, path, 7022 iref, refs_to_drop, 7023 is_data, &last_ref); 7024 if (ret) { 7025 btrfs_abort_transaction(trans, ret); 7026 goto out; 7027 } 7028 } 7029 add_pinned_bytes(info, -num_bytes, owner_objectid, 7030 root_objectid); 7031 } else { 7032 if (found_extent) { 7033 BUG_ON(is_data && refs_to_drop != 7034 extent_data_ref_count(path, iref)); 7035 if (iref) { 7036 BUG_ON(path->slots[0] != extent_slot); 7037 } else { 7038 BUG_ON(path->slots[0] != extent_slot + 1); 7039 path->slots[0] = extent_slot; 7040 num_to_del = 2; 7041 } 7042 } 7043 7044 last_ref = 1; 7045 ret = btrfs_del_items(trans, extent_root, path, path->slots[0], 7046 num_to_del); 7047 if (ret) { 7048 btrfs_abort_transaction(trans, ret); 7049 goto out; 7050 } 7051 btrfs_release_path(path); 7052 7053 if (is_data) { 7054 ret = btrfs_del_csums(trans, info, bytenr, num_bytes); 7055 if (ret) { 7056 btrfs_abort_transaction(trans, ret); 7057 goto out; 7058 } 7059 } 7060 7061 ret = add_to_free_space_tree(trans, info, bytenr, num_bytes); 7062 if (ret) { 7063 btrfs_abort_transaction(trans, ret); 7064 goto out; 7065 } 7066 7067 ret = update_block_group(trans, info, bytenr, num_bytes, 0); 7068 if (ret) { 7069 btrfs_abort_transaction(trans, ret); 7070 goto out; 7071 } 7072 } 7073 btrfs_release_path(path); 7074 7075 out: 7076 btrfs_free_path(path); 7077 return ret; 7078 } 7079 7080 /* 7081 * when we free an block, it is possible (and likely) that we free the last 7082 * delayed ref for that extent as well. This searches the delayed ref tree for 7083 * a given extent, and if there are no other delayed refs to be processed, it 7084 * removes it from the tree. 7085 */ 7086 static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans, 7087 u64 bytenr) 7088 { 7089 struct btrfs_delayed_ref_head *head; 7090 struct btrfs_delayed_ref_root *delayed_refs; 7091 int ret = 0; 7092 7093 delayed_refs = &trans->transaction->delayed_refs; 7094 spin_lock(&delayed_refs->lock); 7095 head = btrfs_find_delayed_ref_head(trans, bytenr); 7096 if (!head) 7097 goto out_delayed_unlock; 7098 7099 spin_lock(&head->lock); 7100 if (!list_empty(&head->ref_list)) 7101 goto out; 7102 7103 if (head->extent_op) { 7104 if (!head->must_insert_reserved) 7105 goto out; 7106 btrfs_free_delayed_extent_op(head->extent_op); 7107 head->extent_op = NULL; 7108 } 7109 7110 /* 7111 * waiting for the lock here would deadlock. If someone else has it 7112 * locked they are already in the process of dropping it anyway 7113 */ 7114 if (!mutex_trylock(&head->mutex)) 7115 goto out; 7116 7117 /* 7118 * at this point we have a head with no other entries. Go 7119 * ahead and process it. 7120 */ 7121 head->node.in_tree = 0; 7122 rb_erase(&head->href_node, &delayed_refs->href_root); 7123 7124 atomic_dec(&delayed_refs->num_entries); 7125 7126 /* 7127 * we don't take a ref on the node because we're removing it from the 7128 * tree, so we just steal the ref the tree was holding. 7129 */ 7130 delayed_refs->num_heads--; 7131 if (head->processing == 0) 7132 delayed_refs->num_heads_ready--; 7133 head->processing = 0; 7134 spin_unlock(&head->lock); 7135 spin_unlock(&delayed_refs->lock); 7136 7137 BUG_ON(head->extent_op); 7138 if (head->must_insert_reserved) 7139 ret = 1; 7140 7141 mutex_unlock(&head->mutex); 7142 btrfs_put_delayed_ref(&head->node); 7143 return ret; 7144 out: 7145 spin_unlock(&head->lock); 7146 7147 out_delayed_unlock: 7148 spin_unlock(&delayed_refs->lock); 7149 return 0; 7150 } 7151 7152 void btrfs_free_tree_block(struct btrfs_trans_handle *trans, 7153 struct btrfs_root *root, 7154 struct extent_buffer *buf, 7155 u64 parent, int last_ref) 7156 { 7157 struct btrfs_fs_info *fs_info = root->fs_info; 7158 int pin = 1; 7159 int ret; 7160 7161 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 7162 ret = btrfs_add_delayed_tree_ref(fs_info, trans, 7163 buf->start, buf->len, 7164 parent, 7165 root->root_key.objectid, 7166 btrfs_header_level(buf), 7167 BTRFS_DROP_DELAYED_REF, NULL); 7168 BUG_ON(ret); /* -ENOMEM */ 7169 } 7170 7171 if (!last_ref) 7172 return; 7173 7174 if (btrfs_header_generation(buf) == trans->transid) { 7175 struct btrfs_block_group_cache *cache; 7176 7177 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 7178 ret = check_ref_cleanup(trans, buf->start); 7179 if (!ret) 7180 goto out; 7181 } 7182 7183 cache = btrfs_lookup_block_group(fs_info, buf->start); 7184 7185 if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) { 7186 pin_down_extent(fs_info, cache, buf->start, 7187 buf->len, 1); 7188 btrfs_put_block_group(cache); 7189 goto out; 7190 } 7191 7192 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)); 7193 7194 btrfs_add_free_space(cache, buf->start, buf->len); 7195 btrfs_free_reserved_bytes(cache, buf->len, 0); 7196 btrfs_put_block_group(cache); 7197 trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len); 7198 pin = 0; 7199 } 7200 out: 7201 if (pin) 7202 add_pinned_bytes(fs_info, buf->len, btrfs_header_level(buf), 7203 root->root_key.objectid); 7204 7205 /* 7206 * Deleting the buffer, clear the corrupt flag since it doesn't matter 7207 * anymore. 7208 */ 7209 clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags); 7210 } 7211 7212 /* Can return -ENOMEM */ 7213 int btrfs_free_extent(struct btrfs_trans_handle *trans, 7214 struct btrfs_fs_info *fs_info, 7215 u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid, 7216 u64 owner, u64 offset) 7217 { 7218 int ret; 7219 7220 if (btrfs_is_testing(fs_info)) 7221 return 0; 7222 7223 add_pinned_bytes(fs_info, num_bytes, owner, root_objectid); 7224 7225 /* 7226 * tree log blocks never actually go into the extent allocation 7227 * tree, just update pinning info and exit early. 7228 */ 7229 if (root_objectid == BTRFS_TREE_LOG_OBJECTID) { 7230 WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID); 7231 /* unlocks the pinned mutex */ 7232 btrfs_pin_extent(fs_info, bytenr, num_bytes, 1); 7233 ret = 0; 7234 } else if (owner < BTRFS_FIRST_FREE_OBJECTID) { 7235 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr, 7236 num_bytes, 7237 parent, root_objectid, (int)owner, 7238 BTRFS_DROP_DELAYED_REF, NULL); 7239 } else { 7240 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr, 7241 num_bytes, 7242 parent, root_objectid, owner, 7243 offset, 0, 7244 BTRFS_DROP_DELAYED_REF, NULL); 7245 } 7246 return ret; 7247 } 7248 7249 /* 7250 * when we wait for progress in the block group caching, its because 7251 * our allocation attempt failed at least once. So, we must sleep 7252 * and let some progress happen before we try again. 7253 * 7254 * This function will sleep at least once waiting for new free space to 7255 * show up, and then it will check the block group free space numbers 7256 * for our min num_bytes. Another option is to have it go ahead 7257 * and look in the rbtree for a free extent of a given size, but this 7258 * is a good start. 7259 * 7260 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using 7261 * any of the information in this block group. 7262 */ 7263 static noinline void 7264 wait_block_group_cache_progress(struct btrfs_block_group_cache *cache, 7265 u64 num_bytes) 7266 { 7267 struct btrfs_caching_control *caching_ctl; 7268 7269 caching_ctl = get_caching_control(cache); 7270 if (!caching_ctl) 7271 return; 7272 7273 wait_event(caching_ctl->wait, block_group_cache_done(cache) || 7274 (cache->free_space_ctl->free_space >= num_bytes)); 7275 7276 put_caching_control(caching_ctl); 7277 } 7278 7279 static noinline int 7280 wait_block_group_cache_done(struct btrfs_block_group_cache *cache) 7281 { 7282 struct btrfs_caching_control *caching_ctl; 7283 int ret = 0; 7284 7285 caching_ctl = get_caching_control(cache); 7286 if (!caching_ctl) 7287 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0; 7288 7289 wait_event(caching_ctl->wait, block_group_cache_done(cache)); 7290 if (cache->cached == BTRFS_CACHE_ERROR) 7291 ret = -EIO; 7292 put_caching_control(caching_ctl); 7293 return ret; 7294 } 7295 7296 int __get_raid_index(u64 flags) 7297 { 7298 if (flags & BTRFS_BLOCK_GROUP_RAID10) 7299 return BTRFS_RAID_RAID10; 7300 else if (flags & BTRFS_BLOCK_GROUP_RAID1) 7301 return BTRFS_RAID_RAID1; 7302 else if (flags & BTRFS_BLOCK_GROUP_DUP) 7303 return BTRFS_RAID_DUP; 7304 else if (flags & BTRFS_BLOCK_GROUP_RAID0) 7305 return BTRFS_RAID_RAID0; 7306 else if (flags & BTRFS_BLOCK_GROUP_RAID5) 7307 return BTRFS_RAID_RAID5; 7308 else if (flags & BTRFS_BLOCK_GROUP_RAID6) 7309 return BTRFS_RAID_RAID6; 7310 7311 return BTRFS_RAID_SINGLE; /* BTRFS_BLOCK_GROUP_SINGLE */ 7312 } 7313 7314 int get_block_group_index(struct btrfs_block_group_cache *cache) 7315 { 7316 return __get_raid_index(cache->flags); 7317 } 7318 7319 static const char *btrfs_raid_type_names[BTRFS_NR_RAID_TYPES] = { 7320 [BTRFS_RAID_RAID10] = "raid10", 7321 [BTRFS_RAID_RAID1] = "raid1", 7322 [BTRFS_RAID_DUP] = "dup", 7323 [BTRFS_RAID_RAID0] = "raid0", 7324 [BTRFS_RAID_SINGLE] = "single", 7325 [BTRFS_RAID_RAID5] = "raid5", 7326 [BTRFS_RAID_RAID6] = "raid6", 7327 }; 7328 7329 static const char *get_raid_name(enum btrfs_raid_types type) 7330 { 7331 if (type >= BTRFS_NR_RAID_TYPES) 7332 return NULL; 7333 7334 return btrfs_raid_type_names[type]; 7335 } 7336 7337 enum btrfs_loop_type { 7338 LOOP_CACHING_NOWAIT = 0, 7339 LOOP_CACHING_WAIT = 1, 7340 LOOP_ALLOC_CHUNK = 2, 7341 LOOP_NO_EMPTY_SIZE = 3, 7342 }; 7343 7344 static inline void 7345 btrfs_lock_block_group(struct btrfs_block_group_cache *cache, 7346 int delalloc) 7347 { 7348 if (delalloc) 7349 down_read(&cache->data_rwsem); 7350 } 7351 7352 static inline void 7353 btrfs_grab_block_group(struct btrfs_block_group_cache *cache, 7354 int delalloc) 7355 { 7356 btrfs_get_block_group(cache); 7357 if (delalloc) 7358 down_read(&cache->data_rwsem); 7359 } 7360 7361 static struct btrfs_block_group_cache * 7362 btrfs_lock_cluster(struct btrfs_block_group_cache *block_group, 7363 struct btrfs_free_cluster *cluster, 7364 int delalloc) 7365 { 7366 struct btrfs_block_group_cache *used_bg = NULL; 7367 7368 spin_lock(&cluster->refill_lock); 7369 while (1) { 7370 used_bg = cluster->block_group; 7371 if (!used_bg) 7372 return NULL; 7373 7374 if (used_bg == block_group) 7375 return used_bg; 7376 7377 btrfs_get_block_group(used_bg); 7378 7379 if (!delalloc) 7380 return used_bg; 7381 7382 if (down_read_trylock(&used_bg->data_rwsem)) 7383 return used_bg; 7384 7385 spin_unlock(&cluster->refill_lock); 7386 7387 down_read(&used_bg->data_rwsem); 7388 7389 spin_lock(&cluster->refill_lock); 7390 if (used_bg == cluster->block_group) 7391 return used_bg; 7392 7393 up_read(&used_bg->data_rwsem); 7394 btrfs_put_block_group(used_bg); 7395 } 7396 } 7397 7398 static inline void 7399 btrfs_release_block_group(struct btrfs_block_group_cache *cache, 7400 int delalloc) 7401 { 7402 if (delalloc) 7403 up_read(&cache->data_rwsem); 7404 btrfs_put_block_group(cache); 7405 } 7406 7407 /* 7408 * walks the btree of allocated extents and find a hole of a given size. 7409 * The key ins is changed to record the hole: 7410 * ins->objectid == start position 7411 * ins->flags = BTRFS_EXTENT_ITEM_KEY 7412 * ins->offset == the size of the hole. 7413 * Any available blocks before search_start are skipped. 7414 * 7415 * If there is no suitable free space, we will record the max size of 7416 * the free space extent currently. 7417 */ 7418 static noinline int find_free_extent(struct btrfs_root *orig_root, 7419 u64 ram_bytes, u64 num_bytes, u64 empty_size, 7420 u64 hint_byte, struct btrfs_key *ins, 7421 u64 flags, int delalloc) 7422 { 7423 struct btrfs_fs_info *fs_info = orig_root->fs_info; 7424 int ret = 0; 7425 struct btrfs_root *root = fs_info->extent_root; 7426 struct btrfs_free_cluster *last_ptr = NULL; 7427 struct btrfs_block_group_cache *block_group = NULL; 7428 u64 search_start = 0; 7429 u64 max_extent_size = 0; 7430 u64 empty_cluster = 0; 7431 struct btrfs_space_info *space_info; 7432 int loop = 0; 7433 int index = __get_raid_index(flags); 7434 bool failed_cluster_refill = false; 7435 bool failed_alloc = false; 7436 bool use_cluster = true; 7437 bool have_caching_bg = false; 7438 bool orig_have_caching_bg = false; 7439 bool full_search = false; 7440 7441 WARN_ON(num_bytes < fs_info->sectorsize); 7442 ins->type = BTRFS_EXTENT_ITEM_KEY; 7443 ins->objectid = 0; 7444 ins->offset = 0; 7445 7446 trace_find_free_extent(fs_info, num_bytes, empty_size, flags); 7447 7448 space_info = __find_space_info(fs_info, flags); 7449 if (!space_info) { 7450 btrfs_err(fs_info, "No space info for %llu", flags); 7451 return -ENOSPC; 7452 } 7453 7454 /* 7455 * If our free space is heavily fragmented we may not be able to make 7456 * big contiguous allocations, so instead of doing the expensive search 7457 * for free space, simply return ENOSPC with our max_extent_size so we 7458 * can go ahead and search for a more manageable chunk. 7459 * 7460 * If our max_extent_size is large enough for our allocation simply 7461 * disable clustering since we will likely not be able to find enough 7462 * space to create a cluster and induce latency trying. 7463 */ 7464 if (unlikely(space_info->max_extent_size)) { 7465 spin_lock(&space_info->lock); 7466 if (space_info->max_extent_size && 7467 num_bytes > space_info->max_extent_size) { 7468 ins->offset = space_info->max_extent_size; 7469 spin_unlock(&space_info->lock); 7470 return -ENOSPC; 7471 } else if (space_info->max_extent_size) { 7472 use_cluster = false; 7473 } 7474 spin_unlock(&space_info->lock); 7475 } 7476 7477 last_ptr = fetch_cluster_info(fs_info, space_info, &empty_cluster); 7478 if (last_ptr) { 7479 spin_lock(&last_ptr->lock); 7480 if (last_ptr->block_group) 7481 hint_byte = last_ptr->window_start; 7482 if (last_ptr->fragmented) { 7483 /* 7484 * We still set window_start so we can keep track of the 7485 * last place we found an allocation to try and save 7486 * some time. 7487 */ 7488 hint_byte = last_ptr->window_start; 7489 use_cluster = false; 7490 } 7491 spin_unlock(&last_ptr->lock); 7492 } 7493 7494 search_start = max(search_start, first_logical_byte(fs_info, 0)); 7495 search_start = max(search_start, hint_byte); 7496 if (search_start == hint_byte) { 7497 block_group = btrfs_lookup_block_group(fs_info, search_start); 7498 /* 7499 * we don't want to use the block group if it doesn't match our 7500 * allocation bits, or if its not cached. 7501 * 7502 * However if we are re-searching with an ideal block group 7503 * picked out then we don't care that the block group is cached. 7504 */ 7505 if (block_group && block_group_bits(block_group, flags) && 7506 block_group->cached != BTRFS_CACHE_NO) { 7507 down_read(&space_info->groups_sem); 7508 if (list_empty(&block_group->list) || 7509 block_group->ro) { 7510 /* 7511 * someone is removing this block group, 7512 * we can't jump into the have_block_group 7513 * target because our list pointers are not 7514 * valid 7515 */ 7516 btrfs_put_block_group(block_group); 7517 up_read(&space_info->groups_sem); 7518 } else { 7519 index = get_block_group_index(block_group); 7520 btrfs_lock_block_group(block_group, delalloc); 7521 goto have_block_group; 7522 } 7523 } else if (block_group) { 7524 btrfs_put_block_group(block_group); 7525 } 7526 } 7527 search: 7528 have_caching_bg = false; 7529 if (index == 0 || index == __get_raid_index(flags)) 7530 full_search = true; 7531 down_read(&space_info->groups_sem); 7532 list_for_each_entry(block_group, &space_info->block_groups[index], 7533 list) { 7534 u64 offset; 7535 int cached; 7536 7537 btrfs_grab_block_group(block_group, delalloc); 7538 search_start = block_group->key.objectid; 7539 7540 /* 7541 * this can happen if we end up cycling through all the 7542 * raid types, but we want to make sure we only allocate 7543 * for the proper type. 7544 */ 7545 if (!block_group_bits(block_group, flags)) { 7546 u64 extra = BTRFS_BLOCK_GROUP_DUP | 7547 BTRFS_BLOCK_GROUP_RAID1 | 7548 BTRFS_BLOCK_GROUP_RAID5 | 7549 BTRFS_BLOCK_GROUP_RAID6 | 7550 BTRFS_BLOCK_GROUP_RAID10; 7551 7552 /* 7553 * if they asked for extra copies and this block group 7554 * doesn't provide them, bail. This does allow us to 7555 * fill raid0 from raid1. 7556 */ 7557 if ((flags & extra) && !(block_group->flags & extra)) 7558 goto loop; 7559 } 7560 7561 have_block_group: 7562 cached = block_group_cache_done(block_group); 7563 if (unlikely(!cached)) { 7564 have_caching_bg = true; 7565 ret = cache_block_group(block_group, 0); 7566 BUG_ON(ret < 0); 7567 ret = 0; 7568 } 7569 7570 if (unlikely(block_group->cached == BTRFS_CACHE_ERROR)) 7571 goto loop; 7572 if (unlikely(block_group->ro)) 7573 goto loop; 7574 7575 /* 7576 * Ok we want to try and use the cluster allocator, so 7577 * lets look there 7578 */ 7579 if (last_ptr && use_cluster) { 7580 struct btrfs_block_group_cache *used_block_group; 7581 unsigned long aligned_cluster; 7582 /* 7583 * the refill lock keeps out other 7584 * people trying to start a new cluster 7585 */ 7586 used_block_group = btrfs_lock_cluster(block_group, 7587 last_ptr, 7588 delalloc); 7589 if (!used_block_group) 7590 goto refill_cluster; 7591 7592 if (used_block_group != block_group && 7593 (used_block_group->ro || 7594 !block_group_bits(used_block_group, flags))) 7595 goto release_cluster; 7596 7597 offset = btrfs_alloc_from_cluster(used_block_group, 7598 last_ptr, 7599 num_bytes, 7600 used_block_group->key.objectid, 7601 &max_extent_size); 7602 if (offset) { 7603 /* we have a block, we're done */ 7604 spin_unlock(&last_ptr->refill_lock); 7605 trace_btrfs_reserve_extent_cluster(fs_info, 7606 used_block_group, 7607 search_start, num_bytes); 7608 if (used_block_group != block_group) { 7609 btrfs_release_block_group(block_group, 7610 delalloc); 7611 block_group = used_block_group; 7612 } 7613 goto checks; 7614 } 7615 7616 WARN_ON(last_ptr->block_group != used_block_group); 7617 release_cluster: 7618 /* If we are on LOOP_NO_EMPTY_SIZE, we can't 7619 * set up a new clusters, so lets just skip it 7620 * and let the allocator find whatever block 7621 * it can find. If we reach this point, we 7622 * will have tried the cluster allocator 7623 * plenty of times and not have found 7624 * anything, so we are likely way too 7625 * fragmented for the clustering stuff to find 7626 * anything. 7627 * 7628 * However, if the cluster is taken from the 7629 * current block group, release the cluster 7630 * first, so that we stand a better chance of 7631 * succeeding in the unclustered 7632 * allocation. */ 7633 if (loop >= LOOP_NO_EMPTY_SIZE && 7634 used_block_group != block_group) { 7635 spin_unlock(&last_ptr->refill_lock); 7636 btrfs_release_block_group(used_block_group, 7637 delalloc); 7638 goto unclustered_alloc; 7639 } 7640 7641 /* 7642 * this cluster didn't work out, free it and 7643 * start over 7644 */ 7645 btrfs_return_cluster_to_free_space(NULL, last_ptr); 7646 7647 if (used_block_group != block_group) 7648 btrfs_release_block_group(used_block_group, 7649 delalloc); 7650 refill_cluster: 7651 if (loop >= LOOP_NO_EMPTY_SIZE) { 7652 spin_unlock(&last_ptr->refill_lock); 7653 goto unclustered_alloc; 7654 } 7655 7656 aligned_cluster = max_t(unsigned long, 7657 empty_cluster + empty_size, 7658 block_group->full_stripe_len); 7659 7660 /* allocate a cluster in this block group */ 7661 ret = btrfs_find_space_cluster(fs_info, block_group, 7662 last_ptr, search_start, 7663 num_bytes, 7664 aligned_cluster); 7665 if (ret == 0) { 7666 /* 7667 * now pull our allocation out of this 7668 * cluster 7669 */ 7670 offset = btrfs_alloc_from_cluster(block_group, 7671 last_ptr, 7672 num_bytes, 7673 search_start, 7674 &max_extent_size); 7675 if (offset) { 7676 /* we found one, proceed */ 7677 spin_unlock(&last_ptr->refill_lock); 7678 trace_btrfs_reserve_extent_cluster(fs_info, 7679 block_group, search_start, 7680 num_bytes); 7681 goto checks; 7682 } 7683 } else if (!cached && loop > LOOP_CACHING_NOWAIT 7684 && !failed_cluster_refill) { 7685 spin_unlock(&last_ptr->refill_lock); 7686 7687 failed_cluster_refill = true; 7688 wait_block_group_cache_progress(block_group, 7689 num_bytes + empty_cluster + empty_size); 7690 goto have_block_group; 7691 } 7692 7693 /* 7694 * at this point we either didn't find a cluster 7695 * or we weren't able to allocate a block from our 7696 * cluster. Free the cluster we've been trying 7697 * to use, and go to the next block group 7698 */ 7699 btrfs_return_cluster_to_free_space(NULL, last_ptr); 7700 spin_unlock(&last_ptr->refill_lock); 7701 goto loop; 7702 } 7703 7704 unclustered_alloc: 7705 /* 7706 * We are doing an unclustered alloc, set the fragmented flag so 7707 * we don't bother trying to setup a cluster again until we get 7708 * more space. 7709 */ 7710 if (unlikely(last_ptr)) { 7711 spin_lock(&last_ptr->lock); 7712 last_ptr->fragmented = 1; 7713 spin_unlock(&last_ptr->lock); 7714 } 7715 spin_lock(&block_group->free_space_ctl->tree_lock); 7716 if (cached && 7717 block_group->free_space_ctl->free_space < 7718 num_bytes + empty_cluster + empty_size) { 7719 if (block_group->free_space_ctl->free_space > 7720 max_extent_size) 7721 max_extent_size = 7722 block_group->free_space_ctl->free_space; 7723 spin_unlock(&block_group->free_space_ctl->tree_lock); 7724 goto loop; 7725 } 7726 spin_unlock(&block_group->free_space_ctl->tree_lock); 7727 7728 offset = btrfs_find_space_for_alloc(block_group, search_start, 7729 num_bytes, empty_size, 7730 &max_extent_size); 7731 /* 7732 * If we didn't find a chunk, and we haven't failed on this 7733 * block group before, and this block group is in the middle of 7734 * caching and we are ok with waiting, then go ahead and wait 7735 * for progress to be made, and set failed_alloc to true. 7736 * 7737 * If failed_alloc is true then we've already waited on this 7738 * block group once and should move on to the next block group. 7739 */ 7740 if (!offset && !failed_alloc && !cached && 7741 loop > LOOP_CACHING_NOWAIT) { 7742 wait_block_group_cache_progress(block_group, 7743 num_bytes + empty_size); 7744 failed_alloc = true; 7745 goto have_block_group; 7746 } else if (!offset) { 7747 goto loop; 7748 } 7749 checks: 7750 search_start = ALIGN(offset, fs_info->stripesize); 7751 7752 /* move on to the next group */ 7753 if (search_start + num_bytes > 7754 block_group->key.objectid + block_group->key.offset) { 7755 btrfs_add_free_space(block_group, offset, num_bytes); 7756 goto loop; 7757 } 7758 7759 if (offset < search_start) 7760 btrfs_add_free_space(block_group, offset, 7761 search_start - offset); 7762 BUG_ON(offset > search_start); 7763 7764 ret = btrfs_add_reserved_bytes(block_group, ram_bytes, 7765 num_bytes, delalloc); 7766 if (ret == -EAGAIN) { 7767 btrfs_add_free_space(block_group, offset, num_bytes); 7768 goto loop; 7769 } 7770 btrfs_inc_block_group_reservations(block_group); 7771 7772 /* we are all good, lets return */ 7773 ins->objectid = search_start; 7774 ins->offset = num_bytes; 7775 7776 trace_btrfs_reserve_extent(fs_info, block_group, 7777 search_start, num_bytes); 7778 btrfs_release_block_group(block_group, delalloc); 7779 break; 7780 loop: 7781 failed_cluster_refill = false; 7782 failed_alloc = false; 7783 BUG_ON(index != get_block_group_index(block_group)); 7784 btrfs_release_block_group(block_group, delalloc); 7785 } 7786 up_read(&space_info->groups_sem); 7787 7788 if ((loop == LOOP_CACHING_NOWAIT) && have_caching_bg 7789 && !orig_have_caching_bg) 7790 orig_have_caching_bg = true; 7791 7792 if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg) 7793 goto search; 7794 7795 if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES) 7796 goto search; 7797 7798 /* 7799 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking 7800 * caching kthreads as we move along 7801 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching 7802 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again 7803 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try 7804 * again 7805 */ 7806 if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) { 7807 index = 0; 7808 if (loop == LOOP_CACHING_NOWAIT) { 7809 /* 7810 * We want to skip the LOOP_CACHING_WAIT step if we 7811 * don't have any uncached bgs and we've already done a 7812 * full search through. 7813 */ 7814 if (orig_have_caching_bg || !full_search) 7815 loop = LOOP_CACHING_WAIT; 7816 else 7817 loop = LOOP_ALLOC_CHUNK; 7818 } else { 7819 loop++; 7820 } 7821 7822 if (loop == LOOP_ALLOC_CHUNK) { 7823 struct btrfs_trans_handle *trans; 7824 int exist = 0; 7825 7826 trans = current->journal_info; 7827 if (trans) 7828 exist = 1; 7829 else 7830 trans = btrfs_join_transaction(root); 7831 7832 if (IS_ERR(trans)) { 7833 ret = PTR_ERR(trans); 7834 goto out; 7835 } 7836 7837 ret = do_chunk_alloc(trans, fs_info, flags, 7838 CHUNK_ALLOC_FORCE); 7839 7840 /* 7841 * If we can't allocate a new chunk we've already looped 7842 * through at least once, move on to the NO_EMPTY_SIZE 7843 * case. 7844 */ 7845 if (ret == -ENOSPC) 7846 loop = LOOP_NO_EMPTY_SIZE; 7847 7848 /* 7849 * Do not bail out on ENOSPC since we 7850 * can do more things. 7851 */ 7852 if (ret < 0 && ret != -ENOSPC) 7853 btrfs_abort_transaction(trans, ret); 7854 else 7855 ret = 0; 7856 if (!exist) 7857 btrfs_end_transaction(trans); 7858 if (ret) 7859 goto out; 7860 } 7861 7862 if (loop == LOOP_NO_EMPTY_SIZE) { 7863 /* 7864 * Don't loop again if we already have no empty_size and 7865 * no empty_cluster. 7866 */ 7867 if (empty_size == 0 && 7868 empty_cluster == 0) { 7869 ret = -ENOSPC; 7870 goto out; 7871 } 7872 empty_size = 0; 7873 empty_cluster = 0; 7874 } 7875 7876 goto search; 7877 } else if (!ins->objectid) { 7878 ret = -ENOSPC; 7879 } else if (ins->objectid) { 7880 if (!use_cluster && last_ptr) { 7881 spin_lock(&last_ptr->lock); 7882 last_ptr->window_start = ins->objectid; 7883 spin_unlock(&last_ptr->lock); 7884 } 7885 ret = 0; 7886 } 7887 out: 7888 if (ret == -ENOSPC) { 7889 spin_lock(&space_info->lock); 7890 space_info->max_extent_size = max_extent_size; 7891 spin_unlock(&space_info->lock); 7892 ins->offset = max_extent_size; 7893 } 7894 return ret; 7895 } 7896 7897 static void dump_space_info(struct btrfs_fs_info *fs_info, 7898 struct btrfs_space_info *info, u64 bytes, 7899 int dump_block_groups) 7900 { 7901 struct btrfs_block_group_cache *cache; 7902 int index = 0; 7903 7904 spin_lock(&info->lock); 7905 btrfs_info(fs_info, "space_info %llu has %llu free, is %sfull", 7906 info->flags, 7907 info->total_bytes - info->bytes_used - info->bytes_pinned - 7908 info->bytes_reserved - info->bytes_readonly - 7909 info->bytes_may_use, (info->full) ? "" : "not "); 7910 btrfs_info(fs_info, 7911 "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu", 7912 info->total_bytes, info->bytes_used, info->bytes_pinned, 7913 info->bytes_reserved, info->bytes_may_use, 7914 info->bytes_readonly); 7915 spin_unlock(&info->lock); 7916 7917 if (!dump_block_groups) 7918 return; 7919 7920 down_read(&info->groups_sem); 7921 again: 7922 list_for_each_entry(cache, &info->block_groups[index], list) { 7923 spin_lock(&cache->lock); 7924 btrfs_info(fs_info, 7925 "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s", 7926 cache->key.objectid, cache->key.offset, 7927 btrfs_block_group_used(&cache->item), cache->pinned, 7928 cache->reserved, cache->ro ? "[readonly]" : ""); 7929 btrfs_dump_free_space(cache, bytes); 7930 spin_unlock(&cache->lock); 7931 } 7932 if (++index < BTRFS_NR_RAID_TYPES) 7933 goto again; 7934 up_read(&info->groups_sem); 7935 } 7936 7937 int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, 7938 u64 num_bytes, u64 min_alloc_size, 7939 u64 empty_size, u64 hint_byte, 7940 struct btrfs_key *ins, int is_data, int delalloc) 7941 { 7942 struct btrfs_fs_info *fs_info = root->fs_info; 7943 bool final_tried = num_bytes == min_alloc_size; 7944 u64 flags; 7945 int ret; 7946 7947 flags = btrfs_get_alloc_profile(root, is_data); 7948 again: 7949 WARN_ON(num_bytes < fs_info->sectorsize); 7950 ret = find_free_extent(root, ram_bytes, num_bytes, empty_size, 7951 hint_byte, ins, flags, delalloc); 7952 if (!ret && !is_data) { 7953 btrfs_dec_block_group_reservations(fs_info, ins->objectid); 7954 } else if (ret == -ENOSPC) { 7955 if (!final_tried && ins->offset) { 7956 num_bytes = min(num_bytes >> 1, ins->offset); 7957 num_bytes = round_down(num_bytes, 7958 fs_info->sectorsize); 7959 num_bytes = max(num_bytes, min_alloc_size); 7960 ram_bytes = num_bytes; 7961 if (num_bytes == min_alloc_size) 7962 final_tried = true; 7963 goto again; 7964 } else if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { 7965 struct btrfs_space_info *sinfo; 7966 7967 sinfo = __find_space_info(fs_info, flags); 7968 btrfs_err(fs_info, 7969 "allocation failed flags %llu, wanted %llu", 7970 flags, num_bytes); 7971 if (sinfo) 7972 dump_space_info(fs_info, sinfo, num_bytes, 1); 7973 } 7974 } 7975 7976 return ret; 7977 } 7978 7979 static int __btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info, 7980 u64 start, u64 len, 7981 int pin, int delalloc) 7982 { 7983 struct btrfs_block_group_cache *cache; 7984 int ret = 0; 7985 7986 cache = btrfs_lookup_block_group(fs_info, start); 7987 if (!cache) { 7988 btrfs_err(fs_info, "Unable to find block group for %llu", 7989 start); 7990 return -ENOSPC; 7991 } 7992 7993 if (pin) 7994 pin_down_extent(fs_info, cache, start, len, 1); 7995 else { 7996 if (btrfs_test_opt(fs_info, DISCARD)) 7997 ret = btrfs_discard_extent(fs_info, start, len, NULL); 7998 btrfs_add_free_space(cache, start, len); 7999 btrfs_free_reserved_bytes(cache, len, delalloc); 8000 trace_btrfs_reserved_extent_free(fs_info, start, len); 8001 } 8002 8003 btrfs_put_block_group(cache); 8004 return ret; 8005 } 8006 8007 int btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info, 8008 u64 start, u64 len, int delalloc) 8009 { 8010 return __btrfs_free_reserved_extent(fs_info, start, len, 0, delalloc); 8011 } 8012 8013 int btrfs_free_and_pin_reserved_extent(struct btrfs_fs_info *fs_info, 8014 u64 start, u64 len) 8015 { 8016 return __btrfs_free_reserved_extent(fs_info, start, len, 1, 0); 8017 } 8018 8019 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, 8020 struct btrfs_fs_info *fs_info, 8021 u64 parent, u64 root_objectid, 8022 u64 flags, u64 owner, u64 offset, 8023 struct btrfs_key *ins, int ref_mod) 8024 { 8025 int ret; 8026 struct btrfs_extent_item *extent_item; 8027 struct btrfs_extent_inline_ref *iref; 8028 struct btrfs_path *path; 8029 struct extent_buffer *leaf; 8030 int type; 8031 u32 size; 8032 8033 if (parent > 0) 8034 type = BTRFS_SHARED_DATA_REF_KEY; 8035 else 8036 type = BTRFS_EXTENT_DATA_REF_KEY; 8037 8038 size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type); 8039 8040 path = btrfs_alloc_path(); 8041 if (!path) 8042 return -ENOMEM; 8043 8044 path->leave_spinning = 1; 8045 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path, 8046 ins, size); 8047 if (ret) { 8048 btrfs_free_path(path); 8049 return ret; 8050 } 8051 8052 leaf = path->nodes[0]; 8053 extent_item = btrfs_item_ptr(leaf, path->slots[0], 8054 struct btrfs_extent_item); 8055 btrfs_set_extent_refs(leaf, extent_item, ref_mod); 8056 btrfs_set_extent_generation(leaf, extent_item, trans->transid); 8057 btrfs_set_extent_flags(leaf, extent_item, 8058 flags | BTRFS_EXTENT_FLAG_DATA); 8059 8060 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1); 8061 btrfs_set_extent_inline_ref_type(leaf, iref, type); 8062 if (parent > 0) { 8063 struct btrfs_shared_data_ref *ref; 8064 ref = (struct btrfs_shared_data_ref *)(iref + 1); 8065 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 8066 btrfs_set_shared_data_ref_count(leaf, ref, ref_mod); 8067 } else { 8068 struct btrfs_extent_data_ref *ref; 8069 ref = (struct btrfs_extent_data_ref *)(&iref->offset); 8070 btrfs_set_extent_data_ref_root(leaf, ref, root_objectid); 8071 btrfs_set_extent_data_ref_objectid(leaf, ref, owner); 8072 btrfs_set_extent_data_ref_offset(leaf, ref, offset); 8073 btrfs_set_extent_data_ref_count(leaf, ref, ref_mod); 8074 } 8075 8076 btrfs_mark_buffer_dirty(path->nodes[0]); 8077 btrfs_free_path(path); 8078 8079 ret = remove_from_free_space_tree(trans, fs_info, ins->objectid, 8080 ins->offset); 8081 if (ret) 8082 return ret; 8083 8084 ret = update_block_group(trans, fs_info, ins->objectid, ins->offset, 1); 8085 if (ret) { /* -ENOENT, logic error */ 8086 btrfs_err(fs_info, "update block group failed for %llu %llu", 8087 ins->objectid, ins->offset); 8088 BUG(); 8089 } 8090 trace_btrfs_reserved_extent_alloc(fs_info, ins->objectid, ins->offset); 8091 return ret; 8092 } 8093 8094 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, 8095 struct btrfs_fs_info *fs_info, 8096 u64 parent, u64 root_objectid, 8097 u64 flags, struct btrfs_disk_key *key, 8098 int level, struct btrfs_key *ins) 8099 { 8100 int ret; 8101 struct btrfs_extent_item *extent_item; 8102 struct btrfs_tree_block_info *block_info; 8103 struct btrfs_extent_inline_ref *iref; 8104 struct btrfs_path *path; 8105 struct extent_buffer *leaf; 8106 u32 size = sizeof(*extent_item) + sizeof(*iref); 8107 u64 num_bytes = ins->offset; 8108 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); 8109 8110 if (!skinny_metadata) 8111 size += sizeof(*block_info); 8112 8113 path = btrfs_alloc_path(); 8114 if (!path) { 8115 btrfs_free_and_pin_reserved_extent(fs_info, ins->objectid, 8116 fs_info->nodesize); 8117 return -ENOMEM; 8118 } 8119 8120 path->leave_spinning = 1; 8121 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path, 8122 ins, size); 8123 if (ret) { 8124 btrfs_free_path(path); 8125 btrfs_free_and_pin_reserved_extent(fs_info, ins->objectid, 8126 fs_info->nodesize); 8127 return ret; 8128 } 8129 8130 leaf = path->nodes[0]; 8131 extent_item = btrfs_item_ptr(leaf, path->slots[0], 8132 struct btrfs_extent_item); 8133 btrfs_set_extent_refs(leaf, extent_item, 1); 8134 btrfs_set_extent_generation(leaf, extent_item, trans->transid); 8135 btrfs_set_extent_flags(leaf, extent_item, 8136 flags | BTRFS_EXTENT_FLAG_TREE_BLOCK); 8137 8138 if (skinny_metadata) { 8139 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1); 8140 num_bytes = fs_info->nodesize; 8141 } else { 8142 block_info = (struct btrfs_tree_block_info *)(extent_item + 1); 8143 btrfs_set_tree_block_key(leaf, block_info, key); 8144 btrfs_set_tree_block_level(leaf, block_info, level); 8145 iref = (struct btrfs_extent_inline_ref *)(block_info + 1); 8146 } 8147 8148 if (parent > 0) { 8149 BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 8150 btrfs_set_extent_inline_ref_type(leaf, iref, 8151 BTRFS_SHARED_BLOCK_REF_KEY); 8152 btrfs_set_extent_inline_ref_offset(leaf, iref, parent); 8153 } else { 8154 btrfs_set_extent_inline_ref_type(leaf, iref, 8155 BTRFS_TREE_BLOCK_REF_KEY); 8156 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid); 8157 } 8158 8159 btrfs_mark_buffer_dirty(leaf); 8160 btrfs_free_path(path); 8161 8162 ret = remove_from_free_space_tree(trans, fs_info, ins->objectid, 8163 num_bytes); 8164 if (ret) 8165 return ret; 8166 8167 ret = update_block_group(trans, fs_info, ins->objectid, 8168 fs_info->nodesize, 1); 8169 if (ret) { /* -ENOENT, logic error */ 8170 btrfs_err(fs_info, "update block group failed for %llu %llu", 8171 ins->objectid, ins->offset); 8172 BUG(); 8173 } 8174 8175 trace_btrfs_reserved_extent_alloc(fs_info, ins->objectid, 8176 fs_info->nodesize); 8177 return ret; 8178 } 8179 8180 int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans, 8181 u64 root_objectid, u64 owner, 8182 u64 offset, u64 ram_bytes, 8183 struct btrfs_key *ins) 8184 { 8185 struct btrfs_fs_info *fs_info = trans->fs_info; 8186 int ret; 8187 8188 BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID); 8189 8190 ret = btrfs_add_delayed_data_ref(fs_info, trans, ins->objectid, 8191 ins->offset, 0, 8192 root_objectid, owner, offset, 8193 ram_bytes, BTRFS_ADD_DELAYED_EXTENT, 8194 NULL); 8195 return ret; 8196 } 8197 8198 /* 8199 * this is used by the tree logging recovery code. It records that 8200 * an extent has been allocated and makes sure to clear the free 8201 * space cache bits as well 8202 */ 8203 int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans, 8204 struct btrfs_fs_info *fs_info, 8205 u64 root_objectid, u64 owner, u64 offset, 8206 struct btrfs_key *ins) 8207 { 8208 int ret; 8209 struct btrfs_block_group_cache *block_group; 8210 struct btrfs_space_info *space_info; 8211 8212 /* 8213 * Mixed block groups will exclude before processing the log so we only 8214 * need to do the exclude dance if this fs isn't mixed. 8215 */ 8216 if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS)) { 8217 ret = __exclude_logged_extent(fs_info, ins->objectid, 8218 ins->offset); 8219 if (ret) 8220 return ret; 8221 } 8222 8223 block_group = btrfs_lookup_block_group(fs_info, ins->objectid); 8224 if (!block_group) 8225 return -EINVAL; 8226 8227 space_info = block_group->space_info; 8228 spin_lock(&space_info->lock); 8229 spin_lock(&block_group->lock); 8230 space_info->bytes_reserved += ins->offset; 8231 block_group->reserved += ins->offset; 8232 spin_unlock(&block_group->lock); 8233 spin_unlock(&space_info->lock); 8234 8235 ret = alloc_reserved_file_extent(trans, fs_info, 0, root_objectid, 8236 0, owner, offset, ins, 1); 8237 btrfs_put_block_group(block_group); 8238 return ret; 8239 } 8240 8241 static struct extent_buffer * 8242 btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, 8243 u64 bytenr, int level) 8244 { 8245 struct btrfs_fs_info *fs_info = root->fs_info; 8246 struct extent_buffer *buf; 8247 8248 buf = btrfs_find_create_tree_block(fs_info, bytenr); 8249 if (IS_ERR(buf)) 8250 return buf; 8251 8252 btrfs_set_header_generation(buf, trans->transid); 8253 btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level); 8254 btrfs_tree_lock(buf); 8255 clean_tree_block(trans, fs_info, buf); 8256 clear_bit(EXTENT_BUFFER_STALE, &buf->bflags); 8257 8258 btrfs_set_lock_blocking(buf); 8259 set_extent_buffer_uptodate(buf); 8260 8261 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) { 8262 buf->log_index = root->log_transid % 2; 8263 /* 8264 * we allow two log transactions at a time, use different 8265 * EXENT bit to differentiate dirty pages. 8266 */ 8267 if (buf->log_index == 0) 8268 set_extent_dirty(&root->dirty_log_pages, buf->start, 8269 buf->start + buf->len - 1, GFP_NOFS); 8270 else 8271 set_extent_new(&root->dirty_log_pages, buf->start, 8272 buf->start + buf->len - 1); 8273 } else { 8274 buf->log_index = -1; 8275 set_extent_dirty(&trans->transaction->dirty_pages, buf->start, 8276 buf->start + buf->len - 1, GFP_NOFS); 8277 } 8278 trans->dirty = true; 8279 /* this returns a buffer locked for blocking */ 8280 return buf; 8281 } 8282 8283 static struct btrfs_block_rsv * 8284 use_block_rsv(struct btrfs_trans_handle *trans, 8285 struct btrfs_root *root, u32 blocksize) 8286 { 8287 struct btrfs_fs_info *fs_info = root->fs_info; 8288 struct btrfs_block_rsv *block_rsv; 8289 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 8290 int ret; 8291 bool global_updated = false; 8292 8293 block_rsv = get_block_rsv(trans, root); 8294 8295 if (unlikely(block_rsv->size == 0)) 8296 goto try_reserve; 8297 again: 8298 ret = block_rsv_use_bytes(block_rsv, blocksize); 8299 if (!ret) 8300 return block_rsv; 8301 8302 if (block_rsv->failfast) 8303 return ERR_PTR(ret); 8304 8305 if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) { 8306 global_updated = true; 8307 update_global_block_rsv(fs_info); 8308 goto again; 8309 } 8310 8311 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { 8312 static DEFINE_RATELIMIT_STATE(_rs, 8313 DEFAULT_RATELIMIT_INTERVAL * 10, 8314 /*DEFAULT_RATELIMIT_BURST*/ 1); 8315 if (__ratelimit(&_rs)) 8316 WARN(1, KERN_DEBUG 8317 "BTRFS: block rsv returned %d\n", ret); 8318 } 8319 try_reserve: 8320 ret = reserve_metadata_bytes(root, block_rsv, blocksize, 8321 BTRFS_RESERVE_NO_FLUSH); 8322 if (!ret) 8323 return block_rsv; 8324 /* 8325 * If we couldn't reserve metadata bytes try and use some from 8326 * the global reserve if its space type is the same as the global 8327 * reservation. 8328 */ 8329 if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL && 8330 block_rsv->space_info == global_rsv->space_info) { 8331 ret = block_rsv_use_bytes(global_rsv, blocksize); 8332 if (!ret) 8333 return global_rsv; 8334 } 8335 return ERR_PTR(ret); 8336 } 8337 8338 static void unuse_block_rsv(struct btrfs_fs_info *fs_info, 8339 struct btrfs_block_rsv *block_rsv, u32 blocksize) 8340 { 8341 block_rsv_add_bytes(block_rsv, blocksize, 0); 8342 block_rsv_release_bytes(fs_info, block_rsv, NULL, 0); 8343 } 8344 8345 /* 8346 * finds a free extent and does all the dirty work required for allocation 8347 * returns the tree buffer or an ERR_PTR on error. 8348 */ 8349 struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans, 8350 struct btrfs_root *root, 8351 u64 parent, u64 root_objectid, 8352 struct btrfs_disk_key *key, int level, 8353 u64 hint, u64 empty_size) 8354 { 8355 struct btrfs_fs_info *fs_info = root->fs_info; 8356 struct btrfs_key ins; 8357 struct btrfs_block_rsv *block_rsv; 8358 struct extent_buffer *buf; 8359 struct btrfs_delayed_extent_op *extent_op; 8360 u64 flags = 0; 8361 int ret; 8362 u32 blocksize = fs_info->nodesize; 8363 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); 8364 8365 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 8366 if (btrfs_is_testing(fs_info)) { 8367 buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr, 8368 level); 8369 if (!IS_ERR(buf)) 8370 root->alloc_bytenr += blocksize; 8371 return buf; 8372 } 8373 #endif 8374 8375 block_rsv = use_block_rsv(trans, root, blocksize); 8376 if (IS_ERR(block_rsv)) 8377 return ERR_CAST(block_rsv); 8378 8379 ret = btrfs_reserve_extent(root, blocksize, blocksize, blocksize, 8380 empty_size, hint, &ins, 0, 0); 8381 if (ret) 8382 goto out_unuse; 8383 8384 buf = btrfs_init_new_buffer(trans, root, ins.objectid, level); 8385 if (IS_ERR(buf)) { 8386 ret = PTR_ERR(buf); 8387 goto out_free_reserved; 8388 } 8389 8390 if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) { 8391 if (parent == 0) 8392 parent = ins.objectid; 8393 flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 8394 } else 8395 BUG_ON(parent > 0); 8396 8397 if (root_objectid != BTRFS_TREE_LOG_OBJECTID) { 8398 extent_op = btrfs_alloc_delayed_extent_op(); 8399 if (!extent_op) { 8400 ret = -ENOMEM; 8401 goto out_free_buf; 8402 } 8403 if (key) 8404 memcpy(&extent_op->key, key, sizeof(extent_op->key)); 8405 else 8406 memset(&extent_op->key, 0, sizeof(extent_op->key)); 8407 extent_op->flags_to_set = flags; 8408 extent_op->update_key = skinny_metadata ? false : true; 8409 extent_op->update_flags = true; 8410 extent_op->is_data = false; 8411 extent_op->level = level; 8412 8413 ret = btrfs_add_delayed_tree_ref(fs_info, trans, 8414 ins.objectid, ins.offset, 8415 parent, root_objectid, level, 8416 BTRFS_ADD_DELAYED_EXTENT, 8417 extent_op); 8418 if (ret) 8419 goto out_free_delayed; 8420 } 8421 return buf; 8422 8423 out_free_delayed: 8424 btrfs_free_delayed_extent_op(extent_op); 8425 out_free_buf: 8426 free_extent_buffer(buf); 8427 out_free_reserved: 8428 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 0); 8429 out_unuse: 8430 unuse_block_rsv(fs_info, block_rsv, blocksize); 8431 return ERR_PTR(ret); 8432 } 8433 8434 struct walk_control { 8435 u64 refs[BTRFS_MAX_LEVEL]; 8436 u64 flags[BTRFS_MAX_LEVEL]; 8437 struct btrfs_key update_progress; 8438 int stage; 8439 int level; 8440 int shared_level; 8441 int update_ref; 8442 int keep_locks; 8443 int reada_slot; 8444 int reada_count; 8445 int for_reloc; 8446 }; 8447 8448 #define DROP_REFERENCE 1 8449 #define UPDATE_BACKREF 2 8450 8451 static noinline void reada_walk_down(struct btrfs_trans_handle *trans, 8452 struct btrfs_root *root, 8453 struct walk_control *wc, 8454 struct btrfs_path *path) 8455 { 8456 struct btrfs_fs_info *fs_info = root->fs_info; 8457 u64 bytenr; 8458 u64 generation; 8459 u64 refs; 8460 u64 flags; 8461 u32 nritems; 8462 struct btrfs_key key; 8463 struct extent_buffer *eb; 8464 int ret; 8465 int slot; 8466 int nread = 0; 8467 8468 if (path->slots[wc->level] < wc->reada_slot) { 8469 wc->reada_count = wc->reada_count * 2 / 3; 8470 wc->reada_count = max(wc->reada_count, 2); 8471 } else { 8472 wc->reada_count = wc->reada_count * 3 / 2; 8473 wc->reada_count = min_t(int, wc->reada_count, 8474 BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 8475 } 8476 8477 eb = path->nodes[wc->level]; 8478 nritems = btrfs_header_nritems(eb); 8479 8480 for (slot = path->slots[wc->level]; slot < nritems; slot++) { 8481 if (nread >= wc->reada_count) 8482 break; 8483 8484 cond_resched(); 8485 bytenr = btrfs_node_blockptr(eb, slot); 8486 generation = btrfs_node_ptr_generation(eb, slot); 8487 8488 if (slot == path->slots[wc->level]) 8489 goto reada; 8490 8491 if (wc->stage == UPDATE_BACKREF && 8492 generation <= root->root_key.offset) 8493 continue; 8494 8495 /* We don't lock the tree block, it's OK to be racy here */ 8496 ret = btrfs_lookup_extent_info(trans, fs_info, bytenr, 8497 wc->level - 1, 1, &refs, 8498 &flags); 8499 /* We don't care about errors in readahead. */ 8500 if (ret < 0) 8501 continue; 8502 BUG_ON(refs == 0); 8503 8504 if (wc->stage == DROP_REFERENCE) { 8505 if (refs == 1) 8506 goto reada; 8507 8508 if (wc->level == 1 && 8509 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8510 continue; 8511 if (!wc->update_ref || 8512 generation <= root->root_key.offset) 8513 continue; 8514 btrfs_node_key_to_cpu(eb, &key, slot); 8515 ret = btrfs_comp_cpu_keys(&key, 8516 &wc->update_progress); 8517 if (ret < 0) 8518 continue; 8519 } else { 8520 if (wc->level == 1 && 8521 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8522 continue; 8523 } 8524 reada: 8525 readahead_tree_block(fs_info, bytenr); 8526 nread++; 8527 } 8528 wc->reada_slot = slot; 8529 } 8530 8531 /* 8532 * helper to process tree block while walking down the tree. 8533 * 8534 * when wc->stage == UPDATE_BACKREF, this function updates 8535 * back refs for pointers in the block. 8536 * 8537 * NOTE: return value 1 means we should stop walking down. 8538 */ 8539 static noinline int walk_down_proc(struct btrfs_trans_handle *trans, 8540 struct btrfs_root *root, 8541 struct btrfs_path *path, 8542 struct walk_control *wc, int lookup_info) 8543 { 8544 struct btrfs_fs_info *fs_info = root->fs_info; 8545 int level = wc->level; 8546 struct extent_buffer *eb = path->nodes[level]; 8547 u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF; 8548 int ret; 8549 8550 if (wc->stage == UPDATE_BACKREF && 8551 btrfs_header_owner(eb) != root->root_key.objectid) 8552 return 1; 8553 8554 /* 8555 * when reference count of tree block is 1, it won't increase 8556 * again. once full backref flag is set, we never clear it. 8557 */ 8558 if (lookup_info && 8559 ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) || 8560 (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) { 8561 BUG_ON(!path->locks[level]); 8562 ret = btrfs_lookup_extent_info(trans, fs_info, 8563 eb->start, level, 1, 8564 &wc->refs[level], 8565 &wc->flags[level]); 8566 BUG_ON(ret == -ENOMEM); 8567 if (ret) 8568 return ret; 8569 BUG_ON(wc->refs[level] == 0); 8570 } 8571 8572 if (wc->stage == DROP_REFERENCE) { 8573 if (wc->refs[level] > 1) 8574 return 1; 8575 8576 if (path->locks[level] && !wc->keep_locks) { 8577 btrfs_tree_unlock_rw(eb, path->locks[level]); 8578 path->locks[level] = 0; 8579 } 8580 return 0; 8581 } 8582 8583 /* wc->stage == UPDATE_BACKREF */ 8584 if (!(wc->flags[level] & flag)) { 8585 BUG_ON(!path->locks[level]); 8586 ret = btrfs_inc_ref(trans, root, eb, 1); 8587 BUG_ON(ret); /* -ENOMEM */ 8588 ret = btrfs_dec_ref(trans, root, eb, 0); 8589 BUG_ON(ret); /* -ENOMEM */ 8590 ret = btrfs_set_disk_extent_flags(trans, fs_info, eb->start, 8591 eb->len, flag, 8592 btrfs_header_level(eb), 0); 8593 BUG_ON(ret); /* -ENOMEM */ 8594 wc->flags[level] |= flag; 8595 } 8596 8597 /* 8598 * the block is shared by multiple trees, so it's not good to 8599 * keep the tree lock 8600 */ 8601 if (path->locks[level] && level > 0) { 8602 btrfs_tree_unlock_rw(eb, path->locks[level]); 8603 path->locks[level] = 0; 8604 } 8605 return 0; 8606 } 8607 8608 /* 8609 * helper to process tree block pointer. 8610 * 8611 * when wc->stage == DROP_REFERENCE, this function checks 8612 * reference count of the block pointed to. if the block 8613 * is shared and we need update back refs for the subtree 8614 * rooted at the block, this function changes wc->stage to 8615 * UPDATE_BACKREF. if the block is shared and there is no 8616 * need to update back, this function drops the reference 8617 * to the block. 8618 * 8619 * NOTE: return value 1 means we should stop walking down. 8620 */ 8621 static noinline int do_walk_down(struct btrfs_trans_handle *trans, 8622 struct btrfs_root *root, 8623 struct btrfs_path *path, 8624 struct walk_control *wc, int *lookup_info) 8625 { 8626 struct btrfs_fs_info *fs_info = root->fs_info; 8627 u64 bytenr; 8628 u64 generation; 8629 u64 parent; 8630 u32 blocksize; 8631 struct btrfs_key key; 8632 struct extent_buffer *next; 8633 int level = wc->level; 8634 int reada = 0; 8635 int ret = 0; 8636 bool need_account = false; 8637 8638 generation = btrfs_node_ptr_generation(path->nodes[level], 8639 path->slots[level]); 8640 /* 8641 * if the lower level block was created before the snapshot 8642 * was created, we know there is no need to update back refs 8643 * for the subtree 8644 */ 8645 if (wc->stage == UPDATE_BACKREF && 8646 generation <= root->root_key.offset) { 8647 *lookup_info = 1; 8648 return 1; 8649 } 8650 8651 bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]); 8652 blocksize = fs_info->nodesize; 8653 8654 next = find_extent_buffer(fs_info, bytenr); 8655 if (!next) { 8656 next = btrfs_find_create_tree_block(fs_info, bytenr); 8657 if (IS_ERR(next)) 8658 return PTR_ERR(next); 8659 8660 btrfs_set_buffer_lockdep_class(root->root_key.objectid, next, 8661 level - 1); 8662 reada = 1; 8663 } 8664 btrfs_tree_lock(next); 8665 btrfs_set_lock_blocking(next); 8666 8667 ret = btrfs_lookup_extent_info(trans, fs_info, bytenr, level - 1, 1, 8668 &wc->refs[level - 1], 8669 &wc->flags[level - 1]); 8670 if (ret < 0) 8671 goto out_unlock; 8672 8673 if (unlikely(wc->refs[level - 1] == 0)) { 8674 btrfs_err(fs_info, "Missing references."); 8675 ret = -EIO; 8676 goto out_unlock; 8677 } 8678 *lookup_info = 0; 8679 8680 if (wc->stage == DROP_REFERENCE) { 8681 if (wc->refs[level - 1] > 1) { 8682 need_account = true; 8683 if (level == 1 && 8684 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8685 goto skip; 8686 8687 if (!wc->update_ref || 8688 generation <= root->root_key.offset) 8689 goto skip; 8690 8691 btrfs_node_key_to_cpu(path->nodes[level], &key, 8692 path->slots[level]); 8693 ret = btrfs_comp_cpu_keys(&key, &wc->update_progress); 8694 if (ret < 0) 8695 goto skip; 8696 8697 wc->stage = UPDATE_BACKREF; 8698 wc->shared_level = level - 1; 8699 } 8700 } else { 8701 if (level == 1 && 8702 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF)) 8703 goto skip; 8704 } 8705 8706 if (!btrfs_buffer_uptodate(next, generation, 0)) { 8707 btrfs_tree_unlock(next); 8708 free_extent_buffer(next); 8709 next = NULL; 8710 *lookup_info = 1; 8711 } 8712 8713 if (!next) { 8714 if (reada && level == 1) 8715 reada_walk_down(trans, root, wc, path); 8716 next = read_tree_block(fs_info, bytenr, generation); 8717 if (IS_ERR(next)) { 8718 return PTR_ERR(next); 8719 } else if (!extent_buffer_uptodate(next)) { 8720 free_extent_buffer(next); 8721 return -EIO; 8722 } 8723 btrfs_tree_lock(next); 8724 btrfs_set_lock_blocking(next); 8725 } 8726 8727 level--; 8728 ASSERT(level == btrfs_header_level(next)); 8729 if (level != btrfs_header_level(next)) { 8730 btrfs_err(root->fs_info, "mismatched level"); 8731 ret = -EIO; 8732 goto out_unlock; 8733 } 8734 path->nodes[level] = next; 8735 path->slots[level] = 0; 8736 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8737 wc->level = level; 8738 if (wc->level == 1) 8739 wc->reada_slot = 0; 8740 return 0; 8741 skip: 8742 wc->refs[level - 1] = 0; 8743 wc->flags[level - 1] = 0; 8744 if (wc->stage == DROP_REFERENCE) { 8745 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 8746 parent = path->nodes[level]->start; 8747 } else { 8748 ASSERT(root->root_key.objectid == 8749 btrfs_header_owner(path->nodes[level])); 8750 if (root->root_key.objectid != 8751 btrfs_header_owner(path->nodes[level])) { 8752 btrfs_err(root->fs_info, 8753 "mismatched block owner"); 8754 ret = -EIO; 8755 goto out_unlock; 8756 } 8757 parent = 0; 8758 } 8759 8760 if (need_account) { 8761 ret = btrfs_qgroup_trace_subtree(trans, root, next, 8762 generation, level - 1); 8763 if (ret) { 8764 btrfs_err_rl(fs_info, 8765 "Error %d accounting shared subtree. Quota is out of sync, rescan required.", 8766 ret); 8767 } 8768 } 8769 ret = btrfs_free_extent(trans, fs_info, bytenr, blocksize, 8770 parent, root->root_key.objectid, 8771 level - 1, 0); 8772 if (ret) 8773 goto out_unlock; 8774 } 8775 8776 *lookup_info = 1; 8777 ret = 1; 8778 8779 out_unlock: 8780 btrfs_tree_unlock(next); 8781 free_extent_buffer(next); 8782 8783 return ret; 8784 } 8785 8786 /* 8787 * helper to process tree block while walking up the tree. 8788 * 8789 * when wc->stage == DROP_REFERENCE, this function drops 8790 * reference count on the block. 8791 * 8792 * when wc->stage == UPDATE_BACKREF, this function changes 8793 * wc->stage back to DROP_REFERENCE if we changed wc->stage 8794 * to UPDATE_BACKREF previously while processing the block. 8795 * 8796 * NOTE: return value 1 means we should stop walking up. 8797 */ 8798 static noinline int walk_up_proc(struct btrfs_trans_handle *trans, 8799 struct btrfs_root *root, 8800 struct btrfs_path *path, 8801 struct walk_control *wc) 8802 { 8803 struct btrfs_fs_info *fs_info = root->fs_info; 8804 int ret; 8805 int level = wc->level; 8806 struct extent_buffer *eb = path->nodes[level]; 8807 u64 parent = 0; 8808 8809 if (wc->stage == UPDATE_BACKREF) { 8810 BUG_ON(wc->shared_level < level); 8811 if (level < wc->shared_level) 8812 goto out; 8813 8814 ret = find_next_key(path, level + 1, &wc->update_progress); 8815 if (ret > 0) 8816 wc->update_ref = 0; 8817 8818 wc->stage = DROP_REFERENCE; 8819 wc->shared_level = -1; 8820 path->slots[level] = 0; 8821 8822 /* 8823 * check reference count again if the block isn't locked. 8824 * we should start walking down the tree again if reference 8825 * count is one. 8826 */ 8827 if (!path->locks[level]) { 8828 BUG_ON(level == 0); 8829 btrfs_tree_lock(eb); 8830 btrfs_set_lock_blocking(eb); 8831 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8832 8833 ret = btrfs_lookup_extent_info(trans, fs_info, 8834 eb->start, level, 1, 8835 &wc->refs[level], 8836 &wc->flags[level]); 8837 if (ret < 0) { 8838 btrfs_tree_unlock_rw(eb, path->locks[level]); 8839 path->locks[level] = 0; 8840 return ret; 8841 } 8842 BUG_ON(wc->refs[level] == 0); 8843 if (wc->refs[level] == 1) { 8844 btrfs_tree_unlock_rw(eb, path->locks[level]); 8845 path->locks[level] = 0; 8846 return 1; 8847 } 8848 } 8849 } 8850 8851 /* wc->stage == DROP_REFERENCE */ 8852 BUG_ON(wc->refs[level] > 1 && !path->locks[level]); 8853 8854 if (wc->refs[level] == 1) { 8855 if (level == 0) { 8856 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) 8857 ret = btrfs_dec_ref(trans, root, eb, 1); 8858 else 8859 ret = btrfs_dec_ref(trans, root, eb, 0); 8860 BUG_ON(ret); /* -ENOMEM */ 8861 ret = btrfs_qgroup_trace_leaf_items(trans, fs_info, eb); 8862 if (ret) { 8863 btrfs_err_rl(fs_info, 8864 "error %d accounting leaf items. Quota is out of sync, rescan required.", 8865 ret); 8866 } 8867 } 8868 /* make block locked assertion in clean_tree_block happy */ 8869 if (!path->locks[level] && 8870 btrfs_header_generation(eb) == trans->transid) { 8871 btrfs_tree_lock(eb); 8872 btrfs_set_lock_blocking(eb); 8873 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 8874 } 8875 clean_tree_block(trans, fs_info, eb); 8876 } 8877 8878 if (eb == root->node) { 8879 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) 8880 parent = eb->start; 8881 else 8882 BUG_ON(root->root_key.objectid != 8883 btrfs_header_owner(eb)); 8884 } else { 8885 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF) 8886 parent = path->nodes[level + 1]->start; 8887 else 8888 BUG_ON(root->root_key.objectid != 8889 btrfs_header_owner(path->nodes[level + 1])); 8890 } 8891 8892 btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1); 8893 out: 8894 wc->refs[level] = 0; 8895 wc->flags[level] = 0; 8896 return 0; 8897 } 8898 8899 static noinline int walk_down_tree(struct btrfs_trans_handle *trans, 8900 struct btrfs_root *root, 8901 struct btrfs_path *path, 8902 struct walk_control *wc) 8903 { 8904 int level = wc->level; 8905 int lookup_info = 1; 8906 int ret; 8907 8908 while (level >= 0) { 8909 ret = walk_down_proc(trans, root, path, wc, lookup_info); 8910 if (ret > 0) 8911 break; 8912 8913 if (level == 0) 8914 break; 8915 8916 if (path->slots[level] >= 8917 btrfs_header_nritems(path->nodes[level])) 8918 break; 8919 8920 ret = do_walk_down(trans, root, path, wc, &lookup_info); 8921 if (ret > 0) { 8922 path->slots[level]++; 8923 continue; 8924 } else if (ret < 0) 8925 return ret; 8926 level = wc->level; 8927 } 8928 return 0; 8929 } 8930 8931 static noinline int walk_up_tree(struct btrfs_trans_handle *trans, 8932 struct btrfs_root *root, 8933 struct btrfs_path *path, 8934 struct walk_control *wc, int max_level) 8935 { 8936 int level = wc->level; 8937 int ret; 8938 8939 path->slots[level] = btrfs_header_nritems(path->nodes[level]); 8940 while (level < max_level && path->nodes[level]) { 8941 wc->level = level; 8942 if (path->slots[level] + 1 < 8943 btrfs_header_nritems(path->nodes[level])) { 8944 path->slots[level]++; 8945 return 0; 8946 } else { 8947 ret = walk_up_proc(trans, root, path, wc); 8948 if (ret > 0) 8949 return 0; 8950 8951 if (path->locks[level]) { 8952 btrfs_tree_unlock_rw(path->nodes[level], 8953 path->locks[level]); 8954 path->locks[level] = 0; 8955 } 8956 free_extent_buffer(path->nodes[level]); 8957 path->nodes[level] = NULL; 8958 level++; 8959 } 8960 } 8961 return 1; 8962 } 8963 8964 /* 8965 * drop a subvolume tree. 8966 * 8967 * this function traverses the tree freeing any blocks that only 8968 * referenced by the tree. 8969 * 8970 * when a shared tree block is found. this function decreases its 8971 * reference count by one. if update_ref is true, this function 8972 * also make sure backrefs for the shared block and all lower level 8973 * blocks are properly updated. 8974 * 8975 * If called with for_reloc == 0, may exit early with -EAGAIN 8976 */ 8977 int btrfs_drop_snapshot(struct btrfs_root *root, 8978 struct btrfs_block_rsv *block_rsv, int update_ref, 8979 int for_reloc) 8980 { 8981 struct btrfs_fs_info *fs_info = root->fs_info; 8982 struct btrfs_path *path; 8983 struct btrfs_trans_handle *trans; 8984 struct btrfs_root *tree_root = fs_info->tree_root; 8985 struct btrfs_root_item *root_item = &root->root_item; 8986 struct walk_control *wc; 8987 struct btrfs_key key; 8988 int err = 0; 8989 int ret; 8990 int level; 8991 bool root_dropped = false; 8992 8993 btrfs_debug(fs_info, "Drop subvolume %llu", root->objectid); 8994 8995 path = btrfs_alloc_path(); 8996 if (!path) { 8997 err = -ENOMEM; 8998 goto out; 8999 } 9000 9001 wc = kzalloc(sizeof(*wc), GFP_NOFS); 9002 if (!wc) { 9003 btrfs_free_path(path); 9004 err = -ENOMEM; 9005 goto out; 9006 } 9007 9008 trans = btrfs_start_transaction(tree_root, 0); 9009 if (IS_ERR(trans)) { 9010 err = PTR_ERR(trans); 9011 goto out_free; 9012 } 9013 9014 if (block_rsv) 9015 trans->block_rsv = block_rsv; 9016 9017 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) { 9018 level = btrfs_header_level(root->node); 9019 path->nodes[level] = btrfs_lock_root_node(root); 9020 btrfs_set_lock_blocking(path->nodes[level]); 9021 path->slots[level] = 0; 9022 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 9023 memset(&wc->update_progress, 0, 9024 sizeof(wc->update_progress)); 9025 } else { 9026 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress); 9027 memcpy(&wc->update_progress, &key, 9028 sizeof(wc->update_progress)); 9029 9030 level = root_item->drop_level; 9031 BUG_ON(level == 0); 9032 path->lowest_level = level; 9033 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 9034 path->lowest_level = 0; 9035 if (ret < 0) { 9036 err = ret; 9037 goto out_end_trans; 9038 } 9039 WARN_ON(ret > 0); 9040 9041 /* 9042 * unlock our path, this is safe because only this 9043 * function is allowed to delete this snapshot 9044 */ 9045 btrfs_unlock_up_safe(path, 0); 9046 9047 level = btrfs_header_level(root->node); 9048 while (1) { 9049 btrfs_tree_lock(path->nodes[level]); 9050 btrfs_set_lock_blocking(path->nodes[level]); 9051 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 9052 9053 ret = btrfs_lookup_extent_info(trans, fs_info, 9054 path->nodes[level]->start, 9055 level, 1, &wc->refs[level], 9056 &wc->flags[level]); 9057 if (ret < 0) { 9058 err = ret; 9059 goto out_end_trans; 9060 } 9061 BUG_ON(wc->refs[level] == 0); 9062 9063 if (level == root_item->drop_level) 9064 break; 9065 9066 btrfs_tree_unlock(path->nodes[level]); 9067 path->locks[level] = 0; 9068 WARN_ON(wc->refs[level] != 1); 9069 level--; 9070 } 9071 } 9072 9073 wc->level = level; 9074 wc->shared_level = -1; 9075 wc->stage = DROP_REFERENCE; 9076 wc->update_ref = update_ref; 9077 wc->keep_locks = 0; 9078 wc->for_reloc = for_reloc; 9079 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info); 9080 9081 while (1) { 9082 9083 ret = walk_down_tree(trans, root, path, wc); 9084 if (ret < 0) { 9085 err = ret; 9086 break; 9087 } 9088 9089 ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL); 9090 if (ret < 0) { 9091 err = ret; 9092 break; 9093 } 9094 9095 if (ret > 0) { 9096 BUG_ON(wc->stage != DROP_REFERENCE); 9097 break; 9098 } 9099 9100 if (wc->stage == DROP_REFERENCE) { 9101 level = wc->level; 9102 btrfs_node_key(path->nodes[level], 9103 &root_item->drop_progress, 9104 path->slots[level]); 9105 root_item->drop_level = level; 9106 } 9107 9108 BUG_ON(wc->level == 0); 9109 if (btrfs_should_end_transaction(trans) || 9110 (!for_reloc && btrfs_need_cleaner_sleep(fs_info))) { 9111 ret = btrfs_update_root(trans, tree_root, 9112 &root->root_key, 9113 root_item); 9114 if (ret) { 9115 btrfs_abort_transaction(trans, ret); 9116 err = ret; 9117 goto out_end_trans; 9118 } 9119 9120 btrfs_end_transaction_throttle(trans); 9121 if (!for_reloc && btrfs_need_cleaner_sleep(fs_info)) { 9122 btrfs_debug(fs_info, 9123 "drop snapshot early exit"); 9124 err = -EAGAIN; 9125 goto out_free; 9126 } 9127 9128 trans = btrfs_start_transaction(tree_root, 0); 9129 if (IS_ERR(trans)) { 9130 err = PTR_ERR(trans); 9131 goto out_free; 9132 } 9133 if (block_rsv) 9134 trans->block_rsv = block_rsv; 9135 } 9136 } 9137 btrfs_release_path(path); 9138 if (err) 9139 goto out_end_trans; 9140 9141 ret = btrfs_del_root(trans, tree_root, &root->root_key); 9142 if (ret) { 9143 btrfs_abort_transaction(trans, ret); 9144 goto out_end_trans; 9145 } 9146 9147 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) { 9148 ret = btrfs_find_root(tree_root, &root->root_key, path, 9149 NULL, NULL); 9150 if (ret < 0) { 9151 btrfs_abort_transaction(trans, ret); 9152 err = ret; 9153 goto out_end_trans; 9154 } else if (ret > 0) { 9155 /* if we fail to delete the orphan item this time 9156 * around, it'll get picked up the next time. 9157 * 9158 * The most common failure here is just -ENOENT. 9159 */ 9160 btrfs_del_orphan_item(trans, tree_root, 9161 root->root_key.objectid); 9162 } 9163 } 9164 9165 if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) { 9166 btrfs_add_dropped_root(trans, root); 9167 } else { 9168 free_extent_buffer(root->node); 9169 free_extent_buffer(root->commit_root); 9170 btrfs_put_fs_root(root); 9171 } 9172 root_dropped = true; 9173 out_end_trans: 9174 btrfs_end_transaction_throttle(trans); 9175 out_free: 9176 kfree(wc); 9177 btrfs_free_path(path); 9178 out: 9179 /* 9180 * So if we need to stop dropping the snapshot for whatever reason we 9181 * need to make sure to add it back to the dead root list so that we 9182 * keep trying to do the work later. This also cleans up roots if we 9183 * don't have it in the radix (like when we recover after a power fail 9184 * or unmount) so we don't leak memory. 9185 */ 9186 if (!for_reloc && root_dropped == false) 9187 btrfs_add_dead_root(root); 9188 if (err && err != -EAGAIN) 9189 btrfs_handle_fs_error(fs_info, err, NULL); 9190 return err; 9191 } 9192 9193 /* 9194 * drop subtree rooted at tree block 'node'. 9195 * 9196 * NOTE: this function will unlock and release tree block 'node' 9197 * only used by relocation code 9198 */ 9199 int btrfs_drop_subtree(struct btrfs_trans_handle *trans, 9200 struct btrfs_root *root, 9201 struct extent_buffer *node, 9202 struct extent_buffer *parent) 9203 { 9204 struct btrfs_fs_info *fs_info = root->fs_info; 9205 struct btrfs_path *path; 9206 struct walk_control *wc; 9207 int level; 9208 int parent_level; 9209 int ret = 0; 9210 int wret; 9211 9212 BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID); 9213 9214 path = btrfs_alloc_path(); 9215 if (!path) 9216 return -ENOMEM; 9217 9218 wc = kzalloc(sizeof(*wc), GFP_NOFS); 9219 if (!wc) { 9220 btrfs_free_path(path); 9221 return -ENOMEM; 9222 } 9223 9224 btrfs_assert_tree_locked(parent); 9225 parent_level = btrfs_header_level(parent); 9226 extent_buffer_get(parent); 9227 path->nodes[parent_level] = parent; 9228 path->slots[parent_level] = btrfs_header_nritems(parent); 9229 9230 btrfs_assert_tree_locked(node); 9231 level = btrfs_header_level(node); 9232 path->nodes[level] = node; 9233 path->slots[level] = 0; 9234 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 9235 9236 wc->refs[parent_level] = 1; 9237 wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF; 9238 wc->level = level; 9239 wc->shared_level = -1; 9240 wc->stage = DROP_REFERENCE; 9241 wc->update_ref = 0; 9242 wc->keep_locks = 1; 9243 wc->for_reloc = 1; 9244 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info); 9245 9246 while (1) { 9247 wret = walk_down_tree(trans, root, path, wc); 9248 if (wret < 0) { 9249 ret = wret; 9250 break; 9251 } 9252 9253 wret = walk_up_tree(trans, root, path, wc, parent_level); 9254 if (wret < 0) 9255 ret = wret; 9256 if (wret != 0) 9257 break; 9258 } 9259 9260 kfree(wc); 9261 btrfs_free_path(path); 9262 return ret; 9263 } 9264 9265 static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags) 9266 { 9267 u64 num_devices; 9268 u64 stripped; 9269 9270 /* 9271 * if restripe for this chunk_type is on pick target profile and 9272 * return, otherwise do the usual balance 9273 */ 9274 stripped = get_restripe_target(fs_info, flags); 9275 if (stripped) 9276 return extended_to_chunk(stripped); 9277 9278 num_devices = fs_info->fs_devices->rw_devices; 9279 9280 stripped = BTRFS_BLOCK_GROUP_RAID0 | 9281 BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 | 9282 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10; 9283 9284 if (num_devices == 1) { 9285 stripped |= BTRFS_BLOCK_GROUP_DUP; 9286 stripped = flags & ~stripped; 9287 9288 /* turn raid0 into single device chunks */ 9289 if (flags & BTRFS_BLOCK_GROUP_RAID0) 9290 return stripped; 9291 9292 /* turn mirroring into duplication */ 9293 if (flags & (BTRFS_BLOCK_GROUP_RAID1 | 9294 BTRFS_BLOCK_GROUP_RAID10)) 9295 return stripped | BTRFS_BLOCK_GROUP_DUP; 9296 } else { 9297 /* they already had raid on here, just return */ 9298 if (flags & stripped) 9299 return flags; 9300 9301 stripped |= BTRFS_BLOCK_GROUP_DUP; 9302 stripped = flags & ~stripped; 9303 9304 /* switch duplicated blocks with raid1 */ 9305 if (flags & BTRFS_BLOCK_GROUP_DUP) 9306 return stripped | BTRFS_BLOCK_GROUP_RAID1; 9307 9308 /* this is drive concat, leave it alone */ 9309 } 9310 9311 return flags; 9312 } 9313 9314 static int inc_block_group_ro(struct btrfs_block_group_cache *cache, int force) 9315 { 9316 struct btrfs_space_info *sinfo = cache->space_info; 9317 u64 num_bytes; 9318 u64 min_allocable_bytes; 9319 int ret = -ENOSPC; 9320 9321 /* 9322 * We need some metadata space and system metadata space for 9323 * allocating chunks in some corner cases until we force to set 9324 * it to be readonly. 9325 */ 9326 if ((sinfo->flags & 9327 (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) && 9328 !force) 9329 min_allocable_bytes = SZ_1M; 9330 else 9331 min_allocable_bytes = 0; 9332 9333 spin_lock(&sinfo->lock); 9334 spin_lock(&cache->lock); 9335 9336 if (cache->ro) { 9337 cache->ro++; 9338 ret = 0; 9339 goto out; 9340 } 9341 9342 num_bytes = cache->key.offset - cache->reserved - cache->pinned - 9343 cache->bytes_super - btrfs_block_group_used(&cache->item); 9344 9345 if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned + 9346 sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes + 9347 min_allocable_bytes <= sinfo->total_bytes) { 9348 sinfo->bytes_readonly += num_bytes; 9349 cache->ro++; 9350 list_add_tail(&cache->ro_list, &sinfo->ro_bgs); 9351 ret = 0; 9352 } 9353 out: 9354 spin_unlock(&cache->lock); 9355 spin_unlock(&sinfo->lock); 9356 return ret; 9357 } 9358 9359 int btrfs_inc_block_group_ro(struct btrfs_root *root, 9360 struct btrfs_block_group_cache *cache) 9361 9362 { 9363 struct btrfs_fs_info *fs_info = root->fs_info; 9364 struct btrfs_trans_handle *trans; 9365 u64 alloc_flags; 9366 int ret; 9367 9368 again: 9369 trans = btrfs_join_transaction(root); 9370 if (IS_ERR(trans)) 9371 return PTR_ERR(trans); 9372 9373 /* 9374 * we're not allowed to set block groups readonly after the dirty 9375 * block groups cache has started writing. If it already started, 9376 * back off and let this transaction commit 9377 */ 9378 mutex_lock(&fs_info->ro_block_group_mutex); 9379 if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) { 9380 u64 transid = trans->transid; 9381 9382 mutex_unlock(&fs_info->ro_block_group_mutex); 9383 btrfs_end_transaction(trans); 9384 9385 ret = btrfs_wait_for_commit(fs_info, transid); 9386 if (ret) 9387 return ret; 9388 goto again; 9389 } 9390 9391 /* 9392 * if we are changing raid levels, try to allocate a corresponding 9393 * block group with the new raid level. 9394 */ 9395 alloc_flags = update_block_group_flags(fs_info, cache->flags); 9396 if (alloc_flags != cache->flags) { 9397 ret = do_chunk_alloc(trans, fs_info, alloc_flags, 9398 CHUNK_ALLOC_FORCE); 9399 /* 9400 * ENOSPC is allowed here, we may have enough space 9401 * already allocated at the new raid level to 9402 * carry on 9403 */ 9404 if (ret == -ENOSPC) 9405 ret = 0; 9406 if (ret < 0) 9407 goto out; 9408 } 9409 9410 ret = inc_block_group_ro(cache, 0); 9411 if (!ret) 9412 goto out; 9413 alloc_flags = get_alloc_profile(fs_info, cache->space_info->flags); 9414 ret = do_chunk_alloc(trans, fs_info, alloc_flags, 9415 CHUNK_ALLOC_FORCE); 9416 if (ret < 0) 9417 goto out; 9418 ret = inc_block_group_ro(cache, 0); 9419 out: 9420 if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) { 9421 alloc_flags = update_block_group_flags(fs_info, cache->flags); 9422 mutex_lock(&fs_info->chunk_mutex); 9423 check_system_chunk(trans, fs_info, alloc_flags); 9424 mutex_unlock(&fs_info->chunk_mutex); 9425 } 9426 mutex_unlock(&fs_info->ro_block_group_mutex); 9427 9428 btrfs_end_transaction(trans); 9429 return ret; 9430 } 9431 9432 int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, 9433 struct btrfs_fs_info *fs_info, u64 type) 9434 { 9435 u64 alloc_flags = get_alloc_profile(fs_info, type); 9436 9437 return do_chunk_alloc(trans, fs_info, alloc_flags, CHUNK_ALLOC_FORCE); 9438 } 9439 9440 /* 9441 * helper to account the unused space of all the readonly block group in the 9442 * space_info. takes mirrors into account. 9443 */ 9444 u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo) 9445 { 9446 struct btrfs_block_group_cache *block_group; 9447 u64 free_bytes = 0; 9448 int factor; 9449 9450 /* It's df, we don't care if it's racy */ 9451 if (list_empty(&sinfo->ro_bgs)) 9452 return 0; 9453 9454 spin_lock(&sinfo->lock); 9455 list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) { 9456 spin_lock(&block_group->lock); 9457 9458 if (!block_group->ro) { 9459 spin_unlock(&block_group->lock); 9460 continue; 9461 } 9462 9463 if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 | 9464 BTRFS_BLOCK_GROUP_RAID10 | 9465 BTRFS_BLOCK_GROUP_DUP)) 9466 factor = 2; 9467 else 9468 factor = 1; 9469 9470 free_bytes += (block_group->key.offset - 9471 btrfs_block_group_used(&block_group->item)) * 9472 factor; 9473 9474 spin_unlock(&block_group->lock); 9475 } 9476 spin_unlock(&sinfo->lock); 9477 9478 return free_bytes; 9479 } 9480 9481 void btrfs_dec_block_group_ro(struct btrfs_block_group_cache *cache) 9482 { 9483 struct btrfs_space_info *sinfo = cache->space_info; 9484 u64 num_bytes; 9485 9486 BUG_ON(!cache->ro); 9487 9488 spin_lock(&sinfo->lock); 9489 spin_lock(&cache->lock); 9490 if (!--cache->ro) { 9491 num_bytes = cache->key.offset - cache->reserved - 9492 cache->pinned - cache->bytes_super - 9493 btrfs_block_group_used(&cache->item); 9494 sinfo->bytes_readonly -= num_bytes; 9495 list_del_init(&cache->ro_list); 9496 } 9497 spin_unlock(&cache->lock); 9498 spin_unlock(&sinfo->lock); 9499 } 9500 9501 /* 9502 * checks to see if its even possible to relocate this block group. 9503 * 9504 * @return - -1 if it's not a good idea to relocate this block group, 0 if its 9505 * ok to go ahead and try. 9506 */ 9507 int btrfs_can_relocate(struct btrfs_fs_info *fs_info, u64 bytenr) 9508 { 9509 struct btrfs_root *root = fs_info->extent_root; 9510 struct btrfs_block_group_cache *block_group; 9511 struct btrfs_space_info *space_info; 9512 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 9513 struct btrfs_device *device; 9514 struct btrfs_trans_handle *trans; 9515 u64 min_free; 9516 u64 dev_min = 1; 9517 u64 dev_nr = 0; 9518 u64 target; 9519 int debug; 9520 int index; 9521 int full = 0; 9522 int ret = 0; 9523 9524 debug = btrfs_test_opt(fs_info, ENOSPC_DEBUG); 9525 9526 block_group = btrfs_lookup_block_group(fs_info, bytenr); 9527 9528 /* odd, couldn't find the block group, leave it alone */ 9529 if (!block_group) { 9530 if (debug) 9531 btrfs_warn(fs_info, 9532 "can't find block group for bytenr %llu", 9533 bytenr); 9534 return -1; 9535 } 9536 9537 min_free = btrfs_block_group_used(&block_group->item); 9538 9539 /* no bytes used, we're good */ 9540 if (!min_free) 9541 goto out; 9542 9543 space_info = block_group->space_info; 9544 spin_lock(&space_info->lock); 9545 9546 full = space_info->full; 9547 9548 /* 9549 * if this is the last block group we have in this space, we can't 9550 * relocate it unless we're able to allocate a new chunk below. 9551 * 9552 * Otherwise, we need to make sure we have room in the space to handle 9553 * all of the extents from this block group. If we can, we're good 9554 */ 9555 if ((space_info->total_bytes != block_group->key.offset) && 9556 (space_info->bytes_used + space_info->bytes_reserved + 9557 space_info->bytes_pinned + space_info->bytes_readonly + 9558 min_free < space_info->total_bytes)) { 9559 spin_unlock(&space_info->lock); 9560 goto out; 9561 } 9562 spin_unlock(&space_info->lock); 9563 9564 /* 9565 * ok we don't have enough space, but maybe we have free space on our 9566 * devices to allocate new chunks for relocation, so loop through our 9567 * alloc devices and guess if we have enough space. if this block 9568 * group is going to be restriped, run checks against the target 9569 * profile instead of the current one. 9570 */ 9571 ret = -1; 9572 9573 /* 9574 * index: 9575 * 0: raid10 9576 * 1: raid1 9577 * 2: dup 9578 * 3: raid0 9579 * 4: single 9580 */ 9581 target = get_restripe_target(fs_info, block_group->flags); 9582 if (target) { 9583 index = __get_raid_index(extended_to_chunk(target)); 9584 } else { 9585 /* 9586 * this is just a balance, so if we were marked as full 9587 * we know there is no space for a new chunk 9588 */ 9589 if (full) { 9590 if (debug) 9591 btrfs_warn(fs_info, 9592 "no space to alloc new chunk for block group %llu", 9593 block_group->key.objectid); 9594 goto out; 9595 } 9596 9597 index = get_block_group_index(block_group); 9598 } 9599 9600 if (index == BTRFS_RAID_RAID10) { 9601 dev_min = 4; 9602 /* Divide by 2 */ 9603 min_free >>= 1; 9604 } else if (index == BTRFS_RAID_RAID1) { 9605 dev_min = 2; 9606 } else if (index == BTRFS_RAID_DUP) { 9607 /* Multiply by 2 */ 9608 min_free <<= 1; 9609 } else if (index == BTRFS_RAID_RAID0) { 9610 dev_min = fs_devices->rw_devices; 9611 min_free = div64_u64(min_free, dev_min); 9612 } 9613 9614 /* We need to do this so that we can look at pending chunks */ 9615 trans = btrfs_join_transaction(root); 9616 if (IS_ERR(trans)) { 9617 ret = PTR_ERR(trans); 9618 goto out; 9619 } 9620 9621 mutex_lock(&fs_info->chunk_mutex); 9622 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { 9623 u64 dev_offset; 9624 9625 /* 9626 * check to make sure we can actually find a chunk with enough 9627 * space to fit our block group in. 9628 */ 9629 if (device->total_bytes > device->bytes_used + min_free && 9630 !device->is_tgtdev_for_dev_replace) { 9631 ret = find_free_dev_extent(trans, device, min_free, 9632 &dev_offset, NULL); 9633 if (!ret) 9634 dev_nr++; 9635 9636 if (dev_nr >= dev_min) 9637 break; 9638 9639 ret = -1; 9640 } 9641 } 9642 if (debug && ret == -1) 9643 btrfs_warn(fs_info, 9644 "no space to allocate a new chunk for block group %llu", 9645 block_group->key.objectid); 9646 mutex_unlock(&fs_info->chunk_mutex); 9647 btrfs_end_transaction(trans); 9648 out: 9649 btrfs_put_block_group(block_group); 9650 return ret; 9651 } 9652 9653 static int find_first_block_group(struct btrfs_fs_info *fs_info, 9654 struct btrfs_path *path, 9655 struct btrfs_key *key) 9656 { 9657 struct btrfs_root *root = fs_info->extent_root; 9658 int ret = 0; 9659 struct btrfs_key found_key; 9660 struct extent_buffer *leaf; 9661 int slot; 9662 9663 ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 9664 if (ret < 0) 9665 goto out; 9666 9667 while (1) { 9668 slot = path->slots[0]; 9669 leaf = path->nodes[0]; 9670 if (slot >= btrfs_header_nritems(leaf)) { 9671 ret = btrfs_next_leaf(root, path); 9672 if (ret == 0) 9673 continue; 9674 if (ret < 0) 9675 goto out; 9676 break; 9677 } 9678 btrfs_item_key_to_cpu(leaf, &found_key, slot); 9679 9680 if (found_key.objectid >= key->objectid && 9681 found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) { 9682 struct extent_map_tree *em_tree; 9683 struct extent_map *em; 9684 9685 em_tree = &root->fs_info->mapping_tree.map_tree; 9686 read_lock(&em_tree->lock); 9687 em = lookup_extent_mapping(em_tree, found_key.objectid, 9688 found_key.offset); 9689 read_unlock(&em_tree->lock); 9690 if (!em) { 9691 btrfs_err(fs_info, 9692 "logical %llu len %llu found bg but no related chunk", 9693 found_key.objectid, found_key.offset); 9694 ret = -ENOENT; 9695 } else { 9696 ret = 0; 9697 } 9698 free_extent_map(em); 9699 goto out; 9700 } 9701 path->slots[0]++; 9702 } 9703 out: 9704 return ret; 9705 } 9706 9707 void btrfs_put_block_group_cache(struct btrfs_fs_info *info) 9708 { 9709 struct btrfs_block_group_cache *block_group; 9710 u64 last = 0; 9711 9712 while (1) { 9713 struct inode *inode; 9714 9715 block_group = btrfs_lookup_first_block_group(info, last); 9716 while (block_group) { 9717 spin_lock(&block_group->lock); 9718 if (block_group->iref) 9719 break; 9720 spin_unlock(&block_group->lock); 9721 block_group = next_block_group(info, block_group); 9722 } 9723 if (!block_group) { 9724 if (last == 0) 9725 break; 9726 last = 0; 9727 continue; 9728 } 9729 9730 inode = block_group->inode; 9731 block_group->iref = 0; 9732 block_group->inode = NULL; 9733 spin_unlock(&block_group->lock); 9734 ASSERT(block_group->io_ctl.inode == NULL); 9735 iput(inode); 9736 last = block_group->key.objectid + block_group->key.offset; 9737 btrfs_put_block_group(block_group); 9738 } 9739 } 9740 9741 int btrfs_free_block_groups(struct btrfs_fs_info *info) 9742 { 9743 struct btrfs_block_group_cache *block_group; 9744 struct btrfs_space_info *space_info; 9745 struct btrfs_caching_control *caching_ctl; 9746 struct rb_node *n; 9747 9748 down_write(&info->commit_root_sem); 9749 while (!list_empty(&info->caching_block_groups)) { 9750 caching_ctl = list_entry(info->caching_block_groups.next, 9751 struct btrfs_caching_control, list); 9752 list_del(&caching_ctl->list); 9753 put_caching_control(caching_ctl); 9754 } 9755 up_write(&info->commit_root_sem); 9756 9757 spin_lock(&info->unused_bgs_lock); 9758 while (!list_empty(&info->unused_bgs)) { 9759 block_group = list_first_entry(&info->unused_bgs, 9760 struct btrfs_block_group_cache, 9761 bg_list); 9762 list_del_init(&block_group->bg_list); 9763 btrfs_put_block_group(block_group); 9764 } 9765 spin_unlock(&info->unused_bgs_lock); 9766 9767 spin_lock(&info->block_group_cache_lock); 9768 while ((n = rb_last(&info->block_group_cache_tree)) != NULL) { 9769 block_group = rb_entry(n, struct btrfs_block_group_cache, 9770 cache_node); 9771 rb_erase(&block_group->cache_node, 9772 &info->block_group_cache_tree); 9773 RB_CLEAR_NODE(&block_group->cache_node); 9774 spin_unlock(&info->block_group_cache_lock); 9775 9776 down_write(&block_group->space_info->groups_sem); 9777 list_del(&block_group->list); 9778 up_write(&block_group->space_info->groups_sem); 9779 9780 if (block_group->cached == BTRFS_CACHE_STARTED) 9781 wait_block_group_cache_done(block_group); 9782 9783 /* 9784 * We haven't cached this block group, which means we could 9785 * possibly have excluded extents on this block group. 9786 */ 9787 if (block_group->cached == BTRFS_CACHE_NO || 9788 block_group->cached == BTRFS_CACHE_ERROR) 9789 free_excluded_extents(info, block_group); 9790 9791 btrfs_remove_free_space_cache(block_group); 9792 ASSERT(list_empty(&block_group->dirty_list)); 9793 ASSERT(list_empty(&block_group->io_list)); 9794 ASSERT(list_empty(&block_group->bg_list)); 9795 ASSERT(atomic_read(&block_group->count) == 1); 9796 btrfs_put_block_group(block_group); 9797 9798 spin_lock(&info->block_group_cache_lock); 9799 } 9800 spin_unlock(&info->block_group_cache_lock); 9801 9802 /* now that all the block groups are freed, go through and 9803 * free all the space_info structs. This is only called during 9804 * the final stages of unmount, and so we know nobody is 9805 * using them. We call synchronize_rcu() once before we start, 9806 * just to be on the safe side. 9807 */ 9808 synchronize_rcu(); 9809 9810 release_global_block_rsv(info); 9811 9812 while (!list_empty(&info->space_info)) { 9813 int i; 9814 9815 space_info = list_entry(info->space_info.next, 9816 struct btrfs_space_info, 9817 list); 9818 9819 /* 9820 * Do not hide this behind enospc_debug, this is actually 9821 * important and indicates a real bug if this happens. 9822 */ 9823 if (WARN_ON(space_info->bytes_pinned > 0 || 9824 space_info->bytes_reserved > 0 || 9825 space_info->bytes_may_use > 0)) 9826 dump_space_info(info, space_info, 0, 0); 9827 list_del(&space_info->list); 9828 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) { 9829 struct kobject *kobj; 9830 kobj = space_info->block_group_kobjs[i]; 9831 space_info->block_group_kobjs[i] = NULL; 9832 if (kobj) { 9833 kobject_del(kobj); 9834 kobject_put(kobj); 9835 } 9836 } 9837 kobject_del(&space_info->kobj); 9838 kobject_put(&space_info->kobj); 9839 } 9840 return 0; 9841 } 9842 9843 static void __link_block_group(struct btrfs_space_info *space_info, 9844 struct btrfs_block_group_cache *cache) 9845 { 9846 int index = get_block_group_index(cache); 9847 bool first = false; 9848 9849 down_write(&space_info->groups_sem); 9850 if (list_empty(&space_info->block_groups[index])) 9851 first = true; 9852 list_add_tail(&cache->list, &space_info->block_groups[index]); 9853 up_write(&space_info->groups_sem); 9854 9855 if (first) { 9856 struct raid_kobject *rkobj; 9857 int ret; 9858 9859 rkobj = kzalloc(sizeof(*rkobj), GFP_NOFS); 9860 if (!rkobj) 9861 goto out_err; 9862 rkobj->raid_type = index; 9863 kobject_init(&rkobj->kobj, &btrfs_raid_ktype); 9864 ret = kobject_add(&rkobj->kobj, &space_info->kobj, 9865 "%s", get_raid_name(index)); 9866 if (ret) { 9867 kobject_put(&rkobj->kobj); 9868 goto out_err; 9869 } 9870 space_info->block_group_kobjs[index] = &rkobj->kobj; 9871 } 9872 9873 return; 9874 out_err: 9875 btrfs_warn(cache->fs_info, 9876 "failed to add kobject for block cache, ignoring"); 9877 } 9878 9879 static struct btrfs_block_group_cache * 9880 btrfs_create_block_group_cache(struct btrfs_fs_info *fs_info, 9881 u64 start, u64 size) 9882 { 9883 struct btrfs_block_group_cache *cache; 9884 9885 cache = kzalloc(sizeof(*cache), GFP_NOFS); 9886 if (!cache) 9887 return NULL; 9888 9889 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl), 9890 GFP_NOFS); 9891 if (!cache->free_space_ctl) { 9892 kfree(cache); 9893 return NULL; 9894 } 9895 9896 cache->key.objectid = start; 9897 cache->key.offset = size; 9898 cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 9899 9900 cache->sectorsize = fs_info->sectorsize; 9901 cache->fs_info = fs_info; 9902 cache->full_stripe_len = btrfs_full_stripe_len(fs_info, 9903 &fs_info->mapping_tree, 9904 start); 9905 set_free_space_tree_thresholds(cache); 9906 9907 atomic_set(&cache->count, 1); 9908 spin_lock_init(&cache->lock); 9909 init_rwsem(&cache->data_rwsem); 9910 INIT_LIST_HEAD(&cache->list); 9911 INIT_LIST_HEAD(&cache->cluster_list); 9912 INIT_LIST_HEAD(&cache->bg_list); 9913 INIT_LIST_HEAD(&cache->ro_list); 9914 INIT_LIST_HEAD(&cache->dirty_list); 9915 INIT_LIST_HEAD(&cache->io_list); 9916 btrfs_init_free_space_ctl(cache); 9917 atomic_set(&cache->trimming, 0); 9918 mutex_init(&cache->free_space_lock); 9919 9920 return cache; 9921 } 9922 9923 int btrfs_read_block_groups(struct btrfs_fs_info *info) 9924 { 9925 struct btrfs_path *path; 9926 int ret; 9927 struct btrfs_block_group_cache *cache; 9928 struct btrfs_space_info *space_info; 9929 struct btrfs_key key; 9930 struct btrfs_key found_key; 9931 struct extent_buffer *leaf; 9932 int need_clear = 0; 9933 u64 cache_gen; 9934 u64 feature; 9935 int mixed; 9936 9937 feature = btrfs_super_incompat_flags(info->super_copy); 9938 mixed = !!(feature & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS); 9939 9940 key.objectid = 0; 9941 key.offset = 0; 9942 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 9943 path = btrfs_alloc_path(); 9944 if (!path) 9945 return -ENOMEM; 9946 path->reada = READA_FORWARD; 9947 9948 cache_gen = btrfs_super_cache_generation(info->super_copy); 9949 if (btrfs_test_opt(info, SPACE_CACHE) && 9950 btrfs_super_generation(info->super_copy) != cache_gen) 9951 need_clear = 1; 9952 if (btrfs_test_opt(info, CLEAR_CACHE)) 9953 need_clear = 1; 9954 9955 while (1) { 9956 ret = find_first_block_group(info, path, &key); 9957 if (ret > 0) 9958 break; 9959 if (ret != 0) 9960 goto error; 9961 9962 leaf = path->nodes[0]; 9963 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 9964 9965 cache = btrfs_create_block_group_cache(info, found_key.objectid, 9966 found_key.offset); 9967 if (!cache) { 9968 ret = -ENOMEM; 9969 goto error; 9970 } 9971 9972 if (need_clear) { 9973 /* 9974 * When we mount with old space cache, we need to 9975 * set BTRFS_DC_CLEAR and set dirty flag. 9976 * 9977 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we 9978 * truncate the old free space cache inode and 9979 * setup a new one. 9980 * b) Setting 'dirty flag' makes sure that we flush 9981 * the new space cache info onto disk. 9982 */ 9983 if (btrfs_test_opt(info, SPACE_CACHE)) 9984 cache->disk_cache_state = BTRFS_DC_CLEAR; 9985 } 9986 9987 read_extent_buffer(leaf, &cache->item, 9988 btrfs_item_ptr_offset(leaf, path->slots[0]), 9989 sizeof(cache->item)); 9990 cache->flags = btrfs_block_group_flags(&cache->item); 9991 if (!mixed && 9992 ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) && 9993 (cache->flags & BTRFS_BLOCK_GROUP_DATA))) { 9994 btrfs_err(info, 9995 "bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups", 9996 cache->key.objectid); 9997 ret = -EINVAL; 9998 goto error; 9999 } 10000 10001 key.objectid = found_key.objectid + found_key.offset; 10002 btrfs_release_path(path); 10003 10004 /* 10005 * We need to exclude the super stripes now so that the space 10006 * info has super bytes accounted for, otherwise we'll think 10007 * we have more space than we actually do. 10008 */ 10009 ret = exclude_super_stripes(info, cache); 10010 if (ret) { 10011 /* 10012 * We may have excluded something, so call this just in 10013 * case. 10014 */ 10015 free_excluded_extents(info, cache); 10016 btrfs_put_block_group(cache); 10017 goto error; 10018 } 10019 10020 /* 10021 * check for two cases, either we are full, and therefore 10022 * don't need to bother with the caching work since we won't 10023 * find any space, or we are empty, and we can just add all 10024 * the space in and be done with it. This saves us _alot_ of 10025 * time, particularly in the full case. 10026 */ 10027 if (found_key.offset == btrfs_block_group_used(&cache->item)) { 10028 cache->last_byte_to_unpin = (u64)-1; 10029 cache->cached = BTRFS_CACHE_FINISHED; 10030 free_excluded_extents(info, cache); 10031 } else if (btrfs_block_group_used(&cache->item) == 0) { 10032 cache->last_byte_to_unpin = (u64)-1; 10033 cache->cached = BTRFS_CACHE_FINISHED; 10034 add_new_free_space(cache, info, 10035 found_key.objectid, 10036 found_key.objectid + 10037 found_key.offset); 10038 free_excluded_extents(info, cache); 10039 } 10040 10041 ret = btrfs_add_block_group_cache(info, cache); 10042 if (ret) { 10043 btrfs_remove_free_space_cache(cache); 10044 btrfs_put_block_group(cache); 10045 goto error; 10046 } 10047 10048 trace_btrfs_add_block_group(info, cache, 0); 10049 ret = update_space_info(info, cache->flags, found_key.offset, 10050 btrfs_block_group_used(&cache->item), 10051 cache->bytes_super, &space_info); 10052 if (ret) { 10053 btrfs_remove_free_space_cache(cache); 10054 spin_lock(&info->block_group_cache_lock); 10055 rb_erase(&cache->cache_node, 10056 &info->block_group_cache_tree); 10057 RB_CLEAR_NODE(&cache->cache_node); 10058 spin_unlock(&info->block_group_cache_lock); 10059 btrfs_put_block_group(cache); 10060 goto error; 10061 } 10062 10063 cache->space_info = space_info; 10064 10065 __link_block_group(space_info, cache); 10066 10067 set_avail_alloc_bits(info, cache->flags); 10068 if (btrfs_chunk_readonly(info, cache->key.objectid)) { 10069 inc_block_group_ro(cache, 1); 10070 } else if (btrfs_block_group_used(&cache->item) == 0) { 10071 spin_lock(&info->unused_bgs_lock); 10072 /* Should always be true but just in case. */ 10073 if (list_empty(&cache->bg_list)) { 10074 btrfs_get_block_group(cache); 10075 list_add_tail(&cache->bg_list, 10076 &info->unused_bgs); 10077 } 10078 spin_unlock(&info->unused_bgs_lock); 10079 } 10080 } 10081 10082 list_for_each_entry_rcu(space_info, &info->space_info, list) { 10083 if (!(get_alloc_profile(info, space_info->flags) & 10084 (BTRFS_BLOCK_GROUP_RAID10 | 10085 BTRFS_BLOCK_GROUP_RAID1 | 10086 BTRFS_BLOCK_GROUP_RAID5 | 10087 BTRFS_BLOCK_GROUP_RAID6 | 10088 BTRFS_BLOCK_GROUP_DUP))) 10089 continue; 10090 /* 10091 * avoid allocating from un-mirrored block group if there are 10092 * mirrored block groups. 10093 */ 10094 list_for_each_entry(cache, 10095 &space_info->block_groups[BTRFS_RAID_RAID0], 10096 list) 10097 inc_block_group_ro(cache, 1); 10098 list_for_each_entry(cache, 10099 &space_info->block_groups[BTRFS_RAID_SINGLE], 10100 list) 10101 inc_block_group_ro(cache, 1); 10102 } 10103 10104 init_global_block_rsv(info); 10105 ret = 0; 10106 error: 10107 btrfs_free_path(path); 10108 return ret; 10109 } 10110 10111 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans, 10112 struct btrfs_fs_info *fs_info) 10113 { 10114 struct btrfs_block_group_cache *block_group, *tmp; 10115 struct btrfs_root *extent_root = fs_info->extent_root; 10116 struct btrfs_block_group_item item; 10117 struct btrfs_key key; 10118 int ret = 0; 10119 bool can_flush_pending_bgs = trans->can_flush_pending_bgs; 10120 10121 trans->can_flush_pending_bgs = false; 10122 list_for_each_entry_safe(block_group, tmp, &trans->new_bgs, bg_list) { 10123 if (ret) 10124 goto next; 10125 10126 spin_lock(&block_group->lock); 10127 memcpy(&item, &block_group->item, sizeof(item)); 10128 memcpy(&key, &block_group->key, sizeof(key)); 10129 spin_unlock(&block_group->lock); 10130 10131 ret = btrfs_insert_item(trans, extent_root, &key, &item, 10132 sizeof(item)); 10133 if (ret) 10134 btrfs_abort_transaction(trans, ret); 10135 ret = btrfs_finish_chunk_alloc(trans, fs_info, key.objectid, 10136 key.offset); 10137 if (ret) 10138 btrfs_abort_transaction(trans, ret); 10139 add_block_group_free_space(trans, fs_info, block_group); 10140 /* already aborted the transaction if it failed. */ 10141 next: 10142 list_del_init(&block_group->bg_list); 10143 } 10144 trans->can_flush_pending_bgs = can_flush_pending_bgs; 10145 } 10146 10147 int btrfs_make_block_group(struct btrfs_trans_handle *trans, 10148 struct btrfs_fs_info *fs_info, u64 bytes_used, 10149 u64 type, u64 chunk_objectid, u64 chunk_offset, 10150 u64 size) 10151 { 10152 struct btrfs_block_group_cache *cache; 10153 int ret; 10154 10155 btrfs_set_log_full_commit(fs_info, trans); 10156 10157 cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size); 10158 if (!cache) 10159 return -ENOMEM; 10160 10161 btrfs_set_block_group_used(&cache->item, bytes_used); 10162 btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid); 10163 btrfs_set_block_group_flags(&cache->item, type); 10164 10165 cache->flags = type; 10166 cache->last_byte_to_unpin = (u64)-1; 10167 cache->cached = BTRFS_CACHE_FINISHED; 10168 cache->needs_free_space = 1; 10169 ret = exclude_super_stripes(fs_info, cache); 10170 if (ret) { 10171 /* 10172 * We may have excluded something, so call this just in 10173 * case. 10174 */ 10175 free_excluded_extents(fs_info, cache); 10176 btrfs_put_block_group(cache); 10177 return ret; 10178 } 10179 10180 add_new_free_space(cache, fs_info, chunk_offset, chunk_offset + size); 10181 10182 free_excluded_extents(fs_info, cache); 10183 10184 #ifdef CONFIG_BTRFS_DEBUG 10185 if (btrfs_should_fragment_free_space(cache)) { 10186 u64 new_bytes_used = size - bytes_used; 10187 10188 bytes_used += new_bytes_used >> 1; 10189 fragment_free_space(cache); 10190 } 10191 #endif 10192 /* 10193 * Call to ensure the corresponding space_info object is created and 10194 * assigned to our block group, but don't update its counters just yet. 10195 * We want our bg to be added to the rbtree with its ->space_info set. 10196 */ 10197 ret = update_space_info(fs_info, cache->flags, 0, 0, 0, 10198 &cache->space_info); 10199 if (ret) { 10200 btrfs_remove_free_space_cache(cache); 10201 btrfs_put_block_group(cache); 10202 return ret; 10203 } 10204 10205 ret = btrfs_add_block_group_cache(fs_info, cache); 10206 if (ret) { 10207 btrfs_remove_free_space_cache(cache); 10208 btrfs_put_block_group(cache); 10209 return ret; 10210 } 10211 10212 /* 10213 * Now that our block group has its ->space_info set and is inserted in 10214 * the rbtree, update the space info's counters. 10215 */ 10216 trace_btrfs_add_block_group(fs_info, cache, 1); 10217 ret = update_space_info(fs_info, cache->flags, size, bytes_used, 10218 cache->bytes_super, &cache->space_info); 10219 if (ret) { 10220 btrfs_remove_free_space_cache(cache); 10221 spin_lock(&fs_info->block_group_cache_lock); 10222 rb_erase(&cache->cache_node, 10223 &fs_info->block_group_cache_tree); 10224 RB_CLEAR_NODE(&cache->cache_node); 10225 spin_unlock(&fs_info->block_group_cache_lock); 10226 btrfs_put_block_group(cache); 10227 return ret; 10228 } 10229 update_global_block_rsv(fs_info); 10230 10231 __link_block_group(cache->space_info, cache); 10232 10233 list_add_tail(&cache->bg_list, &trans->new_bgs); 10234 10235 set_avail_alloc_bits(fs_info, type); 10236 return 0; 10237 } 10238 10239 static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) 10240 { 10241 u64 extra_flags = chunk_to_extended(flags) & 10242 BTRFS_EXTENDED_PROFILE_MASK; 10243 10244 write_seqlock(&fs_info->profiles_lock); 10245 if (flags & BTRFS_BLOCK_GROUP_DATA) 10246 fs_info->avail_data_alloc_bits &= ~extra_flags; 10247 if (flags & BTRFS_BLOCK_GROUP_METADATA) 10248 fs_info->avail_metadata_alloc_bits &= ~extra_flags; 10249 if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 10250 fs_info->avail_system_alloc_bits &= ~extra_flags; 10251 write_sequnlock(&fs_info->profiles_lock); 10252 } 10253 10254 int btrfs_remove_block_group(struct btrfs_trans_handle *trans, 10255 struct btrfs_fs_info *fs_info, u64 group_start, 10256 struct extent_map *em) 10257 { 10258 struct btrfs_root *root = fs_info->extent_root; 10259 struct btrfs_path *path; 10260 struct btrfs_block_group_cache *block_group; 10261 struct btrfs_free_cluster *cluster; 10262 struct btrfs_root *tree_root = fs_info->tree_root; 10263 struct btrfs_key key; 10264 struct inode *inode; 10265 struct kobject *kobj = NULL; 10266 int ret; 10267 int index; 10268 int factor; 10269 struct btrfs_caching_control *caching_ctl = NULL; 10270 bool remove_em; 10271 10272 block_group = btrfs_lookup_block_group(fs_info, group_start); 10273 BUG_ON(!block_group); 10274 BUG_ON(!block_group->ro); 10275 10276 /* 10277 * Free the reserved super bytes from this block group before 10278 * remove it. 10279 */ 10280 free_excluded_extents(fs_info, block_group); 10281 10282 memcpy(&key, &block_group->key, sizeof(key)); 10283 index = get_block_group_index(block_group); 10284 if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP | 10285 BTRFS_BLOCK_GROUP_RAID1 | 10286 BTRFS_BLOCK_GROUP_RAID10)) 10287 factor = 2; 10288 else 10289 factor = 1; 10290 10291 /* make sure this block group isn't part of an allocation cluster */ 10292 cluster = &fs_info->data_alloc_cluster; 10293 spin_lock(&cluster->refill_lock); 10294 btrfs_return_cluster_to_free_space(block_group, cluster); 10295 spin_unlock(&cluster->refill_lock); 10296 10297 /* 10298 * make sure this block group isn't part of a metadata 10299 * allocation cluster 10300 */ 10301 cluster = &fs_info->meta_alloc_cluster; 10302 spin_lock(&cluster->refill_lock); 10303 btrfs_return_cluster_to_free_space(block_group, cluster); 10304 spin_unlock(&cluster->refill_lock); 10305 10306 path = btrfs_alloc_path(); 10307 if (!path) { 10308 ret = -ENOMEM; 10309 goto out; 10310 } 10311 10312 /* 10313 * get the inode first so any iput calls done for the io_list 10314 * aren't the final iput (no unlinks allowed now) 10315 */ 10316 inode = lookup_free_space_inode(tree_root, block_group, path); 10317 10318 mutex_lock(&trans->transaction->cache_write_mutex); 10319 /* 10320 * make sure our free spache cache IO is done before remove the 10321 * free space inode 10322 */ 10323 spin_lock(&trans->transaction->dirty_bgs_lock); 10324 if (!list_empty(&block_group->io_list)) { 10325 list_del_init(&block_group->io_list); 10326 10327 WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode); 10328 10329 spin_unlock(&trans->transaction->dirty_bgs_lock); 10330 btrfs_wait_cache_io(trans, block_group, path); 10331 btrfs_put_block_group(block_group); 10332 spin_lock(&trans->transaction->dirty_bgs_lock); 10333 } 10334 10335 if (!list_empty(&block_group->dirty_list)) { 10336 list_del_init(&block_group->dirty_list); 10337 btrfs_put_block_group(block_group); 10338 } 10339 spin_unlock(&trans->transaction->dirty_bgs_lock); 10340 mutex_unlock(&trans->transaction->cache_write_mutex); 10341 10342 if (!IS_ERR(inode)) { 10343 ret = btrfs_orphan_add(trans, inode); 10344 if (ret) { 10345 btrfs_add_delayed_iput(inode); 10346 goto out; 10347 } 10348 clear_nlink(inode); 10349 /* One for the block groups ref */ 10350 spin_lock(&block_group->lock); 10351 if (block_group->iref) { 10352 block_group->iref = 0; 10353 block_group->inode = NULL; 10354 spin_unlock(&block_group->lock); 10355 iput(inode); 10356 } else { 10357 spin_unlock(&block_group->lock); 10358 } 10359 /* One for our lookup ref */ 10360 btrfs_add_delayed_iput(inode); 10361 } 10362 10363 key.objectid = BTRFS_FREE_SPACE_OBJECTID; 10364 key.offset = block_group->key.objectid; 10365 key.type = 0; 10366 10367 ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1); 10368 if (ret < 0) 10369 goto out; 10370 if (ret > 0) 10371 btrfs_release_path(path); 10372 if (ret == 0) { 10373 ret = btrfs_del_item(trans, tree_root, path); 10374 if (ret) 10375 goto out; 10376 btrfs_release_path(path); 10377 } 10378 10379 spin_lock(&fs_info->block_group_cache_lock); 10380 rb_erase(&block_group->cache_node, 10381 &fs_info->block_group_cache_tree); 10382 RB_CLEAR_NODE(&block_group->cache_node); 10383 10384 if (fs_info->first_logical_byte == block_group->key.objectid) 10385 fs_info->first_logical_byte = (u64)-1; 10386 spin_unlock(&fs_info->block_group_cache_lock); 10387 10388 down_write(&block_group->space_info->groups_sem); 10389 /* 10390 * we must use list_del_init so people can check to see if they 10391 * are still on the list after taking the semaphore 10392 */ 10393 list_del_init(&block_group->list); 10394 if (list_empty(&block_group->space_info->block_groups[index])) { 10395 kobj = block_group->space_info->block_group_kobjs[index]; 10396 block_group->space_info->block_group_kobjs[index] = NULL; 10397 clear_avail_alloc_bits(fs_info, block_group->flags); 10398 } 10399 up_write(&block_group->space_info->groups_sem); 10400 if (kobj) { 10401 kobject_del(kobj); 10402 kobject_put(kobj); 10403 } 10404 10405 if (block_group->has_caching_ctl) 10406 caching_ctl = get_caching_control(block_group); 10407 if (block_group->cached == BTRFS_CACHE_STARTED) 10408 wait_block_group_cache_done(block_group); 10409 if (block_group->has_caching_ctl) { 10410 down_write(&fs_info->commit_root_sem); 10411 if (!caching_ctl) { 10412 struct btrfs_caching_control *ctl; 10413 10414 list_for_each_entry(ctl, 10415 &fs_info->caching_block_groups, list) 10416 if (ctl->block_group == block_group) { 10417 caching_ctl = ctl; 10418 atomic_inc(&caching_ctl->count); 10419 break; 10420 } 10421 } 10422 if (caching_ctl) 10423 list_del_init(&caching_ctl->list); 10424 up_write(&fs_info->commit_root_sem); 10425 if (caching_ctl) { 10426 /* Once for the caching bgs list and once for us. */ 10427 put_caching_control(caching_ctl); 10428 put_caching_control(caching_ctl); 10429 } 10430 } 10431 10432 spin_lock(&trans->transaction->dirty_bgs_lock); 10433 if (!list_empty(&block_group->dirty_list)) { 10434 WARN_ON(1); 10435 } 10436 if (!list_empty(&block_group->io_list)) { 10437 WARN_ON(1); 10438 } 10439 spin_unlock(&trans->transaction->dirty_bgs_lock); 10440 btrfs_remove_free_space_cache(block_group); 10441 10442 spin_lock(&block_group->space_info->lock); 10443 list_del_init(&block_group->ro_list); 10444 10445 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { 10446 WARN_ON(block_group->space_info->total_bytes 10447 < block_group->key.offset); 10448 WARN_ON(block_group->space_info->bytes_readonly 10449 < block_group->key.offset); 10450 WARN_ON(block_group->space_info->disk_total 10451 < block_group->key.offset * factor); 10452 } 10453 block_group->space_info->total_bytes -= block_group->key.offset; 10454 block_group->space_info->bytes_readonly -= block_group->key.offset; 10455 block_group->space_info->disk_total -= block_group->key.offset * factor; 10456 10457 spin_unlock(&block_group->space_info->lock); 10458 10459 memcpy(&key, &block_group->key, sizeof(key)); 10460 10461 mutex_lock(&fs_info->chunk_mutex); 10462 if (!list_empty(&em->list)) { 10463 /* We're in the transaction->pending_chunks list. */ 10464 free_extent_map(em); 10465 } 10466 spin_lock(&block_group->lock); 10467 block_group->removed = 1; 10468 /* 10469 * At this point trimming can't start on this block group, because we 10470 * removed the block group from the tree fs_info->block_group_cache_tree 10471 * so no one can't find it anymore and even if someone already got this 10472 * block group before we removed it from the rbtree, they have already 10473 * incremented block_group->trimming - if they didn't, they won't find 10474 * any free space entries because we already removed them all when we 10475 * called btrfs_remove_free_space_cache(). 10476 * 10477 * And we must not remove the extent map from the fs_info->mapping_tree 10478 * to prevent the same logical address range and physical device space 10479 * ranges from being reused for a new block group. This is because our 10480 * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is 10481 * completely transactionless, so while it is trimming a range the 10482 * currently running transaction might finish and a new one start, 10483 * allowing for new block groups to be created that can reuse the same 10484 * physical device locations unless we take this special care. 10485 * 10486 * There may also be an implicit trim operation if the file system 10487 * is mounted with -odiscard. The same protections must remain 10488 * in place until the extents have been discarded completely when 10489 * the transaction commit has completed. 10490 */ 10491 remove_em = (atomic_read(&block_group->trimming) == 0); 10492 /* 10493 * Make sure a trimmer task always sees the em in the pinned_chunks list 10494 * if it sees block_group->removed == 1 (needs to lock block_group->lock 10495 * before checking block_group->removed). 10496 */ 10497 if (!remove_em) { 10498 /* 10499 * Our em might be in trans->transaction->pending_chunks which 10500 * is protected by fs_info->chunk_mutex ([lock|unlock]_chunks), 10501 * and so is the fs_info->pinned_chunks list. 10502 * 10503 * So at this point we must be holding the chunk_mutex to avoid 10504 * any races with chunk allocation (more specifically at 10505 * volumes.c:contains_pending_extent()), to ensure it always 10506 * sees the em, either in the pending_chunks list or in the 10507 * pinned_chunks list. 10508 */ 10509 list_move_tail(&em->list, &fs_info->pinned_chunks); 10510 } 10511 spin_unlock(&block_group->lock); 10512 10513 if (remove_em) { 10514 struct extent_map_tree *em_tree; 10515 10516 em_tree = &fs_info->mapping_tree.map_tree; 10517 write_lock(&em_tree->lock); 10518 /* 10519 * The em might be in the pending_chunks list, so make sure the 10520 * chunk mutex is locked, since remove_extent_mapping() will 10521 * delete us from that list. 10522 */ 10523 remove_extent_mapping(em_tree, em); 10524 write_unlock(&em_tree->lock); 10525 /* once for the tree */ 10526 free_extent_map(em); 10527 } 10528 10529 mutex_unlock(&fs_info->chunk_mutex); 10530 10531 ret = remove_block_group_free_space(trans, fs_info, block_group); 10532 if (ret) 10533 goto out; 10534 10535 btrfs_put_block_group(block_group); 10536 btrfs_put_block_group(block_group); 10537 10538 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 10539 if (ret > 0) 10540 ret = -EIO; 10541 if (ret < 0) 10542 goto out; 10543 10544 ret = btrfs_del_item(trans, root, path); 10545 out: 10546 btrfs_free_path(path); 10547 return ret; 10548 } 10549 10550 struct btrfs_trans_handle * 10551 btrfs_start_trans_remove_block_group(struct btrfs_fs_info *fs_info, 10552 const u64 chunk_offset) 10553 { 10554 struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree; 10555 struct extent_map *em; 10556 struct map_lookup *map; 10557 unsigned int num_items; 10558 10559 read_lock(&em_tree->lock); 10560 em = lookup_extent_mapping(em_tree, chunk_offset, 1); 10561 read_unlock(&em_tree->lock); 10562 ASSERT(em && em->start == chunk_offset); 10563 10564 /* 10565 * We need to reserve 3 + N units from the metadata space info in order 10566 * to remove a block group (done at btrfs_remove_chunk() and at 10567 * btrfs_remove_block_group()), which are used for: 10568 * 10569 * 1 unit for adding the free space inode's orphan (located in the tree 10570 * of tree roots). 10571 * 1 unit for deleting the block group item (located in the extent 10572 * tree). 10573 * 1 unit for deleting the free space item (located in tree of tree 10574 * roots). 10575 * N units for deleting N device extent items corresponding to each 10576 * stripe (located in the device tree). 10577 * 10578 * In order to remove a block group we also need to reserve units in the 10579 * system space info in order to update the chunk tree (update one or 10580 * more device items and remove one chunk item), but this is done at 10581 * btrfs_remove_chunk() through a call to check_system_chunk(). 10582 */ 10583 map = em->map_lookup; 10584 num_items = 3 + map->num_stripes; 10585 free_extent_map(em); 10586 10587 return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root, 10588 num_items, 1); 10589 } 10590 10591 /* 10592 * Process the unused_bgs list and remove any that don't have any allocated 10593 * space inside of them. 10594 */ 10595 void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) 10596 { 10597 struct btrfs_block_group_cache *block_group; 10598 struct btrfs_space_info *space_info; 10599 struct btrfs_trans_handle *trans; 10600 int ret = 0; 10601 10602 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) 10603 return; 10604 10605 spin_lock(&fs_info->unused_bgs_lock); 10606 while (!list_empty(&fs_info->unused_bgs)) { 10607 u64 start, end; 10608 int trimming; 10609 10610 block_group = list_first_entry(&fs_info->unused_bgs, 10611 struct btrfs_block_group_cache, 10612 bg_list); 10613 list_del_init(&block_group->bg_list); 10614 10615 space_info = block_group->space_info; 10616 10617 if (ret || btrfs_mixed_space_info(space_info)) { 10618 btrfs_put_block_group(block_group); 10619 continue; 10620 } 10621 spin_unlock(&fs_info->unused_bgs_lock); 10622 10623 mutex_lock(&fs_info->delete_unused_bgs_mutex); 10624 10625 /* Don't want to race with allocators so take the groups_sem */ 10626 down_write(&space_info->groups_sem); 10627 spin_lock(&block_group->lock); 10628 if (block_group->reserved || 10629 btrfs_block_group_used(&block_group->item) || 10630 block_group->ro || 10631 list_is_singular(&block_group->list)) { 10632 /* 10633 * We want to bail if we made new allocations or have 10634 * outstanding allocations in this block group. We do 10635 * the ro check in case balance is currently acting on 10636 * this block group. 10637 */ 10638 spin_unlock(&block_group->lock); 10639 up_write(&space_info->groups_sem); 10640 goto next; 10641 } 10642 spin_unlock(&block_group->lock); 10643 10644 /* We don't want to force the issue, only flip if it's ok. */ 10645 ret = inc_block_group_ro(block_group, 0); 10646 up_write(&space_info->groups_sem); 10647 if (ret < 0) { 10648 ret = 0; 10649 goto next; 10650 } 10651 10652 /* 10653 * Want to do this before we do anything else so we can recover 10654 * properly if we fail to join the transaction. 10655 */ 10656 trans = btrfs_start_trans_remove_block_group(fs_info, 10657 block_group->key.objectid); 10658 if (IS_ERR(trans)) { 10659 btrfs_dec_block_group_ro(block_group); 10660 ret = PTR_ERR(trans); 10661 goto next; 10662 } 10663 10664 /* 10665 * We could have pending pinned extents for this block group, 10666 * just delete them, we don't care about them anymore. 10667 */ 10668 start = block_group->key.objectid; 10669 end = start + block_group->key.offset - 1; 10670 /* 10671 * Hold the unused_bg_unpin_mutex lock to avoid racing with 10672 * btrfs_finish_extent_commit(). If we are at transaction N, 10673 * another task might be running finish_extent_commit() for the 10674 * previous transaction N - 1, and have seen a range belonging 10675 * to the block group in freed_extents[] before we were able to 10676 * clear the whole block group range from freed_extents[]. This 10677 * means that task can lookup for the block group after we 10678 * unpinned it from freed_extents[] and removed it, leading to 10679 * a BUG_ON() at btrfs_unpin_extent_range(). 10680 */ 10681 mutex_lock(&fs_info->unused_bg_unpin_mutex); 10682 ret = clear_extent_bits(&fs_info->freed_extents[0], start, end, 10683 EXTENT_DIRTY); 10684 if (ret) { 10685 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 10686 btrfs_dec_block_group_ro(block_group); 10687 goto end_trans; 10688 } 10689 ret = clear_extent_bits(&fs_info->freed_extents[1], start, end, 10690 EXTENT_DIRTY); 10691 if (ret) { 10692 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 10693 btrfs_dec_block_group_ro(block_group); 10694 goto end_trans; 10695 } 10696 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 10697 10698 /* Reset pinned so btrfs_put_block_group doesn't complain */ 10699 spin_lock(&space_info->lock); 10700 spin_lock(&block_group->lock); 10701 10702 space_info->bytes_pinned -= block_group->pinned; 10703 space_info->bytes_readonly += block_group->pinned; 10704 percpu_counter_add(&space_info->total_bytes_pinned, 10705 -block_group->pinned); 10706 block_group->pinned = 0; 10707 10708 spin_unlock(&block_group->lock); 10709 spin_unlock(&space_info->lock); 10710 10711 /* DISCARD can flip during remount */ 10712 trimming = btrfs_test_opt(fs_info, DISCARD); 10713 10714 /* Implicit trim during transaction commit. */ 10715 if (trimming) 10716 btrfs_get_block_group_trimming(block_group); 10717 10718 /* 10719 * Btrfs_remove_chunk will abort the transaction if things go 10720 * horribly wrong. 10721 */ 10722 ret = btrfs_remove_chunk(trans, fs_info, 10723 block_group->key.objectid); 10724 10725 if (ret) { 10726 if (trimming) 10727 btrfs_put_block_group_trimming(block_group); 10728 goto end_trans; 10729 } 10730 10731 /* 10732 * If we're not mounted with -odiscard, we can just forget 10733 * about this block group. Otherwise we'll need to wait 10734 * until transaction commit to do the actual discard. 10735 */ 10736 if (trimming) { 10737 spin_lock(&fs_info->unused_bgs_lock); 10738 /* 10739 * A concurrent scrub might have added us to the list 10740 * fs_info->unused_bgs, so use a list_move operation 10741 * to add the block group to the deleted_bgs list. 10742 */ 10743 list_move(&block_group->bg_list, 10744 &trans->transaction->deleted_bgs); 10745 spin_unlock(&fs_info->unused_bgs_lock); 10746 btrfs_get_block_group(block_group); 10747 } 10748 end_trans: 10749 btrfs_end_transaction(trans); 10750 next: 10751 mutex_unlock(&fs_info->delete_unused_bgs_mutex); 10752 btrfs_put_block_group(block_group); 10753 spin_lock(&fs_info->unused_bgs_lock); 10754 } 10755 spin_unlock(&fs_info->unused_bgs_lock); 10756 } 10757 10758 int btrfs_init_space_info(struct btrfs_fs_info *fs_info) 10759 { 10760 struct btrfs_space_info *space_info; 10761 struct btrfs_super_block *disk_super; 10762 u64 features; 10763 u64 flags; 10764 int mixed = 0; 10765 int ret; 10766 10767 disk_super = fs_info->super_copy; 10768 if (!btrfs_super_root(disk_super)) 10769 return -EINVAL; 10770 10771 features = btrfs_super_incompat_flags(disk_super); 10772 if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) 10773 mixed = 1; 10774 10775 flags = BTRFS_BLOCK_GROUP_SYSTEM; 10776 ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info); 10777 if (ret) 10778 goto out; 10779 10780 if (mixed) { 10781 flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA; 10782 ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info); 10783 } else { 10784 flags = BTRFS_BLOCK_GROUP_METADATA; 10785 ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info); 10786 if (ret) 10787 goto out; 10788 10789 flags = BTRFS_BLOCK_GROUP_DATA; 10790 ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info); 10791 } 10792 out: 10793 return ret; 10794 } 10795 10796 int btrfs_error_unpin_extent_range(struct btrfs_fs_info *fs_info, 10797 u64 start, u64 end) 10798 { 10799 return unpin_extent_range(fs_info, start, end, false); 10800 } 10801 10802 /* 10803 * It used to be that old block groups would be left around forever. 10804 * Iterating over them would be enough to trim unused space. Since we 10805 * now automatically remove them, we also need to iterate over unallocated 10806 * space. 10807 * 10808 * We don't want a transaction for this since the discard may take a 10809 * substantial amount of time. We don't require that a transaction be 10810 * running, but we do need to take a running transaction into account 10811 * to ensure that we're not discarding chunks that were released in 10812 * the current transaction. 10813 * 10814 * Holding the chunks lock will prevent other threads from allocating 10815 * or releasing chunks, but it won't prevent a running transaction 10816 * from committing and releasing the memory that the pending chunks 10817 * list head uses. For that, we need to take a reference to the 10818 * transaction. 10819 */ 10820 static int btrfs_trim_free_extents(struct btrfs_device *device, 10821 u64 minlen, u64 *trimmed) 10822 { 10823 u64 start = 0, len = 0; 10824 int ret; 10825 10826 *trimmed = 0; 10827 10828 /* Not writeable = nothing to do. */ 10829 if (!device->writeable) 10830 return 0; 10831 10832 /* No free space = nothing to do. */ 10833 if (device->total_bytes <= device->bytes_used) 10834 return 0; 10835 10836 ret = 0; 10837 10838 while (1) { 10839 struct btrfs_fs_info *fs_info = device->fs_info; 10840 struct btrfs_transaction *trans; 10841 u64 bytes; 10842 10843 ret = mutex_lock_interruptible(&fs_info->chunk_mutex); 10844 if (ret) 10845 return ret; 10846 10847 down_read(&fs_info->commit_root_sem); 10848 10849 spin_lock(&fs_info->trans_lock); 10850 trans = fs_info->running_transaction; 10851 if (trans) 10852 atomic_inc(&trans->use_count); 10853 spin_unlock(&fs_info->trans_lock); 10854 10855 ret = find_free_dev_extent_start(trans, device, minlen, start, 10856 &start, &len); 10857 if (trans) 10858 btrfs_put_transaction(trans); 10859 10860 if (ret) { 10861 up_read(&fs_info->commit_root_sem); 10862 mutex_unlock(&fs_info->chunk_mutex); 10863 if (ret == -ENOSPC) 10864 ret = 0; 10865 break; 10866 } 10867 10868 ret = btrfs_issue_discard(device->bdev, start, len, &bytes); 10869 up_read(&fs_info->commit_root_sem); 10870 mutex_unlock(&fs_info->chunk_mutex); 10871 10872 if (ret) 10873 break; 10874 10875 start += len; 10876 *trimmed += bytes; 10877 10878 if (fatal_signal_pending(current)) { 10879 ret = -ERESTARTSYS; 10880 break; 10881 } 10882 10883 cond_resched(); 10884 } 10885 10886 return ret; 10887 } 10888 10889 int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) 10890 { 10891 struct btrfs_block_group_cache *cache = NULL; 10892 struct btrfs_device *device; 10893 struct list_head *devices; 10894 u64 group_trimmed; 10895 u64 start; 10896 u64 end; 10897 u64 trimmed = 0; 10898 u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy); 10899 int ret = 0; 10900 10901 /* 10902 * try to trim all FS space, our block group may start from non-zero. 10903 */ 10904 if (range->len == total_bytes) 10905 cache = btrfs_lookup_first_block_group(fs_info, range->start); 10906 else 10907 cache = btrfs_lookup_block_group(fs_info, range->start); 10908 10909 while (cache) { 10910 if (cache->key.objectid >= (range->start + range->len)) { 10911 btrfs_put_block_group(cache); 10912 break; 10913 } 10914 10915 start = max(range->start, cache->key.objectid); 10916 end = min(range->start + range->len, 10917 cache->key.objectid + cache->key.offset); 10918 10919 if (end - start >= range->minlen) { 10920 if (!block_group_cache_done(cache)) { 10921 ret = cache_block_group(cache, 0); 10922 if (ret) { 10923 btrfs_put_block_group(cache); 10924 break; 10925 } 10926 ret = wait_block_group_cache_done(cache); 10927 if (ret) { 10928 btrfs_put_block_group(cache); 10929 break; 10930 } 10931 } 10932 ret = btrfs_trim_block_group(cache, 10933 &group_trimmed, 10934 start, 10935 end, 10936 range->minlen); 10937 10938 trimmed += group_trimmed; 10939 if (ret) { 10940 btrfs_put_block_group(cache); 10941 break; 10942 } 10943 } 10944 10945 cache = next_block_group(fs_info, cache); 10946 } 10947 10948 mutex_lock(&fs_info->fs_devices->device_list_mutex); 10949 devices = &fs_info->fs_devices->alloc_list; 10950 list_for_each_entry(device, devices, dev_alloc_list) { 10951 ret = btrfs_trim_free_extents(device, range->minlen, 10952 &group_trimmed); 10953 if (ret) 10954 break; 10955 10956 trimmed += group_trimmed; 10957 } 10958 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 10959 10960 range->len = trimmed; 10961 return ret; 10962 } 10963 10964 /* 10965 * btrfs_{start,end}_write_no_snapshoting() are similar to 10966 * mnt_{want,drop}_write(), they are used to prevent some tasks from writing 10967 * data into the page cache through nocow before the subvolume is snapshoted, 10968 * but flush the data into disk after the snapshot creation, or to prevent 10969 * operations while snapshoting is ongoing and that cause the snapshot to be 10970 * inconsistent (writes followed by expanding truncates for example). 10971 */ 10972 void btrfs_end_write_no_snapshoting(struct btrfs_root *root) 10973 { 10974 percpu_counter_dec(&root->subv_writers->counter); 10975 /* 10976 * Make sure counter is updated before we wake up waiters. 10977 */ 10978 smp_mb(); 10979 if (waitqueue_active(&root->subv_writers->wait)) 10980 wake_up(&root->subv_writers->wait); 10981 } 10982 10983 int btrfs_start_write_no_snapshoting(struct btrfs_root *root) 10984 { 10985 if (atomic_read(&root->will_be_snapshoted)) 10986 return 0; 10987 10988 percpu_counter_inc(&root->subv_writers->counter); 10989 /* 10990 * Make sure counter is updated before we check for snapshot creation. 10991 */ 10992 smp_mb(); 10993 if (atomic_read(&root->will_be_snapshoted)) { 10994 btrfs_end_write_no_snapshoting(root); 10995 return 0; 10996 } 10997 return 1; 10998 } 10999 11000 static int wait_snapshoting_atomic_t(atomic_t *a) 11001 { 11002 schedule(); 11003 return 0; 11004 } 11005 11006 void btrfs_wait_for_snapshot_creation(struct btrfs_root *root) 11007 { 11008 while (true) { 11009 int ret; 11010 11011 ret = btrfs_start_write_no_snapshoting(root); 11012 if (ret) 11013 break; 11014 wait_on_atomic_t(&root->will_be_snapshoted, 11015 wait_snapshoting_atomic_t, 11016 TASK_UNINTERRUPTIBLE); 11017 } 11018 } 11019