1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "misc.h" 4 #include "ctree.h" 5 #include "block-group.h" 6 #include "space-info.h" 7 #include "disk-io.h" 8 #include "free-space-cache.h" 9 #include "free-space-tree.h" 10 #include "volumes.h" 11 #include "transaction.h" 12 #include "ref-verify.h" 13 #include "sysfs.h" 14 #include "tree-log.h" 15 #include "delalloc-space.h" 16 #include "discard.h" 17 #include "raid56.h" 18 19 /* 20 * Return target flags in extended format or 0 if restripe for this chunk_type 21 * is not in progress 22 * 23 * Should be called with balance_lock held 24 */ 25 static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags) 26 { 27 struct btrfs_balance_control *bctl = fs_info->balance_ctl; 28 u64 target = 0; 29 30 if (!bctl) 31 return 0; 32 33 if (flags & BTRFS_BLOCK_GROUP_DATA && 34 bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) { 35 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target; 36 } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM && 37 bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) { 38 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target; 39 } else if (flags & BTRFS_BLOCK_GROUP_METADATA && 40 bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) { 41 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target; 42 } 43 44 return target; 45 } 46 47 /* 48 * @flags: available profiles in extended format (see ctree.h) 49 * 50 * Return reduced profile in chunk format. If profile changing is in progress 51 * (either running or paused) picks the target profile (if it's already 52 * available), otherwise falls back to plain reducing. 53 */ 54 static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags) 55 { 56 u64 num_devices = fs_info->fs_devices->rw_devices; 57 u64 target; 58 u64 raid_type; 59 u64 allowed = 0; 60 61 /* 62 * See if restripe for this chunk_type is in progress, if so try to 63 * reduce to the target profile 64 */ 65 spin_lock(&fs_info->balance_lock); 66 target = get_restripe_target(fs_info, flags); 67 if (target) { 68 spin_unlock(&fs_info->balance_lock); 69 return extended_to_chunk(target); 70 } 71 spin_unlock(&fs_info->balance_lock); 72 73 /* First, mask out the RAID levels which aren't possible */ 74 for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) { 75 if (num_devices >= btrfs_raid_array[raid_type].devs_min) 76 allowed |= btrfs_raid_array[raid_type].bg_flag; 77 } 78 allowed &= flags; 79 80 if (allowed & BTRFS_BLOCK_GROUP_RAID6) 81 allowed = BTRFS_BLOCK_GROUP_RAID6; 82 else if (allowed & BTRFS_BLOCK_GROUP_RAID5) 83 allowed = BTRFS_BLOCK_GROUP_RAID5; 84 else if (allowed & BTRFS_BLOCK_GROUP_RAID10) 85 allowed = BTRFS_BLOCK_GROUP_RAID10; 86 else if (allowed & BTRFS_BLOCK_GROUP_RAID1) 87 allowed = BTRFS_BLOCK_GROUP_RAID1; 88 else if (allowed & BTRFS_BLOCK_GROUP_RAID0) 89 allowed = BTRFS_BLOCK_GROUP_RAID0; 90 91 flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK; 92 93 return extended_to_chunk(flags | allowed); 94 } 95 96 u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags) 97 { 98 unsigned seq; 99 u64 flags; 100 101 do { 102 flags = orig_flags; 103 seq = read_seqbegin(&fs_info->profiles_lock); 104 105 if (flags & BTRFS_BLOCK_GROUP_DATA) 106 flags |= fs_info->avail_data_alloc_bits; 107 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 108 flags |= fs_info->avail_system_alloc_bits; 109 else if (flags & BTRFS_BLOCK_GROUP_METADATA) 110 flags |= fs_info->avail_metadata_alloc_bits; 111 } while (read_seqretry(&fs_info->profiles_lock, seq)); 112 113 return btrfs_reduce_alloc_profile(fs_info, flags); 114 } 115 116 void btrfs_get_block_group(struct btrfs_block_group *cache) 117 { 118 refcount_inc(&cache->refs); 119 } 120 121 void btrfs_put_block_group(struct btrfs_block_group *cache) 122 { 123 if (refcount_dec_and_test(&cache->refs)) { 124 WARN_ON(cache->pinned > 0); 125 WARN_ON(cache->reserved > 0); 126 127 /* 128 * A block_group shouldn't be on the discard_list anymore. 129 * Remove the block_group from the discard_list to prevent us 130 * from causing a panic due to NULL pointer dereference. 131 */ 132 if (WARN_ON(!list_empty(&cache->discard_list))) 133 btrfs_discard_cancel_work(&cache->fs_info->discard_ctl, 134 cache); 135 136 /* 137 * If not empty, someone is still holding mutex of 138 * full_stripe_lock, which can only be released by caller. 139 * And it will definitely cause use-after-free when caller 140 * tries to release full stripe lock. 141 * 142 * No better way to resolve, but only to warn. 143 */ 144 WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root)); 145 kfree(cache->free_space_ctl); 146 kfree(cache); 147 } 148 } 149 150 /* 151 * This adds the block group to the fs_info rb tree for the block group cache 152 */ 153 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, 154 struct btrfs_block_group *block_group) 155 { 156 struct rb_node **p; 157 struct rb_node *parent = NULL; 158 struct btrfs_block_group *cache; 159 160 ASSERT(block_group->length != 0); 161 162 spin_lock(&info->block_group_cache_lock); 163 p = &info->block_group_cache_tree.rb_node; 164 165 while (*p) { 166 parent = *p; 167 cache = rb_entry(parent, struct btrfs_block_group, cache_node); 168 if (block_group->start < cache->start) { 169 p = &(*p)->rb_left; 170 } else if (block_group->start > cache->start) { 171 p = &(*p)->rb_right; 172 } else { 173 spin_unlock(&info->block_group_cache_lock); 174 return -EEXIST; 175 } 176 } 177 178 rb_link_node(&block_group->cache_node, parent, p); 179 rb_insert_color(&block_group->cache_node, 180 &info->block_group_cache_tree); 181 182 if (info->first_logical_byte > block_group->start) 183 info->first_logical_byte = block_group->start; 184 185 spin_unlock(&info->block_group_cache_lock); 186 187 return 0; 188 } 189 190 /* 191 * This will return the block group at or after bytenr if contains is 0, else 192 * it will return the block group that contains the bytenr 193 */ 194 static struct btrfs_block_group *block_group_cache_tree_search( 195 struct btrfs_fs_info *info, u64 bytenr, int contains) 196 { 197 struct btrfs_block_group *cache, *ret = NULL; 198 struct rb_node *n; 199 u64 end, start; 200 201 spin_lock(&info->block_group_cache_lock); 202 n = info->block_group_cache_tree.rb_node; 203 204 while (n) { 205 cache = rb_entry(n, struct btrfs_block_group, cache_node); 206 end = cache->start + cache->length - 1; 207 start = cache->start; 208 209 if (bytenr < start) { 210 if (!contains && (!ret || start < ret->start)) 211 ret = cache; 212 n = n->rb_left; 213 } else if (bytenr > start) { 214 if (contains && bytenr <= end) { 215 ret = cache; 216 break; 217 } 218 n = n->rb_right; 219 } else { 220 ret = cache; 221 break; 222 } 223 } 224 if (ret) { 225 btrfs_get_block_group(ret); 226 if (bytenr == 0 && info->first_logical_byte > ret->start) 227 info->first_logical_byte = ret->start; 228 } 229 spin_unlock(&info->block_group_cache_lock); 230 231 return ret; 232 } 233 234 /* 235 * Return the block group that starts at or after bytenr 236 */ 237 struct btrfs_block_group *btrfs_lookup_first_block_group( 238 struct btrfs_fs_info *info, u64 bytenr) 239 { 240 return block_group_cache_tree_search(info, bytenr, 0); 241 } 242 243 /* 244 * Return the block group that contains the given bytenr 245 */ 246 struct btrfs_block_group *btrfs_lookup_block_group( 247 struct btrfs_fs_info *info, u64 bytenr) 248 { 249 return block_group_cache_tree_search(info, bytenr, 1); 250 } 251 252 struct btrfs_block_group *btrfs_next_block_group( 253 struct btrfs_block_group *cache) 254 { 255 struct btrfs_fs_info *fs_info = cache->fs_info; 256 struct rb_node *node; 257 258 spin_lock(&fs_info->block_group_cache_lock); 259 260 /* If our block group was removed, we need a full search. */ 261 if (RB_EMPTY_NODE(&cache->cache_node)) { 262 const u64 next_bytenr = cache->start + cache->length; 263 264 spin_unlock(&fs_info->block_group_cache_lock); 265 btrfs_put_block_group(cache); 266 cache = btrfs_lookup_first_block_group(fs_info, next_bytenr); return cache; 267 } 268 node = rb_next(&cache->cache_node); 269 btrfs_put_block_group(cache); 270 if (node) { 271 cache = rb_entry(node, struct btrfs_block_group, cache_node); 272 btrfs_get_block_group(cache); 273 } else 274 cache = NULL; 275 spin_unlock(&fs_info->block_group_cache_lock); 276 return cache; 277 } 278 279 bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr) 280 { 281 struct btrfs_block_group *bg; 282 bool ret = true; 283 284 bg = btrfs_lookup_block_group(fs_info, bytenr); 285 if (!bg) 286 return false; 287 288 spin_lock(&bg->lock); 289 if (bg->ro) 290 ret = false; 291 else 292 atomic_inc(&bg->nocow_writers); 293 spin_unlock(&bg->lock); 294 295 /* No put on block group, done by btrfs_dec_nocow_writers */ 296 if (!ret) 297 btrfs_put_block_group(bg); 298 299 return ret; 300 } 301 302 void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr) 303 { 304 struct btrfs_block_group *bg; 305 306 bg = btrfs_lookup_block_group(fs_info, bytenr); 307 ASSERT(bg); 308 if (atomic_dec_and_test(&bg->nocow_writers)) 309 wake_up_var(&bg->nocow_writers); 310 /* 311 * Once for our lookup and once for the lookup done by a previous call 312 * to btrfs_inc_nocow_writers() 313 */ 314 btrfs_put_block_group(bg); 315 btrfs_put_block_group(bg); 316 } 317 318 void btrfs_wait_nocow_writers(struct btrfs_block_group *bg) 319 { 320 wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers)); 321 } 322 323 void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info, 324 const u64 start) 325 { 326 struct btrfs_block_group *bg; 327 328 bg = btrfs_lookup_block_group(fs_info, start); 329 ASSERT(bg); 330 if (atomic_dec_and_test(&bg->reservations)) 331 wake_up_var(&bg->reservations); 332 btrfs_put_block_group(bg); 333 } 334 335 void btrfs_wait_block_group_reservations(struct btrfs_block_group *bg) 336 { 337 struct btrfs_space_info *space_info = bg->space_info; 338 339 ASSERT(bg->ro); 340 341 if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA)) 342 return; 343 344 /* 345 * Our block group is read only but before we set it to read only, 346 * some task might have had allocated an extent from it already, but it 347 * has not yet created a respective ordered extent (and added it to a 348 * root's list of ordered extents). 349 * Therefore wait for any task currently allocating extents, since the 350 * block group's reservations counter is incremented while a read lock 351 * on the groups' semaphore is held and decremented after releasing 352 * the read access on that semaphore and creating the ordered extent. 353 */ 354 down_write(&space_info->groups_sem); 355 up_write(&space_info->groups_sem); 356 357 wait_var_event(&bg->reservations, !atomic_read(&bg->reservations)); 358 } 359 360 struct btrfs_caching_control *btrfs_get_caching_control( 361 struct btrfs_block_group *cache) 362 { 363 struct btrfs_caching_control *ctl; 364 365 spin_lock(&cache->lock); 366 if (!cache->caching_ctl) { 367 spin_unlock(&cache->lock); 368 return NULL; 369 } 370 371 ctl = cache->caching_ctl; 372 refcount_inc(&ctl->count); 373 spin_unlock(&cache->lock); 374 return ctl; 375 } 376 377 void btrfs_put_caching_control(struct btrfs_caching_control *ctl) 378 { 379 if (refcount_dec_and_test(&ctl->count)) 380 kfree(ctl); 381 } 382 383 /* 384 * When we wait for progress in the block group caching, its because our 385 * allocation attempt failed at least once. So, we must sleep and let some 386 * progress happen before we try again. 387 * 388 * This function will sleep at least once waiting for new free space to show 389 * up, and then it will check the block group free space numbers for our min 390 * num_bytes. Another option is to have it go ahead and look in the rbtree for 391 * a free extent of a given size, but this is a good start. 392 * 393 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using 394 * any of the information in this block group. 395 */ 396 void btrfs_wait_block_group_cache_progress(struct btrfs_block_group *cache, 397 u64 num_bytes) 398 { 399 struct btrfs_caching_control *caching_ctl; 400 401 caching_ctl = btrfs_get_caching_control(cache); 402 if (!caching_ctl) 403 return; 404 405 wait_event(caching_ctl->wait, btrfs_block_group_done(cache) || 406 (cache->free_space_ctl->free_space >= num_bytes)); 407 408 btrfs_put_caching_control(caching_ctl); 409 } 410 411 int btrfs_wait_block_group_cache_done(struct btrfs_block_group *cache) 412 { 413 struct btrfs_caching_control *caching_ctl; 414 int ret = 0; 415 416 caching_ctl = btrfs_get_caching_control(cache); 417 if (!caching_ctl) 418 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0; 419 420 wait_event(caching_ctl->wait, btrfs_block_group_done(cache)); 421 if (cache->cached == BTRFS_CACHE_ERROR) 422 ret = -EIO; 423 btrfs_put_caching_control(caching_ctl); 424 return ret; 425 } 426 427 #ifdef CONFIG_BTRFS_DEBUG 428 static void fragment_free_space(struct btrfs_block_group *block_group) 429 { 430 struct btrfs_fs_info *fs_info = block_group->fs_info; 431 u64 start = block_group->start; 432 u64 len = block_group->length; 433 u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ? 434 fs_info->nodesize : fs_info->sectorsize; 435 u64 step = chunk << 1; 436 437 while (len > chunk) { 438 btrfs_remove_free_space(block_group, start, chunk); 439 start += step; 440 if (len < step) 441 len = 0; 442 else 443 len -= step; 444 } 445 } 446 #endif 447 448 /* 449 * This is only called by btrfs_cache_block_group, since we could have freed 450 * extents we need to check the pinned_extents for any extents that can't be 451 * used yet since their free space will be released as soon as the transaction 452 * commits. 453 */ 454 u64 add_new_free_space(struct btrfs_block_group *block_group, u64 start, u64 end) 455 { 456 struct btrfs_fs_info *info = block_group->fs_info; 457 u64 extent_start, extent_end, size, total_added = 0; 458 int ret; 459 460 while (start < end) { 461 ret = find_first_extent_bit(&info->excluded_extents, start, 462 &extent_start, &extent_end, 463 EXTENT_DIRTY | EXTENT_UPTODATE, 464 NULL); 465 if (ret) 466 break; 467 468 if (extent_start <= start) { 469 start = extent_end + 1; 470 } else if (extent_start > start && extent_start < end) { 471 size = extent_start - start; 472 total_added += size; 473 ret = btrfs_add_free_space_async_trimmed(block_group, 474 start, size); 475 BUG_ON(ret); /* -ENOMEM or logic error */ 476 start = extent_end + 1; 477 } else { 478 break; 479 } 480 } 481 482 if (start < end) { 483 size = end - start; 484 total_added += size; 485 ret = btrfs_add_free_space_async_trimmed(block_group, start, 486 size); 487 BUG_ON(ret); /* -ENOMEM or logic error */ 488 } 489 490 return total_added; 491 } 492 493 static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl) 494 { 495 struct btrfs_block_group *block_group = caching_ctl->block_group; 496 struct btrfs_fs_info *fs_info = block_group->fs_info; 497 struct btrfs_root *extent_root = fs_info->extent_root; 498 struct btrfs_path *path; 499 struct extent_buffer *leaf; 500 struct btrfs_key key; 501 u64 total_found = 0; 502 u64 last = 0; 503 u32 nritems; 504 int ret; 505 bool wakeup = true; 506 507 path = btrfs_alloc_path(); 508 if (!path) 509 return -ENOMEM; 510 511 last = max_t(u64, block_group->start, BTRFS_SUPER_INFO_OFFSET); 512 513 #ifdef CONFIG_BTRFS_DEBUG 514 /* 515 * If we're fragmenting we don't want to make anybody think we can 516 * allocate from this block group until we've had a chance to fragment 517 * the free space. 518 */ 519 if (btrfs_should_fragment_free_space(block_group)) 520 wakeup = false; 521 #endif 522 /* 523 * We don't want to deadlock with somebody trying to allocate a new 524 * extent for the extent root while also trying to search the extent 525 * root to add free space. So we skip locking and search the commit 526 * root, since its read-only 527 */ 528 path->skip_locking = 1; 529 path->search_commit_root = 1; 530 path->reada = READA_FORWARD; 531 532 key.objectid = last; 533 key.offset = 0; 534 key.type = BTRFS_EXTENT_ITEM_KEY; 535 536 next: 537 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); 538 if (ret < 0) 539 goto out; 540 541 leaf = path->nodes[0]; 542 nritems = btrfs_header_nritems(leaf); 543 544 while (1) { 545 if (btrfs_fs_closing(fs_info) > 1) { 546 last = (u64)-1; 547 break; 548 } 549 550 if (path->slots[0] < nritems) { 551 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 552 } else { 553 ret = btrfs_find_next_key(extent_root, path, &key, 0, 0); 554 if (ret) 555 break; 556 557 if (need_resched() || 558 rwsem_is_contended(&fs_info->commit_root_sem)) { 559 if (wakeup) 560 caching_ctl->progress = last; 561 btrfs_release_path(path); 562 up_read(&fs_info->commit_root_sem); 563 mutex_unlock(&caching_ctl->mutex); 564 cond_resched(); 565 mutex_lock(&caching_ctl->mutex); 566 down_read(&fs_info->commit_root_sem); 567 goto next; 568 } 569 570 ret = btrfs_next_leaf(extent_root, path); 571 if (ret < 0) 572 goto out; 573 if (ret) 574 break; 575 leaf = path->nodes[0]; 576 nritems = btrfs_header_nritems(leaf); 577 continue; 578 } 579 580 if (key.objectid < last) { 581 key.objectid = last; 582 key.offset = 0; 583 key.type = BTRFS_EXTENT_ITEM_KEY; 584 585 if (wakeup) 586 caching_ctl->progress = last; 587 btrfs_release_path(path); 588 goto next; 589 } 590 591 if (key.objectid < block_group->start) { 592 path->slots[0]++; 593 continue; 594 } 595 596 if (key.objectid >= block_group->start + block_group->length) 597 break; 598 599 if (key.type == BTRFS_EXTENT_ITEM_KEY || 600 key.type == BTRFS_METADATA_ITEM_KEY) { 601 total_found += add_new_free_space(block_group, last, 602 key.objectid); 603 if (key.type == BTRFS_METADATA_ITEM_KEY) 604 last = key.objectid + 605 fs_info->nodesize; 606 else 607 last = key.objectid + key.offset; 608 609 if (total_found > CACHING_CTL_WAKE_UP) { 610 total_found = 0; 611 if (wakeup) 612 wake_up(&caching_ctl->wait); 613 } 614 } 615 path->slots[0]++; 616 } 617 ret = 0; 618 619 total_found += add_new_free_space(block_group, last, 620 block_group->start + block_group->length); 621 caching_ctl->progress = (u64)-1; 622 623 out: 624 btrfs_free_path(path); 625 return ret; 626 } 627 628 static noinline void caching_thread(struct btrfs_work *work) 629 { 630 struct btrfs_block_group *block_group; 631 struct btrfs_fs_info *fs_info; 632 struct btrfs_caching_control *caching_ctl; 633 int ret; 634 635 caching_ctl = container_of(work, struct btrfs_caching_control, work); 636 block_group = caching_ctl->block_group; 637 fs_info = block_group->fs_info; 638 639 mutex_lock(&caching_ctl->mutex); 640 down_read(&fs_info->commit_root_sem); 641 642 if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) 643 ret = load_free_space_tree(caching_ctl); 644 else 645 ret = load_extent_tree_free(caching_ctl); 646 647 spin_lock(&block_group->lock); 648 block_group->caching_ctl = NULL; 649 block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED; 650 spin_unlock(&block_group->lock); 651 652 #ifdef CONFIG_BTRFS_DEBUG 653 if (btrfs_should_fragment_free_space(block_group)) { 654 u64 bytes_used; 655 656 spin_lock(&block_group->space_info->lock); 657 spin_lock(&block_group->lock); 658 bytes_used = block_group->length - block_group->used; 659 block_group->space_info->bytes_used += bytes_used >> 1; 660 spin_unlock(&block_group->lock); 661 spin_unlock(&block_group->space_info->lock); 662 fragment_free_space(block_group); 663 } 664 #endif 665 666 caching_ctl->progress = (u64)-1; 667 668 up_read(&fs_info->commit_root_sem); 669 btrfs_free_excluded_extents(block_group); 670 mutex_unlock(&caching_ctl->mutex); 671 672 wake_up(&caching_ctl->wait); 673 674 btrfs_put_caching_control(caching_ctl); 675 btrfs_put_block_group(block_group); 676 } 677 678 int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only) 679 { 680 DEFINE_WAIT(wait); 681 struct btrfs_fs_info *fs_info = cache->fs_info; 682 struct btrfs_caching_control *caching_ctl; 683 int ret = 0; 684 685 caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS); 686 if (!caching_ctl) 687 return -ENOMEM; 688 689 INIT_LIST_HEAD(&caching_ctl->list); 690 mutex_init(&caching_ctl->mutex); 691 init_waitqueue_head(&caching_ctl->wait); 692 caching_ctl->block_group = cache; 693 caching_ctl->progress = cache->start; 694 refcount_set(&caching_ctl->count, 1); 695 btrfs_init_work(&caching_ctl->work, caching_thread, NULL, NULL); 696 697 spin_lock(&cache->lock); 698 /* 699 * This should be a rare occasion, but this could happen I think in the 700 * case where one thread starts to load the space cache info, and then 701 * some other thread starts a transaction commit which tries to do an 702 * allocation while the other thread is still loading the space cache 703 * info. The previous loop should have kept us from choosing this block 704 * group, but if we've moved to the state where we will wait on caching 705 * block groups we need to first check if we're doing a fast load here, 706 * so we can wait for it to finish, otherwise we could end up allocating 707 * from a block group who's cache gets evicted for one reason or 708 * another. 709 */ 710 while (cache->cached == BTRFS_CACHE_FAST) { 711 struct btrfs_caching_control *ctl; 712 713 ctl = cache->caching_ctl; 714 refcount_inc(&ctl->count); 715 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE); 716 spin_unlock(&cache->lock); 717 718 schedule(); 719 720 finish_wait(&ctl->wait, &wait); 721 btrfs_put_caching_control(ctl); 722 spin_lock(&cache->lock); 723 } 724 725 if (cache->cached != BTRFS_CACHE_NO) { 726 spin_unlock(&cache->lock); 727 kfree(caching_ctl); 728 return 0; 729 } 730 WARN_ON(cache->caching_ctl); 731 cache->caching_ctl = caching_ctl; 732 cache->cached = BTRFS_CACHE_FAST; 733 spin_unlock(&cache->lock); 734 735 if (btrfs_test_opt(fs_info, SPACE_CACHE)) { 736 mutex_lock(&caching_ctl->mutex); 737 ret = load_free_space_cache(cache); 738 739 spin_lock(&cache->lock); 740 if (ret == 1) { 741 cache->caching_ctl = NULL; 742 cache->cached = BTRFS_CACHE_FINISHED; 743 cache->last_byte_to_unpin = (u64)-1; 744 caching_ctl->progress = (u64)-1; 745 } else { 746 if (load_cache_only) { 747 cache->caching_ctl = NULL; 748 cache->cached = BTRFS_CACHE_NO; 749 } else { 750 cache->cached = BTRFS_CACHE_STARTED; 751 cache->has_caching_ctl = 1; 752 } 753 } 754 spin_unlock(&cache->lock); 755 #ifdef CONFIG_BTRFS_DEBUG 756 if (ret == 1 && 757 btrfs_should_fragment_free_space(cache)) { 758 u64 bytes_used; 759 760 spin_lock(&cache->space_info->lock); 761 spin_lock(&cache->lock); 762 bytes_used = cache->length - cache->used; 763 cache->space_info->bytes_used += bytes_used >> 1; 764 spin_unlock(&cache->lock); 765 spin_unlock(&cache->space_info->lock); 766 fragment_free_space(cache); 767 } 768 #endif 769 mutex_unlock(&caching_ctl->mutex); 770 771 wake_up(&caching_ctl->wait); 772 if (ret == 1) { 773 btrfs_put_caching_control(caching_ctl); 774 btrfs_free_excluded_extents(cache); 775 return 0; 776 } 777 } else { 778 /* 779 * We're either using the free space tree or no caching at all. 780 * Set cached to the appropriate value and wakeup any waiters. 781 */ 782 spin_lock(&cache->lock); 783 if (load_cache_only) { 784 cache->caching_ctl = NULL; 785 cache->cached = BTRFS_CACHE_NO; 786 } else { 787 cache->cached = BTRFS_CACHE_STARTED; 788 cache->has_caching_ctl = 1; 789 } 790 spin_unlock(&cache->lock); 791 wake_up(&caching_ctl->wait); 792 } 793 794 if (load_cache_only) { 795 btrfs_put_caching_control(caching_ctl); 796 return 0; 797 } 798 799 down_write(&fs_info->commit_root_sem); 800 refcount_inc(&caching_ctl->count); 801 list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups); 802 up_write(&fs_info->commit_root_sem); 803 804 btrfs_get_block_group(cache); 805 806 btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work); 807 808 return ret; 809 } 810 811 static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) 812 { 813 u64 extra_flags = chunk_to_extended(flags) & 814 BTRFS_EXTENDED_PROFILE_MASK; 815 816 write_seqlock(&fs_info->profiles_lock); 817 if (flags & BTRFS_BLOCK_GROUP_DATA) 818 fs_info->avail_data_alloc_bits &= ~extra_flags; 819 if (flags & BTRFS_BLOCK_GROUP_METADATA) 820 fs_info->avail_metadata_alloc_bits &= ~extra_flags; 821 if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 822 fs_info->avail_system_alloc_bits &= ~extra_flags; 823 write_sequnlock(&fs_info->profiles_lock); 824 } 825 826 /* 827 * Clear incompat bits for the following feature(s): 828 * 829 * - RAID56 - in case there's neither RAID5 nor RAID6 profile block group 830 * in the whole filesystem 831 * 832 * - RAID1C34 - same as above for RAID1C3 and RAID1C4 block groups 833 */ 834 static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags) 835 { 836 bool found_raid56 = false; 837 bool found_raid1c34 = false; 838 839 if ((flags & BTRFS_BLOCK_GROUP_RAID56_MASK) || 840 (flags & BTRFS_BLOCK_GROUP_RAID1C3) || 841 (flags & BTRFS_BLOCK_GROUP_RAID1C4)) { 842 struct list_head *head = &fs_info->space_info; 843 struct btrfs_space_info *sinfo; 844 845 list_for_each_entry_rcu(sinfo, head, list) { 846 down_read(&sinfo->groups_sem); 847 if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID5])) 848 found_raid56 = true; 849 if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID6])) 850 found_raid56 = true; 851 if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C3])) 852 found_raid1c34 = true; 853 if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C4])) 854 found_raid1c34 = true; 855 up_read(&sinfo->groups_sem); 856 } 857 if (!found_raid56) 858 btrfs_clear_fs_incompat(fs_info, RAID56); 859 if (!found_raid1c34) 860 btrfs_clear_fs_incompat(fs_info, RAID1C34); 861 } 862 } 863 864 static int remove_block_group_item(struct btrfs_trans_handle *trans, 865 struct btrfs_path *path, 866 struct btrfs_block_group *block_group) 867 { 868 struct btrfs_fs_info *fs_info = trans->fs_info; 869 struct btrfs_root *root; 870 struct btrfs_key key; 871 int ret; 872 873 root = fs_info->extent_root; 874 key.objectid = block_group->start; 875 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 876 key.offset = block_group->length; 877 878 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 879 if (ret > 0) 880 ret = -ENOENT; 881 if (ret < 0) 882 return ret; 883 884 ret = btrfs_del_item(trans, root, path); 885 return ret; 886 } 887 888 int btrfs_remove_block_group(struct btrfs_trans_handle *trans, 889 u64 group_start, struct extent_map *em) 890 { 891 struct btrfs_fs_info *fs_info = trans->fs_info; 892 struct btrfs_path *path; 893 struct btrfs_block_group *block_group; 894 struct btrfs_free_cluster *cluster; 895 struct btrfs_root *tree_root = fs_info->tree_root; 896 struct btrfs_key key; 897 struct inode *inode; 898 struct kobject *kobj = NULL; 899 int ret; 900 int index; 901 int factor; 902 struct btrfs_caching_control *caching_ctl = NULL; 903 bool remove_em; 904 bool remove_rsv = false; 905 906 block_group = btrfs_lookup_block_group(fs_info, group_start); 907 BUG_ON(!block_group); 908 BUG_ON(!block_group->ro); 909 910 trace_btrfs_remove_block_group(block_group); 911 /* 912 * Free the reserved super bytes from this block group before 913 * remove it. 914 */ 915 btrfs_free_excluded_extents(block_group); 916 btrfs_free_ref_tree_range(fs_info, block_group->start, 917 block_group->length); 918 919 index = btrfs_bg_flags_to_raid_index(block_group->flags); 920 factor = btrfs_bg_type_to_factor(block_group->flags); 921 922 /* make sure this block group isn't part of an allocation cluster */ 923 cluster = &fs_info->data_alloc_cluster; 924 spin_lock(&cluster->refill_lock); 925 btrfs_return_cluster_to_free_space(block_group, cluster); 926 spin_unlock(&cluster->refill_lock); 927 928 /* 929 * make sure this block group isn't part of a metadata 930 * allocation cluster 931 */ 932 cluster = &fs_info->meta_alloc_cluster; 933 spin_lock(&cluster->refill_lock); 934 btrfs_return_cluster_to_free_space(block_group, cluster); 935 spin_unlock(&cluster->refill_lock); 936 937 path = btrfs_alloc_path(); 938 if (!path) { 939 ret = -ENOMEM; 940 goto out; 941 } 942 943 /* 944 * get the inode first so any iput calls done for the io_list 945 * aren't the final iput (no unlinks allowed now) 946 */ 947 inode = lookup_free_space_inode(block_group, path); 948 949 mutex_lock(&trans->transaction->cache_write_mutex); 950 /* 951 * Make sure our free space cache IO is done before removing the 952 * free space inode 953 */ 954 spin_lock(&trans->transaction->dirty_bgs_lock); 955 if (!list_empty(&block_group->io_list)) { 956 list_del_init(&block_group->io_list); 957 958 WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode); 959 960 spin_unlock(&trans->transaction->dirty_bgs_lock); 961 btrfs_wait_cache_io(trans, block_group, path); 962 btrfs_put_block_group(block_group); 963 spin_lock(&trans->transaction->dirty_bgs_lock); 964 } 965 966 if (!list_empty(&block_group->dirty_list)) { 967 list_del_init(&block_group->dirty_list); 968 remove_rsv = true; 969 btrfs_put_block_group(block_group); 970 } 971 spin_unlock(&trans->transaction->dirty_bgs_lock); 972 mutex_unlock(&trans->transaction->cache_write_mutex); 973 974 if (!IS_ERR(inode)) { 975 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 976 if (ret) { 977 btrfs_add_delayed_iput(inode); 978 goto out; 979 } 980 clear_nlink(inode); 981 /* One for the block groups ref */ 982 spin_lock(&block_group->lock); 983 if (block_group->iref) { 984 block_group->iref = 0; 985 block_group->inode = NULL; 986 spin_unlock(&block_group->lock); 987 iput(inode); 988 } else { 989 spin_unlock(&block_group->lock); 990 } 991 /* One for our lookup ref */ 992 btrfs_add_delayed_iput(inode); 993 } 994 995 key.objectid = BTRFS_FREE_SPACE_OBJECTID; 996 key.type = 0; 997 key.offset = block_group->start; 998 999 ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1); 1000 if (ret < 0) 1001 goto out; 1002 if (ret > 0) 1003 btrfs_release_path(path); 1004 if (ret == 0) { 1005 ret = btrfs_del_item(trans, tree_root, path); 1006 if (ret) 1007 goto out; 1008 btrfs_release_path(path); 1009 } 1010 1011 spin_lock(&fs_info->block_group_cache_lock); 1012 rb_erase(&block_group->cache_node, 1013 &fs_info->block_group_cache_tree); 1014 RB_CLEAR_NODE(&block_group->cache_node); 1015 1016 /* Once for the block groups rbtree */ 1017 btrfs_put_block_group(block_group); 1018 1019 if (fs_info->first_logical_byte == block_group->start) 1020 fs_info->first_logical_byte = (u64)-1; 1021 spin_unlock(&fs_info->block_group_cache_lock); 1022 1023 down_write(&block_group->space_info->groups_sem); 1024 /* 1025 * we must use list_del_init so people can check to see if they 1026 * are still on the list after taking the semaphore 1027 */ 1028 list_del_init(&block_group->list); 1029 if (list_empty(&block_group->space_info->block_groups[index])) { 1030 kobj = block_group->space_info->block_group_kobjs[index]; 1031 block_group->space_info->block_group_kobjs[index] = NULL; 1032 clear_avail_alloc_bits(fs_info, block_group->flags); 1033 } 1034 up_write(&block_group->space_info->groups_sem); 1035 clear_incompat_bg_bits(fs_info, block_group->flags); 1036 if (kobj) { 1037 kobject_del(kobj); 1038 kobject_put(kobj); 1039 } 1040 1041 if (block_group->has_caching_ctl) 1042 caching_ctl = btrfs_get_caching_control(block_group); 1043 if (block_group->cached == BTRFS_CACHE_STARTED) 1044 btrfs_wait_block_group_cache_done(block_group); 1045 if (block_group->has_caching_ctl) { 1046 down_write(&fs_info->commit_root_sem); 1047 if (!caching_ctl) { 1048 struct btrfs_caching_control *ctl; 1049 1050 list_for_each_entry(ctl, 1051 &fs_info->caching_block_groups, list) 1052 if (ctl->block_group == block_group) { 1053 caching_ctl = ctl; 1054 refcount_inc(&caching_ctl->count); 1055 break; 1056 } 1057 } 1058 if (caching_ctl) 1059 list_del_init(&caching_ctl->list); 1060 up_write(&fs_info->commit_root_sem); 1061 if (caching_ctl) { 1062 /* Once for the caching bgs list and once for us. */ 1063 btrfs_put_caching_control(caching_ctl); 1064 btrfs_put_caching_control(caching_ctl); 1065 } 1066 } 1067 1068 spin_lock(&trans->transaction->dirty_bgs_lock); 1069 WARN_ON(!list_empty(&block_group->dirty_list)); 1070 WARN_ON(!list_empty(&block_group->io_list)); 1071 spin_unlock(&trans->transaction->dirty_bgs_lock); 1072 1073 btrfs_remove_free_space_cache(block_group); 1074 1075 spin_lock(&block_group->space_info->lock); 1076 list_del_init(&block_group->ro_list); 1077 1078 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { 1079 WARN_ON(block_group->space_info->total_bytes 1080 < block_group->length); 1081 WARN_ON(block_group->space_info->bytes_readonly 1082 < block_group->length); 1083 WARN_ON(block_group->space_info->disk_total 1084 < block_group->length * factor); 1085 } 1086 block_group->space_info->total_bytes -= block_group->length; 1087 block_group->space_info->bytes_readonly -= block_group->length; 1088 block_group->space_info->disk_total -= block_group->length * factor; 1089 1090 spin_unlock(&block_group->space_info->lock); 1091 1092 /* 1093 * Remove the free space for the block group from the free space tree 1094 * and the block group's item from the extent tree before marking the 1095 * block group as removed. This is to prevent races with tasks that 1096 * freeze and unfreeze a block group, this task and another task 1097 * allocating a new block group - the unfreeze task ends up removing 1098 * the block group's extent map before the task calling this function 1099 * deletes the block group item from the extent tree, allowing for 1100 * another task to attempt to create another block group with the same 1101 * item key (and failing with -EEXIST and a transaction abort). 1102 */ 1103 ret = remove_block_group_free_space(trans, block_group); 1104 if (ret) 1105 goto out; 1106 1107 ret = remove_block_group_item(trans, path, block_group); 1108 if (ret < 0) 1109 goto out; 1110 1111 spin_lock(&block_group->lock); 1112 block_group->removed = 1; 1113 /* 1114 * At this point trimming or scrub can't start on this block group, 1115 * because we removed the block group from the rbtree 1116 * fs_info->block_group_cache_tree so no one can't find it anymore and 1117 * even if someone already got this block group before we removed it 1118 * from the rbtree, they have already incremented block_group->frozen - 1119 * if they didn't, for the trimming case they won't find any free space 1120 * entries because we already removed them all when we called 1121 * btrfs_remove_free_space_cache(). 1122 * 1123 * And we must not remove the extent map from the fs_info->mapping_tree 1124 * to prevent the same logical address range and physical device space 1125 * ranges from being reused for a new block group. This is needed to 1126 * avoid races with trimming and scrub. 1127 * 1128 * An fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is 1129 * completely transactionless, so while it is trimming a range the 1130 * currently running transaction might finish and a new one start, 1131 * allowing for new block groups to be created that can reuse the same 1132 * physical device locations unless we take this special care. 1133 * 1134 * There may also be an implicit trim operation if the file system 1135 * is mounted with -odiscard. The same protections must remain 1136 * in place until the extents have been discarded completely when 1137 * the transaction commit has completed. 1138 */ 1139 remove_em = (atomic_read(&block_group->frozen) == 0); 1140 spin_unlock(&block_group->lock); 1141 1142 if (remove_em) { 1143 struct extent_map_tree *em_tree; 1144 1145 em_tree = &fs_info->mapping_tree; 1146 write_lock(&em_tree->lock); 1147 remove_extent_mapping(em_tree, em); 1148 write_unlock(&em_tree->lock); 1149 /* once for the tree */ 1150 free_extent_map(em); 1151 } 1152 1153 out: 1154 /* Once for the lookup reference */ 1155 btrfs_put_block_group(block_group); 1156 if (remove_rsv) 1157 btrfs_delayed_refs_rsv_release(fs_info, 1); 1158 btrfs_free_path(path); 1159 return ret; 1160 } 1161 1162 struct btrfs_trans_handle *btrfs_start_trans_remove_block_group( 1163 struct btrfs_fs_info *fs_info, const u64 chunk_offset) 1164 { 1165 struct extent_map_tree *em_tree = &fs_info->mapping_tree; 1166 struct extent_map *em; 1167 struct map_lookup *map; 1168 unsigned int num_items; 1169 1170 read_lock(&em_tree->lock); 1171 em = lookup_extent_mapping(em_tree, chunk_offset, 1); 1172 read_unlock(&em_tree->lock); 1173 ASSERT(em && em->start == chunk_offset); 1174 1175 /* 1176 * We need to reserve 3 + N units from the metadata space info in order 1177 * to remove a block group (done at btrfs_remove_chunk() and at 1178 * btrfs_remove_block_group()), which are used for: 1179 * 1180 * 1 unit for adding the free space inode's orphan (located in the tree 1181 * of tree roots). 1182 * 1 unit for deleting the block group item (located in the extent 1183 * tree). 1184 * 1 unit for deleting the free space item (located in tree of tree 1185 * roots). 1186 * N units for deleting N device extent items corresponding to each 1187 * stripe (located in the device tree). 1188 * 1189 * In order to remove a block group we also need to reserve units in the 1190 * system space info in order to update the chunk tree (update one or 1191 * more device items and remove one chunk item), but this is done at 1192 * btrfs_remove_chunk() through a call to check_system_chunk(). 1193 */ 1194 map = em->map_lookup; 1195 num_items = 3 + map->num_stripes; 1196 free_extent_map(em); 1197 1198 return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root, 1199 num_items); 1200 } 1201 1202 /* 1203 * Mark block group @cache read-only, so later write won't happen to block 1204 * group @cache. 1205 * 1206 * If @force is not set, this function will only mark the block group readonly 1207 * if we have enough free space (1M) in other metadata/system block groups. 1208 * If @force is not set, this function will mark the block group readonly 1209 * without checking free space. 1210 * 1211 * NOTE: This function doesn't care if other block groups can contain all the 1212 * data in this block group. That check should be done by relocation routine, 1213 * not this function. 1214 */ 1215 static int inc_block_group_ro(struct btrfs_block_group *cache, int force) 1216 { 1217 struct btrfs_space_info *sinfo = cache->space_info; 1218 u64 num_bytes; 1219 int ret = -ENOSPC; 1220 1221 spin_lock(&sinfo->lock); 1222 spin_lock(&cache->lock); 1223 1224 if (cache->ro) { 1225 cache->ro++; 1226 ret = 0; 1227 goto out; 1228 } 1229 1230 num_bytes = cache->length - cache->reserved - cache->pinned - 1231 cache->bytes_super - cache->used; 1232 1233 /* 1234 * Data never overcommits, even in mixed mode, so do just the straight 1235 * check of left over space in how much we have allocated. 1236 */ 1237 if (force) { 1238 ret = 0; 1239 } else if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA) { 1240 u64 sinfo_used = btrfs_space_info_used(sinfo, true); 1241 1242 /* 1243 * Here we make sure if we mark this bg RO, we still have enough 1244 * free space as buffer. 1245 */ 1246 if (sinfo_used + num_bytes <= sinfo->total_bytes) 1247 ret = 0; 1248 } else { 1249 /* 1250 * We overcommit metadata, so we need to do the 1251 * btrfs_can_overcommit check here, and we need to pass in 1252 * BTRFS_RESERVE_NO_FLUSH to give ourselves the most amount of 1253 * leeway to allow us to mark this block group as read only. 1254 */ 1255 if (btrfs_can_overcommit(cache->fs_info, sinfo, num_bytes, 1256 BTRFS_RESERVE_NO_FLUSH)) 1257 ret = 0; 1258 } 1259 1260 if (!ret) { 1261 sinfo->bytes_readonly += num_bytes; 1262 cache->ro++; 1263 list_add_tail(&cache->ro_list, &sinfo->ro_bgs); 1264 } 1265 out: 1266 spin_unlock(&cache->lock); 1267 spin_unlock(&sinfo->lock); 1268 if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) { 1269 btrfs_info(cache->fs_info, 1270 "unable to make block group %llu ro", cache->start); 1271 btrfs_dump_space_info(cache->fs_info, cache->space_info, 0, 0); 1272 } 1273 return ret; 1274 } 1275 1276 static bool clean_pinned_extents(struct btrfs_trans_handle *trans, 1277 struct btrfs_block_group *bg) 1278 { 1279 struct btrfs_fs_info *fs_info = bg->fs_info; 1280 struct btrfs_transaction *prev_trans = NULL; 1281 const u64 start = bg->start; 1282 const u64 end = start + bg->length - 1; 1283 int ret; 1284 1285 spin_lock(&fs_info->trans_lock); 1286 if (trans->transaction->list.prev != &fs_info->trans_list) { 1287 prev_trans = list_last_entry(&trans->transaction->list, 1288 struct btrfs_transaction, list); 1289 refcount_inc(&prev_trans->use_count); 1290 } 1291 spin_unlock(&fs_info->trans_lock); 1292 1293 /* 1294 * Hold the unused_bg_unpin_mutex lock to avoid racing with 1295 * btrfs_finish_extent_commit(). If we are at transaction N, another 1296 * task might be running finish_extent_commit() for the previous 1297 * transaction N - 1, and have seen a range belonging to the block 1298 * group in pinned_extents before we were able to clear the whole block 1299 * group range from pinned_extents. This means that task can lookup for 1300 * the block group after we unpinned it from pinned_extents and removed 1301 * it, leading to a BUG_ON() at unpin_extent_range(). 1302 */ 1303 mutex_lock(&fs_info->unused_bg_unpin_mutex); 1304 if (prev_trans) { 1305 ret = clear_extent_bits(&prev_trans->pinned_extents, start, end, 1306 EXTENT_DIRTY); 1307 if (ret) 1308 goto out; 1309 } 1310 1311 ret = clear_extent_bits(&trans->transaction->pinned_extents, start, end, 1312 EXTENT_DIRTY); 1313 out: 1314 mutex_unlock(&fs_info->unused_bg_unpin_mutex); 1315 if (prev_trans) 1316 btrfs_put_transaction(prev_trans); 1317 1318 return ret == 0; 1319 } 1320 1321 /* 1322 * Process the unused_bgs list and remove any that don't have any allocated 1323 * space inside of them. 1324 */ 1325 void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) 1326 { 1327 struct btrfs_block_group *block_group; 1328 struct btrfs_space_info *space_info; 1329 struct btrfs_trans_handle *trans; 1330 const bool async_trim_enabled = btrfs_test_opt(fs_info, DISCARD_ASYNC); 1331 int ret = 0; 1332 1333 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) 1334 return; 1335 1336 spin_lock(&fs_info->unused_bgs_lock); 1337 while (!list_empty(&fs_info->unused_bgs)) { 1338 int trimming; 1339 1340 block_group = list_first_entry(&fs_info->unused_bgs, 1341 struct btrfs_block_group, 1342 bg_list); 1343 list_del_init(&block_group->bg_list); 1344 1345 space_info = block_group->space_info; 1346 1347 if (ret || btrfs_mixed_space_info(space_info)) { 1348 btrfs_put_block_group(block_group); 1349 continue; 1350 } 1351 spin_unlock(&fs_info->unused_bgs_lock); 1352 1353 btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group); 1354 1355 mutex_lock(&fs_info->delete_unused_bgs_mutex); 1356 1357 /* Don't want to race with allocators so take the groups_sem */ 1358 down_write(&space_info->groups_sem); 1359 1360 /* 1361 * Async discard moves the final block group discard to be prior 1362 * to the unused_bgs code path. Therefore, if it's not fully 1363 * trimmed, punt it back to the async discard lists. 1364 */ 1365 if (btrfs_test_opt(fs_info, DISCARD_ASYNC) && 1366 !btrfs_is_free_space_trimmed(block_group)) { 1367 trace_btrfs_skip_unused_block_group(block_group); 1368 up_write(&space_info->groups_sem); 1369 /* Requeue if we failed because of async discard */ 1370 btrfs_discard_queue_work(&fs_info->discard_ctl, 1371 block_group); 1372 goto next; 1373 } 1374 1375 spin_lock(&block_group->lock); 1376 if (block_group->reserved || block_group->pinned || 1377 block_group->used || block_group->ro || 1378 list_is_singular(&block_group->list)) { 1379 /* 1380 * We want to bail if we made new allocations or have 1381 * outstanding allocations in this block group. We do 1382 * the ro check in case balance is currently acting on 1383 * this block group. 1384 */ 1385 trace_btrfs_skip_unused_block_group(block_group); 1386 spin_unlock(&block_group->lock); 1387 up_write(&space_info->groups_sem); 1388 goto next; 1389 } 1390 spin_unlock(&block_group->lock); 1391 1392 /* We don't want to force the issue, only flip if it's ok. */ 1393 ret = inc_block_group_ro(block_group, 0); 1394 up_write(&space_info->groups_sem); 1395 if (ret < 0) { 1396 ret = 0; 1397 goto next; 1398 } 1399 1400 /* 1401 * Want to do this before we do anything else so we can recover 1402 * properly if we fail to join the transaction. 1403 */ 1404 trans = btrfs_start_trans_remove_block_group(fs_info, 1405 block_group->start); 1406 if (IS_ERR(trans)) { 1407 btrfs_dec_block_group_ro(block_group); 1408 ret = PTR_ERR(trans); 1409 goto next; 1410 } 1411 1412 /* 1413 * We could have pending pinned extents for this block group, 1414 * just delete them, we don't care about them anymore. 1415 */ 1416 if (!clean_pinned_extents(trans, block_group)) { 1417 btrfs_dec_block_group_ro(block_group); 1418 goto end_trans; 1419 } 1420 1421 /* 1422 * At this point, the block_group is read only and should fail 1423 * new allocations. However, btrfs_finish_extent_commit() can 1424 * cause this block_group to be placed back on the discard 1425 * lists because now the block_group isn't fully discarded. 1426 * Bail here and try again later after discarding everything. 1427 */ 1428 spin_lock(&fs_info->discard_ctl.lock); 1429 if (!list_empty(&block_group->discard_list)) { 1430 spin_unlock(&fs_info->discard_ctl.lock); 1431 btrfs_dec_block_group_ro(block_group); 1432 btrfs_discard_queue_work(&fs_info->discard_ctl, 1433 block_group); 1434 goto end_trans; 1435 } 1436 spin_unlock(&fs_info->discard_ctl.lock); 1437 1438 /* Reset pinned so btrfs_put_block_group doesn't complain */ 1439 spin_lock(&space_info->lock); 1440 spin_lock(&block_group->lock); 1441 1442 btrfs_space_info_update_bytes_pinned(fs_info, space_info, 1443 -block_group->pinned); 1444 space_info->bytes_readonly += block_group->pinned; 1445 percpu_counter_add_batch(&space_info->total_bytes_pinned, 1446 -block_group->pinned, 1447 BTRFS_TOTAL_BYTES_PINNED_BATCH); 1448 block_group->pinned = 0; 1449 1450 spin_unlock(&block_group->lock); 1451 spin_unlock(&space_info->lock); 1452 1453 /* 1454 * The normal path here is an unused block group is passed here, 1455 * then trimming is handled in the transaction commit path. 1456 * Async discard interposes before this to do the trimming 1457 * before coming down the unused block group path as trimming 1458 * will no longer be done later in the transaction commit path. 1459 */ 1460 if (!async_trim_enabled && btrfs_test_opt(fs_info, DISCARD_ASYNC)) 1461 goto flip_async; 1462 1463 /* DISCARD can flip during remount */ 1464 trimming = btrfs_test_opt(fs_info, DISCARD_SYNC); 1465 1466 /* Implicit trim during transaction commit. */ 1467 if (trimming) 1468 btrfs_freeze_block_group(block_group); 1469 1470 /* 1471 * Btrfs_remove_chunk will abort the transaction if things go 1472 * horribly wrong. 1473 */ 1474 ret = btrfs_remove_chunk(trans, block_group->start); 1475 1476 if (ret) { 1477 if (trimming) 1478 btrfs_unfreeze_block_group(block_group); 1479 goto end_trans; 1480 } 1481 1482 /* 1483 * If we're not mounted with -odiscard, we can just forget 1484 * about this block group. Otherwise we'll need to wait 1485 * until transaction commit to do the actual discard. 1486 */ 1487 if (trimming) { 1488 spin_lock(&fs_info->unused_bgs_lock); 1489 /* 1490 * A concurrent scrub might have added us to the list 1491 * fs_info->unused_bgs, so use a list_move operation 1492 * to add the block group to the deleted_bgs list. 1493 */ 1494 list_move(&block_group->bg_list, 1495 &trans->transaction->deleted_bgs); 1496 spin_unlock(&fs_info->unused_bgs_lock); 1497 btrfs_get_block_group(block_group); 1498 } 1499 end_trans: 1500 btrfs_end_transaction(trans); 1501 next: 1502 mutex_unlock(&fs_info->delete_unused_bgs_mutex); 1503 btrfs_put_block_group(block_group); 1504 spin_lock(&fs_info->unused_bgs_lock); 1505 } 1506 spin_unlock(&fs_info->unused_bgs_lock); 1507 return; 1508 1509 flip_async: 1510 btrfs_end_transaction(trans); 1511 mutex_unlock(&fs_info->delete_unused_bgs_mutex); 1512 btrfs_put_block_group(block_group); 1513 btrfs_discard_punt_unused_bgs_list(fs_info); 1514 } 1515 1516 void btrfs_mark_bg_unused(struct btrfs_block_group *bg) 1517 { 1518 struct btrfs_fs_info *fs_info = bg->fs_info; 1519 1520 spin_lock(&fs_info->unused_bgs_lock); 1521 if (list_empty(&bg->bg_list)) { 1522 btrfs_get_block_group(bg); 1523 trace_btrfs_add_unused_block_group(bg); 1524 list_add_tail(&bg->bg_list, &fs_info->unused_bgs); 1525 } 1526 spin_unlock(&fs_info->unused_bgs_lock); 1527 } 1528 1529 static int read_bg_from_eb(struct btrfs_fs_info *fs_info, struct btrfs_key *key, 1530 struct btrfs_path *path) 1531 { 1532 struct extent_map_tree *em_tree; 1533 struct extent_map *em; 1534 struct btrfs_block_group_item bg; 1535 struct extent_buffer *leaf; 1536 int slot; 1537 u64 flags; 1538 int ret = 0; 1539 1540 slot = path->slots[0]; 1541 leaf = path->nodes[0]; 1542 1543 em_tree = &fs_info->mapping_tree; 1544 read_lock(&em_tree->lock); 1545 em = lookup_extent_mapping(em_tree, key->objectid, key->offset); 1546 read_unlock(&em_tree->lock); 1547 if (!em) { 1548 btrfs_err(fs_info, 1549 "logical %llu len %llu found bg but no related chunk", 1550 key->objectid, key->offset); 1551 return -ENOENT; 1552 } 1553 1554 if (em->start != key->objectid || em->len != key->offset) { 1555 btrfs_err(fs_info, 1556 "block group %llu len %llu mismatch with chunk %llu len %llu", 1557 key->objectid, key->offset, em->start, em->len); 1558 ret = -EUCLEAN; 1559 goto out_free_em; 1560 } 1561 1562 read_extent_buffer(leaf, &bg, btrfs_item_ptr_offset(leaf, slot), 1563 sizeof(bg)); 1564 flags = btrfs_stack_block_group_flags(&bg) & 1565 BTRFS_BLOCK_GROUP_TYPE_MASK; 1566 1567 if (flags != (em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK)) { 1568 btrfs_err(fs_info, 1569 "block group %llu len %llu type flags 0x%llx mismatch with chunk type flags 0x%llx", 1570 key->objectid, key->offset, flags, 1571 (BTRFS_BLOCK_GROUP_TYPE_MASK & em->map_lookup->type)); 1572 ret = -EUCLEAN; 1573 } 1574 1575 out_free_em: 1576 free_extent_map(em); 1577 return ret; 1578 } 1579 1580 static int find_first_block_group(struct btrfs_fs_info *fs_info, 1581 struct btrfs_path *path, 1582 struct btrfs_key *key) 1583 { 1584 struct btrfs_root *root = fs_info->extent_root; 1585 int ret; 1586 struct btrfs_key found_key; 1587 struct extent_buffer *leaf; 1588 int slot; 1589 1590 ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 1591 if (ret < 0) 1592 return ret; 1593 1594 while (1) { 1595 slot = path->slots[0]; 1596 leaf = path->nodes[0]; 1597 if (slot >= btrfs_header_nritems(leaf)) { 1598 ret = btrfs_next_leaf(root, path); 1599 if (ret == 0) 1600 continue; 1601 if (ret < 0) 1602 goto out; 1603 break; 1604 } 1605 btrfs_item_key_to_cpu(leaf, &found_key, slot); 1606 1607 if (found_key.objectid >= key->objectid && 1608 found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) { 1609 ret = read_bg_from_eb(fs_info, &found_key, path); 1610 break; 1611 } 1612 1613 path->slots[0]++; 1614 } 1615 out: 1616 return ret; 1617 } 1618 1619 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) 1620 { 1621 u64 extra_flags = chunk_to_extended(flags) & 1622 BTRFS_EXTENDED_PROFILE_MASK; 1623 1624 write_seqlock(&fs_info->profiles_lock); 1625 if (flags & BTRFS_BLOCK_GROUP_DATA) 1626 fs_info->avail_data_alloc_bits |= extra_flags; 1627 if (flags & BTRFS_BLOCK_GROUP_METADATA) 1628 fs_info->avail_metadata_alloc_bits |= extra_flags; 1629 if (flags & BTRFS_BLOCK_GROUP_SYSTEM) 1630 fs_info->avail_system_alloc_bits |= extra_flags; 1631 write_sequnlock(&fs_info->profiles_lock); 1632 } 1633 1634 /** 1635 * btrfs_rmap_block - Map a physical disk address to a list of logical addresses 1636 * @chunk_start: logical address of block group 1637 * @physical: physical address to map to logical addresses 1638 * @logical: return array of logical addresses which map to @physical 1639 * @naddrs: length of @logical 1640 * @stripe_len: size of IO stripe for the given block group 1641 * 1642 * Maps a particular @physical disk address to a list of @logical addresses. 1643 * Used primarily to exclude those portions of a block group that contain super 1644 * block copies. 1645 */ 1646 EXPORT_FOR_TESTS 1647 int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, 1648 u64 physical, u64 **logical, int *naddrs, int *stripe_len) 1649 { 1650 struct extent_map *em; 1651 struct map_lookup *map; 1652 u64 *buf; 1653 u64 bytenr; 1654 u64 data_stripe_length; 1655 u64 io_stripe_size; 1656 int i, nr = 0; 1657 int ret = 0; 1658 1659 em = btrfs_get_chunk_map(fs_info, chunk_start, 1); 1660 if (IS_ERR(em)) 1661 return -EIO; 1662 1663 map = em->map_lookup; 1664 data_stripe_length = em->orig_block_len; 1665 io_stripe_size = map->stripe_len; 1666 1667 /* For RAID5/6 adjust to a full IO stripe length */ 1668 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) 1669 io_stripe_size = map->stripe_len * nr_data_stripes(map); 1670 1671 buf = kcalloc(map->num_stripes, sizeof(u64), GFP_NOFS); 1672 if (!buf) { 1673 ret = -ENOMEM; 1674 goto out; 1675 } 1676 1677 for (i = 0; i < map->num_stripes; i++) { 1678 bool already_inserted = false; 1679 u64 stripe_nr; 1680 int j; 1681 1682 if (!in_range(physical, map->stripes[i].physical, 1683 data_stripe_length)) 1684 continue; 1685 1686 stripe_nr = physical - map->stripes[i].physical; 1687 stripe_nr = div64_u64(stripe_nr, map->stripe_len); 1688 1689 if (map->type & BTRFS_BLOCK_GROUP_RAID10) { 1690 stripe_nr = stripe_nr * map->num_stripes + i; 1691 stripe_nr = div_u64(stripe_nr, map->sub_stripes); 1692 } else if (map->type & BTRFS_BLOCK_GROUP_RAID0) { 1693 stripe_nr = stripe_nr * map->num_stripes + i; 1694 } 1695 /* 1696 * The remaining case would be for RAID56, multiply by 1697 * nr_data_stripes(). Alternatively, just use rmap_len below 1698 * instead of map->stripe_len 1699 */ 1700 1701 bytenr = chunk_start + stripe_nr * io_stripe_size; 1702 1703 /* Ensure we don't add duplicate addresses */ 1704 for (j = 0; j < nr; j++) { 1705 if (buf[j] == bytenr) { 1706 already_inserted = true; 1707 break; 1708 } 1709 } 1710 1711 if (!already_inserted) 1712 buf[nr++] = bytenr; 1713 } 1714 1715 *logical = buf; 1716 *naddrs = nr; 1717 *stripe_len = io_stripe_size; 1718 out: 1719 free_extent_map(em); 1720 return ret; 1721 } 1722 1723 static int exclude_super_stripes(struct btrfs_block_group *cache) 1724 { 1725 struct btrfs_fs_info *fs_info = cache->fs_info; 1726 u64 bytenr; 1727 u64 *logical; 1728 int stripe_len; 1729 int i, nr, ret; 1730 1731 if (cache->start < BTRFS_SUPER_INFO_OFFSET) { 1732 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->start; 1733 cache->bytes_super += stripe_len; 1734 ret = btrfs_add_excluded_extent(fs_info, cache->start, 1735 stripe_len); 1736 if (ret) 1737 return ret; 1738 } 1739 1740 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 1741 bytenr = btrfs_sb_offset(i); 1742 ret = btrfs_rmap_block(fs_info, cache->start, 1743 bytenr, &logical, &nr, &stripe_len); 1744 if (ret) 1745 return ret; 1746 1747 while (nr--) { 1748 u64 len = min_t(u64, stripe_len, 1749 cache->start + cache->length - logical[nr]); 1750 1751 cache->bytes_super += len; 1752 ret = btrfs_add_excluded_extent(fs_info, logical[nr], 1753 len); 1754 if (ret) { 1755 kfree(logical); 1756 return ret; 1757 } 1758 } 1759 1760 kfree(logical); 1761 } 1762 return 0; 1763 } 1764 1765 static void link_block_group(struct btrfs_block_group *cache) 1766 { 1767 struct btrfs_space_info *space_info = cache->space_info; 1768 int index = btrfs_bg_flags_to_raid_index(cache->flags); 1769 bool first = false; 1770 1771 down_write(&space_info->groups_sem); 1772 if (list_empty(&space_info->block_groups[index])) 1773 first = true; 1774 list_add_tail(&cache->list, &space_info->block_groups[index]); 1775 up_write(&space_info->groups_sem); 1776 1777 if (first) 1778 btrfs_sysfs_add_block_group_type(cache); 1779 } 1780 1781 static struct btrfs_block_group *btrfs_create_block_group_cache( 1782 struct btrfs_fs_info *fs_info, u64 start) 1783 { 1784 struct btrfs_block_group *cache; 1785 1786 cache = kzalloc(sizeof(*cache), GFP_NOFS); 1787 if (!cache) 1788 return NULL; 1789 1790 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl), 1791 GFP_NOFS); 1792 if (!cache->free_space_ctl) { 1793 kfree(cache); 1794 return NULL; 1795 } 1796 1797 cache->start = start; 1798 1799 cache->fs_info = fs_info; 1800 cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start); 1801 set_free_space_tree_thresholds(cache); 1802 1803 cache->discard_index = BTRFS_DISCARD_INDEX_UNUSED; 1804 1805 refcount_set(&cache->refs, 1); 1806 spin_lock_init(&cache->lock); 1807 init_rwsem(&cache->data_rwsem); 1808 INIT_LIST_HEAD(&cache->list); 1809 INIT_LIST_HEAD(&cache->cluster_list); 1810 INIT_LIST_HEAD(&cache->bg_list); 1811 INIT_LIST_HEAD(&cache->ro_list); 1812 INIT_LIST_HEAD(&cache->discard_list); 1813 INIT_LIST_HEAD(&cache->dirty_list); 1814 INIT_LIST_HEAD(&cache->io_list); 1815 btrfs_init_free_space_ctl(cache); 1816 atomic_set(&cache->frozen, 0); 1817 mutex_init(&cache->free_space_lock); 1818 btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root); 1819 1820 return cache; 1821 } 1822 1823 /* 1824 * Iterate all chunks and verify that each of them has the corresponding block 1825 * group 1826 */ 1827 static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) 1828 { 1829 struct extent_map_tree *map_tree = &fs_info->mapping_tree; 1830 struct extent_map *em; 1831 struct btrfs_block_group *bg; 1832 u64 start = 0; 1833 int ret = 0; 1834 1835 while (1) { 1836 read_lock(&map_tree->lock); 1837 /* 1838 * lookup_extent_mapping will return the first extent map 1839 * intersecting the range, so setting @len to 1 is enough to 1840 * get the first chunk. 1841 */ 1842 em = lookup_extent_mapping(map_tree, start, 1); 1843 read_unlock(&map_tree->lock); 1844 if (!em) 1845 break; 1846 1847 bg = btrfs_lookup_block_group(fs_info, em->start); 1848 if (!bg) { 1849 btrfs_err(fs_info, 1850 "chunk start=%llu len=%llu doesn't have corresponding block group", 1851 em->start, em->len); 1852 ret = -EUCLEAN; 1853 free_extent_map(em); 1854 break; 1855 } 1856 if (bg->start != em->start || bg->length != em->len || 1857 (bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) != 1858 (em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK)) { 1859 btrfs_err(fs_info, 1860 "chunk start=%llu len=%llu flags=0x%llx doesn't match block group start=%llu len=%llu flags=0x%llx", 1861 em->start, em->len, 1862 em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK, 1863 bg->start, bg->length, 1864 bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK); 1865 ret = -EUCLEAN; 1866 free_extent_map(em); 1867 btrfs_put_block_group(bg); 1868 break; 1869 } 1870 start = em->start + em->len; 1871 free_extent_map(em); 1872 btrfs_put_block_group(bg); 1873 } 1874 return ret; 1875 } 1876 1877 static int read_block_group_item(struct btrfs_block_group *cache, 1878 struct btrfs_path *path, 1879 const struct btrfs_key *key) 1880 { 1881 struct extent_buffer *leaf = path->nodes[0]; 1882 struct btrfs_block_group_item bgi; 1883 int slot = path->slots[0]; 1884 1885 cache->length = key->offset; 1886 1887 read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), 1888 sizeof(bgi)); 1889 cache->used = btrfs_stack_block_group_used(&bgi); 1890 cache->flags = btrfs_stack_block_group_flags(&bgi); 1891 1892 return 0; 1893 } 1894 1895 static int read_one_block_group(struct btrfs_fs_info *info, 1896 struct btrfs_path *path, 1897 const struct btrfs_key *key, 1898 int need_clear) 1899 { 1900 struct btrfs_block_group *cache; 1901 struct btrfs_space_info *space_info; 1902 const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS); 1903 int ret; 1904 1905 ASSERT(key->type == BTRFS_BLOCK_GROUP_ITEM_KEY); 1906 1907 cache = btrfs_create_block_group_cache(info, key->objectid); 1908 if (!cache) 1909 return -ENOMEM; 1910 1911 ret = read_block_group_item(cache, path, key); 1912 if (ret < 0) 1913 goto error; 1914 1915 if (need_clear) { 1916 /* 1917 * When we mount with old space cache, we need to 1918 * set BTRFS_DC_CLEAR and set dirty flag. 1919 * 1920 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we 1921 * truncate the old free space cache inode and 1922 * setup a new one. 1923 * b) Setting 'dirty flag' makes sure that we flush 1924 * the new space cache info onto disk. 1925 */ 1926 if (btrfs_test_opt(info, SPACE_CACHE)) 1927 cache->disk_cache_state = BTRFS_DC_CLEAR; 1928 } 1929 if (!mixed && ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) && 1930 (cache->flags & BTRFS_BLOCK_GROUP_DATA))) { 1931 btrfs_err(info, 1932 "bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups", 1933 cache->start); 1934 ret = -EINVAL; 1935 goto error; 1936 } 1937 1938 /* 1939 * We need to exclude the super stripes now so that the space info has 1940 * super bytes accounted for, otherwise we'll think we have more space 1941 * than we actually do. 1942 */ 1943 ret = exclude_super_stripes(cache); 1944 if (ret) { 1945 /* We may have excluded something, so call this just in case. */ 1946 btrfs_free_excluded_extents(cache); 1947 goto error; 1948 } 1949 1950 /* 1951 * Check for two cases, either we are full, and therefore don't need 1952 * to bother with the caching work since we won't find any space, or we 1953 * are empty, and we can just add all the space in and be done with it. 1954 * This saves us _a_lot_ of time, particularly in the full case. 1955 */ 1956 if (cache->length == cache->used) { 1957 cache->last_byte_to_unpin = (u64)-1; 1958 cache->cached = BTRFS_CACHE_FINISHED; 1959 btrfs_free_excluded_extents(cache); 1960 } else if (cache->used == 0) { 1961 cache->last_byte_to_unpin = (u64)-1; 1962 cache->cached = BTRFS_CACHE_FINISHED; 1963 add_new_free_space(cache, cache->start, 1964 cache->start + cache->length); 1965 btrfs_free_excluded_extents(cache); 1966 } 1967 1968 ret = btrfs_add_block_group_cache(info, cache); 1969 if (ret) { 1970 btrfs_remove_free_space_cache(cache); 1971 goto error; 1972 } 1973 trace_btrfs_add_block_group(info, cache, 0); 1974 btrfs_update_space_info(info, cache->flags, cache->length, 1975 cache->used, cache->bytes_super, &space_info); 1976 1977 cache->space_info = space_info; 1978 1979 link_block_group(cache); 1980 1981 set_avail_alloc_bits(info, cache->flags); 1982 if (btrfs_chunk_readonly(info, cache->start)) { 1983 inc_block_group_ro(cache, 1); 1984 } else if (cache->used == 0) { 1985 ASSERT(list_empty(&cache->bg_list)); 1986 if (btrfs_test_opt(info, DISCARD_ASYNC)) 1987 btrfs_discard_queue_work(&info->discard_ctl, cache); 1988 else 1989 btrfs_mark_bg_unused(cache); 1990 } 1991 return 0; 1992 error: 1993 btrfs_put_block_group(cache); 1994 return ret; 1995 } 1996 1997 int btrfs_read_block_groups(struct btrfs_fs_info *info) 1998 { 1999 struct btrfs_path *path; 2000 int ret; 2001 struct btrfs_block_group *cache; 2002 struct btrfs_space_info *space_info; 2003 struct btrfs_key key; 2004 int need_clear = 0; 2005 u64 cache_gen; 2006 2007 key.objectid = 0; 2008 key.offset = 0; 2009 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 2010 path = btrfs_alloc_path(); 2011 if (!path) 2012 return -ENOMEM; 2013 2014 cache_gen = btrfs_super_cache_generation(info->super_copy); 2015 if (btrfs_test_opt(info, SPACE_CACHE) && 2016 btrfs_super_generation(info->super_copy) != cache_gen) 2017 need_clear = 1; 2018 if (btrfs_test_opt(info, CLEAR_CACHE)) 2019 need_clear = 1; 2020 2021 while (1) { 2022 ret = find_first_block_group(info, path, &key); 2023 if (ret > 0) 2024 break; 2025 if (ret != 0) 2026 goto error; 2027 2028 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); 2029 ret = read_one_block_group(info, path, &key, need_clear); 2030 if (ret < 0) 2031 goto error; 2032 key.objectid += key.offset; 2033 key.offset = 0; 2034 btrfs_release_path(path); 2035 } 2036 2037 rcu_read_lock(); 2038 list_for_each_entry_rcu(space_info, &info->space_info, list) { 2039 if (!(btrfs_get_alloc_profile(info, space_info->flags) & 2040 (BTRFS_BLOCK_GROUP_RAID10 | 2041 BTRFS_BLOCK_GROUP_RAID1_MASK | 2042 BTRFS_BLOCK_GROUP_RAID56_MASK | 2043 BTRFS_BLOCK_GROUP_DUP))) 2044 continue; 2045 /* 2046 * Avoid allocating from un-mirrored block group if there are 2047 * mirrored block groups. 2048 */ 2049 list_for_each_entry(cache, 2050 &space_info->block_groups[BTRFS_RAID_RAID0], 2051 list) 2052 inc_block_group_ro(cache, 1); 2053 list_for_each_entry(cache, 2054 &space_info->block_groups[BTRFS_RAID_SINGLE], 2055 list) 2056 inc_block_group_ro(cache, 1); 2057 } 2058 rcu_read_unlock(); 2059 2060 btrfs_init_global_block_rsv(info); 2061 ret = check_chunk_block_group_mappings(info); 2062 error: 2063 btrfs_free_path(path); 2064 return ret; 2065 } 2066 2067 static int insert_block_group_item(struct btrfs_trans_handle *trans, 2068 struct btrfs_block_group *block_group) 2069 { 2070 struct btrfs_fs_info *fs_info = trans->fs_info; 2071 struct btrfs_block_group_item bgi; 2072 struct btrfs_root *root; 2073 struct btrfs_key key; 2074 2075 spin_lock(&block_group->lock); 2076 btrfs_set_stack_block_group_used(&bgi, block_group->used); 2077 btrfs_set_stack_block_group_chunk_objectid(&bgi, 2078 BTRFS_FIRST_CHUNK_TREE_OBJECTID); 2079 btrfs_set_stack_block_group_flags(&bgi, block_group->flags); 2080 key.objectid = block_group->start; 2081 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 2082 key.offset = block_group->length; 2083 spin_unlock(&block_group->lock); 2084 2085 root = fs_info->extent_root; 2086 return btrfs_insert_item(trans, root, &key, &bgi, sizeof(bgi)); 2087 } 2088 2089 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans) 2090 { 2091 struct btrfs_fs_info *fs_info = trans->fs_info; 2092 struct btrfs_block_group *block_group; 2093 int ret = 0; 2094 2095 if (!trans->can_flush_pending_bgs) 2096 return; 2097 2098 while (!list_empty(&trans->new_bgs)) { 2099 block_group = list_first_entry(&trans->new_bgs, 2100 struct btrfs_block_group, 2101 bg_list); 2102 if (ret) 2103 goto next; 2104 2105 ret = insert_block_group_item(trans, block_group); 2106 if (ret) 2107 btrfs_abort_transaction(trans, ret); 2108 ret = btrfs_finish_chunk_alloc(trans, block_group->start, 2109 block_group->length); 2110 if (ret) 2111 btrfs_abort_transaction(trans, ret); 2112 add_block_group_free_space(trans, block_group); 2113 /* Already aborted the transaction if it failed. */ 2114 next: 2115 btrfs_delayed_refs_rsv_release(fs_info, 1); 2116 list_del_init(&block_group->bg_list); 2117 } 2118 btrfs_trans_release_chunk_metadata(trans); 2119 } 2120 2121 int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, 2122 u64 type, u64 chunk_offset, u64 size) 2123 { 2124 struct btrfs_fs_info *fs_info = trans->fs_info; 2125 struct btrfs_block_group *cache; 2126 int ret; 2127 2128 btrfs_set_log_full_commit(trans); 2129 2130 cache = btrfs_create_block_group_cache(fs_info, chunk_offset); 2131 if (!cache) 2132 return -ENOMEM; 2133 2134 cache->length = size; 2135 cache->used = bytes_used; 2136 cache->flags = type; 2137 cache->last_byte_to_unpin = (u64)-1; 2138 cache->cached = BTRFS_CACHE_FINISHED; 2139 cache->needs_free_space = 1; 2140 ret = exclude_super_stripes(cache); 2141 if (ret) { 2142 /* We may have excluded something, so call this just in case */ 2143 btrfs_free_excluded_extents(cache); 2144 btrfs_put_block_group(cache); 2145 return ret; 2146 } 2147 2148 add_new_free_space(cache, chunk_offset, chunk_offset + size); 2149 2150 btrfs_free_excluded_extents(cache); 2151 2152 #ifdef CONFIG_BTRFS_DEBUG 2153 if (btrfs_should_fragment_free_space(cache)) { 2154 u64 new_bytes_used = size - bytes_used; 2155 2156 bytes_used += new_bytes_used >> 1; 2157 fragment_free_space(cache); 2158 } 2159 #endif 2160 /* 2161 * Ensure the corresponding space_info object is created and 2162 * assigned to our block group. We want our bg to be added to the rbtree 2163 * with its ->space_info set. 2164 */ 2165 cache->space_info = btrfs_find_space_info(fs_info, cache->flags); 2166 ASSERT(cache->space_info); 2167 2168 ret = btrfs_add_block_group_cache(fs_info, cache); 2169 if (ret) { 2170 btrfs_remove_free_space_cache(cache); 2171 btrfs_put_block_group(cache); 2172 return ret; 2173 } 2174 2175 /* 2176 * Now that our block group has its ->space_info set and is inserted in 2177 * the rbtree, update the space info's counters. 2178 */ 2179 trace_btrfs_add_block_group(fs_info, cache, 1); 2180 btrfs_update_space_info(fs_info, cache->flags, size, bytes_used, 2181 cache->bytes_super, &cache->space_info); 2182 btrfs_update_global_block_rsv(fs_info); 2183 2184 link_block_group(cache); 2185 2186 list_add_tail(&cache->bg_list, &trans->new_bgs); 2187 trans->delayed_ref_updates++; 2188 btrfs_update_delayed_refs_rsv(trans); 2189 2190 set_avail_alloc_bits(fs_info, type); 2191 return 0; 2192 } 2193 2194 /* 2195 * Mark one block group RO, can be called several times for the same block 2196 * group. 2197 * 2198 * @cache: the destination block group 2199 * @do_chunk_alloc: whether need to do chunk pre-allocation, this is to 2200 * ensure we still have some free space after marking this 2201 * block group RO. 2202 */ 2203 int btrfs_inc_block_group_ro(struct btrfs_block_group *cache, 2204 bool do_chunk_alloc) 2205 { 2206 struct btrfs_fs_info *fs_info = cache->fs_info; 2207 struct btrfs_trans_handle *trans; 2208 u64 alloc_flags; 2209 int ret; 2210 2211 again: 2212 trans = btrfs_join_transaction(fs_info->extent_root); 2213 if (IS_ERR(trans)) 2214 return PTR_ERR(trans); 2215 2216 /* 2217 * we're not allowed to set block groups readonly after the dirty 2218 * block groups cache has started writing. If it already started, 2219 * back off and let this transaction commit 2220 */ 2221 mutex_lock(&fs_info->ro_block_group_mutex); 2222 if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) { 2223 u64 transid = trans->transid; 2224 2225 mutex_unlock(&fs_info->ro_block_group_mutex); 2226 btrfs_end_transaction(trans); 2227 2228 ret = btrfs_wait_for_commit(fs_info, transid); 2229 if (ret) 2230 return ret; 2231 goto again; 2232 } 2233 2234 if (do_chunk_alloc) { 2235 /* 2236 * If we are changing raid levels, try to allocate a 2237 * corresponding block group with the new raid level. 2238 */ 2239 alloc_flags = btrfs_get_alloc_profile(fs_info, cache->flags); 2240 if (alloc_flags != cache->flags) { 2241 ret = btrfs_chunk_alloc(trans, alloc_flags, 2242 CHUNK_ALLOC_FORCE); 2243 /* 2244 * ENOSPC is allowed here, we may have enough space 2245 * already allocated at the new raid level to carry on 2246 */ 2247 if (ret == -ENOSPC) 2248 ret = 0; 2249 if (ret < 0) 2250 goto out; 2251 } 2252 } 2253 2254 ret = inc_block_group_ro(cache, 0); 2255 if (!do_chunk_alloc) 2256 goto unlock_out; 2257 if (!ret) 2258 goto out; 2259 alloc_flags = btrfs_get_alloc_profile(fs_info, cache->space_info->flags); 2260 ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE); 2261 if (ret < 0) 2262 goto out; 2263 ret = inc_block_group_ro(cache, 0); 2264 out: 2265 if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) { 2266 alloc_flags = btrfs_get_alloc_profile(fs_info, cache->flags); 2267 mutex_lock(&fs_info->chunk_mutex); 2268 check_system_chunk(trans, alloc_flags); 2269 mutex_unlock(&fs_info->chunk_mutex); 2270 } 2271 unlock_out: 2272 mutex_unlock(&fs_info->ro_block_group_mutex); 2273 2274 btrfs_end_transaction(trans); 2275 return ret; 2276 } 2277 2278 void btrfs_dec_block_group_ro(struct btrfs_block_group *cache) 2279 { 2280 struct btrfs_space_info *sinfo = cache->space_info; 2281 u64 num_bytes; 2282 2283 BUG_ON(!cache->ro); 2284 2285 spin_lock(&sinfo->lock); 2286 spin_lock(&cache->lock); 2287 if (!--cache->ro) { 2288 num_bytes = cache->length - cache->reserved - 2289 cache->pinned - cache->bytes_super - cache->used; 2290 sinfo->bytes_readonly -= num_bytes; 2291 list_del_init(&cache->ro_list); 2292 } 2293 spin_unlock(&cache->lock); 2294 spin_unlock(&sinfo->lock); 2295 } 2296 2297 static int update_block_group_item(struct btrfs_trans_handle *trans, 2298 struct btrfs_path *path, 2299 struct btrfs_block_group *cache) 2300 { 2301 struct btrfs_fs_info *fs_info = trans->fs_info; 2302 int ret; 2303 struct btrfs_root *root = fs_info->extent_root; 2304 unsigned long bi; 2305 struct extent_buffer *leaf; 2306 struct btrfs_block_group_item bgi; 2307 struct btrfs_key key; 2308 2309 key.objectid = cache->start; 2310 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; 2311 key.offset = cache->length; 2312 2313 ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 2314 if (ret) { 2315 if (ret > 0) 2316 ret = -ENOENT; 2317 goto fail; 2318 } 2319 2320 leaf = path->nodes[0]; 2321 bi = btrfs_item_ptr_offset(leaf, path->slots[0]); 2322 btrfs_set_stack_block_group_used(&bgi, cache->used); 2323 btrfs_set_stack_block_group_chunk_objectid(&bgi, 2324 BTRFS_FIRST_CHUNK_TREE_OBJECTID); 2325 btrfs_set_stack_block_group_flags(&bgi, cache->flags); 2326 write_extent_buffer(leaf, &bgi, bi, sizeof(bgi)); 2327 btrfs_mark_buffer_dirty(leaf); 2328 fail: 2329 btrfs_release_path(path); 2330 return ret; 2331 2332 } 2333 2334 static int cache_save_setup(struct btrfs_block_group *block_group, 2335 struct btrfs_trans_handle *trans, 2336 struct btrfs_path *path) 2337 { 2338 struct btrfs_fs_info *fs_info = block_group->fs_info; 2339 struct btrfs_root *root = fs_info->tree_root; 2340 struct inode *inode = NULL; 2341 struct extent_changeset *data_reserved = NULL; 2342 u64 alloc_hint = 0; 2343 int dcs = BTRFS_DC_ERROR; 2344 u64 num_pages = 0; 2345 int retries = 0; 2346 int ret = 0; 2347 2348 /* 2349 * If this block group is smaller than 100 megs don't bother caching the 2350 * block group. 2351 */ 2352 if (block_group->length < (100 * SZ_1M)) { 2353 spin_lock(&block_group->lock); 2354 block_group->disk_cache_state = BTRFS_DC_WRITTEN; 2355 spin_unlock(&block_group->lock); 2356 return 0; 2357 } 2358 2359 if (TRANS_ABORTED(trans)) 2360 return 0; 2361 again: 2362 inode = lookup_free_space_inode(block_group, path); 2363 if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) { 2364 ret = PTR_ERR(inode); 2365 btrfs_release_path(path); 2366 goto out; 2367 } 2368 2369 if (IS_ERR(inode)) { 2370 BUG_ON(retries); 2371 retries++; 2372 2373 if (block_group->ro) 2374 goto out_free; 2375 2376 ret = create_free_space_inode(trans, block_group, path); 2377 if (ret) 2378 goto out_free; 2379 goto again; 2380 } 2381 2382 /* 2383 * We want to set the generation to 0, that way if anything goes wrong 2384 * from here on out we know not to trust this cache when we load up next 2385 * time. 2386 */ 2387 BTRFS_I(inode)->generation = 0; 2388 ret = btrfs_update_inode(trans, root, inode); 2389 if (ret) { 2390 /* 2391 * So theoretically we could recover from this, simply set the 2392 * super cache generation to 0 so we know to invalidate the 2393 * cache, but then we'd have to keep track of the block groups 2394 * that fail this way so we know we _have_ to reset this cache 2395 * before the next commit or risk reading stale cache. So to 2396 * limit our exposure to horrible edge cases lets just abort the 2397 * transaction, this only happens in really bad situations 2398 * anyway. 2399 */ 2400 btrfs_abort_transaction(trans, ret); 2401 goto out_put; 2402 } 2403 WARN_ON(ret); 2404 2405 /* We've already setup this transaction, go ahead and exit */ 2406 if (block_group->cache_generation == trans->transid && 2407 i_size_read(inode)) { 2408 dcs = BTRFS_DC_SETUP; 2409 goto out_put; 2410 } 2411 2412 if (i_size_read(inode) > 0) { 2413 ret = btrfs_check_trunc_cache_free_space(fs_info, 2414 &fs_info->global_block_rsv); 2415 if (ret) 2416 goto out_put; 2417 2418 ret = btrfs_truncate_free_space_cache(trans, NULL, inode); 2419 if (ret) 2420 goto out_put; 2421 } 2422 2423 spin_lock(&block_group->lock); 2424 if (block_group->cached != BTRFS_CACHE_FINISHED || 2425 !btrfs_test_opt(fs_info, SPACE_CACHE)) { 2426 /* 2427 * don't bother trying to write stuff out _if_ 2428 * a) we're not cached, 2429 * b) we're with nospace_cache mount option, 2430 * c) we're with v2 space_cache (FREE_SPACE_TREE). 2431 */ 2432 dcs = BTRFS_DC_WRITTEN; 2433 spin_unlock(&block_group->lock); 2434 goto out_put; 2435 } 2436 spin_unlock(&block_group->lock); 2437 2438 /* 2439 * We hit an ENOSPC when setting up the cache in this transaction, just 2440 * skip doing the setup, we've already cleared the cache so we're safe. 2441 */ 2442 if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) { 2443 ret = -ENOSPC; 2444 goto out_put; 2445 } 2446 2447 /* 2448 * Try to preallocate enough space based on how big the block group is. 2449 * Keep in mind this has to include any pinned space which could end up 2450 * taking up quite a bit since it's not folded into the other space 2451 * cache. 2452 */ 2453 num_pages = div_u64(block_group->length, SZ_256M); 2454 if (!num_pages) 2455 num_pages = 1; 2456 2457 num_pages *= 16; 2458 num_pages *= PAGE_SIZE; 2459 2460 ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 0, 2461 num_pages); 2462 if (ret) 2463 goto out_put; 2464 2465 ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages, 2466 num_pages, num_pages, 2467 &alloc_hint); 2468 /* 2469 * Our cache requires contiguous chunks so that we don't modify a bunch 2470 * of metadata or split extents when writing the cache out, which means 2471 * we can enospc if we are heavily fragmented in addition to just normal 2472 * out of space conditions. So if we hit this just skip setting up any 2473 * other block groups for this transaction, maybe we'll unpin enough 2474 * space the next time around. 2475 */ 2476 if (!ret) 2477 dcs = BTRFS_DC_SETUP; 2478 else if (ret == -ENOSPC) 2479 set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags); 2480 2481 out_put: 2482 iput(inode); 2483 out_free: 2484 btrfs_release_path(path); 2485 out: 2486 spin_lock(&block_group->lock); 2487 if (!ret && dcs == BTRFS_DC_SETUP) 2488 block_group->cache_generation = trans->transid; 2489 block_group->disk_cache_state = dcs; 2490 spin_unlock(&block_group->lock); 2491 2492 extent_changeset_free(data_reserved); 2493 return ret; 2494 } 2495 2496 int btrfs_setup_space_cache(struct btrfs_trans_handle *trans) 2497 { 2498 struct btrfs_fs_info *fs_info = trans->fs_info; 2499 struct btrfs_block_group *cache, *tmp; 2500 struct btrfs_transaction *cur_trans = trans->transaction; 2501 struct btrfs_path *path; 2502 2503 if (list_empty(&cur_trans->dirty_bgs) || 2504 !btrfs_test_opt(fs_info, SPACE_CACHE)) 2505 return 0; 2506 2507 path = btrfs_alloc_path(); 2508 if (!path) 2509 return -ENOMEM; 2510 2511 /* Could add new block groups, use _safe just in case */ 2512 list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs, 2513 dirty_list) { 2514 if (cache->disk_cache_state == BTRFS_DC_CLEAR) 2515 cache_save_setup(cache, trans, path); 2516 } 2517 2518 btrfs_free_path(path); 2519 return 0; 2520 } 2521 2522 /* 2523 * Transaction commit does final block group cache writeback during a critical 2524 * section where nothing is allowed to change the FS. This is required in 2525 * order for the cache to actually match the block group, but can introduce a 2526 * lot of latency into the commit. 2527 * 2528 * So, btrfs_start_dirty_block_groups is here to kick off block group cache IO. 2529 * There's a chance we'll have to redo some of it if the block group changes 2530 * again during the commit, but it greatly reduces the commit latency by 2531 * getting rid of the easy block groups while we're still allowing others to 2532 * join the commit. 2533 */ 2534 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans) 2535 { 2536 struct btrfs_fs_info *fs_info = trans->fs_info; 2537 struct btrfs_block_group *cache; 2538 struct btrfs_transaction *cur_trans = trans->transaction; 2539 int ret = 0; 2540 int should_put; 2541 struct btrfs_path *path = NULL; 2542 LIST_HEAD(dirty); 2543 struct list_head *io = &cur_trans->io_bgs; 2544 int num_started = 0; 2545 int loops = 0; 2546 2547 spin_lock(&cur_trans->dirty_bgs_lock); 2548 if (list_empty(&cur_trans->dirty_bgs)) { 2549 spin_unlock(&cur_trans->dirty_bgs_lock); 2550 return 0; 2551 } 2552 list_splice_init(&cur_trans->dirty_bgs, &dirty); 2553 spin_unlock(&cur_trans->dirty_bgs_lock); 2554 2555 again: 2556 /* Make sure all the block groups on our dirty list actually exist */ 2557 btrfs_create_pending_block_groups(trans); 2558 2559 if (!path) { 2560 path = btrfs_alloc_path(); 2561 if (!path) 2562 return -ENOMEM; 2563 } 2564 2565 /* 2566 * cache_write_mutex is here only to save us from balance or automatic 2567 * removal of empty block groups deleting this block group while we are 2568 * writing out the cache 2569 */ 2570 mutex_lock(&trans->transaction->cache_write_mutex); 2571 while (!list_empty(&dirty)) { 2572 bool drop_reserve = true; 2573 2574 cache = list_first_entry(&dirty, struct btrfs_block_group, 2575 dirty_list); 2576 /* 2577 * This can happen if something re-dirties a block group that 2578 * is already under IO. Just wait for it to finish and then do 2579 * it all again 2580 */ 2581 if (!list_empty(&cache->io_list)) { 2582 list_del_init(&cache->io_list); 2583 btrfs_wait_cache_io(trans, cache, path); 2584 btrfs_put_block_group(cache); 2585 } 2586 2587 2588 /* 2589 * btrfs_wait_cache_io uses the cache->dirty_list to decide if 2590 * it should update the cache_state. Don't delete until after 2591 * we wait. 2592 * 2593 * Since we're not running in the commit critical section 2594 * we need the dirty_bgs_lock to protect from update_block_group 2595 */ 2596 spin_lock(&cur_trans->dirty_bgs_lock); 2597 list_del_init(&cache->dirty_list); 2598 spin_unlock(&cur_trans->dirty_bgs_lock); 2599 2600 should_put = 1; 2601 2602 cache_save_setup(cache, trans, path); 2603 2604 if (cache->disk_cache_state == BTRFS_DC_SETUP) { 2605 cache->io_ctl.inode = NULL; 2606 ret = btrfs_write_out_cache(trans, cache, path); 2607 if (ret == 0 && cache->io_ctl.inode) { 2608 num_started++; 2609 should_put = 0; 2610 2611 /* 2612 * The cache_write_mutex is protecting the 2613 * io_list, also refer to the definition of 2614 * btrfs_transaction::io_bgs for more details 2615 */ 2616 list_add_tail(&cache->io_list, io); 2617 } else { 2618 /* 2619 * If we failed to write the cache, the 2620 * generation will be bad and life goes on 2621 */ 2622 ret = 0; 2623 } 2624 } 2625 if (!ret) { 2626 ret = update_block_group_item(trans, path, cache); 2627 /* 2628 * Our block group might still be attached to the list 2629 * of new block groups in the transaction handle of some 2630 * other task (struct btrfs_trans_handle->new_bgs). This 2631 * means its block group item isn't yet in the extent 2632 * tree. If this happens ignore the error, as we will 2633 * try again later in the critical section of the 2634 * transaction commit. 2635 */ 2636 if (ret == -ENOENT) { 2637 ret = 0; 2638 spin_lock(&cur_trans->dirty_bgs_lock); 2639 if (list_empty(&cache->dirty_list)) { 2640 list_add_tail(&cache->dirty_list, 2641 &cur_trans->dirty_bgs); 2642 btrfs_get_block_group(cache); 2643 drop_reserve = false; 2644 } 2645 spin_unlock(&cur_trans->dirty_bgs_lock); 2646 } else if (ret) { 2647 btrfs_abort_transaction(trans, ret); 2648 } 2649 } 2650 2651 /* If it's not on the io list, we need to put the block group */ 2652 if (should_put) 2653 btrfs_put_block_group(cache); 2654 if (drop_reserve) 2655 btrfs_delayed_refs_rsv_release(fs_info, 1); 2656 2657 if (ret) 2658 break; 2659 2660 /* 2661 * Avoid blocking other tasks for too long. It might even save 2662 * us from writing caches for block groups that are going to be 2663 * removed. 2664 */ 2665 mutex_unlock(&trans->transaction->cache_write_mutex); 2666 mutex_lock(&trans->transaction->cache_write_mutex); 2667 } 2668 mutex_unlock(&trans->transaction->cache_write_mutex); 2669 2670 /* 2671 * Go through delayed refs for all the stuff we've just kicked off 2672 * and then loop back (just once) 2673 */ 2674 ret = btrfs_run_delayed_refs(trans, 0); 2675 if (!ret && loops == 0) { 2676 loops++; 2677 spin_lock(&cur_trans->dirty_bgs_lock); 2678 list_splice_init(&cur_trans->dirty_bgs, &dirty); 2679 /* 2680 * dirty_bgs_lock protects us from concurrent block group 2681 * deletes too (not just cache_write_mutex). 2682 */ 2683 if (!list_empty(&dirty)) { 2684 spin_unlock(&cur_trans->dirty_bgs_lock); 2685 goto again; 2686 } 2687 spin_unlock(&cur_trans->dirty_bgs_lock); 2688 } else if (ret < 0) { 2689 btrfs_cleanup_dirty_bgs(cur_trans, fs_info); 2690 } 2691 2692 btrfs_free_path(path); 2693 return ret; 2694 } 2695 2696 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) 2697 { 2698 struct btrfs_fs_info *fs_info = trans->fs_info; 2699 struct btrfs_block_group *cache; 2700 struct btrfs_transaction *cur_trans = trans->transaction; 2701 int ret = 0; 2702 int should_put; 2703 struct btrfs_path *path; 2704 struct list_head *io = &cur_trans->io_bgs; 2705 int num_started = 0; 2706 2707 path = btrfs_alloc_path(); 2708 if (!path) 2709 return -ENOMEM; 2710 2711 /* 2712 * Even though we are in the critical section of the transaction commit, 2713 * we can still have concurrent tasks adding elements to this 2714 * transaction's list of dirty block groups. These tasks correspond to 2715 * endio free space workers started when writeback finishes for a 2716 * space cache, which run inode.c:btrfs_finish_ordered_io(), and can 2717 * allocate new block groups as a result of COWing nodes of the root 2718 * tree when updating the free space inode. The writeback for the space 2719 * caches is triggered by an earlier call to 2720 * btrfs_start_dirty_block_groups() and iterations of the following 2721 * loop. 2722 * Also we want to do the cache_save_setup first and then run the 2723 * delayed refs to make sure we have the best chance at doing this all 2724 * in one shot. 2725 */ 2726 spin_lock(&cur_trans->dirty_bgs_lock); 2727 while (!list_empty(&cur_trans->dirty_bgs)) { 2728 cache = list_first_entry(&cur_trans->dirty_bgs, 2729 struct btrfs_block_group, 2730 dirty_list); 2731 2732 /* 2733 * This can happen if cache_save_setup re-dirties a block group 2734 * that is already under IO. Just wait for it to finish and 2735 * then do it all again 2736 */ 2737 if (!list_empty(&cache->io_list)) { 2738 spin_unlock(&cur_trans->dirty_bgs_lock); 2739 list_del_init(&cache->io_list); 2740 btrfs_wait_cache_io(trans, cache, path); 2741 btrfs_put_block_group(cache); 2742 spin_lock(&cur_trans->dirty_bgs_lock); 2743 } 2744 2745 /* 2746 * Don't remove from the dirty list until after we've waited on 2747 * any pending IO 2748 */ 2749 list_del_init(&cache->dirty_list); 2750 spin_unlock(&cur_trans->dirty_bgs_lock); 2751 should_put = 1; 2752 2753 cache_save_setup(cache, trans, path); 2754 2755 if (!ret) 2756 ret = btrfs_run_delayed_refs(trans, 2757 (unsigned long) -1); 2758 2759 if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) { 2760 cache->io_ctl.inode = NULL; 2761 ret = btrfs_write_out_cache(trans, cache, path); 2762 if (ret == 0 && cache->io_ctl.inode) { 2763 num_started++; 2764 should_put = 0; 2765 list_add_tail(&cache->io_list, io); 2766 } else { 2767 /* 2768 * If we failed to write the cache, the 2769 * generation will be bad and life goes on 2770 */ 2771 ret = 0; 2772 } 2773 } 2774 if (!ret) { 2775 ret = update_block_group_item(trans, path, cache); 2776 /* 2777 * One of the free space endio workers might have 2778 * created a new block group while updating a free space 2779 * cache's inode (at inode.c:btrfs_finish_ordered_io()) 2780 * and hasn't released its transaction handle yet, in 2781 * which case the new block group is still attached to 2782 * its transaction handle and its creation has not 2783 * finished yet (no block group item in the extent tree 2784 * yet, etc). If this is the case, wait for all free 2785 * space endio workers to finish and retry. This is a 2786 * a very rare case so no need for a more efficient and 2787 * complex approach. 2788 */ 2789 if (ret == -ENOENT) { 2790 wait_event(cur_trans->writer_wait, 2791 atomic_read(&cur_trans->num_writers) == 1); 2792 ret = update_block_group_item(trans, path, cache); 2793 } 2794 if (ret) 2795 btrfs_abort_transaction(trans, ret); 2796 } 2797 2798 /* If its not on the io list, we need to put the block group */ 2799 if (should_put) 2800 btrfs_put_block_group(cache); 2801 btrfs_delayed_refs_rsv_release(fs_info, 1); 2802 spin_lock(&cur_trans->dirty_bgs_lock); 2803 } 2804 spin_unlock(&cur_trans->dirty_bgs_lock); 2805 2806 /* 2807 * Refer to the definition of io_bgs member for details why it's safe 2808 * to use it without any locking 2809 */ 2810 while (!list_empty(io)) { 2811 cache = list_first_entry(io, struct btrfs_block_group, 2812 io_list); 2813 list_del_init(&cache->io_list); 2814 btrfs_wait_cache_io(trans, cache, path); 2815 btrfs_put_block_group(cache); 2816 } 2817 2818 btrfs_free_path(path); 2819 return ret; 2820 } 2821 2822 int btrfs_update_block_group(struct btrfs_trans_handle *trans, 2823 u64 bytenr, u64 num_bytes, int alloc) 2824 { 2825 struct btrfs_fs_info *info = trans->fs_info; 2826 struct btrfs_block_group *cache = NULL; 2827 u64 total = num_bytes; 2828 u64 old_val; 2829 u64 byte_in_group; 2830 int factor; 2831 int ret = 0; 2832 2833 /* Block accounting for super block */ 2834 spin_lock(&info->delalloc_root_lock); 2835 old_val = btrfs_super_bytes_used(info->super_copy); 2836 if (alloc) 2837 old_val += num_bytes; 2838 else 2839 old_val -= num_bytes; 2840 btrfs_set_super_bytes_used(info->super_copy, old_val); 2841 spin_unlock(&info->delalloc_root_lock); 2842 2843 while (total) { 2844 cache = btrfs_lookup_block_group(info, bytenr); 2845 if (!cache) { 2846 ret = -ENOENT; 2847 break; 2848 } 2849 factor = btrfs_bg_type_to_factor(cache->flags); 2850 2851 /* 2852 * If this block group has free space cache written out, we 2853 * need to make sure to load it if we are removing space. This 2854 * is because we need the unpinning stage to actually add the 2855 * space back to the block group, otherwise we will leak space. 2856 */ 2857 if (!alloc && !btrfs_block_group_done(cache)) 2858 btrfs_cache_block_group(cache, 1); 2859 2860 byte_in_group = bytenr - cache->start; 2861 WARN_ON(byte_in_group > cache->length); 2862 2863 spin_lock(&cache->space_info->lock); 2864 spin_lock(&cache->lock); 2865 2866 if (btrfs_test_opt(info, SPACE_CACHE) && 2867 cache->disk_cache_state < BTRFS_DC_CLEAR) 2868 cache->disk_cache_state = BTRFS_DC_CLEAR; 2869 2870 old_val = cache->used; 2871 num_bytes = min(total, cache->length - byte_in_group); 2872 if (alloc) { 2873 old_val += num_bytes; 2874 cache->used = old_val; 2875 cache->reserved -= num_bytes; 2876 cache->space_info->bytes_reserved -= num_bytes; 2877 cache->space_info->bytes_used += num_bytes; 2878 cache->space_info->disk_used += num_bytes * factor; 2879 spin_unlock(&cache->lock); 2880 spin_unlock(&cache->space_info->lock); 2881 } else { 2882 old_val -= num_bytes; 2883 cache->used = old_val; 2884 cache->pinned += num_bytes; 2885 btrfs_space_info_update_bytes_pinned(info, 2886 cache->space_info, num_bytes); 2887 cache->space_info->bytes_used -= num_bytes; 2888 cache->space_info->disk_used -= num_bytes * factor; 2889 spin_unlock(&cache->lock); 2890 spin_unlock(&cache->space_info->lock); 2891 2892 percpu_counter_add_batch( 2893 &cache->space_info->total_bytes_pinned, 2894 num_bytes, 2895 BTRFS_TOTAL_BYTES_PINNED_BATCH); 2896 set_extent_dirty(&trans->transaction->pinned_extents, 2897 bytenr, bytenr + num_bytes - 1, 2898 GFP_NOFS | __GFP_NOFAIL); 2899 } 2900 2901 spin_lock(&trans->transaction->dirty_bgs_lock); 2902 if (list_empty(&cache->dirty_list)) { 2903 list_add_tail(&cache->dirty_list, 2904 &trans->transaction->dirty_bgs); 2905 trans->delayed_ref_updates++; 2906 btrfs_get_block_group(cache); 2907 } 2908 spin_unlock(&trans->transaction->dirty_bgs_lock); 2909 2910 /* 2911 * No longer have used bytes in this block group, queue it for 2912 * deletion. We do this after adding the block group to the 2913 * dirty list to avoid races between cleaner kthread and space 2914 * cache writeout. 2915 */ 2916 if (!alloc && old_val == 0) { 2917 if (!btrfs_test_opt(info, DISCARD_ASYNC)) 2918 btrfs_mark_bg_unused(cache); 2919 } 2920 2921 btrfs_put_block_group(cache); 2922 total -= num_bytes; 2923 bytenr += num_bytes; 2924 } 2925 2926 /* Modified block groups are accounted for in the delayed_refs_rsv. */ 2927 btrfs_update_delayed_refs_rsv(trans); 2928 return ret; 2929 } 2930 2931 /** 2932 * btrfs_add_reserved_bytes - update the block_group and space info counters 2933 * @cache: The cache we are manipulating 2934 * @ram_bytes: The number of bytes of file content, and will be same to 2935 * @num_bytes except for the compress path. 2936 * @num_bytes: The number of bytes in question 2937 * @delalloc: The blocks are allocated for the delalloc write 2938 * 2939 * This is called by the allocator when it reserves space. If this is a 2940 * reservation and the block group has become read only we cannot make the 2941 * reservation and return -EAGAIN, otherwise this function always succeeds. 2942 */ 2943 int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, 2944 u64 ram_bytes, u64 num_bytes, int delalloc) 2945 { 2946 struct btrfs_space_info *space_info = cache->space_info; 2947 int ret = 0; 2948 2949 spin_lock(&space_info->lock); 2950 spin_lock(&cache->lock); 2951 if (cache->ro) { 2952 ret = -EAGAIN; 2953 } else { 2954 cache->reserved += num_bytes; 2955 space_info->bytes_reserved += num_bytes; 2956 trace_btrfs_space_reservation(cache->fs_info, "space_info", 2957 space_info->flags, num_bytes, 1); 2958 btrfs_space_info_update_bytes_may_use(cache->fs_info, 2959 space_info, -ram_bytes); 2960 if (delalloc) 2961 cache->delalloc_bytes += num_bytes; 2962 } 2963 spin_unlock(&cache->lock); 2964 spin_unlock(&space_info->lock); 2965 return ret; 2966 } 2967 2968 /** 2969 * btrfs_free_reserved_bytes - update the block_group and space info counters 2970 * @cache: The cache we are manipulating 2971 * @num_bytes: The number of bytes in question 2972 * @delalloc: The blocks are allocated for the delalloc write 2973 * 2974 * This is called by somebody who is freeing space that was never actually used 2975 * on disk. For example if you reserve some space for a new leaf in transaction 2976 * A and before transaction A commits you free that leaf, you call this with 2977 * reserve set to 0 in order to clear the reservation. 2978 */ 2979 void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, 2980 u64 num_bytes, int delalloc) 2981 { 2982 struct btrfs_space_info *space_info = cache->space_info; 2983 2984 spin_lock(&space_info->lock); 2985 spin_lock(&cache->lock); 2986 if (cache->ro) 2987 space_info->bytes_readonly += num_bytes; 2988 cache->reserved -= num_bytes; 2989 space_info->bytes_reserved -= num_bytes; 2990 space_info->max_extent_size = 0; 2991 2992 if (delalloc) 2993 cache->delalloc_bytes -= num_bytes; 2994 spin_unlock(&cache->lock); 2995 spin_unlock(&space_info->lock); 2996 } 2997 2998 static void force_metadata_allocation(struct btrfs_fs_info *info) 2999 { 3000 struct list_head *head = &info->space_info; 3001 struct btrfs_space_info *found; 3002 3003 rcu_read_lock(); 3004 list_for_each_entry_rcu(found, head, list) { 3005 if (found->flags & BTRFS_BLOCK_GROUP_METADATA) 3006 found->force_alloc = CHUNK_ALLOC_FORCE; 3007 } 3008 rcu_read_unlock(); 3009 } 3010 3011 static int should_alloc_chunk(struct btrfs_fs_info *fs_info, 3012 struct btrfs_space_info *sinfo, int force) 3013 { 3014 u64 bytes_used = btrfs_space_info_used(sinfo, false); 3015 u64 thresh; 3016 3017 if (force == CHUNK_ALLOC_FORCE) 3018 return 1; 3019 3020 /* 3021 * in limited mode, we want to have some free space up to 3022 * about 1% of the FS size. 3023 */ 3024 if (force == CHUNK_ALLOC_LIMITED) { 3025 thresh = btrfs_super_total_bytes(fs_info->super_copy); 3026 thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1)); 3027 3028 if (sinfo->total_bytes - bytes_used < thresh) 3029 return 1; 3030 } 3031 3032 if (bytes_used + SZ_2M < div_factor(sinfo->total_bytes, 8)) 3033 return 0; 3034 return 1; 3035 } 3036 3037 int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type) 3038 { 3039 u64 alloc_flags = btrfs_get_alloc_profile(trans->fs_info, type); 3040 3041 return btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE); 3042 } 3043 3044 /* 3045 * If force is CHUNK_ALLOC_FORCE: 3046 * - return 1 if it successfully allocates a chunk, 3047 * - return errors including -ENOSPC otherwise. 3048 * If force is NOT CHUNK_ALLOC_FORCE: 3049 * - return 0 if it doesn't need to allocate a new chunk, 3050 * - return 1 if it successfully allocates a chunk, 3051 * - return errors including -ENOSPC otherwise. 3052 */ 3053 int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, 3054 enum btrfs_chunk_alloc_enum force) 3055 { 3056 struct btrfs_fs_info *fs_info = trans->fs_info; 3057 struct btrfs_space_info *space_info; 3058 bool wait_for_alloc = false; 3059 bool should_alloc = false; 3060 int ret = 0; 3061 3062 /* Don't re-enter if we're already allocating a chunk */ 3063 if (trans->allocating_chunk) 3064 return -ENOSPC; 3065 3066 space_info = btrfs_find_space_info(fs_info, flags); 3067 ASSERT(space_info); 3068 3069 do { 3070 spin_lock(&space_info->lock); 3071 if (force < space_info->force_alloc) 3072 force = space_info->force_alloc; 3073 should_alloc = should_alloc_chunk(fs_info, space_info, force); 3074 if (space_info->full) { 3075 /* No more free physical space */ 3076 if (should_alloc) 3077 ret = -ENOSPC; 3078 else 3079 ret = 0; 3080 spin_unlock(&space_info->lock); 3081 return ret; 3082 } else if (!should_alloc) { 3083 spin_unlock(&space_info->lock); 3084 return 0; 3085 } else if (space_info->chunk_alloc) { 3086 /* 3087 * Someone is already allocating, so we need to block 3088 * until this someone is finished and then loop to 3089 * recheck if we should continue with our allocation 3090 * attempt. 3091 */ 3092 wait_for_alloc = true; 3093 spin_unlock(&space_info->lock); 3094 mutex_lock(&fs_info->chunk_mutex); 3095 mutex_unlock(&fs_info->chunk_mutex); 3096 } else { 3097 /* Proceed with allocation */ 3098 space_info->chunk_alloc = 1; 3099 wait_for_alloc = false; 3100 spin_unlock(&space_info->lock); 3101 } 3102 3103 cond_resched(); 3104 } while (wait_for_alloc); 3105 3106 mutex_lock(&fs_info->chunk_mutex); 3107 trans->allocating_chunk = true; 3108 3109 /* 3110 * If we have mixed data/metadata chunks we want to make sure we keep 3111 * allocating mixed chunks instead of individual chunks. 3112 */ 3113 if (btrfs_mixed_space_info(space_info)) 3114 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA); 3115 3116 /* 3117 * if we're doing a data chunk, go ahead and make sure that 3118 * we keep a reasonable number of metadata chunks allocated in the 3119 * FS as well. 3120 */ 3121 if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) { 3122 fs_info->data_chunk_allocations++; 3123 if (!(fs_info->data_chunk_allocations % 3124 fs_info->metadata_ratio)) 3125 force_metadata_allocation(fs_info); 3126 } 3127 3128 /* 3129 * Check if we have enough space in SYSTEM chunk because we may need 3130 * to update devices. 3131 */ 3132 check_system_chunk(trans, flags); 3133 3134 ret = btrfs_alloc_chunk(trans, flags); 3135 trans->allocating_chunk = false; 3136 3137 spin_lock(&space_info->lock); 3138 if (ret < 0) { 3139 if (ret == -ENOSPC) 3140 space_info->full = 1; 3141 else 3142 goto out; 3143 } else { 3144 ret = 1; 3145 space_info->max_extent_size = 0; 3146 } 3147 3148 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; 3149 out: 3150 space_info->chunk_alloc = 0; 3151 spin_unlock(&space_info->lock); 3152 mutex_unlock(&fs_info->chunk_mutex); 3153 /* 3154 * When we allocate a new chunk we reserve space in the chunk block 3155 * reserve to make sure we can COW nodes/leafs in the chunk tree or 3156 * add new nodes/leafs to it if we end up needing to do it when 3157 * inserting the chunk item and updating device items as part of the 3158 * second phase of chunk allocation, performed by 3159 * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a 3160 * large number of new block groups to create in our transaction 3161 * handle's new_bgs list to avoid exhausting the chunk block reserve 3162 * in extreme cases - like having a single transaction create many new 3163 * block groups when starting to write out the free space caches of all 3164 * the block groups that were made dirty during the lifetime of the 3165 * transaction. 3166 */ 3167 if (trans->chunk_bytes_reserved >= (u64)SZ_2M) 3168 btrfs_create_pending_block_groups(trans); 3169 3170 return ret; 3171 } 3172 3173 static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type) 3174 { 3175 u64 num_dev; 3176 3177 num_dev = btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)].devs_max; 3178 if (!num_dev) 3179 num_dev = fs_info->fs_devices->rw_devices; 3180 3181 return num_dev; 3182 } 3183 3184 /* 3185 * Reserve space in the system space for allocating or removing a chunk 3186 */ 3187 void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) 3188 { 3189 struct btrfs_fs_info *fs_info = trans->fs_info; 3190 struct btrfs_space_info *info; 3191 u64 left; 3192 u64 thresh; 3193 int ret = 0; 3194 u64 num_devs; 3195 3196 /* 3197 * Needed because we can end up allocating a system chunk and for an 3198 * atomic and race free space reservation in the chunk block reserve. 3199 */ 3200 lockdep_assert_held(&fs_info->chunk_mutex); 3201 3202 info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); 3203 spin_lock(&info->lock); 3204 left = info->total_bytes - btrfs_space_info_used(info, true); 3205 spin_unlock(&info->lock); 3206 3207 num_devs = get_profile_num_devs(fs_info, type); 3208 3209 /* num_devs device items to update and 1 chunk item to add or remove */ 3210 thresh = btrfs_calc_metadata_size(fs_info, num_devs) + 3211 btrfs_calc_insert_metadata_size(fs_info, 1); 3212 3213 if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { 3214 btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu", 3215 left, thresh, type); 3216 btrfs_dump_space_info(fs_info, info, 0, 0); 3217 } 3218 3219 if (left < thresh) { 3220 u64 flags = btrfs_system_alloc_profile(fs_info); 3221 3222 /* 3223 * Ignore failure to create system chunk. We might end up not 3224 * needing it, as we might not need to COW all nodes/leafs from 3225 * the paths we visit in the chunk tree (they were already COWed 3226 * or created in the current transaction for example). 3227 */ 3228 ret = btrfs_alloc_chunk(trans, flags); 3229 } 3230 3231 if (!ret) { 3232 ret = btrfs_block_rsv_add(fs_info->chunk_root, 3233 &fs_info->chunk_block_rsv, 3234 thresh, BTRFS_RESERVE_NO_FLUSH); 3235 if (!ret) 3236 trans->chunk_bytes_reserved += thresh; 3237 } 3238 } 3239 3240 void btrfs_put_block_group_cache(struct btrfs_fs_info *info) 3241 { 3242 struct btrfs_block_group *block_group; 3243 u64 last = 0; 3244 3245 while (1) { 3246 struct inode *inode; 3247 3248 block_group = btrfs_lookup_first_block_group(info, last); 3249 while (block_group) { 3250 btrfs_wait_block_group_cache_done(block_group); 3251 spin_lock(&block_group->lock); 3252 if (block_group->iref) 3253 break; 3254 spin_unlock(&block_group->lock); 3255 block_group = btrfs_next_block_group(block_group); 3256 } 3257 if (!block_group) { 3258 if (last == 0) 3259 break; 3260 last = 0; 3261 continue; 3262 } 3263 3264 inode = block_group->inode; 3265 block_group->iref = 0; 3266 block_group->inode = NULL; 3267 spin_unlock(&block_group->lock); 3268 ASSERT(block_group->io_ctl.inode == NULL); 3269 iput(inode); 3270 last = block_group->start + block_group->length; 3271 btrfs_put_block_group(block_group); 3272 } 3273 } 3274 3275 /* 3276 * Must be called only after stopping all workers, since we could have block 3277 * group caching kthreads running, and therefore they could race with us if we 3278 * freed the block groups before stopping them. 3279 */ 3280 int btrfs_free_block_groups(struct btrfs_fs_info *info) 3281 { 3282 struct btrfs_block_group *block_group; 3283 struct btrfs_space_info *space_info; 3284 struct btrfs_caching_control *caching_ctl; 3285 struct rb_node *n; 3286 3287 down_write(&info->commit_root_sem); 3288 while (!list_empty(&info->caching_block_groups)) { 3289 caching_ctl = list_entry(info->caching_block_groups.next, 3290 struct btrfs_caching_control, list); 3291 list_del(&caching_ctl->list); 3292 btrfs_put_caching_control(caching_ctl); 3293 } 3294 up_write(&info->commit_root_sem); 3295 3296 spin_lock(&info->unused_bgs_lock); 3297 while (!list_empty(&info->unused_bgs)) { 3298 block_group = list_first_entry(&info->unused_bgs, 3299 struct btrfs_block_group, 3300 bg_list); 3301 list_del_init(&block_group->bg_list); 3302 btrfs_put_block_group(block_group); 3303 } 3304 spin_unlock(&info->unused_bgs_lock); 3305 3306 spin_lock(&info->block_group_cache_lock); 3307 while ((n = rb_last(&info->block_group_cache_tree)) != NULL) { 3308 block_group = rb_entry(n, struct btrfs_block_group, 3309 cache_node); 3310 rb_erase(&block_group->cache_node, 3311 &info->block_group_cache_tree); 3312 RB_CLEAR_NODE(&block_group->cache_node); 3313 spin_unlock(&info->block_group_cache_lock); 3314 3315 down_write(&block_group->space_info->groups_sem); 3316 list_del(&block_group->list); 3317 up_write(&block_group->space_info->groups_sem); 3318 3319 /* 3320 * We haven't cached this block group, which means we could 3321 * possibly have excluded extents on this block group. 3322 */ 3323 if (block_group->cached == BTRFS_CACHE_NO || 3324 block_group->cached == BTRFS_CACHE_ERROR) 3325 btrfs_free_excluded_extents(block_group); 3326 3327 btrfs_remove_free_space_cache(block_group); 3328 ASSERT(block_group->cached != BTRFS_CACHE_STARTED); 3329 ASSERT(list_empty(&block_group->dirty_list)); 3330 ASSERT(list_empty(&block_group->io_list)); 3331 ASSERT(list_empty(&block_group->bg_list)); 3332 ASSERT(refcount_read(&block_group->refs) == 1); 3333 btrfs_put_block_group(block_group); 3334 3335 spin_lock(&info->block_group_cache_lock); 3336 } 3337 spin_unlock(&info->block_group_cache_lock); 3338 3339 /* 3340 * Now that all the block groups are freed, go through and free all the 3341 * space_info structs. This is only called during the final stages of 3342 * unmount, and so we know nobody is using them. We call 3343 * synchronize_rcu() once before we start, just to be on the safe side. 3344 */ 3345 synchronize_rcu(); 3346 3347 btrfs_release_global_block_rsv(info); 3348 3349 while (!list_empty(&info->space_info)) { 3350 space_info = list_entry(info->space_info.next, 3351 struct btrfs_space_info, 3352 list); 3353 3354 /* 3355 * Do not hide this behind enospc_debug, this is actually 3356 * important and indicates a real bug if this happens. 3357 */ 3358 if (WARN_ON(space_info->bytes_pinned > 0 || 3359 space_info->bytes_reserved > 0 || 3360 space_info->bytes_may_use > 0)) 3361 btrfs_dump_space_info(info, space_info, 0, 0); 3362 WARN_ON(space_info->reclaim_size > 0); 3363 list_del(&space_info->list); 3364 btrfs_sysfs_remove_space_info(space_info); 3365 } 3366 return 0; 3367 } 3368 3369 void btrfs_freeze_block_group(struct btrfs_block_group *cache) 3370 { 3371 atomic_inc(&cache->frozen); 3372 } 3373 3374 void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) 3375 { 3376 struct btrfs_fs_info *fs_info = block_group->fs_info; 3377 struct extent_map_tree *em_tree; 3378 struct extent_map *em; 3379 bool cleanup; 3380 3381 spin_lock(&block_group->lock); 3382 cleanup = (atomic_dec_and_test(&block_group->frozen) && 3383 block_group->removed); 3384 spin_unlock(&block_group->lock); 3385 3386 if (cleanup) { 3387 em_tree = &fs_info->mapping_tree; 3388 write_lock(&em_tree->lock); 3389 em = lookup_extent_mapping(em_tree, block_group->start, 3390 1); 3391 BUG_ON(!em); /* logic error, can't happen */ 3392 remove_extent_mapping(em_tree, em); 3393 write_unlock(&em_tree->lock); 3394 3395 /* once for us and once for the tree */ 3396 free_extent_map(em); 3397 free_extent_map(em); 3398 3399 /* 3400 * We may have left one free space entry and other possible 3401 * tasks trimming this block group have left 1 entry each one. 3402 * Free them if any. 3403 */ 3404 __btrfs_remove_free_space_cache(block_group->free_space_ctl); 3405 } 3406 } 3407