Lines Matching refs:ctl
47 static int link_free_space(struct btrfs_free_space_ctl *ctl,
49 static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
51 static int search_bitmap(struct btrfs_free_space_ctl *ctl,
54 static void free_bitmap(struct btrfs_free_space_ctl *ctl,
56 static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
60 static void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache() argument
65 while ((node = rb_last(&ctl->free_space_offset)) != NULL) { in __btrfs_remove_free_space_cache()
68 unlink_free_space(ctl, info, true); in __btrfs_remove_free_space_cache()
71 free_bitmap(ctl, info); in __btrfs_remove_free_space_cache()
74 cond_resched_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
689 static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) in recalculate_thresholds() argument
691 struct btrfs_block_group *block_group = ctl->block_group; in recalculate_thresholds()
696 u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; in recalculate_thresholds()
701 if (ctl->total_bitmaps > max_bitmaps) in recalculate_thresholds()
705 ctl->total_bitmaps, ctl->unit, max_bitmaps, in recalculate_thresholds()
707 ASSERT(ctl->total_bitmaps <= max_bitmaps); in recalculate_thresholds()
720 bitmap_bytes = ctl->total_bitmaps * ctl->unit; in recalculate_thresholds()
729 ctl->extents_thresh = in recalculate_thresholds()
734 struct btrfs_free_space_ctl *ctl, in __load_free_space_cache() argument
832 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
833 ret = link_free_space(ctl, e); in __load_free_space_cache()
834 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
852 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
853 ret = link_free_space(ctl, e); in __load_free_space_cache()
855 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
862 ctl->total_bitmaps++; in __load_free_space_cache()
863 recalculate_thresholds(ctl); in __load_free_space_cache()
864 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
892 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
893 __btrfs_remove_free_space_cache(ctl); in __load_free_space_cache()
894 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
899 struct btrfs_free_space_ctl *ctl) in copy_free_space_cache() argument
905 while (!ret && (n = rb_first(&ctl->free_space_offset)) != NULL) { in copy_free_space_cache()
911 unlink_free_space(ctl, info, true); in copy_free_space_cache()
912 spin_unlock(&ctl->tree_lock); in copy_free_space_cache()
915 spin_lock(&ctl->tree_lock); in copy_free_space_cache()
918 u64 bytes = ctl->unit; in copy_free_space_cache()
920 ret = search_bitmap(ctl, info, &offset, &bytes, false); in copy_free_space_cache()
922 bitmap_clear_bits(ctl, info, offset, bytes, true); in copy_free_space_cache()
923 spin_unlock(&ctl->tree_lock); in copy_free_space_cache()
926 spin_lock(&ctl->tree_lock); in copy_free_space_cache()
928 free_bitmap(ctl, info); in copy_free_space_cache()
932 cond_resched_lock(&ctl->tree_lock); in copy_free_space_cache()
942 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in load_free_space_cache() local
1061 spin_lock(&ctl->tree_lock); in load_free_space_cache()
1063 spin_unlock(&ctl->tree_lock); in load_free_space_cache()
1070 struct btrfs_free_space_ctl *ctl, in write_cache_extent_entries() argument
1078 struct rb_node *node = rb_first(&ctl->free_space_offset); in write_cache_extent_entries()
1130 list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) { in write_cache_extent_entries()
1378 struct btrfs_free_space_ctl *ctl, in __btrfs_write_out_cache() argument
1423 mutex_lock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1425 spin_lock(&ctl->tree_lock); in __btrfs_write_out_cache()
1426 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1450 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1451 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1493 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1494 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1521 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_write_out_cache() local
1536 ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl, in btrfs_write_out_cache()
1571 static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, in offset_to_bitmap() argument
1577 bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; in offset_to_bitmap()
1578 bitmap_start = offset - ctl->start; in offset_to_bitmap()
1581 bitmap_start += ctl->start; in offset_to_bitmap()
1586 static int tree_insert_offset(struct btrfs_free_space_ctl *ctl, in tree_insert_offset() argument
1594 lockdep_assert_held(&ctl->tree_lock); in tree_insert_offset()
1600 root = &ctl->free_space_offset; in tree_insert_offset()
1703 tree_search_offset(struct btrfs_free_space_ctl *ctl, in tree_search_offset() argument
1706 struct rb_node *n = ctl->free_space_offset.rb_node; in tree_search_offset()
1709 lockdep_assert_held(&ctl->tree_lock); in tree_search_offset()
1791 if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) in tree_search_offset()
1806 ctl->unit > offset) in tree_search_offset()
1816 static inline void unlink_free_space(struct btrfs_free_space_ctl *ctl, in unlink_free_space() argument
1820 lockdep_assert_held(&ctl->tree_lock); in unlink_free_space()
1822 rb_erase(&info->offset_index, &ctl->free_space_offset); in unlink_free_space()
1823 rb_erase_cached(&info->bytes_index, &ctl->free_space_bytes); in unlink_free_space()
1824 ctl->free_extents--; in unlink_free_space()
1827 ctl->discardable_extents[BTRFS_STAT_CURR]--; in unlink_free_space()
1828 ctl->discardable_bytes[BTRFS_STAT_CURR] -= info->bytes; in unlink_free_space()
1832 ctl->free_space -= info->bytes; in unlink_free_space()
1835 static int link_free_space(struct btrfs_free_space_ctl *ctl, in link_free_space() argument
1840 lockdep_assert_held(&ctl->tree_lock); in link_free_space()
1843 ret = tree_insert_offset(ctl, NULL, info); in link_free_space()
1847 rb_add_cached(&info->bytes_index, &ctl->free_space_bytes, entry_less); in link_free_space()
1850 ctl->discardable_extents[BTRFS_STAT_CURR]++; in link_free_space()
1851 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in link_free_space()
1854 ctl->free_space += info->bytes; in link_free_space()
1855 ctl->free_extents++; in link_free_space()
1859 static void relink_bitmap_entry(struct btrfs_free_space_ctl *ctl, in relink_bitmap_entry() argument
1871 lockdep_assert_held(&ctl->tree_lock); in relink_bitmap_entry()
1873 rb_erase_cached(&info->bytes_index, &ctl->free_space_bytes); in relink_bitmap_entry()
1874 rb_add_cached(&info->bytes_index, &ctl->free_space_bytes, entry_less); in relink_bitmap_entry()
1877 static inline void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in bitmap_clear_bits() argument
1884 start = offset_to_bit(info->offset, ctl->unit, offset); in bitmap_clear_bits()
1885 count = bytes_to_bits(bytes, ctl->unit); in bitmap_clear_bits()
1892 if (info->max_extent_size > ctl->unit) in bitmap_clear_bits()
1895 relink_bitmap_entry(ctl, info); in bitmap_clear_bits()
1905 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in bitmap_clear_bits()
1906 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in bitmap_clear_bits()
1910 ctl->free_space -= bytes; in bitmap_clear_bits()
1913 static void btrfs_bitmap_set_bits(struct btrfs_free_space_ctl *ctl, in btrfs_bitmap_set_bits() argument
1920 start = offset_to_bit(info->offset, ctl->unit, offset); in btrfs_bitmap_set_bits()
1921 count = bytes_to_bits(bytes, ctl->unit); in btrfs_bitmap_set_bits()
1933 ctl->free_space += bytes; in btrfs_bitmap_set_bits()
1935 relink_bitmap_entry(ctl, info); in btrfs_bitmap_set_bits()
1945 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in btrfs_bitmap_set_bits()
1946 ctl->discardable_bytes[BTRFS_STAT_CURR] += bytes; in btrfs_bitmap_set_bits()
1954 static int search_bitmap(struct btrfs_free_space_ctl *ctl, in search_bitmap() argument
1975 i = offset_to_bit(bitmap_info->offset, ctl->unit, in search_bitmap()
1977 bits = bytes_to_bits(*bytes, ctl->unit); in search_bitmap()
1997 *offset = (u64)(i * ctl->unit) + bitmap_info->offset; in search_bitmap()
1998 *bytes = (u64)(found_bits) * ctl->unit; in search_bitmap()
2002 *bytes = (u64)(max_bits) * ctl->unit; in search_bitmap()
2004 relink_bitmap_entry(ctl, bitmap_info); in search_bitmap()
2010 find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes, in find_free_space() argument
2019 if (!ctl->free_space_offset.rb_node) in find_free_space()
2023 node = rb_first_cached(&ctl->free_space_bytes); in find_free_space()
2025 entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), in find_free_space()
2060 tmp = entry->offset - ctl->start + align - 1; in find_free_space()
2062 tmp = tmp * align + ctl->start; in find_free_space()
2086 ret = search_bitmap(ctl, entry, &tmp, &size, true); in find_free_space()
2116 static void add_new_bitmap(struct btrfs_free_space_ctl *ctl, in add_new_bitmap() argument
2119 info->offset = offset_to_bitmap(ctl, offset); in add_new_bitmap()
2123 link_free_space(ctl, info); in add_new_bitmap()
2124 ctl->total_bitmaps++; in add_new_bitmap()
2125 recalculate_thresholds(ctl); in add_new_bitmap()
2128 static void free_bitmap(struct btrfs_free_space_ctl *ctl, in free_bitmap() argument
2138 ctl->discardable_extents[BTRFS_STAT_CURR] -= in free_bitmap()
2140 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bitmap_info->bytes; in free_bitmap()
2143 unlink_free_space(ctl, bitmap_info, true); in free_bitmap()
2146 ctl->total_bitmaps--; in free_bitmap()
2147 recalculate_thresholds(ctl); in free_bitmap()
2150 static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, in remove_from_bitmap() argument
2159 end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; in remove_from_bitmap()
2168 search_bytes = ctl->unit; in remove_from_bitmap()
2170 ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, in remove_from_bitmap()
2181 bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes, true); in remove_from_bitmap()
2188 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2214 search_bytes = ctl->unit; in remove_from_bitmap()
2215 ret = search_bitmap(ctl, bitmap_info, &search_start, in remove_from_bitmap()
2222 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2227 static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, in add_bytes_to_bitmap() argument
2240 ctl->discardable_extents[BTRFS_STAT_CURR] += in add_bytes_to_bitmap()
2242 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in add_bytes_to_bitmap()
2247 end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); in add_bytes_to_bitmap()
2251 btrfs_bitmap_set_bits(ctl, info, offset, bytes_to_set); in add_bytes_to_bitmap()
2257 static bool use_bitmap(struct btrfs_free_space_ctl *ctl, in use_bitmap() argument
2260 struct btrfs_block_group *block_group = ctl->block_group; in use_bitmap()
2277 if (!forced && ctl->free_extents < ctl->extents_thresh) { in use_bitmap()
2286 if (ctl->free_extents * 3 <= ctl->extents_thresh) in use_bitmap()
2301 if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length) in use_bitmap()
2311 static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, in insert_into_bitmap() argument
2325 if (!ctl->op->use_bitmap(ctl, info)) in insert_into_bitmap()
2328 if (ctl->op == &free_space_op) in insert_into_bitmap()
2329 block_group = ctl->block_group; in insert_into_bitmap()
2357 if (entry->offset == offset_to_bitmap(ctl, offset)) { in insert_into_bitmap()
2358 bytes_added = add_bytes_to_bitmap(ctl, entry, offset, in insert_into_bitmap()
2371 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in insert_into_bitmap()
2378 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in insert_into_bitmap()
2392 add_new_bitmap(ctl, info, offset); in insert_into_bitmap()
2397 spin_unlock(&ctl->tree_lock); in insert_into_bitmap()
2404 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2414 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2449 static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl, in try_merge_free_space() argument
2465 right_info = tree_search_offset(ctl, offset + bytes, 0, 0); in try_merge_free_space()
2472 left_info = tree_search_offset(ctl, offset - 1, 0, 0); in try_merge_free_space()
2477 unlink_free_space(ctl, right_info, update_stat); in try_merge_free_space()
2487 unlink_free_space(ctl, left_info, update_stat); in try_merge_free_space()
2497 static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_end() argument
2505 const u64 bitmap_offset = offset_to_bitmap(ctl, end); in steal_from_bitmap_to_end()
2508 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_end()
2512 i = offset_to_bit(bitmap->offset, ctl->unit, end); in steal_from_bitmap_to_end()
2516 bytes = (j - i) * ctl->unit; in steal_from_bitmap_to_end()
2523 bitmap_clear_bits(ctl, bitmap, end, bytes, update_stat); in steal_from_bitmap_to_end()
2526 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_end()
2531 static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_front() argument
2542 bitmap_offset = offset_to_bitmap(ctl, info->offset); in steal_from_bitmap_to_front()
2547 bitmap_offset = offset_to_bitmap(ctl, info->offset - 1); in steal_from_bitmap_to_front()
2550 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_front()
2554 i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; in steal_from_bitmap_to_front()
2566 bytes = (i + 1) * ctl->unit; in steal_from_bitmap_to_front()
2568 bytes = (i - prev_j) * ctl->unit; in steal_from_bitmap_to_front()
2577 bitmap_clear_bits(ctl, bitmap, info->offset, bytes, update_stat); in steal_from_bitmap_to_front()
2580 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_front()
2596 static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap() argument
2607 if (ctl->total_bitmaps > 0) { in steal_from_bitmap()
2611 stole_end = steal_from_bitmap_to_end(ctl, info, update_stat); in steal_from_bitmap()
2612 if (ctl->total_bitmaps > 0) in steal_from_bitmap()
2613 stole_front = steal_from_bitmap_to_front(ctl, info, in steal_from_bitmap()
2617 try_merge_free_space(ctl, info, update_stat); in steal_from_bitmap()
2626 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_add_free_space() local
2643 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space()
2645 if (try_merge_free_space(ctl, info, true)) in __btrfs_add_free_space()
2653 ret = insert_into_bitmap(ctl, info); in __btrfs_add_free_space()
2667 steal_from_bitmap(ctl, info, true); in __btrfs_add_free_space()
2671 ret = link_free_space(ctl, info); in __btrfs_add_free_space()
2676 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space()
2695 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_add_free_space_zoned() local
2721 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space_zoned()
2722 ctl->free_space += to_free; in __btrfs_add_free_space_zoned()
2723 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space_zoned()
2801 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space() local
2826 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space()
2833 info = tree_search_offset(ctl, offset, 0, 0); in btrfs_remove_free_space()
2839 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in btrfs_remove_free_space()
2854 unlink_free_space(ctl, info, true); in btrfs_remove_free_space()
2861 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2874 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2888 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2899 ret = remove_from_bitmap(ctl, info, &offset, &bytes); in btrfs_remove_free_space()
2906 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2915 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_dump_free_space() local
2932 spin_lock(&ctl->tree_lock); in btrfs_dump_free_space()
2933 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in btrfs_dump_free_space()
2941 spin_unlock(&ctl->tree_lock); in btrfs_dump_free_space()
2950 struct btrfs_free_space_ctl *ctl) in btrfs_init_free_space_ctl() argument
2954 spin_lock_init(&ctl->tree_lock); in btrfs_init_free_space_ctl()
2955 ctl->unit = fs_info->sectorsize; in btrfs_init_free_space_ctl()
2956 ctl->start = block_group->start; in btrfs_init_free_space_ctl()
2957 ctl->block_group = block_group; in btrfs_init_free_space_ctl()
2958 ctl->op = &free_space_op; in btrfs_init_free_space_ctl()
2959 ctl->free_space_bytes = RB_ROOT_CACHED; in btrfs_init_free_space_ctl()
2960 INIT_LIST_HEAD(&ctl->trimming_ranges); in btrfs_init_free_space_ctl()
2961 mutex_init(&ctl->cache_writeout_mutex); in btrfs_init_free_space_ctl()
2968 ctl->extents_thresh = (SZ_32K / 2) / sizeof(struct btrfs_free_space); in btrfs_init_free_space_ctl()
2981 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_return_cluster_to_free_space() local
2984 lockdep_assert_held(&ctl->tree_lock); in __btrfs_return_cluster_to_free_space()
3008 ctl->discardable_extents[BTRFS_STAT_CURR]--; in __btrfs_return_cluster_to_free_space()
3009 ctl->discardable_bytes[BTRFS_STAT_CURR] -= in __btrfs_return_cluster_to_free_space()
3013 try_merge_free_space(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
3014 steal_from_bitmap(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
3018 ctl->discardable_extents[BTRFS_STAT_CURR]++; in __btrfs_return_cluster_to_free_space()
3019 ctl->discardable_bytes[BTRFS_STAT_CURR] += in __btrfs_return_cluster_to_free_space()
3023 tree_insert_offset(ctl, NULL, entry); in __btrfs_return_cluster_to_free_space()
3024 rb_add_cached(&entry->bytes_index, &ctl->free_space_bytes, in __btrfs_return_cluster_to_free_space()
3034 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space_cache() local
3038 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
3047 cond_resched_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
3049 __btrfs_remove_free_space_cache(ctl); in btrfs_remove_free_space_cache()
3051 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
3060 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_is_free_space_trimmed() local
3065 spin_lock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
3066 node = rb_first(&ctl->free_space_offset); in btrfs_is_free_space_trimmed()
3079 spin_unlock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
3087 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_for_alloc() local
3100 spin_lock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
3101 entry = find_free_space(ctl, &offset, &bytes_search, in btrfs_find_space_for_alloc()
3109 bitmap_clear_bits(ctl, entry, offset, bytes, true); in btrfs_find_space_for_alloc()
3115 free_bitmap(ctl, entry); in btrfs_find_space_for_alloc()
3117 unlink_free_space(ctl, entry, true); in btrfs_find_space_for_alloc()
3132 link_free_space(ctl, entry); in btrfs_find_space_for_alloc()
3136 spin_unlock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
3156 struct btrfs_free_space_ctl *ctl; in btrfs_return_cluster_to_free_space() local
3174 ctl = block_group->free_space_ctl; in btrfs_return_cluster_to_free_space()
3177 spin_lock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
3179 spin_unlock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
3193 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_bitmap() local
3202 err = search_bitmap(ctl, entry, &search_start, &search_bytes, true); in btrfs_alloc_from_bitmap()
3210 bitmap_clear_bits(ctl, entry, ret, bytes, false); in btrfs_alloc_from_bitmap()
3224 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_cluster() local
3289 spin_lock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3294 ctl->free_space -= bytes; in btrfs_alloc_from_cluster()
3296 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in btrfs_alloc_from_cluster()
3301 ctl->free_extents--; in btrfs_alloc_from_cluster()
3305 ctl->total_bitmaps--; in btrfs_alloc_from_cluster()
3306 recalculate_thresholds(ctl); in btrfs_alloc_from_cluster()
3308 ctl->discardable_extents[BTRFS_STAT_CURR]--; in btrfs_alloc_from_cluster()
3314 spin_unlock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3325 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_bitmap_cluster() local
3336 lockdep_assert_held(&ctl->tree_lock); in btrfs_bitmap_cluster()
3338 i = offset_to_bit(entry->offset, ctl->unit, in btrfs_bitmap_cluster()
3340 want_bits = bytes_to_bits(bytes, ctl->unit); in btrfs_bitmap_cluster()
3341 min_bits = bytes_to_bits(min_bytes, ctl->unit); in btrfs_bitmap_cluster()
3367 entry->max_extent_size = (u64)max_bits * ctl->unit; in btrfs_bitmap_cluster()
3378 if (cluster->max_size < found_bits * ctl->unit) in btrfs_bitmap_cluster()
3379 cluster->max_size = found_bits * ctl->unit; in btrfs_bitmap_cluster()
3386 cluster->window_start = start * ctl->unit + entry->offset; in btrfs_bitmap_cluster()
3387 rb_erase(&entry->offset_index, &ctl->free_space_offset); in btrfs_bitmap_cluster()
3388 rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); in btrfs_bitmap_cluster()
3399 ret = tree_insert_offset(ctl, cluster, entry); in btrfs_bitmap_cluster()
3403 total_found * ctl->unit, 1); in btrfs_bitmap_cluster()
3418 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_no_bitmap() local
3427 lockdep_assert_held(&ctl->tree_lock); in setup_cluster_no_bitmap()
3429 entry = tree_search_offset(ctl, offset, 0, 1); in setup_cluster_no_bitmap()
3489 rb_erase(&entry->offset_index, &ctl->free_space_offset); in setup_cluster_no_bitmap()
3490 rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); in setup_cluster_no_bitmap()
3491 ret = tree_insert_offset(ctl, cluster, entry); in setup_cluster_no_bitmap()
3511 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_bitmap() local
3514 u64 bitmap_offset = offset_to_bitmap(ctl, offset); in setup_cluster_bitmap()
3516 if (ctl->total_bitmaps == 0) in setup_cluster_bitmap()
3527 entry = tree_search_offset(ctl, bitmap_offset, 1, 0); in setup_cluster_bitmap()
3561 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_cluster() local
3585 spin_lock(&ctl->tree_lock); in btrfs_find_space_cluster()
3591 if (ctl->free_space < bytes) { in btrfs_find_space_cluster()
3592 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3629 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3656 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in do_trimming() local
3680 mutex_lock(&ctl->cache_writeout_mutex); in do_trimming()
3690 mutex_unlock(&ctl->cache_writeout_mutex); in do_trimming()
3715 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_no_bitmap() local
3728 mutex_lock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3729 spin_lock(&ctl->tree_lock); in trim_no_bitmap()
3731 if (ctl->free_space < minlen) in trim_no_bitmap()
3734 entry = tree_search_offset(ctl, start, 0, 1); in trim_no_bitmap()
3758 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3759 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3762 unlink_free_space(ctl, entry, true); in trim_no_bitmap()
3775 link_free_space(ctl, entry); in trim_no_bitmap()
3783 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3784 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3788 unlink_free_space(ctl, entry, true); in trim_no_bitmap()
3792 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3795 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_no_bitmap()
3796 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3823 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3824 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3843 static void reset_trimming_bitmap(struct btrfs_free_space_ctl *ctl, u64 offset) in reset_trimming_bitmap() argument
3847 spin_lock(&ctl->tree_lock); in reset_trimming_bitmap()
3848 entry = tree_search_offset(ctl, offset, 1, 0); in reset_trimming_bitmap()
3851 ctl->discardable_extents[BTRFS_STAT_CURR] += in reset_trimming_bitmap()
3853 ctl->discardable_bytes[BTRFS_STAT_CURR] += entry->bytes; in reset_trimming_bitmap()
3858 spin_unlock(&ctl->tree_lock); in reset_trimming_bitmap()
3861 static void end_trimming_bitmap(struct btrfs_free_space_ctl *ctl, in end_trimming_bitmap() argument
3866 ctl->discardable_extents[BTRFS_STAT_CURR] -= in end_trimming_bitmap()
3868 ctl->discardable_bytes[BTRFS_STAT_CURR] -= entry->bytes; in end_trimming_bitmap()
3881 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_bitmaps() local
3886 u64 offset = offset_to_bitmap(ctl, start); in trim_bitmaps()
3893 mutex_lock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3894 spin_lock(&ctl->tree_lock); in trim_bitmaps()
3896 if (ctl->free_space < minlen) { in trim_bitmaps()
3899 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3900 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3904 entry = tree_search_offset(ctl, offset, 1, 0); in trim_bitmaps()
3915 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3916 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3931 ret2 = search_bitmap(ctl, entry, &start, &bytes, false); in trim_bitmaps()
3938 end_trimming_bitmap(ctl, entry); in trim_bitmaps()
3941 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3942 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3952 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3953 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3959 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3960 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3975 bitmap_clear_bits(ctl, entry, start, bytes, true); in trim_bitmaps()
3977 free_bitmap(ctl, entry); in trim_bitmaps()
3979 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3982 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_bitmaps()
3983 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3988 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
3995 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
4004 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
4022 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_trim_block_group() local
4043 div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem); in btrfs_trim_block_group()
4046 reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); in btrfs_trim_block_group()
4193 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_add_free_space_entry() local
4208 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4212 ret = link_free_space(ctl, info); in test_add_free_space_entry()
4213 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4227 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4228 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_add_free_space_entry()
4233 add_new_bitmap(ctl, info, offset); in test_add_free_space_entry()
4238 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in test_add_free_space_entry()
4243 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4263 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_check_exists() local
4267 spin_lock(&ctl->tree_lock); in test_check_exists()
4268 info = tree_search_offset(ctl, offset, 0, 0); in test_check_exists()
4270 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_check_exists()
4283 bit_bytes = ctl->unit; in test_check_exists()
4284 ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); in test_check_exists()
4336 spin_unlock(&ctl->tree_lock); in test_check_exists()