node.c (521187439abfb3e1c946796dc2187c443e5457ab) node.c (94c821fb286b545d37549ff30a0c341e066f0d6c)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/node.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 2199 unchanged lines hidden (view full) ---

2208 case FREE_NID:
2209 list_add_tail(&i->list, &nm_i->free_nid_list);
2210 break;
2211 default:
2212 BUG_ON(1);
2213 }
2214}
2215
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/node.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 2199 unchanged lines hidden (view full) ---

2208 case FREE_NID:
2209 list_add_tail(&i->list, &nm_i->free_nid_list);
2210 break;
2211 default:
2212 BUG_ON(1);
2213 }
2214}
2215
2216bool f2fs_nat_bitmap_enabled(struct f2fs_sb_info *sbi)
2217{
2218 struct f2fs_nm_info *nm_i = NM_I(sbi);
2219 unsigned int i;
2220 bool ret = true;
2221
2222 down_read(&nm_i->nat_tree_lock);
2223 for (i = 0; i < nm_i->nat_blocks; i++) {
2224 if (!test_bit_le(i, nm_i->nat_block_bitmap)) {
2225 ret = false;
2226 break;
2227 }
2228 }
2229 up_read(&nm_i->nat_tree_lock);
2230
2231 return ret;
2232}
2233
2216static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid,
2217 bool set, bool build)
2218{
2219 struct f2fs_nm_info *nm_i = NM_I(sbi);
2220 unsigned int nat_ofs = NAT_BLOCK_OFFSET(nid);
2221 unsigned int nid_ofs = nid - START_NID(nid);
2222
2223 if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap))

--- 655 unchanged lines hidden (view full) ---

2879 list_add(&nes->set_list, cur->set_list.prev);
2880 return;
2881 }
2882 }
2883add_out:
2884 list_add_tail(&nes->set_list, head);
2885}
2886
2234static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid,
2235 bool set, bool build)
2236{
2237 struct f2fs_nm_info *nm_i = NM_I(sbi);
2238 unsigned int nat_ofs = NAT_BLOCK_OFFSET(nid);
2239 unsigned int nid_ofs = nid - START_NID(nid);
2240
2241 if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap))

--- 655 unchanged lines hidden (view full) ---

2897 list_add(&nes->set_list, cur->set_list.prev);
2898 return;
2899 }
2900 }
2901add_out:
2902 list_add_tail(&nes->set_list, head);
2903}
2904
2887static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid,
2905static void __update_nat_bits(struct f2fs_nm_info *nm_i, unsigned int nat_ofs,
2906 unsigned int valid)
2907{
2908 if (valid == 0) {
2909 __set_bit_le(nat_ofs, nm_i->empty_nat_bits);
2910 __clear_bit_le(nat_ofs, nm_i->full_nat_bits);
2911 return;
2912 }
2913
2914 __clear_bit_le(nat_ofs, nm_i->empty_nat_bits);
2915 if (valid == NAT_ENTRY_PER_BLOCK)
2916 __set_bit_le(nat_ofs, nm_i->full_nat_bits);
2917 else
2918 __clear_bit_le(nat_ofs, nm_i->full_nat_bits);
2919}
2920
2921static void update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid,
2888 struct page *page)
2889{
2890 struct f2fs_nm_info *nm_i = NM_I(sbi);
2891 unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK;
2892 struct f2fs_nat_block *nat_blk = page_address(page);
2893 int valid = 0;
2894 int i = 0;
2895
2922 struct page *page)
2923{
2924 struct f2fs_nm_info *nm_i = NM_I(sbi);
2925 unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK;
2926 struct f2fs_nat_block *nat_blk = page_address(page);
2927 int valid = 0;
2928 int i = 0;
2929
2896 if (!enabled_nat_bits(sbi, NULL))
2930 if (!is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG))
2897 return;
2898
2899 if (nat_index == 0) {
2900 valid = 1;
2901 i = 1;
2902 }
2903 for (; i < NAT_ENTRY_PER_BLOCK; i++) {
2904 if (le32_to_cpu(nat_blk->entries[i].block_addr) != NULL_ADDR)
2905 valid++;
2906 }
2931 return;
2932
2933 if (nat_index == 0) {
2934 valid = 1;
2935 i = 1;
2936 }
2937 for (; i < NAT_ENTRY_PER_BLOCK; i++) {
2938 if (le32_to_cpu(nat_blk->entries[i].block_addr) != NULL_ADDR)
2939 valid++;
2940 }
2907 if (valid == 0) {
2908 __set_bit_le(nat_index, nm_i->empty_nat_bits);
2909 __clear_bit_le(nat_index, nm_i->full_nat_bits);
2910 return;
2941
2942 __update_nat_bits(nm_i, nat_index, valid);
2943}
2944
2945void f2fs_enable_nat_bits(struct f2fs_sb_info *sbi)
2946{
2947 struct f2fs_nm_info *nm_i = NM_I(sbi);
2948 unsigned int nat_ofs;
2949
2950 down_read(&nm_i->nat_tree_lock);
2951
2952 for (nat_ofs = 0; nat_ofs < nm_i->nat_blocks; nat_ofs++) {
2953 unsigned int valid = 0, nid_ofs = 0;
2954
2955 /* handle nid zero due to it should never be used */
2956 if (unlikely(nat_ofs == 0)) {
2957 valid = 1;
2958 nid_ofs = 1;
2959 }
2960
2961 for (; nid_ofs < NAT_ENTRY_PER_BLOCK; nid_ofs++) {
2962 if (!test_bit_le(nid_ofs,
2963 nm_i->free_nid_bitmap[nat_ofs]))
2964 valid++;
2965 }
2966
2967 __update_nat_bits(nm_i, nat_ofs, valid);
2911 }
2912
2968 }
2969
2913 __clear_bit_le(nat_index, nm_i->empty_nat_bits);
2914 if (valid == NAT_ENTRY_PER_BLOCK)
2915 __set_bit_le(nat_index, nm_i->full_nat_bits);
2916 else
2917 __clear_bit_le(nat_index, nm_i->full_nat_bits);
2970 up_read(&nm_i->nat_tree_lock);
2918}
2919
2920static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
2921 struct nat_entry_set *set, struct cp_control *cpc)
2922{
2923 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2924 struct f2fs_journal *journal = curseg->journal;
2925 nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;
2926 bool to_journal = true;
2927 struct f2fs_nat_block *nat_blk;
2928 struct nat_entry *ne, *cur;
2929 struct page *page = NULL;
2930
2931 /*
2932 * there are two steps to flush nat entries:
2933 * #1, flush nat entries to journal in current hot data summary block.
2934 * #2, flush nat entries to nat page.
2935 */
2971}
2972
2973static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
2974 struct nat_entry_set *set, struct cp_control *cpc)
2975{
2976 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2977 struct f2fs_journal *journal = curseg->journal;
2978 nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;
2979 bool to_journal = true;
2980 struct f2fs_nat_block *nat_blk;
2981 struct nat_entry *ne, *cur;
2982 struct page *page = NULL;
2983
2984 /*
2985 * there are two steps to flush nat entries:
2986 * #1, flush nat entries to journal in current hot data summary block.
2987 * #2, flush nat entries to nat page.
2988 */
2936 if (enabled_nat_bits(sbi, cpc) ||
2989 if ((cpc->reason & CP_UMOUNT) ||
2937 !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL))
2938 to_journal = false;
2939
2940 if (to_journal) {
2941 down_write(&curseg->journal_rwsem);
2942 } else {
2943 page = get_next_nat_page(sbi, start_nid);
2944 if (IS_ERR(page))

--- 30 unchanged lines hidden (view full) ---

2975 update_free_nid_bitmap(sbi, nid, false, false);
2976 spin_unlock(&NM_I(sbi)->nid_list_lock);
2977 }
2978 }
2979
2980 if (to_journal) {
2981 up_write(&curseg->journal_rwsem);
2982 } else {
2990 !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL))
2991 to_journal = false;
2992
2993 if (to_journal) {
2994 down_write(&curseg->journal_rwsem);
2995 } else {
2996 page = get_next_nat_page(sbi, start_nid);
2997 if (IS_ERR(page))

--- 30 unchanged lines hidden (view full) ---

3028 update_free_nid_bitmap(sbi, nid, false, false);
3029 spin_unlock(&NM_I(sbi)->nid_list_lock);
3030 }
3031 }
3032
3033 if (to_journal) {
3034 up_write(&curseg->journal_rwsem);
3035 } else {
2983 __update_nat_bits(sbi, start_nid, page);
3036 update_nat_bits(sbi, start_nid, page);
2984 f2fs_put_page(page, 1);
2985 }
2986
2987 /* Allow dirty nats by node block allocation in write_begin */
2988 if (!set->entry_cnt) {
2989 radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set);
2990 kmem_cache_free(nat_entry_set_slab, set);
2991 }

--- 14 unchanged lines hidden (view full) ---

3006 nid_t set_idx = 0;
3007 LIST_HEAD(sets);
3008 int err = 0;
3009
3010 /*
3011 * during unmount, let's flush nat_bits before checking
3012 * nat_cnt[DIRTY_NAT].
3013 */
3037 f2fs_put_page(page, 1);
3038 }
3039
3040 /* Allow dirty nats by node block allocation in write_begin */
3041 if (!set->entry_cnt) {
3042 radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set);
3043 kmem_cache_free(nat_entry_set_slab, set);
3044 }

--- 14 unchanged lines hidden (view full) ---

3059 nid_t set_idx = 0;
3060 LIST_HEAD(sets);
3061 int err = 0;
3062
3063 /*
3064 * during unmount, let's flush nat_bits before checking
3065 * nat_cnt[DIRTY_NAT].
3066 */
3014 if (enabled_nat_bits(sbi, cpc)) {
3067 if (cpc->reason & CP_UMOUNT) {
3015 down_write(&nm_i->nat_tree_lock);
3016 remove_nats_in_journal(sbi);
3017 up_write(&nm_i->nat_tree_lock);
3018 }
3019
3020 if (!nm_i->nat_cnt[DIRTY_NAT])
3021 return 0;
3022
3023 down_write(&nm_i->nat_tree_lock);
3024
3025 /*
3026 * if there are no enough space in journal to store dirty nat
3027 * entries, remove all entries from journal and merge them
3028 * into nat entry set.
3029 */
3068 down_write(&nm_i->nat_tree_lock);
3069 remove_nats_in_journal(sbi);
3070 up_write(&nm_i->nat_tree_lock);
3071 }
3072
3073 if (!nm_i->nat_cnt[DIRTY_NAT])
3074 return 0;
3075
3076 down_write(&nm_i->nat_tree_lock);
3077
3078 /*
3079 * if there are no enough space in journal to store dirty nat
3080 * entries, remove all entries from journal and merge them
3081 * into nat entry set.
3082 */
3030 if (enabled_nat_bits(sbi, cpc) ||
3083 if (cpc->reason & CP_UMOUNT ||
3031 !__has_cursum_space(journal,
3032 nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL))
3033 remove_nats_in_journal(sbi);
3034
3035 while ((found = __gang_lookup_nat_set(nm_i,
3036 set_idx, SETVEC_SIZE, setvec))) {
3037 unsigned idx;
3038

--- 20 unchanged lines hidden (view full) ---

3059{
3060 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3061 struct f2fs_nm_info *nm_i = NM_I(sbi);
3062 unsigned int nat_bits_bytes = nm_i->nat_blocks / BITS_PER_BYTE;
3063 unsigned int i;
3064 __u64 cp_ver = cur_cp_version(ckpt);
3065 block_t nat_bits_addr;
3066
3084 !__has_cursum_space(journal,
3085 nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL))
3086 remove_nats_in_journal(sbi);
3087
3088 while ((found = __gang_lookup_nat_set(nm_i,
3089 set_idx, SETVEC_SIZE, setvec))) {
3090 unsigned idx;
3091

--- 20 unchanged lines hidden (view full) ---

3112{
3113 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3114 struct f2fs_nm_info *nm_i = NM_I(sbi);
3115 unsigned int nat_bits_bytes = nm_i->nat_blocks / BITS_PER_BYTE;
3116 unsigned int i;
3117 __u64 cp_ver = cur_cp_version(ckpt);
3118 block_t nat_bits_addr;
3119
3067 if (!enabled_nat_bits(sbi, NULL))
3068 return 0;
3069
3070 nm_i->nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8);
3071 nm_i->nat_bits = f2fs_kvzalloc(sbi,
3072 nm_i->nat_bits_blocks << F2FS_BLKSIZE_BITS, GFP_KERNEL);
3073 if (!nm_i->nat_bits)
3074 return -ENOMEM;
3075
3120 nm_i->nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8);
3121 nm_i->nat_bits = f2fs_kvzalloc(sbi,
3122 nm_i->nat_bits_blocks << F2FS_BLKSIZE_BITS, GFP_KERNEL);
3123 if (!nm_i->nat_bits)
3124 return -ENOMEM;
3125
3126 nm_i->full_nat_bits = nm_i->nat_bits + 8;
3127 nm_i->empty_nat_bits = nm_i->full_nat_bits + nat_bits_bytes;
3128
3129 if (!is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG))
3130 return 0;
3131
3076 nat_bits_addr = __start_cp_addr(sbi) + sbi->blocks_per_seg -
3077 nm_i->nat_bits_blocks;
3078 for (i = 0; i < nm_i->nat_bits_blocks; i++) {
3079 struct page *page;
3080
3081 page = f2fs_get_meta_page(sbi, nat_bits_addr++);
3082 if (IS_ERR(page))
3083 return PTR_ERR(page);
3084
3085 memcpy(nm_i->nat_bits + (i << F2FS_BLKSIZE_BITS),
3086 page_address(page), F2FS_BLKSIZE);
3087 f2fs_put_page(page, 1);
3088 }
3089
3090 cp_ver |= (cur_cp_crc(ckpt) << 32);
3091 if (cpu_to_le64(cp_ver) != *(__le64 *)nm_i->nat_bits) {
3132 nat_bits_addr = __start_cp_addr(sbi) + sbi->blocks_per_seg -
3133 nm_i->nat_bits_blocks;
3134 for (i = 0; i < nm_i->nat_bits_blocks; i++) {
3135 struct page *page;
3136
3137 page = f2fs_get_meta_page(sbi, nat_bits_addr++);
3138 if (IS_ERR(page))
3139 return PTR_ERR(page);
3140
3141 memcpy(nm_i->nat_bits + (i << F2FS_BLKSIZE_BITS),
3142 page_address(page), F2FS_BLKSIZE);
3143 f2fs_put_page(page, 1);
3144 }
3145
3146 cp_ver |= (cur_cp_crc(ckpt) << 32);
3147 if (cpu_to_le64(cp_ver) != *(__le64 *)nm_i->nat_bits) {
3092 disable_nat_bits(sbi, true);
3148 clear_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
3149 f2fs_notice(sbi, "Disable nat_bits due to incorrect cp_ver (%llu, %llu)",
3150 cp_ver, le64_to_cpu(*(__le64 *)nm_i->nat_bits));
3093 return 0;
3094 }
3095
3151 return 0;
3152 }
3153
3096 nm_i->full_nat_bits = nm_i->nat_bits + 8;
3097 nm_i->empty_nat_bits = nm_i->full_nat_bits + nat_bits_bytes;
3098
3099 f2fs_notice(sbi, "Found nat_bits in checkpoint");
3100 return 0;
3101}
3102
3103static inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi)
3104{
3105 struct f2fs_nm_info *nm_i = NM_I(sbi);
3106 unsigned int i = 0;
3107 nid_t nid, last_nid;
3108
3154 f2fs_notice(sbi, "Found nat_bits in checkpoint");
3155 return 0;
3156}
3157
3158static inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi)
3159{
3160 struct f2fs_nm_info *nm_i = NM_I(sbi);
3161 unsigned int i = 0;
3162 nid_t nid, last_nid;
3163
3109 if (!enabled_nat_bits(sbi, NULL))
3164 if (!is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG))
3110 return;
3111
3112 for (i = 0; i < nm_i->nat_blocks; i++) {
3113 i = find_next_bit_le(nm_i->empty_nat_bits, nm_i->nat_blocks, i);
3114 if (i >= nm_i->nat_blocks)
3115 break;
3116
3117 __set_bit_le(i, nm_i->nat_block_bitmap);

--- 249 unchanged lines hidden --- 		3165 return;
3166
3167 for (i = 0; i < nm_i->nat_blocks; i++) {
3168 i = find_next_bit_le(nm_i->empty_nat_bits, nm_i->nat_blocks, i);
3169 if (i >= nm_i->nat_blocks)
3170 break;
3171
3172 __set_bit_le(i, nm_i->nat_block_bitmap);

--- 249 unchanged lines hidden ---