17c1a000dSChao Yu // SPDX-License-Identifier: GPL-2.0
20a8165d7SJaegeuk Kim /*
37bc09003SJaegeuk Kim * fs/f2fs/gc.c
47bc09003SJaegeuk Kim *
57bc09003SJaegeuk Kim * Copyright (c) 2012 Samsung Electronics Co., Ltd.
67bc09003SJaegeuk Kim * http://www.samsung.com/
77bc09003SJaegeuk Kim */
87bc09003SJaegeuk Kim #include <linux/fs.h>
97bc09003SJaegeuk Kim #include <linux/module.h>
107bc09003SJaegeuk Kim #include <linux/init.h>
117bc09003SJaegeuk Kim #include <linux/f2fs_fs.h>
127bc09003SJaegeuk Kim #include <linux/kthread.h>
137bc09003SJaegeuk Kim #include <linux/delay.h>
147bc09003SJaegeuk Kim #include <linux/freezer.h>
15b4b10061SJaegeuk Kim #include <linux/sched/signal.h>
166691d940SDaeho Jeong #include <linux/random.h>
174034247aSNeilBrown #include <linux/sched/mm.h>
187bc09003SJaegeuk Kim
197bc09003SJaegeuk Kim #include "f2fs.h"
207bc09003SJaegeuk Kim #include "node.h"
217bc09003SJaegeuk Kim #include "segment.h"
227bc09003SJaegeuk Kim #include "gc.h"
2352118743SDaeho Jeong #include "iostat.h"
248e46b3edSNamjae Jeon #include <trace/events/f2fs.h>
257bc09003SJaegeuk Kim
26093749e2SChao Yu static struct kmem_cache *victim_entry_slab;
27093749e2SChao Yu
28da52f8adSJack Qiu static unsigned int count_bits(const unsigned long *addr,
29da52f8adSJack Qiu unsigned int offset, unsigned int len);
30da52f8adSJack Qiu
gc_thread_func(void * data)317bc09003SJaegeuk Kim static int gc_thread_func(void *data)
327bc09003SJaegeuk Kim {
337bc09003SJaegeuk Kim struct f2fs_sb_info *sbi = data;
34b59d0baeSNamjae Jeon struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
357bc09003SJaegeuk Kim wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head;
365911d2d1SChao Yu wait_queue_head_t *fggc_wq = &sbi->gc_thread->fggc_wq;
37b8c502b8SChao Yu unsigned int wait_ms;
38d147ea4aSJaegeuk Kim struct f2fs_gc_control gc_control = {
39d147ea4aSJaegeuk Kim .victim_segno = NULL_SEGNO,
40c58d7c55SJaegeuk Kim .should_migrate_blocks = false,
41c58d7c55SJaegeuk Kim .err_gc_skipped = false };
427bc09003SJaegeuk Kim
43b59d0baeSNamjae Jeon wait_ms = gc_th->min_sleep_time;
447bc09003SJaegeuk Kim
451d7be270SJaegeuk Kim set_freezable();
467bc09003SJaegeuk Kim do {
475911d2d1SChao Yu bool sync_mode, foreground = false;
48bbbc34fdSChao Yu
491d7be270SJaegeuk Kim wait_event_interruptible_timeout(*wq,
50d9872a69SJaegeuk Kim kthread_should_stop() || freezing(current) ||
515911d2d1SChao Yu waitqueue_active(fggc_wq) ||
52d9872a69SJaegeuk Kim gc_th->gc_wake,
531d7be270SJaegeuk Kim msecs_to_jiffies(wait_ms));
541d7be270SJaegeuk Kim
555911d2d1SChao Yu if (test_opt(sbi, GC_MERGE) && waitqueue_active(fggc_wq))
565911d2d1SChao Yu foreground = true;
575911d2d1SChao Yu
58d9872a69SJaegeuk Kim /* give it a try one time */
59d9872a69SJaegeuk Kim if (gc_th->gc_wake)
6045c98f5aSYangtao Li gc_th->gc_wake = false;
61d9872a69SJaegeuk Kim
62b62e71beSChao Yu if (try_to_freeze() || f2fs_readonly(sbi->sb)) {
63274bd9baSChao Yu stat_other_skip_bggc_count(sbi);
647bc09003SJaegeuk Kim continue;
65274bd9baSChao Yu }
667bc09003SJaegeuk Kim if (kthread_should_stop())
677bc09003SJaegeuk Kim break;
687bc09003SJaegeuk Kim
69d6212a5fSChangman Lee if (sbi->sb->s_writers.frozen >= SB_FREEZE_WRITE) {
7088dd8934SChao Yu increase_sleep_time(gc_th, &wait_ms);
71274bd9baSChao Yu stat_other_skip_bggc_count(sbi);
72d6212a5fSChangman Lee continue;
73d6212a5fSChangman Lee }
74d6212a5fSChangman Lee
75c40e15a9SYangtao Li if (time_to_inject(sbi, FAULT_CHECKPOINT))
76a9cfee0eSChao Yu f2fs_stop_checkpoint(sbi, false,
77a9cfee0eSChao Yu STOP_CP_REASON_FAULT_INJECT);
780f348028SChao Yu
79274bd9baSChao Yu if (!sb_start_write_trylock(sbi->sb)) {
80274bd9baSChao Yu stat_other_skip_bggc_count(sbi);
81dc6febb6SChao Yu continue;
82274bd9baSChao Yu }
83dc6febb6SChao Yu
847bc09003SJaegeuk Kim /*
857bc09003SJaegeuk Kim * [GC triggering condition]
867bc09003SJaegeuk Kim * 0. GC is not conducted currently.
877bc09003SJaegeuk Kim * 1. There are enough dirty segments.
887bc09003SJaegeuk Kim * 2. IO subsystem is idle by checking the # of writeback pages.
897bc09003SJaegeuk Kim * 3. IO subsystem is idle by checking the # of requests in
907bc09003SJaegeuk Kim * bdev's request list.
917bc09003SJaegeuk Kim *
92e1c42045Sarter97 * Note) We have to avoid triggering GCs frequently.
937bc09003SJaegeuk Kim * Because it is possible that some segments can be
947bc09003SJaegeuk Kim * invalidated soon after by user update or deletion.
957bc09003SJaegeuk Kim * So, I'd like to wait some time to collect dirty segments.
967bc09003SJaegeuk Kim */
97d98af5f4SDaeho Jeong if (sbi->gc_mode == GC_URGENT_HIGH ||
98d98af5f4SDaeho Jeong sbi->gc_mode == GC_URGENT_MID) {
99d9872a69SJaegeuk Kim wait_ms = gc_th->urgent_sleep_time;
100e4544b63STim Murray f2fs_down_write(&sbi->gc_lock);
101d9872a69SJaegeuk Kim goto do_gc;
102d9872a69SJaegeuk Kim }
103d9872a69SJaegeuk Kim
1045911d2d1SChao Yu if (foreground) {
105e4544b63STim Murray f2fs_down_write(&sbi->gc_lock);
1065911d2d1SChao Yu goto do_gc;
107e4544b63STim Murray } else if (!f2fs_down_write_trylock(&sbi->gc_lock)) {
108274bd9baSChao Yu stat_other_skip_bggc_count(sbi);
10969babac0SJaegeuk Kim goto next;
110274bd9baSChao Yu }
11169babac0SJaegeuk Kim
112a7d10cf3SSahitya Tummala if (!is_idle(sbi, GC_TIME)) {
11388dd8934SChao Yu increase_sleep_time(gc_th, &wait_ms);
114e4544b63STim Murray f2fs_up_write(&sbi->gc_lock);
115274bd9baSChao Yu stat_io_skip_bggc_count(sbi);
116dc6febb6SChao Yu goto next;
1177bc09003SJaegeuk Kim }
1187bc09003SJaegeuk Kim
1197bc09003SJaegeuk Kim if (has_enough_invalid_blocks(sbi))
12088dd8934SChao Yu decrease_sleep_time(gc_th, &wait_ms);
1217bc09003SJaegeuk Kim else
12288dd8934SChao Yu increase_sleep_time(gc_th, &wait_ms);
123d9872a69SJaegeuk Kim do_gc:
1249bf1dcbdSChao Yu stat_inc_gc_call_count(sbi, foreground ?
1259bf1dcbdSChao Yu FOREGROUND : BACKGROUND);
1267bc09003SJaegeuk Kim
127bbbc34fdSChao Yu sync_mode = F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC;
128bbbc34fdSChao Yu
1295911d2d1SChao Yu /* foreground GC was been triggered via f2fs_balance_fs() */
1305911d2d1SChao Yu if (foreground)
1315911d2d1SChao Yu sync_mode = false;
1325911d2d1SChao Yu
133d147ea4aSJaegeuk Kim gc_control.init_gc_type = sync_mode ? FG_GC : BG_GC;
134d147ea4aSJaegeuk Kim gc_control.no_bg_gc = foreground;
135c81d5baeSJaegeuk Kim gc_control.nr_free_secs = foreground ? 1 : 0;
136d147ea4aSJaegeuk Kim
13743727527SJaegeuk Kim /* if return value is not zero, no victim was selected */
1381adaa71eSqixiaoyu1 if (f2fs_gc(sbi, &gc_control)) {
1391adaa71eSqixiaoyu1 /* don't bother wait_ms by foreground gc */
1401adaa71eSqixiaoyu1 if (!foreground)
141b59d0baeSNamjae Jeon wait_ms = gc_th->no_gc_sleep_time;
14226a8057aSYuwei Guan } else {
14326a8057aSYuwei Guan /* reset wait_ms to default sleep time */
14426a8057aSYuwei Guan if (wait_ms == gc_th->no_gc_sleep_time)
14526a8057aSYuwei Guan wait_ms = gc_th->min_sleep_time;
1461adaa71eSqixiaoyu1 }
14781eb8d6eSJaegeuk Kim
1485911d2d1SChao Yu if (foreground)
1495911d2d1SChao Yu wake_up_all(&gc_th->fggc_wq);
1505911d2d1SChao Yu
15184e4214fSJaegeuk Kim trace_f2fs_background_gc(sbi->sb, wait_ms,
15284e4214fSJaegeuk Kim prefree_segments(sbi), free_segments(sbi));
15384e4214fSJaegeuk Kim
1544660f9c0SJaegeuk Kim /* balancing f2fs's metadata periodically */
1557bcd0cfaSChao Yu f2fs_balance_fs_bg(sbi, true);
156dc6febb6SChao Yu next:
157e5a0db6aSYangtao Li if (sbi->gc_mode != GC_NORMAL) {
158e5a0db6aSYangtao Li spin_lock(&sbi->gc_remaining_trials_lock);
159e5a0db6aSYangtao Li if (sbi->gc_remaining_trials) {
160e5a0db6aSYangtao Li sbi->gc_remaining_trials--;
161e5a0db6aSYangtao Li if (!sbi->gc_remaining_trials)
1626359a1aaSYangtao Li sbi->gc_mode = GC_NORMAL;
1636359a1aaSYangtao Li }
164e5a0db6aSYangtao Li spin_unlock(&sbi->gc_remaining_trials_lock);
1656359a1aaSYangtao Li }
166dc6febb6SChao Yu sb_end_write(sbi->sb);
16781eb8d6eSJaegeuk Kim
1687bc09003SJaegeuk Kim } while (!kthread_should_stop());
1697bc09003SJaegeuk Kim return 0;
1707bc09003SJaegeuk Kim }
1717bc09003SJaegeuk Kim
f2fs_start_gc_thread(struct f2fs_sb_info * sbi)1724d57b86dSChao Yu int f2fs_start_gc_thread(struct f2fs_sb_info *sbi)
1737bc09003SJaegeuk Kim {
1741042d60fSNamjae Jeon struct f2fs_gc_kthread *gc_th;
175ec7b1f2dSNamjae Jeon dev_t dev = sbi->sb->s_bdev->bd_dev;
1767bc09003SJaegeuk Kim
1771ecc0c5cSChao Yu gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL);
178146dbcbfSYangtao Li if (!gc_th)
179146dbcbfSYangtao Li return -ENOMEM;
1807bc09003SJaegeuk Kim
181d9872a69SJaegeuk Kim gc_th->urgent_sleep_time = DEF_GC_THREAD_URGENT_SLEEP_TIME;
182b59d0baeSNamjae Jeon gc_th->min_sleep_time = DEF_GC_THREAD_MIN_SLEEP_TIME;
183b59d0baeSNamjae Jeon gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME;
184b59d0baeSNamjae Jeon gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME;
185b59d0baeSNamjae Jeon
18645c98f5aSYangtao Li gc_th->gc_wake = false;
187d2dc095fSNamjae Jeon
1887bc09003SJaegeuk Kim sbi->gc_thread = gc_th;
1897bc09003SJaegeuk Kim init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head);
1905911d2d1SChao Yu init_waitqueue_head(&sbi->gc_thread->fggc_wq);
1917bc09003SJaegeuk Kim sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi,
192ec7b1f2dSNamjae Jeon "f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev));
1937bc09003SJaegeuk Kim if (IS_ERR(gc_th->f2fs_gc_task)) {
194146dbcbfSYangtao Li int err = PTR_ERR(gc_th->f2fs_gc_task);
195146dbcbfSYangtao Li
196c8eb7024SChao Yu kfree(gc_th);
19725718423SNamjae Jeon sbi->gc_thread = NULL;
1987a267f8dSNamjae Jeon return err;
1997bc09003SJaegeuk Kim }
2007bc09003SJaegeuk Kim
201146dbcbfSYangtao Li return 0;
202146dbcbfSYangtao Li }
203146dbcbfSYangtao Li
f2fs_stop_gc_thread(struct f2fs_sb_info * sbi)2044d57b86dSChao Yu void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi)
2057bc09003SJaegeuk Kim {
2067bc09003SJaegeuk Kim struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
2075f029c04SYi Zhuang
2087bc09003SJaegeuk Kim if (!gc_th)
2097bc09003SJaegeuk Kim return;
2107bc09003SJaegeuk Kim kthread_stop(gc_th->f2fs_gc_task);
2115911d2d1SChao Yu wake_up_all(&gc_th->fggc_wq);
212c8eb7024SChao Yu kfree(gc_th);
2137bc09003SJaegeuk Kim sbi->gc_thread = NULL;
2147bc09003SJaegeuk Kim }
2157bc09003SJaegeuk Kim
select_gc_type(struct f2fs_sb_info * sbi,int gc_type)2165b0e9539SJaegeuk Kim static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type)
2177bc09003SJaegeuk Kim {
218093749e2SChao Yu int gc_mode;
219093749e2SChao Yu
220093749e2SChao Yu if (gc_type == BG_GC) {
221093749e2SChao Yu if (sbi->am.atgc_enabled)
222093749e2SChao Yu gc_mode = GC_AT;
223093749e2SChao Yu else
224093749e2SChao Yu gc_mode = GC_CB;
225093749e2SChao Yu } else {
226093749e2SChao Yu gc_mode = GC_GREEDY;
227093749e2SChao Yu }
228d2dc095fSNamjae Jeon
2295b0e9539SJaegeuk Kim switch (sbi->gc_mode) {
2305b0e9539SJaegeuk Kim case GC_IDLE_CB:
231d2dc095fSNamjae Jeon gc_mode = GC_CB;
2325b0e9539SJaegeuk Kim break;
2335b0e9539SJaegeuk Kim case GC_IDLE_GREEDY:
2340e5e8111SDaeho Jeong case GC_URGENT_HIGH:
235d2dc095fSNamjae Jeon gc_mode = GC_GREEDY;
2365b0e9539SJaegeuk Kim break;
237093749e2SChao Yu case GC_IDLE_AT:
238093749e2SChao Yu gc_mode = GC_AT;
239093749e2SChao Yu break;
240d2dc095fSNamjae Jeon }
241093749e2SChao Yu
242d2dc095fSNamjae Jeon return gc_mode;
2437bc09003SJaegeuk Kim }
2447bc09003SJaegeuk Kim
select_policy(struct f2fs_sb_info * sbi,int gc_type,int type,struct victim_sel_policy * p)2457bc09003SJaegeuk Kim static void select_policy(struct f2fs_sb_info *sbi, int gc_type,
2467bc09003SJaegeuk Kim int type, struct victim_sel_policy *p)
2477bc09003SJaegeuk Kim {
2487bc09003SJaegeuk Kim struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2497bc09003SJaegeuk Kim
2504ebefc44SJaegeuk Kim if (p->alloc_mode == SSR) {
2517bc09003SJaegeuk Kim p->gc_mode = GC_GREEDY;
252da52f8adSJack Qiu p->dirty_bitmap = dirty_i->dirty_segmap[type];
253a26b7c8aSJin Xu p->max_search = dirty_i->nr_dirty[type];
2547bc09003SJaegeuk Kim p->ofs_unit = 1;
255093749e2SChao Yu } else if (p->alloc_mode == AT_SSR) {
256093749e2SChao Yu p->gc_mode = GC_GREEDY;
257093749e2SChao Yu p->dirty_bitmap = dirty_i->dirty_segmap[type];
258093749e2SChao Yu p->max_search = dirty_i->nr_dirty[type];
259093749e2SChao Yu p->ofs_unit = 1;
2607bc09003SJaegeuk Kim } else {
2615b0e9539SJaegeuk Kim p->gc_mode = select_gc_type(sbi, gc_type);
2627bc09003SJaegeuk Kim p->ofs_unit = sbi->segs_per_sec;
263da52f8adSJack Qiu if (__is_large_section(sbi)) {
264da52f8adSJack Qiu p->dirty_bitmap = dirty_i->dirty_secmap;
265da52f8adSJack Qiu p->max_search = count_bits(p->dirty_bitmap,
266da52f8adSJack Qiu 0, MAIN_SECS(sbi));
267da52f8adSJack Qiu } else {
268da52f8adSJack Qiu p->dirty_bitmap = dirty_i->dirty_segmap[DIRTY];
269da52f8adSJack Qiu p->max_search = dirty_i->nr_dirty[DIRTY];
270da52f8adSJack Qiu }
2717bc09003SJaegeuk Kim }
272a26b7c8aSJin Xu
2737a88ddb5SChao Yu /*
2747a88ddb5SChao Yu * adjust candidates range, should select all dirty segments for
2757a88ddb5SChao Yu * foreground GC and urgent GC cases.
2767a88ddb5SChao Yu */
277b27bc809SJaegeuk Kim if (gc_type != FG_GC &&
2780e5e8111SDaeho Jeong (sbi->gc_mode != GC_URGENT_HIGH) &&
279093749e2SChao Yu (p->gc_mode != GC_AT && p->alloc_mode != AT_SSR) &&
280b27bc809SJaegeuk Kim p->max_search > sbi->max_victim_search)
281b1c57c1cSJaegeuk Kim p->max_search = sbi->max_victim_search;
282a26b7c8aSJin Xu
283b94929d9SYunlong Song /* let's select beginning hot/small space first in no_heap mode*/
2846691d940SDaeho Jeong if (f2fs_need_rand_seg(sbi))
2858032bf12SJason A. Donenfeld p->offset = get_random_u32_below(MAIN_SECS(sbi) * sbi->segs_per_sec);
2866691d940SDaeho Jeong else if (test_opt(sbi, NOHEAP) &&
287b94929d9SYunlong Song (type == CURSEG_HOT_DATA || IS_NODESEG(type)))
2887a20b8a6SJaegeuk Kim p->offset = 0;
2897a20b8a6SJaegeuk Kim else
290e066b83cSJaegeuk Kim p->offset = SIT_I(sbi)->last_victim[p->gc_mode];
2917bc09003SJaegeuk Kim }
2927bc09003SJaegeuk Kim
get_max_cost(struct f2fs_sb_info * sbi,struct victim_sel_policy * p)2937bc09003SJaegeuk Kim static unsigned int get_max_cost(struct f2fs_sb_info *sbi,
2947bc09003SJaegeuk Kim struct victim_sel_policy *p)
2957bc09003SJaegeuk Kim {
296b7250d2dSJaegeuk Kim /* SSR allocates in a segment unit */
297b7250d2dSJaegeuk Kim if (p->alloc_mode == SSR)
2983519e3f9SChao Yu return sbi->blocks_per_seg;
299093749e2SChao Yu else if (p->alloc_mode == AT_SSR)
300093749e2SChao Yu return UINT_MAX;
301093749e2SChao Yu
302093749e2SChao Yu /* LFS */
3037bc09003SJaegeuk Kim if (p->gc_mode == GC_GREEDY)
304c541a51bSJaegeuk Kim return 2 * sbi->blocks_per_seg * p->ofs_unit;
3057bc09003SJaegeuk Kim else if (p->gc_mode == GC_CB)
3067bc09003SJaegeuk Kim return UINT_MAX;
307093749e2SChao Yu else if (p->gc_mode == GC_AT)
308093749e2SChao Yu return UINT_MAX;
3097bc09003SJaegeuk Kim else /* No other gc_mode */
3107bc09003SJaegeuk Kim return 0;
3117bc09003SJaegeuk Kim }
3127bc09003SJaegeuk Kim
check_bg_victims(struct f2fs_sb_info * sbi)3137bc09003SJaegeuk Kim static unsigned int check_bg_victims(struct f2fs_sb_info *sbi)
3147bc09003SJaegeuk Kim {
3157bc09003SJaegeuk Kim struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
3165ec4e49fSJaegeuk Kim unsigned int secno;
3177bc09003SJaegeuk Kim
3187bc09003SJaegeuk Kim /*
3197bc09003SJaegeuk Kim * If the gc_type is FG_GC, we can select victim segments
3207bc09003SJaegeuk Kim * selected by background GC before.
3217bc09003SJaegeuk Kim * Those segments guarantee they have small valid blocks.
3227bc09003SJaegeuk Kim */
3237cd8558bSJaegeuk Kim for_each_set_bit(secno, dirty_i->victim_secmap, MAIN_SECS(sbi)) {
3245ec4e49fSJaegeuk Kim if (sec_usage_check(sbi, secno))
325b65ee148SChao Yu continue;
3265ec4e49fSJaegeuk Kim clear_bit(secno, dirty_i->victim_secmap);
3274ddb1a4dSJaegeuk Kim return GET_SEG_FROM_SEC(sbi, secno);
3287bc09003SJaegeuk Kim }
3297bc09003SJaegeuk Kim return NULL_SEGNO;
3307bc09003SJaegeuk Kim }
3317bc09003SJaegeuk Kim
get_cb_cost(struct f2fs_sb_info * sbi,unsigned int segno)3327bc09003SJaegeuk Kim static unsigned int get_cb_cost(struct f2fs_sb_info *sbi, unsigned int segno)
3337bc09003SJaegeuk Kim {
3347bc09003SJaegeuk Kim struct sit_info *sit_i = SIT_I(sbi);
3354ddb1a4dSJaegeuk Kim unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
3364ddb1a4dSJaegeuk Kim unsigned int start = GET_SEG_FROM_SEC(sbi, secno);
3377bc09003SJaegeuk Kim unsigned long long mtime = 0;
3387bc09003SJaegeuk Kim unsigned int vblocks;
3397bc09003SJaegeuk Kim unsigned char age = 0;
3407bc09003SJaegeuk Kim unsigned char u;
3417bc09003SJaegeuk Kim unsigned int i;
342de881df9SAravind Ramesh unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi, segno);
3437bc09003SJaegeuk Kim
344de881df9SAravind Ramesh for (i = 0; i < usable_segs_per_sec; i++)
3457bc09003SJaegeuk Kim mtime += get_seg_entry(sbi, start + i)->mtime;
346302bd348SJaegeuk Kim vblocks = get_valid_blocks(sbi, segno, true);
3477bc09003SJaegeuk Kim
348de881df9SAravind Ramesh mtime = div_u64(mtime, usable_segs_per_sec);
349de881df9SAravind Ramesh vblocks = div_u64(vblocks, usable_segs_per_sec);
3507bc09003SJaegeuk Kim
3517bc09003SJaegeuk Kim u = (vblocks * 100) >> sbi->log_blocks_per_seg;
3527bc09003SJaegeuk Kim
353e1c42045Sarter97 /* Handle if the system time has changed by the user */
3547bc09003SJaegeuk Kim if (mtime < sit_i->min_mtime)
3557bc09003SJaegeuk Kim sit_i->min_mtime = mtime;
3567bc09003SJaegeuk Kim if (mtime > sit_i->max_mtime)
3577bc09003SJaegeuk Kim sit_i->max_mtime = mtime;
3587bc09003SJaegeuk Kim if (sit_i->max_mtime != sit_i->min_mtime)
3597bc09003SJaegeuk Kim age = 100 - div64_u64(100 * (mtime - sit_i->min_mtime),
3607bc09003SJaegeuk Kim sit_i->max_mtime - sit_i->min_mtime);
3617bc09003SJaegeuk Kim
3627bc09003SJaegeuk Kim return UINT_MAX - ((100 * (100 - u) * age) / (100 + u));
3637bc09003SJaegeuk Kim }
3647bc09003SJaegeuk Kim
get_gc_cost(struct f2fs_sb_info * sbi,unsigned int segno,struct victim_sel_policy * p)365a57e564dSJin Xu static inline unsigned int get_gc_cost(struct f2fs_sb_info *sbi,
366a57e564dSJin Xu unsigned int segno, struct victim_sel_policy *p)
3677bc09003SJaegeuk Kim {
3687bc09003SJaegeuk Kim if (p->alloc_mode == SSR)
3692afce76aSYunlong Song return get_seg_entry(sbi, segno)->ckpt_valid_blocks;
3707bc09003SJaegeuk Kim
3717bc09003SJaegeuk Kim /* alloc_mode == LFS */
3727bc09003SJaegeuk Kim if (p->gc_mode == GC_GREEDY)
37391f4382bSYunlong Song return get_valid_blocks(sbi, segno, true);
374093749e2SChao Yu else if (p->gc_mode == GC_CB)
3757bc09003SJaegeuk Kim return get_cb_cost(sbi, segno);
376093749e2SChao Yu
377093749e2SChao Yu f2fs_bug_on(sbi, 1);
378093749e2SChao Yu return 0;
3797bc09003SJaegeuk Kim }
3807bc09003SJaegeuk Kim
count_bits(const unsigned long * addr,unsigned int offset,unsigned int len)381688159b6SFan Li static unsigned int count_bits(const unsigned long *addr,
382688159b6SFan Li unsigned int offset, unsigned int len)
383688159b6SFan Li {
384688159b6SFan Li unsigned int end = offset + len, sum = 0;
385688159b6SFan Li
386688159b6SFan Li while (offset < end) {
387688159b6SFan Li if (test_bit(offset++, addr))
388688159b6SFan Li ++sum;
389688159b6SFan Li }
390688159b6SFan Li return sum;
391688159b6SFan Li }
392688159b6SFan Li
f2fs_check_victim_tree(struct f2fs_sb_info * sbi,struct rb_root_cached * root)393043d2d00SJaegeuk Kim static bool f2fs_check_victim_tree(struct f2fs_sb_info *sbi,
394043d2d00SJaegeuk Kim struct rb_root_cached *root)
395043d2d00SJaegeuk Kim {
396043d2d00SJaegeuk Kim #ifdef CONFIG_F2FS_CHECK_FS
397043d2d00SJaegeuk Kim struct rb_node *cur = rb_first_cached(root), *next;
398043d2d00SJaegeuk Kim struct victim_entry *cur_ve, *next_ve;
399043d2d00SJaegeuk Kim
400043d2d00SJaegeuk Kim while (cur) {
401043d2d00SJaegeuk Kim next = rb_next(cur);
402043d2d00SJaegeuk Kim if (!next)
403043d2d00SJaegeuk Kim return true;
404043d2d00SJaegeuk Kim
405043d2d00SJaegeuk Kim cur_ve = rb_entry(cur, struct victim_entry, rb_node);
406043d2d00SJaegeuk Kim next_ve = rb_entry(next, struct victim_entry, rb_node);
407043d2d00SJaegeuk Kim
408043d2d00SJaegeuk Kim if (cur_ve->mtime > next_ve->mtime) {
409043d2d00SJaegeuk Kim f2fs_info(sbi, "broken victim_rbtree, "
410043d2d00SJaegeuk Kim "cur_mtime(%llu) next_mtime(%llu)",
411043d2d00SJaegeuk Kim cur_ve->mtime, next_ve->mtime);
412043d2d00SJaegeuk Kim return false;
413043d2d00SJaegeuk Kim }
414043d2d00SJaegeuk Kim cur = next;
415043d2d00SJaegeuk Kim }
416043d2d00SJaegeuk Kim #endif
417043d2d00SJaegeuk Kim return true;
418043d2d00SJaegeuk Kim }
419043d2d00SJaegeuk Kim
__lookup_victim_entry(struct f2fs_sb_info * sbi,unsigned long long mtime)420043d2d00SJaegeuk Kim static struct victim_entry *__lookup_victim_entry(struct f2fs_sb_info *sbi,
421043d2d00SJaegeuk Kim unsigned long long mtime)
422043d2d00SJaegeuk Kim {
423043d2d00SJaegeuk Kim struct atgc_management *am = &sbi->am;
424043d2d00SJaegeuk Kim struct rb_node *node = am->root.rb_root.rb_node;
425043d2d00SJaegeuk Kim struct victim_entry *ve = NULL;
426043d2d00SJaegeuk Kim
427043d2d00SJaegeuk Kim while (node) {
428043d2d00SJaegeuk Kim ve = rb_entry(node, struct victim_entry, rb_node);
429043d2d00SJaegeuk Kim
430043d2d00SJaegeuk Kim if (mtime < ve->mtime)
431043d2d00SJaegeuk Kim node = node->rb_left;
432043d2d00SJaegeuk Kim else
433043d2d00SJaegeuk Kim node = node->rb_right;
434043d2d00SJaegeuk Kim }
435043d2d00SJaegeuk Kim return ve;
436043d2d00SJaegeuk Kim }
437043d2d00SJaegeuk Kim
__create_victim_entry(struct f2fs_sb_info * sbi,unsigned long long mtime,unsigned int segno)438043d2d00SJaegeuk Kim static struct victim_entry *__create_victim_entry(struct f2fs_sb_info *sbi,
439043d2d00SJaegeuk Kim unsigned long long mtime, unsigned int segno)
440093749e2SChao Yu {
441093749e2SChao Yu struct atgc_management *am = &sbi->am;
442093749e2SChao Yu struct victim_entry *ve;
443093749e2SChao Yu
444043d2d00SJaegeuk Kim ve = f2fs_kmem_cache_alloc(victim_entry_slab, GFP_NOFS, true, NULL);
445093749e2SChao Yu
446093749e2SChao Yu ve->mtime = mtime;
447093749e2SChao Yu ve->segno = segno;
448093749e2SChao Yu
449093749e2SChao Yu list_add_tail(&ve->list, &am->victim_list);
450093749e2SChao Yu am->victim_count++;
451093749e2SChao Yu
452093749e2SChao Yu return ve;
453093749e2SChao Yu }
454093749e2SChao Yu
__insert_victim_entry(struct f2fs_sb_info * sbi,unsigned long long mtime,unsigned int segno)455043d2d00SJaegeuk Kim static void __insert_victim_entry(struct f2fs_sb_info *sbi,
456093749e2SChao Yu unsigned long long mtime, unsigned int segno)
457093749e2SChao Yu {
458093749e2SChao Yu struct atgc_management *am = &sbi->am;
459043d2d00SJaegeuk Kim struct rb_root_cached *root = &am->root;
460043d2d00SJaegeuk Kim struct rb_node **p = &root->rb_root.rb_node;
461093749e2SChao Yu struct rb_node *parent = NULL;
462043d2d00SJaegeuk Kim struct victim_entry *ve;
463093749e2SChao Yu bool left_most = true;
464093749e2SChao Yu
465043d2d00SJaegeuk Kim /* look up rb tree to find parent node */
466043d2d00SJaegeuk Kim while (*p) {
467043d2d00SJaegeuk Kim parent = *p;
468043d2d00SJaegeuk Kim ve = rb_entry(parent, struct victim_entry, rb_node);
469043d2d00SJaegeuk Kim
470043d2d00SJaegeuk Kim if (mtime < ve->mtime) {
471043d2d00SJaegeuk Kim p = &(*p)->rb_left;
472043d2d00SJaegeuk Kim } else {
473043d2d00SJaegeuk Kim p = &(*p)->rb_right;
474043d2d00SJaegeuk Kim left_most = false;
475043d2d00SJaegeuk Kim }
476043d2d00SJaegeuk Kim }
477043d2d00SJaegeuk Kim
478043d2d00SJaegeuk Kim ve = __create_victim_entry(sbi, mtime, segno);
479043d2d00SJaegeuk Kim
480043d2d00SJaegeuk Kim rb_link_node(&ve->rb_node, parent, p);
481043d2d00SJaegeuk Kim rb_insert_color_cached(&ve->rb_node, root, left_most);
482093749e2SChao Yu }
483093749e2SChao Yu
add_victim_entry(struct f2fs_sb_info * sbi,struct victim_sel_policy * p,unsigned int segno)484093749e2SChao Yu static void add_victim_entry(struct f2fs_sb_info *sbi,
485093749e2SChao Yu struct victim_sel_policy *p, unsigned int segno)
486093749e2SChao Yu {
487093749e2SChao Yu struct sit_info *sit_i = SIT_I(sbi);
488093749e2SChao Yu unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
489093749e2SChao Yu unsigned int start = GET_SEG_FROM_SEC(sbi, secno);
490093749e2SChao Yu unsigned long long mtime = 0;
491093749e2SChao Yu unsigned int i;
492093749e2SChao Yu
493093749e2SChao Yu if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
494093749e2SChao Yu if (p->gc_mode == GC_AT &&
495093749e2SChao Yu get_valid_blocks(sbi, segno, true) == 0)
496093749e2SChao Yu return;
497093749e2SChao Yu }
498093749e2SChao Yu
499093749e2SChao Yu for (i = 0; i < sbi->segs_per_sec; i++)
500093749e2SChao Yu mtime += get_seg_entry(sbi, start + i)->mtime;
501093749e2SChao Yu mtime = div_u64(mtime, sbi->segs_per_sec);
502093749e2SChao Yu
503093749e2SChao Yu /* Handle if the system time has changed by the user */
504093749e2SChao Yu if (mtime < sit_i->min_mtime)
505093749e2SChao Yu sit_i->min_mtime = mtime;
506093749e2SChao Yu if (mtime > sit_i->max_mtime)
507093749e2SChao Yu sit_i->max_mtime = mtime;
508093749e2SChao Yu if (mtime < sit_i->dirty_min_mtime)
509093749e2SChao Yu sit_i->dirty_min_mtime = mtime;
510093749e2SChao Yu if (mtime > sit_i->dirty_max_mtime)
511093749e2SChao Yu sit_i->dirty_max_mtime = mtime;
512093749e2SChao Yu
513093749e2SChao Yu /* don't choose young section as candidate */
514093749e2SChao Yu if (sit_i->dirty_max_mtime - mtime < p->age_threshold)
515093749e2SChao Yu return;
516093749e2SChao Yu
517043d2d00SJaegeuk Kim __insert_victim_entry(sbi, mtime, segno);
518093749e2SChao Yu }
519093749e2SChao Yu
atgc_lookup_victim(struct f2fs_sb_info * sbi,struct victim_sel_policy * p)520093749e2SChao Yu static void atgc_lookup_victim(struct f2fs_sb_info *sbi,
521093749e2SChao Yu struct victim_sel_policy *p)
522093749e2SChao Yu {
523093749e2SChao Yu struct sit_info *sit_i = SIT_I(sbi);
524093749e2SChao Yu struct atgc_management *am = &sbi->am;
525093749e2SChao Yu struct rb_root_cached *root = &am->root;
526093749e2SChao Yu struct rb_node *node;
527093749e2SChao Yu struct victim_entry *ve;
528093749e2SChao Yu unsigned long long total_time;
529093749e2SChao Yu unsigned long long age, u, accu;
530093749e2SChao Yu unsigned long long max_mtime = sit_i->dirty_max_mtime;
531093749e2SChao Yu unsigned long long min_mtime = sit_i->dirty_min_mtime;
532074b5ea2SJaegeuk Kim unsigned int sec_blocks = CAP_BLKS_PER_SEC(sbi);
533093749e2SChao Yu unsigned int vblocks;
534093749e2SChao Yu unsigned int dirty_threshold = max(am->max_candidate_count,
535093749e2SChao Yu am->candidate_ratio *
536093749e2SChao Yu am->victim_count / 100);
537093749e2SChao Yu unsigned int age_weight = am->age_weight;
538093749e2SChao Yu unsigned int cost;
539093749e2SChao Yu unsigned int iter = 0;
540093749e2SChao Yu
541093749e2SChao Yu if (max_mtime < min_mtime)
542093749e2SChao Yu return;
543093749e2SChao Yu
544093749e2SChao Yu max_mtime += 1;
545093749e2SChao Yu total_time = max_mtime - min_mtime;
546093749e2SChao Yu
547093749e2SChao Yu accu = div64_u64(ULLONG_MAX, total_time);
548093749e2SChao Yu accu = min_t(unsigned long long, div_u64(accu, 100),
549093749e2SChao Yu DEFAULT_ACCURACY_CLASS);
550093749e2SChao Yu
551093749e2SChao Yu node = rb_first_cached(root);
552093749e2SChao Yu next:
553043d2d00SJaegeuk Kim ve = rb_entry_safe(node, struct victim_entry, rb_node);
554043d2d00SJaegeuk Kim if (!ve)
555093749e2SChao Yu return;
556093749e2SChao Yu
557093749e2SChao Yu if (ve->mtime >= max_mtime || ve->mtime < min_mtime)
558093749e2SChao Yu goto skip;
559093749e2SChao Yu
560093749e2SChao Yu /* age = 10000 * x% * 60 */
561093749e2SChao Yu age = div64_u64(accu * (max_mtime - ve->mtime), total_time) *
562093749e2SChao Yu age_weight;
563093749e2SChao Yu
564093749e2SChao Yu vblocks = get_valid_blocks(sbi, ve->segno, true);
565093749e2SChao Yu f2fs_bug_on(sbi, !vblocks || vblocks == sec_blocks);
566093749e2SChao Yu
567093749e2SChao Yu /* u = 10000 * x% * 40 */
568093749e2SChao Yu u = div64_u64(accu * (sec_blocks - vblocks), sec_blocks) *
569093749e2SChao Yu (100 - age_weight);
570093749e2SChao Yu
571093749e2SChao Yu f2fs_bug_on(sbi, age + u >= UINT_MAX);
572093749e2SChao Yu
573093749e2SChao Yu cost = UINT_MAX - (age + u);
574093749e2SChao Yu iter++;
575093749e2SChao Yu
576093749e2SChao Yu if (cost < p->min_cost ||
577093749e2SChao Yu (cost == p->min_cost && age > p->oldest_age)) {
578093749e2SChao Yu p->min_cost = cost;
579093749e2SChao Yu p->oldest_age = age;
580093749e2SChao Yu p->min_segno = ve->segno;
581093749e2SChao Yu }
582093749e2SChao Yu skip:
583093749e2SChao Yu if (iter < dirty_threshold) {
584093749e2SChao Yu node = rb_next(node);
585093749e2SChao Yu goto next;
586093749e2SChao Yu }
587093749e2SChao Yu }
588093749e2SChao Yu
589093749e2SChao Yu /*
590093749e2SChao Yu * select candidates around source section in range of
591093749e2SChao Yu * [target - dirty_threshold, target + dirty_threshold]
592093749e2SChao Yu */
atssr_lookup_victim(struct f2fs_sb_info * sbi,struct victim_sel_policy * p)593093749e2SChao Yu static void atssr_lookup_victim(struct f2fs_sb_info *sbi,
594093749e2SChao Yu struct victim_sel_policy *p)
595093749e2SChao Yu {
596093749e2SChao Yu struct sit_info *sit_i = SIT_I(sbi);
597093749e2SChao Yu struct atgc_management *am = &sbi->am;
598093749e2SChao Yu struct victim_entry *ve;
599093749e2SChao Yu unsigned long long age;
600093749e2SChao Yu unsigned long long max_mtime = sit_i->dirty_max_mtime;
601093749e2SChao Yu unsigned long long min_mtime = sit_i->dirty_min_mtime;
602093749e2SChao Yu unsigned int seg_blocks = sbi->blocks_per_seg;
603093749e2SChao Yu unsigned int vblocks;
604093749e2SChao Yu unsigned int dirty_threshold = max(am->max_candidate_count,
605093749e2SChao Yu am->candidate_ratio *
606093749e2SChao Yu am->victim_count / 100);
607043d2d00SJaegeuk Kim unsigned int cost, iter;
608093749e2SChao Yu int stage = 0;
609093749e2SChao Yu
610093749e2SChao Yu if (max_mtime < min_mtime)
611093749e2SChao Yu return;
612093749e2SChao Yu max_mtime += 1;
613093749e2SChao Yu next_stage:
614043d2d00SJaegeuk Kim iter = 0;
615043d2d00SJaegeuk Kim ve = __lookup_victim_entry(sbi, p->age);
616093749e2SChao Yu next_node:
617043d2d00SJaegeuk Kim if (!ve) {
618043d2d00SJaegeuk Kim if (stage++ == 0)
619043d2d00SJaegeuk Kim goto next_stage;
620093749e2SChao Yu return;
621093749e2SChao Yu }
622093749e2SChao Yu
623093749e2SChao Yu if (ve->mtime >= max_mtime || ve->mtime < min_mtime)
624093749e2SChao Yu goto skip_node;
625093749e2SChao Yu
626093749e2SChao Yu age = max_mtime - ve->mtime;
627093749e2SChao Yu
628093749e2SChao Yu vblocks = get_seg_entry(sbi, ve->segno)->ckpt_valid_blocks;
629093749e2SChao Yu f2fs_bug_on(sbi, !vblocks);
630093749e2SChao Yu
631093749e2SChao Yu /* rare case */
632093749e2SChao Yu if (vblocks == seg_blocks)
633093749e2SChao Yu goto skip_node;
634093749e2SChao Yu
635093749e2SChao Yu iter++;
636093749e2SChao Yu
637093749e2SChao Yu age = max_mtime - abs(p->age - age);
638093749e2SChao Yu cost = UINT_MAX - vblocks;
639093749e2SChao Yu
640093749e2SChao Yu if (cost < p->min_cost ||
641093749e2SChao Yu (cost == p->min_cost && age > p->oldest_age)) {
642093749e2SChao Yu p->min_cost = cost;
643093749e2SChao Yu p->oldest_age = age;
644093749e2SChao Yu p->min_segno = ve->segno;
645093749e2SChao Yu }
646093749e2SChao Yu skip_node:
647093749e2SChao Yu if (iter < dirty_threshold) {
648043d2d00SJaegeuk Kim ve = rb_entry(stage == 0 ? rb_prev(&ve->rb_node) :
649043d2d00SJaegeuk Kim rb_next(&ve->rb_node),
650043d2d00SJaegeuk Kim struct victim_entry, rb_node);
651093749e2SChao Yu goto next_node;
652093749e2SChao Yu }
653043d2d00SJaegeuk Kim
654043d2d00SJaegeuk Kim if (stage++ == 0)
655093749e2SChao Yu goto next_stage;
656093749e2SChao Yu }
657043d2d00SJaegeuk Kim
lookup_victim_by_age(struct f2fs_sb_info * sbi,struct victim_sel_policy * p)658093749e2SChao Yu static void lookup_victim_by_age(struct f2fs_sb_info *sbi,
659093749e2SChao Yu struct victim_sel_policy *p)
660093749e2SChao Yu {
661043d2d00SJaegeuk Kim f2fs_bug_on(sbi, !f2fs_check_victim_tree(sbi, &sbi->am.root));
662093749e2SChao Yu
663093749e2SChao Yu if (p->gc_mode == GC_AT)
664093749e2SChao Yu atgc_lookup_victim(sbi, p);
665093749e2SChao Yu else if (p->alloc_mode == AT_SSR)
666093749e2SChao Yu atssr_lookup_victim(sbi, p);
667093749e2SChao Yu else
668093749e2SChao Yu f2fs_bug_on(sbi, 1);
669093749e2SChao Yu }
670093749e2SChao Yu
release_victim_entry(struct f2fs_sb_info * sbi)671093749e2SChao Yu static void release_victim_entry(struct f2fs_sb_info *sbi)
672093749e2SChao Yu {
673093749e2SChao Yu struct atgc_management *am = &sbi->am;
674093749e2SChao Yu struct victim_entry *ve, *tmp;
675093749e2SChao Yu
676093749e2SChao Yu list_for_each_entry_safe(ve, tmp, &am->victim_list, list) {
677093749e2SChao Yu list_del(&ve->list);
678093749e2SChao Yu kmem_cache_free(victim_entry_slab, ve);
679093749e2SChao Yu am->victim_count--;
680093749e2SChao Yu }
681093749e2SChao Yu
682093749e2SChao Yu am->root = RB_ROOT_CACHED;
683093749e2SChao Yu
684093749e2SChao Yu f2fs_bug_on(sbi, am->victim_count);
685093749e2SChao Yu f2fs_bug_on(sbi, !list_empty(&am->victim_list));
686093749e2SChao Yu }
687093749e2SChao Yu
f2fs_pin_section(struct f2fs_sb_info * sbi,unsigned int segno)68871419129SChao Yu static bool f2fs_pin_section(struct f2fs_sb_info *sbi, unsigned int segno)
68971419129SChao Yu {
69071419129SChao Yu struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
69171419129SChao Yu unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
69271419129SChao Yu
69371419129SChao Yu if (!dirty_i->enable_pin_section)
69471419129SChao Yu return false;
69571419129SChao Yu if (!test_and_set_bit(secno, dirty_i->pinned_secmap))
69671419129SChao Yu dirty_i->pinned_secmap_cnt++;
69771419129SChao Yu return true;
69871419129SChao Yu }
69971419129SChao Yu
f2fs_pinned_section_exists(struct dirty_seglist_info * dirty_i)70071419129SChao Yu static bool f2fs_pinned_section_exists(struct dirty_seglist_info *dirty_i)
70171419129SChao Yu {
70271419129SChao Yu return dirty_i->pinned_secmap_cnt;
70371419129SChao Yu }
70471419129SChao Yu
f2fs_section_is_pinned(struct dirty_seglist_info * dirty_i,unsigned int secno)70571419129SChao Yu static bool f2fs_section_is_pinned(struct dirty_seglist_info *dirty_i,
70671419129SChao Yu unsigned int secno)
70771419129SChao Yu {
70871419129SChao Yu return dirty_i->enable_pin_section &&
70971419129SChao Yu f2fs_pinned_section_exists(dirty_i) &&
71071419129SChao Yu test_bit(secno, dirty_i->pinned_secmap);
71171419129SChao Yu }
71271419129SChao Yu
f2fs_unpin_all_sections(struct f2fs_sb_info * sbi,bool enable)71371419129SChao Yu static void f2fs_unpin_all_sections(struct f2fs_sb_info *sbi, bool enable)
71471419129SChao Yu {
71571419129SChao Yu unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
71671419129SChao Yu
71771419129SChao Yu if (f2fs_pinned_section_exists(DIRTY_I(sbi))) {
71871419129SChao Yu memset(DIRTY_I(sbi)->pinned_secmap, 0, bitmap_size);
71971419129SChao Yu DIRTY_I(sbi)->pinned_secmap_cnt = 0;
72071419129SChao Yu }
72171419129SChao Yu DIRTY_I(sbi)->enable_pin_section = enable;
72271419129SChao Yu }
72371419129SChao Yu
f2fs_gc_pinned_control(struct inode * inode,int gc_type,unsigned int segno)72471419129SChao Yu static int f2fs_gc_pinned_control(struct inode *inode, int gc_type,
72571419129SChao Yu unsigned int segno)
72671419129SChao Yu {
72771419129SChao Yu if (!f2fs_is_pinned_file(inode))
72871419129SChao Yu return 0;
72971419129SChao Yu if (gc_type != FG_GC)
73071419129SChao Yu return -EBUSY;
73171419129SChao Yu if (!f2fs_pin_section(F2FS_I_SB(inode), segno))
73271419129SChao Yu f2fs_pin_file_control(inode, true);
73371419129SChao Yu return -EAGAIN;
73471419129SChao Yu }
73571419129SChao Yu
7360a8165d7SJaegeuk Kim /*
737111d2495SMasanari Iida * This function is called from two paths.
7387bc09003SJaegeuk Kim * One is garbage collection and the other is SSR segment selection.
7397bc09003SJaegeuk Kim * When it is called during GC, it just gets a victim segment
7407bc09003SJaegeuk Kim * and it does not remove it from dirty seglist.
7417bc09003SJaegeuk Kim * When it is called from SSR segment selection, it finds a segment
7427bc09003SJaegeuk Kim * which has minimum valid blocks and removes it from dirty seglist.
7437bc09003SJaegeuk Kim */
f2fs_get_victim(struct f2fs_sb_info * sbi,unsigned int * result,int gc_type,int type,char alloc_mode,unsigned long long age)74419e0e21aSYangtao Li int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result,
74519e0e21aSYangtao Li int gc_type, int type, char alloc_mode,
74619e0e21aSYangtao Li unsigned long long age)
7477bc09003SJaegeuk Kim {
7487bc09003SJaegeuk Kim struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
749e066b83cSJaegeuk Kim struct sit_info *sm = SIT_I(sbi);
7507bc09003SJaegeuk Kim struct victim_sel_policy p;
7513fa56503SSheng Yong unsigned int secno, last_victim;
75204f0b2eaSQiuyang Sun unsigned int last_segment;
753093749e2SChao Yu unsigned int nsearched;
754093749e2SChao Yu bool is_atgc;
75597767500SQilong Zhang int ret = 0;
7567bc09003SJaegeuk Kim
757210f41bcSChao Yu mutex_lock(&dirty_i->seglist_lock);
75804f0b2eaSQiuyang Sun last_segment = MAIN_SECS(sbi) * sbi->segs_per_sec;
759210f41bcSChao Yu
7607bc09003SJaegeuk Kim p.alloc_mode = alloc_mode;
761093749e2SChao Yu p.age = age;
762093749e2SChao Yu p.age_threshold = sbi->am.age_threshold;
7637bc09003SJaegeuk Kim
764093749e2SChao Yu retry:
765093749e2SChao Yu select_policy(sbi, gc_type, type, &p);
7667bc09003SJaegeuk Kim p.min_segno = NULL_SEGNO;
767093749e2SChao Yu p.oldest_age = 0;
7683fa56503SSheng Yong p.min_cost = get_max_cost(sbi, &p);
7697bc09003SJaegeuk Kim
770093749e2SChao Yu is_atgc = (p.gc_mode == GC_AT || p.alloc_mode == AT_SSR);
771093749e2SChao Yu nsearched = 0;
772093749e2SChao Yu
773093749e2SChao Yu if (is_atgc)
774093749e2SChao Yu SIT_I(sbi)->dirty_min_mtime = ULLONG_MAX;
775093749e2SChao Yu
776e066b83cSJaegeuk Kim if (*result != NULL_SEGNO) {
77797767500SQilong Zhang if (!get_valid_blocks(sbi, *result, false)) {
77897767500SQilong Zhang ret = -ENODATA;
77997767500SQilong Zhang goto out;
78097767500SQilong Zhang }
78197767500SQilong Zhang
78297767500SQilong Zhang if (sec_usage_check(sbi, GET_SEC_FROM_SEG(sbi, *result)))
78397767500SQilong Zhang ret = -EBUSY;
78497767500SQilong Zhang else
785e066b83cSJaegeuk Kim p.min_segno = *result;
786e066b83cSJaegeuk Kim goto out;
787e066b83cSJaegeuk Kim }
788e066b83cSJaegeuk Kim
78997767500SQilong Zhang ret = -ENODATA;
7903342bb30SChao Yu if (p.max_search == 0)
7913342bb30SChao Yu goto out;
7923342bb30SChao Yu
793e3080b01SChao Yu if (__is_large_section(sbi) && p.alloc_mode == LFS) {
794e3080b01SChao Yu if (sbi->next_victim_seg[BG_GC] != NULL_SEGNO) {
795e3080b01SChao Yu p.min_segno = sbi->next_victim_seg[BG_GC];
796e3080b01SChao Yu *result = p.min_segno;
797e3080b01SChao Yu sbi->next_victim_seg[BG_GC] = NULL_SEGNO;
798e3080b01SChao Yu goto got_result;
799e3080b01SChao Yu }
800e3080b01SChao Yu if (gc_type == FG_GC &&
801e3080b01SChao Yu sbi->next_victim_seg[FG_GC] != NULL_SEGNO) {
802e3080b01SChao Yu p.min_segno = sbi->next_victim_seg[FG_GC];
803e3080b01SChao Yu *result = p.min_segno;
804e3080b01SChao Yu sbi->next_victim_seg[FG_GC] = NULL_SEGNO;
805e3080b01SChao Yu goto got_result;
806e3080b01SChao Yu }
807e3080b01SChao Yu }
808e3080b01SChao Yu
809e066b83cSJaegeuk Kim last_victim = sm->last_victim[p.gc_mode];
8107bc09003SJaegeuk Kim if (p.alloc_mode == LFS && gc_type == FG_GC) {
8117bc09003SJaegeuk Kim p.min_segno = check_bg_victims(sbi);
8127bc09003SJaegeuk Kim if (p.min_segno != NULL_SEGNO)
8137bc09003SJaegeuk Kim goto got_it;
8147bc09003SJaegeuk Kim }
8157bc09003SJaegeuk Kim
8167bc09003SJaegeuk Kim while (1) {
817da52f8adSJack Qiu unsigned long cost, *dirty_bitmap;
818da52f8adSJack Qiu unsigned int unit_no, segno;
8197bc09003SJaegeuk Kim
820da52f8adSJack Qiu dirty_bitmap = p.dirty_bitmap;
821da52f8adSJack Qiu unit_no = find_next_bit(dirty_bitmap,
822da52f8adSJack Qiu last_segment / p.ofs_unit,
823da52f8adSJack Qiu p.offset / p.ofs_unit);
824da52f8adSJack Qiu segno = unit_no * p.ofs_unit;
825a43f7ec3SChao Yu if (segno >= last_segment) {
826e066b83cSJaegeuk Kim if (sm->last_victim[p.gc_mode]) {
827e066b83cSJaegeuk Kim last_segment =
828e066b83cSJaegeuk Kim sm->last_victim[p.gc_mode];
829e066b83cSJaegeuk Kim sm->last_victim[p.gc_mode] = 0;
8307bc09003SJaegeuk Kim p.offset = 0;
8317bc09003SJaegeuk Kim continue;
8327bc09003SJaegeuk Kim }
8337bc09003SJaegeuk Kim break;
8347bc09003SJaegeuk Kim }
835a57e564dSJin Xu
836a57e564dSJin Xu p.offset = segno + p.ofs_unit;
837688159b6SFan Li nsearched++;
838688159b6SFan Li
839bbf9f7d9SSahitya Tummala #ifdef CONFIG_F2FS_CHECK_FS
840bbf9f7d9SSahitya Tummala /*
841bbf9f7d9SSahitya Tummala * skip selecting the invalid segno (that is failed due to block
842bbf9f7d9SSahitya Tummala * validity check failure during GC) to avoid endless GC loop in
843bbf9f7d9SSahitya Tummala * such cases.
844bbf9f7d9SSahitya Tummala */
845bbf9f7d9SSahitya Tummala if (test_bit(segno, sm->invalid_segmap))
846bbf9f7d9SSahitya Tummala goto next;
847bbf9f7d9SSahitya Tummala #endif
848bbf9f7d9SSahitya Tummala
8494ddb1a4dSJaegeuk Kim secno = GET_SEC_FROM_SEG(sbi, segno);
8507bc09003SJaegeuk Kim
8515ec4e49fSJaegeuk Kim if (sec_usage_check(sbi, secno))
852688159b6SFan Li goto next;
85361461fc9SChao Yu
8544354994fSDaniel Rosenberg /* Don't touch checkpointed data */
85561461fc9SChao Yu if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
85661461fc9SChao Yu if (p.alloc_mode == LFS) {
85761461fc9SChao Yu /*
85861461fc9SChao Yu * LFS is set to find source section during GC.
85961461fc9SChao Yu * The victim should have no checkpointed data.
86061461fc9SChao Yu */
86161461fc9SChao Yu if (get_ckpt_valid_blocks(sbi, segno, true))
8624354994fSDaniel Rosenberg goto next;
86361461fc9SChao Yu } else {
86461461fc9SChao Yu /*
86561461fc9SChao Yu * SSR | AT_SSR are set to find target segment
86661461fc9SChao Yu * for writes which can be full by checkpointed
86761461fc9SChao Yu * and newly written blocks.
86861461fc9SChao Yu */
86961461fc9SChao Yu if (!f2fs_segment_has_free_slot(sbi, segno))
87061461fc9SChao Yu goto next;
87161461fc9SChao Yu }
87261461fc9SChao Yu }
87361461fc9SChao Yu
8745ec4e49fSJaegeuk Kim if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap))
875688159b6SFan Li goto next;
8767bc09003SJaegeuk Kim
87771419129SChao Yu if (gc_type == FG_GC && f2fs_section_is_pinned(dirty_i, secno))
87871419129SChao Yu goto next;
87971419129SChao Yu
880093749e2SChao Yu if (is_atgc) {
881093749e2SChao Yu add_victim_entry(sbi, &p, segno);
882093749e2SChao Yu goto next;
883093749e2SChao Yu }
884093749e2SChao Yu
8857bc09003SJaegeuk Kim cost = get_gc_cost(sbi, segno, &p);
8867bc09003SJaegeuk Kim
8877bc09003SJaegeuk Kim if (p.min_cost > cost) {
8887bc09003SJaegeuk Kim p.min_segno = segno;
8897bc09003SJaegeuk Kim p.min_cost = cost;
890a57e564dSJin Xu }
891688159b6SFan Li next:
892688159b6SFan Li if (nsearched >= p.max_search) {
893e066b83cSJaegeuk Kim if (!sm->last_victim[p.gc_mode] && segno <= last_victim)
894da52f8adSJack Qiu sm->last_victim[p.gc_mode] =
895da52f8adSJack Qiu last_victim + p.ofs_unit;
8964ce53776SJaegeuk Kim else
897da52f8adSJack Qiu sm->last_victim[p.gc_mode] = segno + p.ofs_unit;
89804f0b2eaSQiuyang Sun sm->last_victim[p.gc_mode] %=
89904f0b2eaSQiuyang Sun (MAIN_SECS(sbi) * sbi->segs_per_sec);
9007bc09003SJaegeuk Kim break;
9017bc09003SJaegeuk Kim }
9027bc09003SJaegeuk Kim }
903093749e2SChao Yu
904093749e2SChao Yu /* get victim for GC_AT/AT_SSR */
905093749e2SChao Yu if (is_atgc) {
906093749e2SChao Yu lookup_victim_by_age(sbi, &p);
907093749e2SChao Yu release_victim_entry(sbi);
908093749e2SChao Yu }
909093749e2SChao Yu
910093749e2SChao Yu if (is_atgc && p.min_segno == NULL_SEGNO &&
911093749e2SChao Yu sm->elapsed_time < p.age_threshold) {
912093749e2SChao Yu p.age_threshold = 0;
913093749e2SChao Yu goto retry;
914093749e2SChao Yu }
915093749e2SChao Yu
9167bc09003SJaegeuk Kim if (p.min_segno != NULL_SEGNO) {
917b2b3460aSNamjae Jeon got_it:
918e3080b01SChao Yu *result = (p.min_segno / p.ofs_unit) * p.ofs_unit;
919e3080b01SChao Yu got_result:
9207bc09003SJaegeuk Kim if (p.alloc_mode == LFS) {
9214ddb1a4dSJaegeuk Kim secno = GET_SEC_FROM_SEG(sbi, p.min_segno);
9225ec4e49fSJaegeuk Kim if (gc_type == FG_GC)
9235ec4e49fSJaegeuk Kim sbi->cur_victim_sec = secno;
9245ec4e49fSJaegeuk Kim else
9255ec4e49fSJaegeuk Kim set_bit(secno, dirty_i->victim_secmap);
9267bc09003SJaegeuk Kim }
92797767500SQilong Zhang ret = 0;
9288e46b3edSNamjae Jeon
929e3c59108SSahitya Tummala }
930e3c59108SSahitya Tummala out:
931e3c59108SSahitya Tummala if (p.min_segno != NULL_SEGNO)
9328e46b3edSNamjae Jeon trace_f2fs_get_victim(sbi->sb, type, gc_type, &p,
9338e46b3edSNamjae Jeon sbi->cur_victim_sec,
9348e46b3edSNamjae Jeon prefree_segments(sbi), free_segments(sbi));
9357bc09003SJaegeuk Kim mutex_unlock(&dirty_i->seglist_lock);
9367bc09003SJaegeuk Kim
93797767500SQilong Zhang return ret;
9387bc09003SJaegeuk Kim }
9397bc09003SJaegeuk Kim
find_gc_inode(struct gc_inode_list * gc_list,nid_t ino)9407dda2af8SChangman Lee static struct inode *find_gc_inode(struct gc_inode_list *gc_list, nid_t ino)
9417bc09003SJaegeuk Kim {
9427bc09003SJaegeuk Kim struct inode_entry *ie;
9437bc09003SJaegeuk Kim
9447dda2af8SChangman Lee ie = radix_tree_lookup(&gc_list->iroot, ino);
9457dda2af8SChangman Lee if (ie)
9467bc09003SJaegeuk Kim return ie->inode;
9477bc09003SJaegeuk Kim return NULL;
9487bc09003SJaegeuk Kim }
9497bc09003SJaegeuk Kim
add_gc_inode(struct gc_inode_list * gc_list,struct inode * inode)9507dda2af8SChangman Lee static void add_gc_inode(struct gc_inode_list *gc_list, struct inode *inode)
9517bc09003SJaegeuk Kim {
9526cc4af56SGu Zheng struct inode_entry *new_ie;
9537bc09003SJaegeuk Kim
9547dda2af8SChangman Lee if (inode == find_gc_inode(gc_list, inode->i_ino)) {
9557bc09003SJaegeuk Kim iput(inode);
9567bc09003SJaegeuk Kim return;
9577bc09003SJaegeuk Kim }
95832410577SChao Yu new_ie = f2fs_kmem_cache_alloc(f2fs_inode_entry_slab,
95932410577SChao Yu GFP_NOFS, true, NULL);
9607bc09003SJaegeuk Kim new_ie->inode = inode;
961f28e5034SChao Yu
962f28e5034SChao Yu f2fs_radix_tree_insert(&gc_list->iroot, inode->i_ino, new_ie);
9637dda2af8SChangman Lee list_add_tail(&new_ie->list, &gc_list->ilist);
9647bc09003SJaegeuk Kim }
9657bc09003SJaegeuk Kim
put_gc_inode(struct gc_inode_list * gc_list)9667dda2af8SChangman Lee static void put_gc_inode(struct gc_inode_list *gc_list)
9677bc09003SJaegeuk Kim {
9687bc09003SJaegeuk Kim struct inode_entry *ie, *next_ie;
9695f029c04SYi Zhuang
9707dda2af8SChangman Lee list_for_each_entry_safe(ie, next_ie, &gc_list->ilist, list) {
9717dda2af8SChangman Lee radix_tree_delete(&gc_list->iroot, ie->inode->i_ino);
9727bc09003SJaegeuk Kim iput(ie->inode);
9737bc09003SJaegeuk Kim list_del(&ie->list);
9744d57b86dSChao Yu kmem_cache_free(f2fs_inode_entry_slab, ie);
9757bc09003SJaegeuk Kim }
9767bc09003SJaegeuk Kim }
9777bc09003SJaegeuk Kim
check_valid_map(struct f2fs_sb_info * sbi,unsigned int segno,int offset)9787bc09003SJaegeuk Kim static int check_valid_map(struct f2fs_sb_info *sbi,
9797bc09003SJaegeuk Kim unsigned int segno, int offset)
9807bc09003SJaegeuk Kim {
9817bc09003SJaegeuk Kim struct sit_info *sit_i = SIT_I(sbi);
9827bc09003SJaegeuk Kim struct seg_entry *sentry;
9837bc09003SJaegeuk Kim int ret;
9847bc09003SJaegeuk Kim
9853d26fa6bSChao Yu down_read(&sit_i->sentry_lock);
9867bc09003SJaegeuk Kim sentry = get_seg_entry(sbi, segno);
9877bc09003SJaegeuk Kim ret = f2fs_test_bit(offset, sentry->cur_valid_map);
9883d26fa6bSChao Yu up_read(&sit_i->sentry_lock);
98943727527SJaegeuk Kim return ret;
9907bc09003SJaegeuk Kim }
9917bc09003SJaegeuk Kim
9920a8165d7SJaegeuk Kim /*
9937bc09003SJaegeuk Kim * This function compares node address got in summary with that in NAT.
9947bc09003SJaegeuk Kim * On validity, copy that node with cold status, otherwise (invalid node)
9957bc09003SJaegeuk Kim * ignore that.
9967bc09003SJaegeuk Kim */
gc_node_segment(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,unsigned int segno,int gc_type)99748018b4cSChao Yu static int gc_node_segment(struct f2fs_sb_info *sbi,
9987bc09003SJaegeuk Kim struct f2fs_summary *sum, unsigned int segno, int gc_type)
9997bc09003SJaegeuk Kim {
10007bc09003SJaegeuk Kim struct f2fs_summary *entry;
100126d58599SJaegeuk Kim block_t start_addr;
10027bc09003SJaegeuk Kim int off;
10037ea984b0SChao Yu int phase = 0;
1004c29fd0c0SChao Yu bool fggc = (gc_type == FG_GC);
100548018b4cSChao Yu int submitted = 0;
1006de881df9SAravind Ramesh unsigned int usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
10077bc09003SJaegeuk Kim
100826d58599SJaegeuk Kim start_addr = START_BLOCK(sbi, segno);
100926d58599SJaegeuk Kim
10107bc09003SJaegeuk Kim next_step:
10117bc09003SJaegeuk Kim entry = sum;
1012c718379bSJaegeuk Kim
1013c29fd0c0SChao Yu if (fggc && phase == 2)
1014c29fd0c0SChao Yu atomic_inc(&sbi->wb_sync_req[NODE]);
1015c29fd0c0SChao Yu
1016de881df9SAravind Ramesh for (off = 0; off < usable_blks_in_seg; off++, entry++) {
10177bc09003SJaegeuk Kim nid_t nid = le32_to_cpu(entry->nid);
10187bc09003SJaegeuk Kim struct page *node_page;
101926d58599SJaegeuk Kim struct node_info ni;
102048018b4cSChao Yu int err;
10217bc09003SJaegeuk Kim
102243727527SJaegeuk Kim /* stop BG_GC if there is not enough free sections. */
10237f3037a5SJaegeuk Kim if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0))
102448018b4cSChao Yu return submitted;
10257bc09003SJaegeuk Kim
102643727527SJaegeuk Kim if (check_valid_map(sbi, segno, off) == 0)
10277bc09003SJaegeuk Kim continue;
10287bc09003SJaegeuk Kim
10297ea984b0SChao Yu if (phase == 0) {
10304d57b86dSChao Yu f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
10317ea984b0SChao Yu META_NAT, true);
10327ea984b0SChao Yu continue;
10337ea984b0SChao Yu }
10347ea984b0SChao Yu
10357ea984b0SChao Yu if (phase == 1) {
10364d57b86dSChao Yu f2fs_ra_node_page(sbi, nid);
10377bc09003SJaegeuk Kim continue;
10387bc09003SJaegeuk Kim }
10397ea984b0SChao Yu
10407ea984b0SChao Yu /* phase == 2 */
10414d57b86dSChao Yu node_page = f2fs_get_node_page(sbi, nid);
10427bc09003SJaegeuk Kim if (IS_ERR(node_page))
10437bc09003SJaegeuk Kim continue;
10447bc09003SJaegeuk Kim
10454d57b86dSChao Yu /* block may become invalid during f2fs_get_node_page */
10469a01b56bSHuang Ying if (check_valid_map(sbi, segno, off) == 0) {
10479a01b56bSHuang Ying f2fs_put_page(node_page, 1);
10489a01b56bSHuang Ying continue;
10499a01b56bSHuang Ying }
10509a01b56bSHuang Ying
1051a9419b63SJaegeuk Kim if (f2fs_get_node_info(sbi, nid, &ni, false)) {
10527735730dSChao Yu f2fs_put_page(node_page, 1);
10537735730dSChao Yu continue;
10547735730dSChao Yu }
10557735730dSChao Yu
105626d58599SJaegeuk Kim if (ni.blk_addr != start_addr + off) {
105726d58599SJaegeuk Kim f2fs_put_page(node_page, 1);
105826d58599SJaegeuk Kim continue;
105926d58599SJaegeuk Kim }
106026d58599SJaegeuk Kim
106148018b4cSChao Yu err = f2fs_move_node_page(node_page, gc_type);
106248018b4cSChao Yu if (!err && gc_type == FG_GC)
106348018b4cSChao Yu submitted++;
1064e1235983SChangman Lee stat_inc_node_blk_count(sbi, 1, gc_type);
10657bc09003SJaegeuk Kim }
1066c718379bSJaegeuk Kim
10677ea984b0SChao Yu if (++phase < 3)
10687bc09003SJaegeuk Kim goto next_step;
1069c29fd0c0SChao Yu
1070c29fd0c0SChao Yu if (fggc)
1071c29fd0c0SChao Yu atomic_dec(&sbi->wb_sync_req[NODE]);
107248018b4cSChao Yu return submitted;
10737bc09003SJaegeuk Kim }
10747bc09003SJaegeuk Kim
10750a8165d7SJaegeuk Kim /*
10769af45ef5SJaegeuk Kim * Calculate start block index indicating the given node offset.
10779af45ef5SJaegeuk Kim * Be careful, caller should give this node offset only indicating direct node
10789af45ef5SJaegeuk Kim * blocks. If any node offsets, which point the other types of node blocks such
10799af45ef5SJaegeuk Kim * as indirect or double indirect node blocks, are given, it must be a caller's
10809af45ef5SJaegeuk Kim * bug.
10817bc09003SJaegeuk Kim */
f2fs_start_bidx_of_node(unsigned int node_ofs,struct inode * inode)10824d57b86dSChao Yu block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode)
10837bc09003SJaegeuk Kim {
1084ce19a5d4SJaegeuk Kim unsigned int indirect_blks = 2 * NIDS_PER_BLOCK + 4;
1085ce19a5d4SJaegeuk Kim unsigned int bidx;
10867bc09003SJaegeuk Kim
1087ce19a5d4SJaegeuk Kim if (node_ofs == 0)
1088ce19a5d4SJaegeuk Kim return 0;
10897bc09003SJaegeuk Kim
1090ce19a5d4SJaegeuk Kim if (node_ofs <= 2) {
10917bc09003SJaegeuk Kim bidx = node_ofs - 1;
10927bc09003SJaegeuk Kim } else if (node_ofs <= indirect_blks) {
1093ce19a5d4SJaegeuk Kim int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1);
10945f029c04SYi Zhuang
10957bc09003SJaegeuk Kim bidx = node_ofs - 2 - dec;
10967bc09003SJaegeuk Kim } else {
1097ce19a5d4SJaegeuk Kim int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1);
10985f029c04SYi Zhuang
10997bc09003SJaegeuk Kim bidx = node_ofs - 5 - dec;
11007bc09003SJaegeuk Kim }
1101d02a6e61SChao Yu return bidx * ADDRS_PER_BLOCK(inode) + ADDRS_PER_INODE(inode);
11027bc09003SJaegeuk Kim }
11037bc09003SJaegeuk Kim
is_alive(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,struct node_info * dni,block_t blkaddr,unsigned int * nofs)1104c1079892SNicholas Krause static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
11057bc09003SJaegeuk Kim struct node_info *dni, block_t blkaddr, unsigned int *nofs)
11067bc09003SJaegeuk Kim {
11077bc09003SJaegeuk Kim struct page *node_page;
11087bc09003SJaegeuk Kim nid_t nid;
1109d3b7b4afSChao Yu unsigned int ofs_in_node, max_addrs, base;
11107bc09003SJaegeuk Kim block_t source_blkaddr;
11117bc09003SJaegeuk Kim
11127bc09003SJaegeuk Kim nid = le32_to_cpu(sum->nid);
11137bc09003SJaegeuk Kim ofs_in_node = le16_to_cpu(sum->ofs_in_node);
11147bc09003SJaegeuk Kim
11154d57b86dSChao Yu node_page = f2fs_get_node_page(sbi, nid);
11167bc09003SJaegeuk Kim if (IS_ERR(node_page))
1117c1079892SNicholas Krause return false;
11187bc09003SJaegeuk Kim
1119a9419b63SJaegeuk Kim if (f2fs_get_node_info(sbi, nid, dni, false)) {
11207735730dSChao Yu f2fs_put_page(node_page, 1);
11217735730dSChao Yu return false;
11227735730dSChao Yu }
11237bc09003SJaegeuk Kim
11247bc09003SJaegeuk Kim if (sum->version != dni->version) {
1125dcbb4c10SJoe Perches f2fs_warn(sbi, "%s: valid data with mismatched node version.",
1126c13ff37eSJaegeuk Kim __func__);
1127c13ff37eSJaegeuk Kim set_sbi_flag(sbi, SBI_NEED_FSCK);
11287bc09003SJaegeuk Kim }
11297bc09003SJaegeuk Kim
11306d18762eSChao Yu if (f2fs_check_nid_range(sbi, dni->ino)) {
11316d18762eSChao Yu f2fs_put_page(node_page, 1);
113277900c45SChao Yu return false;
11336d18762eSChao Yu }
113477900c45SChao Yu
1135d3b7b4afSChao Yu if (IS_INODE(node_page)) {
1136d3b7b4afSChao Yu base = offset_in_addr(F2FS_INODE(node_page));
1137d3b7b4afSChao Yu max_addrs = DEF_ADDRS_PER_INODE;
1138d3b7b4afSChao Yu } else {
1139d3b7b4afSChao Yu base = 0;
1140d3b7b4afSChao Yu max_addrs = DEF_ADDRS_PER_BLOCK;
1141d3b7b4afSChao Yu }
1142d3b7b4afSChao Yu
1143d3b7b4afSChao Yu if (base + ofs_in_node >= max_addrs) {
1144d3b7b4afSChao Yu f2fs_err(sbi, "Inconsistent blkaddr offset: base:%u, ofs_in_node:%u, max:%u, ino:%u, nid:%u",
1145d3b7b4afSChao Yu base, ofs_in_node, max_addrs, dni->ino, dni->nid);
1146c3db3c2fSPavel Machek f2fs_put_page(node_page, 1);
1147c6ad7fd1SChao Yu return false;
1148c6ad7fd1SChao Yu }
1149c6ad7fd1SChao Yu
11507bc09003SJaegeuk Kim *nofs = ofs_of_node(node_page);
1151a2ced1ceSChao Yu source_blkaddr = data_blkaddr(NULL, node_page, ofs_in_node);
11527bc09003SJaegeuk Kim f2fs_put_page(node_page, 1);
11537bc09003SJaegeuk Kim
1154bbf9f7d9SSahitya Tummala if (source_blkaddr != blkaddr) {
1155bbf9f7d9SSahitya Tummala #ifdef CONFIG_F2FS_CHECK_FS
1156bbf9f7d9SSahitya Tummala unsigned int segno = GET_SEGNO(sbi, blkaddr);
1157bbf9f7d9SSahitya Tummala unsigned long offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
1158bbf9f7d9SSahitya Tummala
1159bbf9f7d9SSahitya Tummala if (unlikely(check_valid_map(sbi, segno, offset))) {
1160bbf9f7d9SSahitya Tummala if (!test_and_set_bit(segno, SIT_I(sbi)->invalid_segmap)) {
1161833dcd35SJoe Perches f2fs_err(sbi, "mismatched blkaddr %u (source_blkaddr %u) in seg %u",
1162bbf9f7d9SSahitya Tummala blkaddr, source_blkaddr, segno);
1163f6db4307SChao Yu set_sbi_flag(sbi, SBI_NEED_FSCK);
1164bbf9f7d9SSahitya Tummala }
1165bbf9f7d9SSahitya Tummala }
1166bbf9f7d9SSahitya Tummala #endif
1167c1079892SNicholas Krause return false;
1168bbf9f7d9SSahitya Tummala }
1169c1079892SNicholas Krause return true;
11707bc09003SJaegeuk Kim }
11717bc09003SJaegeuk Kim
ra_data_block(struct inode * inode,pgoff_t index)11726aa58d8aSChao Yu static int ra_data_block(struct inode *inode, pgoff_t index)
11736aa58d8aSChao Yu {
11746aa58d8aSChao Yu struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
11756aa58d8aSChao Yu struct address_space *mapping = inode->i_mapping;
11766aa58d8aSChao Yu struct dnode_of_data dn;
11776aa58d8aSChao Yu struct page *page;
11786aa58d8aSChao Yu struct f2fs_io_info fio = {
11796aa58d8aSChao Yu .sbi = sbi,
11806aa58d8aSChao Yu .ino = inode->i_ino,
11816aa58d8aSChao Yu .type = DATA,
11826aa58d8aSChao Yu .temp = COLD,
11836aa58d8aSChao Yu .op = REQ_OP_READ,
11846aa58d8aSChao Yu .op_flags = 0,
11856aa58d8aSChao Yu .encrypted_page = NULL,
11862eae077eSChao Yu .in_list = 0,
11872eae077eSChao Yu .retry = 0,
11886aa58d8aSChao Yu };
11896aa58d8aSChao Yu int err;
11906aa58d8aSChao Yu
11916aa58d8aSChao Yu page = f2fs_grab_cache_page(mapping, index, true);
11926aa58d8aSChao Yu if (!page)
11936aa58d8aSChao Yu return -ENOMEM;
11946aa58d8aSChao Yu
119504a91ab0SChristoph Hellwig if (f2fs_lookup_read_extent_cache_block(inode, index,
119604a91ab0SChristoph Hellwig &dn.data_blkaddr)) {
119793770ab7SChao Yu if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr,
119893770ab7SChao Yu DATA_GENERIC_ENHANCE_READ))) {
119910f966bbSChao Yu err = -EFSCORRUPTED;
120095fa90c9SChao Yu f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
120193770ab7SChao Yu goto put_page;
120293770ab7SChao Yu }
12036aa58d8aSChao Yu goto got_it;
12046aa58d8aSChao Yu }
12056aa58d8aSChao Yu
12066aa58d8aSChao Yu set_new_dnode(&dn, inode, NULL, NULL, 0);
12076aa58d8aSChao Yu err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
12086aa58d8aSChao Yu if (err)
12096aa58d8aSChao Yu goto put_page;
12106aa58d8aSChao Yu f2fs_put_dnode(&dn);
12116aa58d8aSChao Yu
121293770ab7SChao Yu if (!__is_valid_data_blkaddr(dn.data_blkaddr)) {
121393770ab7SChao Yu err = -ENOENT;
121493770ab7SChao Yu goto put_page;
121593770ab7SChao Yu }
12166aa58d8aSChao Yu if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr,
121793770ab7SChao Yu DATA_GENERIC_ENHANCE))) {
121810f966bbSChao Yu err = -EFSCORRUPTED;
121995fa90c9SChao Yu f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
12206aa58d8aSChao Yu goto put_page;
12216aa58d8aSChao Yu }
12226aa58d8aSChao Yu got_it:
12236aa58d8aSChao Yu /* read page */
12246aa58d8aSChao Yu fio.page = page;
12256aa58d8aSChao Yu fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
12266aa58d8aSChao Yu
12279bf1a3f7SYunlong Song /*
12289bf1a3f7SYunlong Song * don't cache encrypted data into meta inode until previous dirty
12299bf1a3f7SYunlong Song * data were writebacked to avoid racing between GC and flush.
12309bf1a3f7SYunlong Song */
1231bae0ee7aSChao Yu f2fs_wait_on_page_writeback(page, DATA, true, true);
12329bf1a3f7SYunlong Song
12339bf1a3f7SYunlong Song f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
12349bf1a3f7SYunlong Song
12356aa58d8aSChao Yu fio.encrypted_page = f2fs_pagecache_get_page(META_MAPPING(sbi),
12366aa58d8aSChao Yu dn.data_blkaddr,
12376aa58d8aSChao Yu FGP_LOCK | FGP_CREAT, GFP_NOFS);
12386aa58d8aSChao Yu if (!fio.encrypted_page) {
12396aa58d8aSChao Yu err = -ENOMEM;
12406aa58d8aSChao Yu goto put_page;
12416aa58d8aSChao Yu }
12426aa58d8aSChao Yu
12436aa58d8aSChao Yu err = f2fs_submit_page_bio(&fio);
12446aa58d8aSChao Yu if (err)
12456aa58d8aSChao Yu goto put_encrypted_page;
12466aa58d8aSChao Yu f2fs_put_page(fio.encrypted_page, 0);
12476aa58d8aSChao Yu f2fs_put_page(page, 1);
12488b83ac81SChao Yu
124934a23525SChao Yu f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE);
125034a23525SChao Yu f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE);
12518b83ac81SChao Yu
12526aa58d8aSChao Yu return 0;
12536aa58d8aSChao Yu put_encrypted_page:
12546aa58d8aSChao Yu f2fs_put_page(fio.encrypted_page, 1);
12556aa58d8aSChao Yu put_page:
12566aa58d8aSChao Yu f2fs_put_page(page, 1);
12576aa58d8aSChao Yu return err;
12586aa58d8aSChao Yu }
12596aa58d8aSChao Yu
1260d4c759eeSJaegeuk Kim /*
1261d4c759eeSJaegeuk Kim * Move data block via META_MAPPING while keeping locked data page.
1262d4c759eeSJaegeuk Kim * This can be used to move blocks, aka LBAs, directly on disk.
1263d4c759eeSJaegeuk Kim */
move_data_block(struct inode * inode,block_t bidx,int gc_type,unsigned int segno,int off)126448018b4cSChao Yu static int move_data_block(struct inode *inode, block_t bidx,
12652ef79ecbSChao Yu int gc_type, unsigned int segno, int off)
12664375a336SJaegeuk Kim {
12674375a336SJaegeuk Kim struct f2fs_io_info fio = {
12684375a336SJaegeuk Kim .sbi = F2FS_I_SB(inode),
126939d787beSChao Yu .ino = inode->i_ino,
12704375a336SJaegeuk Kim .type = DATA,
1271a912b54dSJaegeuk Kim .temp = COLD,
127204d328deSMike Christie .op = REQ_OP_READ,
127370fd7614SChristoph Hellwig .op_flags = 0,
12744375a336SJaegeuk Kim .encrypted_page = NULL,
12752eae077eSChao Yu .in_list = 0,
12762eae077eSChao Yu .retry = 0,
12774375a336SJaegeuk Kim };
12784375a336SJaegeuk Kim struct dnode_of_data dn;
12794375a336SJaegeuk Kim struct f2fs_summary sum;
12804375a336SJaegeuk Kim struct node_info ni;
12816aa58d8aSChao Yu struct page *page, *mpage;
12824356e48eSChao Yu block_t newaddr;
128348018b4cSChao Yu int err = 0;
1284b0332a0fSChao Yu bool lfs_mode = f2fs_lfs_mode(fio.sbi);
1285ac2d750bSWeichao Guo int type = fio.sbi->am.atgc_enabled && (gc_type == BG_GC) &&
1286ac2d750bSWeichao Guo (fio.sbi->gc_mode != GC_URGENT_HIGH) ?
1287093749e2SChao Yu CURSEG_ALL_DATA_ATGC : CURSEG_COLD_DATA;
12884375a336SJaegeuk Kim
12894375a336SJaegeuk Kim /* do not read out */
1290a56c7c6fSJaegeuk Kim page = f2fs_grab_cache_page(inode->i_mapping, bidx, false);
12914375a336SJaegeuk Kim if (!page)
129248018b4cSChao Yu return -ENOMEM;
12934375a336SJaegeuk Kim
129448018b4cSChao Yu if (!check_valid_map(F2FS_I_SB(inode), segno, off)) {
129548018b4cSChao Yu err = -ENOENT;
129620614711SYunlei He goto out;
129748018b4cSChao Yu }
129820614711SYunlei He
129971419129SChao Yu err = f2fs_gc_pinned_control(inode, gc_type, segno);
130071419129SChao Yu if (err)
13011ad71a27SJaegeuk Kim goto out;
13021ad71a27SJaegeuk Kim
13034375a336SJaegeuk Kim set_new_dnode(&dn, inode, NULL, NULL, 0);
13044d57b86dSChao Yu err = f2fs_get_dnode_of_data(&dn, bidx, LOOKUP_NODE);
13054375a336SJaegeuk Kim if (err)
13064375a336SJaegeuk Kim goto out;
13074375a336SJaegeuk Kim
130808b39fbdSChao Yu if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
130908b39fbdSChao Yu ClearPageUptodate(page);
131048018b4cSChao Yu err = -ENOENT;
13114375a336SJaegeuk Kim goto put_out;
131208b39fbdSChao Yu }
131308b39fbdSChao Yu
131408b39fbdSChao Yu /*
131508b39fbdSChao Yu * don't cache encrypted data into meta inode until previous dirty
131608b39fbdSChao Yu * data were writebacked to avoid racing between GC and flush.
131708b39fbdSChao Yu */
1318bae0ee7aSChao Yu f2fs_wait_on_page_writeback(page, DATA, true, true);
13194375a336SJaegeuk Kim
13209bf1a3f7SYunlong Song f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
13219bf1a3f7SYunlong Song
1322a9419b63SJaegeuk Kim err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false);
13237735730dSChao Yu if (err)
13247735730dSChao Yu goto put_out;
13257735730dSChao Yu
13264375a336SJaegeuk Kim /* read page */
13274375a336SJaegeuk Kim fio.page = page;
13287a9d7548SChao Yu fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
13294375a336SJaegeuk Kim
1330107a805dSChao Yu if (lfs_mode)
1331e4544b63STim Murray f2fs_down_write(&fio.sbi->io_order_lock);
1332107a805dSChao Yu
1333543b8c46SJaegeuk Kim mpage = f2fs_grab_cache_page(META_MAPPING(fio.sbi),
1334543b8c46SJaegeuk Kim fio.old_blkaddr, false);
1335d7cd3702SChao Yu if (!mpage) {
1336d7cd3702SChao Yu err = -ENOMEM;
1337543b8c46SJaegeuk Kim goto up_out;
1338d7cd3702SChao Yu }
1339543b8c46SJaegeuk Kim
1340543b8c46SJaegeuk Kim fio.encrypted_page = mpage;
1341543b8c46SJaegeuk Kim
1342543b8c46SJaegeuk Kim /* read source block in mpage */
1343543b8c46SJaegeuk Kim if (!PageUptodate(mpage)) {
1344543b8c46SJaegeuk Kim err = f2fs_submit_page_bio(&fio);
1345543b8c46SJaegeuk Kim if (err) {
1346543b8c46SJaegeuk Kim f2fs_put_page(mpage, 1);
1347543b8c46SJaegeuk Kim goto up_out;
1348543b8c46SJaegeuk Kim }
13498b83ac81SChao Yu
135034a23525SChao Yu f2fs_update_iostat(fio.sbi, inode, FS_DATA_READ_IO,
135134a23525SChao Yu F2FS_BLKSIZE);
135234a23525SChao Yu f2fs_update_iostat(fio.sbi, NULL, FS_GDATA_READ_IO,
135334a23525SChao Yu F2FS_BLKSIZE);
13548b83ac81SChao Yu
1355543b8c46SJaegeuk Kim lock_page(mpage);
1356543b8c46SJaegeuk Kim if (unlikely(mpage->mapping != META_MAPPING(fio.sbi) ||
1357543b8c46SJaegeuk Kim !PageUptodate(mpage))) {
1358543b8c46SJaegeuk Kim err = -EIO;
1359543b8c46SJaegeuk Kim f2fs_put_page(mpage, 1);
1360543b8c46SJaegeuk Kim goto up_out;
1361543b8c46SJaegeuk Kim }
1362543b8c46SJaegeuk Kim }
1363543b8c46SJaegeuk Kim
1364cf740403SChao Yu set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version);
1365cf740403SChao Yu
1366cf740403SChao Yu /* allocate block address */
13674d57b86dSChao Yu f2fs_allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,
1368093749e2SChao Yu &sum, type, NULL);
13694356e48eSChao Yu
137001eccef7SChao Yu fio.encrypted_page = f2fs_pagecache_get_page(META_MAPPING(fio.sbi),
137101eccef7SChao Yu newaddr, FGP_LOCK | FGP_CREAT, GFP_NOFS);
13724356e48eSChao Yu if (!fio.encrypted_page) {
13734356e48eSChao Yu err = -ENOMEM;
1374543b8c46SJaegeuk Kim f2fs_put_page(mpage, 1);
13754356e48eSChao Yu goto recover_block;
13764356e48eSChao Yu }
13774375a336SJaegeuk Kim
1378543b8c46SJaegeuk Kim /* write target block */
1379543b8c46SJaegeuk Kim f2fs_wait_on_page_writeback(fio.encrypted_page, DATA, true, true);
13806aa58d8aSChao Yu memcpy(page_address(fio.encrypted_page),
13816aa58d8aSChao Yu page_address(mpage), PAGE_SIZE);
13826aa58d8aSChao Yu f2fs_put_page(mpage, 1);
1383*cfd217f6SChao Yu
1384*cfd217f6SChao Yu f2fs_invalidate_internal_cache(fio.sbi, fio.old_blkaddr);
13856aa58d8aSChao Yu
13868d64d365SChao Yu set_page_dirty(fio.encrypted_page);
13876282adbfSJaegeuk Kim if (clear_page_dirty_for_io(fio.encrypted_page))
13886282adbfSJaegeuk Kim dec_page_count(fio.sbi, F2FS_DIRTY_META);
13896282adbfSJaegeuk Kim
13904375a336SJaegeuk Kim set_page_writeback(fio.encrypted_page);
1391548aedacSJaegeuk Kim
139204d328deSMike Christie fio.op = REQ_OP_WRITE;
139370fd7614SChristoph Hellwig fio.op_flags = REQ_SYNC;
13944356e48eSChao Yu fio.new_blkaddr = newaddr;
1395fe16efe6SChao Yu f2fs_submit_page_write(&fio);
1396fe16efe6SChao Yu if (fio.retry) {
139748018b4cSChao Yu err = -EAGAIN;
1398a9d572c7SSheng Yong if (PageWriteback(fio.encrypted_page))
1399a9d572c7SSheng Yong end_page_writeback(fio.encrypted_page);
1400a9d572c7SSheng Yong goto put_page_out;
1401a9d572c7SSheng Yong }
14024375a336SJaegeuk Kim
140334a23525SChao Yu f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, F2FS_BLKSIZE);
1404b0af6d49SChao Yu
1405f28b3434SChao Yu f2fs_update_data_blkaddr(&dn, newaddr);
140691942321SJaegeuk Kim set_inode_flag(inode, FI_APPEND_WRITE);
1407548aedacSJaegeuk Kim put_page_out:
14084375a336SJaegeuk Kim f2fs_put_page(fio.encrypted_page, 1);
14094356e48eSChao Yu recover_block:
14104356e48eSChao Yu if (err)
14114d57b86dSChao Yu f2fs_do_replace_block(fio.sbi, &sum, newaddr, fio.old_blkaddr,
1412c5d02785SChao Yu true, true, true);
1413543b8c46SJaegeuk Kim up_out:
1414543b8c46SJaegeuk Kim if (lfs_mode)
1415e4544b63STim Murray f2fs_up_write(&fio.sbi->io_order_lock);
14164375a336SJaegeuk Kim put_out:
14174375a336SJaegeuk Kim f2fs_put_dnode(&dn);
14184375a336SJaegeuk Kim out:
14194375a336SJaegeuk Kim f2fs_put_page(page, 1);
142048018b4cSChao Yu return err;
14214375a336SJaegeuk Kim }
14224375a336SJaegeuk Kim
move_data_page(struct inode * inode,block_t bidx,int gc_type,unsigned int segno,int off)142348018b4cSChao Yu static int move_data_page(struct inode *inode, block_t bidx, int gc_type,
142420614711SYunlei He unsigned int segno, int off)
14257bc09003SJaegeuk Kim {
1426c879f90dSJaegeuk Kim struct page *page;
142748018b4cSChao Yu int err = 0;
1428c879f90dSJaegeuk Kim
14294d57b86dSChao Yu page = f2fs_get_lock_data_page(inode, bidx, true);
1430c879f90dSJaegeuk Kim if (IS_ERR(page))
143148018b4cSChao Yu return PTR_ERR(page);
143263a0b7cbSFan Li
143348018b4cSChao Yu if (!check_valid_map(F2FS_I_SB(inode), segno, off)) {
143448018b4cSChao Yu err = -ENOENT;
143520614711SYunlei He goto out;
143648018b4cSChao Yu }
143720614711SYunlei He
143871419129SChao Yu err = f2fs_gc_pinned_control(inode, gc_type, segno);
143971419129SChao Yu if (err)
14401ad71a27SJaegeuk Kim goto out;
14415fe45743SChao Yu
14424ebefc44SJaegeuk Kim if (gc_type == BG_GC) {
144348018b4cSChao Yu if (PageWriteback(page)) {
144448018b4cSChao Yu err = -EAGAIN;
14457bc09003SJaegeuk Kim goto out;
144648018b4cSChao Yu }
14477bc09003SJaegeuk Kim set_page_dirty(page);
1448b763f3beSChao Yu set_page_private_gcing(page);
14497bc09003SJaegeuk Kim } else {
1450c879f90dSJaegeuk Kim struct f2fs_io_info fio = {
1451c879f90dSJaegeuk Kim .sbi = F2FS_I_SB(inode),
145239d787beSChao Yu .ino = inode->i_ino,
1453c879f90dSJaegeuk Kim .type = DATA,
1454a912b54dSJaegeuk Kim .temp = COLD,
145504d328deSMike Christie .op = REQ_OP_WRITE,
145670fd7614SChristoph Hellwig .op_flags = REQ_SYNC,
1457e959c8f5SHou Pengyang .old_blkaddr = NULL_ADDR,
1458c879f90dSJaegeuk Kim .page = page,
14594375a336SJaegeuk Kim .encrypted_page = NULL,
1460cc15620bSJaegeuk Kim .need_lock = LOCK_REQ,
1461b0af6d49SChao Yu .io_type = FS_GC_DATA_IO,
1462c879f90dSJaegeuk Kim };
146372e1c797SChao Yu bool is_dirty = PageDirty(page);
146472e1c797SChao Yu
146572e1c797SChao Yu retry:
1466bae0ee7aSChao Yu f2fs_wait_on_page_writeback(page, DATA, true, true);
14678d64d365SChao Yu
14688d64d365SChao Yu set_page_dirty(page);
1469933439c8SChao Yu if (clear_page_dirty_for_io(page)) {
1470a7ffdbe2SJaegeuk Kim inode_dec_dirty_pages(inode);
14714d57b86dSChao Yu f2fs_remove_dirty_inode(inode);
1472933439c8SChao Yu }
147372e1c797SChao Yu
1474b763f3beSChao Yu set_page_private_gcing(page);
147572e1c797SChao Yu
14764d57b86dSChao Yu err = f2fs_do_write_data_page(&fio);
147714a28559SChao Yu if (err) {
1478b763f3beSChao Yu clear_page_private_gcing(page);
147914a28559SChao Yu if (err == -ENOMEM) {
14804034247aSNeilBrown memalloc_retry_wait(GFP_NOFS);
148172e1c797SChao Yu goto retry;
148272e1c797SChao Yu }
148314a28559SChao Yu if (is_dirty)
148414a28559SChao Yu set_page_dirty(page);
148514a28559SChao Yu }
14867bc09003SJaegeuk Kim }
14877bc09003SJaegeuk Kim out:
14887bc09003SJaegeuk Kim f2fs_put_page(page, 1);
148948018b4cSChao Yu return err;
14907bc09003SJaegeuk Kim }
14917bc09003SJaegeuk Kim
14920a8165d7SJaegeuk Kim /*
14937bc09003SJaegeuk Kim * This function tries to get parent node of victim data block, and identifies
14947bc09003SJaegeuk Kim * data block validity. If the block is valid, copy that with cold status and
14957bc09003SJaegeuk Kim * modify parent node.
14967bc09003SJaegeuk Kim * If the parent node is not valid or the data block address is different,
14977bc09003SJaegeuk Kim * the victim data block is ignored.
14987bc09003SJaegeuk Kim */
gc_data_segment(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,struct gc_inode_list * gc_list,unsigned int segno,int gc_type,bool force_migrate)149948018b4cSChao Yu static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
15007dede886SChao Yu struct gc_inode_list *gc_list, unsigned int segno, int gc_type,
15017dede886SChao Yu bool force_migrate)
15027bc09003SJaegeuk Kim {
15037bc09003SJaegeuk Kim struct super_block *sb = sbi->sb;
15047bc09003SJaegeuk Kim struct f2fs_summary *entry;
15057bc09003SJaegeuk Kim block_t start_addr;
150643727527SJaegeuk Kim int off;
15077bc09003SJaegeuk Kim int phase = 0;
150848018b4cSChao Yu int submitted = 0;
1509de881df9SAravind Ramesh unsigned int usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
15107bc09003SJaegeuk Kim
15117bc09003SJaegeuk Kim start_addr = START_BLOCK(sbi, segno);
15127bc09003SJaegeuk Kim
15137bc09003SJaegeuk Kim next_step:
15147bc09003SJaegeuk Kim entry = sum;
1515c718379bSJaegeuk Kim
1516de881df9SAravind Ramesh for (off = 0; off < usable_blks_in_seg; off++, entry++) {
15177bc09003SJaegeuk Kim struct page *data_page;
15187bc09003SJaegeuk Kim struct inode *inode;
15197bc09003SJaegeuk Kim struct node_info dni; /* dnode info for the data */
15207bc09003SJaegeuk Kim unsigned int ofs_in_node, nofs;
15217bc09003SJaegeuk Kim block_t start_bidx;
15227ea984b0SChao Yu nid_t nid = le32_to_cpu(entry->nid);
15237bc09003SJaegeuk Kim
1524803e74beSJaegeuk Kim /*
1525803e74beSJaegeuk Kim * stop BG_GC if there is not enough free sections.
1526803e74beSJaegeuk Kim * Or, stop GC if the segment becomes fully valid caused by
1527803e74beSJaegeuk Kim * race condition along with SSR block allocation.
1528803e74beSJaegeuk Kim */
1529803e74beSJaegeuk Kim if ((gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) ||
15307dede886SChao Yu (!force_migrate && get_valid_blocks(sbi, segno, true) ==
1531074b5ea2SJaegeuk Kim CAP_BLKS_PER_SEC(sbi)))
153248018b4cSChao Yu return submitted;
15337bc09003SJaegeuk Kim
153443727527SJaegeuk Kim if (check_valid_map(sbi, segno, off) == 0)
15357bc09003SJaegeuk Kim continue;
15367bc09003SJaegeuk Kim
15377bc09003SJaegeuk Kim if (phase == 0) {
15384d57b86dSChao Yu f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
15397ea984b0SChao Yu META_NAT, true);
15407ea984b0SChao Yu continue;
15417ea984b0SChao Yu }
15427ea984b0SChao Yu
15437ea984b0SChao Yu if (phase == 1) {
15444d57b86dSChao Yu f2fs_ra_node_page(sbi, nid);
15457bc09003SJaegeuk Kim continue;
15467bc09003SJaegeuk Kim }
15477bc09003SJaegeuk Kim
15487bc09003SJaegeuk Kim /* Get an inode by ino with checking validity */
1549c1079892SNicholas Krause if (!is_alive(sbi, entry, &dni, start_addr + off, &nofs))
15507bc09003SJaegeuk Kim continue;
15517bc09003SJaegeuk Kim
15527ea984b0SChao Yu if (phase == 2) {
15534d57b86dSChao Yu f2fs_ra_node_page(sbi, dni.ino);
15547bc09003SJaegeuk Kim continue;
15557bc09003SJaegeuk Kim }
15567bc09003SJaegeuk Kim
15577bc09003SJaegeuk Kim ofs_in_node = le16_to_cpu(entry->ofs_in_node);
15587bc09003SJaegeuk Kim
15597ea984b0SChao Yu if (phase == 3) {
156071419129SChao Yu int err;
156171419129SChao Yu
1562d4686d56SJaegeuk Kim inode = f2fs_iget(sb, dni.ino);
15639056d648SChao Yu if (IS_ERR(inode) || is_bad_inode(inode) ||
15649056d648SChao Yu special_file(inode->i_mode))
15657bc09003SJaegeuk Kim continue;
15667bc09003SJaegeuk Kim
156771419129SChao Yu err = f2fs_gc_pinned_control(inode, gc_type, segno);
156871419129SChao Yu if (err == -EAGAIN) {
1569a22bb552SChao Yu iput(inode);
1570a22bb552SChao Yu return submitted;
1571a22bb552SChao Yu }
1572a22bb552SChao Yu
1573e4544b63STim Murray if (!f2fs_down_write_trylock(
1574b2532c69SChao Yu &F2FS_I(inode)->i_gc_rwsem[WRITE])) {
1575bb06664aSChao Yu iput(inode);
15766f8d4455SJaegeuk Kim sbi->skipped_gc_rwsem++;
1577bb06664aSChao Yu continue;
1578bb06664aSChao Yu }
1579bb06664aSChao Yu
15806aa58d8aSChao Yu start_bidx = f2fs_start_bidx_of_node(nofs, inode) +
15816aa58d8aSChao Yu ofs_in_node;
15826aa58d8aSChao Yu
15836aa58d8aSChao Yu if (f2fs_post_read_required(inode)) {
15846aa58d8aSChao Yu int err = ra_data_block(inode, start_bidx);
15856aa58d8aSChao Yu
1586e4544b63STim Murray f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
15876aa58d8aSChao Yu if (err) {
15886aa58d8aSChao Yu iput(inode);
15896aa58d8aSChao Yu continue;
15906aa58d8aSChao Yu }
15916aa58d8aSChao Yu add_gc_inode(gc_list, inode);
15926aa58d8aSChao Yu continue;
15936aa58d8aSChao Yu }
15946aa58d8aSChao Yu
159559237a21SChao Yu data_page = f2fs_get_read_data_page(inode, start_bidx,
159659237a21SChao Yu REQ_RAHEAD, true, NULL);
1597e4544b63STim Murray f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
159831a32688SChangman Lee if (IS_ERR(data_page)) {
159931a32688SChangman Lee iput(inode);
160031a32688SChangman Lee continue;
160131a32688SChangman Lee }
16027bc09003SJaegeuk Kim
16037bc09003SJaegeuk Kim f2fs_put_page(data_page, 0);
16047dda2af8SChangman Lee add_gc_inode(gc_list, inode);
160531a32688SChangman Lee continue;
160631a32688SChangman Lee }
160731a32688SChangman Lee
16087ea984b0SChao Yu /* phase 4 */
16097dda2af8SChangman Lee inode = find_gc_inode(gc_list, dni.ino);
16107bc09003SJaegeuk Kim if (inode) {
161182e0a5aaSChao Yu struct f2fs_inode_info *fi = F2FS_I(inode);
161282e0a5aaSChao Yu bool locked = false;
161348018b4cSChao Yu int err;
161482e0a5aaSChao Yu
161582e0a5aaSChao Yu if (S_ISREG(inode->i_mode)) {
16166fd257cbSChao Yu if (!f2fs_down_write_trylock(&fi->i_gc_rwsem[WRITE])) {
1617ad126ebdSChao Yu sbi->skipped_gc_rwsem++;
161882e0a5aaSChao Yu continue;
1619ad126ebdSChao Yu }
1620e4544b63STim Murray if (!f2fs_down_write_trylock(
16216fd257cbSChao Yu &fi->i_gc_rwsem[READ])) {
16226f8d4455SJaegeuk Kim sbi->skipped_gc_rwsem++;
16236fd257cbSChao Yu f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
162482e0a5aaSChao Yu continue;
162582e0a5aaSChao Yu }
162682e0a5aaSChao Yu locked = true;
162773ac2f4eSChao Yu
162873ac2f4eSChao Yu /* wait for all inflight aio data */
162973ac2f4eSChao Yu inode_dio_wait(inode);
163082e0a5aaSChao Yu }
163182e0a5aaSChao Yu
16324d57b86dSChao Yu start_bidx = f2fs_start_bidx_of_node(nofs, inode)
1633c879f90dSJaegeuk Kim + ofs_in_node;
16346dbb1796SEric Biggers if (f2fs_post_read_required(inode))
163548018b4cSChao Yu err = move_data_block(inode, start_bidx,
163648018b4cSChao Yu gc_type, segno, off);
16374375a336SJaegeuk Kim else
163848018b4cSChao Yu err = move_data_page(inode, start_bidx, gc_type,
1639d4c759eeSJaegeuk Kim segno, off);
164082e0a5aaSChao Yu
164148018b4cSChao Yu if (!err && (gc_type == FG_GC ||
164248018b4cSChao Yu f2fs_post_read_required(inode)))
164348018b4cSChao Yu submitted++;
164448018b4cSChao Yu
164582e0a5aaSChao Yu if (locked) {
1646e4544b63STim Murray f2fs_up_write(&fi->i_gc_rwsem[READ]);
16476fd257cbSChao Yu f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
164882e0a5aaSChao Yu }
164982e0a5aaSChao Yu
1650e1235983SChangman Lee stat_inc_data_blk_count(sbi, 1, gc_type);
16517bc09003SJaegeuk Kim }
16527bc09003SJaegeuk Kim }
1653c718379bSJaegeuk Kim
16547ea984b0SChao Yu if (++phase < 5)
16557bc09003SJaegeuk Kim goto next_step;
165648018b4cSChao Yu
165748018b4cSChao Yu return submitted;
16587bc09003SJaegeuk Kim }
16597bc09003SJaegeuk Kim
__get_victim(struct f2fs_sb_info * sbi,unsigned int * victim,int gc_type)16607bc09003SJaegeuk Kim static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim,
16618a2d0aceSGu Zheng int gc_type)
16627bc09003SJaegeuk Kim {
16637bc09003SJaegeuk Kim struct sit_info *sit_i = SIT_I(sbi);
16647bc09003SJaegeuk Kim int ret;
16658a2d0aceSGu Zheng
16663d26fa6bSChao Yu down_write(&sit_i->sentry_lock);
166719e0e21aSYangtao Li ret = f2fs_get_victim(sbi, victim, gc_type, NO_CHECK_TYPE, LFS, 0);
16683d26fa6bSChao Yu up_write(&sit_i->sentry_lock);
16697bc09003SJaegeuk Kim return ret;
16707bc09003SJaegeuk Kim }
16717bc09003SJaegeuk Kim
do_garbage_collect(struct f2fs_sb_info * sbi,unsigned int start_segno,struct gc_inode_list * gc_list,int gc_type,bool force_migrate)1672718e53faSChao Yu static int do_garbage_collect(struct f2fs_sb_info *sbi,
1673718e53faSChao Yu unsigned int start_segno,
16747dede886SChao Yu struct gc_inode_list *gc_list, int gc_type,
16757dede886SChao Yu bool force_migrate)
16767bc09003SJaegeuk Kim {
16777bc09003SJaegeuk Kim struct page *sum_page;
16787bc09003SJaegeuk Kim struct f2fs_summary_block *sum;
1679c718379bSJaegeuk Kim struct blk_plug plug;
1680718e53faSChao Yu unsigned int segno = start_segno;
1681718e53faSChao Yu unsigned int end_segno = start_segno + sbi->segs_per_sec;
1682e3080b01SChao Yu int seg_freed = 0, migrated = 0;
1683718e53faSChao Yu unsigned char type = IS_DATASEG(get_seg_entry(sbi, segno)->type) ?
1684718e53faSChao Yu SUM_TYPE_DATA : SUM_TYPE_NODE;
16859bf1dcbdSChao Yu unsigned char data_type = (type == SUM_TYPE_DATA) ? DATA : NODE;
168648018b4cSChao Yu int submitted = 0;
16877bc09003SJaegeuk Kim
1688e3080b01SChao Yu if (__is_large_section(sbi))
1689e3080b01SChao Yu end_segno = rounddown(end_segno, sbi->segs_per_sec);
1690e3080b01SChao Yu
1691de881df9SAravind Ramesh /*
1692de881df9SAravind Ramesh * zone-capacity can be less than zone-size in zoned devices,
1693de881df9SAravind Ramesh * resulting in less than expected usable segments in the zone,
1694de881df9SAravind Ramesh * calculate the end segno in the zone which can be garbage collected
1695de881df9SAravind Ramesh */
1696de881df9SAravind Ramesh if (f2fs_sb_has_blkzoned(sbi))
1697de881df9SAravind Ramesh end_segno -= sbi->segs_per_sec -
1698de881df9SAravind Ramesh f2fs_usable_segs_in_sec(sbi, segno);
1699de881df9SAravind Ramesh
1700093749e2SChao Yu sanity_check_seg_type(sbi, get_seg_entry(sbi, segno)->type);
1701093749e2SChao Yu
1702718e53faSChao Yu /* readahead multi ssa blocks those have contiguous address */
17032c70c5e3SChao Yu if (__is_large_section(sbi))
17044d57b86dSChao Yu f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno),
1705e3080b01SChao Yu end_segno - segno, META_SSA, true);
1706718e53faSChao Yu
1707718e53faSChao Yu /* reference all summary page */
1708718e53faSChao Yu while (segno < end_segno) {
17094d57b86dSChao Yu sum_page = f2fs_get_sum_page(sbi, segno++);
1710edc55aafSJaegeuk Kim if (IS_ERR(sum_page)) {
1711edc55aafSJaegeuk Kim int err = PTR_ERR(sum_page);
1712edc55aafSJaegeuk Kim
1713edc55aafSJaegeuk Kim end_segno = segno - 1;
1714edc55aafSJaegeuk Kim for (segno = start_segno; segno < end_segno; segno++) {
1715edc55aafSJaegeuk Kim sum_page = find_get_page(META_MAPPING(sbi),
1716edc55aafSJaegeuk Kim GET_SUM_BLOCK(sbi, segno));
1717edc55aafSJaegeuk Kim f2fs_put_page(sum_page, 0);
1718edc55aafSJaegeuk Kim f2fs_put_page(sum_page, 0);
1719edc55aafSJaegeuk Kim }
1720edc55aafSJaegeuk Kim return err;
1721edc55aafSJaegeuk Kim }
1722718e53faSChao Yu unlock_page(sum_page);
1723718e53faSChao Yu }
17247bc09003SJaegeuk Kim
1725c718379bSJaegeuk Kim blk_start_plug(&plug);
1726c718379bSJaegeuk Kim
1727718e53faSChao Yu for (segno = start_segno; segno < end_segno; segno++) {
1728aa987273SJaegeuk Kim
1729718e53faSChao Yu /* find segment summary of victim */
1730718e53faSChao Yu sum_page = find_get_page(META_MAPPING(sbi),
1731718e53faSChao Yu GET_SUM_BLOCK(sbi, segno));
1732718e53faSChao Yu f2fs_put_page(sum_page, 0);
1733718e53faSChao Yu
1734d6c66cd1SYunlong Song if (get_valid_blocks(sbi, segno, false) == 0)
1735d6c66cd1SYunlong Song goto freed;
1736dabfbbc8SJaegeuk Kim if (gc_type == BG_GC && __is_large_section(sbi) &&
1737e3080b01SChao Yu migrated >= sbi->migration_granularity)
1738e3080b01SChao Yu goto skip;
1739d6c66cd1SYunlong Song if (!PageUptodate(sum_page) || unlikely(f2fs_cp_error(sbi)))
1740e3080b01SChao Yu goto skip;
1741de0dcc40SJaegeuk Kim
17427bc09003SJaegeuk Kim sum = page_address(sum_page);
174310d255c3SChao Yu if (type != GET_SUM_TYPE((&sum->footer))) {
1744dcbb4c10SJoe Perches f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SSA and SIT",
174510d255c3SChao Yu segno, type, GET_SUM_TYPE((&sum->footer)));
174610d255c3SChao Yu set_sbi_flag(sbi, SBI_NEED_FSCK);
1747a9cfee0eSChao Yu f2fs_stop_checkpoint(sbi, false,
1748a9cfee0eSChao Yu STOP_CP_REASON_CORRUPTED_SUMMARY);
1749e3080b01SChao Yu goto skip;
175010d255c3SChao Yu }
17517bc09003SJaegeuk Kim
17529236cac5SJaegeuk Kim /*
17539236cac5SJaegeuk Kim * this is to avoid deadlock:
17549236cac5SJaegeuk Kim * - lock_page(sum_page) - f2fs_replace_block
17553d26fa6bSChao Yu * - check_valid_map() - down_write(sentry_lock)
17563d26fa6bSChao Yu * - down_read(sentry_lock) - change_curseg()
17579236cac5SJaegeuk Kim * - lock_page(sum_page)
17589236cac5SJaegeuk Kim */
1759718e53faSChao Yu if (type == SUM_TYPE_NODE)
176048018b4cSChao Yu submitted += gc_node_segment(sbi, sum->entries, segno,
1761718e53faSChao Yu gc_type);
176248018b4cSChao Yu else
176348018b4cSChao Yu submitted += gc_data_segment(sbi, sum->entries, gc_list,
17647dede886SChao Yu segno, gc_type,
17657dede886SChao Yu force_migrate);
1766c718379bSJaegeuk Kim
17679bf1dcbdSChao Yu stat_inc_gc_seg_count(sbi, data_type, gc_type);
176807c6b593SDaeho Jeong sbi->gc_reclaimed_segs[sbi->gc_mode]++;
17698c7b9ac1SJaegeuk Kim migrated++;
1770c56f16daSChao Yu
1771d6c66cd1SYunlong Song freed:
1772c56f16daSChao Yu if (gc_type == FG_GC &&
1773c56f16daSChao Yu get_valid_blocks(sbi, segno, false) == 0)
1774c56f16daSChao Yu seg_freed++;
1775e3080b01SChao Yu
1776e219aecfSYonggil Song if (__is_large_section(sbi))
1777e219aecfSYonggil Song sbi->next_victim_seg[gc_type] =
1778e219aecfSYonggil Song (segno + 1 < end_segno) ? segno + 1 : NULL_SEGNO;
1779e3080b01SChao Yu skip:
17809236cac5SJaegeuk Kim f2fs_put_page(sum_page, 0);
1781718e53faSChao Yu }
1782718e53faSChao Yu
178348018b4cSChao Yu if (submitted)
17849bf1dcbdSChao Yu f2fs_submit_merged_write(sbi, data_type);
1785718e53faSChao Yu
1786718e53faSChao Yu blk_finish_plug(&plug);
1787718e53faSChao Yu
17889bf1dcbdSChao Yu if (migrated)
17899bf1dcbdSChao Yu stat_inc_gc_sec_count(sbi, data_type, gc_type);
179017d899dfSChao Yu
1791c56f16daSChao Yu return seg_freed;
17927bc09003SJaegeuk Kim }
17937bc09003SJaegeuk Kim
f2fs_gc(struct f2fs_sb_info * sbi,struct f2fs_gc_control * gc_control)1794d147ea4aSJaegeuk Kim int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control)
17957bc09003SJaegeuk Kim {
1796d147ea4aSJaegeuk Kim int gc_type = gc_control->init_gc_type;
1797d147ea4aSJaegeuk Kim unsigned int segno = gc_control->victim_segno;
179836ded4c1SYonggil Song int sec_freed = 0, seg_freed = 0, total_freed = 0, total_sec_freed = 0;
1799c56f16daSChao Yu int ret = 0;
1800d5053a34SJaegeuk Kim struct cp_control cpc;
18017dda2af8SChangman Lee struct gc_inode_list gc_list = {
18027dda2af8SChangman Lee .ilist = LIST_HEAD_INIT(gc_list.ilist),
1803f6bb2a2cSMatthew Wilcox .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS),
18047dda2af8SChangman Lee };
18052ef79ecbSChao Yu unsigned int skipped_round = 0, round = 0;
1806d11cef14SYonggil Song unsigned int upper_secs;
1807d5053a34SJaegeuk Kim
1808d147ea4aSJaegeuk Kim trace_f2fs_gc_begin(sbi->sb, gc_type, gc_control->no_bg_gc,
1809c81d5baeSJaegeuk Kim gc_control->nr_free_secs,
1810c56f16daSChao Yu get_pages(sbi, F2FS_DIRTY_NODES),
1811c56f16daSChao Yu get_pages(sbi, F2FS_DIRTY_DENTS),
1812c56f16daSChao Yu get_pages(sbi, F2FS_DIRTY_IMETA),
1813c56f16daSChao Yu free_sections(sbi),
1814c56f16daSChao Yu free_segments(sbi),
1815c56f16daSChao Yu reserved_segments(sbi),
1816c56f16daSChao Yu prefree_segments(sbi));
1817c56f16daSChao Yu
1818119ee914SJaegeuk Kim cpc.reason = __get_cp_reason(sbi);
18197bc09003SJaegeuk Kim gc_more:
1820c17caf0bSYonggil Song sbi->skipped_gc_rwsem = 0;
18211751e8a6SLinus Torvalds if (unlikely(!(sbi->sb->s_flags & SB_ACTIVE))) {
1822e5dbd956SWeichao Guo ret = -EINVAL;
1823408e9375SJaegeuk Kim goto stop;
1824e5dbd956SWeichao Guo }
18256d5a1495SChao Yu if (unlikely(f2fs_cp_error(sbi))) {
18266d5a1495SChao Yu ret = -EIO;
1827203681f6SJaegeuk Kim goto stop;
18286d5a1495SChao Yu }
18297bc09003SJaegeuk Kim
18302d3f197bSJaegeuk Kim /* Let's run FG_GC, if we don't have enough space. */
18312d3f197bSJaegeuk Kim if (has_not_enough_free_secs(sbi, 0, 0)) {
18322d3f197bSJaegeuk Kim gc_type = FG_GC;
18332d3f197bSJaegeuk Kim
18346e17bfbcSJaegeuk Kim /*
183519f4e688SHou Pengyang * For example, if there are many prefree_segments below given
183619f4e688SHou Pengyang * threshold, we can make them free by checkpoint. Then, we
183719f4e688SHou Pengyang * secure free segments which doesn't need fggc any more.
18386e17bfbcSJaegeuk Kim */
1839d147ea4aSJaegeuk Kim if (prefree_segments(sbi)) {
1840eb61c2ccSChao Yu stat_inc_cp_call_count(sbi, TOTAL_CALL);
18414d57b86dSChao Yu ret = f2fs_write_checkpoint(sbi, &cpc);
18422956e450SJaegeuk Kim if (ret)
18432956e450SJaegeuk Kim goto stop;
184436ded4c1SYonggil Song /* Reset due to checkpoint */
184536ded4c1SYonggil Song sec_freed = 0;
18468fd5a37eSJaegeuk Kim }
1847fe94793eSYunlei He }
18487bc09003SJaegeuk Kim
184919f4e688SHou Pengyang /* f2fs_balance_fs doesn't need to do BG_GC in critical path. */
1850d147ea4aSJaegeuk Kim if (gc_type == BG_GC && gc_control->no_bg_gc) {
1851c56f16daSChao Yu ret = -EINVAL;
185219f4e688SHou Pengyang goto stop;
1853c56f16daSChao Yu }
185471419129SChao Yu retry:
185597767500SQilong Zhang ret = __get_victim(sbi, &segno, gc_type);
185671419129SChao Yu if (ret) {
185771419129SChao Yu /* allow to search victim from sections has pinned data */
185871419129SChao Yu if (ret == -ENODATA && gc_type == FG_GC &&
185971419129SChao Yu f2fs_pinned_section_exists(DIRTY_I(sbi))) {
186071419129SChao Yu f2fs_unpin_all_sections(sbi, false);
186171419129SChao Yu goto retry;
186271419129SChao Yu }
1863408e9375SJaegeuk Kim goto stop;
186471419129SChao Yu }
18657bc09003SJaegeuk Kim
1866d147ea4aSJaegeuk Kim seg_freed = do_garbage_collect(sbi, segno, &gc_list, gc_type,
1867d147ea4aSJaegeuk Kim gc_control->should_migrate_blocks);
1868c56f16daSChao Yu total_freed += seg_freed;
186943727527SJaegeuk Kim
187036ded4c1SYonggil Song if (seg_freed == f2fs_usable_segs_in_sec(sbi, segno)) {
1871d147ea4aSJaegeuk Kim sec_freed++;
187236ded4c1SYonggil Song total_sec_freed++;
187336ded4c1SYonggil Song }
18742ef79ecbSChao Yu
18752d3f197bSJaegeuk Kim if (gc_type == FG_GC) {
18765ec4e49fSJaegeuk Kim sbi->cur_victim_sec = NULL_SEGNO;
187743727527SJaegeuk Kim
1878c1660d88SYangtao Li if (has_enough_free_secs(sbi, sec_freed, 0)) {
18792d3f197bSJaegeuk Kim if (!gc_control->no_bg_gc &&
188036ded4c1SYonggil Song total_sec_freed < gc_control->nr_free_secs)
1881c81d5baeSJaegeuk Kim goto go_gc_more;
18826f8d4455SJaegeuk Kim goto stop;
1883c81d5baeSJaegeuk Kim }
1884d147ea4aSJaegeuk Kim if (sbi->skipped_gc_rwsem)
1885d147ea4aSJaegeuk Kim skipped_round++;
1886d147ea4aSJaegeuk Kim round++;
1887d147ea4aSJaegeuk Kim if (skipped_round > MAX_SKIP_GC_COUNT &&
1888d147ea4aSJaegeuk Kim skipped_round * 2 >= round) {
1889eb61c2ccSChao Yu stat_inc_cp_call_count(sbi, TOTAL_CALL);
1890d147ea4aSJaegeuk Kim ret = f2fs_write_checkpoint(sbi, &cpc);
1891d147ea4aSJaegeuk Kim goto stop;
1892d147ea4aSJaegeuk Kim }
1893c1660d88SYangtao Li } else if (has_enough_free_secs(sbi, 0, 0)) {
18942d3f197bSJaegeuk Kim goto stop;
1895d147ea4aSJaegeuk Kim }
1896a9163b94SByungki Lee
1897d11cef14SYonggil Song __get_secs_required(sbi, NULL, &upper_secs, NULL);
1898d11cef14SYonggil Song
1899d11cef14SYonggil Song /*
1900d11cef14SYonggil Song * Write checkpoint to reclaim prefree segments.
1901d11cef14SYonggil Song * We need more three extra sections for writer's data/node/dentry.
1902d11cef14SYonggil Song */
1903d11cef14SYonggil Song if (free_sections(sbi) <= upper_secs + NR_GC_CHECKPOINT_SECS &&
1904d147ea4aSJaegeuk Kim prefree_segments(sbi)) {
1905eb61c2ccSChao Yu stat_inc_cp_call_count(sbi, TOTAL_CALL);
1906a9163b94SByungki Lee ret = f2fs_write_checkpoint(sbi, &cpc);
1907a9163b94SByungki Lee if (ret)
1908a9163b94SByungki Lee goto stop;
190936ded4c1SYonggil Song /* Reset due to checkpoint */
191036ded4c1SYonggil Song sec_freed = 0;
1911a9163b94SByungki Lee }
1912c81d5baeSJaegeuk Kim go_gc_more:
1913e066b83cSJaegeuk Kim segno = NULL_SEGNO;
19147bc09003SJaegeuk Kim goto gc_more;
1915d147ea4aSJaegeuk Kim
1916408e9375SJaegeuk Kim stop:
1917e066b83cSJaegeuk Kim SIT_I(sbi)->last_victim[ALLOC_NEXT] = 0;
1918d147ea4aSJaegeuk Kim SIT_I(sbi)->last_victim[FLUSH_DEVICE] = gc_control->victim_segno;
1919c56f16daSChao Yu
192071419129SChao Yu if (gc_type == FG_GC)
192171419129SChao Yu f2fs_unpin_all_sections(sbi, true);
192271419129SChao Yu
192336ded4c1SYonggil Song trace_f2fs_gc_end(sbi->sb, ret, total_freed, total_sec_freed,
1924c56f16daSChao Yu get_pages(sbi, F2FS_DIRTY_NODES),
1925c56f16daSChao Yu get_pages(sbi, F2FS_DIRTY_DENTS),
1926c56f16daSChao Yu get_pages(sbi, F2FS_DIRTY_IMETA),
1927c56f16daSChao Yu free_sections(sbi),
1928c56f16daSChao Yu free_segments(sbi),
1929c56f16daSChao Yu reserved_segments(sbi),
1930c56f16daSChao Yu prefree_segments(sbi));
1931c56f16daSChao Yu
1932e4544b63STim Murray f2fs_up_write(&sbi->gc_lock);
19337bc09003SJaegeuk Kim
19347dda2af8SChangman Lee put_gc_inode(&gc_list);
1935d530d4d8SChao Yu
1936d147ea4aSJaegeuk Kim if (gc_control->err_gc_skipped && !ret)
193736ded4c1SYonggil Song ret = total_sec_freed ? 0 : -EAGAIN;
193843727527SJaegeuk Kim return ret;
19397bc09003SJaegeuk Kim }
19407bc09003SJaegeuk Kim
f2fs_create_garbage_collection_cache(void)1941093749e2SChao Yu int __init f2fs_create_garbage_collection_cache(void)
1942093749e2SChao Yu {
1943093749e2SChao Yu victim_entry_slab = f2fs_kmem_cache_create("f2fs_victim_entry",
1944093749e2SChao Yu sizeof(struct victim_entry));
1945870af777SYangtao Li return victim_entry_slab ? 0 : -ENOMEM;
1946093749e2SChao Yu }
1947093749e2SChao Yu
f2fs_destroy_garbage_collection_cache(void)1948093749e2SChao Yu void f2fs_destroy_garbage_collection_cache(void)
1949093749e2SChao Yu {
1950093749e2SChao Yu kmem_cache_destroy(victim_entry_slab);
1951093749e2SChao Yu }
1952093749e2SChao Yu
init_atgc_management(struct f2fs_sb_info * sbi)1953093749e2SChao Yu static void init_atgc_management(struct f2fs_sb_info *sbi)
1954093749e2SChao Yu {
1955093749e2SChao Yu struct atgc_management *am = &sbi->am;
1956093749e2SChao Yu
1957093749e2SChao Yu if (test_opt(sbi, ATGC) &&
1958093749e2SChao Yu SIT_I(sbi)->elapsed_time >= DEF_GC_THREAD_AGE_THRESHOLD)
1959093749e2SChao Yu am->atgc_enabled = true;
1960093749e2SChao Yu
1961093749e2SChao Yu am->root = RB_ROOT_CACHED;
1962093749e2SChao Yu INIT_LIST_HEAD(&am->victim_list);
1963093749e2SChao Yu am->victim_count = 0;
1964093749e2SChao Yu
1965093749e2SChao Yu am->candidate_ratio = DEF_GC_THREAD_CANDIDATE_RATIO;
1966093749e2SChao Yu am->max_candidate_count = DEF_GC_THREAD_MAX_CANDIDATE_COUNT;
1967093749e2SChao Yu am->age_weight = DEF_GC_THREAD_AGE_WEIGHT;
196889e53ff1SChao Yu am->age_threshold = DEF_GC_THREAD_AGE_THRESHOLD;
1969093749e2SChao Yu }
1970093749e2SChao Yu
f2fs_build_gc_manager(struct f2fs_sb_info * sbi)19714d57b86dSChao Yu void f2fs_build_gc_manager(struct f2fs_sb_info *sbi)
19727bc09003SJaegeuk Kim {
19731ad71a27SJaegeuk Kim sbi->gc_pin_file_threshold = DEF_GC_FAILED_PINNED_FILES;
1974d5793249SJaegeuk Kim
1975d5793249SJaegeuk Kim /* give warm/cold data area from slower device */
19760916878dSDamien Le Moal if (f2fs_is_multi_device(sbi) && !__is_large_section(sbi))
1977d5793249SJaegeuk Kim SIT_I(sbi)->last_victim[ALLOC_NEXT] =
1978d5793249SJaegeuk Kim GET_SEGNO(sbi, FDEV(0).end_blk) + 1;
1979093749e2SChao Yu
1980093749e2SChao Yu init_atgc_management(sbi);
19817bc09003SJaegeuk Kim }
198204f0b2eaSQiuyang Sun
free_segment_range(struct f2fs_sb_info * sbi,unsigned int secs,bool gc_only)1983b4b10061SJaegeuk Kim static int free_segment_range(struct f2fs_sb_info *sbi,
1984b4b10061SJaegeuk Kim unsigned int secs, bool gc_only)
198504f0b2eaSQiuyang Sun {
1986b4b10061SJaegeuk Kim unsigned int segno, next_inuse, start, end;
1987b4b10061SJaegeuk Kim struct cp_control cpc = { CP_RESIZE, 0, 0, 0 };
1988b4b10061SJaegeuk Kim int gc_mode, gc_type;
198904f0b2eaSQiuyang Sun int err = 0;
1990b4b10061SJaegeuk Kim int type;
1991b4b10061SJaegeuk Kim
1992b4b10061SJaegeuk Kim /* Force block allocation for GC */
1993b4b10061SJaegeuk Kim MAIN_SECS(sbi) -= secs;
1994b4b10061SJaegeuk Kim start = MAIN_SECS(sbi) * sbi->segs_per_sec;
1995b4b10061SJaegeuk Kim end = MAIN_SEGS(sbi) - 1;
1996b4b10061SJaegeuk Kim
1997b4b10061SJaegeuk Kim mutex_lock(&DIRTY_I(sbi)->seglist_lock);
1998b4b10061SJaegeuk Kim for (gc_mode = 0; gc_mode < MAX_GC_POLICY; gc_mode++)
1999b4b10061SJaegeuk Kim if (SIT_I(sbi)->last_victim[gc_mode] >= start)
2000b4b10061SJaegeuk Kim SIT_I(sbi)->last_victim[gc_mode] = 0;
2001b4b10061SJaegeuk Kim
2002b4b10061SJaegeuk Kim for (gc_type = BG_GC; gc_type <= FG_GC; gc_type++)
2003b4b10061SJaegeuk Kim if (sbi->next_victim_seg[gc_type] >= start)
2004b4b10061SJaegeuk Kim sbi->next_victim_seg[gc_type] = NULL_SEGNO;
2005b4b10061SJaegeuk Kim mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
200604f0b2eaSQiuyang Sun
200704f0b2eaSQiuyang Sun /* Move out cursegs from the target range */
2008d0b9e42aSChao Yu for (type = CURSEG_HOT_DATA; type < NR_CURSEG_PERSIST_TYPE; type++)
20090ef81833SChao Yu f2fs_allocate_segment_for_resize(sbi, type, start, end);
201004f0b2eaSQiuyang Sun
201104f0b2eaSQiuyang Sun /* do GC to move out valid blocks in the range */
201204f0b2eaSQiuyang Sun for (segno = start; segno <= end; segno += sbi->segs_per_sec) {
201304f0b2eaSQiuyang Sun struct gc_inode_list gc_list = {
201404f0b2eaSQiuyang Sun .ilist = LIST_HEAD_INIT(gc_list.ilist),
201504f0b2eaSQiuyang Sun .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS),
201604f0b2eaSQiuyang Sun };
201704f0b2eaSQiuyang Sun
20187dede886SChao Yu do_garbage_collect(sbi, segno, &gc_list, FG_GC, true);
201904f0b2eaSQiuyang Sun put_gc_inode(&gc_list);
202004f0b2eaSQiuyang Sun
2021b4b10061SJaegeuk Kim if (!gc_only && get_valid_blocks(sbi, segno, true)) {
2022b4b10061SJaegeuk Kim err = -EAGAIN;
2023b4b10061SJaegeuk Kim goto out;
202404f0b2eaSQiuyang Sun }
2025b4b10061SJaegeuk Kim if (fatal_signal_pending(current)) {
2026b4b10061SJaegeuk Kim err = -ERESTARTSYS;
2027b4b10061SJaegeuk Kim goto out;
2028b4b10061SJaegeuk Kim }
2029b4b10061SJaegeuk Kim }
2030b4b10061SJaegeuk Kim if (gc_only)
2031b4b10061SJaegeuk Kim goto out;
203204f0b2eaSQiuyang Sun
2033eb61c2ccSChao Yu stat_inc_cp_call_count(sbi, TOTAL_CALL);
2034b4b10061SJaegeuk Kim err = f2fs_write_checkpoint(sbi, &cpc);
203504f0b2eaSQiuyang Sun if (err)
2036b4b10061SJaegeuk Kim goto out;
203704f0b2eaSQiuyang Sun
203804f0b2eaSQiuyang Sun next_inuse = find_next_inuse(FREE_I(sbi), end + 1, start);
203904f0b2eaSQiuyang Sun if (next_inuse <= end) {
2040dcbb4c10SJoe Perches f2fs_err(sbi, "segno %u should be free but still inuse!",
2041dcbb4c10SJoe Perches next_inuse);
204204f0b2eaSQiuyang Sun f2fs_bug_on(sbi, 1);
204304f0b2eaSQiuyang Sun }
2044b4b10061SJaegeuk Kim out:
2045b4b10061SJaegeuk Kim MAIN_SECS(sbi) += secs;
204604f0b2eaSQiuyang Sun return err;
204704f0b2eaSQiuyang Sun }
204804f0b2eaSQiuyang Sun
update_sb_metadata(struct f2fs_sb_info * sbi,int secs)204904f0b2eaSQiuyang Sun static void update_sb_metadata(struct f2fs_sb_info *sbi, int secs)
205004f0b2eaSQiuyang Sun {
205104f0b2eaSQiuyang Sun struct f2fs_super_block *raw_sb = F2FS_RAW_SUPER(sbi);
2052a4ba5dfcSChao Yu int section_count;
2053a4ba5dfcSChao Yu int segment_count;
2054a4ba5dfcSChao Yu int segment_count_main;
2055a4ba5dfcSChao Yu long long block_count;
205604f0b2eaSQiuyang Sun int segs = secs * sbi->segs_per_sec;
205704f0b2eaSQiuyang Sun
2058e4544b63STim Murray f2fs_down_write(&sbi->sb_lock);
2059a4ba5dfcSChao Yu
2060a4ba5dfcSChao Yu section_count = le32_to_cpu(raw_sb->section_count);
2061a4ba5dfcSChao Yu segment_count = le32_to_cpu(raw_sb->segment_count);
2062a4ba5dfcSChao Yu segment_count_main = le32_to_cpu(raw_sb->segment_count_main);
2063a4ba5dfcSChao Yu block_count = le64_to_cpu(raw_sb->block_count);
2064a4ba5dfcSChao Yu
206504f0b2eaSQiuyang Sun raw_sb->section_count = cpu_to_le32(section_count + secs);
206604f0b2eaSQiuyang Sun raw_sb->segment_count = cpu_to_le32(segment_count + segs);
206704f0b2eaSQiuyang Sun raw_sb->segment_count_main = cpu_to_le32(segment_count_main + segs);
206804f0b2eaSQiuyang Sun raw_sb->block_count = cpu_to_le64(block_count +
206904f0b2eaSQiuyang Sun (long long)segs * sbi->blocks_per_seg);
207046d9ce19SQiuyang Sun if (f2fs_is_multi_device(sbi)) {
207146d9ce19SQiuyang Sun int last_dev = sbi->s_ndevs - 1;
207246d9ce19SQiuyang Sun int dev_segs =
207346d9ce19SQiuyang Sun le32_to_cpu(raw_sb->devs[last_dev].total_segments);
207446d9ce19SQiuyang Sun
207546d9ce19SQiuyang Sun raw_sb->devs[last_dev].total_segments =
207646d9ce19SQiuyang Sun cpu_to_le32(dev_segs + segs);
207746d9ce19SQiuyang Sun }
2078a4ba5dfcSChao Yu
2079e4544b63STim Murray f2fs_up_write(&sbi->sb_lock);
208004f0b2eaSQiuyang Sun }
208104f0b2eaSQiuyang Sun
update_fs_metadata(struct f2fs_sb_info * sbi,int secs)208204f0b2eaSQiuyang Sun static void update_fs_metadata(struct f2fs_sb_info *sbi, int secs)
208304f0b2eaSQiuyang Sun {
208404f0b2eaSQiuyang Sun int segs = secs * sbi->segs_per_sec;
208546d9ce19SQiuyang Sun long long blks = (long long)segs * sbi->blocks_per_seg;
208604f0b2eaSQiuyang Sun long long user_block_count =
208704f0b2eaSQiuyang Sun le64_to_cpu(F2FS_CKPT(sbi)->user_block_count);
208804f0b2eaSQiuyang Sun
208904f0b2eaSQiuyang Sun SM_I(sbi)->segment_count = (int)SM_I(sbi)->segment_count + segs;
209004f0b2eaSQiuyang Sun MAIN_SEGS(sbi) = (int)MAIN_SEGS(sbi) + segs;
2091b4b10061SJaegeuk Kim MAIN_SECS(sbi) += secs;
209204f0b2eaSQiuyang Sun FREE_I(sbi)->free_sections = (int)FREE_I(sbi)->free_sections + secs;
209304f0b2eaSQiuyang Sun FREE_I(sbi)->free_segments = (int)FREE_I(sbi)->free_segments + segs;
209446d9ce19SQiuyang Sun F2FS_CKPT(sbi)->user_block_count = cpu_to_le64(user_block_count + blks);
209546d9ce19SQiuyang Sun
209646d9ce19SQiuyang Sun if (f2fs_is_multi_device(sbi)) {
209746d9ce19SQiuyang Sun int last_dev = sbi->s_ndevs - 1;
209846d9ce19SQiuyang Sun
209946d9ce19SQiuyang Sun FDEV(last_dev).total_segments =
210046d9ce19SQiuyang Sun (int)FDEV(last_dev).total_segments + segs;
210146d9ce19SQiuyang Sun FDEV(last_dev).end_blk =
210246d9ce19SQiuyang Sun (long long)FDEV(last_dev).end_blk + blks;
210346d9ce19SQiuyang Sun #ifdef CONFIG_BLK_DEV_ZONED
21042e2c6e9bSJaegeuk Kim FDEV(last_dev).nr_blkz = FDEV(last_dev).nr_blkz +
21052e2c6e9bSJaegeuk Kim div_u64(blks, sbi->blocks_per_blkz);
210646d9ce19SQiuyang Sun #endif
210746d9ce19SQiuyang Sun }
210804f0b2eaSQiuyang Sun }
210904f0b2eaSQiuyang Sun
f2fs_resize_fs(struct file * filp,__u64 block_count)2110d8189834SChao Yu int f2fs_resize_fs(struct file *filp, __u64 block_count)
211104f0b2eaSQiuyang Sun {
2112d8189834SChao Yu struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(filp));
211304f0b2eaSQiuyang Sun __u64 old_block_count, shrunk_blocks;
2114b4b10061SJaegeuk Kim struct cp_control cpc = { CP_RESIZE, 0, 0, 0 };
211504f0b2eaSQiuyang Sun unsigned int secs;
211604f0b2eaSQiuyang Sun int err = 0;
211704f0b2eaSQiuyang Sun __u32 rem;
211804f0b2eaSQiuyang Sun
211904f0b2eaSQiuyang Sun old_block_count = le64_to_cpu(F2FS_RAW_SUPER(sbi)->block_count);
212004f0b2eaSQiuyang Sun if (block_count > old_block_count)
212104f0b2eaSQiuyang Sun return -EINVAL;
212204f0b2eaSQiuyang Sun
212346d9ce19SQiuyang Sun if (f2fs_is_multi_device(sbi)) {
212446d9ce19SQiuyang Sun int last_dev = sbi->s_ndevs - 1;
212546d9ce19SQiuyang Sun __u64 last_segs = FDEV(last_dev).total_segments;
212646d9ce19SQiuyang Sun
212746d9ce19SQiuyang Sun if (block_count + last_segs * sbi->blocks_per_seg <=
212846d9ce19SQiuyang Sun old_block_count)
212946d9ce19SQiuyang Sun return -EINVAL;
213046d9ce19SQiuyang Sun }
213146d9ce19SQiuyang Sun
213204f0b2eaSQiuyang Sun /* new fs size should align to section size */
213304f0b2eaSQiuyang Sun div_u64_rem(block_count, BLKS_PER_SEC(sbi), &rem);
213404f0b2eaSQiuyang Sun if (rem)
213504f0b2eaSQiuyang Sun return -EINVAL;
213604f0b2eaSQiuyang Sun
213704f0b2eaSQiuyang Sun if (block_count == old_block_count)
213804f0b2eaSQiuyang Sun return 0;
213904f0b2eaSQiuyang Sun
214004f0b2eaSQiuyang Sun if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
2141dcbb4c10SJoe Perches f2fs_err(sbi, "Should run fsck to repair first.");
214210f966bbSChao Yu return -EFSCORRUPTED;
214304f0b2eaSQiuyang Sun }
214404f0b2eaSQiuyang Sun
214504f0b2eaSQiuyang Sun if (test_opt(sbi, DISABLE_CHECKPOINT)) {
2146dcbb4c10SJoe Perches f2fs_err(sbi, "Checkpoint should be enabled.");
214704f0b2eaSQiuyang Sun return -EINVAL;
214804f0b2eaSQiuyang Sun }
214904f0b2eaSQiuyang Sun
2150d8189834SChao Yu err = mnt_want_write_file(filp);
2151d8189834SChao Yu if (err)
2152d8189834SChao Yu return err;
2153d8189834SChao Yu
215404f0b2eaSQiuyang Sun shrunk_blocks = old_block_count - block_count;
215504f0b2eaSQiuyang Sun secs = div_u64(shrunk_blocks, BLKS_PER_SEC(sbi));
2156b4b10061SJaegeuk Kim
2157b4b10061SJaegeuk Kim /* stop other GC */
2158d8189834SChao Yu if (!f2fs_down_write_trylock(&sbi->gc_lock)) {
2159d8189834SChao Yu err = -EAGAIN;
2160d8189834SChao Yu goto out_drop_write;
2161d8189834SChao Yu }
2162b4b10061SJaegeuk Kim
2163b4b10061SJaegeuk Kim /* stop CP to protect MAIN_SEC in free_segment_range */
2164b4b10061SJaegeuk Kim f2fs_lock_op(sbi);
21653ab0598eSChao Yu
21663ab0598eSChao Yu spin_lock(&sbi->stat_lock);
21673ab0598eSChao Yu if (shrunk_blocks + valid_user_blocks(sbi) +
21683ab0598eSChao Yu sbi->current_reserved_blocks + sbi->unusable_block_count +
21693ab0598eSChao Yu F2FS_OPTION(sbi).root_reserved_blocks > sbi->user_block_count)
21703ab0598eSChao Yu err = -ENOSPC;
21713ab0598eSChao Yu spin_unlock(&sbi->stat_lock);
21723ab0598eSChao Yu
21733ab0598eSChao Yu if (err)
21743ab0598eSChao Yu goto out_unlock;
21753ab0598eSChao Yu
2176b4b10061SJaegeuk Kim err = free_segment_range(sbi, secs, true);
21773ab0598eSChao Yu
21783ab0598eSChao Yu out_unlock:
2179b4b10061SJaegeuk Kim f2fs_unlock_op(sbi);
2180e4544b63STim Murray f2fs_up_write(&sbi->gc_lock);
2181d8189834SChao Yu out_drop_write:
2182d8189834SChao Yu mnt_drop_write_file(filp);
2183b4b10061SJaegeuk Kim if (err)
2184b4b10061SJaegeuk Kim return err;
2185b4b10061SJaegeuk Kim
2186880b9577SDarrick J. Wong err = freeze_super(sbi->sb, FREEZE_HOLDER_USERSPACE);
21878bec7dd1SChao Yu if (err)
21888bec7dd1SChao Yu return err;
2189d8189834SChao Yu
2190d8189834SChao Yu if (f2fs_readonly(sbi->sb)) {
2191880b9577SDarrick J. Wong err = thaw_super(sbi->sb, FREEZE_HOLDER_USERSPACE);
2192880b9577SDarrick J. Wong if (err)
2193880b9577SDarrick J. Wong return err;
2194d8189834SChao Yu return -EROFS;
2195d8189834SChao Yu }
2196d8189834SChao Yu
2197e4544b63STim Murray f2fs_down_write(&sbi->gc_lock);
2198e4544b63STim Murray f2fs_down_write(&sbi->cp_global_sem);
2199b4b10061SJaegeuk Kim
220004f0b2eaSQiuyang Sun spin_lock(&sbi->stat_lock);
220104f0b2eaSQiuyang Sun if (shrunk_blocks + valid_user_blocks(sbi) +
220204f0b2eaSQiuyang Sun sbi->current_reserved_blocks + sbi->unusable_block_count +
220304f0b2eaSQiuyang Sun F2FS_OPTION(sbi).root_reserved_blocks > sbi->user_block_count)
220404f0b2eaSQiuyang Sun err = -ENOSPC;
220504f0b2eaSQiuyang Sun else
220604f0b2eaSQiuyang Sun sbi->user_block_count -= shrunk_blocks;
220704f0b2eaSQiuyang Sun spin_unlock(&sbi->stat_lock);
220804f0b2eaSQiuyang Sun if (err)
2209b4b10061SJaegeuk Kim goto out_err;
2210b4b10061SJaegeuk Kim
221128fc4e90SZhang Qilong set_sbi_flag(sbi, SBI_IS_RESIZEFS);
2212b4b10061SJaegeuk Kim err = free_segment_range(sbi, secs, false);
2213b4b10061SJaegeuk Kim if (err)
2214b4b10061SJaegeuk Kim goto recover_out;
221504f0b2eaSQiuyang Sun
221604f0b2eaSQiuyang Sun update_sb_metadata(sbi, -secs);
221704f0b2eaSQiuyang Sun
221804f0b2eaSQiuyang Sun err = f2fs_commit_super(sbi, false);
221904f0b2eaSQiuyang Sun if (err) {
222004f0b2eaSQiuyang Sun update_sb_metadata(sbi, secs);
2221b4b10061SJaegeuk Kim goto recover_out;
222204f0b2eaSQiuyang Sun }
222304f0b2eaSQiuyang Sun
222404f0b2eaSQiuyang Sun update_fs_metadata(sbi, -secs);
222504f0b2eaSQiuyang Sun clear_sbi_flag(sbi, SBI_IS_RESIZEFS);
222668275682SSahitya Tummala set_sbi_flag(sbi, SBI_IS_DIRTY);
222768275682SSahitya Tummala
2228eb61c2ccSChao Yu stat_inc_cp_call_count(sbi, TOTAL_CALL);
2229b4b10061SJaegeuk Kim err = f2fs_write_checkpoint(sbi, &cpc);
223004f0b2eaSQiuyang Sun if (err) {
223104f0b2eaSQiuyang Sun update_fs_metadata(sbi, secs);
223204f0b2eaSQiuyang Sun update_sb_metadata(sbi, secs);
223304f0b2eaSQiuyang Sun f2fs_commit_super(sbi, false);
223404f0b2eaSQiuyang Sun }
2235b4b10061SJaegeuk Kim recover_out:
223628fc4e90SZhang Qilong clear_sbi_flag(sbi, SBI_IS_RESIZEFS);
223704f0b2eaSQiuyang Sun if (err) {
223804f0b2eaSQiuyang Sun set_sbi_flag(sbi, SBI_NEED_FSCK);
2239dcbb4c10SJoe Perches f2fs_err(sbi, "resize_fs failed, should run fsck to repair!");
224004f0b2eaSQiuyang Sun
224104f0b2eaSQiuyang Sun spin_lock(&sbi->stat_lock);
224204f0b2eaSQiuyang Sun sbi->user_block_count += shrunk_blocks;
224304f0b2eaSQiuyang Sun spin_unlock(&sbi->stat_lock);
224404f0b2eaSQiuyang Sun }
2245b4b10061SJaegeuk Kim out_err:
2246e4544b63STim Murray f2fs_up_write(&sbi->cp_global_sem);
2247e4544b63STim Murray f2fs_up_write(&sbi->gc_lock);
2248880b9577SDarrick J. Wong thaw_super(sbi->sb, FREEZE_HOLDER_USERSPACE);
224904f0b2eaSQiuyang Sun return err;
225004f0b2eaSQiuyang Sun }
2251