xref: /openbmc/linux/fs/f2fs/f2fs.h (revision c277991d7cf07cf68673abea52d4a96c4ee6272d)
1d29fbcdbSNishad Kamdar /* SPDX-License-Identifier: GPL-2.0 */
20a8165d7SJaegeuk Kim /*
339a53e0cSJaegeuk Kim  * fs/f2fs/f2fs.h
439a53e0cSJaegeuk Kim  *
539a53e0cSJaegeuk Kim  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
639a53e0cSJaegeuk Kim  *             http://www.samsung.com/
739a53e0cSJaegeuk Kim  */
839a53e0cSJaegeuk Kim #ifndef _LINUX_F2FS_H
939a53e0cSJaegeuk Kim #define _LINUX_F2FS_H
1039a53e0cSJaegeuk Kim 
11f847c699SChao Yu #include <linux/uio.h>
1239a53e0cSJaegeuk Kim #include <linux/types.h>
1339a53e0cSJaegeuk Kim #include <linux/page-flags.h>
1439a53e0cSJaegeuk Kim #include <linux/buffer_head.h>
1539a53e0cSJaegeuk Kim #include <linux/slab.h>
1639a53e0cSJaegeuk Kim #include <linux/crc32.h>
1739a53e0cSJaegeuk Kim #include <linux/magic.h>
18c2d715d1SJaegeuk Kim #include <linux/kobject.h>
197bd59381SGu Zheng #include <linux/sched.h>
207c2e5963SJaegeuk Kim #include <linux/cred.h>
21df808180SMatthew Wilcox (Oracle) #include <linux/sched/mm.h>
2239307a8eSJaegeuk Kim #include <linux/vmalloc.h>
23740432f8SJaegeuk Kim #include <linux/bio.h>
24d0239e1bSJaegeuk Kim #include <linux/blkdev.h>
250abd675eSChao Yu #include <linux/quotaops.h>
26c6a564ffSChristoph Hellwig #include <linux/part_stat.h>
2743b6573bSKeith Mok #include <crypto/hash.h>
2839a53e0cSJaegeuk Kim 
29734f0d24SDave Chinner #include <linux/fscrypt.h>
3095ae251fSEric Biggers #include <linux/fsverity.h>
31734f0d24SDave Chinner 
3251dcbdacSMatthew Wilcox (Oracle) struct pagevec;
3351dcbdacSMatthew Wilcox (Oracle) 
345d56b671SJaegeuk Kim #ifdef CONFIG_F2FS_CHECK_FS
359850cf4aSJaegeuk Kim #define f2fs_bug_on(sbi, condition)	BUG_ON(condition)
365d56b671SJaegeuk Kim #else
379850cf4aSJaegeuk Kim #define f2fs_bug_on(sbi, condition)					\
389850cf4aSJaegeuk Kim 	do {								\
39db489652SYangtao Li 		if (WARN_ON(condition))					\
40caf0047eSChao Yu 			set_sbi_flag(sbi, SBI_NEED_FSCK);		\
419850cf4aSJaegeuk Kim 	} while (0)
425d56b671SJaegeuk Kim #endif
435d56b671SJaegeuk Kim 
442c63feadSJaegeuk Kim enum {
452c63feadSJaegeuk Kim 	FAULT_KMALLOC,
46628b3d14SChao Yu 	FAULT_KVMALLOC,
47c41f3cc3SJaegeuk Kim 	FAULT_PAGE_ALLOC,
4801eccef7SChao Yu 	FAULT_PAGE_GET,
49b96d9b3bSChao Yu 	FAULT_ALLOC_BIO,	/* it's obsolete due to bio_alloc() will never fail */
50cb78942bSJaegeuk Kim 	FAULT_ALLOC_NID,
51cb78942bSJaegeuk Kim 	FAULT_ORPHAN,
52cb78942bSJaegeuk Kim 	FAULT_BLOCK,
53cb78942bSJaegeuk Kim 	FAULT_DIR_DEPTH,
5453aa6bbfSJaegeuk Kim 	FAULT_EVICT_INODE,
5514b44d23SJaegeuk Kim 	FAULT_TRUNCATE,
566f5c2ed0SChao Yu 	FAULT_READ_IO,
570f348028SChao Yu 	FAULT_CHECKPOINT,
58b83dcfe6SChao Yu 	FAULT_DISCARD,
596f5c2ed0SChao Yu 	FAULT_WRITE_IO,
6032410577SChao Yu 	FAULT_SLAB_ALLOC,
6110a26878SChao Yu 	FAULT_DQUOT_INIT,
623e020389SChao Yu 	FAULT_LOCK_OP,
6318792e64SChao Yu 	FAULT_BLKADDR,
642c63feadSJaegeuk Kim 	FAULT_MAX,
652c63feadSJaegeuk Kim };
662c63feadSJaegeuk Kim 
677fa750a1SArnd Bergmann #ifdef CONFIG_F2FS_FAULT_INJECTION
68447286ebSYangtao Li #define F2FS_ALL_FAULT_TYPE		(GENMASK(FAULT_MAX - 1, 0))
69d494500aSChao Yu 
7008796897SSheng Yong struct f2fs_fault_info {
7108796897SSheng Yong 	atomic_t inject_ops;
7208796897SSheng Yong 	unsigned int inject_rate;
7308796897SSheng Yong 	unsigned int inject_type;
7408796897SSheng Yong };
7508796897SSheng Yong 
7619880e6eSAlexey Dobriyan extern const char *f2fs_fault_name[FAULT_MAX];
77447286ebSYangtao Li #define IS_FAULT_SET(fi, type) ((fi)->inject_type & BIT(type))
782c63feadSJaegeuk Kim #endif
792c63feadSJaegeuk Kim 
8039a53e0cSJaegeuk Kim /*
8139a53e0cSJaegeuk Kim  * For mount options
8239a53e0cSJaegeuk Kim  */
8339a53e0cSJaegeuk Kim #define F2FS_MOUNT_DISABLE_ROLL_FORWARD	0x00000002
8439a53e0cSJaegeuk Kim #define F2FS_MOUNT_DISCARD		0x00000004
8539a53e0cSJaegeuk Kim #define F2FS_MOUNT_NOHEAP		0x00000008
8639a53e0cSJaegeuk Kim #define F2FS_MOUNT_XATTR_USER		0x00000010
8739a53e0cSJaegeuk Kim #define F2FS_MOUNT_POSIX_ACL		0x00000020
8839a53e0cSJaegeuk Kim #define F2FS_MOUNT_DISABLE_EXT_IDENTIFY	0x00000040
89444c580fSJaegeuk Kim #define F2FS_MOUNT_INLINE_XATTR		0x00000080
901001b347SHuajun Li #define F2FS_MOUNT_INLINE_DATA		0x00000100
9134d67debSChao Yu #define F2FS_MOUNT_INLINE_DENTRY	0x00000200
9234d67debSChao Yu #define F2FS_MOUNT_FLUSH_MERGE		0x00000400
9334d67debSChao Yu #define F2FS_MOUNT_NOBARRIER		0x00000800
94d5053a34SJaegeuk Kim #define F2FS_MOUNT_FASTBOOT		0x00001000
9512607c1bSJaegeuk Kim #define F2FS_MOUNT_READ_EXTENT_CACHE	0x00002000
96343f40f0SChao Yu #define F2FS_MOUNT_DATA_FLUSH		0x00008000
9773faec4dSJaegeuk Kim #define F2FS_MOUNT_FAULT_INJECTION	0x00010000
980abd675eSChao Yu #define F2FS_MOUNT_USRQUOTA		0x00080000
990abd675eSChao Yu #define F2FS_MOUNT_GRPQUOTA		0x00100000
1005c57132eSChao Yu #define F2FS_MOUNT_PRJQUOTA		0x00200000
1014b2414d0SChao Yu #define F2FS_MOUNT_QUOTA		0x00400000
1026afc662eSChao Yu #define F2FS_MOUNT_INLINE_XATTR_SIZE	0x00800000
1037e65be49SJaegeuk Kim #define F2FS_MOUNT_RESERVE_ROOT		0x01000000
1044354994fSDaniel Rosenberg #define F2FS_MOUNT_DISABLE_CHECKPOINT	0x02000000
105a9117ecaSChao Yu #define F2FS_MOUNT_NORECOVERY		0x04000000
106093749e2SChao Yu #define F2FS_MOUNT_ATGC			0x08000000
107261eeb9cSDaeho Jeong #define F2FS_MOUNT_MERGE_CHECKPOINT	0x10000000
1085911d2d1SChao Yu #define	F2FS_MOUNT_GC_MERGE		0x20000000
1096ce19affSChao Yu #define F2FS_MOUNT_COMPRESS_CACHE	0x40000000
11071644dffSJaegeuk Kim #define F2FS_MOUNT_AGE_EXTENT_CACHE	0x80000000
11139a53e0cSJaegeuk Kim 
11263189b78SChao Yu #define F2FS_OPTION(sbi)	((sbi)->mount_opt)
11363189b78SChao Yu #define clear_opt(sbi, option)	(F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option)
11463189b78SChao Yu #define set_opt(sbi, option)	(F2FS_OPTION(sbi).opt |= F2FS_MOUNT_##option)
11563189b78SChao Yu #define test_opt(sbi, option)	(F2FS_OPTION(sbi).opt & F2FS_MOUNT_##option)
11639a53e0cSJaegeuk Kim 
11739a53e0cSJaegeuk Kim #define ver_after(a, b)	(typecheck(unsigned long long, a) &&		\
11839a53e0cSJaegeuk Kim 		typecheck(unsigned long long, b) &&			\
11939a53e0cSJaegeuk Kim 		((long long)((a) - (b)) > 0))
12039a53e0cSJaegeuk Kim 
121a9841c4dSJaegeuk Kim typedef u32 block_t;	/*
122a9841c4dSJaegeuk Kim 			 * should not change u32, since it is the on-disk block
123a9841c4dSJaegeuk Kim 			 * address format, __le32.
124a9841c4dSJaegeuk Kim 			 */
12539a53e0cSJaegeuk Kim typedef u32 nid_t;
12639a53e0cSJaegeuk Kim 
1274c8ff709SChao Yu #define COMPRESS_EXT_NUM		16
1284c8ff709SChao Yu 
129e4544b63STim Murray /*
130e4544b63STim Murray  * An implementation of an rwsem that is explicitly unfair to readers. This
131e4544b63STim Murray  * prevents priority inversion when a low-priority reader acquires the read lock
132e4544b63STim Murray  * while sleeping on the write lock but the write lock is needed by
133e4544b63STim Murray  * higher-priority clients.
134e4544b63STim Murray  */
135e4544b63STim Murray 
136e4544b63STim Murray struct f2fs_rwsem {
137e4544b63STim Murray         struct rw_semaphore internal_rwsem;
1387f8e249dSJaegeuk Kim #ifdef CONFIG_F2FS_UNFAIR_RWSEM
139e4544b63STim Murray         wait_queue_head_t read_waiters;
1407f8e249dSJaegeuk Kim #endif
141e4544b63STim Murray };
142e4544b63STim Murray 
14339a53e0cSJaegeuk Kim struct f2fs_mount_info {
14439a53e0cSJaegeuk Kim 	unsigned int opt;
14563189b78SChao Yu 	int write_io_size_bits;		/* Write IO size bits */
14663189b78SChao Yu 	block_t root_reserved_blocks;	/* root reserved blocks */
14763189b78SChao Yu 	kuid_t s_resuid;		/* reserved blocks for uid */
14863189b78SChao Yu 	kgid_t s_resgid;		/* reserved blocks for gid */
14963189b78SChao Yu 	int active_logs;		/* # of active logs */
15063189b78SChao Yu 	int inline_xattr_size;		/* inline xattr size */
15163189b78SChao Yu #ifdef CONFIG_F2FS_FAULT_INJECTION
15263189b78SChao Yu 	struct f2fs_fault_info fault_info;	/* For fault injection */
15363189b78SChao Yu #endif
15463189b78SChao Yu #ifdef CONFIG_QUOTA
15563189b78SChao Yu 	/* Names of quota files with journalled quota */
15663189b78SChao Yu 	char *s_qf_names[MAXQUOTAS];
15763189b78SChao Yu 	int s_jquota_fmt;			/* Format of quota to use */
15863189b78SChao Yu #endif
15963189b78SChao Yu 	/* For which write hints are passed down to block layer */
16063189b78SChao Yu 	int alloc_mode;			/* segment allocation policy */
16163189b78SChao Yu 	int fsync_mode;			/* fsync policy */
162b0332a0fSChao Yu 	int fs_mode;			/* fs mode: LFS or ADAPTIVE */
163bbbc34fdSChao Yu 	int bggc_mode;			/* bggc mode: off, on or sync */
1647a8fc586SDaeho Jeong 	int memory_mode;		/* memory mode */
1654f993264SChao Yu 	int discard_unit;		/*
1664f993264SChao Yu 					 * discard command's offset/size should
1674f993264SChao Yu 					 * be aligned to this unit: block,
1684f993264SChao Yu 					 * segment or section
1694f993264SChao Yu 					 */
170ac4acb1fSEric Biggers 	struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
1711ae18f71SJaegeuk Kim 	block_t unusable_cap_perc;	/* percentage for cap */
1724d3aed70SDaniel Rosenberg 	block_t unusable_cap;		/* Amount of space allowed to be
1734d3aed70SDaniel Rosenberg 					 * unusable when disabling checkpoint
1744d3aed70SDaniel Rosenberg 					 */
1754c8ff709SChao Yu 
1764c8ff709SChao Yu 	/* For compression */
1774c8ff709SChao Yu 	unsigned char compress_algorithm;	/* algorithm type */
178b28f047bSChao Yu 	unsigned char compress_log_size;	/* cluster log size */
1793fde13f8SChao Yu 	unsigned char compress_level;		/* compress level */
180b28f047bSChao Yu 	bool compress_chksum;			/* compressed data chksum */
1814c8ff709SChao Yu 	unsigned char compress_ext_cnt;		/* extension count */
182151b1982SFengnan Chang 	unsigned char nocompress_ext_cnt;		/* nocompress extension count */
183602a16d5SDaeho Jeong 	int compress_mode;			/* compression mode */
1844c8ff709SChao Yu 	unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN];	/* extensions */
185151b1982SFengnan Chang 	unsigned char noextensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */
18639a53e0cSJaegeuk Kim };
18739a53e0cSJaegeuk Kim 
188cde4de12SJaegeuk Kim #define F2FS_FEATURE_ENCRYPT		0x0001
1890bfd7a09SDamien Le Moal #define F2FS_FEATURE_BLKZONED		0x0002
190e65ef207SJaegeuk Kim #define F2FS_FEATURE_ATOMIC_WRITE	0x0004
1917a2af766SChao Yu #define F2FS_FEATURE_EXTRA_ATTR		0x0008
1925c57132eSChao Yu #define F2FS_FEATURE_PRJQUOTA		0x0010
193704956ecSChao Yu #define F2FS_FEATURE_INODE_CHKSUM	0x0020
1946afc662eSChao Yu #define F2FS_FEATURE_FLEXIBLE_INLINE_XATTR	0x0040
195234a9689SJaegeuk Kim #define F2FS_FEATURE_QUOTA_INO		0x0080
1961c1d35dfSChao Yu #define F2FS_FEATURE_INODE_CRTIME	0x0100
197b7c409deSSheng Yong #define F2FS_FEATURE_LOST_FOUND		0x0200
19895ae251fSEric Biggers #define F2FS_FEATURE_VERITY		0x0400
199d440c52dSJunling Zheng #define F2FS_FEATURE_SB_CHKSUM		0x0800
2005aba5430SDaniel Rosenberg #define F2FS_FEATURE_CASEFOLD		0x1000
2014c8ff709SChao Yu #define F2FS_FEATURE_COMPRESSION	0x2000
202a7d9fe3cSJaegeuk Kim #define F2FS_FEATURE_RO			0x4000
203cde4de12SJaegeuk Kim 
2047beb01f7SChao Yu #define __F2FS_HAS_FEATURE(raw_super, mask)				\
2057beb01f7SChao Yu 	((raw_super->feature & cpu_to_le32(mask)) != 0)
2067beb01f7SChao Yu #define F2FS_HAS_FEATURE(sbi, mask)	__F2FS_HAS_FEATURE(sbi->raw_super, mask)
20776f105a2SJaegeuk Kim 
20839a53e0cSJaegeuk Kim /*
2097c2e5963SJaegeuk Kim  * Default values for user and/or group using reserved blocks
2107c2e5963SJaegeuk Kim  */
2117c2e5963SJaegeuk Kim #define	F2FS_DEF_RESUID		0
2127c2e5963SJaegeuk Kim #define	F2FS_DEF_RESGID		0
2137c2e5963SJaegeuk Kim 
2147c2e5963SJaegeuk Kim /*
21539a53e0cSJaegeuk Kim  * For checkpoint manager
21639a53e0cSJaegeuk Kim  */
21739a53e0cSJaegeuk Kim enum {
21839a53e0cSJaegeuk Kim 	NAT_BITMAP,
21939a53e0cSJaegeuk Kim 	SIT_BITMAP
22039a53e0cSJaegeuk Kim };
22139a53e0cSJaegeuk Kim 
222c473f1a9SChao Yu #define	CP_UMOUNT	0x00000001
223c473f1a9SChao Yu #define	CP_FASTBOOT	0x00000002
224c473f1a9SChao Yu #define	CP_SYNC		0x00000004
225c473f1a9SChao Yu #define	CP_RECOVERY	0x00000008
226c473f1a9SChao Yu #define	CP_DISCARD	0x00000010
2271f43e2adSChao Yu #define CP_TRIMMED	0x00000020
2284354994fSDaniel Rosenberg #define CP_PAUSE	0x00000040
229b4b10061SJaegeuk Kim #define CP_RESIZE 	0x00000080
23075ab4cb8SJaegeuk Kim 
231ecc9aa00SChao Yu #define DEF_MAX_DISCARD_REQUEST		8	/* issue 8 discards per round */
232969d1b18SChao Yu #define DEF_MIN_DISCARD_ISSUE_TIME	50	/* 50 ms, if exists */
233f9d1dcedSYunlei He #define DEF_MID_DISCARD_ISSUE_TIME	500	/* 500 ms, if device busy */
234969d1b18SChao Yu #define DEF_MAX_DISCARD_ISSUE_TIME	60000	/* 60 s, if no candidates */
2358bb4f253SJaegeuk Kim #define DEF_DISCARD_URGENT_UTIL		80	/* do more discard over 80% */
23660b99b48SJaegeuk Kim #define DEF_CP_INTERVAL			60	/* 60 secs */
237dcf25fe8SChao Yu #define DEF_IDLE_INTERVAL		5	/* 5 secs */
2384354994fSDaniel Rosenberg #define DEF_DISABLE_INTERVAL		5	/* 5 secs */
239db610a64SJaegeuk Kim #define DEF_DISABLE_QUICK_INTERVAL	1	/* 1 secs */
24003f2c02dSJaegeuk Kim #define DEF_UMOUNT_DISCARD_TIMEOUT	5	/* 5 secs */
241bba681cbSJaegeuk Kim 
24275ab4cb8SJaegeuk Kim struct cp_control {
24375ab4cb8SJaegeuk Kim 	int reason;
2444b2fecc8SJaegeuk Kim 	__u64 trim_start;
2454b2fecc8SJaegeuk Kim 	__u64 trim_end;
2464b2fecc8SJaegeuk Kim 	__u64 trim_minlen;
24775ab4cb8SJaegeuk Kim };
24875ab4cb8SJaegeuk Kim 
249662befdaSChao Yu /*
250e1da7872SChao Yu  * indicate meta/data type
251662befdaSChao Yu  */
252662befdaSChao Yu enum {
253662befdaSChao Yu 	META_CP,
254662befdaSChao Yu 	META_NAT,
25581c1a0f1SChao Yu 	META_SIT,
2564c521f49SJaegeuk Kim 	META_SSA,
257b63e7be5SChao Yu 	META_MAX,
2584c521f49SJaegeuk Kim 	META_POR,
25993770ab7SChao Yu 	DATA_GENERIC,		/* check range only */
26093770ab7SChao Yu 	DATA_GENERIC_ENHANCE,	/* strong check on range and segment bitmap */
26193770ab7SChao Yu 	DATA_GENERIC_ENHANCE_READ,	/*
26293770ab7SChao Yu 					 * strong check on range and segment
26393770ab7SChao Yu 					 * bitmap but no warning due to race
26493770ab7SChao Yu 					 * condition of read on truncated area
26593770ab7SChao Yu 					 * by extent_cache
26693770ab7SChao Yu 					 */
2670ef4ca04SChao Yu 	DATA_GENERIC_ENHANCE_UPDATE,	/*
2680ef4ca04SChao Yu 					 * strong check on range and segment
2690ef4ca04SChao Yu 					 * bitmap for update case
2700ef4ca04SChao Yu 					 */
271e1da7872SChao Yu 	META_GENERIC,
272662befdaSChao Yu };
273662befdaSChao Yu 
2746451e041SJaegeuk Kim /* for the list of ino */
2756451e041SJaegeuk Kim enum {
2766451e041SJaegeuk Kim 	ORPHAN_INO,		/* for orphan ino list */
277fff04f90SJaegeuk Kim 	APPEND_INO,		/* for append ino list */
278fff04f90SJaegeuk Kim 	UPDATE_INO,		/* for update ino list */
279d382e369SYonggil Song 	TRANS_DIR_INO,		/* for transactions dir ino list */
28039d787beSChao Yu 	FLUSH_INO,		/* for multiple device flushing */
2816451e041SJaegeuk Kim 	MAX_INO_ENTRY,		/* max. list */
2826451e041SJaegeuk Kim };
2836451e041SJaegeuk Kim 
2846451e041SJaegeuk Kim struct ino_entry {
28539a53e0cSJaegeuk Kim 	struct list_head list;		/* list head */
28639a53e0cSJaegeuk Kim 	nid_t ino;			/* inode number */
28739d787beSChao Yu 	unsigned int dirty_device;	/* dirty device bitmap */
28839a53e0cSJaegeuk Kim };
28939a53e0cSJaegeuk Kim 
2902710fd7eSChao Yu /* for the list of inodes to be GCed */
29106292073SChao Yu struct inode_entry {
29239a53e0cSJaegeuk Kim 	struct list_head list;	/* list head */
29339a53e0cSJaegeuk Kim 	struct inode *inode;	/* vfs inode pointer */
29439a53e0cSJaegeuk Kim };
29539a53e0cSJaegeuk Kim 
29650fa53ecSChao Yu struct fsync_node_entry {
29750fa53ecSChao Yu 	struct list_head list;	/* list head */
29850fa53ecSChao Yu 	struct page *page;	/* warm node page pointer */
29950fa53ecSChao Yu 	unsigned int seq_id;	/* sequence id */
30050fa53ecSChao Yu };
30150fa53ecSChao Yu 
302261eeb9cSDaeho Jeong struct ckpt_req {
303261eeb9cSDaeho Jeong 	struct completion wait;		/* completion for checkpoint done */
304261eeb9cSDaeho Jeong 	struct llist_node llnode;	/* llist_node to be linked in wait queue */
305261eeb9cSDaeho Jeong 	int ret;			/* return code of checkpoint */
306261eeb9cSDaeho Jeong 	ktime_t queue_time;		/* request queued time */
307261eeb9cSDaeho Jeong };
308261eeb9cSDaeho Jeong 
309261eeb9cSDaeho Jeong struct ckpt_req_control {
310261eeb9cSDaeho Jeong 	struct task_struct *f2fs_issue_ckpt;	/* checkpoint task */
311e6592066SDaeho Jeong 	int ckpt_thread_ioprio;			/* checkpoint merge thread ioprio */
312261eeb9cSDaeho Jeong 	wait_queue_head_t ckpt_wait_queue;	/* waiting queue for wake-up */
313261eeb9cSDaeho Jeong 	atomic_t issued_ckpt;		/* # of actually issued ckpts */
314261eeb9cSDaeho Jeong 	atomic_t total_ckpt;		/* # of total ckpts */
315261eeb9cSDaeho Jeong 	atomic_t queued_ckpt;		/* # of queued ckpts */
316261eeb9cSDaeho Jeong 	struct llist_head issue_list;	/* list for command issue */
317261eeb9cSDaeho Jeong 	spinlock_t stat_lock;		/* lock for below checkpoint time stats */
318261eeb9cSDaeho Jeong 	unsigned int cur_time;		/* cur wait time in msec for currently issued checkpoint */
319261eeb9cSDaeho Jeong 	unsigned int peak_time;		/* peak wait time in msec until now */
320261eeb9cSDaeho Jeong };
321261eeb9cSDaeho Jeong 
322a7eeb823SChao Yu /* for the bitmap indicate blocks to be discarded */
3237fd9e544SJaegeuk Kim struct discard_entry {
3247fd9e544SJaegeuk Kim 	struct list_head list;	/* list head */
325a7eeb823SChao Yu 	block_t start_blkaddr;	/* start blockaddr of current segment */
326a7eeb823SChao Yu 	unsigned char discard_map[SIT_VBLOCK_MAP_SIZE];	/* segment discard bitmap */
3277fd9e544SJaegeuk Kim };
3287fd9e544SJaegeuk Kim 
3291cd2e6d5SYangtao Li /* minimum discard granularity, unit: block count */
3301cd2e6d5SYangtao Li #define MIN_DISCARD_GRANULARITY		1
331969d1b18SChao Yu /* default discard granularity of inner discard thread, unit: block count */
332969d1b18SChao Yu #define DEFAULT_DISCARD_GRANULARITY		16
333c46867e9SYangtao Li /* default maximum discard granularity of ordered discard, unit: block count */
334c46867e9SYangtao Li #define DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY	16
335969d1b18SChao Yu 
336ba48a33eSChao Yu /* max discard pend list number */
337ba48a33eSChao Yu #define MAX_PLIST_NUM		512
338ba48a33eSChao Yu #define plist_idx(blk_num)	((blk_num) >= MAX_PLIST_NUM ?		\
3392f84babfSSheng Yong 					(MAX_PLIST_NUM - 1) : ((blk_num) - 1))
340ba48a33eSChao Yu 
34115469963SJaegeuk Kim enum {
34235ec7d57SChao Yu 	D_PREP,			/* initial */
34335ec7d57SChao Yu 	D_PARTIAL,		/* partially submitted */
34435ec7d57SChao Yu 	D_SUBMIT,		/* all submitted */
34535ec7d57SChao Yu 	D_DONE,			/* finished */
34615469963SJaegeuk Kim };
34715469963SJaegeuk Kim 
348004b6862SChao Yu struct discard_info {
349b01a9201SJaegeuk Kim 	block_t lstart;			/* logical start address */
350b01a9201SJaegeuk Kim 	block_t len;			/* length */
351004b6862SChao Yu 	block_t start;			/* actual start address in dev */
352004b6862SChao Yu };
353004b6862SChao Yu 
354b01a9201SJaegeuk Kim struct discard_cmd {
355004b6862SChao Yu 	struct rb_node rb_node;		/* rb node located in rb-tree */
356004b6862SChao Yu 	struct discard_info di;		/* discard info */
357b01a9201SJaegeuk Kim 	struct list_head list;		/* command list */
358b01a9201SJaegeuk Kim 	struct completion wait;		/* compleation */
359c81abe34SJaegeuk Kim 	struct block_device *bdev;	/* bdev */
360ec9895adSChao Yu 	unsigned short ref;		/* reference count */
3619a744b92SChao Yu 	unsigned char state;		/* state */
36272691af6SJaegeuk Kim 	unsigned char queued;		/* queued discard */
363c81abe34SJaegeuk Kim 	int error;			/* bio error */
36435ec7d57SChao Yu 	spinlock_t lock;		/* for state/bio_ref updating */
36535ec7d57SChao Yu 	unsigned short bio_ref;		/* bio reference count */
366275b66b0SChao Yu };
367275b66b0SChao Yu 
36878997b56SChao Yu enum {
36978997b56SChao Yu 	DPOLICY_BG,
37078997b56SChao Yu 	DPOLICY_FORCE,
37178997b56SChao Yu 	DPOLICY_FSTRIM,
37278997b56SChao Yu 	DPOLICY_UMOUNT,
37378997b56SChao Yu 	MAX_DPOLICY,
37478997b56SChao Yu };
37578997b56SChao Yu 
376ecc9aa00SChao Yu struct discard_policy {
37778997b56SChao Yu 	int type;			/* type of discard */
378ecc9aa00SChao Yu 	unsigned int min_interval;	/* used for candidates exist */
379f9d1dcedSYunlei He 	unsigned int mid_interval;	/* used for device busy */
380ecc9aa00SChao Yu 	unsigned int max_interval;	/* used for candidates not exist */
381ecc9aa00SChao Yu 	unsigned int max_requests;	/* # of discards issued per round */
382ecc9aa00SChao Yu 	unsigned int io_aware_gran;	/* minimum granularity discard not be aware of I/O */
383ecc9aa00SChao Yu 	bool io_aware;			/* issue discard in idle time */
384ecc9aa00SChao Yu 	bool sync;			/* submit discard with REQ_SYNC flag */
38520ee4382SChao Yu 	bool ordered;			/* issue discard by lba order */
3866ce48b0cSChao Yu 	bool timeout;			/* discard timeout for put_super */
38778997b56SChao Yu 	unsigned int granularity;	/* discard granularity */
388ecc9aa00SChao Yu };
389ecc9aa00SChao Yu 
3900b54fb84SJaegeuk Kim struct discard_cmd_control {
39115469963SJaegeuk Kim 	struct task_struct *f2fs_issue_discard;	/* discard thread */
39246f84c2cSChao Yu 	struct list_head entry_list;		/* 4KB discard entry list */
393ba48a33eSChao Yu 	struct list_head pend_list[MAX_PLIST_NUM];/* store pending entries */
39446f84c2cSChao Yu 	struct list_head wait_list;		/* store on-flushing entries */
3958412663dSChao Yu 	struct list_head fstrim_list;		/* in-flight discard from fstrim */
39615469963SJaegeuk Kim 	wait_queue_head_t discard_wait_queue;	/* waiting queue for wake-up */
39715469963SJaegeuk Kim 	struct mutex cmd_lock;
398d618ebafSChao Yu 	unsigned int nr_discards;		/* # of discards in the list */
399d618ebafSChao Yu 	unsigned int max_discards;		/* max. discards to be issued */
400d2d8e896SKonstantin Vyshetsky 	unsigned int max_discard_request;	/* max. discard request per round */
401d2d8e896SKonstantin Vyshetsky 	unsigned int min_discard_issue_time;	/* min. interval between discard issue */
402d2d8e896SKonstantin Vyshetsky 	unsigned int mid_discard_issue_time;	/* mid. interval between discard issue */
403d2d8e896SKonstantin Vyshetsky 	unsigned int max_discard_issue_time;	/* max. interval between discard issue */
404120e0ea1SYangtao Li 	unsigned int discard_io_aware_gran; /* minimum discard granularity not be aware of I/O */
4058a47d228SYangtao Li 	unsigned int discard_urgent_util;	/* utilization which issue discard proactively */
406969d1b18SChao Yu 	unsigned int discard_granularity;	/* discard granularity */
407c46867e9SYangtao Li 	unsigned int max_ordered_discard;	/* maximum discard granularity issued by lba order */
408d84d1cbdSChao Yu 	unsigned int undiscard_blks;		/* # of undiscard blocks */
40920ee4382SChao Yu 	unsigned int next_pos;			/* next discard position */
4108b8dd65fSChao Yu 	atomic_t issued_discard;		/* # of issued discard */
41172691af6SJaegeuk Kim 	atomic_t queued_discard;		/* # of queued discard */
4125f32366aSChao Yu 	atomic_t discard_cmd_cnt;		/* # of cached cmd count */
4134dada3fdSChao Yu 	struct rb_root_cached root;		/* root of discard rb-tree */
41467fce70bSChao Yu 	bool rbtree_check;			/* config for consistence check */
41545c98f5aSYangtao Li 	bool discard_wake;			/* to wake up discard thread */
41639a53e0cSJaegeuk Kim };
41739a53e0cSJaegeuk Kim 
41839a53e0cSJaegeuk Kim /* for the list of fsync inodes, used only during recovery */
41939a53e0cSJaegeuk Kim struct fsync_inode_entry {
42039a53e0cSJaegeuk Kim 	struct list_head list;	/* list head */
42139a53e0cSJaegeuk Kim 	struct inode *inode;	/* vfs inode pointer */
422c52e1b10SJaegeuk Kim 	block_t blkaddr;	/* block address locating the last fsync */
423c52e1b10SJaegeuk Kim 	block_t last_dentry;	/* block address locating the last dentry */
42439a53e0cSJaegeuk Kim };
42539a53e0cSJaegeuk Kim 
42668afcf2dSTomohiro Kusumi #define nats_in_cursum(jnl)		(le16_to_cpu((jnl)->n_nats))
42768afcf2dSTomohiro Kusumi #define sits_in_cursum(jnl)		(le16_to_cpu((jnl)->n_sits))
42839a53e0cSJaegeuk Kim 
42968afcf2dSTomohiro Kusumi #define nat_in_journal(jnl, i)		((jnl)->nat_j.entries[i].ne)
43068afcf2dSTomohiro Kusumi #define nid_in_journal(jnl, i)		((jnl)->nat_j.entries[i].nid)
43168afcf2dSTomohiro Kusumi #define sit_in_journal(jnl, i)		((jnl)->sit_j.entries[i].se)
43268afcf2dSTomohiro Kusumi #define segno_in_journal(jnl, i)	((jnl)->sit_j.entries[i].segno)
43339a53e0cSJaegeuk Kim 
434dfc08a12SChao Yu #define MAX_NAT_JENTRIES(jnl)	(NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl))
435dfc08a12SChao Yu #define MAX_SIT_JENTRIES(jnl)	(SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl))
436309cc2b6SJaegeuk Kim 
437dfc08a12SChao Yu static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i)
43839a53e0cSJaegeuk Kim {
439dfc08a12SChao Yu 	int before = nats_in_cursum(journal);
440cac5a3d8SDongOh Shin 
441dfc08a12SChao Yu 	journal->n_nats = cpu_to_le16(before + i);
44239a53e0cSJaegeuk Kim 	return before;
44339a53e0cSJaegeuk Kim }
44439a53e0cSJaegeuk Kim 
445dfc08a12SChao Yu static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i)
44639a53e0cSJaegeuk Kim {
447dfc08a12SChao Yu 	int before = sits_in_cursum(journal);
448cac5a3d8SDongOh Shin 
449dfc08a12SChao Yu 	journal->n_sits = cpu_to_le16(before + i);
45039a53e0cSJaegeuk Kim 	return before;
45139a53e0cSJaegeuk Kim }
45239a53e0cSJaegeuk Kim 
453dfc08a12SChao Yu static inline bool __has_cursum_space(struct f2fs_journal *journal,
454dfc08a12SChao Yu 							int size, int type)
455184a5cd2SChao Yu {
456184a5cd2SChao Yu 	if (type == NAT_JOURNAL)
457dfc08a12SChao Yu 		return size <= MAX_NAT_JENTRIES(journal);
458dfc08a12SChao Yu 	return size <= MAX_SIT_JENTRIES(journal);
459184a5cd2SChao Yu }
460184a5cd2SChao Yu 
461f2470371SChao Yu /* for inline stuff */
462f2470371SChao Yu #define DEF_INLINE_RESERVED_SIZE	1
4637a2af766SChao Yu static inline int get_extra_isize(struct inode *inode);
4646afc662eSChao Yu static inline int get_inline_xattr_addrs(struct inode *inode);
4657a2af766SChao Yu #define MAX_INLINE_DATA(inode)	(sizeof(__le32) *			\
4667a2af766SChao Yu 				(CUR_ADDRS_PER_INODE(inode) -		\
467b323fd28SChao Yu 				get_inline_xattr_addrs(inode) -	\
4686afc662eSChao Yu 				DEF_INLINE_RESERVED_SIZE))
469f2470371SChao Yu 
470f2470371SChao Yu /* for inline dir */
471f2470371SChao Yu #define NR_INLINE_DENTRY(inode)	(MAX_INLINE_DATA(inode) * BITS_PER_BYTE / \
472f2470371SChao Yu 				((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
473f2470371SChao Yu 				BITS_PER_BYTE + 1))
474f91108b8SGeert Uytterhoeven #define INLINE_DENTRY_BITMAP_SIZE(inode) \
475f91108b8SGeert Uytterhoeven 	DIV_ROUND_UP(NR_INLINE_DENTRY(inode), BITS_PER_BYTE)
476f2470371SChao Yu #define INLINE_RESERVED_SIZE(inode)	(MAX_INLINE_DATA(inode) - \
477f2470371SChao Yu 				((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
478f2470371SChao Yu 				NR_INLINE_DENTRY(inode) + \
479f2470371SChao Yu 				INLINE_DENTRY_BITMAP_SIZE(inode)))
480f2470371SChao Yu 
481e9750824SNamjae Jeon /*
48239a53e0cSJaegeuk Kim  * For INODE and NODE manager
48339a53e0cSJaegeuk Kim  */
4847b3cd7d6SJaegeuk Kim /* for directory operations */
48543c780baSEric Biggers 
48643c780baSEric Biggers struct f2fs_filename {
48743c780baSEric Biggers 	/*
48843c780baSEric Biggers 	 * The filename the user specified.  This is NULL for some
48943c780baSEric Biggers 	 * filesystem-internal operations, e.g. converting an inline directory
49043c780baSEric Biggers 	 * to a non-inline one, or roll-forward recovering an encrypted dentry.
49143c780baSEric Biggers 	 */
49243c780baSEric Biggers 	const struct qstr *usr_fname;
49343c780baSEric Biggers 
49443c780baSEric Biggers 	/*
49543c780baSEric Biggers 	 * The on-disk filename.  For encrypted directories, this is encrypted.
49643c780baSEric Biggers 	 * This may be NULL for lookups in an encrypted dir without the key.
49743c780baSEric Biggers 	 */
49843c780baSEric Biggers 	struct fscrypt_str disk_name;
49943c780baSEric Biggers 
50043c780baSEric Biggers 	/* The dirhash of this filename */
50143c780baSEric Biggers 	f2fs_hash_t hash;
50243c780baSEric Biggers 
50343c780baSEric Biggers #ifdef CONFIG_FS_ENCRYPTION
50443c780baSEric Biggers 	/*
50543c780baSEric Biggers 	 * For lookups in encrypted directories: either the buffer backing
50643c780baSEric Biggers 	 * disk_name, or a buffer that holds the decoded no-key name.
50743c780baSEric Biggers 	 */
50843c780baSEric Biggers 	struct fscrypt_str crypto_buf;
50943c780baSEric Biggers #endif
5105298d4bfSChristoph Hellwig #if IS_ENABLED(CONFIG_UNICODE)
51143c780baSEric Biggers 	/*
51243c780baSEric Biggers 	 * For casefolded directories: the casefolded name, but it's left NULL
513b5639bb4SEric Biggers 	 * if the original name is not valid Unicode, if the original name is
514b5639bb4SEric Biggers 	 * "." or "..", if the directory is both casefolded and encrypted and
515b5639bb4SEric Biggers 	 * its encryption key is unavailable, or if the filesystem is doing an
516b5639bb4SEric Biggers 	 * internal operation where usr_fname is also NULL.  In all these cases
517b5639bb4SEric Biggers 	 * we fall back to treating the name as an opaque byte sequence.
51843c780baSEric Biggers 	 */
51943c780baSEric Biggers 	struct fscrypt_str cf_name;
52043c780baSEric Biggers #endif
52143c780baSEric Biggers };
52243c780baSEric Biggers 
5237b3cd7d6SJaegeuk Kim struct f2fs_dentry_ptr {
524d8c6822aSJaegeuk Kim 	struct inode *inode;
52576a9dd85SChao Yu 	void *bitmap;
5267b3cd7d6SJaegeuk Kim 	struct f2fs_dir_entry *dentry;
5277b3cd7d6SJaegeuk Kim 	__u8 (*filename)[F2FS_SLOT_LEN];
5287b3cd7d6SJaegeuk Kim 	int max;
52976a9dd85SChao Yu 	int nr_bitmap;
5307b3cd7d6SJaegeuk Kim };
5317b3cd7d6SJaegeuk Kim 
53264c24ecbSTomohiro Kusumi static inline void make_dentry_ptr_block(struct inode *inode,
53364c24ecbSTomohiro Kusumi 		struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t)
5347b3cd7d6SJaegeuk Kim {
535d8c6822aSJaegeuk Kim 	d->inode = inode;
5367b3cd7d6SJaegeuk Kim 	d->max = NR_DENTRY_IN_BLOCK;
53776a9dd85SChao Yu 	d->nr_bitmap = SIZE_OF_DENTRY_BITMAP;
538c79d1520SYunlong Song 	d->bitmap = t->dentry_bitmap;
5397b3cd7d6SJaegeuk Kim 	d->dentry = t->dentry;
5407b3cd7d6SJaegeuk Kim 	d->filename = t->filename;
54164c24ecbSTomohiro Kusumi }
542cac5a3d8SDongOh Shin 
54364c24ecbSTomohiro Kusumi static inline void make_dentry_ptr_inline(struct inode *inode,
544f2470371SChao Yu 					struct f2fs_dentry_ptr *d, void *t)
54564c24ecbSTomohiro Kusumi {
546f2470371SChao Yu 	int entry_cnt = NR_INLINE_DENTRY(inode);
547f2470371SChao Yu 	int bitmap_size = INLINE_DENTRY_BITMAP_SIZE(inode);
548f2470371SChao Yu 	int reserved_size = INLINE_RESERVED_SIZE(inode);
549f2470371SChao Yu 
55064c24ecbSTomohiro Kusumi 	d->inode = inode;
551f2470371SChao Yu 	d->max = entry_cnt;
552f2470371SChao Yu 	d->nr_bitmap = bitmap_size;
553f2470371SChao Yu 	d->bitmap = t;
554f2470371SChao Yu 	d->dentry = t + bitmap_size + reserved_size;
555f2470371SChao Yu 	d->filename = t + bitmap_size + reserved_size +
556f2470371SChao Yu 					SIZE_OF_DIR_ENTRY * entry_cnt;
5577b3cd7d6SJaegeuk Kim }
5587b3cd7d6SJaegeuk Kim 
559dbe6a5ffSJaegeuk Kim /*
560dbe6a5ffSJaegeuk Kim  * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1
561dbe6a5ffSJaegeuk Kim  * as its node offset to distinguish from index node blocks.
562dbe6a5ffSJaegeuk Kim  * But some bits are used to mark the node block.
56339a53e0cSJaegeuk Kim  */
564dbe6a5ffSJaegeuk Kim #define XATTR_NODE_OFFSET	((((unsigned int)-1) << OFFSET_BIT_SHIFT) \
565dbe6a5ffSJaegeuk Kim 				>> OFFSET_BIT_SHIFT)
566266e97a8SJaegeuk Kim enum {
567266e97a8SJaegeuk Kim 	ALLOC_NODE,			/* allocate a new node page if needed */
568266e97a8SJaegeuk Kim 	LOOKUP_NODE,			/* look up a node without readahead */
569266e97a8SJaegeuk Kim 	LOOKUP_NODE_RA,			/*
570266e97a8SJaegeuk Kim 					 * look up a node with readahead called
5714f4124d0SChao Yu 					 * by get_data_block.
57239a53e0cSJaegeuk Kim 					 */
573266e97a8SJaegeuk Kim };
574266e97a8SJaegeuk Kim 
57591803392SChao Yu #define DEFAULT_RETRY_IO_COUNT	8	/* maximum retry read IO or flush count */
5767735730dSChao Yu 
5775df7731fSChao Yu /* congestion wait timeout value, default: 20ms */
5785df7731fSChao Yu #define	DEFAULT_IO_TIMEOUT	(msecs_to_jiffies(20))
5795df7731fSChao Yu 
580af033b2aSChao Yu /* maximum retry quota flush count */
581af033b2aSChao Yu #define DEFAULT_RETRY_QUOTA_FLUSH_COUNT		8
582af033b2aSChao Yu 
583a7b8618aSJaegeuk Kim /* maximum retry of EIO'ed page */
584a7b8618aSJaegeuk Kim #define MAX_RETRY_PAGE_EIO			100
58550c63009SJaegeuk Kim 
586a6db67f0SJaegeuk Kim #define F2FS_LINK_MAX	0xffffffff	/* maximum link count per file */
58739a53e0cSJaegeuk Kim 
588817202d9SChao Yu #define MAX_DIR_RA_PAGES	4	/* maximum ra pages of dir */
589817202d9SChao Yu 
590287b1406SChao Yu /* dirty segments threshold for triggering CP */
591287b1406SChao Yu #define DEFAULT_DIRTY_THRESHOLD		4
592287b1406SChao Yu 
593e7547dacSJaegeuk Kim #define RECOVERY_MAX_RA_BLOCKS		BIO_MAX_VECS
594e7547dacSJaegeuk Kim #define RECOVERY_MIN_RA_BLOCKS		1
595e7547dacSJaegeuk Kim 
596e7547dacSJaegeuk Kim #define F2FS_ONSTACK_PAGES	16	/* nr of onstack pages */
597e7547dacSJaegeuk Kim 
59839a53e0cSJaegeuk Kim /* for in-memory extent cache entry */
59913054c54SChao Yu #define F2FS_MIN_EXTENT_LEN	64	/* minimum extent length */
60013054c54SChao Yu 
60113054c54SChao Yu /* number of extent info in extent cache we try to shrink */
60212607c1bSJaegeuk Kim #define READ_EXTENT_CACHE_SHRINK_NUMBER	128
603c11abd1aSJaegeuk Kim 
60471644dffSJaegeuk Kim /* number of age extent info in extent cache we try to shrink */
60571644dffSJaegeuk Kim #define AGE_EXTENT_CACHE_SHRINK_NUMBER	128
60671644dffSJaegeuk Kim #define LAST_AGE_WEIGHT			30
60771644dffSJaegeuk Kim #define SAME_AGE_REGION			1024
60871644dffSJaegeuk Kim 
60971644dffSJaegeuk Kim /*
61071644dffSJaegeuk Kim  * Define data block with age less than 1GB as hot data
61171644dffSJaegeuk Kim  * define data block with age less than 10GB but more than 1GB as warm data
61271644dffSJaegeuk Kim  */
61371644dffSJaegeuk Kim #define DEF_HOT_DATA_AGE_THRESHOLD	262144
61471644dffSJaegeuk Kim #define DEF_WARM_DATA_AGE_THRESHOLD	2621440
61571644dffSJaegeuk Kim 
616e7547dacSJaegeuk Kim /* extent cache type */
617e7547dacSJaegeuk Kim enum extent_type {
618e7547dacSJaegeuk Kim 	EX_READ,
61971644dffSJaegeuk Kim 	EX_BLOCK_AGE,
620e7547dacSJaegeuk Kim 	NR_EXTENT_CACHES,
621e7547dacSJaegeuk Kim };
62201fc4b9aSFengnan Chang 
62339a53e0cSJaegeuk Kim struct extent_info {
62439a53e0cSJaegeuk Kim 	unsigned int fofs;		/* start offset in a file */
625111d2495SMasanari Iida 	unsigned int len;		/* length of the extent */
626e7547dacSJaegeuk Kim 	union {
627e7547dacSJaegeuk Kim 		/* read extent_cache */
628e7547dacSJaegeuk Kim 		struct {
629e7547dacSJaegeuk Kim 			/* start block address of the extent */
630e7547dacSJaegeuk Kim 			block_t blk;
63194afd6d6SChao Yu #ifdef CONFIG_F2FS_FS_COMPRESSION
632e7547dacSJaegeuk Kim 			/* physical extent length of compressed blocks */
633e7547dacSJaegeuk Kim 			unsigned int c_len;
63494afd6d6SChao Yu #endif
63539a53e0cSJaegeuk Kim 		};
63671644dffSJaegeuk Kim 		/* block age extent_cache */
63771644dffSJaegeuk Kim 		struct {
63871644dffSJaegeuk Kim 			/* block age of the extent */
63971644dffSJaegeuk Kim 			unsigned long long age;
64071644dffSJaegeuk Kim 			/* last total blocks allocated */
64171644dffSJaegeuk Kim 			unsigned long long last_blocks;
64271644dffSJaegeuk Kim 		};
643e7547dacSJaegeuk Kim 	};
644e7547dacSJaegeuk Kim };
64539a53e0cSJaegeuk Kim 
64613054c54SChao Yu struct extent_node {
647c0362117SChao Yu 	struct rb_node rb_node;		/* rb node located in rb-tree */
64813054c54SChao Yu 	struct extent_info ei;		/* extent info */
64954c2258cSChao Yu 	struct list_head list;		/* node in global extent list of sbi */
650201ef5e0SHou Pengyang 	struct extent_tree *et;		/* extent tree pointer */
65113054c54SChao Yu };
65213054c54SChao Yu 
65313054c54SChao Yu struct extent_tree {
65413054c54SChao Yu 	nid_t ino;			/* inode number */
655e7547dacSJaegeuk Kim 	enum extent_type type;		/* keep the extent tree type */
6564dada3fdSChao Yu 	struct rb_root_cached root;	/* root of extent info rb-tree */
65762c8af65SChao Yu 	struct extent_node *cached_en;	/* recently accessed extent node */
658137d09f0SJaegeuk Kim 	struct list_head list;		/* to be used by sbi->zombie_list */
65913054c54SChao Yu 	rwlock_t lock;			/* protect extent info rb-tree */
66068e35385SChao Yu 	atomic_t node_cnt;		/* # of extent node in rb-tree*/
661b430f726SZhikang Zhang 	bool largest_updated;		/* largest extent updated */
662e7547dacSJaegeuk Kim 	struct extent_info largest;	/* largest cached extent for EX_READ */
663e7547dacSJaegeuk Kim };
664e7547dacSJaegeuk Kim 
665e7547dacSJaegeuk Kim struct extent_tree_info {
666e7547dacSJaegeuk Kim 	struct radix_tree_root extent_tree_root;/* cache extent cache entries */
667e7547dacSJaegeuk Kim 	struct mutex extent_tree_lock;	/* locking extent radix tree */
668e7547dacSJaegeuk Kim 	struct list_head extent_list;		/* lru list for shrinker */
669e7547dacSJaegeuk Kim 	spinlock_t extent_lock;			/* locking extent lru list */
670e7547dacSJaegeuk Kim 	atomic_t total_ext_tree;		/* extent tree count */
671e7547dacSJaegeuk Kim 	struct list_head zombie_list;		/* extent zombie tree list */
672e7547dacSJaegeuk Kim 	atomic_t total_zombie_tree;		/* extent zombie tree count */
673e7547dacSJaegeuk Kim 	atomic_t total_ext_node;		/* extent info count */
67413054c54SChao Yu };
67513054c54SChao Yu 
67639a53e0cSJaegeuk Kim /*
67762a134bdSChristoph Hellwig  * State of block returned by f2fs_map_blocks.
678003a3e1dSJaegeuk Kim  */
67962a134bdSChristoph Hellwig #define F2FS_MAP_NEW		(1U << 0)
68062a134bdSChristoph Hellwig #define F2FS_MAP_MAPPED		(1U << 1)
681da8c7fecSChristoph Hellwig #define F2FS_MAP_DELALLOC	(1U << 2)
6827f63eb77SJaegeuk Kim #define F2FS_MAP_FLAGS		(F2FS_MAP_NEW | F2FS_MAP_MAPPED |\
683da8c7fecSChristoph Hellwig 				F2FS_MAP_DELALLOC)
684003a3e1dSJaegeuk Kim 
685003a3e1dSJaegeuk Kim struct f2fs_map_blocks {
68671f2c820SChao Yu 	struct block_device *m_bdev;	/* for multi-device dio */
687003a3e1dSJaegeuk Kim 	block_t m_pblk;
688003a3e1dSJaegeuk Kim 	block_t m_lblk;
689003a3e1dSJaegeuk Kim 	unsigned int m_len;
690003a3e1dSJaegeuk Kim 	unsigned int m_flags;
691da85985cSChao Yu 	pgoff_t *m_next_pgofs;		/* point next possible non-hole pgofs */
692c4020b2dSChao Yu 	pgoff_t *m_next_extent;		/* point to next possible extent */
693d5097be5SHyunchul Lee 	int m_seg_type;
694f9d6d059SChao Yu 	bool m_may_create;		/* indicate it is from write path */
69571f2c820SChao Yu 	bool m_multidev_dio;		/* indicate it allows multi-device dio */
696003a3e1dSJaegeuk Kim };
697003a3e1dSJaegeuk Kim 
698e2b4e2bcSChao Yu /* for flag in get_data_block */
699f2220c7fSQiuyang Sun enum {
700f2220c7fSQiuyang Sun 	F2FS_GET_BLOCK_DEFAULT,
701f2220c7fSQiuyang Sun 	F2FS_GET_BLOCK_FIEMAP,
702f2220c7fSQiuyang Sun 	F2FS_GET_BLOCK_BMAP,
7030a4daae5SJaegeuk Kim 	F2FS_GET_BLOCK_DIO,
704f2220c7fSQiuyang Sun 	F2FS_GET_BLOCK_PRE_DIO,
705f2220c7fSQiuyang Sun 	F2FS_GET_BLOCK_PRE_AIO,
706c4020b2dSChao Yu 	F2FS_GET_BLOCK_PRECACHE,
707f2220c7fSQiuyang Sun };
708e2b4e2bcSChao Yu 
709003a3e1dSJaegeuk Kim /*
71039a53e0cSJaegeuk Kim  * i_advise uses FADVISE_XXX_BIT. We can add additional hints later.
71139a53e0cSJaegeuk Kim  */
71239a53e0cSJaegeuk Kim #define FADVISE_COLD_BIT	0x01
713354a3399SJaegeuk Kim #define FADVISE_LOST_PINO_BIT	0x02
714cde4de12SJaegeuk Kim #define FADVISE_ENCRYPT_BIT	0x04
715e7d55452SJaegeuk Kim #define FADVISE_ENC_NAME_BIT	0x08
71626787236SJaegeuk Kim #define FADVISE_KEEP_SIZE_BIT	0x10
717b6a06cbbSChao Yu #define FADVISE_HOT_BIT		0x20
71895ae251fSEric Biggers #define FADVISE_VERITY_BIT	0x40
719d4dd19ecSJaegeuk Kim #define FADVISE_TRUNC_BIT	0x80
72039a53e0cSJaegeuk Kim 
721797c1cb5SChao Yu #define FADVISE_MODIFIABLE_BITS	(FADVISE_COLD_BIT | FADVISE_HOT_BIT)
722797c1cb5SChao Yu 
723b5492af7SJaegeuk Kim #define file_is_cold(inode)	is_file(inode, FADVISE_COLD_BIT)
724b5492af7SJaegeuk Kim #define file_set_cold(inode)	set_file(inode, FADVISE_COLD_BIT)
725b5492af7SJaegeuk Kim #define file_clear_cold(inode)	clear_file(inode, FADVISE_COLD_BIT)
7261153db09SChao Yu 
7271153db09SChao Yu #define file_wrong_pino(inode)	is_file(inode, FADVISE_LOST_PINO_BIT)
7281153db09SChao Yu #define file_lost_pino(inode)	set_file(inode, FADVISE_LOST_PINO_BIT)
729b5492af7SJaegeuk Kim #define file_got_pino(inode)	clear_file(inode, FADVISE_LOST_PINO_BIT)
7301153db09SChao Yu 
731cde4de12SJaegeuk Kim #define file_is_encrypt(inode)	is_file(inode, FADVISE_ENCRYPT_BIT)
732cde4de12SJaegeuk Kim #define file_set_encrypt(inode)	set_file(inode, FADVISE_ENCRYPT_BIT)
7331153db09SChao Yu 
734e7d55452SJaegeuk Kim #define file_enc_name(inode)	is_file(inode, FADVISE_ENC_NAME_BIT)
735e7d55452SJaegeuk Kim #define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
7361153db09SChao Yu 
73726787236SJaegeuk Kim #define file_keep_isize(inode)	is_file(inode, FADVISE_KEEP_SIZE_BIT)
73826787236SJaegeuk Kim #define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
7391153db09SChao Yu 
740b6a06cbbSChao Yu #define file_is_hot(inode)	is_file(inode, FADVISE_HOT_BIT)
741b6a06cbbSChao Yu #define file_set_hot(inode)	set_file(inode, FADVISE_HOT_BIT)
742b6a06cbbSChao Yu #define file_clear_hot(inode)	clear_file(inode, FADVISE_HOT_BIT)
7431153db09SChao Yu 
74495ae251fSEric Biggers #define file_is_verity(inode)	is_file(inode, FADVISE_VERITY_BIT)
74595ae251fSEric Biggers #define file_set_verity(inode)	set_file(inode, FADVISE_VERITY_BIT)
746cde4de12SJaegeuk Kim 
747d4dd19ecSJaegeuk Kim #define file_should_truncate(inode)	is_file(inode, FADVISE_TRUNC_BIT)
748d4dd19ecSJaegeuk Kim #define file_need_truncate(inode)	set_file(inode, FADVISE_TRUNC_BIT)
749d4dd19ecSJaegeuk Kim #define file_dont_truncate(inode)	clear_file(inode, FADVISE_TRUNC_BIT)
750d4dd19ecSJaegeuk Kim 
751ab9fa662SJaegeuk Kim #define DEF_DIR_LEVEL		0
752ab9fa662SJaegeuk Kim 
7532ef79ecbSChao Yu enum {
7542ef79ecbSChao Yu 	GC_FAILURE_PIN,
7552ef79ecbSChao Yu 	MAX_GC_FAILURE
7562ef79ecbSChao Yu };
7572ef79ecbSChao Yu 
7587653b9d8SChao Yu /* used for f2fs_inode_info->flags */
7597653b9d8SChao Yu enum {
7607653b9d8SChao Yu 	FI_NEW_INODE,		/* indicate newly allocated inode */
7617653b9d8SChao Yu 	FI_DIRTY_INODE,		/* indicate inode is dirty or not */
7627653b9d8SChao Yu 	FI_AUTO_RECOVER,	/* indicate inode is recoverable */
7637653b9d8SChao Yu 	FI_DIRTY_DIR,		/* indicate directory has dirty pages */
7647653b9d8SChao Yu 	FI_INC_LINK,		/* need to increment i_nlink */
7657653b9d8SChao Yu 	FI_ACL_MODE,		/* indicate acl mode */
7667653b9d8SChao Yu 	FI_NO_ALLOC,		/* should not allocate any blocks */
7677653b9d8SChao Yu 	FI_FREE_NID,		/* free allocated nide */
7687653b9d8SChao Yu 	FI_NO_EXTENT,		/* not to use the extent cache */
7697653b9d8SChao Yu 	FI_INLINE_XATTR,	/* used for inline xattr */
7707653b9d8SChao Yu 	FI_INLINE_DATA,		/* used for inline data*/
7717653b9d8SChao Yu 	FI_INLINE_DENTRY,	/* used for inline dentry */
7727653b9d8SChao Yu 	FI_APPEND_WRITE,	/* inode has appended data */
7737653b9d8SChao Yu 	FI_UPDATE_WRITE,	/* inode has in-place-update data */
7747653b9d8SChao Yu 	FI_NEED_IPU,		/* used for ipu per file */
7757653b9d8SChao Yu 	FI_ATOMIC_FILE,		/* indicate atomic file */
7767653b9d8SChao Yu 	FI_FIRST_BLOCK_WRITTEN,	/* indicate #0 data block was written */
7777653b9d8SChao Yu 	FI_DROP_CACHE,		/* drop dirty page cache */
7787653b9d8SChao Yu 	FI_DATA_EXIST,		/* indicate data exists */
7797653b9d8SChao Yu 	FI_INLINE_DOTS,		/* indicate inline dot dentries */
7801018a546SChao Yu 	FI_SKIP_WRITES,		/* should skip data page writeback */
7811018a546SChao Yu 	FI_OPU_WRITE,		/* used for opu per file */
7827653b9d8SChao Yu 	FI_DIRTY_FILE,		/* indicate regular/symlink has dirty pages */
7833d697a4aSEric Biggers 	FI_PREALLOCATED_ALL,	/* all blocks for write were preallocated */
7847653b9d8SChao Yu 	FI_HOT_DATA,		/* indicate file is hot */
7857653b9d8SChao Yu 	FI_EXTRA_ATTR,		/* indicate file has extra attribute */
7867653b9d8SChao Yu 	FI_PROJ_INHERIT,	/* indicate file inherits projectid */
7877653b9d8SChao Yu 	FI_PIN_FILE,		/* indicate file should not be gced */
7887653b9d8SChao Yu 	FI_VERITY_IN_PROGRESS,	/* building fs-verity Merkle tree */
7897653b9d8SChao Yu 	FI_COMPRESSED_FILE,	/* indicate file's data can be compressed */
790b28f047bSChao Yu 	FI_COMPRESS_CORRUPT,	/* indicate compressed cluster is corrupted */
7917653b9d8SChao Yu 	FI_MMAP_FILE,		/* indicate file was mmapped */
792602a16d5SDaeho Jeong 	FI_ENABLE_COMPRESS,	/* enable compression in "user" compression mode */
793c6140415SJaegeuk Kim 	FI_COMPRESS_RELEASED,	/* compressed blocks were released */
794859fca6bSChao Yu 	FI_ALIGNED_WRITE,	/* enable aligned write */
7954a2c5b79SYe Bin 	FI_COW_FILE,		/* indicate COW file */
7964d8d45dfSDaeho Jeong 	FI_ATOMIC_COMMITTED,	/* indicate atomic commit completed except disk sync */
79741e8f85aSDaeho Jeong 	FI_ATOMIC_REPLACE,	/* indicate atomic replace */
7987653b9d8SChao Yu 	FI_MAX,			/* max flag, never be used */
7997653b9d8SChao Yu };
8007653b9d8SChao Yu 
80139a53e0cSJaegeuk Kim struct f2fs_inode_info {
80239a53e0cSJaegeuk Kim 	struct inode vfs_inode;		/* serve a vfs inode */
80339a53e0cSJaegeuk Kim 	unsigned long i_flags;		/* keep an inode flags for ioctl */
80439a53e0cSJaegeuk Kim 	unsigned char i_advise;		/* use to give file attribute hints */
80538431545SJaegeuk Kim 	unsigned char i_dir_level;	/* use for dentry level for large dir */
8061ad71a27SJaegeuk Kim 	unsigned int i_current_depth;	/* only for directory depth */
8072ef79ecbSChao Yu 	/* for gc failure statistic */
8082ef79ecbSChao Yu 	unsigned int i_gc_failures[MAX_GC_FAILURE];
8096666e6aaSJaegeuk Kim 	unsigned int i_pino;		/* parent inode number */
81039a53e0cSJaegeuk Kim 	umode_t i_acl_mode;		/* keep file acl mode temporarily */
81139a53e0cSJaegeuk Kim 
81239a53e0cSJaegeuk Kim 	/* Use below internally in f2fs*/
8137653b9d8SChao Yu 	unsigned long flags[BITS_TO_LONGS(FI_MAX)];	/* use to pass per-file flags */
814e4544b63STim Murray 	struct f2fs_rwsem i_sem;	/* protect fi info */
815204706c7SJaegeuk Kim 	atomic_t dirty_pages;		/* # of dirty pages */
81639a53e0cSJaegeuk Kim 	f2fs_hash_t chash;		/* hash value of given file name */
81739a53e0cSJaegeuk Kim 	unsigned int clevel;		/* maximum level of given file name */
81888c5c13aSJaegeuk Kim 	struct task_struct *task;	/* lookup and create consistency */
819b0af6d49SChao Yu 	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
820d80afefbSChao Yu 	struct task_struct *wb_task;	/* indicate inode is in context of writeback */
82139a53e0cSJaegeuk Kim 	nid_t i_xattr_nid;		/* node id that contains xattrs */
82226de9b11SJaegeuk Kim 	loff_t	last_disk_size;		/* lastly written file size */
823c10c9820SChao Yu 	spinlock_t i_size_lock;		/* protect last_disk_size */
82488b88a66SJaegeuk Kim 
8250abd675eSChao Yu #ifdef CONFIG_QUOTA
8260abd675eSChao Yu 	struct dquot *i_dquot[MAXQUOTAS];
8270abd675eSChao Yu 
8280abd675eSChao Yu 	/* quota space reservation, managed internally by quota code */
8290abd675eSChao Yu 	qsize_t i_reserved_quota;
8300abd675eSChao Yu #endif
8310f18b462SJaegeuk Kim 	struct list_head dirty_list;	/* dirty list for dirs and files */
8320f18b462SJaegeuk Kim 	struct list_head gdirty_list;	/* linked in global dirty list */
8333db1de0eSDaeho Jeong 	struct task_struct *atomic_write_task;	/* store atomic write task */
834e7547dacSJaegeuk Kim 	struct extent_tree *extent_tree[NR_EXTENT_CACHES];
835e7547dacSJaegeuk Kim 					/* cached extent_tree entry */
8363db1de0eSDaeho Jeong 	struct inode *cow_inode;	/* copy-on-write inode for atomic write */
837b2532c69SChao Yu 
838b2532c69SChao Yu 	/* avoid racing between foreground op and gc */
839e4544b63STim Murray 	struct f2fs_rwsem i_gc_rwsem[2];
840e4544b63STim Murray 	struct f2fs_rwsem i_xattr_sem; /* avoid racing between reading and changing EAs */
841f2470371SChao Yu 
8427a2af766SChao Yu 	int i_extra_isize;		/* size of extra space located in i_addr */
8435c57132eSChao Yu 	kprojid_t i_projid;		/* id for project quota */
8446afc662eSChao Yu 	int i_inline_xattr_size;	/* inline xattr size */
84524b81dfcSArnd Bergmann 	struct timespec64 i_crtime;	/* inode creation time */
846*c277991dSChao Yu 	struct timespec64 i_disk_time[3];/* inode disk times */
8474c8ff709SChao Yu 
8484c8ff709SChao Yu 	/* for file compress */
849c2759ebaSDaeho Jeong 	atomic_t i_compr_blocks;		/* # of compressed blocks */
8504c8ff709SChao Yu 	unsigned char i_compress_algorithm;	/* algorithm type */
8514c8ff709SChao Yu 	unsigned char i_log_cluster_size;	/* log of cluster size */
8523fde13f8SChao Yu 	unsigned char i_compress_level;		/* compress level (lz4hc,zstd) */
853b90e5086SChao Yu 	unsigned char i_compress_flag;		/* compress flag */
8544c8ff709SChao Yu 	unsigned int i_cluster_size;		/* cluster size */
855f8e2f32bSDaeho Jeong 
856f8e2f32bSDaeho Jeong 	unsigned int atomic_write_cnt;
8574d8d45dfSDaeho Jeong 	loff_t original_i_size;		/* original i_size before atomic write */
85839a53e0cSJaegeuk Kim };
85939a53e0cSJaegeuk Kim 
86012607c1bSJaegeuk Kim static inline void get_read_extent_info(struct extent_info *ext,
861bd933d4fSChao Yu 					struct f2fs_extent *i_ext)
86239a53e0cSJaegeuk Kim {
863bd933d4fSChao Yu 	ext->fofs = le32_to_cpu(i_ext->fofs);
864bd933d4fSChao Yu 	ext->blk = le32_to_cpu(i_ext->blk);
865bd933d4fSChao Yu 	ext->len = le32_to_cpu(i_ext->len);
86639a53e0cSJaegeuk Kim }
86739a53e0cSJaegeuk Kim 
86812607c1bSJaegeuk Kim static inline void set_raw_read_extent(struct extent_info *ext,
86939a53e0cSJaegeuk Kim 					struct f2fs_extent *i_ext)
87039a53e0cSJaegeuk Kim {
87139a53e0cSJaegeuk Kim 	i_ext->fofs = cpu_to_le32(ext->fofs);
8724d0b0bd4SChao Yu 	i_ext->blk = cpu_to_le32(ext->blk);
87339a53e0cSJaegeuk Kim 	i_ext->len = cpu_to_le32(ext->len);
87439a53e0cSJaegeuk Kim }
87539a53e0cSJaegeuk Kim 
876004b6862SChao Yu static inline bool __is_discard_mergeable(struct discard_info *back,
87735ec7d57SChao Yu 			struct discard_info *front, unsigned int max_len)
878004b6862SChao Yu {
8799a997188SJaegeuk Kim 	return (back->lstart + back->len == front->lstart) &&
88035ec7d57SChao Yu 		(back->len + front->len <= max_len);
881004b6862SChao Yu }
882004b6862SChao Yu 
883004b6862SChao Yu static inline bool __is_discard_back_mergeable(struct discard_info *cur,
88435ec7d57SChao Yu 			struct discard_info *back, unsigned int max_len)
885004b6862SChao Yu {
88635ec7d57SChao Yu 	return __is_discard_mergeable(back, cur, max_len);
887004b6862SChao Yu }
888004b6862SChao Yu 
889004b6862SChao Yu static inline bool __is_discard_front_mergeable(struct discard_info *cur,
89035ec7d57SChao Yu 			struct discard_info *front, unsigned int max_len)
891004b6862SChao Yu {
89235ec7d57SChao Yu 	return __is_discard_mergeable(cur, front, max_len);
893004b6862SChao Yu }
894004b6862SChao Yu 
8959a4ffdf5SChao Yu /*
8969a4ffdf5SChao Yu  * For free nid management
8979a4ffdf5SChao Yu  */
8989a4ffdf5SChao Yu enum nid_state {
8999a4ffdf5SChao Yu 	FREE_NID,		/* newly added to free nid list */
9009a4ffdf5SChao Yu 	PREALLOC_NID,		/* it is preallocated */
9019a4ffdf5SChao Yu 	MAX_NID_STATE,
902b8559dc2SChao Yu };
903b8559dc2SChao Yu 
904a95ba66aSJaegeuk Kim enum nat_state {
905a95ba66aSJaegeuk Kim 	TOTAL_NAT,
906a95ba66aSJaegeuk Kim 	DIRTY_NAT,
907a95ba66aSJaegeuk Kim 	RECLAIMABLE_NAT,
908a95ba66aSJaegeuk Kim 	MAX_NAT_STATE,
909a95ba66aSJaegeuk Kim };
910a95ba66aSJaegeuk Kim 
91139a53e0cSJaegeuk Kim struct f2fs_nm_info {
91239a53e0cSJaegeuk Kim 	block_t nat_blkaddr;		/* base disk address of NAT */
91339a53e0cSJaegeuk Kim 	nid_t max_nid;			/* maximum possible node ids */
91404d47e67SChao Yu 	nid_t available_nids;		/* # of available node ids */
91539a53e0cSJaegeuk Kim 	nid_t next_scan_nid;		/* the next nid to be scanned */
91647c8ebccSJaegeuk Kim 	nid_t max_rf_node_blocks;	/* max # of nodes for recovery */
917cdfc41c1SJaegeuk Kim 	unsigned int ram_thresh;	/* control the memory footprint */
918ea1a29a0SChao Yu 	unsigned int ra_nid_pages;	/* # of nid pages to be readaheaded */
9192304cb0cSChao Yu 	unsigned int dirty_nats_ratio;	/* control dirty nats ratio threshold */
92039a53e0cSJaegeuk Kim 
92139a53e0cSJaegeuk Kim 	/* NAT cache management */
92239a53e0cSJaegeuk Kim 	struct radix_tree_root nat_root;/* root of the nat entry cache */
923309cc2b6SJaegeuk Kim 	struct radix_tree_root nat_set_root;/* root of the nat set cache */
924e4544b63STim Murray 	struct f2fs_rwsem nat_tree_lock;	/* protect nat entry tree */
92539a53e0cSJaegeuk Kim 	struct list_head nat_entries;	/* cached nat entry list (clean) */
92622969158SChao Yu 	spinlock_t nat_list_lock;	/* protect clean nat entry list */
927a95ba66aSJaegeuk Kim 	unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */
92822ad0b6aSJaegeuk Kim 	unsigned int nat_blocks;	/* # of nat blocks */
92939a53e0cSJaegeuk Kim 
93039a53e0cSJaegeuk Kim 	/* free node ids management */
9318a7ed66aSJaegeuk Kim 	struct radix_tree_root free_nid_root;/* root of the free_nid cache */
9329a4ffdf5SChao Yu 	struct list_head free_nid_list;		/* list for free nids excluding preallocated nids */
9339a4ffdf5SChao Yu 	unsigned int nid_cnt[MAX_NID_STATE];	/* the number of free node id */
934b8559dc2SChao Yu 	spinlock_t nid_list_lock;	/* protect nid lists ops */
93539a53e0cSJaegeuk Kim 	struct mutex build_lock;	/* lock for build free nids */
936bb1105e4SJaegeuk Kim 	unsigned char **free_nid_bitmap;
9374ac91242SChao Yu 	unsigned char *nat_block_bitmap;
938586d1492SChao Yu 	unsigned short *free_nid_count;	/* free nid count of NAT block */
93939a53e0cSJaegeuk Kim 
94039a53e0cSJaegeuk Kim 	/* for checkpoint */
94139a53e0cSJaegeuk Kim 	char *nat_bitmap;		/* NAT bitmap pointer */
94222ad0b6aSJaegeuk Kim 
94322ad0b6aSJaegeuk Kim 	unsigned int nat_bits_blocks;	/* # of nat bits blocks */
94422ad0b6aSJaegeuk Kim 	unsigned char *nat_bits;	/* NAT bits blocks */
94522ad0b6aSJaegeuk Kim 	unsigned char *full_nat_bits;	/* full NAT pages */
94622ad0b6aSJaegeuk Kim 	unsigned char *empty_nat_bits;	/* empty NAT pages */
947599a09b2SChao Yu #ifdef CONFIG_F2FS_CHECK_FS
948599a09b2SChao Yu 	char *nat_bitmap_mir;		/* NAT bitmap mirror */
949599a09b2SChao Yu #endif
95039a53e0cSJaegeuk Kim 	int bitmap_size;		/* bitmap size */
95139a53e0cSJaegeuk Kim };
95239a53e0cSJaegeuk Kim 
95339a53e0cSJaegeuk Kim /*
95439a53e0cSJaegeuk Kim  * this structure is used as one of function parameters.
95539a53e0cSJaegeuk Kim  * all the information are dedicated to a given direct node block determined
95639a53e0cSJaegeuk Kim  * by the data offset in a file.
95739a53e0cSJaegeuk Kim  */
95839a53e0cSJaegeuk Kim struct dnode_of_data {
95939a53e0cSJaegeuk Kim 	struct inode *inode;		/* vfs inode pointer */
96039a53e0cSJaegeuk Kim 	struct page *inode_page;	/* its inode page, NULL is possible */
96139a53e0cSJaegeuk Kim 	struct page *node_page;		/* cached direct node page */
96239a53e0cSJaegeuk Kim 	nid_t nid;			/* node id of the direct node block */
96339a53e0cSJaegeuk Kim 	unsigned int ofs_in_node;	/* data offset in the node page */
96439a53e0cSJaegeuk Kim 	bool inode_page_locked;		/* inode page is locked or not */
96593bae099SJaegeuk Kim 	bool node_changed;		/* is node block changed */
9663cf45747SChao Yu 	char cur_level;			/* level of hole node page */
9673cf45747SChao Yu 	char max_level;			/* level of current page located */
96839a53e0cSJaegeuk Kim 	block_t	data_blkaddr;		/* block address of the node block */
96939a53e0cSJaegeuk Kim };
97039a53e0cSJaegeuk Kim 
97139a53e0cSJaegeuk Kim static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode,
97239a53e0cSJaegeuk Kim 		struct page *ipage, struct page *npage, nid_t nid)
97339a53e0cSJaegeuk Kim {
974d66d1f76SJaegeuk Kim 	memset(dn, 0, sizeof(*dn));
97539a53e0cSJaegeuk Kim 	dn->inode = inode;
97639a53e0cSJaegeuk Kim 	dn->inode_page = ipage;
97739a53e0cSJaegeuk Kim 	dn->node_page = npage;
97839a53e0cSJaegeuk Kim 	dn->nid = nid;
97939a53e0cSJaegeuk Kim }
98039a53e0cSJaegeuk Kim 
98139a53e0cSJaegeuk Kim /*
98239a53e0cSJaegeuk Kim  * For SIT manager
98339a53e0cSJaegeuk Kim  *
98439a53e0cSJaegeuk Kim  * By default, there are 6 active log areas across the whole main area.
98539a53e0cSJaegeuk Kim  * When considering hot and cold data separation to reduce cleaning overhead,
98639a53e0cSJaegeuk Kim  * we split 3 for data logs and 3 for node logs as hot, warm, and cold types,
98739a53e0cSJaegeuk Kim  * respectively.
98839a53e0cSJaegeuk Kim  * In the current design, you should not change the numbers intentionally.
98939a53e0cSJaegeuk Kim  * Instead, as a mount option such as active_logs=x, you can use 2, 4, and 6
99039a53e0cSJaegeuk Kim  * logs individually according to the underlying devices. (default: 6)
99139a53e0cSJaegeuk Kim  * Just in case, on-disk layout covers maximum 16 logs that consist of 8 for
99239a53e0cSJaegeuk Kim  * data and 8 for node logs.
99339a53e0cSJaegeuk Kim  */
99439a53e0cSJaegeuk Kim #define	NR_CURSEG_DATA_TYPE	(3)
99539a53e0cSJaegeuk Kim #define NR_CURSEG_NODE_TYPE	(3)
996093749e2SChao Yu #define NR_CURSEG_INMEM_TYPE	(2)
997a7d9fe3cSJaegeuk Kim #define NR_CURSEG_RO_TYPE	(2)
998d0b9e42aSChao Yu #define NR_CURSEG_PERSIST_TYPE	(NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
999d0b9e42aSChao Yu #define NR_CURSEG_TYPE		(NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE)
100039a53e0cSJaegeuk Kim 
100139a53e0cSJaegeuk Kim enum {
100239a53e0cSJaegeuk Kim 	CURSEG_HOT_DATA	= 0,	/* directory entry blocks */
100339a53e0cSJaegeuk Kim 	CURSEG_WARM_DATA,	/* data blocks */
100439a53e0cSJaegeuk Kim 	CURSEG_COLD_DATA,	/* multimedia or GCed data blocks */
100539a53e0cSJaegeuk Kim 	CURSEG_HOT_NODE,	/* direct node blocks of directory files */
100639a53e0cSJaegeuk Kim 	CURSEG_WARM_NODE,	/* direct node blocks of normal files */
100739a53e0cSJaegeuk Kim 	CURSEG_COLD_NODE,	/* indirect node blocks */
1008d0b9e42aSChao Yu 	NR_PERSISTENT_LOG,	/* number of persistent log */
1009d0b9e42aSChao Yu 	CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG,
1010d0b9e42aSChao Yu 				/* pinned file that needs consecutive block address */
1011093749e2SChao Yu 	CURSEG_ALL_DATA_ATGC,	/* SSR alloctor in hot/warm/cold data area */
1012d0b9e42aSChao Yu 	NO_CHECK_TYPE,		/* number of persistent & inmem log */
101339a53e0cSJaegeuk Kim };
101439a53e0cSJaegeuk Kim 
10156b4afdd7SJaegeuk Kim struct flush_cmd {
10166b4afdd7SJaegeuk Kim 	struct completion wait;
1017721bd4d5SGu Zheng 	struct llist_node llnode;
101839d787beSChao Yu 	nid_t ino;
10196b4afdd7SJaegeuk Kim 	int ret;
10206b4afdd7SJaegeuk Kim };
10216b4afdd7SJaegeuk Kim 
1022a688b9d9SGu Zheng struct flush_cmd_control {
1023a688b9d9SGu Zheng 	struct task_struct *f2fs_issue_flush;	/* flush thread */
1024a688b9d9SGu Zheng 	wait_queue_head_t flush_wait_queue;	/* waiting queue for wake-up */
10258b8dd65fSChao Yu 	atomic_t issued_flush;			/* # of issued flushes */
102672691af6SJaegeuk Kim 	atomic_t queued_flush;			/* # of queued flushes */
1027721bd4d5SGu Zheng 	struct llist_head issue_list;		/* list for command issue */
1028721bd4d5SGu Zheng 	struct llist_node *dispatch_list;	/* list for command dispatch */
1029a688b9d9SGu Zheng };
1030a688b9d9SGu Zheng 
103139a53e0cSJaegeuk Kim struct f2fs_sm_info {
103239a53e0cSJaegeuk Kim 	struct sit_info *sit_info;		/* whole segment information */
103339a53e0cSJaegeuk Kim 	struct free_segmap_info *free_info;	/* free segment information */
103439a53e0cSJaegeuk Kim 	struct dirty_seglist_info *dirty_info;	/* dirty segment information */
103539a53e0cSJaegeuk Kim 	struct curseg_info *curseg_array;	/* active segment information */
103639a53e0cSJaegeuk Kim 
1037e4544b63STim Murray 	struct f2fs_rwsem curseg_lock;	/* for preventing curseg change */
10382b60311dSChao Yu 
103939a53e0cSJaegeuk Kim 	block_t seg0_blkaddr;		/* block address of 0'th segment */
104039a53e0cSJaegeuk Kim 	block_t main_blkaddr;		/* start block address of main area */
104139a53e0cSJaegeuk Kim 	block_t ssa_blkaddr;		/* start block address of SSA area */
104239a53e0cSJaegeuk Kim 
104339a53e0cSJaegeuk Kim 	unsigned int segment_count;	/* total # of segments */
104439a53e0cSJaegeuk Kim 	unsigned int main_segments;	/* # of segments in main area */
104539a53e0cSJaegeuk Kim 	unsigned int reserved_segments;	/* # of reserved segments */
1046300a8429SChao Yu 	unsigned int additional_reserved_segments;/* reserved segs for IO align feature */
104739a53e0cSJaegeuk Kim 	unsigned int ovp_segments;	/* # of overprovision segments */
104881eb8d6eSJaegeuk Kim 
104981eb8d6eSJaegeuk Kim 	/* a threshold to reclaim prefree segments */
105081eb8d6eSJaegeuk Kim 	unsigned int rec_prefree_segments;
10517fd9e544SJaegeuk Kim 
1052184a5cd2SChao Yu 	struct list_head sit_entry_set;	/* sit entry set list */
1053184a5cd2SChao Yu 
1054216fbd64SJaegeuk Kim 	unsigned int ipu_policy;	/* in-place-update policy */
1055216fbd64SJaegeuk Kim 	unsigned int min_ipu_util;	/* in-place-update threshold */
1056c1ce1b02SJaegeuk Kim 	unsigned int min_fsync_blocks;	/* threshold for fsync */
1057853137ceSJaegeuk Kim 	unsigned int min_seq_blocks;	/* threshold for sequential blocks */
1058ef095d19SJaegeuk Kim 	unsigned int min_hot_blocks;	/* threshold for hot block allocation */
1059a2a12b67SChao Yu 	unsigned int min_ssr_sections;	/* threshold to trigger SSR allocation */
10606b4afdd7SJaegeuk Kim 
10616b4afdd7SJaegeuk Kim 	/* for flush command control */
1062b01a9201SJaegeuk Kim 	struct flush_cmd_control *fcc_info;
1063a688b9d9SGu Zheng 
10640b54fb84SJaegeuk Kim 	/* for discard command control */
10650b54fb84SJaegeuk Kim 	struct discard_cmd_control *dcc_info;
106639a53e0cSJaegeuk Kim };
106739a53e0cSJaegeuk Kim 
106839a53e0cSJaegeuk Kim /*
106939a53e0cSJaegeuk Kim  * For superblock
107039a53e0cSJaegeuk Kim  */
107139a53e0cSJaegeuk Kim /*
107239a53e0cSJaegeuk Kim  * COUNT_TYPE for monitoring
107339a53e0cSJaegeuk Kim  *
107439a53e0cSJaegeuk Kim  * f2fs monitors the number of several block types such as on-writeback,
107539a53e0cSJaegeuk Kim  * dirty dentry blocks, dirty node blocks, and dirty meta blocks.
107639a53e0cSJaegeuk Kim  */
107736951b38SChao Yu #define WB_DATA_TYPE(p)	(__is_cp_guaranteed(p) ? F2FS_WB_CP_DATA : F2FS_WB_DATA)
107839a53e0cSJaegeuk Kim enum count_type {
107939a53e0cSJaegeuk Kim 	F2FS_DIRTY_DENTS,
1080c227f912SChao Yu 	F2FS_DIRTY_DATA,
10812c8a4a28SJaegeuk Kim 	F2FS_DIRTY_QDATA,
108239a53e0cSJaegeuk Kim 	F2FS_DIRTY_NODES,
108339a53e0cSJaegeuk Kim 	F2FS_DIRTY_META,
10840f18b462SJaegeuk Kim 	F2FS_DIRTY_IMETA,
108536951b38SChao Yu 	F2FS_WB_CP_DATA,
108636951b38SChao Yu 	F2FS_WB_DATA,
10875f9abab4SJaegeuk Kim 	F2FS_RD_DATA,
10885f9abab4SJaegeuk Kim 	F2FS_RD_NODE,
10895f9abab4SJaegeuk Kim 	F2FS_RD_META,
109002b16d0aSChao Yu 	F2FS_DIO_WRITE,
109102b16d0aSChao Yu 	F2FS_DIO_READ,
109239a53e0cSJaegeuk Kim 	NR_COUNT_TYPE,
109339a53e0cSJaegeuk Kim };
109439a53e0cSJaegeuk Kim 
109539a53e0cSJaegeuk Kim /*
1096e1c42045Sarter97  * The below are the page types of bios used in submit_bio().
109739a53e0cSJaegeuk Kim  * The available types are:
109839a53e0cSJaegeuk Kim  * DATA			User data pages. It operates as async mode.
109939a53e0cSJaegeuk Kim  * NODE			Node pages. It operates as async mode.
110039a53e0cSJaegeuk Kim  * META			FS metadata pages such as SIT, NAT, CP.
110139a53e0cSJaegeuk Kim  * NR_PAGE_TYPE		The number of page types.
110239a53e0cSJaegeuk Kim  * META_FLUSH		Make sure the previous pages are written
110339a53e0cSJaegeuk Kim  *			with waiting the bio's completion
110439a53e0cSJaegeuk Kim  * ...			Only can be used with META.
110539a53e0cSJaegeuk Kim  */
11067d5e5109SJaegeuk Kim #define PAGE_TYPE_OF_BIO(type)	((type) > META ? META : (type))
110739a53e0cSJaegeuk Kim enum page_type {
11086b8beca0SChao Yu 	DATA = 0,
11096b8beca0SChao Yu 	NODE = 1,	/* should not change this */
111039a53e0cSJaegeuk Kim 	META,
111139a53e0cSJaegeuk Kim 	NR_PAGE_TYPE,
111239a53e0cSJaegeuk Kim 	META_FLUSH,
11133db1de0eSDaeho Jeong 	IPU,		/* the below types are used by tracepoints only. */
11148ce67cb0SJaegeuk Kim 	OPU,
111539a53e0cSJaegeuk Kim };
111639a53e0cSJaegeuk Kim 
1117a912b54dSJaegeuk Kim enum temp_type {
1118a912b54dSJaegeuk Kim 	HOT = 0,	/* must be zero for meta bio */
1119a912b54dSJaegeuk Kim 	WARM,
1120a912b54dSJaegeuk Kim 	COLD,
1121a912b54dSJaegeuk Kim 	NR_TEMP_TYPE,
1122a912b54dSJaegeuk Kim };
1123a912b54dSJaegeuk Kim 
1124cc15620bSJaegeuk Kim enum need_lock_type {
1125cc15620bSJaegeuk Kim 	LOCK_REQ = 0,
1126cc15620bSJaegeuk Kim 	LOCK_DONE,
1127cc15620bSJaegeuk Kim 	LOCK_RETRY,
1128cc15620bSJaegeuk Kim };
1129cc15620bSJaegeuk Kim 
1130a5fd5050SChao Yu enum cp_reason_type {
1131a5fd5050SChao Yu 	CP_NO_NEEDED,
1132a5fd5050SChao Yu 	CP_NON_REGULAR,
11334c8ff709SChao Yu 	CP_COMPRESSED,
1134a5fd5050SChao Yu 	CP_HARDLINK,
1135a5fd5050SChao Yu 	CP_SB_NEED_CP,
1136a5fd5050SChao Yu 	CP_WRONG_PINO,
1137a5fd5050SChao Yu 	CP_NO_SPC_ROLL,
1138a5fd5050SChao Yu 	CP_NODE_NEED_CP,
1139a5fd5050SChao Yu 	CP_FASTBOOT_MODE,
1140a5fd5050SChao Yu 	CP_SPEC_LOG_NUM,
11410a007b97SJaegeuk Kim 	CP_RECOVER_DIR,
1142a5fd5050SChao Yu };
1143a5fd5050SChao Yu 
1144b0af6d49SChao Yu enum iostat_type {
11458b83ac81SChao Yu 	/* WRITE IO */
11468b83ac81SChao Yu 	APP_DIRECT_IO,			/* app direct write IOs */
11478b83ac81SChao Yu 	APP_BUFFERED_IO,		/* app buffered write IOs */
1148b0af6d49SChao Yu 	APP_WRITE_IO,			/* app write IOs */
1149b0af6d49SChao Yu 	APP_MAPPED_IO,			/* app mapped IOs */
115034a23525SChao Yu 	APP_BUFFERED_CDATA_IO,		/* app buffered write IOs on compressed file */
115134a23525SChao Yu 	APP_MAPPED_CDATA_IO,		/* app mapped write IOs on compressed file */
1152b0af6d49SChao Yu 	FS_DATA_IO,			/* data IOs from kworker/fsync/reclaimer */
115334a23525SChao Yu 	FS_CDATA_IO,			/* data IOs from kworker/fsync/reclaimer on compressed file */
1154b0af6d49SChao Yu 	FS_NODE_IO,			/* node IOs from kworker/fsync/reclaimer */
1155b0af6d49SChao Yu 	FS_META_IO,			/* meta IOs from kworker/reclaimer */
1156b0af6d49SChao Yu 	FS_GC_DATA_IO,			/* data IOs from forground gc */
1157b0af6d49SChao Yu 	FS_GC_NODE_IO,			/* node IOs from forground gc */
1158b0af6d49SChao Yu 	FS_CP_DATA_IO,			/* data IOs from checkpoint */
1159b0af6d49SChao Yu 	FS_CP_NODE_IO,			/* node IOs from checkpoint */
1160b0af6d49SChao Yu 	FS_CP_META_IO,			/* meta IOs from checkpoint */
11618b83ac81SChao Yu 
11628b83ac81SChao Yu 	/* READ IO */
11638b83ac81SChao Yu 	APP_DIRECT_READ_IO,		/* app direct read IOs */
11648b83ac81SChao Yu 	APP_BUFFERED_READ_IO,		/* app buffered read IOs */
11658b83ac81SChao Yu 	APP_READ_IO,			/* app read IOs */
11668b83ac81SChao Yu 	APP_MAPPED_READ_IO,		/* app mapped read IOs */
116734a23525SChao Yu 	APP_BUFFERED_CDATA_READ_IO,	/* app buffered read IOs on compressed file  */
116834a23525SChao Yu 	APP_MAPPED_CDATA_READ_IO,	/* app mapped read IOs on compressed file  */
11698b83ac81SChao Yu 	FS_DATA_READ_IO,		/* data read IOs */
11709c122384SChao Yu 	FS_GDATA_READ_IO,		/* data read IOs from background gc */
11719c122384SChao Yu 	FS_CDATA_READ_IO,		/* compressed data read IOs */
11728b83ac81SChao Yu 	FS_NODE_READ_IO,		/* node read IOs */
11738b83ac81SChao Yu 	FS_META_READ_IO,		/* meta read IOs */
11748b83ac81SChao Yu 
11758b83ac81SChao Yu 	/* other */
11767a2b15cfSYangtao Li 	FS_DISCARD_IO,			/* discard */
1177193a639fSYangtao Li 	FS_FLUSH_IO,			/* flush */
1178b0af6d49SChao Yu 	NR_IO_TYPE,
1179b0af6d49SChao Yu };
1180b0af6d49SChao Yu 
1181458e6197SJaegeuk Kim struct f2fs_io_info {
118205ca3632SJaegeuk Kim 	struct f2fs_sb_info *sbi;	/* f2fs_sb_info pointer */
118339d787beSChao Yu 	nid_t ino;		/* inode number */
1184458e6197SJaegeuk Kim 	enum page_type type;	/* contains DATA/NODE/META/META_FLUSH */
1185a912b54dSJaegeuk Kim 	enum temp_type temp;	/* contains HOT/WARM/COLD */
11867649c873SBart Van Assche 	enum req_op op;		/* contains REQ_OP_ */
11877649c873SBart Van Assche 	blk_opf_t op_flags;	/* req_flag_bits */
11887a9d7548SChao Yu 	block_t new_blkaddr;	/* new block address to be written */
118928bc106bSChao Yu 	block_t old_blkaddr;	/* old block address before Cow */
119005ca3632SJaegeuk Kim 	struct page *page;	/* page to be written */
11914375a336SJaegeuk Kim 	struct page *encrypted_page;	/* encrypted page */
11924c8ff709SChao Yu 	struct page *compressed_page;	/* compressed page */
1193fb830fc5SChao Yu 	struct list_head list;		/* serialize IOs */
11942eae077eSChao Yu 	unsigned int compr_blocks;	/* # of compressed block addresses */
11952eae077eSChao Yu 	unsigned int need_lock:8;	/* indicate we need to lock cp_rwsem */
11962eae077eSChao Yu 	unsigned int version:8;		/* version of the node */
11972eae077eSChao Yu 	unsigned int submitted:1;	/* indicate IO submission */
11982eae077eSChao Yu 	unsigned int in_list:1;		/* indicate fio is in io_list */
11992eae077eSChao Yu 	unsigned int is_por:1;		/* indicate IO is from recovery or not */
12002eae077eSChao Yu 	unsigned int retry:1;		/* need to reallocate block address */
12012eae077eSChao Yu 	unsigned int encrypted:1;	/* indicate file is encrypted */
12022eae077eSChao Yu 	unsigned int post_read:1;	/* require post read */
1203b0af6d49SChao Yu 	enum iostat_type io_type;	/* io type */
1204578c6478SYufen Yu 	struct writeback_control *io_wbc; /* writeback control */
12058648de2cSChao Yu 	struct bio **bio;		/* bio for ipu */
12068648de2cSChao Yu 	sector_t *last_block;		/* last block number in bio */
1207458e6197SJaegeuk Kim };
1208458e6197SJaegeuk Kim 
12090b20fcecSChao Yu struct bio_entry {
12100b20fcecSChao Yu 	struct bio *bio;
12110b20fcecSChao Yu 	struct list_head list;
12120b20fcecSChao Yu };
12130b20fcecSChao Yu 
121468afcf2dSTomohiro Kusumi #define is_read_io(rw) ((rw) == READ)
12151ff7bd3bSJaegeuk Kim struct f2fs_bio_info {
1216458e6197SJaegeuk Kim 	struct f2fs_sb_info *sbi;	/* f2fs superblock */
12171ff7bd3bSJaegeuk Kim 	struct bio *bio;		/* bios to merge */
12181ff7bd3bSJaegeuk Kim 	sector_t last_block_in_bio;	/* last block number */
1219458e6197SJaegeuk Kim 	struct f2fs_io_info fio;	/* store buffered io info. */
1220e4544b63STim Murray 	struct f2fs_rwsem io_rwsem;	/* blocking op for bio */
1221fb830fc5SChao Yu 	spinlock_t io_lock;		/* serialize DATA/NODE IOs */
1222fb830fc5SChao Yu 	struct list_head io_list;	/* track fios */
12230b20fcecSChao Yu 	struct list_head bio_list;	/* bio entry list head */
1224e4544b63STim Murray 	struct f2fs_rwsem bio_list_lock;	/* lock to protect bio entry list */
12251ff7bd3bSJaegeuk Kim };
12261ff7bd3bSJaegeuk Kim 
12273c62be17SJaegeuk Kim #define FDEV(i)				(sbi->devs[i])
12283c62be17SJaegeuk Kim #define RDEV(i)				(raw_super->devs[i])
12293c62be17SJaegeuk Kim struct f2fs_dev_info {
12303c62be17SJaegeuk Kim 	struct block_device *bdev;
12313c62be17SJaegeuk Kim 	char path[MAX_PATH_LEN];
12323c62be17SJaegeuk Kim 	unsigned int total_segments;
12333c62be17SJaegeuk Kim 	block_t start_blk;
12343c62be17SJaegeuk Kim 	block_t end_blk;
12353c62be17SJaegeuk Kim #ifdef CONFIG_BLK_DEV_ZONED
12363c62be17SJaegeuk Kim 	unsigned int nr_blkz;		/* Total number of zones */
123795175dafSDamien Le Moal 	unsigned long *blkz_seq;	/* Bitmap indicating sequential zones */
12383c62be17SJaegeuk Kim #endif
12393c62be17SJaegeuk Kim };
12403c62be17SJaegeuk Kim 
1241c227f912SChao Yu enum inode_type {
1242c227f912SChao Yu 	DIR_INODE,			/* for dirty dir inode */
1243c227f912SChao Yu 	FILE_INODE,			/* for dirty regular/symlink inode */
12440f18b462SJaegeuk Kim 	DIRTY_META,			/* for all dirtied inode metadata */
1245c227f912SChao Yu 	NR_INODE_TYPE,
1246c227f912SChao Yu };
1247c227f912SChao Yu 
124867298804SChao Yu /* for inner inode cache management */
124967298804SChao Yu struct inode_management {
125067298804SChao Yu 	struct radix_tree_root ino_root;	/* ino entry array */
125167298804SChao Yu 	spinlock_t ino_lock;			/* for ino entry lock */
125267298804SChao Yu 	struct list_head ino_list;		/* inode list head */
125367298804SChao Yu 	unsigned long ino_num;			/* number of entries */
125467298804SChao Yu };
125567298804SChao Yu 
1256093749e2SChao Yu /* for GC_AT */
1257093749e2SChao Yu struct atgc_management {
1258093749e2SChao Yu 	bool atgc_enabled;			/* ATGC is enabled or not */
1259093749e2SChao Yu 	struct rb_root_cached root;		/* root of victim rb-tree */
1260093749e2SChao Yu 	struct list_head victim_list;		/* linked with all victim entries */
1261093749e2SChao Yu 	unsigned int victim_count;		/* victim count in rb-tree */
1262093749e2SChao Yu 	unsigned int candidate_ratio;		/* candidate ratio */
1263093749e2SChao Yu 	unsigned int max_candidate_count;	/* max candidate count */
1264093749e2SChao Yu 	unsigned int age_weight;		/* age weight, vblock_weight = 100 - age_weight */
1265093749e2SChao Yu 	unsigned long long age_threshold;	/* age threshold */
1266093749e2SChao Yu };
1267093749e2SChao Yu 
1268d147ea4aSJaegeuk Kim struct f2fs_gc_control {
1269d147ea4aSJaegeuk Kim 	unsigned int victim_segno;	/* target victim segment number */
1270d147ea4aSJaegeuk Kim 	int init_gc_type;		/* FG_GC or BG_GC */
1271d147ea4aSJaegeuk Kim 	bool no_bg_gc;			/* check the space and stop bg_gc */
1272d147ea4aSJaegeuk Kim 	bool should_migrate_blocks;	/* should migrate blocks */
1273d147ea4aSJaegeuk Kim 	bool err_gc_skipped;		/* return EAGAIN if GC skipped */
1274c81d5baeSJaegeuk Kim 	unsigned int nr_free_secs;	/* # of free sections to do GC */
1275d147ea4aSJaegeuk Kim };
1276d147ea4aSJaegeuk Kim 
12775bb9c111SYangtao Li /*
12785bb9c111SYangtao Li  * For s_flag in struct f2fs_sb_info
12795bb9c111SYangtao Li  * Modification on enum should be synchronized with s_flag array
12805bb9c111SYangtao Li  */
1281caf0047eSChao Yu enum {
1282caf0047eSChao Yu 	SBI_IS_DIRTY,				/* dirty flag for checkpoint */
1283caf0047eSChao Yu 	SBI_IS_CLOSE,				/* specify unmounting */
1284caf0047eSChao Yu 	SBI_NEED_FSCK,				/* need fsck.f2fs to fix */
1285caf0047eSChao Yu 	SBI_POR_DOING,				/* recovery is doing or not */
1286df728b0fSJaegeuk Kim 	SBI_NEED_SB_WRITE,			/* need to recover superblock */
1287bbf156f7SJaegeuk Kim 	SBI_NEED_CP,				/* need to checkpoint */
128883a3bfdbSJaegeuk Kim 	SBI_IS_SHUTDOWN,			/* shutdown by ioctl */
12891378752bSChao Yu 	SBI_IS_RECOVERED,			/* recovered orphan/data */
12904354994fSDaniel Rosenberg 	SBI_CP_DISABLED,			/* CP was disabled last mount */
1291db610a64SJaegeuk Kim 	SBI_CP_DISABLED_QUICK,			/* CP was disabled quickly */
1292af033b2aSChao Yu 	SBI_QUOTA_NEED_FLUSH,			/* need to flush quota info in CP */
1293af033b2aSChao Yu 	SBI_QUOTA_SKIP_FLUSH,			/* skip flushing quota in current CP */
1294af033b2aSChao Yu 	SBI_QUOTA_NEED_REPAIR,			/* quota file may be corrupted */
129504f0b2eaSQiuyang Sun 	SBI_IS_RESIZEFS,			/* resizefs is in process */
1296ba900534SJaegeuk Kim 	SBI_IS_FREEZING,			/* freezefs is in process */
12975bb9c111SYangtao Li 	MAX_SBI_FLAG,
1298caf0047eSChao Yu };
1299caf0047eSChao Yu 
13006beceb54SJaegeuk Kim enum {
13016beceb54SJaegeuk Kim 	CP_TIME,
1302d0239e1bSJaegeuk Kim 	REQ_TIME,
1303a7d10cf3SSahitya Tummala 	DISCARD_TIME,
1304a7d10cf3SSahitya Tummala 	GC_TIME,
13054354994fSDaniel Rosenberg 	DISABLE_TIME,
130603f2c02dSJaegeuk Kim 	UMOUNT_DISCARD_TIMEOUT,
13076beceb54SJaegeuk Kim 	MAX_TIME,
13086beceb54SJaegeuk Kim };
13096beceb54SJaegeuk Kim 
1310a3951cd1SYangtao Li /* Note that you need to keep synchronization with this gc_mode_names array */
13110cdd3195SHyunchul Lee enum {
13125b0e9539SJaegeuk Kim 	GC_NORMAL,
13135b0e9539SJaegeuk Kim 	GC_IDLE_CB,
13145b0e9539SJaegeuk Kim 	GC_IDLE_GREEDY,
1315093749e2SChao Yu 	GC_IDLE_AT,
13160e5e8111SDaeho Jeong 	GC_URGENT_HIGH,
13170e5e8111SDaeho Jeong 	GC_URGENT_LOW,
1318d98af5f4SDaeho Jeong 	GC_URGENT_MID,
131907c6b593SDaeho Jeong 	MAX_GC_MODE,
13205b0e9539SJaegeuk Kim };
13215b0e9539SJaegeuk Kim 
13225b0e9539SJaegeuk Kim enum {
1323bbbc34fdSChao Yu 	BGGC_MODE_ON,		/* background gc is on */
1324bbbc34fdSChao Yu 	BGGC_MODE_OFF,		/* background gc is off */
1325bbbc34fdSChao Yu 	BGGC_MODE_SYNC,		/*
1326bbbc34fdSChao Yu 				 * background gc is on, migrating blocks
1327bbbc34fdSChao Yu 				 * like foreground gc
1328bbbc34fdSChao Yu 				 */
1329bbbc34fdSChao Yu };
1330bbbc34fdSChao Yu 
1331bbbc34fdSChao Yu enum {
1332b0332a0fSChao Yu 	FS_MODE_ADAPTIVE,		/* use both lfs/ssr allocation */
1333b0332a0fSChao Yu 	FS_MODE_LFS,			/* use lfs allocation only */
13346691d940SDaeho Jeong 	FS_MODE_FRAGMENT_SEG,		/* segment fragmentation mode */
13356691d940SDaeho Jeong 	FS_MODE_FRAGMENT_BLK,		/* block fragmentation mode */
1336b0332a0fSChao Yu };
1337b0332a0fSChao Yu 
1338b0332a0fSChao Yu enum {
133907939627SJaegeuk Kim 	ALLOC_MODE_DEFAULT,	/* stay default */
134007939627SJaegeuk Kim 	ALLOC_MODE_REUSE,	/* reuse segments as much as possible */
134107939627SJaegeuk Kim };
134207939627SJaegeuk Kim 
134393cf93f1SJunling Zheng enum fsync_mode {
134493cf93f1SJunling Zheng 	FSYNC_MODE_POSIX,	/* fsync follows posix semantics */
134593cf93f1SJunling Zheng 	FSYNC_MODE_STRICT,	/* fsync behaves in line with ext4 */
1346d6290814SJaegeuk Kim 	FSYNC_MODE_NOBARRIER,	/* fsync behaves nobarrier based on posix */
134793cf93f1SJunling Zheng };
134893cf93f1SJunling Zheng 
1349602a16d5SDaeho Jeong enum {
1350602a16d5SDaeho Jeong 	COMPR_MODE_FS,		/*
1351602a16d5SDaeho Jeong 				 * automatically compress compression
1352602a16d5SDaeho Jeong 				 * enabled files
1353602a16d5SDaeho Jeong 				 */
1354602a16d5SDaeho Jeong 	COMPR_MODE_USER,	/*
1355602a16d5SDaeho Jeong 				 * automatical compression is disabled.
1356602a16d5SDaeho Jeong 				 * user can control the file compression
1357602a16d5SDaeho Jeong 				 * using ioctls
1358602a16d5SDaeho Jeong 				 */
1359602a16d5SDaeho Jeong };
1360602a16d5SDaeho Jeong 
13614f993264SChao Yu enum {
13624f993264SChao Yu 	DISCARD_UNIT_BLOCK,	/* basic discard unit is block */
13634f993264SChao Yu 	DISCARD_UNIT_SEGMENT,	/* basic discard unit is segment */
13644f993264SChao Yu 	DISCARD_UNIT_SECTION,	/* basic discard unit is section */
13654f993264SChao Yu };
13664f993264SChao Yu 
13677a8fc586SDaeho Jeong enum {
13687a8fc586SDaeho Jeong 	MEMORY_MODE_NORMAL,	/* memory mode for normal devices */
13697a8fc586SDaeho Jeong 	MEMORY_MODE_LOW,	/* memory mode for low memry devices */
13707a8fc586SDaeho Jeong };
13717a8fc586SDaeho Jeong 
1372b763f3beSChao Yu static inline int f2fs_test_bit(unsigned int nr, char *addr);
1373b763f3beSChao Yu static inline void f2fs_set_bit(unsigned int nr, char *addr);
1374b763f3beSChao Yu static inline void f2fs_clear_bit(unsigned int nr, char *addr);
13754c8ff709SChao Yu 
1376b763f3beSChao Yu /*
1377b763f3beSChao Yu  * Layout of f2fs page.private:
1378b763f3beSChao Yu  *
1379b763f3beSChao Yu  * Layout A: lowest bit should be 1
1380b763f3beSChao Yu  * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... |
1381b763f3beSChao Yu  * bit 0	PAGE_PRIVATE_NOT_POINTER
1382b3107b38SChao Yu  * bit 1	PAGE_PRIVATE_DUMMY_WRITE
1383b3107b38SChao Yu  * bit 2	PAGE_PRIVATE_ONGOING_MIGRATION
1384b3107b38SChao Yu  * bit 3	PAGE_PRIVATE_INLINE_INODE
1385b3107b38SChao Yu  * bit 4	PAGE_PRIVATE_REF_RESOURCE
1386b3107b38SChao Yu  * bit 5-	f2fs private data
1387b763f3beSChao Yu  *
1388b763f3beSChao Yu  * Layout B: lowest bit should be 0
1389b763f3beSChao Yu  * page.private is a wrapped pointer.
1390b763f3beSChao Yu  */
1391b763f3beSChao Yu enum {
1392b763f3beSChao Yu 	PAGE_PRIVATE_NOT_POINTER,		/* private contains non-pointer data */
1393b763f3beSChao Yu 	PAGE_PRIVATE_DUMMY_WRITE,		/* data page for padding aligned IO */
1394b763f3beSChao Yu 	PAGE_PRIVATE_ONGOING_MIGRATION,		/* data page which is on-going migrating */
1395b763f3beSChao Yu 	PAGE_PRIVATE_INLINE_INODE,		/* inode page contains inline data */
1396b763f3beSChao Yu 	PAGE_PRIVATE_REF_RESOURCE,		/* dirty page has referenced resources */
1397b763f3beSChao Yu 	PAGE_PRIVATE_MAX
1398b763f3beSChao Yu };
1399b763f3beSChao Yu 
1400b763f3beSChao Yu #define PAGE_PRIVATE_GET_FUNC(name, flagname) \
1401b763f3beSChao Yu static inline bool page_private_##name(struct page *page) \
1402b763f3beSChao Yu { \
1403c9ebd3dfSJaegeuk Kim 	return PagePrivate(page) && \
1404c9ebd3dfSJaegeuk Kim 		test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \
1405b763f3beSChao Yu 		test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1406b763f3beSChao Yu }
1407b763f3beSChao Yu 
1408b763f3beSChao Yu #define PAGE_PRIVATE_SET_FUNC(name, flagname) \
1409b763f3beSChao Yu static inline void set_page_private_##name(struct page *page) \
1410b763f3beSChao Yu { \
1411b763f3beSChao Yu 	if (!PagePrivate(page)) { \
1412b763f3beSChao Yu 		get_page(page); \
1413b763f3beSChao Yu 		SetPagePrivate(page); \
1414c9ebd3dfSJaegeuk Kim 		set_page_private(page, 0); \
1415b763f3beSChao Yu 	} \
1416b763f3beSChao Yu 	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \
1417b763f3beSChao Yu 	set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1418b763f3beSChao Yu }
1419b763f3beSChao Yu 
1420b763f3beSChao Yu #define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \
1421b763f3beSChao Yu static inline void clear_page_private_##name(struct page *page) \
1422b763f3beSChao Yu { \
1423b763f3beSChao Yu 	clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1424447286ebSYangtao Li 	if (page_private(page) == BIT(PAGE_PRIVATE_NOT_POINTER)) { \
1425b763f3beSChao Yu 		set_page_private(page, 0); \
1426b763f3beSChao Yu 		if (PagePrivate(page)) { \
1427b763f3beSChao Yu 			ClearPagePrivate(page); \
1428b763f3beSChao Yu 			put_page(page); \
1429b763f3beSChao Yu 		}\
1430b763f3beSChao Yu 	} \
1431b763f3beSChao Yu }
1432b763f3beSChao Yu 
1433b763f3beSChao Yu PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
1434b763f3beSChao Yu PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE);
1435b763f3beSChao Yu PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
1436b763f3beSChao Yu PAGE_PRIVATE_GET_FUNC(dummy, DUMMY_WRITE);
1437b763f3beSChao Yu 
1438b763f3beSChao Yu PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
1439b763f3beSChao Yu PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE);
1440b763f3beSChao Yu PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
1441b763f3beSChao Yu PAGE_PRIVATE_SET_FUNC(dummy, DUMMY_WRITE);
1442b763f3beSChao Yu 
1443b763f3beSChao Yu PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
1444b763f3beSChao Yu PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE);
1445b763f3beSChao Yu PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
1446b763f3beSChao Yu PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE);
14474c8ff709SChao Yu 
14486ce19affSChao Yu static inline unsigned long get_page_private_data(struct page *page)
14496ce19affSChao Yu {
14506ce19affSChao Yu 	unsigned long data = page_private(page);
14516ce19affSChao Yu 
14526ce19affSChao Yu 	if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
14536ce19affSChao Yu 		return 0;
14546ce19affSChao Yu 	return data >> PAGE_PRIVATE_MAX;
14556ce19affSChao Yu }
14566ce19affSChao Yu 
14576ce19affSChao Yu static inline void set_page_private_data(struct page *page, unsigned long data)
14586ce19affSChao Yu {
14596ce19affSChao Yu 	if (!PagePrivate(page)) {
14606ce19affSChao Yu 		get_page(page);
14616ce19affSChao Yu 		SetPagePrivate(page);
1462c9ebd3dfSJaegeuk Kim 		set_page_private(page, 0);
14636ce19affSChao Yu 	}
14646ce19affSChao Yu 	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page));
14656ce19affSChao Yu 	page_private(page) |= data << PAGE_PRIVATE_MAX;
14666ce19affSChao Yu }
14676ce19affSChao Yu 
14686ce19affSChao Yu static inline void clear_page_private_data(struct page *page)
14696ce19affSChao Yu {
1470447286ebSYangtao Li 	page_private(page) &= GENMASK(PAGE_PRIVATE_MAX - 1, 0);
1471447286ebSYangtao Li 	if (page_private(page) == BIT(PAGE_PRIVATE_NOT_POINTER)) {
14726ce19affSChao Yu 		set_page_private(page, 0);
14736ce19affSChao Yu 		if (PagePrivate(page)) {
14746ce19affSChao Yu 			ClearPagePrivate(page);
14756ce19affSChao Yu 			put_page(page);
14766ce19affSChao Yu 		}
14776ce19affSChao Yu 	}
14786ce19affSChao Yu }
14796ce19affSChao Yu 
14804c8ff709SChao Yu /* For compression */
14814c8ff709SChao Yu enum compress_algorithm_type {
14824c8ff709SChao Yu 	COMPRESS_LZO,
14834c8ff709SChao Yu 	COMPRESS_LZ4,
148450cfa66fSChao Yu 	COMPRESS_ZSTD,
14856d92b201SChao Yu 	COMPRESS_LZORLE,
14864c8ff709SChao Yu 	COMPRESS_MAX,
14874c8ff709SChao Yu };
14884c8ff709SChao Yu 
1489b28f047bSChao Yu enum compress_flag {
1490b28f047bSChao Yu 	COMPRESS_CHKSUM,
1491b28f047bSChao Yu 	COMPRESS_MAX_FLAG,
1492b28f047bSChao Yu };
1493b28f047bSChao Yu 
14946ce19affSChao Yu #define	COMPRESS_WATERMARK			20
14956ce19affSChao Yu #define	COMPRESS_PERCENT			20
14966ce19affSChao Yu 
1497b28f047bSChao Yu #define COMPRESS_DATA_RESERVED_SIZE		4
14984c8ff709SChao Yu struct compress_data {
14994c8ff709SChao Yu 	__le32 clen;			/* compressed data size */
1500b28f047bSChao Yu 	__le32 chksum;			/* compressed data chksum */
15014c8ff709SChao Yu 	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
15024c8ff709SChao Yu 	u8 cdata[];			/* compressed data */
15034c8ff709SChao Yu };
15044c8ff709SChao Yu 
15054c8ff709SChao Yu #define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
15064c8ff709SChao Yu 
15074c8ff709SChao Yu #define F2FS_COMPRESSED_PAGE_MAGIC	0xF5F2C000
15084c8ff709SChao Yu 
15093fde13f8SChao Yu #define	COMPRESS_LEVEL_OFFSET	8
15103fde13f8SChao Yu 
15114c8ff709SChao Yu /* compress context */
15124c8ff709SChao Yu struct compress_ctx {
15134c8ff709SChao Yu 	struct inode *inode;		/* inode the context belong to */
15144c8ff709SChao Yu 	pgoff_t cluster_idx;		/* cluster index number */
15154c8ff709SChao Yu 	unsigned int cluster_size;	/* page count in cluster */
15164c8ff709SChao Yu 	unsigned int log_cluster_size;	/* log of cluster size */
15174c8ff709SChao Yu 	struct page **rpages;		/* pages store raw data in cluster */
15184c8ff709SChao Yu 	unsigned int nr_rpages;		/* total page number in rpages */
15194c8ff709SChao Yu 	struct page **cpages;		/* pages store compressed data in cluster */
15204c8ff709SChao Yu 	unsigned int nr_cpages;		/* total page number in cpages */
15213271d7ebSFengnan Chang 	unsigned int valid_nr_cpages;	/* valid page number in cpages */
15224c8ff709SChao Yu 	void *rbuf;			/* virtual mapped address on rpages */
15234c8ff709SChao Yu 	struct compress_data *cbuf;	/* virtual mapped address on cpages */
15244c8ff709SChao Yu 	size_t rlen;			/* valid data length in rbuf */
15254c8ff709SChao Yu 	size_t clen;			/* valid data length in cbuf */
15264c8ff709SChao Yu 	void *private;			/* payload buffer for specified compression algorithm */
152750cfa66fSChao Yu 	void *private2;			/* extra payload buffer */
15284c8ff709SChao Yu };
15294c8ff709SChao Yu 
15304c8ff709SChao Yu /* compress context for write IO path */
15314c8ff709SChao Yu struct compress_io_ctx {
15324c8ff709SChao Yu 	u32 magic;			/* magic number to indicate page is compressed */
15334c8ff709SChao Yu 	struct inode *inode;		/* inode the context belong to */
15344c8ff709SChao Yu 	struct page **rpages;		/* pages store raw data in cluster */
15354c8ff709SChao Yu 	unsigned int nr_rpages;		/* total page number in rpages */
1536e6c3948dSChao Yu 	atomic_t pending_pages;		/* in-flight compressed page count */
15374c8ff709SChao Yu };
15384c8ff709SChao Yu 
15397f59b277SEric Biggers /* Context for decompressing one cluster on the read IO path */
15404c8ff709SChao Yu struct decompress_io_ctx {
15414c8ff709SChao Yu 	u32 magic;			/* magic number to indicate page is compressed */
15424c8ff709SChao Yu 	struct inode *inode;		/* inode the context belong to */
15434c8ff709SChao Yu 	pgoff_t cluster_idx;		/* cluster index number */
15444c8ff709SChao Yu 	unsigned int cluster_size;	/* page count in cluster */
15454c8ff709SChao Yu 	unsigned int log_cluster_size;	/* log of cluster size */
15464c8ff709SChao Yu 	struct page **rpages;		/* pages store raw data in cluster */
15474c8ff709SChao Yu 	unsigned int nr_rpages;		/* total page number in rpages */
15484c8ff709SChao Yu 	struct page **cpages;		/* pages store compressed data in cluster */
15494c8ff709SChao Yu 	unsigned int nr_cpages;		/* total page number in cpages */
15504c8ff709SChao Yu 	struct page **tpages;		/* temp pages to pad holes in cluster */
15514c8ff709SChao Yu 	void *rbuf;			/* virtual mapped address on rpages */
15524c8ff709SChao Yu 	struct compress_data *cbuf;	/* virtual mapped address on cpages */
15534c8ff709SChao Yu 	size_t rlen;			/* valid data length in rbuf */
15544c8ff709SChao Yu 	size_t clen;			/* valid data length in cbuf */
15557f59b277SEric Biggers 
15567f59b277SEric Biggers 	/*
15577f59b277SEric Biggers 	 * The number of compressed pages remaining to be read in this cluster.
15587f59b277SEric Biggers 	 * This is initially nr_cpages.  It is decremented by 1 each time a page
15597f59b277SEric Biggers 	 * has been read (or failed to be read).  When it reaches 0, the cluster
15607f59b277SEric Biggers 	 * is decompressed (or an error is reported).
15617f59b277SEric Biggers 	 *
15627f59b277SEric Biggers 	 * If an error occurs before all the pages have been submitted for I/O,
15637f59b277SEric Biggers 	 * then this will never reach 0.  In this case the I/O submitter is
15647f59b277SEric Biggers 	 * responsible for calling f2fs_decompress_end_io() instead.
15657f59b277SEric Biggers 	 */
15667f59b277SEric Biggers 	atomic_t remaining_pages;
15677f59b277SEric Biggers 
15687f59b277SEric Biggers 	/*
15697f59b277SEric Biggers 	 * Number of references to this decompress_io_ctx.
15707f59b277SEric Biggers 	 *
15717f59b277SEric Biggers 	 * One reference is held for I/O completion.  This reference is dropped
15727f59b277SEric Biggers 	 * after the pagecache pages are updated and unlocked -- either after
15737f59b277SEric Biggers 	 * decompression (and verity if enabled), or after an error.
15747f59b277SEric Biggers 	 *
15757f59b277SEric Biggers 	 * In addition, each compressed page holds a reference while it is in a
15767f59b277SEric Biggers 	 * bio.  These references are necessary prevent compressed pages from
15777f59b277SEric Biggers 	 * being freed while they are still in a bio.
15787f59b277SEric Biggers 	 */
15797f59b277SEric Biggers 	refcount_t refcnt;
15807f59b277SEric Biggers 
15817f59b277SEric Biggers 	bool failed;			/* IO error occurred before decompression? */
15827f59b277SEric Biggers 	bool need_verity;		/* need fs-verity verification after decompression? */
158350cfa66fSChao Yu 	void *private;			/* payload buffer for specified decompression algorithm */
158450cfa66fSChao Yu 	void *private2;			/* extra payload buffer */
15857f59b277SEric Biggers 	struct work_struct verity_work;	/* work to verify the decompressed pages */
1586bff139b4SDaeho Jeong 	struct work_struct free_work;	/* work for late free this structure itself */
15874c8ff709SChao Yu };
15884c8ff709SChao Yu 
15894c8ff709SChao Yu #define NULL_CLUSTER			((unsigned int)(~0))
15904c8ff709SChao Yu #define MIN_COMPRESS_LOG_SIZE		2
15914c8ff709SChao Yu #define MAX_COMPRESS_LOG_SIZE		8
15920e2b7385SChao Yu #define MAX_COMPRESS_WINDOW_SIZE(log_size)	((PAGE_SIZE) << (log_size))
15934c8ff709SChao Yu 
159439a53e0cSJaegeuk Kim struct f2fs_sb_info {
159539a53e0cSJaegeuk Kim 	struct super_block *sb;			/* pointer to VFS super block */
15965e176d54SJaegeuk Kim 	struct proc_dir_entry *s_proc;		/* proc entry */
159739a53e0cSJaegeuk Kim 	struct f2fs_super_block *raw_super;	/* raw super block pointer */
1598e4544b63STim Murray 	struct f2fs_rwsem sb_lock;		/* lock for raw super block */
1599e8240f65SChao Yu 	int valid_super_block;			/* valid super block no */
1600fadb2fb8SChao Yu 	unsigned long s_flag;				/* flags for sbi */
1601853137ceSJaegeuk Kim 	struct mutex writepages;		/* mutex for writepages() */
160239a53e0cSJaegeuk Kim 
1603178053e2SDamien Le Moal #ifdef CONFIG_BLK_DEV_ZONED
1604178053e2SDamien Le Moal 	unsigned int blocks_per_blkz;		/* F2FS blocks per zone */
1605178053e2SDamien Le Moal 	unsigned int log_blocks_per_blkz;	/* log2 F2FS blocks per zone */
1606178053e2SDamien Le Moal #endif
1607178053e2SDamien Le Moal 
160839a53e0cSJaegeuk Kim 	/* for node-related operations */
160939a53e0cSJaegeuk Kim 	struct f2fs_nm_info *nm_info;		/* node manager */
161039a53e0cSJaegeuk Kim 	struct inode *node_inode;		/* cache node blocks */
161139a53e0cSJaegeuk Kim 
161239a53e0cSJaegeuk Kim 	/* for segment-related operations */
161339a53e0cSJaegeuk Kim 	struct f2fs_sm_info *sm_info;		/* segment manager */
16141ff7bd3bSJaegeuk Kim 
16151ff7bd3bSJaegeuk Kim 	/* for bio operations */
1616a912b54dSJaegeuk Kim 	struct f2fs_bio_info *write_io[NR_PAGE_TYPE];	/* for write bios */
1617107a805dSChao Yu 	/* keep migration IO order for LFS mode */
1618e4544b63STim Murray 	struct f2fs_rwsem io_order_lock;
16190a595ebaSJaegeuk Kim 	mempool_t *write_io_dummy;		/* Dummy pages */
1620a7b8618aSJaegeuk Kim 	pgoff_t page_eio_ofs[NR_PAGE_TYPE];	/* EIO page offset */
1621a7b8618aSJaegeuk Kim 	int page_eio_cnt[NR_PAGE_TYPE];		/* EIO count */
162239a53e0cSJaegeuk Kim 
162339a53e0cSJaegeuk Kim 	/* for checkpoint */
162439a53e0cSJaegeuk Kim 	struct f2fs_checkpoint *ckpt;		/* raw checkpoint pointer */
16258508e44aSJaegeuk Kim 	int cur_cp_pack;			/* remain current cp pack */
1626aaec2b1dSChao Yu 	spinlock_t cp_lock;			/* for flag in ckpt */
162739a53e0cSJaegeuk Kim 	struct inode *meta_inode;		/* cache meta blocks */
1628e4544b63STim Murray 	struct f2fs_rwsem cp_global_sem;	/* checkpoint procedure lock */
1629e4544b63STim Murray 	struct f2fs_rwsem cp_rwsem;		/* blocking FS operations */
1630e4544b63STim Murray 	struct f2fs_rwsem node_write;		/* locking node writes */
1631e4544b63STim Murray 	struct f2fs_rwsem node_change;	/* locking node change */
1632fb51b5efSChangman Lee 	wait_queue_head_t cp_wait;
16336beceb54SJaegeuk Kim 	unsigned long last_time[MAX_TIME];	/* to store time in jiffies */
16346beceb54SJaegeuk Kim 	long interval_time[MAX_TIME];		/* to store thresholds */
1635261eeb9cSDaeho Jeong 	struct ckpt_req_control cprc_info;	/* for checkpoint request control */
163639a53e0cSJaegeuk Kim 
163767298804SChao Yu 	struct inode_management im[MAX_INO_ENTRY];	/* manage inode cache */
16386451e041SJaegeuk Kim 
163950fa53ecSChao Yu 	spinlock_t fsync_node_lock;		/* for node entry lock */
164050fa53ecSChao Yu 	struct list_head fsync_node_list;	/* node list head */
164150fa53ecSChao Yu 	unsigned int fsync_seg_id;		/* sequence id */
164250fa53ecSChao Yu 	unsigned int fsync_node_num;		/* number of node entries */
164350fa53ecSChao Yu 
16446451e041SJaegeuk Kim 	/* for orphan inode, use 0'th array */
16450d47c1adSGu Zheng 	unsigned int max_orphans;		/* max orphan inodes */
164639a53e0cSJaegeuk Kim 
1647c227f912SChao Yu 	/* for inode management */
1648c227f912SChao Yu 	struct list_head inode_list[NR_INODE_TYPE];	/* dirty inode list */
1649c227f912SChao Yu 	spinlock_t inode_lock[NR_INODE_TYPE];	/* for dirty inode list lock */
1650040d2bb3SChao Yu 	struct mutex flush_lock;		/* for flush exclusion */
165139a53e0cSJaegeuk Kim 
165213054c54SChao Yu 	/* for extent tree cache */
1653e7547dacSJaegeuk Kim 	struct extent_tree_info extent_tree[NR_EXTENT_CACHES];
165471644dffSJaegeuk Kim 	atomic64_t allocated_data_blocks;	/* for block age extent_cache */
165571644dffSJaegeuk Kim 
165671644dffSJaegeuk Kim 	/* The threshold used for hot and warm data seperation*/
165771644dffSJaegeuk Kim 	unsigned int hot_data_age_threshold;
165871644dffSJaegeuk Kim 	unsigned int warm_data_age_threshold;
1659d23be468Sqixiaoyu1 	unsigned int last_age_weight;
166013054c54SChao Yu 
166139a53e0cSJaegeuk Kim 	/* basic filesystem units */
166239a53e0cSJaegeuk Kim 	unsigned int log_sectors_per_block;	/* log2 sectors per block */
166339a53e0cSJaegeuk Kim 	unsigned int log_blocksize;		/* log2 block size */
166439a53e0cSJaegeuk Kim 	unsigned int blocksize;			/* block size */
166539a53e0cSJaegeuk Kim 	unsigned int root_ino_num;		/* root inode number*/
166639a53e0cSJaegeuk Kim 	unsigned int node_ino_num;		/* node inode number*/
166739a53e0cSJaegeuk Kim 	unsigned int meta_ino_num;		/* meta inode number*/
166839a53e0cSJaegeuk Kim 	unsigned int log_blocks_per_seg;	/* log2 blocks per segment */
166939a53e0cSJaegeuk Kim 	unsigned int blocks_per_seg;		/* blocks per segment */
1670b771aadcSJaegeuk Kim 	unsigned int unusable_blocks_per_sec;	/* unusable blocks per section */
167139a53e0cSJaegeuk Kim 	unsigned int segs_per_sec;		/* segments per section */
167239a53e0cSJaegeuk Kim 	unsigned int secs_per_zone;		/* sections per zone */
167339a53e0cSJaegeuk Kim 	unsigned int total_sections;		/* total section count */
167439a53e0cSJaegeuk Kim 	unsigned int total_node_count;		/* total node block count */
167539a53e0cSJaegeuk Kim 	unsigned int total_valid_node_count;	/* valid node block count */
1676ab9fa662SJaegeuk Kim 	int dir_level;				/* directory level */
167766aee5aaSYuwei Guan 	bool readdir_ra;			/* readahead inode in readdir */
167810208567SJaegeuk Kim 	u64 max_io_bytes;			/* max io bytes to merge IOs */
167939a53e0cSJaegeuk Kim 
168039a53e0cSJaegeuk Kim 	block_t user_block_count;		/* # of user blocks */
168139a53e0cSJaegeuk Kim 	block_t total_valid_block_count;	/* # of valid blocks */
1682a66cdd98SJaegeuk Kim 	block_t discard_blks;			/* discard command candidats */
168339a53e0cSJaegeuk Kim 	block_t last_valid_block_count;		/* for recovery */
1684daeb433eSChao Yu 	block_t reserved_blocks;		/* configurable reserved blocks */
168580d42145SYunlong Song 	block_t current_reserved_blocks;	/* current reserved blocks */
1686daeb433eSChao Yu 
16874354994fSDaniel Rosenberg 	/* Additional tracking for no checkpoint mode */
16884354994fSDaniel Rosenberg 	block_t unusable_block_count;		/* # of blocks saved by last cp */
16894354994fSDaniel Rosenberg 
1690292c196aSChao Yu 	unsigned int nquota_files;		/* # of quota sysfile */
1691e4544b63STim Murray 	struct f2fs_rwsem quota_sem;		/* blocking cp for flags */
1692292c196aSChao Yu 
1693523be8a6SJaegeuk Kim 	/* # of pages, see count_type */
169435782b23SJaegeuk Kim 	atomic_t nr_pages[NR_COUNT_TYPE];
169541382ec4SJaegeuk Kim 	/* # of allocated blocks */
169641382ec4SJaegeuk Kim 	struct percpu_counter alloc_valid_block_count;
169747c8ebccSJaegeuk Kim 	/* # of node block writes as roll forward recovery */
169847c8ebccSJaegeuk Kim 	struct percpu_counter rf_node_block_count;
169939a53e0cSJaegeuk Kim 
1700687de7f1SJaegeuk Kim 	/* writeback control */
1701c29fd0c0SChao Yu 	atomic_t wb_sync_req[META];	/* count # of WB_SYNC threads */
1702687de7f1SJaegeuk Kim 
1703513c5f37SJaegeuk Kim 	/* valid inode count */
1704513c5f37SJaegeuk Kim 	struct percpu_counter total_valid_inode_count;
1705513c5f37SJaegeuk Kim 
170639a53e0cSJaegeuk Kim 	struct f2fs_mount_info mount_opt;	/* mount options */
170739a53e0cSJaegeuk Kim 
170839a53e0cSJaegeuk Kim 	/* for cleaning operations */
1709e4544b63STim Murray 	struct f2fs_rwsem gc_lock;		/*
1710fb24fea7SChao Yu 						 * semaphore for GC, avoid
1711fb24fea7SChao Yu 						 * race between GC and GC or CP
1712fb24fea7SChao Yu 						 */
171339a53e0cSJaegeuk Kim 	struct f2fs_gc_kthread	*gc_thread;	/* GC thread */
1714093749e2SChao Yu 	struct atgc_management am;		/* atgc management */
17155ec4e49fSJaegeuk Kim 	unsigned int cur_victim_sec;		/* current victim section num */
17165b0e9539SJaegeuk Kim 	unsigned int gc_mode;			/* current GC state */
1717e3080b01SChao Yu 	unsigned int next_victim_seg[2];	/* next segment in victim section */
1718e5a0db6aSYangtao Li 	spinlock_t gc_remaining_trials_lock;
1719e5a0db6aSYangtao Li 	/* remaining trial count for GC_URGENT_* and GC_IDLE_* */
1720e5a0db6aSYangtao Li 	unsigned int gc_remaining_trials;
17210e5e8111SDaeho Jeong 
17222ef79ecbSChao Yu 	/* for skip statistic */
17236f8d4455SJaegeuk Kim 	unsigned long long skipped_gc_rwsem;		/* FG_GC only */
172439a53e0cSJaegeuk Kim 
17251ad71a27SJaegeuk Kim 	/* threshold for gc trials on pinned files */
17261ad71a27SJaegeuk Kim 	u64 gc_pin_file_threshold;
1727e4544b63STim Murray 	struct f2fs_rwsem pin_sem;
17281ad71a27SJaegeuk Kim 
1729b1c57c1cSJaegeuk Kim 	/* maximum # of trials to find a victim segment for SSR and GC */
1730b1c57c1cSJaegeuk Kim 	unsigned int max_victim_search;
1731e3080b01SChao Yu 	/* migration granularity of garbage collection, unit: segment */
1732e3080b01SChao Yu 	unsigned int migration_granularity;
1733b1c57c1cSJaegeuk Kim 
173439a53e0cSJaegeuk Kim 	/*
173539a53e0cSJaegeuk Kim 	 * for stat information.
173639a53e0cSJaegeuk Kim 	 * one is for the LFS mode, and the other is for the SSR mode.
173739a53e0cSJaegeuk Kim 	 */
173835b09d82SNamjae Jeon #ifdef CONFIG_F2FS_STAT_FS
173939a53e0cSJaegeuk Kim 	struct f2fs_stat_info *stat_info;	/* FS status information */
1740b63e7be5SChao Yu 	atomic_t meta_count[META_MAX];		/* # of meta blocks */
174139a53e0cSJaegeuk Kim 	unsigned int segment_count[2];		/* # of allocated segments */
174239a53e0cSJaegeuk Kim 	unsigned int block_count[2];		/* # of allocated blocks */
1743b9a2c252SChangman Lee 	atomic_t inplace_count;		/* # of inplace update */
1744e7547dacSJaegeuk Kim 	/* # of lookup extent cache */
1745e7547dacSJaegeuk Kim 	atomic64_t total_hit_ext[NR_EXTENT_CACHES];
1746e7547dacSJaegeuk Kim 	/* # of hit rbtree extent node */
1747e7547dacSJaegeuk Kim 	atomic64_t read_hit_rbtree[NR_EXTENT_CACHES];
1748e7547dacSJaegeuk Kim 	/* # of hit cached extent node */
1749e7547dacSJaegeuk Kim 	atomic64_t read_hit_cached[NR_EXTENT_CACHES];
1750e7547dacSJaegeuk Kim 	/* # of hit largest extent node in read extent cache */
1751e7547dacSJaegeuk Kim 	atomic64_t read_hit_largest;
1752d5e8f6c9SChao Yu 	atomic_t inline_xattr;			/* # of inline_xattr inodes */
175303e14d52SChao Yu 	atomic_t inline_inode;			/* # of inline_data inodes */
175403e14d52SChao Yu 	atomic_t inline_dir;			/* # of inline_dentry inodes */
17554c8ff709SChao Yu 	atomic_t compr_inode;			/* # of compressed inodes */
1756ae999bb9SDaeho Jeong 	atomic64_t compr_blocks;		/* # of compressed blocks */
17578ec071c3SChao Yu 	atomic_t swapfile_inode;		/* # of swapfile inodes */
1758b4dac120SChao Yu 	atomic_t atomic_files;			/* # of opened atomic file */
175926a28a0cSJaegeuk Kim 	atomic_t max_aw_cnt;			/* max # of atomic writes */
1760274bd9baSChao Yu 	unsigned int io_skip_bggc;		/* skip background gc for in-flight IO */
1761274bd9baSChao Yu 	unsigned int other_skip_bggc;		/* skip background gc for other reasons */
176233fbd510SChao Yu 	unsigned int ndirty_inode[NR_INODE_TYPE];	/* # of dirty inodes */
176335b09d82SNamjae Jeon #endif
176439a53e0cSJaegeuk Kim 	spinlock_t stat_lock;			/* lock for stat operations */
1765b59d0baeSNamjae Jeon 
1766da9953b7SJaegeuk Kim 	/* to attach REQ_META|REQ_FUA flags */
1767da9953b7SJaegeuk Kim 	unsigned int data_io_flag;
176832b6aba8SJaegeuk Kim 	unsigned int node_io_flag;
1769b0af6d49SChao Yu 
1770759820c9SJulia Lawall 	/* For sysfs support */
17715d4daa57SChao Yu 	struct kobject s_kobj;			/* /sys/fs/f2fs/<devname> */
1772b59d0baeSNamjae Jeon 	struct completion s_kobj_unregister;
17732658e50dSJaegeuk Kim 
17745d4daa57SChao Yu 	struct kobject s_stat_kobj;		/* /sys/fs/f2fs/<devname>/stat */
17755d4daa57SChao Yu 	struct completion s_stat_kobj_unregister;
17765d4daa57SChao Yu 
17774c89b53dSJaegeuk Kim 	struct kobject s_feature_list_kobj;		/* /sys/fs/f2fs/<devname>/feature_list */
17784c89b53dSJaegeuk Kim 	struct completion s_feature_list_kobj_unregister;
17794c89b53dSJaegeuk Kim 
17802658e50dSJaegeuk Kim 	/* For shrinker support */
17812658e50dSJaegeuk Kim 	struct list_head s_list;
178271f2c820SChao Yu 	struct mutex umount_mutex;
178371f2c820SChao Yu 	unsigned int shrinker_run_no;
178471f2c820SChao Yu 
178571f2c820SChao Yu 	/* For multi devices */
17863c62be17SJaegeuk Kim 	int s_ndevs;				/* number of devices */
17873c62be17SJaegeuk Kim 	struct f2fs_dev_info *devs;		/* for device list */
17881228b482SChao Yu 	unsigned int dirty_device;		/* for checkpoint data flush */
17891228b482SChao Yu 	spinlock_t dev_lock;			/* protect dirty_device */
179071f2c820SChao Yu 	bool aligned_blksize;			/* all devices has the same logical blksize */
17918f1dbbbbSShuoran Liu 
17928f1dbbbbSShuoran Liu 	/* For write statistics */
17938f1dbbbbSShuoran Liu 	u64 sectors_written_start;
17948f1dbbbbSShuoran Liu 	u64 kbytes_written;
179543b6573bSKeith Mok 
179643b6573bSKeith Mok 	/* Reference to checksum algorithm driver via cryptoapi */
179743b6573bSKeith Mok 	struct crypto_shash *s_chksum_driver;
17981ecc0c5cSChao Yu 
1799704956ecSChao Yu 	/* Precomputed FS UUID checksum for seeding other checksums */
1800704956ecSChao Yu 	__u32 s_chksum_seed;
18014c8ff709SChao Yu 
18024c8ff709SChao Yu 	struct workqueue_struct *post_read_wq;	/* post read workqueue */
1803a999150fSChao Yu 
180495fa90c9SChao Yu 	unsigned char errors[MAX_F2FS_ERRORS];	/* error flags */
180595fa90c9SChao Yu 	spinlock_t error_lock;			/* protect errors array */
180695fa90c9SChao Yu 	bool error_dirty;			/* errors of sb is dirty */
180795fa90c9SChao Yu 
1808a999150fSChao Yu 	struct kmem_cache *inline_xattr_slab;	/* inline xattr entry */
1809a999150fSChao Yu 	unsigned int inline_xattr_slab_size;	/* default inline xattr slab size */
181031083031SChao Yu 
181107c6b593SDaeho Jeong 	/* For reclaimed segs statistics per each GC mode */
181207c6b593SDaeho Jeong 	unsigned int gc_segment_mode;		/* GC state for reclaimed segments */
181307c6b593SDaeho Jeong 	unsigned int gc_reclaimed_segs[MAX_GC_MODE];	/* Reclaimed segs for each mode */
181407c6b593SDaeho Jeong 
18150f6b56ecSDaeho Jeong 	unsigned long seq_file_ra_mul;		/* multiplier for ra_pages of seq. files in fadvise */
18160f6b56ecSDaeho Jeong 
18176691d940SDaeho Jeong 	int max_fragment_chunk;			/* max chunk size for block fragmentation mode */
18186691d940SDaeho Jeong 	int max_fragment_hole;			/* max hole size for block fragmentation mode */
18196691d940SDaeho Jeong 
1820f8e2f32bSDaeho Jeong 	/* For atomic write statistics */
1821f8e2f32bSDaeho Jeong 	atomic64_t current_atomic_write;
1822f8e2f32bSDaeho Jeong 	s64 peak_atomic_write;
1823f8e2f32bSDaeho Jeong 	u64 committed_atomic_block;
1824f8e2f32bSDaeho Jeong 	u64 revoked_atomic_block;
1825f8e2f32bSDaeho Jeong 
182631083031SChao Yu #ifdef CONFIG_F2FS_FS_COMPRESSION
182731083031SChao Yu 	struct kmem_cache *page_array_slab;	/* page array entry */
182831083031SChao Yu 	unsigned int page_array_slab_size;	/* default page array slab size */
18295ac443e2SDaeho Jeong 
18305ac443e2SDaeho Jeong 	/* For runtime compression statistics */
18315ac443e2SDaeho Jeong 	u64 compr_written_block;
18325ac443e2SDaeho Jeong 	u64 compr_saved_block;
18335ac443e2SDaeho Jeong 	u32 compr_new_inode;
18346ce19affSChao Yu 
18356ce19affSChao Yu 	/* For compressed block cache */
18366ce19affSChao Yu 	struct inode *compress_inode;		/* cache compressed blocks */
18376ce19affSChao Yu 	unsigned int compress_percent;		/* cache page percentage */
18386ce19affSChao Yu 	unsigned int compress_watermark;	/* cache page watermark */
18396ce19affSChao Yu 	atomic_t compress_page_hit;		/* cache hit count */
184031083031SChao Yu #endif
184152118743SDaeho Jeong 
184252118743SDaeho Jeong #ifdef CONFIG_F2FS_IOSTAT
184352118743SDaeho Jeong 	/* For app/fs IO statistics */
184452118743SDaeho Jeong 	spinlock_t iostat_lock;
18457a2b15cfSYangtao Li 	unsigned long long iostat_count[NR_IO_TYPE];
18467a2b15cfSYangtao Li 	unsigned long long iostat_bytes[NR_IO_TYPE];
18477a2b15cfSYangtao Li 	unsigned long long prev_iostat_bytes[NR_IO_TYPE];
184852118743SDaeho Jeong 	bool iostat_enable;
184952118743SDaeho Jeong 	unsigned long iostat_next_period;
185052118743SDaeho Jeong 	unsigned int iostat_period_ms;
1851a4b68176SDaeho Jeong 
1852a4b68176SDaeho Jeong 	/* For io latency related statistics info in one iostat period */
1853a4b68176SDaeho Jeong 	spinlock_t iostat_lat_lock;
1854a4b68176SDaeho Jeong 	struct iostat_lat_info *iostat_io_lat;
185552118743SDaeho Jeong #endif
185639a53e0cSJaegeuk Kim };
185739a53e0cSJaegeuk Kim 
18581ecc0c5cSChao Yu #ifdef CONFIG_F2FS_FAULT_INJECTION
1859c40e15a9SYangtao Li #define time_to_inject(sbi, type) __time_to_inject(sbi, type, __func__,	\
1860c40e15a9SYangtao Li 									__builtin_return_address(0))
1861c40e15a9SYangtao Li static inline bool __time_to_inject(struct f2fs_sb_info *sbi, int type,
1862c40e15a9SYangtao Li 				const char *func, const char *parent_func)
18631ecc0c5cSChao Yu {
186463189b78SChao Yu 	struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info;
18651ecc0c5cSChao Yu 
18661ecc0c5cSChao Yu 	if (!ffi->inject_rate)
18671ecc0c5cSChao Yu 		return false;
18681ecc0c5cSChao Yu 
18691ecc0c5cSChao Yu 	if (!IS_FAULT_SET(ffi, type))
18701ecc0c5cSChao Yu 		return false;
18711ecc0c5cSChao Yu 
18721ecc0c5cSChao Yu 	atomic_inc(&ffi->inject_ops);
18731ecc0c5cSChao Yu 	if (atomic_read(&ffi->inject_ops) >= ffi->inject_rate) {
18741ecc0c5cSChao Yu 		atomic_set(&ffi->inject_ops, 0);
1875c40e15a9SYangtao Li 		printk_ratelimited("%sF2FS-fs (%s) : inject %s in %s of %pS\n",
1876c40e15a9SYangtao Li 			KERN_INFO, sbi->sb->s_id, f2fs_fault_name[type],
1877c40e15a9SYangtao Li 			func, parent_func);
18781ecc0c5cSChao Yu 		return true;
18791ecc0c5cSChao Yu 	}
18801ecc0c5cSChao Yu 	return false;
18811ecc0c5cSChao Yu }
18827fa750a1SArnd Bergmann #else
18837fa750a1SArnd Bergmann static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
18847fa750a1SArnd Bergmann {
18857fa750a1SArnd Bergmann 	return false;
18867fa750a1SArnd Bergmann }
18871ecc0c5cSChao Yu #endif
18881ecc0c5cSChao Yu 
18890916878dSDamien Le Moal /*
18900916878dSDamien Le Moal  * Test if the mounted volume is a multi-device volume.
18910916878dSDamien Le Moal  *   - For a single regular disk volume, sbi->s_ndevs is 0.
18920916878dSDamien Le Moal  *   - For a single zoned disk volume, sbi->s_ndevs is 1.
18930916878dSDamien Le Moal  *   - For a multi-device volume, sbi->s_ndevs is always 2 or more.
18940916878dSDamien Le Moal  */
18950916878dSDamien Le Moal static inline bool f2fs_is_multi_device(struct f2fs_sb_info *sbi)
18960916878dSDamien Le Moal {
18970916878dSDamien Le Moal 	return sbi->s_ndevs > 1;
18980916878dSDamien Le Moal }
18990916878dSDamien Le Moal 
19006beceb54SJaegeuk Kim static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
19016beceb54SJaegeuk Kim {
1902a7d10cf3SSahitya Tummala 	unsigned long now = jiffies;
1903a7d10cf3SSahitya Tummala 
1904a7d10cf3SSahitya Tummala 	sbi->last_time[type] = now;
1905a7d10cf3SSahitya Tummala 
1906a7d10cf3SSahitya Tummala 	/* DISCARD_TIME and GC_TIME are based on REQ_TIME */
1907a7d10cf3SSahitya Tummala 	if (type == REQ_TIME) {
1908a7d10cf3SSahitya Tummala 		sbi->last_time[DISCARD_TIME] = now;
1909a7d10cf3SSahitya Tummala 		sbi->last_time[GC_TIME] = now;
1910a7d10cf3SSahitya Tummala 	}
19116beceb54SJaegeuk Kim }
19126beceb54SJaegeuk Kim 
19136beceb54SJaegeuk Kim static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type)
19146beceb54SJaegeuk Kim {
19156bccfa19SArnd Bergmann 	unsigned long interval = sbi->interval_time[type] * HZ;
19166beceb54SJaegeuk Kim 
19176beceb54SJaegeuk Kim 	return time_after(jiffies, sbi->last_time[type] + interval);
19186beceb54SJaegeuk Kim }
19196beceb54SJaegeuk Kim 
1920a7d10cf3SSahitya Tummala static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi,
1921a7d10cf3SSahitya Tummala 						int type)
1922a7d10cf3SSahitya Tummala {
1923a7d10cf3SSahitya Tummala 	unsigned long interval = sbi->interval_time[type] * HZ;
1924a7d10cf3SSahitya Tummala 	unsigned int wait_ms = 0;
1925a7d10cf3SSahitya Tummala 	long delta;
1926a7d10cf3SSahitya Tummala 
1927a7d10cf3SSahitya Tummala 	delta = (sbi->last_time[type] + interval) - jiffies;
1928a7d10cf3SSahitya Tummala 	if (delta > 0)
1929a7d10cf3SSahitya Tummala 		wait_ms = jiffies_to_msecs(delta);
1930a7d10cf3SSahitya Tummala 
1931a7d10cf3SSahitya Tummala 	return wait_ms;
1932a7d10cf3SSahitya Tummala }
1933a7d10cf3SSahitya Tummala 
193439a53e0cSJaegeuk Kim /*
193539a53e0cSJaegeuk Kim  * Inline functions
193639a53e0cSJaegeuk Kim  */
1937416d2dbbSChao Yu static inline u32 __f2fs_crc32(struct f2fs_sb_info *sbi, u32 crc,
1938704956ecSChao Yu 			      const void *address, unsigned int length)
1939704956ecSChao Yu {
1940704956ecSChao Yu 	struct {
1941704956ecSChao Yu 		struct shash_desc shash;
1942704956ecSChao Yu 		char ctx[4];
1943704956ecSChao Yu 	} desc;
1944704956ecSChao Yu 	int err;
1945704956ecSChao Yu 
1946704956ecSChao Yu 	BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx));
1947704956ecSChao Yu 
1948704956ecSChao Yu 	desc.shash.tfm = sbi->s_chksum_driver;
1949704956ecSChao Yu 	*(u32 *)desc.ctx = crc;
1950704956ecSChao Yu 
1951704956ecSChao Yu 	err = crypto_shash_update(&desc.shash, address, length);
1952704956ecSChao Yu 	BUG_ON(err);
1953704956ecSChao Yu 
1954704956ecSChao Yu 	return *(u32 *)desc.ctx;
1955704956ecSChao Yu }
1956704956ecSChao Yu 
1957416d2dbbSChao Yu static inline u32 f2fs_crc32(struct f2fs_sb_info *sbi, const void *address,
1958416d2dbbSChao Yu 			   unsigned int length)
1959416d2dbbSChao Yu {
1960416d2dbbSChao Yu 	return __f2fs_crc32(sbi, F2FS_SUPER_MAGIC, address, length);
1961416d2dbbSChao Yu }
1962416d2dbbSChao Yu 
1963416d2dbbSChao Yu static inline bool f2fs_crc_valid(struct f2fs_sb_info *sbi, __u32 blk_crc,
1964416d2dbbSChao Yu 				  void *buf, size_t buf_size)
1965416d2dbbSChao Yu {
1966416d2dbbSChao Yu 	return f2fs_crc32(sbi, buf, buf_size) == blk_crc;
1967416d2dbbSChao Yu }
1968416d2dbbSChao Yu 
1969416d2dbbSChao Yu static inline u32 f2fs_chksum(struct f2fs_sb_info *sbi, u32 crc,
1970416d2dbbSChao Yu 			      const void *address, unsigned int length)
1971416d2dbbSChao Yu {
1972416d2dbbSChao Yu 	return __f2fs_crc32(sbi, crc, address, length);
1973416d2dbbSChao Yu }
1974416d2dbbSChao Yu 
197539a53e0cSJaegeuk Kim static inline struct f2fs_inode_info *F2FS_I(struct inode *inode)
197639a53e0cSJaegeuk Kim {
197739a53e0cSJaegeuk Kim 	return container_of(inode, struct f2fs_inode_info, vfs_inode);
197839a53e0cSJaegeuk Kim }
197939a53e0cSJaegeuk Kim 
198039a53e0cSJaegeuk Kim static inline struct f2fs_sb_info *F2FS_SB(struct super_block *sb)
198139a53e0cSJaegeuk Kim {
198239a53e0cSJaegeuk Kim 	return sb->s_fs_info;
198339a53e0cSJaegeuk Kim }
198439a53e0cSJaegeuk Kim 
19854081363fSJaegeuk Kim static inline struct f2fs_sb_info *F2FS_I_SB(struct inode *inode)
19864081363fSJaegeuk Kim {
19874081363fSJaegeuk Kim 	return F2FS_SB(inode->i_sb);
19884081363fSJaegeuk Kim }
19894081363fSJaegeuk Kim 
19904081363fSJaegeuk Kim static inline struct f2fs_sb_info *F2FS_M_SB(struct address_space *mapping)
19914081363fSJaegeuk Kim {
19924081363fSJaegeuk Kim 	return F2FS_I_SB(mapping->host);
19934081363fSJaegeuk Kim }
19944081363fSJaegeuk Kim 
19954081363fSJaegeuk Kim static inline struct f2fs_sb_info *F2FS_P_SB(struct page *page)
19964081363fSJaegeuk Kim {
19974969c06aSJaegeuk Kim 	return F2FS_M_SB(page_file_mapping(page));
19984081363fSJaegeuk Kim }
19994081363fSJaegeuk Kim 
200039a53e0cSJaegeuk Kim static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi)
200139a53e0cSJaegeuk Kim {
200239a53e0cSJaegeuk Kim 	return (struct f2fs_super_block *)(sbi->raw_super);
200339a53e0cSJaegeuk Kim }
200439a53e0cSJaegeuk Kim 
200539a53e0cSJaegeuk Kim static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi)
200639a53e0cSJaegeuk Kim {
200739a53e0cSJaegeuk Kim 	return (struct f2fs_checkpoint *)(sbi->ckpt);
200839a53e0cSJaegeuk Kim }
200939a53e0cSJaegeuk Kim 
201045590710SGu Zheng static inline struct f2fs_node *F2FS_NODE(struct page *page)
201145590710SGu Zheng {
201245590710SGu Zheng 	return (struct f2fs_node *)page_address(page);
201345590710SGu Zheng }
201445590710SGu Zheng 
201558bfaf44SJaegeuk Kim static inline struct f2fs_inode *F2FS_INODE(struct page *page)
201658bfaf44SJaegeuk Kim {
201758bfaf44SJaegeuk Kim 	return &((struct f2fs_node *)page_address(page))->i;
201858bfaf44SJaegeuk Kim }
201958bfaf44SJaegeuk Kim 
202039a53e0cSJaegeuk Kim static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi)
202139a53e0cSJaegeuk Kim {
202239a53e0cSJaegeuk Kim 	return (struct f2fs_nm_info *)(sbi->nm_info);
202339a53e0cSJaegeuk Kim }
202439a53e0cSJaegeuk Kim 
202539a53e0cSJaegeuk Kim static inline struct f2fs_sm_info *SM_I(struct f2fs_sb_info *sbi)
202639a53e0cSJaegeuk Kim {
202739a53e0cSJaegeuk Kim 	return (struct f2fs_sm_info *)(sbi->sm_info);
202839a53e0cSJaegeuk Kim }
202939a53e0cSJaegeuk Kim 
203039a53e0cSJaegeuk Kim static inline struct sit_info *SIT_I(struct f2fs_sb_info *sbi)
203139a53e0cSJaegeuk Kim {
203239a53e0cSJaegeuk Kim 	return (struct sit_info *)(SM_I(sbi)->sit_info);
203339a53e0cSJaegeuk Kim }
203439a53e0cSJaegeuk Kim 
203539a53e0cSJaegeuk Kim static inline struct free_segmap_info *FREE_I(struct f2fs_sb_info *sbi)
203639a53e0cSJaegeuk Kim {
203739a53e0cSJaegeuk Kim 	return (struct free_segmap_info *)(SM_I(sbi)->free_info);
203839a53e0cSJaegeuk Kim }
203939a53e0cSJaegeuk Kim 
204039a53e0cSJaegeuk Kim static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi)
204139a53e0cSJaegeuk Kim {
204239a53e0cSJaegeuk Kim 	return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info);
204339a53e0cSJaegeuk Kim }
204439a53e0cSJaegeuk Kim 
20459df27d98SGu Zheng static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi)
20469df27d98SGu Zheng {
20479df27d98SGu Zheng 	return sbi->meta_inode->i_mapping;
20489df27d98SGu Zheng }
20499df27d98SGu Zheng 
20504ef51a8fSJaegeuk Kim static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi)
20514ef51a8fSJaegeuk Kim {
20524ef51a8fSJaegeuk Kim 	return sbi->node_inode->i_mapping;
20534ef51a8fSJaegeuk Kim }
20544ef51a8fSJaegeuk Kim 
2055caf0047eSChao Yu static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type)
205639a53e0cSJaegeuk Kim {
2057fadb2fb8SChao Yu 	return test_bit(type, &sbi->s_flag);
205839a53e0cSJaegeuk Kim }
205939a53e0cSJaegeuk Kim 
2060caf0047eSChao Yu static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
206139a53e0cSJaegeuk Kim {
2062fadb2fb8SChao Yu 	set_bit(type, &sbi->s_flag);
2063caf0047eSChao Yu }
2064caf0047eSChao Yu 
2065caf0047eSChao Yu static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
2066caf0047eSChao Yu {
2067fadb2fb8SChao Yu 	clear_bit(type, &sbi->s_flag);
206839a53e0cSJaegeuk Kim }
206939a53e0cSJaegeuk Kim 
2070d71b5564SJaegeuk Kim static inline unsigned long long cur_cp_version(struct f2fs_checkpoint *cp)
2071d71b5564SJaegeuk Kim {
2072d71b5564SJaegeuk Kim 	return le64_to_cpu(cp->checkpoint_ver);
2073d71b5564SJaegeuk Kim }
2074d71b5564SJaegeuk Kim 
2075ea676733SJaegeuk Kim static inline unsigned long f2fs_qf_ino(struct super_block *sb, int type)
2076ea676733SJaegeuk Kim {
2077ea676733SJaegeuk Kim 	if (type < F2FS_MAX_QUOTAS)
2078ea676733SJaegeuk Kim 		return le32_to_cpu(F2FS_SB(sb)->raw_super->qf_ino[type]);
2079ea676733SJaegeuk Kim 	return 0;
2080ea676733SJaegeuk Kim }
2081ea676733SJaegeuk Kim 
2082ced2c7eaSKinglong Mee static inline __u64 cur_cp_crc(struct f2fs_checkpoint *cp)
2083ced2c7eaSKinglong Mee {
2084ced2c7eaSKinglong Mee 	size_t crc_offset = le32_to_cpu(cp->checksum_offset);
2085ced2c7eaSKinglong Mee 	return le32_to_cpu(*((__le32 *)((unsigned char *)cp + crc_offset)));
2086ced2c7eaSKinglong Mee }
2087ced2c7eaSKinglong Mee 
2088aaec2b1dSChao Yu static inline bool __is_set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
208925ca923bSJaegeuk Kim {
209025ca923bSJaegeuk Kim 	unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags);
2091aaec2b1dSChao Yu 
209225ca923bSJaegeuk Kim 	return ckpt_flags & f;
209325ca923bSJaegeuk Kim }
209425ca923bSJaegeuk Kim 
2095aaec2b1dSChao Yu static inline bool is_set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
209625ca923bSJaegeuk Kim {
2097aaec2b1dSChao Yu 	return __is_set_ckpt_flags(F2FS_CKPT(sbi), f);
2098aaec2b1dSChao Yu }
2099aaec2b1dSChao Yu 
2100aaec2b1dSChao Yu static inline void __set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2101aaec2b1dSChao Yu {
2102aaec2b1dSChao Yu 	unsigned int ckpt_flags;
2103aaec2b1dSChao Yu 
2104aaec2b1dSChao Yu 	ckpt_flags = le32_to_cpu(cp->ckpt_flags);
210525ca923bSJaegeuk Kim 	ckpt_flags |= f;
210625ca923bSJaegeuk Kim 	cp->ckpt_flags = cpu_to_le32(ckpt_flags);
210725ca923bSJaegeuk Kim }
210825ca923bSJaegeuk Kim 
2109aaec2b1dSChao Yu static inline void set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
211025ca923bSJaegeuk Kim {
2111d1aa2453SChao Yu 	unsigned long flags;
2112d1aa2453SChao Yu 
2113d1aa2453SChao Yu 	spin_lock_irqsave(&sbi->cp_lock, flags);
2114aaec2b1dSChao Yu 	__set_ckpt_flags(F2FS_CKPT(sbi), f);
2115d1aa2453SChao Yu 	spin_unlock_irqrestore(&sbi->cp_lock, flags);
2116aaec2b1dSChao Yu }
2117aaec2b1dSChao Yu 
2118aaec2b1dSChao Yu static inline void __clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
2119aaec2b1dSChao Yu {
2120aaec2b1dSChao Yu 	unsigned int ckpt_flags;
2121aaec2b1dSChao Yu 
2122aaec2b1dSChao Yu 	ckpt_flags = le32_to_cpu(cp->ckpt_flags);
212325ca923bSJaegeuk Kim 	ckpt_flags &= (~f);
212425ca923bSJaegeuk Kim 	cp->ckpt_flags = cpu_to_le32(ckpt_flags);
212525ca923bSJaegeuk Kim }
212625ca923bSJaegeuk Kim 
2127aaec2b1dSChao Yu static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
2128aaec2b1dSChao Yu {
2129d1aa2453SChao Yu 	unsigned long flags;
2130d1aa2453SChao Yu 
2131d1aa2453SChao Yu 	spin_lock_irqsave(&sbi->cp_lock, flags);
2132aaec2b1dSChao Yu 	__clear_ckpt_flags(F2FS_CKPT(sbi), f);
2133d1aa2453SChao Yu 	spin_unlock_irqrestore(&sbi->cp_lock, flags);
2134aaec2b1dSChao Yu }
2135aaec2b1dSChao Yu 
2136c7f91bd4SBart Van Assche #define init_f2fs_rwsem(sem)					\
2137c7f91bd4SBart Van Assche do {								\
2138c7f91bd4SBart Van Assche 	static struct lock_class_key __key;			\
2139c7f91bd4SBart Van Assche 								\
2140c7f91bd4SBart Van Assche 	__init_f2fs_rwsem((sem), #sem, &__key);			\
2141c7f91bd4SBart Van Assche } while (0)
2142c7f91bd4SBart Van Assche 
2143c7f91bd4SBart Van Assche static inline void __init_f2fs_rwsem(struct f2fs_rwsem *sem,
2144c7f91bd4SBart Van Assche 		const char *sem_name, struct lock_class_key *key)
2145e4544b63STim Murray {
2146c7f91bd4SBart Van Assche 	__init_rwsem(&sem->internal_rwsem, sem_name, key);
21477f8e249dSJaegeuk Kim #ifdef CONFIG_F2FS_UNFAIR_RWSEM
2148e4544b63STim Murray 	init_waitqueue_head(&sem->read_waiters);
21497f8e249dSJaegeuk Kim #endif
2150e4544b63STim Murray }
2151e4544b63STim Murray 
2152e4544b63STim Murray static inline int f2fs_rwsem_is_locked(struct f2fs_rwsem *sem)
2153e4544b63STim Murray {
2154e4544b63STim Murray 	return rwsem_is_locked(&sem->internal_rwsem);
2155e4544b63STim Murray }
2156e4544b63STim Murray 
2157e4544b63STim Murray static inline int f2fs_rwsem_is_contended(struct f2fs_rwsem *sem)
2158e4544b63STim Murray {
2159e4544b63STim Murray 	return rwsem_is_contended(&sem->internal_rwsem);
2160e4544b63STim Murray }
2161e4544b63STim Murray 
2162e4544b63STim Murray static inline void f2fs_down_read(struct f2fs_rwsem *sem)
2163e4544b63STim Murray {
21647f8e249dSJaegeuk Kim #ifdef CONFIG_F2FS_UNFAIR_RWSEM
2165e4544b63STim Murray 	wait_event(sem->read_waiters, down_read_trylock(&sem->internal_rwsem));
21667f8e249dSJaegeuk Kim #else
21677f8e249dSJaegeuk Kim 	down_read(&sem->internal_rwsem);
21687f8e249dSJaegeuk Kim #endif
2169e4544b63STim Murray }
2170e4544b63STim Murray 
2171e4544b63STim Murray static inline int f2fs_down_read_trylock(struct f2fs_rwsem *sem)
2172e4544b63STim Murray {
2173e4544b63STim Murray 	return down_read_trylock(&sem->internal_rwsem);
2174e4544b63STim Murray }
2175e4544b63STim Murray 
2176e4544b63STim Murray #ifdef CONFIG_DEBUG_LOCK_ALLOC
2177e4544b63STim Murray static inline void f2fs_down_read_nested(struct f2fs_rwsem *sem, int subclass)
2178e4544b63STim Murray {
2179e4544b63STim Murray 	down_read_nested(&sem->internal_rwsem, subclass);
2180e4544b63STim Murray }
2181e4544b63STim Murray #else
2182e4544b63STim Murray #define f2fs_down_read_nested(sem, subclass) f2fs_down_read(sem)
2183e4544b63STim Murray #endif
2184e4544b63STim Murray 
2185e4544b63STim Murray static inline void f2fs_up_read(struct f2fs_rwsem *sem)
2186e4544b63STim Murray {
2187e4544b63STim Murray 	up_read(&sem->internal_rwsem);
2188e4544b63STim Murray }
2189e4544b63STim Murray 
2190e4544b63STim Murray static inline void f2fs_down_write(struct f2fs_rwsem *sem)
2191e4544b63STim Murray {
2192e4544b63STim Murray 	down_write(&sem->internal_rwsem);
2193e4544b63STim Murray }
2194e4544b63STim Murray 
2195e4544b63STim Murray static inline int f2fs_down_write_trylock(struct f2fs_rwsem *sem)
2196e4544b63STim Murray {
2197e4544b63STim Murray 	return down_write_trylock(&sem->internal_rwsem);
2198e4544b63STim Murray }
2199e4544b63STim Murray 
2200e4544b63STim Murray static inline void f2fs_up_write(struct f2fs_rwsem *sem)
2201e4544b63STim Murray {
2202e4544b63STim Murray 	up_write(&sem->internal_rwsem);
22037f8e249dSJaegeuk Kim #ifdef CONFIG_F2FS_UNFAIR_RWSEM
2204e4544b63STim Murray 	wake_up_all(&sem->read_waiters);
22057f8e249dSJaegeuk Kim #endif
2206e4544b63STim Murray }
2207e4544b63STim Murray 
2208e479556bSGu Zheng static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
220939a53e0cSJaegeuk Kim {
2210e4544b63STim Murray 	f2fs_down_read(&sbi->cp_rwsem);
221139a53e0cSJaegeuk Kim }
221239a53e0cSJaegeuk Kim 
2213cc15620bSJaegeuk Kim static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi)
2214cc15620bSJaegeuk Kim {
2215c40e15a9SYangtao Li 	if (time_to_inject(sbi, FAULT_LOCK_OP))
22163e020389SChao Yu 		return 0;
2217e4544b63STim Murray 	return f2fs_down_read_trylock(&sbi->cp_rwsem);
2218cc15620bSJaegeuk Kim }
2219cc15620bSJaegeuk Kim 
2220e479556bSGu Zheng static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi)
222139a53e0cSJaegeuk Kim {
2222e4544b63STim Murray 	f2fs_up_read(&sbi->cp_rwsem);
222339936837SJaegeuk Kim }
222439936837SJaegeuk Kim 
2225e479556bSGu Zheng static inline void f2fs_lock_all(struct f2fs_sb_info *sbi)
222639936837SJaegeuk Kim {
2227e4544b63STim Murray 	f2fs_down_write(&sbi->cp_rwsem);
222839936837SJaegeuk Kim }
222939936837SJaegeuk Kim 
2230e479556bSGu Zheng static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi)
223139936837SJaegeuk Kim {
2232e4544b63STim Murray 	f2fs_up_write(&sbi->cp_rwsem);
223339a53e0cSJaegeuk Kim }
223439a53e0cSJaegeuk Kim 
2235119ee914SJaegeuk Kim static inline int __get_cp_reason(struct f2fs_sb_info *sbi)
2236119ee914SJaegeuk Kim {
2237119ee914SJaegeuk Kim 	int reason = CP_SYNC;
2238119ee914SJaegeuk Kim 
2239119ee914SJaegeuk Kim 	if (test_opt(sbi, FASTBOOT))
2240119ee914SJaegeuk Kim 		reason = CP_FASTBOOT;
2241119ee914SJaegeuk Kim 	if (is_sbi_flag_set(sbi, SBI_IS_CLOSE))
2242119ee914SJaegeuk Kim 		reason = CP_UMOUNT;
2243119ee914SJaegeuk Kim 	return reason;
2244119ee914SJaegeuk Kim }
2245119ee914SJaegeuk Kim 
2246119ee914SJaegeuk Kim static inline bool __remain_node_summaries(int reason)
2247119ee914SJaegeuk Kim {
2248c473f1a9SChao Yu 	return (reason & (CP_UMOUNT | CP_FASTBOOT));
2249119ee914SJaegeuk Kim }
2250119ee914SJaegeuk Kim 
2251119ee914SJaegeuk Kim static inline bool __exist_node_summaries(struct f2fs_sb_info *sbi)
2252119ee914SJaegeuk Kim {
2253aaec2b1dSChao Yu 	return (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG) ||
2254aaec2b1dSChao Yu 			is_set_ckpt_flags(sbi, CP_FASTBOOT_FLAG));
2255119ee914SJaegeuk Kim }
2256119ee914SJaegeuk Kim 
225739a53e0cSJaegeuk Kim /*
225839a53e0cSJaegeuk Kim  * Check whether the inode has blocks or not
225939a53e0cSJaegeuk Kim  */
226039a53e0cSJaegeuk Kim static inline int F2FS_HAS_BLOCKS(struct inode *inode)
226139a53e0cSJaegeuk Kim {
22620eb0adadSChao Yu 	block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0;
22630eb0adadSChao Yu 
2264000519f2SChao Yu 	return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block;
226539a53e0cSJaegeuk Kim }
226639a53e0cSJaegeuk Kim 
22674bc8e9bcSChao Yu static inline bool f2fs_has_xattr_block(unsigned int ofs)
22684bc8e9bcSChao Yu {
22694bc8e9bcSChao Yu 	return ofs == XATTR_NODE_OFFSET;
22704bc8e9bcSChao Yu }
22714bc8e9bcSChao Yu 
2272d8a9a229SJaegeuk Kim static inline bool __allow_reserved_blocks(struct f2fs_sb_info *sbi,
2273a90a0884SJaegeuk Kim 					struct inode *inode, bool cap)
22747c2e5963SJaegeuk Kim {
2275d8a9a229SJaegeuk Kim 	if (!inode)
2276d8a9a229SJaegeuk Kim 		return true;
22777c2e5963SJaegeuk Kim 	if (!test_opt(sbi, RESERVE_ROOT))
22787c2e5963SJaegeuk Kim 		return false;
2279d8a9a229SJaegeuk Kim 	if (IS_NOQUOTA(inode))
2280d8a9a229SJaegeuk Kim 		return true;
228163189b78SChao Yu 	if (uid_eq(F2FS_OPTION(sbi).s_resuid, current_fsuid()))
22827c2e5963SJaegeuk Kim 		return true;
228363189b78SChao Yu 	if (!gid_eq(F2FS_OPTION(sbi).s_resgid, GLOBAL_ROOT_GID) &&
228463189b78SChao Yu 					in_group_p(F2FS_OPTION(sbi).s_resgid))
22857c2e5963SJaegeuk Kim 		return true;
2286a90a0884SJaegeuk Kim 	if (cap && capable(CAP_SYS_RESOURCE))
2287162b27aeSJaegeuk Kim 		return true;
22887c2e5963SJaegeuk Kim 	return false;
22897c2e5963SJaegeuk Kim }
22907c2e5963SJaegeuk Kim 
22910abd675eSChao Yu static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool);
22920abd675eSChao Yu static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
229346008c6dSChao Yu 				 struct inode *inode, blkcnt_t *count)
229439a53e0cSJaegeuk Kim {
22950abd675eSChao Yu 	blkcnt_t diff = 0, release = 0;
2296daeb433eSChao Yu 	block_t avail_user_block_count;
22970abd675eSChao Yu 	int ret;
22980abd675eSChao Yu 
22990abd675eSChao Yu 	ret = dquot_reserve_block(inode, *count);
23000abd675eSChao Yu 	if (ret)
23010abd675eSChao Yu 		return ret;
2302dd11a5dfSJaegeuk Kim 
230355523519SChao Yu 	if (time_to_inject(sbi, FAULT_BLOCK)) {
23040abd675eSChao Yu 		release = *count;
23059ea2f0beSChao Yu 		goto release_quota;
230655523519SChao Yu 	}
23077fa750a1SArnd Bergmann 
2308dd11a5dfSJaegeuk Kim 	/*
2309dd11a5dfSJaegeuk Kim 	 * let's increase this in prior to actual block count change in order
2310dd11a5dfSJaegeuk Kim 	 * for f2fs_sync_file to avoid data races when deciding checkpoint.
2311dd11a5dfSJaegeuk Kim 	 */
2312dd11a5dfSJaegeuk Kim 	percpu_counter_add(&sbi->alloc_valid_block_count, (*count));
231339a53e0cSJaegeuk Kim 
231439a53e0cSJaegeuk Kim 	spin_lock(&sbi->stat_lock);
23152555a2d5SJaegeuk Kim 	sbi->total_valid_block_count += (block_t)(*count);
231680d42145SYunlong Song 	avail_user_block_count = sbi->user_block_count -
231780d42145SYunlong Song 					sbi->current_reserved_blocks;
23187e65be49SJaegeuk Kim 
2319a90a0884SJaegeuk Kim 	if (!__allow_reserved_blocks(sbi, inode, true))
232063189b78SChao Yu 		avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
2321300a8429SChao Yu 
2322300a8429SChao Yu 	if (F2FS_IO_ALIGNED(sbi))
2323300a8429SChao Yu 		avail_user_block_count -= sbi->blocks_per_seg *
2324300a8429SChao Yu 				SM_I(sbi)->additional_reserved_segments;
2325300a8429SChao Yu 
2326a4c3ecaaSDaniel Rosenberg 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2327a4c3ecaaSDaniel Rosenberg 		if (avail_user_block_count > sbi->unusable_block_count)
23284354994fSDaniel Rosenberg 			avail_user_block_count -= sbi->unusable_block_count;
2329a4c3ecaaSDaniel Rosenberg 		else
2330a4c3ecaaSDaniel Rosenberg 			avail_user_block_count = 0;
2331a4c3ecaaSDaniel Rosenberg 	}
2332daeb433eSChao Yu 	if (unlikely(sbi->total_valid_block_count > avail_user_block_count)) {
2333daeb433eSChao Yu 		diff = sbi->total_valid_block_count - avail_user_block_count;
23347e65be49SJaegeuk Kim 		if (diff > *count)
23357e65be49SJaegeuk Kim 			diff = *count;
2336dd11a5dfSJaegeuk Kim 		*count -= diff;
23370abd675eSChao Yu 		release = diff;
23387e65be49SJaegeuk Kim 		sbi->total_valid_block_count -= diff;
233946008c6dSChao Yu 		if (!*count) {
234039a53e0cSJaegeuk Kim 			spin_unlock(&sbi->stat_lock);
23410abd675eSChao Yu 			goto enospc;
234239a53e0cSJaegeuk Kim 		}
234346008c6dSChao Yu 	}
234439a53e0cSJaegeuk Kim 	spin_unlock(&sbi->stat_lock);
234541382ec4SJaegeuk Kim 
234636b877afSDaniel Rosenberg 	if (unlikely(release)) {
234736b877afSDaniel Rosenberg 		percpu_counter_sub(&sbi->alloc_valid_block_count, release);
23480abd675eSChao Yu 		dquot_release_reservation_block(inode, release);
234936b877afSDaniel Rosenberg 	}
23500abd675eSChao Yu 	f2fs_i_blocks_write(inode, *count, true, true);
23510abd675eSChao Yu 	return 0;
23520abd675eSChao Yu 
23530abd675eSChao Yu enospc:
235436b877afSDaniel Rosenberg 	percpu_counter_sub(&sbi->alloc_valid_block_count, release);
23559ea2f0beSChao Yu release_quota:
23560abd675eSChao Yu 	dquot_release_reservation_block(inode, release);
23570abd675eSChao Yu 	return -ENOSPC;
235839a53e0cSJaegeuk Kim }
235939a53e0cSJaegeuk Kim 
2360dcbb4c10SJoe Perches __printf(2, 3)
2361dcbb4c10SJoe Perches void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...);
2362dcbb4c10SJoe Perches 
2363dcbb4c10SJoe Perches #define f2fs_err(sbi, fmt, ...)						\
2364dcbb4c10SJoe Perches 	f2fs_printk(sbi, KERN_ERR fmt, ##__VA_ARGS__)
2365dcbb4c10SJoe Perches #define f2fs_warn(sbi, fmt, ...)					\
2366dcbb4c10SJoe Perches 	f2fs_printk(sbi, KERN_WARNING fmt, ##__VA_ARGS__)
2367dcbb4c10SJoe Perches #define f2fs_notice(sbi, fmt, ...)					\
2368dcbb4c10SJoe Perches 	f2fs_printk(sbi, KERN_NOTICE fmt, ##__VA_ARGS__)
2369dcbb4c10SJoe Perches #define f2fs_info(sbi, fmt, ...)					\
2370dcbb4c10SJoe Perches 	f2fs_printk(sbi, KERN_INFO fmt, ##__VA_ARGS__)
2371dcbb4c10SJoe Perches #define f2fs_debug(sbi, fmt, ...)					\
2372dcbb4c10SJoe Perches 	f2fs_printk(sbi, KERN_DEBUG fmt, ##__VA_ARGS__)
2373dcbb4c10SJoe Perches 
2374da19b0dcSGu Zheng static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
237539a53e0cSJaegeuk Kim 						struct inode *inode,
23760eb0adadSChao Yu 						block_t count)
237739a53e0cSJaegeuk Kim {
23780eb0adadSChao Yu 	blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK;
23790eb0adadSChao Yu 
238039a53e0cSJaegeuk Kim 	spin_lock(&sbi->stat_lock);
23819850cf4aSJaegeuk Kim 	f2fs_bug_on(sbi, sbi->total_valid_block_count < (block_t) count);
238239a53e0cSJaegeuk Kim 	sbi->total_valid_block_count -= (block_t)count;
238380d42145SYunlong Song 	if (sbi->reserved_blocks &&
238480d42145SYunlong Song 		sbi->current_reserved_blocks < sbi->reserved_blocks)
238580d42145SYunlong Song 		sbi->current_reserved_blocks = min(sbi->reserved_blocks,
238680d42145SYunlong Song 					sbi->current_reserved_blocks + count);
238739a53e0cSJaegeuk Kim 	spin_unlock(&sbi->stat_lock);
23885e159cd3SChao Yu 	if (unlikely(inode->i_blocks < sectors)) {
2389dcbb4c10SJoe Perches 		f2fs_warn(sbi, "Inconsistent i_blocks, ino:%lu, iblocks:%llu, sectors:%llu",
23905e159cd3SChao Yu 			  inode->i_ino,
23915e159cd3SChao Yu 			  (unsigned long long)inode->i_blocks,
23925e159cd3SChao Yu 			  (unsigned long long)sectors);
23935e159cd3SChao Yu 		set_sbi_flag(sbi, SBI_NEED_FSCK);
23945e159cd3SChao Yu 		return;
23955e159cd3SChao Yu 	}
23960abd675eSChao Yu 	f2fs_i_blocks_write(inode, count, false, true);
239739a53e0cSJaegeuk Kim }
239839a53e0cSJaegeuk Kim 
239939a53e0cSJaegeuk Kim static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
240039a53e0cSJaegeuk Kim {
240135782b23SJaegeuk Kim 	atomic_inc(&sbi->nr_pages[count_type]);
24027c4abcbeSChao Yu 
240320109873SChao Yu 	if (count_type == F2FS_DIRTY_DENTS ||
240420109873SChao Yu 			count_type == F2FS_DIRTY_NODES ||
240520109873SChao Yu 			count_type == F2FS_DIRTY_META ||
240620109873SChao Yu 			count_type == F2FS_DIRTY_QDATA ||
240720109873SChao Yu 			count_type == F2FS_DIRTY_IMETA)
2408caf0047eSChao Yu 		set_sbi_flag(sbi, SBI_IS_DIRTY);
240939a53e0cSJaegeuk Kim }
241039a53e0cSJaegeuk Kim 
2411a7ffdbe2SJaegeuk Kim static inline void inode_inc_dirty_pages(struct inode *inode)
241239a53e0cSJaegeuk Kim {
2413204706c7SJaegeuk Kim 	atomic_inc(&F2FS_I(inode)->dirty_pages);
2414c227f912SChao Yu 	inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2415c227f912SChao Yu 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
24162c8a4a28SJaegeuk Kim 	if (IS_NOQUOTA(inode))
24172c8a4a28SJaegeuk Kim 		inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
241839a53e0cSJaegeuk Kim }
241939a53e0cSJaegeuk Kim 
242039a53e0cSJaegeuk Kim static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
242139a53e0cSJaegeuk Kim {
242235782b23SJaegeuk Kim 	atomic_dec(&sbi->nr_pages[count_type]);
242339a53e0cSJaegeuk Kim }
242439a53e0cSJaegeuk Kim 
2425a7ffdbe2SJaegeuk Kim static inline void inode_dec_dirty_pages(struct inode *inode)
242639a53e0cSJaegeuk Kim {
24275ac9f36fSChao Yu 	if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
24285ac9f36fSChao Yu 			!S_ISLNK(inode->i_mode))
24291fe54f9dSJaegeuk Kim 		return;
24301fe54f9dSJaegeuk Kim 
2431204706c7SJaegeuk Kim 	atomic_dec(&F2FS_I(inode)->dirty_pages);
2432c227f912SChao Yu 	dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2433c227f912SChao Yu 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
24342c8a4a28SJaegeuk Kim 	if (IS_NOQUOTA(inode))
24352c8a4a28SJaegeuk Kim 		dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
243639a53e0cSJaegeuk Kim }
243739a53e0cSJaegeuk Kim 
2438f8e2f32bSDaeho Jeong static inline void inc_atomic_write_cnt(struct inode *inode)
2439f8e2f32bSDaeho Jeong {
2440f8e2f32bSDaeho Jeong 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2441f8e2f32bSDaeho Jeong 	struct f2fs_inode_info *fi = F2FS_I(inode);
2442f8e2f32bSDaeho Jeong 	u64 current_write;
2443f8e2f32bSDaeho Jeong 
2444f8e2f32bSDaeho Jeong 	fi->atomic_write_cnt++;
2445f8e2f32bSDaeho Jeong 	atomic64_inc(&sbi->current_atomic_write);
2446f8e2f32bSDaeho Jeong 	current_write = atomic64_read(&sbi->current_atomic_write);
2447f8e2f32bSDaeho Jeong 	if (current_write > sbi->peak_atomic_write)
2448f8e2f32bSDaeho Jeong 		sbi->peak_atomic_write = current_write;
2449f8e2f32bSDaeho Jeong }
2450f8e2f32bSDaeho Jeong 
2451f8e2f32bSDaeho Jeong static inline void release_atomic_write_cnt(struct inode *inode)
2452f8e2f32bSDaeho Jeong {
2453f8e2f32bSDaeho Jeong 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2454f8e2f32bSDaeho Jeong 	struct f2fs_inode_info *fi = F2FS_I(inode);
2455f8e2f32bSDaeho Jeong 
2456f8e2f32bSDaeho Jeong 	atomic64_sub(fi->atomic_write_cnt, &sbi->current_atomic_write);
2457f8e2f32bSDaeho Jeong 	fi->atomic_write_cnt = 0;
2458f8e2f32bSDaeho Jeong }
2459f8e2f32bSDaeho Jeong 
2460523be8a6SJaegeuk Kim static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
246139a53e0cSJaegeuk Kim {
246235782b23SJaegeuk Kim 	return atomic_read(&sbi->nr_pages[count_type]);
246339a53e0cSJaegeuk Kim }
246439a53e0cSJaegeuk Kim 
2465204706c7SJaegeuk Kim static inline int get_dirty_pages(struct inode *inode)
2466f8b2c1f9SJaegeuk Kim {
2467204706c7SJaegeuk Kim 	return atomic_read(&F2FS_I(inode)->dirty_pages);
2468f8b2c1f9SJaegeuk Kim }
2469f8b2c1f9SJaegeuk Kim 
24705ac206cfSNamjae Jeon static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type)
24715ac206cfSNamjae Jeon {
24723519e3f9SChao Yu 	unsigned int pages_per_sec = sbi->segs_per_sec * sbi->blocks_per_seg;
2473523be8a6SJaegeuk Kim 	unsigned int segs = (get_pages(sbi, block_type) + pages_per_sec - 1) >>
2474523be8a6SJaegeuk Kim 						sbi->log_blocks_per_seg;
2475523be8a6SJaegeuk Kim 
2476523be8a6SJaegeuk Kim 	return segs / sbi->segs_per_sec;
24775ac206cfSNamjae Jeon }
24785ac206cfSNamjae Jeon 
247939a53e0cSJaegeuk Kim static inline block_t valid_user_blocks(struct f2fs_sb_info *sbi)
248039a53e0cSJaegeuk Kim {
24818b8343faSJaegeuk Kim 	return sbi->total_valid_block_count;
248239a53e0cSJaegeuk Kim }
248339a53e0cSJaegeuk Kim 
2484f83a2584SYunlei He static inline block_t discard_blocks(struct f2fs_sb_info *sbi)
2485f83a2584SYunlei He {
2486f83a2584SYunlei He 	return sbi->discard_blks;
2487f83a2584SYunlei He }
2488f83a2584SYunlei He 
248939a53e0cSJaegeuk Kim static inline unsigned long __bitmap_size(struct f2fs_sb_info *sbi, int flag)
249039a53e0cSJaegeuk Kim {
249139a53e0cSJaegeuk Kim 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
249239a53e0cSJaegeuk Kim 
249339a53e0cSJaegeuk Kim 	/* return NAT or SIT bitmap */
249439a53e0cSJaegeuk Kim 	if (flag == NAT_BITMAP)
249539a53e0cSJaegeuk Kim 		return le32_to_cpu(ckpt->nat_ver_bitmap_bytesize);
249639a53e0cSJaegeuk Kim 	else if (flag == SIT_BITMAP)
249739a53e0cSJaegeuk Kim 		return le32_to_cpu(ckpt->sit_ver_bitmap_bytesize);
249839a53e0cSJaegeuk Kim 
249939a53e0cSJaegeuk Kim 	return 0;
250039a53e0cSJaegeuk Kim }
250139a53e0cSJaegeuk Kim 
250255141486SWanpeng Li static inline block_t __cp_payload(struct f2fs_sb_info *sbi)
250355141486SWanpeng Li {
250455141486SWanpeng Li 	return le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
250555141486SWanpeng Li }
250655141486SWanpeng Li 
250739a53e0cSJaegeuk Kim static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
250839a53e0cSJaegeuk Kim {
250939a53e0cSJaegeuk Kim 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
25104260c406SGustavo A. R. Silva 	void *tmp_ptr = &ckpt->sit_nat_version_bitmap;
25111dbe4152SChangman Lee 	int offset;
25121dbe4152SChangman Lee 
2513199bc3feSChao Yu 	if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) {
2514199bc3feSChao Yu 		offset = (flag == SIT_BITMAP) ?
2515199bc3feSChao Yu 			le32_to_cpu(ckpt->nat_ver_bitmap_bytesize) : 0;
2516b471eb99SChao Yu 		/*
2517b471eb99SChao Yu 		 * if large_nat_bitmap feature is enabled, leave checksum
2518b471eb99SChao Yu 		 * protection for all nat/sit bitmaps.
2519b471eb99SChao Yu 		 */
25204260c406SGustavo A. R. Silva 		return tmp_ptr + offset + sizeof(__le32);
2521199bc3feSChao Yu 	}
2522199bc3feSChao Yu 
252355141486SWanpeng Li 	if (__cp_payload(sbi) > 0) {
25241dbe4152SChangman Lee 		if (flag == NAT_BITMAP)
2525280dfeaeSZhang Qilong 			return tmp_ptr;
25261dbe4152SChangman Lee 		else
252765b85cccSJaegeuk Kim 			return (unsigned char *)ckpt + F2FS_BLKSIZE;
25281dbe4152SChangman Lee 	} else {
25291dbe4152SChangman Lee 		offset = (flag == NAT_BITMAP) ?
253025ca923bSJaegeuk Kim 			le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
25314260c406SGustavo A. R. Silva 		return tmp_ptr + offset;
253239a53e0cSJaegeuk Kim 	}
25331dbe4152SChangman Lee }
253439a53e0cSJaegeuk Kim 
253539a53e0cSJaegeuk Kim static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi)
253639a53e0cSJaegeuk Kim {
25378508e44aSJaegeuk Kim 	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
253839a53e0cSJaegeuk Kim 
25398508e44aSJaegeuk Kim 	if (sbi->cur_cp_pack == 2)
254039a53e0cSJaegeuk Kim 		start_addr += sbi->blocks_per_seg;
254139a53e0cSJaegeuk Kim 	return start_addr;
254239a53e0cSJaegeuk Kim }
254339a53e0cSJaegeuk Kim 
25448508e44aSJaegeuk Kim static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi)
25458508e44aSJaegeuk Kim {
25468508e44aSJaegeuk Kim 	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
25478508e44aSJaegeuk Kim 
25488508e44aSJaegeuk Kim 	if (sbi->cur_cp_pack == 1)
25498508e44aSJaegeuk Kim 		start_addr += sbi->blocks_per_seg;
25508508e44aSJaegeuk Kim 	return start_addr;
25518508e44aSJaegeuk Kim }
25528508e44aSJaegeuk Kim 
25538508e44aSJaegeuk Kim static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi)
25548508e44aSJaegeuk Kim {
25558508e44aSJaegeuk Kim 	sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1;
25568508e44aSJaegeuk Kim }
25578508e44aSJaegeuk Kim 
255839a53e0cSJaegeuk Kim static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi)
255939a53e0cSJaegeuk Kim {
256039a53e0cSJaegeuk Kim 	return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
256139a53e0cSJaegeuk Kim }
256239a53e0cSJaegeuk Kim 
25633bac20a8SJaegeuk Kim extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
25640abd675eSChao Yu static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
2565000519f2SChao Yu 					struct inode *inode, bool is_inode)
256639a53e0cSJaegeuk Kim {
256739a53e0cSJaegeuk Kim 	block_t	valid_block_count;
2568a4c3ecaaSDaniel Rosenberg 	unsigned int valid_node_count, user_block_count;
2569af033b2aSChao Yu 	int err;
25700abd675eSChao Yu 
2571af033b2aSChao Yu 	if (is_inode) {
2572af033b2aSChao Yu 		if (inode) {
2573af033b2aSChao Yu 			err = dquot_alloc_inode(inode);
2574af033b2aSChao Yu 			if (err)
2575af033b2aSChao Yu 				return err;
2576af033b2aSChao Yu 		}
2577af033b2aSChao Yu 	} else {
2578af033b2aSChao Yu 		err = dquot_reserve_block(inode, 1);
2579af033b2aSChao Yu 		if (err)
2580af033b2aSChao Yu 			return err;
25810abd675eSChao Yu 	}
258239a53e0cSJaegeuk Kim 
2583c40e15a9SYangtao Li 	if (time_to_inject(sbi, FAULT_BLOCK))
2584812c6056SChao Yu 		goto enospc;
2585812c6056SChao Yu 
258639a53e0cSJaegeuk Kim 	spin_lock(&sbi->stat_lock);
258739a53e0cSJaegeuk Kim 
25887e65be49SJaegeuk Kim 	valid_block_count = sbi->total_valid_block_count +
25897e65be49SJaegeuk Kim 					sbi->current_reserved_blocks + 1;
25907e65be49SJaegeuk Kim 
2591a90a0884SJaegeuk Kim 	if (!__allow_reserved_blocks(sbi, inode, false))
259263189b78SChao Yu 		valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
2593300a8429SChao Yu 
2594300a8429SChao Yu 	if (F2FS_IO_ALIGNED(sbi))
2595300a8429SChao Yu 		valid_block_count += sbi->blocks_per_seg *
2596300a8429SChao Yu 				SM_I(sbi)->additional_reserved_segments;
2597300a8429SChao Yu 
2598a4c3ecaaSDaniel Rosenberg 	user_block_count = sbi->user_block_count;
25994354994fSDaniel Rosenberg 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2600a4c3ecaaSDaniel Rosenberg 		user_block_count -= sbi->unusable_block_count;
26017e65be49SJaegeuk Kim 
2602a4c3ecaaSDaniel Rosenberg 	if (unlikely(valid_block_count > user_block_count)) {
260339a53e0cSJaegeuk Kim 		spin_unlock(&sbi->stat_lock);
26040abd675eSChao Yu 		goto enospc;
260539a53e0cSJaegeuk Kim 	}
260639a53e0cSJaegeuk Kim 
2607ef86d709SGu Zheng 	valid_node_count = sbi->total_valid_node_count + 1;
2608cfb271d4SChao Yu 	if (unlikely(valid_node_count > sbi->total_node_count)) {
260939a53e0cSJaegeuk Kim 		spin_unlock(&sbi->stat_lock);
26100abd675eSChao Yu 		goto enospc;
261139a53e0cSJaegeuk Kim 	}
261239a53e0cSJaegeuk Kim 
2613ef86d709SGu Zheng 	sbi->total_valid_node_count++;
2614ef86d709SGu Zheng 	sbi->total_valid_block_count++;
261539a53e0cSJaegeuk Kim 	spin_unlock(&sbi->stat_lock);
261639a53e0cSJaegeuk Kim 
26170abd675eSChao Yu 	if (inode) {
26180abd675eSChao Yu 		if (is_inode)
26190abd675eSChao Yu 			f2fs_mark_inode_dirty_sync(inode, true);
26200abd675eSChao Yu 		else
26210abd675eSChao Yu 			f2fs_i_blocks_write(inode, 1, true, true);
26220abd675eSChao Yu 	}
26230abd675eSChao Yu 
262441382ec4SJaegeuk Kim 	percpu_counter_inc(&sbi->alloc_valid_block_count);
26250abd675eSChao Yu 	return 0;
26260abd675eSChao Yu 
26270abd675eSChao Yu enospc:
2628af033b2aSChao Yu 	if (is_inode) {
2629af033b2aSChao Yu 		if (inode)
2630af033b2aSChao Yu 			dquot_free_inode(inode);
2631af033b2aSChao Yu 	} else {
26320abd675eSChao Yu 		dquot_release_reservation_block(inode, 1);
2633af033b2aSChao Yu 	}
26340abd675eSChao Yu 	return -ENOSPC;
263539a53e0cSJaegeuk Kim }
263639a53e0cSJaegeuk Kim 
263739a53e0cSJaegeuk Kim static inline void dec_valid_node_count(struct f2fs_sb_info *sbi,
2638000519f2SChao Yu 					struct inode *inode, bool is_inode)
263939a53e0cSJaegeuk Kim {
264039a53e0cSJaegeuk Kim 	spin_lock(&sbi->stat_lock);
264139a53e0cSJaegeuk Kim 
26424d17e6feSChao Yu 	if (unlikely(!sbi->total_valid_block_count ||
26434d17e6feSChao Yu 			!sbi->total_valid_node_count)) {
26444d17e6feSChao Yu 		f2fs_warn(sbi, "dec_valid_node_count: inconsistent block counts, total_valid_block:%u, total_valid_node:%u",
26454d17e6feSChao Yu 			  sbi->total_valid_block_count,
26464d17e6feSChao Yu 			  sbi->total_valid_node_count);
26474d17e6feSChao Yu 		set_sbi_flag(sbi, SBI_NEED_FSCK);
26484d17e6feSChao Yu 	} else {
2649ef86d709SGu Zheng 		sbi->total_valid_block_count--;
26504d17e6feSChao Yu 		sbi->total_valid_node_count--;
26514d17e6feSChao Yu 	}
26524d17e6feSChao Yu 
265380d42145SYunlong Song 	if (sbi->reserved_blocks &&
265480d42145SYunlong Song 		sbi->current_reserved_blocks < sbi->reserved_blocks)
265580d42145SYunlong Song 		sbi->current_reserved_blocks++;
265639a53e0cSJaegeuk Kim 
265739a53e0cSJaegeuk Kim 	spin_unlock(&sbi->stat_lock);
26580abd675eSChao Yu 
2659ea6d7e72SChao Yu 	if (is_inode) {
2660af033b2aSChao Yu 		dquot_free_inode(inode);
2661ea6d7e72SChao Yu 	} else {
2662ea6d7e72SChao Yu 		if (unlikely(inode->i_blocks == 0)) {
2663097a7686SChao Yu 			f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%lu, iblocks:%llu",
2664ea6d7e72SChao Yu 				  inode->i_ino,
2665ea6d7e72SChao Yu 				  (unsigned long long)inode->i_blocks);
2666ea6d7e72SChao Yu 			set_sbi_flag(sbi, SBI_NEED_FSCK);
2667ea6d7e72SChao Yu 			return;
2668ea6d7e72SChao Yu 		}
26690abd675eSChao Yu 		f2fs_i_blocks_write(inode, 1, false, true);
267039a53e0cSJaegeuk Kim 	}
2671ea6d7e72SChao Yu }
267239a53e0cSJaegeuk Kim 
267339a53e0cSJaegeuk Kim static inline unsigned int valid_node_count(struct f2fs_sb_info *sbi)
267439a53e0cSJaegeuk Kim {
26758b8343faSJaegeuk Kim 	return sbi->total_valid_node_count;
267639a53e0cSJaegeuk Kim }
267739a53e0cSJaegeuk Kim 
267839a53e0cSJaegeuk Kim static inline void inc_valid_inode_count(struct f2fs_sb_info *sbi)
267939a53e0cSJaegeuk Kim {
2680513c5f37SJaegeuk Kim 	percpu_counter_inc(&sbi->total_valid_inode_count);
268139a53e0cSJaegeuk Kim }
268239a53e0cSJaegeuk Kim 
26830e80220aSJaegeuk Kim static inline void dec_valid_inode_count(struct f2fs_sb_info *sbi)
268439a53e0cSJaegeuk Kim {
2685513c5f37SJaegeuk Kim 	percpu_counter_dec(&sbi->total_valid_inode_count);
268639a53e0cSJaegeuk Kim }
268739a53e0cSJaegeuk Kim 
2688513c5f37SJaegeuk Kim static inline s64 valid_inode_count(struct f2fs_sb_info *sbi)
268939a53e0cSJaegeuk Kim {
2690513c5f37SJaegeuk Kim 	return percpu_counter_sum_positive(&sbi->total_valid_inode_count);
269139a53e0cSJaegeuk Kim }
269239a53e0cSJaegeuk Kim 
2693a56c7c6fSJaegeuk Kim static inline struct page *f2fs_grab_cache_page(struct address_space *mapping,
2694a56c7c6fSJaegeuk Kim 						pgoff_t index, bool for_write)
2695a56c7c6fSJaegeuk Kim {
269682cf4f13SChao Yu 	struct page *page;
2697df808180SMatthew Wilcox (Oracle) 	unsigned int flags;
2698cac5a3d8SDongOh Shin 
26997fa750a1SArnd Bergmann 	if (IS_ENABLED(CONFIG_F2FS_FAULT_INJECTION)) {
270082cf4f13SChao Yu 		if (!for_write)
270182cf4f13SChao Yu 			page = find_get_page_flags(mapping, index,
270282cf4f13SChao Yu 							FGP_LOCK | FGP_ACCESSED);
270382cf4f13SChao Yu 		else
270482cf4f13SChao Yu 			page = find_lock_page(mapping, index);
2705c41f3cc3SJaegeuk Kim 		if (page)
2706c41f3cc3SJaegeuk Kim 			return page;
2707c41f3cc3SJaegeuk Kim 
2708c40e15a9SYangtao Li 		if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC))
2709c41f3cc3SJaegeuk Kim 			return NULL;
271055523519SChao Yu 	}
27117fa750a1SArnd Bergmann 
2712a56c7c6fSJaegeuk Kim 	if (!for_write)
2713a56c7c6fSJaegeuk Kim 		return grab_cache_page(mapping, index);
2714df808180SMatthew Wilcox (Oracle) 
2715df808180SMatthew Wilcox (Oracle) 	flags = memalloc_nofs_save();
2716b7446e7cSMatthew Wilcox (Oracle) 	page = grab_cache_page_write_begin(mapping, index);
2717df808180SMatthew Wilcox (Oracle) 	memalloc_nofs_restore(flags);
2718df808180SMatthew Wilcox (Oracle) 
2719df808180SMatthew Wilcox (Oracle) 	return page;
2720a56c7c6fSJaegeuk Kim }
2721a56c7c6fSJaegeuk Kim 
272201eccef7SChao Yu static inline struct page *f2fs_pagecache_get_page(
272301eccef7SChao Yu 				struct address_space *mapping, pgoff_t index,
272401eccef7SChao Yu 				int fgp_flags, gfp_t gfp_mask)
272501eccef7SChao Yu {
2726c40e15a9SYangtao Li 	if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET))
272701eccef7SChao Yu 		return NULL;
27287fa750a1SArnd Bergmann 
272901eccef7SChao Yu 	return pagecache_get_page(mapping, index, fgp_flags, gfp_mask);
273001eccef7SChao Yu }
273101eccef7SChao Yu 
273239a53e0cSJaegeuk Kim static inline void f2fs_put_page(struct page *page, int unlock)
273339a53e0cSJaegeuk Kim {
2734031fa8ccSJaegeuk Kim 	if (!page)
273539a53e0cSJaegeuk Kim 		return;
273639a53e0cSJaegeuk Kim 
273739a53e0cSJaegeuk Kim 	if (unlock) {
27389850cf4aSJaegeuk Kim 		f2fs_bug_on(F2FS_P_SB(page), !PageLocked(page));
273939a53e0cSJaegeuk Kim 		unlock_page(page);
274039a53e0cSJaegeuk Kim 	}
274109cbfeafSKirill A. Shutemov 	put_page(page);
274239a53e0cSJaegeuk Kim }
274339a53e0cSJaegeuk Kim 
274439a53e0cSJaegeuk Kim static inline void f2fs_put_dnode(struct dnode_of_data *dn)
274539a53e0cSJaegeuk Kim {
274639a53e0cSJaegeuk Kim 	if (dn->node_page)
274739a53e0cSJaegeuk Kim 		f2fs_put_page(dn->node_page, 1);
274839a53e0cSJaegeuk Kim 	if (dn->inode_page && dn->node_page != dn->inode_page)
274939a53e0cSJaegeuk Kim 		f2fs_put_page(dn->inode_page, 0);
275039a53e0cSJaegeuk Kim 	dn->node_page = NULL;
275139a53e0cSJaegeuk Kim 	dn->inode_page = NULL;
275239a53e0cSJaegeuk Kim }
275339a53e0cSJaegeuk Kim 
275439a53e0cSJaegeuk Kim static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name,
2755e8512d2eSGu Zheng 					size_t size)
275639a53e0cSJaegeuk Kim {
2757e8512d2eSGu Zheng 	return kmem_cache_create(name, size, 0, SLAB_RECLAIM_ACCOUNT, NULL);
275839a53e0cSJaegeuk Kim }
275939a53e0cSJaegeuk Kim 
276032410577SChao Yu static inline void *f2fs_kmem_cache_alloc_nofail(struct kmem_cache *cachep,
27617bd59381SGu Zheng 						gfp_t flags)
27627bd59381SGu Zheng {
27637bd59381SGu Zheng 	void *entry;
27647bd59381SGu Zheng 
276580c54505SJaegeuk Kim 	entry = kmem_cache_alloc(cachep, flags);
276680c54505SJaegeuk Kim 	if (!entry)
276780c54505SJaegeuk Kim 		entry = kmem_cache_alloc(cachep, flags | __GFP_NOFAIL);
27687bd59381SGu Zheng 	return entry;
27697bd59381SGu Zheng }
27707bd59381SGu Zheng 
277132410577SChao Yu static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep,
277232410577SChao Yu 			gfp_t flags, bool nofail, struct f2fs_sb_info *sbi)
277332410577SChao Yu {
277432410577SChao Yu 	if (nofail)
277532410577SChao Yu 		return f2fs_kmem_cache_alloc_nofail(cachep, flags);
277632410577SChao Yu 
2777c40e15a9SYangtao Li 	if (time_to_inject(sbi, FAULT_SLAB_ALLOC))
277832410577SChao Yu 		return NULL;
277932410577SChao Yu 
278032410577SChao Yu 	return kmem_cache_alloc(cachep, flags);
278132410577SChao Yu }
278232410577SChao Yu 
2783493720a4SChao Yu static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
27845f9abab4SJaegeuk Kim {
27855f9abab4SJaegeuk Kim 	if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) ||
27865f9abab4SJaegeuk Kim 		get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) ||
2787fef4129eSChao Yu 		get_pages(sbi, F2FS_WB_CP_DATA) ||
2788fef4129eSChao Yu 		get_pages(sbi, F2FS_DIO_READ) ||
278911ac8ef8SJaegeuk Kim 		get_pages(sbi, F2FS_DIO_WRITE))
2790493720a4SChao Yu 		return true;
279111ac8ef8SJaegeuk Kim 
279256659ce8SSahitya Tummala 	if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
279311ac8ef8SJaegeuk Kim 			atomic_read(&SM_I(sbi)->dcc_info->queued_discard))
2794493720a4SChao Yu 		return true;
279511ac8ef8SJaegeuk Kim 
279611ac8ef8SJaegeuk Kim 	if (SM_I(sbi) && SM_I(sbi)->fcc_info &&
279772691af6SJaegeuk Kim 			atomic_read(&SM_I(sbi)->fcc_info->queued_flush))
2798493720a4SChao Yu 		return true;
2799493720a4SChao Yu 	return false;
2800493720a4SChao Yu }
2801493720a4SChao Yu 
2802493720a4SChao Yu static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2803493720a4SChao Yu {
2804493720a4SChao Yu 	if (sbi->gc_mode == GC_URGENT_HIGH)
2805493720a4SChao Yu 		return true;
2806493720a4SChao Yu 
2807493720a4SChao Yu 	if (is_inflight_io(sbi, type))
28085f9abab4SJaegeuk Kim 		return false;
280911ac8ef8SJaegeuk Kim 
2810d98af5f4SDaeho Jeong 	if (sbi->gc_mode == GC_URGENT_MID)
2811d98af5f4SDaeho Jeong 		return true;
2812d98af5f4SDaeho Jeong 
28130e5e8111SDaeho Jeong 	if (sbi->gc_mode == GC_URGENT_LOW &&
28140e5e8111SDaeho Jeong 			(type == DISCARD_TIME || type == GC_TIME))
28150e5e8111SDaeho Jeong 		return true;
28160e5e8111SDaeho Jeong 
28175f9abab4SJaegeuk Kim 	return f2fs_time_over(sbi, type);
28185f9abab4SJaegeuk Kim }
28195f9abab4SJaegeuk Kim 
28209be32d72SJaegeuk Kim static inline void f2fs_radix_tree_insert(struct radix_tree_root *root,
28219be32d72SJaegeuk Kim 				unsigned long index, void *item)
28229be32d72SJaegeuk Kim {
28239be32d72SJaegeuk Kim 	while (radix_tree_insert(root, index, item))
28249be32d72SJaegeuk Kim 		cond_resched();
28259be32d72SJaegeuk Kim }
28269be32d72SJaegeuk Kim 
282739a53e0cSJaegeuk Kim #define RAW_IS_INODE(p)	((p)->footer.nid == (p)->footer.ino)
282839a53e0cSJaegeuk Kim 
282939a53e0cSJaegeuk Kim static inline bool IS_INODE(struct page *page)
283039a53e0cSJaegeuk Kim {
283145590710SGu Zheng 	struct f2fs_node *p = F2FS_NODE(page);
2832cac5a3d8SDongOh Shin 
283339a53e0cSJaegeuk Kim 	return RAW_IS_INODE(p);
283439a53e0cSJaegeuk Kim }
283539a53e0cSJaegeuk Kim 
28367a2af766SChao Yu static inline int offset_in_addr(struct f2fs_inode *i)
28377a2af766SChao Yu {
28387a2af766SChao Yu 	return (i->i_inline & F2FS_EXTRA_ATTR) ?
28397a2af766SChao Yu 			(le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0;
28407a2af766SChao Yu }
28417a2af766SChao Yu 
284239a53e0cSJaegeuk Kim static inline __le32 *blkaddr_in_node(struct f2fs_node *node)
284339a53e0cSJaegeuk Kim {
284439a53e0cSJaegeuk Kim 	return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr;
284539a53e0cSJaegeuk Kim }
284639a53e0cSJaegeuk Kim 
28477a2af766SChao Yu static inline int f2fs_has_extra_attr(struct inode *inode);
2848a2ced1ceSChao Yu static inline block_t data_blkaddr(struct inode *inode,
28497a2af766SChao Yu 			struct page *node_page, unsigned int offset)
285039a53e0cSJaegeuk Kim {
285139a53e0cSJaegeuk Kim 	struct f2fs_node *raw_node;
285239a53e0cSJaegeuk Kim 	__le32 *addr_array;
28537a2af766SChao Yu 	int base = 0;
28547a2af766SChao Yu 	bool is_inode = IS_INODE(node_page);
2855cac5a3d8SDongOh Shin 
285645590710SGu Zheng 	raw_node = F2FS_NODE(node_page);
28577a2af766SChao Yu 
2858979f492fSLiFan 	if (is_inode) {
2859979f492fSLiFan 		if (!inode)
28607a88ddb5SChao Yu 			/* from GC path only */
28617a2af766SChao Yu 			base = offset_in_addr(&raw_node->i);
2862979f492fSLiFan 		else if (f2fs_has_extra_attr(inode))
28637a2af766SChao Yu 			base = get_extra_isize(inode);
28647a2af766SChao Yu 	}
28657a2af766SChao Yu 
286639a53e0cSJaegeuk Kim 	addr_array = blkaddr_in_node(raw_node);
28677a2af766SChao Yu 	return le32_to_cpu(addr_array[base + offset]);
286839a53e0cSJaegeuk Kim }
286939a53e0cSJaegeuk Kim 
2870a2ced1ceSChao Yu static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn)
2871a2ced1ceSChao Yu {
2872a2ced1ceSChao Yu 	return data_blkaddr(dn->inode, dn->node_page, dn->ofs_in_node);
2873a2ced1ceSChao Yu }
2874a2ced1ceSChao Yu 
287539a53e0cSJaegeuk Kim static inline int f2fs_test_bit(unsigned int nr, char *addr)
287639a53e0cSJaegeuk Kim {
287739a53e0cSJaegeuk Kim 	int mask;
287839a53e0cSJaegeuk Kim 
287939a53e0cSJaegeuk Kim 	addr += (nr >> 3);
2880447286ebSYangtao Li 	mask = BIT(7 - (nr & 0x07));
288139a53e0cSJaegeuk Kim 	return mask & *addr;
288239a53e0cSJaegeuk Kim }
288339a53e0cSJaegeuk Kim 
2884a66cdd98SJaegeuk Kim static inline void f2fs_set_bit(unsigned int nr, char *addr)
2885a66cdd98SJaegeuk Kim {
2886a66cdd98SJaegeuk Kim 	int mask;
2887a66cdd98SJaegeuk Kim 
2888a66cdd98SJaegeuk Kim 	addr += (nr >> 3);
2889447286ebSYangtao Li 	mask = BIT(7 - (nr & 0x07));
2890a66cdd98SJaegeuk Kim 	*addr |= mask;
2891a66cdd98SJaegeuk Kim }
2892a66cdd98SJaegeuk Kim 
2893a66cdd98SJaegeuk Kim static inline void f2fs_clear_bit(unsigned int nr, char *addr)
2894a66cdd98SJaegeuk Kim {
2895a66cdd98SJaegeuk Kim 	int mask;
2896a66cdd98SJaegeuk Kim 
2897a66cdd98SJaegeuk Kim 	addr += (nr >> 3);
2898447286ebSYangtao Li 	mask = BIT(7 - (nr & 0x07));
2899a66cdd98SJaegeuk Kim 	*addr &= ~mask;
2900a66cdd98SJaegeuk Kim }
2901a66cdd98SJaegeuk Kim 
290252aca074SGu Zheng static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr)
290339a53e0cSJaegeuk Kim {
290439a53e0cSJaegeuk Kim 	int mask;
290539a53e0cSJaegeuk Kim 	int ret;
290639a53e0cSJaegeuk Kim 
290739a53e0cSJaegeuk Kim 	addr += (nr >> 3);
2908447286ebSYangtao Li 	mask = BIT(7 - (nr & 0x07));
290939a53e0cSJaegeuk Kim 	ret = mask & *addr;
291039a53e0cSJaegeuk Kim 	*addr |= mask;
291139a53e0cSJaegeuk Kim 	return ret;
291239a53e0cSJaegeuk Kim }
291339a53e0cSJaegeuk Kim 
291452aca074SGu Zheng static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr)
291539a53e0cSJaegeuk Kim {
291639a53e0cSJaegeuk Kim 	int mask;
291739a53e0cSJaegeuk Kim 	int ret;
291839a53e0cSJaegeuk Kim 
291939a53e0cSJaegeuk Kim 	addr += (nr >> 3);
2920447286ebSYangtao Li 	mask = BIT(7 - (nr & 0x07));
292139a53e0cSJaegeuk Kim 	ret = mask & *addr;
292239a53e0cSJaegeuk Kim 	*addr &= ~mask;
292339a53e0cSJaegeuk Kim 	return ret;
292439a53e0cSJaegeuk Kim }
292539a53e0cSJaegeuk Kim 
2926c6ac4c0eSGu Zheng static inline void f2fs_change_bit(unsigned int nr, char *addr)
2927c6ac4c0eSGu Zheng {
2928c6ac4c0eSGu Zheng 	int mask;
2929c6ac4c0eSGu Zheng 
2930c6ac4c0eSGu Zheng 	addr += (nr >> 3);
2931447286ebSYangtao Li 	mask = BIT(7 - (nr & 0x07));
2932c6ac4c0eSGu Zheng 	*addr ^= mask;
2933c6ac4c0eSGu Zheng }
2934c6ac4c0eSGu Zheng 
293559c84408SChao Yu /*
293636098557SEric Biggers  * On-disk inode flags (f2fs_inode::i_flags)
293759c84408SChao Yu  */
29384c8ff709SChao Yu #define F2FS_COMPR_FL			0x00000004 /* Compress file */
293959c84408SChao Yu #define F2FS_SYNC_FL			0x00000008 /* Synchronous updates */
294059c84408SChao Yu #define F2FS_IMMUTABLE_FL		0x00000010 /* Immutable file */
294159c84408SChao Yu #define F2FS_APPEND_FL			0x00000020 /* writes to file may only append */
294259c84408SChao Yu #define F2FS_NODUMP_FL			0x00000040 /* do not dump file */
294359c84408SChao Yu #define F2FS_NOATIME_FL			0x00000080 /* do not update atime */
29444c8ff709SChao Yu #define F2FS_NOCOMP_FL			0x00000400 /* Don't compress */
294559c84408SChao Yu #define F2FS_INDEX_FL			0x00001000 /* hash-indexed directory */
294659c84408SChao Yu #define F2FS_DIRSYNC_FL			0x00010000 /* dirsync behaviour (directories only) */
294759c84408SChao Yu #define F2FS_PROJINHERIT_FL		0x20000000 /* Create with parents projid */
29482c2eb7a3SDaniel Rosenberg #define F2FS_CASEFOLD_FL		0x40000000 /* Casefolded file */
294959c84408SChao Yu 
295059c84408SChao Yu /* Flags that should be inherited by new inodes from their parent. */
295136098557SEric Biggers #define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \
29522c2eb7a3SDaniel Rosenberg 			   F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
2953787caf1bSSheng Yong 			   F2FS_CASEFOLD_FL)
295459c84408SChao Yu 
295559c84408SChao Yu /* Flags that are appropriate for regular files (all but dir-specific ones). */
29562c2eb7a3SDaniel Rosenberg #define F2FS_REG_FLMASK		(~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
29572c2eb7a3SDaniel Rosenberg 				F2FS_CASEFOLD_FL))
295859c84408SChao Yu 
295959c84408SChao Yu /* Flags that are appropriate for non-directories/regular files. */
296059c84408SChao Yu #define F2FS_OTHER_FLMASK	(F2FS_NODUMP_FL | F2FS_NOATIME_FL)
29615c57132eSChao Yu 
29625c57132eSChao Yu static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags)
29635c57132eSChao Yu {
29645c57132eSChao Yu 	if (S_ISDIR(mode))
29655c57132eSChao Yu 		return flags;
29665c57132eSChao Yu 	else if (S_ISREG(mode))
29675c57132eSChao Yu 		return flags & F2FS_REG_FLMASK;
29685c57132eSChao Yu 	else
29695c57132eSChao Yu 		return flags & F2FS_OTHER_FLMASK;
29705c57132eSChao Yu }
29715c57132eSChao Yu 
2972205b9822SJaegeuk Kim static inline void __mark_inode_dirty_flag(struct inode *inode,
2973205b9822SJaegeuk Kim 						int flag, bool set)
297439a53e0cSJaegeuk Kim {
2975205b9822SJaegeuk Kim 	switch (flag) {
2976205b9822SJaegeuk Kim 	case FI_INLINE_XATTR:
2977205b9822SJaegeuk Kim 	case FI_INLINE_DATA:
2978205b9822SJaegeuk Kim 	case FI_INLINE_DENTRY:
29799ac1e2d8SDaeho Jeong 	case FI_NEW_INODE:
2980205b9822SJaegeuk Kim 		if (set)
2981205b9822SJaegeuk Kim 			return;
2982df561f66SGustavo A. R. Silva 		fallthrough;
2983205b9822SJaegeuk Kim 	case FI_DATA_EXIST:
2984205b9822SJaegeuk Kim 	case FI_INLINE_DOTS:
29851ad71a27SJaegeuk Kim 	case FI_PIN_FILE:
2986c6140415SJaegeuk Kim 	case FI_COMPRESS_RELEASED:
29877c45729aSJaegeuk Kim 		f2fs_mark_inode_dirty_sync(inode, true);
2988205b9822SJaegeuk Kim 	}
298939a53e0cSJaegeuk Kim }
299039a53e0cSJaegeuk Kim 
299191942321SJaegeuk Kim static inline void set_inode_flag(struct inode *inode, int flag)
299239a53e0cSJaegeuk Kim {
299358f7e00fSYufen Yu 	set_bit(flag, F2FS_I(inode)->flags);
2994205b9822SJaegeuk Kim 	__mark_inode_dirty_flag(inode, flag, true);
299539a53e0cSJaegeuk Kim }
299639a53e0cSJaegeuk Kim 
299791942321SJaegeuk Kim static inline int is_inode_flag_set(struct inode *inode, int flag)
299839a53e0cSJaegeuk Kim {
29997653b9d8SChao Yu 	return test_bit(flag, F2FS_I(inode)->flags);
300039a53e0cSJaegeuk Kim }
300139a53e0cSJaegeuk Kim 
300291942321SJaegeuk Kim static inline void clear_inode_flag(struct inode *inode, int flag)
300339a53e0cSJaegeuk Kim {
300458f7e00fSYufen Yu 	clear_bit(flag, F2FS_I(inode)->flags);
3005205b9822SJaegeuk Kim 	__mark_inode_dirty_flag(inode, flag, false);
300639a53e0cSJaegeuk Kim }
300739a53e0cSJaegeuk Kim 
300895ae251fSEric Biggers static inline bool f2fs_verity_in_progress(struct inode *inode)
300995ae251fSEric Biggers {
301095ae251fSEric Biggers 	return IS_ENABLED(CONFIG_FS_VERITY) &&
301195ae251fSEric Biggers 	       is_inode_flag_set(inode, FI_VERITY_IN_PROGRESS);
301295ae251fSEric Biggers }
301395ae251fSEric Biggers 
301491942321SJaegeuk Kim static inline void set_acl_inode(struct inode *inode, umode_t mode)
3015444c580fSJaegeuk Kim {
301691942321SJaegeuk Kim 	F2FS_I(inode)->i_acl_mode = mode;
301791942321SJaegeuk Kim 	set_inode_flag(inode, FI_ACL_MODE);
30187c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, false);
301939a53e0cSJaegeuk Kim }
302039a53e0cSJaegeuk Kim 
3021a1961246SJaegeuk Kim static inline void f2fs_i_links_write(struct inode *inode, bool inc)
3022a1961246SJaegeuk Kim {
3023a1961246SJaegeuk Kim 	if (inc)
3024a1961246SJaegeuk Kim 		inc_nlink(inode);
3025a1961246SJaegeuk Kim 	else
3026a1961246SJaegeuk Kim 		drop_nlink(inode);
30277c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
3028a1961246SJaegeuk Kim }
3029a1961246SJaegeuk Kim 
30308edd03c8SJaegeuk Kim static inline void f2fs_i_blocks_write(struct inode *inode,
30310abd675eSChao Yu 					block_t diff, bool add, bool claim)
30328edd03c8SJaegeuk Kim {
303326de9b11SJaegeuk Kim 	bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
303426de9b11SJaegeuk Kim 	bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
303526de9b11SJaegeuk Kim 
30360abd675eSChao Yu 	/* add = 1, claim = 1 should be dquot_reserve_block in pair */
30370abd675eSChao Yu 	if (add) {
30380abd675eSChao Yu 		if (claim)
30390abd675eSChao Yu 			dquot_claim_block(inode, diff);
30400abd675eSChao Yu 		else
30410abd675eSChao Yu 			dquot_alloc_block_nofail(inode, diff);
30420abd675eSChao Yu 	} else {
30430abd675eSChao Yu 		dquot_free_block(inode, diff);
30440abd675eSChao Yu 	}
30450abd675eSChao Yu 
30467c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
304726de9b11SJaegeuk Kim 	if (clean || recover)
304826de9b11SJaegeuk Kim 		set_inode_flag(inode, FI_AUTO_RECOVER);
30498edd03c8SJaegeuk Kim }
30508edd03c8SJaegeuk Kim 
30514d8d45dfSDaeho Jeong static inline bool f2fs_is_atomic_file(struct inode *inode);
30524d8d45dfSDaeho Jeong 
3053fc9581c8SJaegeuk Kim static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size)
3054fc9581c8SJaegeuk Kim {
305526de9b11SJaegeuk Kim 	bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
305626de9b11SJaegeuk Kim 	bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
305726de9b11SJaegeuk Kim 
3058fc9581c8SJaegeuk Kim 	if (i_size_read(inode) == i_size)
3059fc9581c8SJaegeuk Kim 		return;
3060fc9581c8SJaegeuk Kim 
3061fc9581c8SJaegeuk Kim 	i_size_write(inode, i_size);
30624d8d45dfSDaeho Jeong 
30634d8d45dfSDaeho Jeong 	if (f2fs_is_atomic_file(inode))
30644d8d45dfSDaeho Jeong 		return;
30654d8d45dfSDaeho Jeong 
30667c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
306726de9b11SJaegeuk Kim 	if (clean || recover)
306826de9b11SJaegeuk Kim 		set_inode_flag(inode, FI_AUTO_RECOVER);
306926de9b11SJaegeuk Kim }
307026de9b11SJaegeuk Kim 
3071205b9822SJaegeuk Kim static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth)
3072205b9822SJaegeuk Kim {
3073205b9822SJaegeuk Kim 	F2FS_I(inode)->i_current_depth = depth;
30747c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
3075205b9822SJaegeuk Kim }
3076205b9822SJaegeuk Kim 
30771ad71a27SJaegeuk Kim static inline void f2fs_i_gc_failures_write(struct inode *inode,
30781ad71a27SJaegeuk Kim 					unsigned int count)
30791ad71a27SJaegeuk Kim {
30802ef79ecbSChao Yu 	F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN] = count;
30811ad71a27SJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
30821ad71a27SJaegeuk Kim }
30831ad71a27SJaegeuk Kim 
3084205b9822SJaegeuk Kim static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid)
3085205b9822SJaegeuk Kim {
3086205b9822SJaegeuk Kim 	F2FS_I(inode)->i_xattr_nid = xnid;
30877c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
3088205b9822SJaegeuk Kim }
3089205b9822SJaegeuk Kim 
3090205b9822SJaegeuk Kim static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino)
3091205b9822SJaegeuk Kim {
3092205b9822SJaegeuk Kim 	F2FS_I(inode)->i_pino = pino;
30937c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
3094205b9822SJaegeuk Kim }
3095205b9822SJaegeuk Kim 
309691942321SJaegeuk Kim static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri)
3097444c580fSJaegeuk Kim {
3098205b9822SJaegeuk Kim 	struct f2fs_inode_info *fi = F2FS_I(inode);
3099205b9822SJaegeuk Kim 
3100444c580fSJaegeuk Kim 	if (ri->i_inline & F2FS_INLINE_XATTR)
31017653b9d8SChao Yu 		set_bit(FI_INLINE_XATTR, fi->flags);
31021001b347SHuajun Li 	if (ri->i_inline & F2FS_INLINE_DATA)
31037653b9d8SChao Yu 		set_bit(FI_INLINE_DATA, fi->flags);
310434d67debSChao Yu 	if (ri->i_inline & F2FS_INLINE_DENTRY)
31057653b9d8SChao Yu 		set_bit(FI_INLINE_DENTRY, fi->flags);
3106b3d208f9SJaegeuk Kim 	if (ri->i_inline & F2FS_DATA_EXIST)
31077653b9d8SChao Yu 		set_bit(FI_DATA_EXIST, fi->flags);
3108510022a8SJaegeuk Kim 	if (ri->i_inline & F2FS_INLINE_DOTS)
31097653b9d8SChao Yu 		set_bit(FI_INLINE_DOTS, fi->flags);
31107a2af766SChao Yu 	if (ri->i_inline & F2FS_EXTRA_ATTR)
31117653b9d8SChao Yu 		set_bit(FI_EXTRA_ATTR, fi->flags);
31121ad71a27SJaegeuk Kim 	if (ri->i_inline & F2FS_PIN_FILE)
31137653b9d8SChao Yu 		set_bit(FI_PIN_FILE, fi->flags);
3114c6140415SJaegeuk Kim 	if (ri->i_inline & F2FS_COMPRESS_RELEASED)
3115c6140415SJaegeuk Kim 		set_bit(FI_COMPRESS_RELEASED, fi->flags);
3116444c580fSJaegeuk Kim }
3117444c580fSJaegeuk Kim 
311891942321SJaegeuk Kim static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri)
3119444c580fSJaegeuk Kim {
3120444c580fSJaegeuk Kim 	ri->i_inline = 0;
3121444c580fSJaegeuk Kim 
312291942321SJaegeuk Kim 	if (is_inode_flag_set(inode, FI_INLINE_XATTR))
3123444c580fSJaegeuk Kim 		ri->i_inline |= F2FS_INLINE_XATTR;
312491942321SJaegeuk Kim 	if (is_inode_flag_set(inode, FI_INLINE_DATA))
31251001b347SHuajun Li 		ri->i_inline |= F2FS_INLINE_DATA;
312691942321SJaegeuk Kim 	if (is_inode_flag_set(inode, FI_INLINE_DENTRY))
312734d67debSChao Yu 		ri->i_inline |= F2FS_INLINE_DENTRY;
312891942321SJaegeuk Kim 	if (is_inode_flag_set(inode, FI_DATA_EXIST))
3129b3d208f9SJaegeuk Kim 		ri->i_inline |= F2FS_DATA_EXIST;
313091942321SJaegeuk Kim 	if (is_inode_flag_set(inode, FI_INLINE_DOTS))
3131510022a8SJaegeuk Kim 		ri->i_inline |= F2FS_INLINE_DOTS;
31327a2af766SChao Yu 	if (is_inode_flag_set(inode, FI_EXTRA_ATTR))
31337a2af766SChao Yu 		ri->i_inline |= F2FS_EXTRA_ATTR;
31341ad71a27SJaegeuk Kim 	if (is_inode_flag_set(inode, FI_PIN_FILE))
31351ad71a27SJaegeuk Kim 		ri->i_inline |= F2FS_PIN_FILE;
3136c6140415SJaegeuk Kim 	if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED))
3137c6140415SJaegeuk Kim 		ri->i_inline |= F2FS_COMPRESS_RELEASED;
31387a2af766SChao Yu }
31397a2af766SChao Yu 
31407a2af766SChao Yu static inline int f2fs_has_extra_attr(struct inode *inode)
31417a2af766SChao Yu {
31427a2af766SChao Yu 	return is_inode_flag_set(inode, FI_EXTRA_ATTR);
3143444c580fSJaegeuk Kim }
3144444c580fSJaegeuk Kim 
3145987c7c31SChao Yu static inline int f2fs_has_inline_xattr(struct inode *inode)
3146987c7c31SChao Yu {
314791942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_INLINE_XATTR);
3148987c7c31SChao Yu }
3149987c7c31SChao Yu 
31504c8ff709SChao Yu static inline int f2fs_compressed_file(struct inode *inode)
31514c8ff709SChao Yu {
31524c8ff709SChao Yu 	return S_ISREG(inode->i_mode) &&
31534c8ff709SChao Yu 		is_inode_flag_set(inode, FI_COMPRESSED_FILE);
31544c8ff709SChao Yu }
31554c8ff709SChao Yu 
3156602a16d5SDaeho Jeong static inline bool f2fs_need_compress_data(struct inode *inode)
3157602a16d5SDaeho Jeong {
3158602a16d5SDaeho Jeong 	int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode;
3159602a16d5SDaeho Jeong 
3160602a16d5SDaeho Jeong 	if (!f2fs_compressed_file(inode))
3161602a16d5SDaeho Jeong 		return false;
3162602a16d5SDaeho Jeong 
3163602a16d5SDaeho Jeong 	if (compress_mode == COMPR_MODE_FS)
3164602a16d5SDaeho Jeong 		return true;
3165602a16d5SDaeho Jeong 	else if (compress_mode == COMPR_MODE_USER &&
3166602a16d5SDaeho Jeong 			is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
3167602a16d5SDaeho Jeong 		return true;
3168602a16d5SDaeho Jeong 
3169602a16d5SDaeho Jeong 	return false;
3170602a16d5SDaeho Jeong }
3171602a16d5SDaeho Jeong 
317281ca7350SChao Yu static inline unsigned int addrs_per_inode(struct inode *inode)
3173de93653fSJaegeuk Kim {
3174d02a6e61SChao Yu 	unsigned int addrs = CUR_ADDRS_PER_INODE(inode) -
3175d02a6e61SChao Yu 				get_inline_xattr_addrs(inode);
31764c8ff709SChao Yu 
31774c8ff709SChao Yu 	if (!f2fs_compressed_file(inode))
31784c8ff709SChao Yu 		return addrs;
31794c8ff709SChao Yu 	return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size);
3180d02a6e61SChao Yu }
3181d02a6e61SChao Yu 
3182d02a6e61SChao Yu static inline unsigned int addrs_per_block(struct inode *inode)
3183d02a6e61SChao Yu {
31844c8ff709SChao Yu 	if (!f2fs_compressed_file(inode))
31854c8ff709SChao Yu 		return DEF_ADDRS_PER_BLOCK;
31864c8ff709SChao Yu 	return ALIGN_DOWN(DEF_ADDRS_PER_BLOCK, F2FS_I(inode)->i_cluster_size);
3187de93653fSJaegeuk Kim }
3188de93653fSJaegeuk Kim 
31896afc662eSChao Yu static inline void *inline_xattr_addr(struct inode *inode, struct page *page)
319065985d93SJaegeuk Kim {
3191695fd1edSChao Yu 	struct f2fs_inode *ri = F2FS_INODE(page);
3192cac5a3d8SDongOh Shin 
319365985d93SJaegeuk Kim 	return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE -
3194b323fd28SChao Yu 					get_inline_xattr_addrs(inode)]);
319565985d93SJaegeuk Kim }
319665985d93SJaegeuk Kim 
319765985d93SJaegeuk Kim static inline int inline_xattr_size(struct inode *inode)
319865985d93SJaegeuk Kim {
3199622927f3SChao Yu 	if (f2fs_has_inline_xattr(inode))
32006afc662eSChao Yu 		return get_inline_xattr_addrs(inode) * sizeof(__le32);
3201622927f3SChao Yu 	return 0;
320265985d93SJaegeuk Kim }
320365985d93SJaegeuk Kim 
3204d9c454abSChao Liu /*
3205d9c454abSChao Liu  * Notice: check inline_data flag without inode page lock is unsafe.
3206d9c454abSChao Liu  * It could change at any time by f2fs_convert_inline_page().
3207d9c454abSChao Liu  */
32080dbdc2aeSJaegeuk Kim static inline int f2fs_has_inline_data(struct inode *inode)
32090dbdc2aeSJaegeuk Kim {
321091942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_INLINE_DATA);
32110dbdc2aeSJaegeuk Kim }
32120dbdc2aeSJaegeuk Kim 
3213b3d208f9SJaegeuk Kim static inline int f2fs_exist_data(struct inode *inode)
3214b3d208f9SJaegeuk Kim {
321591942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_DATA_EXIST);
3216b3d208f9SJaegeuk Kim }
3217b3d208f9SJaegeuk Kim 
3218510022a8SJaegeuk Kim static inline int f2fs_has_inline_dots(struct inode *inode)
3219510022a8SJaegeuk Kim {
322091942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_INLINE_DOTS);
3221510022a8SJaegeuk Kim }
3222510022a8SJaegeuk Kim 
32234c8ff709SChao Yu static inline int f2fs_is_mmap_file(struct inode *inode)
32244c8ff709SChao Yu {
32254c8ff709SChao Yu 	return is_inode_flag_set(inode, FI_MMAP_FILE);
32264c8ff709SChao Yu }
32274c8ff709SChao Yu 
32281ad71a27SJaegeuk Kim static inline bool f2fs_is_pinned_file(struct inode *inode)
32291ad71a27SJaegeuk Kim {
32301ad71a27SJaegeuk Kim 	return is_inode_flag_set(inode, FI_PIN_FILE);
32311ad71a27SJaegeuk Kim }
32321ad71a27SJaegeuk Kim 
323388b88a66SJaegeuk Kim static inline bool f2fs_is_atomic_file(struct inode *inode)
323488b88a66SJaegeuk Kim {
323591942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_ATOMIC_FILE);
323688b88a66SJaegeuk Kim }
323788b88a66SJaegeuk Kim 
32384a2c5b79SYe Bin static inline bool f2fs_is_cow_file(struct inode *inode)
32394a2c5b79SYe Bin {
32404a2c5b79SYe Bin 	return is_inode_flag_set(inode, FI_COW_FILE);
32414a2c5b79SYe Bin }
32424a2c5b79SYe Bin 
32433c6c2bebSJaegeuk Kim static inline bool f2fs_is_first_block_written(struct inode *inode)
32443c6c2bebSJaegeuk Kim {
324591942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_FIRST_BLOCK_WRITTEN);
32463c6c2bebSJaegeuk Kim }
32473c6c2bebSJaegeuk Kim 
32481e84371fSJaegeuk Kim static inline bool f2fs_is_drop_cache(struct inode *inode)
32491e84371fSJaegeuk Kim {
325091942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_DROP_CACHE);
32511e84371fSJaegeuk Kim }
32521e84371fSJaegeuk Kim 
3253f2470371SChao Yu static inline void *inline_data_addr(struct inode *inode, struct page *page)
32541001b347SHuajun Li {
3255695fd1edSChao Yu 	struct f2fs_inode *ri = F2FS_INODE(page);
32567a2af766SChao Yu 	int extra_size = get_extra_isize(inode);
3257cac5a3d8SDongOh Shin 
32587a2af766SChao Yu 	return (void *)&(ri->i_addr[extra_size + DEF_INLINE_RESERVED_SIZE]);
32591001b347SHuajun Li }
32601001b347SHuajun Li 
326134d67debSChao Yu static inline int f2fs_has_inline_dentry(struct inode *inode)
326234d67debSChao Yu {
326391942321SJaegeuk Kim 	return is_inode_flag_set(inode, FI_INLINE_DENTRY);
326434d67debSChao Yu }
326534d67debSChao Yu 
3266b5492af7SJaegeuk Kim static inline int is_file(struct inode *inode, int type)
3267b5492af7SJaegeuk Kim {
3268b5492af7SJaegeuk Kim 	return F2FS_I(inode)->i_advise & type;
3269b5492af7SJaegeuk Kim }
3270b5492af7SJaegeuk Kim 
3271b5492af7SJaegeuk Kim static inline void set_file(struct inode *inode, int type)
3272b5492af7SJaegeuk Kim {
3273766c6639SJaegeuk Kim 	if (is_file(inode, type))
3274766c6639SJaegeuk Kim 		return;
3275b5492af7SJaegeuk Kim 	F2FS_I(inode)->i_advise |= type;
32767c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
3277b5492af7SJaegeuk Kim }
3278b5492af7SJaegeuk Kim 
3279b5492af7SJaegeuk Kim static inline void clear_file(struct inode *inode, int type)
3280b5492af7SJaegeuk Kim {
3281766c6639SJaegeuk Kim 	if (!is_file(inode, type))
3282766c6639SJaegeuk Kim 		return;
3283b5492af7SJaegeuk Kim 	F2FS_I(inode)->i_advise &= ~type;
32847c45729aSJaegeuk Kim 	f2fs_mark_inode_dirty_sync(inode, true);
3285b5492af7SJaegeuk Kim }
3286b5492af7SJaegeuk Kim 
3287fe1897eaSChao Yu static inline bool f2fs_is_time_consistent(struct inode *inode)
3288fe1897eaSChao Yu {
3289fe1897eaSChao Yu 	if (!timespec64_equal(F2FS_I(inode)->i_disk_time, &inode->i_atime))
3290fe1897eaSChao Yu 		return false;
3291fe1897eaSChao Yu 	if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 1, &inode->i_ctime))
3292fe1897eaSChao Yu 		return false;
3293fe1897eaSChao Yu 	if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 2, &inode->i_mtime))
3294fe1897eaSChao Yu 		return false;
3295fe1897eaSChao Yu 	return true;
3296fe1897eaSChao Yu }
3297fe1897eaSChao Yu 
329826787236SJaegeuk Kim static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
329926787236SJaegeuk Kim {
3300a0d00fadSChao Yu 	bool ret;
3301a0d00fadSChao Yu 
330226787236SJaegeuk Kim 	if (dsync) {
330326787236SJaegeuk Kim 		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
330426787236SJaegeuk Kim 
330526787236SJaegeuk Kim 		spin_lock(&sbi->inode_lock[DIRTY_META]);
330626787236SJaegeuk Kim 		ret = list_empty(&F2FS_I(inode)->gdirty_list);
330726787236SJaegeuk Kim 		spin_unlock(&sbi->inode_lock[DIRTY_META]);
330826787236SJaegeuk Kim 		return ret;
330926787236SJaegeuk Kim 	}
331026787236SJaegeuk Kim 	if (!is_inode_flag_set(inode, FI_AUTO_RECOVER) ||
331126787236SJaegeuk Kim 			file_keep_isize(inode) ||
3312235831d7SJunling Zheng 			i_size_read(inode) & ~PAGE_MASK)
331326787236SJaegeuk Kim 		return false;
3314a0d00fadSChao Yu 
3315fe1897eaSChao Yu 	if (!f2fs_is_time_consistent(inode))
3316214c2461SJaegeuk Kim 		return false;
3317214c2461SJaegeuk Kim 
3318c10c9820SChao Yu 	spin_lock(&F2FS_I(inode)->i_size_lock);
3319a0d00fadSChao Yu 	ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
3320c10c9820SChao Yu 	spin_unlock(&F2FS_I(inode)->i_size_lock);
3321a0d00fadSChao Yu 
3322a0d00fadSChao Yu 	return ret;
3323267378d4SChao Yu }
3324267378d4SChao Yu 
3325deeedd71SChao Yu static inline bool f2fs_readonly(struct super_block *sb)
332677888c1eSJaegeuk Kim {
3327deeedd71SChao Yu 	return sb_rdonly(sb);
332877888c1eSJaegeuk Kim }
332977888c1eSJaegeuk Kim 
3330744602cfSJaegeuk Kim static inline bool f2fs_cp_error(struct f2fs_sb_info *sbi)
3331744602cfSJaegeuk Kim {
3332aaec2b1dSChao Yu 	return is_set_ckpt_flags(sbi, CP_ERROR_FLAG);
3333744602cfSJaegeuk Kim }
3334744602cfSJaegeuk Kim 
333543c780baSEric Biggers static inline bool is_dot_dotdot(const u8 *name, size_t len)
3336eaa693f4SJaegeuk Kim {
333743c780baSEric Biggers 	if (len == 1 && name[0] == '.')
3338eaa693f4SJaegeuk Kim 		return true;
3339eaa693f4SJaegeuk Kim 
334043c780baSEric Biggers 	if (len == 2 && name[0] == '.' && name[1] == '.')
3341eaa693f4SJaegeuk Kim 		return true;
3342eaa693f4SJaegeuk Kim 
3343eaa693f4SJaegeuk Kim 	return false;
3344eaa693f4SJaegeuk Kim }
3345eaa693f4SJaegeuk Kim 
33461ecc0c5cSChao Yu static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi,
33471ecc0c5cSChao Yu 					size_t size, gfp_t flags)
33480414b004SJaegeuk Kim {
3349c40e15a9SYangtao Li 	if (time_to_inject(sbi, FAULT_KMALLOC))
33502c63feadSJaegeuk Kim 		return NULL;
33517fa750a1SArnd Bergmann 
33520b6d4ca0SEric Biggers 	return kmalloc(size, flags);
33530414b004SJaegeuk Kim }
33540414b004SJaegeuk Kim 
3355acbf054dSChao Yu static inline void *f2fs_kzalloc(struct f2fs_sb_info *sbi,
3356acbf054dSChao Yu 					size_t size, gfp_t flags)
3357acbf054dSChao Yu {
3358acbf054dSChao Yu 	return f2fs_kmalloc(sbi, size, flags | __GFP_ZERO);
3359acbf054dSChao Yu }
3360acbf054dSChao Yu 
3361628b3d14SChao Yu static inline void *f2fs_kvmalloc(struct f2fs_sb_info *sbi,
3362628b3d14SChao Yu 					size_t size, gfp_t flags)
3363628b3d14SChao Yu {
3364c40e15a9SYangtao Li 	if (time_to_inject(sbi, FAULT_KVMALLOC))
3365628b3d14SChao Yu 		return NULL;
33667fa750a1SArnd Bergmann 
3367628b3d14SChao Yu 	return kvmalloc(size, flags);
3368628b3d14SChao Yu }
3369628b3d14SChao Yu 
3370628b3d14SChao Yu static inline void *f2fs_kvzalloc(struct f2fs_sb_info *sbi,
3371628b3d14SChao Yu 					size_t size, gfp_t flags)
3372628b3d14SChao Yu {
3373628b3d14SChao Yu 	return f2fs_kvmalloc(sbi, size, flags | __GFP_ZERO);
3374628b3d14SChao Yu }
3375628b3d14SChao Yu 
33767a2af766SChao Yu static inline int get_extra_isize(struct inode *inode)
3377f2470371SChao Yu {
33787a2af766SChao Yu 	return F2FS_I(inode)->i_extra_isize / sizeof(__le32);
3379f2470371SChao Yu }
3380f2470371SChao Yu 
33816afc662eSChao Yu static inline int get_inline_xattr_addrs(struct inode *inode)
33826afc662eSChao Yu {
33836afc662eSChao Yu 	return F2FS_I(inode)->i_inline_xattr_size;
33846afc662eSChao Yu }
33856afc662eSChao Yu 
33864d57b86dSChao Yu #define f2fs_get_inode_mode(i) \
338791942321SJaegeuk Kim 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
3388a6dda0e6SChristoph Hellwig 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
3389a6dda0e6SChristoph Hellwig 
33907a2af766SChao Yu #define F2FS_TOTAL_EXTRA_ATTR_SIZE			\
33917a2af766SChao Yu 	(offsetof(struct f2fs_inode, i_extra_end) -	\
33927a2af766SChao Yu 	offsetof(struct f2fs_inode, i_extra_isize))	\
33937a2af766SChao Yu 
33945c57132eSChao Yu #define F2FS_OLD_ATTRIBUTE_SIZE	(offsetof(struct f2fs_inode, i_addr))
33955c57132eSChao Yu #define F2FS_FITS_IN_INODE(f2fs_inode, extra_isize, field)		\
33962f84babfSSheng Yong 		((offsetof(typeof(*(f2fs_inode)), field) +	\
33975c57132eSChao Yu 		sizeof((f2fs_inode)->field))			\
33982f84babfSSheng Yong 		<= (F2FS_OLD_ATTRIBUTE_SIZE + (extra_isize)))	\
33995c57132eSChao Yu 
34002c70c5e3SChao Yu #define __is_large_section(sbi)		((sbi)->segs_per_sec > 1)
34012c70c5e3SChao Yu 
34026dc3a126SChao Yu #define __is_meta_io(fio) (PAGE_TYPE_OF_BIO((fio)->type) == META)
3403c9b60788SChao Yu 
3404e1da7872SChao Yu bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3405e1da7872SChao Yu 					block_t blkaddr, int type);
3406e1da7872SChao Yu static inline void verify_blkaddr(struct f2fs_sb_info *sbi,
3407e1da7872SChao Yu 					block_t blkaddr, int type)
3408e1da7872SChao Yu {
3409e1da7872SChao Yu 	if (!f2fs_is_valid_blkaddr(sbi, blkaddr, type)) {
3410dcbb4c10SJoe Perches 		f2fs_err(sbi, "invalid blkaddr: %u, type: %d, run fsck to fix.",
3411e1da7872SChao Yu 			 blkaddr, type);
3412e1da7872SChao Yu 		f2fs_bug_on(sbi, 1);
3413e1da7872SChao Yu 	}
3414e1da7872SChao Yu }
3415e1da7872SChao Yu 
3416e1da7872SChao Yu static inline bool __is_valid_data_blkaddr(block_t blkaddr)
34177b525dd0SChao Yu {
34184c8ff709SChao Yu 	if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR ||
34194c8ff709SChao Yu 			blkaddr == COMPRESS_ADDR)
34207b525dd0SChao Yu 		return false;
34217b525dd0SChao Yu 	return true;
34227b525dd0SChao Yu }
34237b525dd0SChao Yu 
342439a53e0cSJaegeuk Kim /*
342539a53e0cSJaegeuk Kim  * file.c
342639a53e0cSJaegeuk Kim  */
3427cac5a3d8SDongOh Shin int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync);
34284d57b86dSChao Yu void f2fs_truncate_data_blocks(struct dnode_of_data *dn);
34293265d3dbSChao Yu int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock);
3430c42d28ceSChao Yu int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock);
3431cac5a3d8SDongOh Shin int f2fs_truncate(struct inode *inode);
3432b74d24f7SChristian Brauner int f2fs_getattr(struct mnt_idmap *idmap, const struct path *path,
3433549c7297SChristian Brauner 		 struct kstat *stat, u32 request_mask, unsigned int flags);
3434c1632a0fSChristian Brauner int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
3435549c7297SChristian Brauner 		 struct iattr *attr);
34364d57b86dSChao Yu int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end);
34374d57b86dSChao Yu void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count);
3438c4020b2dSChao Yu int f2fs_precache_extents(struct inode *inode);
34399b1bb01cSMiklos Szeredi int f2fs_fileattr_get(struct dentry *dentry, struct fileattr *fa);
34408782a9aeSChristian Brauner int f2fs_fileattr_set(struct mnt_idmap *idmap,
34419b1bb01cSMiklos Szeredi 		      struct dentry *dentry, struct fileattr *fa);
3442cac5a3d8SDongOh Shin long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
3443cac5a3d8SDongOh Shin long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
344478130819SChao Yu int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid);
34451ad71a27SJaegeuk Kim int f2fs_pin_file_control(struct inode *inode, bool inc);
344639a53e0cSJaegeuk Kim 
344739a53e0cSJaegeuk Kim /*
344839a53e0cSJaegeuk Kim  * inode.c
344939a53e0cSJaegeuk Kim  */
3450cac5a3d8SDongOh Shin void f2fs_set_inode_flags(struct inode *inode);
3451704956ecSChao Yu bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct page *page);
3452704956ecSChao Yu void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct page *page);
3453cac5a3d8SDongOh Shin struct inode *f2fs_iget(struct super_block *sb, unsigned long ino);
3454cac5a3d8SDongOh Shin struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino);
34554d57b86dSChao Yu int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink);
34564d57b86dSChao Yu void f2fs_update_inode(struct inode *inode, struct page *node_page);
34574d57b86dSChao Yu void f2fs_update_inode_page(struct inode *inode);
3458cac5a3d8SDongOh Shin int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc);
3459cac5a3d8SDongOh Shin void f2fs_evict_inode(struct inode *inode);
34604d57b86dSChao Yu void f2fs_handle_failed_inode(struct inode *inode);
346139a53e0cSJaegeuk Kim 
346239a53e0cSJaegeuk Kim /*
346339a53e0cSJaegeuk Kim  * namei.c
346439a53e0cSJaegeuk Kim  */
34654d57b86dSChao Yu int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
3466b6a06cbbSChao Yu 							bool hot, bool set);
346739a53e0cSJaegeuk Kim struct dentry *f2fs_get_parent(struct dentry *child);
3468f2d40141SChristian Brauner int f2fs_get_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
34693db1de0eSDaeho Jeong 		     struct inode **new_inode);
347039a53e0cSJaegeuk Kim 
347139a53e0cSJaegeuk Kim /*
347239a53e0cSJaegeuk Kim  * dir.c
347339a53e0cSJaegeuk Kim  */
347443c780baSEric Biggers int f2fs_init_casefolded_name(const struct inode *dir,
347543c780baSEric Biggers 			      struct f2fs_filename *fname);
347643c780baSEric Biggers int f2fs_setup_filename(struct inode *dir, const struct qstr *iname,
347743c780baSEric Biggers 			int lookup, struct f2fs_filename *fname);
347843c780baSEric Biggers int f2fs_prepare_lookup(struct inode *dir, struct dentry *dentry,
347943c780baSEric Biggers 			struct f2fs_filename *fname);
348043c780baSEric Biggers void f2fs_free_filename(struct f2fs_filename *fname);
348143c780baSEric Biggers struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d,
348243c780baSEric Biggers 			const struct f2fs_filename *fname, int *max_slots);
3483cac5a3d8SDongOh Shin int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
3484cac5a3d8SDongOh Shin 			unsigned int start_pos, struct fscrypt_str *fstr);
34854d57b86dSChao Yu void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
3486cac5a3d8SDongOh Shin 			struct f2fs_dentry_ptr *d);
34874d57b86dSChao Yu struct page *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
348843c780baSEric Biggers 			const struct f2fs_filename *fname, struct page *dpage);
34894d57b86dSChao Yu void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode,
3490cac5a3d8SDongOh Shin 			unsigned int current_depth);
34914d57b86dSChao Yu int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots);
3492cac5a3d8SDongOh Shin void f2fs_drop_nlink(struct inode *dir, struct inode *inode);
3493cac5a3d8SDongOh Shin struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
349443c780baSEric Biggers 					 const struct f2fs_filename *fname,
349543c780baSEric Biggers 					 struct page **res_page);
3496cac5a3d8SDongOh Shin struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
3497cac5a3d8SDongOh Shin 			const struct qstr *child, struct page **res_page);
3498cac5a3d8SDongOh Shin struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct page **p);
3499cac5a3d8SDongOh Shin ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr,
3500cac5a3d8SDongOh Shin 			struct page **page);
3501cac5a3d8SDongOh Shin void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
3502cac5a3d8SDongOh Shin 			struct page *page, struct inode *inode);
3503b06af2afSJaegeuk Kim bool f2fs_has_enough_room(struct inode *dir, struct page *ipage,
350443c780baSEric Biggers 			  const struct f2fs_filename *fname);
3505cac5a3d8SDongOh Shin void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d,
350643c780baSEric Biggers 			const struct fscrypt_str *name, f2fs_hash_t name_hash,
3507cac5a3d8SDongOh Shin 			unsigned int bit_pos);
350843c780baSEric Biggers int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
3509cac5a3d8SDongOh Shin 			struct inode *inode, nid_t ino, umode_t mode);
351043c780baSEric Biggers int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname,
3511cac5a3d8SDongOh Shin 			struct inode *inode, nid_t ino, umode_t mode);
35124d57b86dSChao Yu int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
3513cac5a3d8SDongOh Shin 			struct inode *inode, nid_t ino, umode_t mode);
3514cac5a3d8SDongOh Shin void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
3515cac5a3d8SDongOh Shin 			struct inode *dir, struct inode *inode);
3516cac5a3d8SDongOh Shin int f2fs_do_tmpfile(struct inode *inode, struct inode *dir);
3517cac5a3d8SDongOh Shin bool f2fs_empty_dir(struct inode *dir);
351839a53e0cSJaegeuk Kim 
3519b7f7a5e0SAl Viro static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode)
3520b7f7a5e0SAl Viro {
3521bfc2b7e8SEric Biggers 	if (fscrypt_is_nokey_name(dentry))
3522bfc2b7e8SEric Biggers 		return -ENOKEY;
35234d57b86dSChao Yu 	return f2fs_do_add_link(d_inode(dentry->d_parent), &dentry->d_name,
3524510022a8SJaegeuk Kim 				inode, inode->i_ino, inode->i_mode);
3525b7f7a5e0SAl Viro }
3526b7f7a5e0SAl Viro 
352739a53e0cSJaegeuk Kim /*
352839a53e0cSJaegeuk Kim  * super.c
352939a53e0cSJaegeuk Kim  */
3530cac5a3d8SDongOh Shin int f2fs_inode_dirtied(struct inode *inode, bool sync);
3531cac5a3d8SDongOh Shin void f2fs_inode_synced(struct inode *inode);
353210a26878SChao Yu int f2fs_dquot_initialize(struct inode *inode);
3533ea676733SJaegeuk Kim int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
3534af033b2aSChao Yu int f2fs_quota_sync(struct super_block *sb, int type);
35356d1451bfSChengguang Xu loff_t max_file_blocks(struct inode *inode);
35364b2414d0SChao Yu void f2fs_quota_off_umount(struct super_block *sb);
3537a9cfee0eSChao Yu void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason);
35381aa161e4SJaegeuk Kim void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag);
353995fa90c9SChao Yu void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error);
3540cac5a3d8SDongOh Shin int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
3541cac5a3d8SDongOh Shin int f2fs_sync_fs(struct super_block *sb, int sync);
35424d57b86dSChao Yu int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
354339a53e0cSJaegeuk Kim 
354439a53e0cSJaegeuk Kim /*
354539a53e0cSJaegeuk Kim  * hash.c
354639a53e0cSJaegeuk Kim  */
354743c780baSEric Biggers void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname);
354839a53e0cSJaegeuk Kim 
354939a53e0cSJaegeuk Kim /*
355039a53e0cSJaegeuk Kim  * node.c
355139a53e0cSJaegeuk Kim  */
355239a53e0cSJaegeuk Kim struct node_info;
355339a53e0cSJaegeuk Kim 
35544d57b86dSChao Yu int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
35554d57b86dSChao Yu bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type);
355650fa53ecSChao Yu bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct page *page);
355750fa53ecSChao Yu void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi);
355850fa53ecSChao Yu void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page);
355950fa53ecSChao Yu void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi);
35604d57b86dSChao Yu int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid);
35614d57b86dSChao Yu bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
35624d57b86dSChao Yu bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino);
35637735730dSChao Yu int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
3564a9419b63SJaegeuk Kim 				struct node_info *ni, bool checkpoint_context);
35654d57b86dSChao Yu pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
35664d57b86dSChao Yu int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
35674d57b86dSChao Yu int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
35684d57b86dSChao Yu int f2fs_truncate_xattr_node(struct inode *inode);
356950fa53ecSChao Yu int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
357050fa53ecSChao Yu 					unsigned int seq_id);
357194c821fbSChao Yu bool f2fs_nat_bitmap_enabled(struct f2fs_sb_info *sbi);
35724d57b86dSChao Yu int f2fs_remove_inode_page(struct inode *inode);
35734d57b86dSChao Yu struct page *f2fs_new_inode_page(struct inode *inode);
35744d57b86dSChao Yu struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs);
35754d57b86dSChao Yu void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid);
35764d57b86dSChao Yu struct page *f2fs_get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid);
35774d57b86dSChao Yu struct page *f2fs_get_node_page_ra(struct page *parent, int start);
357848018b4cSChao Yu int f2fs_move_node_page(struct page *node_page, int gc_type);
357968e79bafSJia Yang void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
35804d57b86dSChao Yu int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
358150fa53ecSChao Yu 			struct writeback_control *wbc, bool atomic,
358250fa53ecSChao Yu 			unsigned int *seq_id);
35834d57b86dSChao Yu int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
35844d57b86dSChao Yu 			struct writeback_control *wbc,
3585b0af6d49SChao Yu 			bool do_balance, enum iostat_type io_type);
3586e2374015SChao Yu int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount);
35874d57b86dSChao Yu bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid);
35884d57b86dSChao Yu void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid);
35894d57b86dSChao Yu void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid);
35904d57b86dSChao Yu int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink);
35919627a7b3SChao Yu int f2fs_recover_inline_xattr(struct inode *inode, struct page *page);
35924d57b86dSChao Yu int f2fs_recover_xattr_data(struct inode *inode, struct page *page);
35934d57b86dSChao Yu int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct page *page);
35947735730dSChao Yu int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
3595cac5a3d8SDongOh Shin 			unsigned int segno, struct f2fs_summary_block *sum);
359694c821fbSChao Yu void f2fs_enable_nat_bits(struct f2fs_sb_info *sbi);
3597edc55aafSJaegeuk Kim int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
35984d57b86dSChao Yu int f2fs_build_node_manager(struct f2fs_sb_info *sbi);
35994d57b86dSChao Yu void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi);
36004d57b86dSChao Yu int __init f2fs_create_node_manager_caches(void);
36014d57b86dSChao Yu void f2fs_destroy_node_manager_caches(void);
360239a53e0cSJaegeuk Kim 
360339a53e0cSJaegeuk Kim /*
360439a53e0cSJaegeuk Kim  * segment.c
360539a53e0cSJaegeuk Kim  */
36064d57b86dSChao Yu bool f2fs_need_SSR(struct f2fs_sb_info *sbi);
36073db1de0eSDaeho Jeong int f2fs_commit_atomic_write(struct inode *inode);
36083db1de0eSDaeho Jeong void f2fs_abort_atomic_write(struct inode *inode, bool clean);
3609cac5a3d8SDongOh Shin void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need);
36107bcd0cfaSChao Yu void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg);
361139d787beSChao Yu int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino);
36124d57b86dSChao Yu int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi);
36131228b482SChao Yu int f2fs_flush_device_cache(struct f2fs_sb_info *sbi);
36144d57b86dSChao Yu void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free);
36154d57b86dSChao Yu void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr);
36164d57b86dSChao Yu bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr);
36174d674904SFengnan Chang int f2fs_start_discard_thread(struct f2fs_sb_info *sbi);
36184d57b86dSChao Yu void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi);
36194d57b86dSChao Yu void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi);
362003f2c02dSJaegeuk Kim bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi);
36214d57b86dSChao Yu void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
36224d57b86dSChao Yu 					struct cp_control *cpc);
36234354994fSDaniel Rosenberg void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi);
36244d3aed70SDaniel Rosenberg block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi);
36254d3aed70SDaniel Rosenberg int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
36264d57b86dSChao Yu void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
36274d57b86dSChao Yu int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
362861461fc9SChao Yu bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno);
3629093749e2SChao Yu void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
3630093749e2SChao Yu void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi);
3631093749e2SChao Yu void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi);
3632093749e2SChao Yu void f2fs_get_new_segment(struct f2fs_sb_info *sbi,
3633093749e2SChao Yu 			unsigned int *newseg, bool new_sec, int dir);
36340ef81833SChao Yu void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
363504f0b2eaSQiuyang Sun 					unsigned int start, unsigned int end);
3636509f1010SChao Yu void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force);
3637901d745fSChao Yu void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
3638cac5a3d8SDongOh Shin int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
36394d57b86dSChao Yu bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
36404d57b86dSChao Yu 					struct cp_control *cpc);
36414d57b86dSChao Yu struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno);
36424d57b86dSChao Yu void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src,
36434d57b86dSChao Yu 					block_t blk_addr);
36444d57b86dSChao Yu void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3645b0af6d49SChao Yu 						enum iostat_type io_type);
36464d57b86dSChao Yu void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio);
36474d57b86dSChao Yu void f2fs_outplace_write_data(struct dnode_of_data *dn,
36484d57b86dSChao Yu 			struct f2fs_io_info *fio);
36494d57b86dSChao Yu int f2fs_inplace_write_data(struct f2fs_io_info *fio);
36504d57b86dSChao Yu void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3651cac5a3d8SDongOh Shin 			block_t old_blkaddr, block_t new_blkaddr,
3652c5d02785SChao Yu 			bool recover_curseg, bool recover_newaddr,
3653c5d02785SChao Yu 			bool from_gc);
3654cac5a3d8SDongOh Shin void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3655cac5a3d8SDongOh Shin 			block_t old_addr, block_t new_addr,
3656cac5a3d8SDongOh Shin 			unsigned char version, bool recover_curseg,
3657cac5a3d8SDongOh Shin 			bool recover_newaddr);
36584d57b86dSChao Yu void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3659cac5a3d8SDongOh Shin 			block_t old_blkaddr, block_t *new_blkaddr,
3660fb830fc5SChao Yu 			struct f2fs_summary *sum, int type,
3661f608c38cSChao Yu 			struct f2fs_io_info *fio);
366271f2c820SChao Yu void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
366371f2c820SChao Yu 					block_t blkaddr, unsigned int blkcnt);
3664cac5a3d8SDongOh Shin void f2fs_wait_on_page_writeback(struct page *page,
3665bae0ee7aSChao Yu 			enum page_type type, bool ordered, bool locked);
36660ded69f6SJaegeuk Kim void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
36671e78e8bdSSahitya Tummala void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
36681e78e8bdSSahitya Tummala 								block_t len);
36694d57b86dSChao Yu void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
36704d57b86dSChao Yu void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
36714d57b86dSChao Yu int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3672cac5a3d8SDongOh Shin 			unsigned int val, int alloc);
36734d57b86dSChao Yu void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3674c426d991SShin'ichiro Kawasaki int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi);
3675d508c94eSShin'ichiro Kawasaki int f2fs_check_write_pointer(struct f2fs_sb_info *sbi);
36764d57b86dSChao Yu int f2fs_build_segment_manager(struct f2fs_sb_info *sbi);
36774d57b86dSChao Yu void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi);
36784d57b86dSChao Yu int __init f2fs_create_segment_manager_caches(void);
36794d57b86dSChao Yu void f2fs_destroy_segment_manager_caches(void);
36804d57b86dSChao Yu int f2fs_rw_hint_to_seg_type(enum rw_hint hint);
3681de881df9SAravind Ramesh unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
3682de881df9SAravind Ramesh 			unsigned int segno);
3683de881df9SAravind Ramesh unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
3684de881df9SAravind Ramesh 			unsigned int segno);
368539a53e0cSJaegeuk Kim 
36866691d940SDaeho Jeong #define DEF_FRAGMENT_SIZE	4
36876691d940SDaeho Jeong #define MIN_FRAGMENT_SIZE	1
36886691d940SDaeho Jeong #define MAX_FRAGMENT_SIZE	512
36896691d940SDaeho Jeong 
36906691d940SDaeho Jeong static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi)
36916691d940SDaeho Jeong {
36926691d940SDaeho Jeong 	return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG ||
36936691d940SDaeho Jeong 		F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK;
36946691d940SDaeho Jeong }
36956691d940SDaeho Jeong 
369639a53e0cSJaegeuk Kim /*
369739a53e0cSJaegeuk Kim  * checkpoint.c
369839a53e0cSJaegeuk Kim  */
3699a9cfee0eSChao Yu void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
3700a9cfee0eSChao Yu 							unsigned char reason);
3701c7b58576SJaegeuk Kim void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi);
37024d57b86dSChao Yu struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
37034d57b86dSChao Yu struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
370486f33603SJaegeuk Kim struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index);
37054d57b86dSChao Yu struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index);
3706e1da7872SChao Yu bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
37074d57b86dSChao Yu 					block_t blkaddr, int type);
37084d57b86dSChao Yu int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
3709cac5a3d8SDongOh Shin 			int type, bool sync);
3710430f163bSChao Yu void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index,
3711430f163bSChao Yu 							unsigned int ra_blocks);
37124d57b86dSChao Yu long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
3713b0af6d49SChao Yu 			long nr_to_write, enum iostat_type io_type);
37144d57b86dSChao Yu void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
37154d57b86dSChao Yu void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
37164d57b86dSChao Yu void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all);
37174d57b86dSChao Yu bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode);
37184d57b86dSChao Yu void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
371939d787beSChao Yu 					unsigned int devidx, int type);
37204d57b86dSChao Yu bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
372139d787beSChao Yu 					unsigned int devidx, int type);
37224d57b86dSChao Yu int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi);
37234d57b86dSChao Yu void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi);
37244d57b86dSChao Yu void f2fs_add_orphan_inode(struct inode *inode);
37254d57b86dSChao Yu void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino);
37264d57b86dSChao Yu int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi);
37274d57b86dSChao Yu int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
37284f5e34f7SMatthew Wilcox (Oracle) void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio);
37294d57b86dSChao Yu void f2fs_remove_dirty_inode(struct inode *inode);
3730d80afefbSChao Yu int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
3731d80afefbSChao Yu 								bool from_cp);
3732bf22c3ccSSahitya Tummala void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
37333a0a9cbcSChao Yu u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
37344d57b86dSChao Yu int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
37354d57b86dSChao Yu void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
37364d57b86dSChao Yu int __init f2fs_create_checkpoint_caches(void);
37374d57b86dSChao Yu void f2fs_destroy_checkpoint_caches(void);
3738261eeb9cSDaeho Jeong int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi);
3739261eeb9cSDaeho Jeong int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi);
3740261eeb9cSDaeho Jeong void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi);
3741261eeb9cSDaeho Jeong void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi);
374239a53e0cSJaegeuk Kim 
374339a53e0cSJaegeuk Kim /*
374439a53e0cSJaegeuk Kim  * data.c
374539a53e0cSJaegeuk Kim  */
3746f543805fSChao Yu int __init f2fs_init_bioset(void);
3747f543805fSChao Yu void f2fs_destroy_bioset(void);
37480b20fcecSChao Yu int f2fs_init_bio_entry_cache(void);
37490b20fcecSChao Yu void f2fs_destroy_bio_entry_cache(void);
3750bc29835aSChristoph Hellwig void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
3751bc29835aSChristoph Hellwig 			  enum page_type type);
3752908ea654SYufen Yu int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi);
3753b9109b0eSJaegeuk Kim void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
3754b9109b0eSJaegeuk Kim void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
3755bab475c5SChao Yu 				struct inode *inode, struct page *page,
3756bab475c5SChao Yu 				nid_t ino, enum page_type type);
37570b20fcecSChao Yu void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
37580b20fcecSChao Yu 					struct bio **bio, struct page *page);
3759b9109b0eSJaegeuk Kim void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
3760cac5a3d8SDongOh Shin int f2fs_submit_page_bio(struct f2fs_io_info *fio);
37618648de2cSChao Yu int f2fs_merge_page_bio(struct f2fs_io_info *fio);
3762fe16efe6SChao Yu void f2fs_submit_page_write(struct f2fs_io_info *fio);
3763cac5a3d8SDongOh Shin struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
37645189810aSChristoph Hellwig 		block_t blk_addr, sector_t *sector);
3765cac5a3d8SDongOh Shin int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr);
37664d57b86dSChao Yu void f2fs_set_data_blkaddr(struct dnode_of_data *dn);
3767cac5a3d8SDongOh Shin void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr);
37684d57b86dSChao Yu int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count);
37694d57b86dSChao Yu int f2fs_reserve_new_block(struct dnode_of_data *dn);
3770cf342d3bSChristoph Hellwig int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index);
3771cac5a3d8SDongOh Shin int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
37724d57b86dSChao Yu struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
377359237a21SChao Yu 			blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs);
377459237a21SChao Yu struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index,
377559237a21SChao Yu 							pgoff_t *next_pgofs);
37764d57b86dSChao Yu struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
3777cac5a3d8SDongOh Shin 			bool for_write);
37784d57b86dSChao Yu struct page *f2fs_get_new_data_page(struct inode *inode,
3779cac5a3d8SDongOh Shin 			struct page *ipage, pgoff_t index, bool new_i_size);
37804d57b86dSChao Yu int f2fs_do_write_data_page(struct f2fs_io_info *fio);
3781cd8fc522SChristoph Hellwig int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag);
3782cac5a3d8SDongOh Shin int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3783cac5a3d8SDongOh Shin 			u64 start, u64 len);
37844c8ff709SChao Yu int f2fs_encrypt_one_page(struct f2fs_io_info *fio);
37854d57b86dSChao Yu bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio);
37864d57b86dSChao Yu bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio);
37874c8ff709SChao Yu int f2fs_write_single_data_page(struct page *page, int *submitted,
37884c8ff709SChao Yu 				struct bio **bio, sector_t *last_block,
37894c8ff709SChao Yu 				struct writeback_control *wbc,
37904c8ff709SChao Yu 				enum iostat_type io_type,
37913afae09fSChao Yu 				int compr_blocks, bool allow_balance);
3792a1e09b03SEric Biggers void f2fs_write_failed(struct inode *inode, loff_t to);
379391503996SMatthew Wilcox (Oracle) void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length);
3794c26cd045SMatthew Wilcox (Oracle) bool f2fs_release_folio(struct folio *folio, gfp_t wait);
3795b91050a8SHyunchul Lee bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
37965ec2d99dSMatthew Wilcox void f2fs_clear_page_cache_dirty_tag(struct page *page);
37974c8ff709SChao Yu int f2fs_init_post_read_processing(void);
37984c8ff709SChao Yu void f2fs_destroy_post_read_processing(void);
37994c8ff709SChao Yu int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi);
38004c8ff709SChao Yu void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi);
38011517c1a7SEric Biggers extern const struct iomap_ops f2fs_iomap_ops;
380239a53e0cSJaegeuk Kim 
380339a53e0cSJaegeuk Kim /*
380439a53e0cSJaegeuk Kim  * gc.c
380539a53e0cSJaegeuk Kim  */
38064d57b86dSChao Yu int f2fs_start_gc_thread(struct f2fs_sb_info *sbi);
38074d57b86dSChao Yu void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi);
38084d57b86dSChao Yu block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode);
3809d147ea4aSJaegeuk Kim int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control);
38104d57b86dSChao Yu void f2fs_build_gc_manager(struct f2fs_sb_info *sbi);
381104f0b2eaSQiuyang Sun int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count);
3812093749e2SChao Yu int __init f2fs_create_garbage_collection_cache(void);
3813093749e2SChao Yu void f2fs_destroy_garbage_collection_cache(void);
381419e0e21aSYangtao Li /* victim selection function for cleaning and SSR */
381519e0e21aSYangtao Li int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result,
381619e0e21aSYangtao Li 			int gc_type, int type, char alloc_mode,
381719e0e21aSYangtao Li 			unsigned long long age);
381839a53e0cSJaegeuk Kim 
381939a53e0cSJaegeuk Kim /*
382039a53e0cSJaegeuk Kim  * recovery.c
382139a53e0cSJaegeuk Kim  */
38224d57b86dSChao Yu int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only);
38234d57b86dSChao Yu bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi);
3824cad83c96SChao Yu int __init f2fs_create_recovery_cache(void);
3825cad83c96SChao Yu void f2fs_destroy_recovery_cache(void);
382639a53e0cSJaegeuk Kim 
382739a53e0cSJaegeuk Kim /*
382839a53e0cSJaegeuk Kim  * debug.c
382939a53e0cSJaegeuk Kim  */
383039a53e0cSJaegeuk Kim #ifdef CONFIG_F2FS_STAT_FS
383139a53e0cSJaegeuk Kim struct f2fs_stat_info {
383239a53e0cSJaegeuk Kim 	struct list_head stat_list;
383339a53e0cSJaegeuk Kim 	struct f2fs_sb_info *sbi;
383439a53e0cSJaegeuk Kim 	int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs;
383539a53e0cSJaegeuk Kim 	int main_area_segs, main_area_sections, main_area_zones;
3836e7547dacSJaegeuk Kim 	unsigned long long hit_cached[NR_EXTENT_CACHES];
3837e7547dacSJaegeuk Kim 	unsigned long long hit_rbtree[NR_EXTENT_CACHES];
3838e7547dacSJaegeuk Kim 	unsigned long long total_ext[NR_EXTENT_CACHES];
3839e7547dacSJaegeuk Kim 	unsigned long long hit_total[NR_EXTENT_CACHES];
3840e7547dacSJaegeuk Kim 	int ext_tree[NR_EXTENT_CACHES];
3841e7547dacSJaegeuk Kim 	int zombie_tree[NR_EXTENT_CACHES];
3842e7547dacSJaegeuk Kim 	int ext_node[NR_EXTENT_CACHES];
3843e7547dacSJaegeuk Kim 	/* to count memory footprint */
3844e7547dacSJaegeuk Kim 	unsigned long long ext_mem[NR_EXTENT_CACHES];
3845e7547dacSJaegeuk Kim 	/* for read extent cache */
3846e7547dacSJaegeuk Kim 	unsigned long long hit_largest;
384771644dffSJaegeuk Kim 	/* for block age extent cache */
384871644dffSJaegeuk Kim 	unsigned long long allocated_data_blocks;
38492c8a4a28SJaegeuk Kim 	int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta;
38502c8a4a28SJaegeuk Kim 	int ndirty_data, ndirty_qdata;
38512c8a4a28SJaegeuk Kim 	unsigned int ndirty_dirs, ndirty_files, nquota_files, ndirty_all;
38525b0ef73cSJaegeuk Kim 	int nats, dirty_nats, sits, dirty_sits;
38535b0ef73cSJaegeuk Kim 	int free_nids, avail_nids, alloc_nids;
385439a53e0cSJaegeuk Kim 	int total_count, utilization;
38558b8dd65fSChao Yu 	int bg_gc, nr_wb_cp_data, nr_wb_data;
38565f9abab4SJaegeuk Kim 	int nr_rd_data, nr_rd_node, nr_rd_meta;
385702b16d0aSChao Yu 	int nr_dio_read, nr_dio_write;
3858274bd9baSChao Yu 	unsigned int io_skip_bggc, other_skip_bggc;
385914d8d5f7SChao Yu 	int nr_flushing, nr_flushed, flush_list_empty;
386014d8d5f7SChao Yu 	int nr_discarding, nr_discarded;
38615f32366aSChao Yu 	int nr_discard_cmd;
3862d84d1cbdSChao Yu 	unsigned int undiscard_blks;
3863261eeb9cSDaeho Jeong 	int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
3864261eeb9cSDaeho Jeong 	unsigned int cur_ckpt_time, peak_ckpt_time;
3865a00861dbSJaegeuk Kim 	int inline_xattr, inline_inode, inline_dir, append, update, orphans;
38668ec071c3SChao Yu 	int compr_inode, swapfile_inode;
3867ae999bb9SDaeho Jeong 	unsigned long long compr_blocks;
38687bc155feSJaegeuk Kim 	int aw_cnt, max_aw_cnt;
3869f83a2584SYunlei He 	unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
387039a53e0cSJaegeuk Kim 	unsigned int bimodal, avg_vblocks;
387139a53e0cSJaegeuk Kim 	int util_free, util_valid, util_invalid;
387239a53e0cSJaegeuk Kim 	int rsvd_segs, overp_segs;
38736ce19affSChao Yu 	int dirty_count, node_pages, meta_pages, compress_pages;
38746ce19affSChao Yu 	int compress_page_hit;
387542190d2aSJaegeuk Kim 	int prefree_count, call_count, cp_count, bg_cp_count;
387639a53e0cSJaegeuk Kim 	int tot_segs, node_segs, data_segs, free_segs, free_secs;
3877e1235983SChangman Lee 	int bg_node_segs, bg_data_segs;
387839a53e0cSJaegeuk Kim 	int tot_blks, data_blks, node_blks;
3879e1235983SChangman Lee 	int bg_data_blks, bg_node_blks;
388039a53e0cSJaegeuk Kim 	int curseg[NR_CURSEG_TYPE];
388139a53e0cSJaegeuk Kim 	int cursec[NR_CURSEG_TYPE];
388239a53e0cSJaegeuk Kim 	int curzone[NR_CURSEG_TYPE];
38830759e2c1SChao Yu 	unsigned int dirty_seg[NR_CURSEG_TYPE];
38840759e2c1SChao Yu 	unsigned int full_seg[NR_CURSEG_TYPE];
38850759e2c1SChao Yu 	unsigned int valid_blks[NR_CURSEG_TYPE];
388639a53e0cSJaegeuk Kim 
3887b63e7be5SChao Yu 	unsigned int meta_count[META_MAX];
388839a53e0cSJaegeuk Kim 	unsigned int segment_count[2];
388939a53e0cSJaegeuk Kim 	unsigned int block_count[2];
3890b9a2c252SChangman Lee 	unsigned int inplace_count;
38919edcdabfSChao Yu 	unsigned long long base_mem, cache_mem, page_mem;
389239a53e0cSJaegeuk Kim };
389339a53e0cSJaegeuk Kim 
3894963d4f7dSGu Zheng static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi)
3895963d4f7dSGu Zheng {
3896963d4f7dSGu Zheng 	return (struct f2fs_stat_info *)sbi->stat_info;
3897963d4f7dSGu Zheng }
3898963d4f7dSGu Zheng 
3899942e0be6SChangman Lee #define stat_inc_cp_count(si)		((si)->cp_count++)
390042190d2aSJaegeuk Kim #define stat_inc_bg_cp_count(si)	((si)->bg_cp_count++)
390139a53e0cSJaegeuk Kim #define stat_inc_call_count(si)		((si)->call_count++)
3902fc7100eaSHridya Valsaraju #define stat_inc_bggc_count(si)		((si)->bg_gc++)
3903274bd9baSChao Yu #define stat_io_skip_bggc_count(sbi)	((sbi)->io_skip_bggc++)
3904274bd9baSChao Yu #define stat_other_skip_bggc_count(sbi)	((sbi)->other_skip_bggc++)
390533fbd510SChao Yu #define stat_inc_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]++)
390633fbd510SChao Yu #define stat_dec_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]--)
3907e7547dacSJaegeuk Kim #define stat_inc_total_hit(sbi, type)		(atomic64_inc(&(sbi)->total_hit_ext[type]))
3908e7547dacSJaegeuk Kim #define stat_inc_rbtree_node_hit(sbi, type)	(atomic64_inc(&(sbi)->read_hit_rbtree[type]))
39095b7ee374SChao Yu #define stat_inc_largest_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_largest))
3910e7547dacSJaegeuk Kim #define stat_inc_cached_node_hit(sbi, type)	(atomic64_inc(&(sbi)->read_hit_cached[type]))
3911d5e8f6c9SChao Yu #define stat_inc_inline_xattr(inode)					\
3912d5e8f6c9SChao Yu 	do {								\
3913d5e8f6c9SChao Yu 		if (f2fs_has_inline_xattr(inode))			\
3914d5e8f6c9SChao Yu 			(atomic_inc(&F2FS_I_SB(inode)->inline_xattr));	\
3915d5e8f6c9SChao Yu 	} while (0)
3916d5e8f6c9SChao Yu #define stat_dec_inline_xattr(inode)					\
3917d5e8f6c9SChao Yu 	do {								\
3918d5e8f6c9SChao Yu 		if (f2fs_has_inline_xattr(inode))			\
3919d5e8f6c9SChao Yu 			(atomic_dec(&F2FS_I_SB(inode)->inline_xattr));	\
3920d5e8f6c9SChao Yu 	} while (0)
39210dbdc2aeSJaegeuk Kim #define stat_inc_inline_inode(inode)					\
39220dbdc2aeSJaegeuk Kim 	do {								\
39230dbdc2aeSJaegeuk Kim 		if (f2fs_has_inline_data(inode))			\
392403e14d52SChao Yu 			(atomic_inc(&F2FS_I_SB(inode)->inline_inode));	\
39250dbdc2aeSJaegeuk Kim 	} while (0)
39260dbdc2aeSJaegeuk Kim #define stat_dec_inline_inode(inode)					\
39270dbdc2aeSJaegeuk Kim 	do {								\
39280dbdc2aeSJaegeuk Kim 		if (f2fs_has_inline_data(inode))			\
392903e14d52SChao Yu 			(atomic_dec(&F2FS_I_SB(inode)->inline_inode));	\
39300dbdc2aeSJaegeuk Kim 	} while (0)
39313289c061SJaegeuk Kim #define stat_inc_inline_dir(inode)					\
39323289c061SJaegeuk Kim 	do {								\
39333289c061SJaegeuk Kim 		if (f2fs_has_inline_dentry(inode))			\
393403e14d52SChao Yu 			(atomic_inc(&F2FS_I_SB(inode)->inline_dir));	\
39353289c061SJaegeuk Kim 	} while (0)
39363289c061SJaegeuk Kim #define stat_dec_inline_dir(inode)					\
39373289c061SJaegeuk Kim 	do {								\
39383289c061SJaegeuk Kim 		if (f2fs_has_inline_dentry(inode))			\
393903e14d52SChao Yu 			(atomic_dec(&F2FS_I_SB(inode)->inline_dir));	\
39403289c061SJaegeuk Kim 	} while (0)
39414c8ff709SChao Yu #define stat_inc_compr_inode(inode)					\
39424c8ff709SChao Yu 	do {								\
39434c8ff709SChao Yu 		if (f2fs_compressed_file(inode))			\
39444c8ff709SChao Yu 			(atomic_inc(&F2FS_I_SB(inode)->compr_inode));	\
39454c8ff709SChao Yu 	} while (0)
39464c8ff709SChao Yu #define stat_dec_compr_inode(inode)					\
39474c8ff709SChao Yu 	do {								\
39484c8ff709SChao Yu 		if (f2fs_compressed_file(inode))			\
39494c8ff709SChao Yu 			(atomic_dec(&F2FS_I_SB(inode)->compr_inode));	\
39504c8ff709SChao Yu 	} while (0)
39514c8ff709SChao Yu #define stat_add_compr_blocks(inode, blocks)				\
3952ae999bb9SDaeho Jeong 		(atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
39534c8ff709SChao Yu #define stat_sub_compr_blocks(inode, blocks)				\
3954ae999bb9SDaeho Jeong 		(atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
39558ec071c3SChao Yu #define stat_inc_swapfile_inode(inode)					\
39568ec071c3SChao Yu 		(atomic_inc(&F2FS_I_SB(inode)->swapfile_inode))
39578ec071c3SChao Yu #define stat_dec_swapfile_inode(inode)					\
39588ec071c3SChao Yu 		(atomic_dec(&F2FS_I_SB(inode)->swapfile_inode))
3959b4dac120SChao Yu #define stat_inc_atomic_inode(inode)					\
3960b4dac120SChao Yu 			(atomic_inc(&F2FS_I_SB(inode)->atomic_files))
3961b4dac120SChao Yu #define stat_dec_atomic_inode(inode)					\
3962b4dac120SChao Yu 			(atomic_dec(&F2FS_I_SB(inode)->atomic_files))
3963b63e7be5SChao Yu #define stat_inc_meta_count(sbi, blkaddr)				\
3964b63e7be5SChao Yu 	do {								\
3965b63e7be5SChao Yu 		if (blkaddr < SIT_I(sbi)->sit_base_addr)		\
3966b63e7be5SChao Yu 			atomic_inc(&(sbi)->meta_count[META_CP]);	\
3967b63e7be5SChao Yu 		else if (blkaddr < NM_I(sbi)->nat_blkaddr)		\
3968b63e7be5SChao Yu 			atomic_inc(&(sbi)->meta_count[META_SIT]);	\
3969b63e7be5SChao Yu 		else if (blkaddr < SM_I(sbi)->ssa_blkaddr)		\
3970b63e7be5SChao Yu 			atomic_inc(&(sbi)->meta_count[META_NAT]);	\
3971b63e7be5SChao Yu 		else if (blkaddr < SM_I(sbi)->main_blkaddr)		\
3972b63e7be5SChao Yu 			atomic_inc(&(sbi)->meta_count[META_SSA]);	\
3973b63e7be5SChao Yu 	} while (0)
3974dcdfff65SJaegeuk Kim #define stat_inc_seg_type(sbi, curseg)					\
3975dcdfff65SJaegeuk Kim 		((sbi)->segment_count[(curseg)->alloc_type]++)
3976dcdfff65SJaegeuk Kim #define stat_inc_block_count(sbi, curseg)				\
3977dcdfff65SJaegeuk Kim 		((sbi)->block_count[(curseg)->alloc_type]++)
3978b9a2c252SChangman Lee #define stat_inc_inplace_blocks(sbi)					\
3979b9a2c252SChangman Lee 		(atomic_inc(&(sbi)->inplace_count))
398026a28a0cSJaegeuk Kim #define stat_update_max_atomic_write(inode)				\
398126a28a0cSJaegeuk Kim 	do {								\
3982b4dac120SChao Yu 		int cur = atomic_read(&F2FS_I_SB(inode)->atomic_files);	\
398326a28a0cSJaegeuk Kim 		int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt);	\
398426a28a0cSJaegeuk Kim 		if (cur > max)						\
398526a28a0cSJaegeuk Kim 			atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur);	\
398626a28a0cSJaegeuk Kim 	} while (0)
3987e1235983SChangman Lee #define stat_inc_seg_count(sbi, type, gc_type)				\
398839a53e0cSJaegeuk Kim 	do {								\
3989963d4f7dSGu Zheng 		struct f2fs_stat_info *si = F2FS_STAT(sbi);		\
399068afcf2dSTomohiro Kusumi 		si->tot_segs++;						\
399168afcf2dSTomohiro Kusumi 		if ((type) == SUM_TYPE_DATA) {				\
399239a53e0cSJaegeuk Kim 			si->data_segs++;				\
3993e1235983SChangman Lee 			si->bg_data_segs += (gc_type == BG_GC) ? 1 : 0;	\
3994e1235983SChangman Lee 		} else {						\
399539a53e0cSJaegeuk Kim 			si->node_segs++;				\
3996e1235983SChangman Lee 			si->bg_node_segs += (gc_type == BG_GC) ? 1 : 0;	\
3997e1235983SChangman Lee 		}							\
399839a53e0cSJaegeuk Kim 	} while (0)
399939a53e0cSJaegeuk Kim 
400039a53e0cSJaegeuk Kim #define stat_inc_tot_blk_count(si, blks)				\
400168afcf2dSTomohiro Kusumi 	((si)->tot_blks += (blks))
400239a53e0cSJaegeuk Kim 
4003e1235983SChangman Lee #define stat_inc_data_blk_count(sbi, blks, gc_type)			\
400439a53e0cSJaegeuk Kim 	do {								\
4005963d4f7dSGu Zheng 		struct f2fs_stat_info *si = F2FS_STAT(sbi);		\
400639a53e0cSJaegeuk Kim 		stat_inc_tot_blk_count(si, blks);			\
400739a53e0cSJaegeuk Kim 		si->data_blks += (blks);				\
400868afcf2dSTomohiro Kusumi 		si->bg_data_blks += ((gc_type) == BG_GC) ? (blks) : 0;	\
400939a53e0cSJaegeuk Kim 	} while (0)
401039a53e0cSJaegeuk Kim 
4011e1235983SChangman Lee #define stat_inc_node_blk_count(sbi, blks, gc_type)			\
401239a53e0cSJaegeuk Kim 	do {								\
4013963d4f7dSGu Zheng 		struct f2fs_stat_info *si = F2FS_STAT(sbi);		\
401439a53e0cSJaegeuk Kim 		stat_inc_tot_blk_count(si, blks);			\
401539a53e0cSJaegeuk Kim 		si->node_blks += (blks);				\
401668afcf2dSTomohiro Kusumi 		si->bg_node_blks += ((gc_type) == BG_GC) ? (blks) : 0;	\
401739a53e0cSJaegeuk Kim 	} while (0)
401839a53e0cSJaegeuk Kim 
4019cac5a3d8SDongOh Shin int f2fs_build_stats(struct f2fs_sb_info *sbi);
4020cac5a3d8SDongOh Shin void f2fs_destroy_stats(struct f2fs_sb_info *sbi);
402121acc07dSGreg Kroah-Hartman void __init f2fs_create_root_stats(void);
40224589d25dSNamjae Jeon void f2fs_destroy_root_stats(void);
4023fc7100eaSHridya Valsaraju void f2fs_update_sit_info(struct f2fs_sb_info *sbi);
402439a53e0cSJaegeuk Kim #else
4025d66450e7SArnd Bergmann #define stat_inc_cp_count(si)				do { } while (0)
4026d66450e7SArnd Bergmann #define stat_inc_bg_cp_count(si)			do { } while (0)
4027d66450e7SArnd Bergmann #define stat_inc_call_count(si)				do { } while (0)
4028d66450e7SArnd Bergmann #define stat_inc_bggc_count(si)				do { } while (0)
4029274bd9baSChao Yu #define stat_io_skip_bggc_count(sbi)			do { } while (0)
4030274bd9baSChao Yu #define stat_other_skip_bggc_count(sbi)			do { } while (0)
4031d66450e7SArnd Bergmann #define stat_inc_dirty_inode(sbi, type)			do { } while (0)
4032d66450e7SArnd Bergmann #define stat_dec_dirty_inode(sbi, type)			do { } while (0)
4033e7547dacSJaegeuk Kim #define stat_inc_total_hit(sbi, type)			do { } while (0)
4034e7547dacSJaegeuk Kim #define stat_inc_rbtree_node_hit(sbi, type)		do { } while (0)
4035d66450e7SArnd Bergmann #define stat_inc_largest_node_hit(sbi)			do { } while (0)
4036e7547dacSJaegeuk Kim #define stat_inc_cached_node_hit(sbi, type)		do { } while (0)
4037d66450e7SArnd Bergmann #define stat_inc_inline_xattr(inode)			do { } while (0)
4038d66450e7SArnd Bergmann #define stat_dec_inline_xattr(inode)			do { } while (0)
4039d66450e7SArnd Bergmann #define stat_inc_inline_inode(inode)			do { } while (0)
4040d66450e7SArnd Bergmann #define stat_dec_inline_inode(inode)			do { } while (0)
4041d66450e7SArnd Bergmann #define stat_inc_inline_dir(inode)			do { } while (0)
4042d66450e7SArnd Bergmann #define stat_dec_inline_dir(inode)			do { } while (0)
40434c8ff709SChao Yu #define stat_inc_compr_inode(inode)			do { } while (0)
40444c8ff709SChao Yu #define stat_dec_compr_inode(inode)			do { } while (0)
40454c8ff709SChao Yu #define stat_add_compr_blocks(inode, blocks)		do { } while (0)
40464c8ff709SChao Yu #define stat_sub_compr_blocks(inode, blocks)		do { } while (0)
40478ec071c3SChao Yu #define stat_inc_swapfile_inode(inode)			do { } while (0)
40488ec071c3SChao Yu #define stat_dec_swapfile_inode(inode)			do { } while (0)
4049b4dac120SChao Yu #define stat_inc_atomic_inode(inode)			do { } while (0)
4050b4dac120SChao Yu #define stat_dec_atomic_inode(inode)			do { } while (0)
4051d66450e7SArnd Bergmann #define stat_update_max_atomic_write(inode)		do { } while (0)
4052b63e7be5SChao Yu #define stat_inc_meta_count(sbi, blkaddr)		do { } while (0)
4053d66450e7SArnd Bergmann #define stat_inc_seg_type(sbi, curseg)			do { } while (0)
4054d66450e7SArnd Bergmann #define stat_inc_block_count(sbi, curseg)		do { } while (0)
4055d66450e7SArnd Bergmann #define stat_inc_inplace_blocks(sbi)			do { } while (0)
4056d66450e7SArnd Bergmann #define stat_inc_seg_count(sbi, type, gc_type)		do { } while (0)
4057d66450e7SArnd Bergmann #define stat_inc_tot_blk_count(si, blks)		do { } while (0)
4058d66450e7SArnd Bergmann #define stat_inc_data_blk_count(sbi, blks, gc_type)	do { } while (0)
4059d66450e7SArnd Bergmann #define stat_inc_node_blk_count(sbi, blks, gc_type)	do { } while (0)
406039a53e0cSJaegeuk Kim 
406139a53e0cSJaegeuk Kim static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; }
406239a53e0cSJaegeuk Kim static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { }
406321acc07dSGreg Kroah-Hartman static inline void __init f2fs_create_root_stats(void) { }
40644589d25dSNamjae Jeon static inline void f2fs_destroy_root_stats(void) { }
406548abe91aSYueHaibing static inline void f2fs_update_sit_info(struct f2fs_sb_info *sbi) {}
406639a53e0cSJaegeuk Kim #endif
406739a53e0cSJaegeuk Kim 
406839a53e0cSJaegeuk Kim extern const struct file_operations f2fs_dir_operations;
406939a53e0cSJaegeuk Kim extern const struct file_operations f2fs_file_operations;
407039a53e0cSJaegeuk Kim extern const struct inode_operations f2fs_file_inode_operations;
407139a53e0cSJaegeuk Kim extern const struct address_space_operations f2fs_dblock_aops;
407239a53e0cSJaegeuk Kim extern const struct address_space_operations f2fs_node_aops;
407339a53e0cSJaegeuk Kim extern const struct address_space_operations f2fs_meta_aops;
407439a53e0cSJaegeuk Kim extern const struct inode_operations f2fs_dir_inode_operations;
407539a53e0cSJaegeuk Kim extern const struct inode_operations f2fs_symlink_inode_operations;
4076cbaf042aSJaegeuk Kim extern const struct inode_operations f2fs_encrypted_symlink_inode_operations;
407739a53e0cSJaegeuk Kim extern const struct inode_operations f2fs_special_inode_operations;
40784d57b86dSChao Yu extern struct kmem_cache *f2fs_inode_entry_slab;
40791001b347SHuajun Li 
4080e18c65b2SHuajun Li /*
4081e18c65b2SHuajun Li  * inline.c
4082e18c65b2SHuajun Li  */
4083cac5a3d8SDongOh Shin bool f2fs_may_inline_data(struct inode *inode);
4084677a82b4SChao Yu bool f2fs_sanity_check_inline_data(struct inode *inode);
4085cac5a3d8SDongOh Shin bool f2fs_may_inline_dentry(struct inode *inode);
40864d57b86dSChao Yu void f2fs_do_read_inline_data(struct page *page, struct page *ipage);
40874d57b86dSChao Yu void f2fs_truncate_inline_inode(struct inode *inode,
40884d57b86dSChao Yu 						struct page *ipage, u64 from);
4089cac5a3d8SDongOh Shin int f2fs_read_inline_data(struct inode *inode, struct page *page);
4090cac5a3d8SDongOh Shin int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page);
4091cac5a3d8SDongOh Shin int f2fs_convert_inline_inode(struct inode *inode);
4092b06af2afSJaegeuk Kim int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry);
4093cac5a3d8SDongOh Shin int f2fs_write_inline_data(struct inode *inode, struct page *page);
40949627a7b3SChao Yu int f2fs_recover_inline_data(struct inode *inode, struct page *npage);
40954d57b86dSChao Yu struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
409643c780baSEric Biggers 					const struct f2fs_filename *fname,
409743c780baSEric Biggers 					struct page **res_page);
40984d57b86dSChao Yu int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
4099cac5a3d8SDongOh Shin 			struct page *ipage);
410043c780baSEric Biggers int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
4101cac5a3d8SDongOh Shin 			struct inode *inode, nid_t ino, umode_t mode);
41024d57b86dSChao Yu void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
41034d57b86dSChao Yu 				struct page *page, struct inode *dir,
41044d57b86dSChao Yu 				struct inode *inode);
4105cac5a3d8SDongOh Shin bool f2fs_empty_inline_dir(struct inode *dir);
4106cac5a3d8SDongOh Shin int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
4107cac5a3d8SDongOh Shin 			struct fscrypt_str *fstr);
4108cac5a3d8SDongOh Shin int f2fs_inline_data_fiemap(struct inode *inode,
4109cac5a3d8SDongOh Shin 			struct fiemap_extent_info *fieinfo,
4110cac5a3d8SDongOh Shin 			__u64 start, __u64 len);
4111cde4de12SJaegeuk Kim 
4112cde4de12SJaegeuk Kim /*
41132658e50dSJaegeuk Kim  * shrinker.c
41142658e50dSJaegeuk Kim  */
4115cac5a3d8SDongOh Shin unsigned long f2fs_shrink_count(struct shrinker *shrink,
4116cac5a3d8SDongOh Shin 			struct shrink_control *sc);
4117cac5a3d8SDongOh Shin unsigned long f2fs_shrink_scan(struct shrinker *shrink,
4118cac5a3d8SDongOh Shin 			struct shrink_control *sc);
4119cac5a3d8SDongOh Shin void f2fs_join_shrinker(struct f2fs_sb_info *sbi);
4120cac5a3d8SDongOh Shin void f2fs_leave_shrinker(struct f2fs_sb_info *sbi);
41212658e50dSJaegeuk Kim 
41222658e50dSJaegeuk Kim /*
4123a28ef1f5SChao Yu  * extent_cache.c
4124a28ef1f5SChao Yu  */
4125269d1194SChao Yu bool sanity_check_extent_cache(struct inode *inode);
412672840cccSJaegeuk Kim void f2fs_init_extent_tree(struct inode *inode);
4127cac5a3d8SDongOh Shin void f2fs_drop_extent_tree(struct inode *inode);
4128e7547dacSJaegeuk Kim void f2fs_destroy_extent_node(struct inode *inode);
4129cac5a3d8SDongOh Shin void f2fs_destroy_extent_tree(struct inode *inode);
41304d57b86dSChao Yu void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi);
41314d57b86dSChao Yu int __init f2fs_create_extent_cache(void);
41324d57b86dSChao Yu void f2fs_destroy_extent_cache(void);
4133a28ef1f5SChao Yu 
4134e7547dacSJaegeuk Kim /* read extent cache ops */
413572840cccSJaegeuk Kim void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage);
4136e7547dacSJaegeuk Kim bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs,
4137e7547dacSJaegeuk Kim 			struct extent_info *ei);
413804a91ab0SChristoph Hellwig bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index,
413904a91ab0SChristoph Hellwig 			block_t *blkaddr);
4140e7547dacSJaegeuk Kim void f2fs_update_read_extent_cache(struct dnode_of_data *dn);
4141e7547dacSJaegeuk Kim void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn,
4142e7547dacSJaegeuk Kim 			pgoff_t fofs, block_t blkaddr, unsigned int len);
4143e7547dacSJaegeuk Kim unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi,
4144e7547dacSJaegeuk Kim 			int nr_shrink);
4145e7547dacSJaegeuk Kim 
414671644dffSJaegeuk Kim /* block age extent cache ops */
414771644dffSJaegeuk Kim void f2fs_init_age_extent_tree(struct inode *inode);
414871644dffSJaegeuk Kim bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs,
414971644dffSJaegeuk Kim 			struct extent_info *ei);
415071644dffSJaegeuk Kim void f2fs_update_age_extent_cache(struct dnode_of_data *dn);
415171644dffSJaegeuk Kim void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn,
415271644dffSJaegeuk Kim 			pgoff_t fofs, unsigned int len);
415371644dffSJaegeuk Kim unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi,
415471644dffSJaegeuk Kim 			int nr_shrink);
415571644dffSJaegeuk Kim 
4156a28ef1f5SChao Yu /*
41578ceffcb2SChao Yu  * sysfs.c
41588ceffcb2SChao Yu  */
41590f6b56ecSDaeho Jeong #define MIN_RA_MUL	2
41600f6b56ecSDaeho Jeong #define MAX_RA_MUL	256
41610f6b56ecSDaeho Jeong 
4162dc6b2055SJaegeuk Kim int __init f2fs_init_sysfs(void);
4163dc6b2055SJaegeuk Kim void f2fs_exit_sysfs(void);
4164dc6b2055SJaegeuk Kim int f2fs_register_sysfs(struct f2fs_sb_info *sbi);
4165dc6b2055SJaegeuk Kim void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi);
41668ceffcb2SChao Yu 
416795ae251fSEric Biggers /* verity.c */
416895ae251fSEric Biggers extern const struct fsverity_operations f2fs_verityops;
416995ae251fSEric Biggers 
41708ceffcb2SChao Yu /*
4171cde4de12SJaegeuk Kim  * crypto support
4172cde4de12SJaegeuk Kim  */
41731958593eSJaegeuk Kim static inline bool f2fs_encrypted_file(struct inode *inode)
41741958593eSJaegeuk Kim {
417562230e0dSChandan Rajendra 	return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
41761958593eSJaegeuk Kim }
41771958593eSJaegeuk Kim 
4178cde4de12SJaegeuk Kim static inline void f2fs_set_encrypted_inode(struct inode *inode)
4179cde4de12SJaegeuk Kim {
4180643fa961SChandan Rajendra #ifdef CONFIG_FS_ENCRYPTION
4181cde4de12SJaegeuk Kim 	file_set_encrypt(inode);
41829149a5ebSChao Yu 	f2fs_set_inode_flags(inode);
4183cde4de12SJaegeuk Kim #endif
4184cde4de12SJaegeuk Kim }
4185cde4de12SJaegeuk Kim 
41866dbb1796SEric Biggers /*
41876dbb1796SEric Biggers  * Returns true if the reads of the inode's data need to undergo some
41886dbb1796SEric Biggers  * postprocessing step, like decryption or authenticity verification.
41896dbb1796SEric Biggers  */
41906dbb1796SEric Biggers static inline bool f2fs_post_read_required(struct inode *inode)
4191cde4de12SJaegeuk Kim {
41924c8ff709SChao Yu 	return f2fs_encrypted_file(inode) || fsverity_active(inode) ||
41934c8ff709SChao Yu 		f2fs_compressed_file(inode);
41944c8ff709SChao Yu }
41954c8ff709SChao Yu 
41964c8ff709SChao Yu /*
41974c8ff709SChao Yu  * compress.c
41984c8ff709SChao Yu  */
41994c8ff709SChao Yu #ifdef CONFIG_F2FS_FS_COMPRESSION
42004c8ff709SChao Yu bool f2fs_is_compressed_page(struct page *page);
42014c8ff709SChao Yu struct page *f2fs_compress_control_page(struct page *page);
42024c8ff709SChao Yu int f2fs_prepare_compress_overwrite(struct inode *inode,
42034c8ff709SChao Yu 			struct page **pagep, pgoff_t index, void **fsdata);
42044c8ff709SChao Yu bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
42054c8ff709SChao Yu 					pgoff_t index, unsigned copied);
42063265d3dbSChao Yu int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock);
42074c8ff709SChao Yu void f2fs_compress_write_end_io(struct bio *bio, struct page *page);
42084c8ff709SChao Yu bool f2fs_is_compress_backend_ready(struct inode *inode);
4209a1357a91SYangtao Li int __init f2fs_init_compress_mempool(void);
42105e6bbde9SChao Yu void f2fs_destroy_compress_mempool(void);
4211bff139b4SDaeho Jeong void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task);
42126ce19affSChao Yu void f2fs_end_read_compressed_page(struct page *page, bool failed,
4213bff139b4SDaeho Jeong 				block_t blkaddr, bool in_task);
42144c8ff709SChao Yu bool f2fs_cluster_is_empty(struct compress_ctx *cc);
42154c8ff709SChao Yu bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
421601fc4b9aSFengnan Chang bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages,
42174f8219f8SFengnan Chang 				int index, int nr_pages, bool uptodate);
4218bbe1da7eSChao Yu bool f2fs_sanity_check_cluster(struct dnode_of_data *dn);
42194c8ff709SChao Yu void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page);
42204c8ff709SChao Yu int f2fs_write_multi_pages(struct compress_ctx *cc,
42214c8ff709SChao Yu 						int *submitted,
42224c8ff709SChao Yu 						struct writeback_control *wbc,
42234c8ff709SChao Yu 						enum iostat_type io_type);
42244c8ff709SChao Yu int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index);
4225e7547dacSJaegeuk Kim void f2fs_update_read_extent_tree_range_compressed(struct inode *inode,
4226e7547dacSJaegeuk Kim 				pgoff_t fofs, block_t blkaddr,
4227e7547dacSJaegeuk Kim 				unsigned int llen, unsigned int c_len);
42284c8ff709SChao Yu int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
42294c8ff709SChao Yu 				unsigned nr_pages, sector_t *last_block_in_bio,
42300683728aSChao Yu 				bool is_readahead, bool for_write);
42314c8ff709SChao Yu struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc);
4232bff139b4SDaeho Jeong void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
4233bff139b4SDaeho Jeong 				bool in_task);
4234bff139b4SDaeho Jeong void f2fs_put_page_dic(struct page *page, bool in_task);
423594afd6d6SChao Yu unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn);
42364c8ff709SChao Yu int f2fs_init_compress_ctx(struct compress_ctx *cc);
42378bfbfb0dSChao Yu void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
42384c8ff709SChao Yu void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
42396ce19affSChao Yu int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
42406ce19affSChao Yu void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
424131083031SChao Yu int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
424231083031SChao Yu void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
4243c68d6c88SChao Yu int __init f2fs_init_compress_cache(void);
4244c68d6c88SChao Yu void f2fs_destroy_compress_cache(void);
42456ce19affSChao Yu struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi);
42466ce19affSChao Yu void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr);
42476ce19affSChao Yu void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
42486ce19affSChao Yu 						nid_t ino, block_t blkaddr);
42496ce19affSChao Yu bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
42506ce19affSChao Yu 								block_t blkaddr);
42516ce19affSChao Yu void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino);
42525ac443e2SDaeho Jeong #define inc_compr_inode_stat(inode)					\
42535ac443e2SDaeho Jeong 	do {								\
42545ac443e2SDaeho Jeong 		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);		\
42555ac443e2SDaeho Jeong 		sbi->compr_new_inode++;					\
42565ac443e2SDaeho Jeong 	} while (0)
42575ac443e2SDaeho Jeong #define add_compr_block_stat(inode, blocks)				\
42585ac443e2SDaeho Jeong 	do {								\
42595ac443e2SDaeho Jeong 		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);		\
42605ac443e2SDaeho Jeong 		int diff = F2FS_I(inode)->i_cluster_size - blocks;	\
42615ac443e2SDaeho Jeong 		sbi->compr_written_block += blocks;			\
42625ac443e2SDaeho Jeong 		sbi->compr_saved_block += diff;				\
42635ac443e2SDaeho Jeong 	} while (0)
42644c8ff709SChao Yu #else
42654c8ff709SChao Yu static inline bool f2fs_is_compressed_page(struct page *page) { return false; }
42664c8ff709SChao Yu static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
42674c8ff709SChao Yu {
42684c8ff709SChao Yu 	if (!f2fs_compressed_file(inode))
42694c8ff709SChao Yu 		return true;
42704c8ff709SChao Yu 	/* not support compression */
42714c8ff709SChao Yu 	return false;
42724c8ff709SChao Yu }
42734c8ff709SChao Yu static inline struct page *f2fs_compress_control_page(struct page *page)
42744c8ff709SChao Yu {
42754c8ff709SChao Yu 	WARN_ON_ONCE(1);
42764c8ff709SChao Yu 	return ERR_PTR(-EINVAL);
42774c8ff709SChao Yu }
4278a1357a91SYangtao Li static inline int __init f2fs_init_compress_mempool(void) { return 0; }
42795e6bbde9SChao Yu static inline void f2fs_destroy_compress_mempool(void) { }
4280bff139b4SDaeho Jeong static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic,
4281bff139b4SDaeho Jeong 				bool in_task) { }
42826ce19affSChao Yu static inline void f2fs_end_read_compressed_page(struct page *page,
4283bff139b4SDaeho Jeong 				bool failed, block_t blkaddr, bool in_task)
42847f59b277SEric Biggers {
42857f59b277SEric Biggers 	WARN_ON_ONCE(1);
42867f59b277SEric Biggers }
4287bff139b4SDaeho Jeong static inline void f2fs_put_page_dic(struct page *page, bool in_task)
42887f59b277SEric Biggers {
42897f59b277SEric Biggers 	WARN_ON_ONCE(1);
42907f59b277SEric Biggers }
429194afd6d6SChao Yu static inline unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn) { return 0; }
4292bbe1da7eSChao Yu static inline bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) { return false; }
42936ce19affSChao Yu static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
42946ce19affSChao Yu static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
429531083031SChao Yu static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
429631083031SChao Yu static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
4297c68d6c88SChao Yu static inline int __init f2fs_init_compress_cache(void) { return 0; }
4298c68d6c88SChao Yu static inline void f2fs_destroy_compress_cache(void) { }
42996ce19affSChao Yu static inline void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi,
43006ce19affSChao Yu 				block_t blkaddr) { }
43016ce19affSChao Yu static inline void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
43026ce19affSChao Yu 				struct page *page, nid_t ino, block_t blkaddr) { }
43036ce19affSChao Yu static inline bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi,
43046ce19affSChao Yu 				struct page *page, block_t blkaddr) { return false; }
43056ce19affSChao Yu static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi,
43066ce19affSChao Yu 							nid_t ino) { }
43075ac443e2SDaeho Jeong #define inc_compr_inode_stat(inode)		do { } while (0)
4308e7547dacSJaegeuk Kim static inline void f2fs_update_read_extent_tree_range_compressed(
4309e7547dacSJaegeuk Kim 				struct inode *inode,
4310e7547dacSJaegeuk Kim 				pgoff_t fofs, block_t blkaddr,
4311e7547dacSJaegeuk Kim 				unsigned int llen, unsigned int c_len) { }
43124c8ff709SChao Yu #endif
43134c8ff709SChao Yu 
4314912f0d65SJaegeuk Kim static inline int set_compress_context(struct inode *inode)
43154c8ff709SChao Yu {
4316912f0d65SJaegeuk Kim #ifdef CONFIG_F2FS_FS_COMPRESSION
43174c8ff709SChao Yu 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
43184c8ff709SChao Yu 
43194c8ff709SChao Yu 	F2FS_I(inode)->i_compress_algorithm =
43204c8ff709SChao Yu 			F2FS_OPTION(sbi).compress_algorithm;
43214c8ff709SChao Yu 	F2FS_I(inode)->i_log_cluster_size =
43224c8ff709SChao Yu 			F2FS_OPTION(sbi).compress_log_size;
4323b28f047bSChao Yu 	F2FS_I(inode)->i_compress_flag =
4324b28f047bSChao Yu 			F2FS_OPTION(sbi).compress_chksum ?
4325447286ebSYangtao Li 				BIT(COMPRESS_CHKSUM) : 0;
43264c8ff709SChao Yu 	F2FS_I(inode)->i_cluster_size =
4327447286ebSYangtao Li 			BIT(F2FS_I(inode)->i_log_cluster_size);
432801f6afd0SChao Yu 	if ((F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 ||
432901f6afd0SChao Yu 		F2FS_I(inode)->i_compress_algorithm == COMPRESS_ZSTD) &&
43303fde13f8SChao Yu 			F2FS_OPTION(sbi).compress_level)
4331b90e5086SChao Yu 		F2FS_I(inode)->i_compress_level =
4332b90e5086SChao Yu 				F2FS_OPTION(sbi).compress_level;
43334c8ff709SChao Yu 	F2FS_I(inode)->i_flags |= F2FS_COMPR_FL;
43344c8ff709SChao Yu 	set_inode_flag(inode, FI_COMPRESSED_FILE);
43354c8ff709SChao Yu 	stat_inc_compr_inode(inode);
43365ac443e2SDaeho Jeong 	inc_compr_inode_stat(inode);
4337530e0704SChao Yu 	f2fs_mark_inode_dirty_sync(inode, true);
4338912f0d65SJaegeuk Kim 	return 0;
4339912f0d65SJaegeuk Kim #else
4340912f0d65SJaegeuk Kim 	return -EOPNOTSUPP;
4341912f0d65SJaegeuk Kim #endif
43424c8ff709SChao Yu }
43434c8ff709SChao Yu 
434478134d03SDaeho Jeong static inline bool f2fs_disable_compressed_file(struct inode *inode)
43454c8ff709SChao Yu {
43464c8ff709SChao Yu 	struct f2fs_inode_info *fi = F2FS_I(inode);
43474c8ff709SChao Yu 
43484c8ff709SChao Yu 	if (!f2fs_compressed_file(inode))
434978134d03SDaeho Jeong 		return true;
435002d58cd2SHyeong-Jun Kim 	if (S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))
435178134d03SDaeho Jeong 		return false;
43524c8ff709SChao Yu 
43534c8ff709SChao Yu 	fi->i_flags &= ~F2FS_COMPR_FL;
43544c8ff709SChao Yu 	stat_dec_compr_inode(inode);
435596f5b4faSChao Yu 	clear_inode_flag(inode, FI_COMPRESSED_FILE);
4356530e0704SChao Yu 	f2fs_mark_inode_dirty_sync(inode, true);
435778134d03SDaeho Jeong 	return true;
4358cde4de12SJaegeuk Kim }
4359cde4de12SJaegeuk Kim 
4360ccd31cb2SSheng Yong #define F2FS_FEATURE_FUNCS(name, flagname) \
4361e4807519SYangtao Li static inline bool f2fs_sb_has_##name(struct f2fs_sb_info *sbi) \
4362ccd31cb2SSheng Yong { \
43637beb01f7SChao Yu 	return F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_##flagname); \
4364cde4de12SJaegeuk Kim }
4365f424f664SJaegeuk Kim 
4366ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(encrypt, ENCRYPT);
4367ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(blkzoned, BLKZONED);
4368ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(extra_attr, EXTRA_ATTR);
4369ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(project_quota, PRJQUOTA);
4370ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(inode_chksum, INODE_CHKSUM);
4371ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(flexible_inline_xattr, FLEXIBLE_INLINE_XATTR);
4372ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO);
4373ccd31cb2SSheng Yong F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME);
4374b7c409deSSheng Yong F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND);
437595ae251fSEric Biggers F2FS_FEATURE_FUNCS(verity, VERITY);
4376d440c52dSJunling Zheng F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
43775aba5430SDaniel Rosenberg F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
43784c8ff709SChao Yu F2FS_FEATURE_FUNCS(compression, COMPRESSION);
4379a7d9fe3cSJaegeuk Kim F2FS_FEATURE_FUNCS(readonly, RO);
43801c1d35dfSChao Yu 
4381178053e2SDamien Le Moal #ifdef CONFIG_BLK_DEV_ZONED
438295175dafSDamien Le Moal static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
438395175dafSDamien Le Moal 				    block_t blkaddr)
4384178053e2SDamien Le Moal {
4385178053e2SDamien Le Moal 	unsigned int zno = blkaddr >> sbi->log_blocks_per_blkz;
4386178053e2SDamien Le Moal 
438795175dafSDamien Le Moal 	return test_bit(zno, FDEV(devi).blkz_seq);
4388178053e2SDamien Le Moal }
4389178053e2SDamien Le Moal #endif
4390178053e2SDamien Le Moal 
43917d20c8abSChao Yu static inline bool f2fs_hw_should_discard(struct f2fs_sb_info *sbi)
439296ba2decSDamien Le Moal {
43937beb01f7SChao Yu 	return f2fs_sb_has_blkzoned(sbi);
43947d20c8abSChao Yu }
439596ba2decSDamien Le Moal 
43967f3d7719SDamien Le Moal static inline bool f2fs_bdev_support_discard(struct block_device *bdev)
43977f3d7719SDamien Le Moal {
439870200574SChristoph Hellwig 	return bdev_max_discard_sectors(bdev) || bdev_is_zoned(bdev);
43997f3d7719SDamien Le Moal }
44007f3d7719SDamien Le Moal 
44017d20c8abSChao Yu static inline bool f2fs_hw_support_discard(struct f2fs_sb_info *sbi)
44027d20c8abSChao Yu {
44037f3d7719SDamien Le Moal 	int i;
44047f3d7719SDamien Le Moal 
44057f3d7719SDamien Le Moal 	if (!f2fs_is_multi_device(sbi))
44067f3d7719SDamien Le Moal 		return f2fs_bdev_support_discard(sbi->sb->s_bdev);
44077f3d7719SDamien Le Moal 
44087f3d7719SDamien Le Moal 	for (i = 0; i < sbi->s_ndevs; i++)
44097f3d7719SDamien Le Moal 		if (f2fs_bdev_support_discard(FDEV(i).bdev))
44107f3d7719SDamien Le Moal 			return true;
44117f3d7719SDamien Le Moal 	return false;
44127d20c8abSChao Yu }
44137d20c8abSChao Yu 
44147d20c8abSChao Yu static inline bool f2fs_realtime_discard_enable(struct f2fs_sb_info *sbi)
44157d20c8abSChao Yu {
44167d20c8abSChao Yu 	return (test_opt(sbi, DISCARD) && f2fs_hw_support_discard(sbi)) ||
44177d20c8abSChao Yu 					f2fs_hw_should_discard(sbi);
441852763a4bSJaegeuk Kim }
441952763a4bSJaegeuk Kim 
4420f824deb5SChao Yu static inline bool f2fs_hw_is_readonly(struct f2fs_sb_info *sbi)
4421f824deb5SChao Yu {
4422f824deb5SChao Yu 	int i;
4423f824deb5SChao Yu 
4424f824deb5SChao Yu 	if (!f2fs_is_multi_device(sbi))
4425f824deb5SChao Yu 		return bdev_read_only(sbi->sb->s_bdev);
4426f824deb5SChao Yu 
4427f824deb5SChao Yu 	for (i = 0; i < sbi->s_ndevs; i++)
4428f824deb5SChao Yu 		if (bdev_read_only(FDEV(i).bdev))
4429f824deb5SChao Yu 			return true;
4430f824deb5SChao Yu 	return false;
4431f824deb5SChao Yu }
4432f824deb5SChao Yu 
4433b0332a0fSChao Yu static inline bool f2fs_lfs_mode(struct f2fs_sb_info *sbi)
443452763a4bSJaegeuk Kim {
4435b0332a0fSChao Yu 	return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
443652763a4bSJaegeuk Kim }
443752763a4bSJaegeuk Kim 
44387a8fc586SDaeho Jeong static inline bool f2fs_low_mem_mode(struct f2fs_sb_info *sbi)
44397a8fc586SDaeho Jeong {
44407a8fc586SDaeho Jeong 	return F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_LOW;
44417a8fc586SDaeho Jeong }
44427a8fc586SDaeho Jeong 
44434c8ff709SChao Yu static inline bool f2fs_may_compress(struct inode *inode)
44444c8ff709SChao Yu {
44454c8ff709SChao Yu 	if (IS_SWAPFILE(inode) || f2fs_is_pinned_file(inode) ||
44467165841dSChao Yu 		f2fs_is_atomic_file(inode) || f2fs_has_inline_data(inode))
44474c8ff709SChao Yu 		return false;
44484c8ff709SChao Yu 	return S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode);
44494c8ff709SChao Yu }
44504c8ff709SChao Yu 
44514c8ff709SChao Yu static inline void f2fs_i_compr_blocks_update(struct inode *inode,
44524c8ff709SChao Yu 						u64 blocks, bool add)
44534c8ff709SChao Yu {
4454c2759ebaSDaeho Jeong 	struct f2fs_inode_info *fi = F2FS_I(inode);
4455054cb289SYufen Yu 	int diff = fi->i_cluster_size - blocks;
44564c8ff709SChao Yu 
4457ef8d563fSChao Yu 	/* don't update i_compr_blocks if saved blocks were released */
4458c2759ebaSDaeho Jeong 	if (!add && !atomic_read(&fi->i_compr_blocks))
4459ef8d563fSChao Yu 		return;
4460ef8d563fSChao Yu 
44614c8ff709SChao Yu 	if (add) {
4462c2759ebaSDaeho Jeong 		atomic_add(diff, &fi->i_compr_blocks);
44634c8ff709SChao Yu 		stat_add_compr_blocks(inode, diff);
44644c8ff709SChao Yu 	} else {
4465c2759ebaSDaeho Jeong 		atomic_sub(diff, &fi->i_compr_blocks);
44664c8ff709SChao Yu 		stat_sub_compr_blocks(inode, diff);
44674c8ff709SChao Yu 	}
44684c8ff709SChao Yu 	f2fs_mark_inode_dirty_sync(inode, true);
44694c8ff709SChao Yu }
44704c8ff709SChao Yu 
447171f2c820SChao Yu static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi,
447271f2c820SChao Yu 								int flag)
447371f2c820SChao Yu {
447471f2c820SChao Yu 	if (!f2fs_is_multi_device(sbi))
447571f2c820SChao Yu 		return false;
447671f2c820SChao Yu 	if (flag != F2FS_GET_BLOCK_DIO)
447771f2c820SChao Yu 		return false;
447871f2c820SChao Yu 	return sbi->aligned_blksize;
447971f2c820SChao Yu }
448071f2c820SChao Yu 
44817f59b277SEric Biggers static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
44827f59b277SEric Biggers {
44837f59b277SEric Biggers 	return fsverity_active(inode) &&
44847f59b277SEric Biggers 	       idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
44857f59b277SEric Biggers }
44867f59b277SEric Biggers 
448783a3bfdbSJaegeuk Kim #ifdef CONFIG_F2FS_FAULT_INJECTION
4488d494500aSChao Yu extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate,
4489d494500aSChao Yu 							unsigned int type);
449083a3bfdbSJaegeuk Kim #else
4491d494500aSChao Yu #define f2fs_build_fault_attr(sbi, rate, type)		do { } while (0)
449283a3bfdbSJaegeuk Kim #endif
449383a3bfdbSJaegeuk Kim 
4494af033b2aSChao Yu static inline bool is_journalled_quota(struct f2fs_sb_info *sbi)
4495af033b2aSChao Yu {
4496af033b2aSChao Yu #ifdef CONFIG_QUOTA
44977beb01f7SChao Yu 	if (f2fs_sb_has_quota_ino(sbi))
4498af033b2aSChao Yu 		return true;
4499af033b2aSChao Yu 	if (F2FS_OPTION(sbi).s_qf_names[USRQUOTA] ||
4500af033b2aSChao Yu 		F2FS_OPTION(sbi).s_qf_names[GRPQUOTA] ||
4501af033b2aSChao Yu 		F2FS_OPTION(sbi).s_qf_names[PRJQUOTA])
4502af033b2aSChao Yu 		return true;
4503af033b2aSChao Yu #endif
4504af033b2aSChao Yu 	return false;
4505af033b2aSChao Yu }
45060af725fcSJaegeuk Kim 
45074f993264SChao Yu static inline bool f2fs_block_unit_discard(struct f2fs_sb_info *sbi)
45084f993264SChao Yu {
45094f993264SChao Yu 	return F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK;
45104f993264SChao Yu }
45114f993264SChao Yu 
4512a64239d0SNeilBrown static inline void f2fs_io_schedule_timeout(long timeout)
4513a64239d0SNeilBrown {
4514a64239d0SNeilBrown 	set_current_state(TASK_UNINTERRUPTIBLE);
4515a64239d0SNeilBrown 	io_schedule_timeout(timeout);
4516a64239d0SNeilBrown }
4517a64239d0SNeilBrown 
4518a7b8618aSJaegeuk Kim static inline void f2fs_handle_page_eio(struct f2fs_sb_info *sbi, pgoff_t ofs,
4519a7b8618aSJaegeuk Kim 					enum page_type type)
4520a7b8618aSJaegeuk Kim {
4521a7b8618aSJaegeuk Kim 	if (unlikely(f2fs_cp_error(sbi)))
4522a7b8618aSJaegeuk Kim 		return;
4523a7b8618aSJaegeuk Kim 
4524a7b8618aSJaegeuk Kim 	if (ofs == sbi->page_eio_ofs[type]) {
4525a7b8618aSJaegeuk Kim 		if (sbi->page_eio_cnt[type]++ == MAX_RETRY_PAGE_EIO)
4526a7b8618aSJaegeuk Kim 			set_ckpt_flags(sbi, CP_ERROR_FLAG);
4527a7b8618aSJaegeuk Kim 	} else {
4528a7b8618aSJaegeuk Kim 		sbi->page_eio_ofs[type] = ofs;
4529a7b8618aSJaegeuk Kim 		sbi->page_eio_cnt[type] = 0;
4530a7b8618aSJaegeuk Kim 	}
4531a7b8618aSJaegeuk Kim }
4532a7b8618aSJaegeuk Kim 
4533ed8ac22bSYangtao Li static inline bool f2fs_is_readonly(struct f2fs_sb_info *sbi)
4534ed8ac22bSYangtao Li {
4535ed8ac22bSYangtao Li 	return f2fs_sb_has_readonly(sbi) || f2fs_readonly(sbi->sb);
4536ed8ac22bSYangtao Li }
4537ed8ac22bSYangtao Li 
453810f966bbSChao Yu #define EFSBADCRC	EBADMSG		/* Bad CRC detected */
453910f966bbSChao Yu #define EFSCORRUPTED	EUCLEAN		/* Filesystem is corrupted */
454010f966bbSChao Yu 
4541e1074d4bSChao Yu #endif /* _LINUX_F2FS_H */
4542