xref: /openbmc/linux/fs/f2fs/f2fs.h (revision 7fd53f41)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * fs/f2fs/f2fs.h
4  *
5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6  *             http://www.samsung.com/
7  */
8 #ifndef _LINUX_F2FS_H
9 #define _LINUX_F2FS_H
10 
11 #include <linux/uio.h>
12 #include <linux/types.h>
13 #include <linux/page-flags.h>
14 #include <linux/buffer_head.h>
15 #include <linux/slab.h>
16 #include <linux/crc32.h>
17 #include <linux/magic.h>
18 #include <linux/kobject.h>
19 #include <linux/sched.h>
20 #include <linux/cred.h>
21 #include <linux/vmalloc.h>
22 #include <linux/bio.h>
23 #include <linux/blkdev.h>
24 #include <linux/quotaops.h>
25 #include <linux/part_stat.h>
26 #include <crypto/hash.h>
27 
28 #include <linux/fscrypt.h>
29 #include <linux/fsverity.h>
30 
31 #ifdef CONFIG_F2FS_CHECK_FS
32 #define f2fs_bug_on(sbi, condition)	BUG_ON(condition)
33 #else
34 #define f2fs_bug_on(sbi, condition)					\
35 	do {								\
36 		if (WARN_ON(condition))					\
37 			set_sbi_flag(sbi, SBI_NEED_FSCK);		\
38 	} while (0)
39 #endif
40 
41 enum {
42 	FAULT_KMALLOC,
43 	FAULT_KVMALLOC,
44 	FAULT_PAGE_ALLOC,
45 	FAULT_PAGE_GET,
46 	FAULT_ALLOC_NID,
47 	FAULT_ORPHAN,
48 	FAULT_BLOCK,
49 	FAULT_DIR_DEPTH,
50 	FAULT_EVICT_INODE,
51 	FAULT_TRUNCATE,
52 	FAULT_READ_IO,
53 	FAULT_CHECKPOINT,
54 	FAULT_DISCARD,
55 	FAULT_WRITE_IO,
56 	FAULT_MAX,
57 };
58 
59 #ifdef CONFIG_F2FS_FAULT_INJECTION
60 #define F2FS_ALL_FAULT_TYPE		((1 << FAULT_MAX) - 1)
61 
62 struct f2fs_fault_info {
63 	atomic_t inject_ops;
64 	unsigned int inject_rate;
65 	unsigned int inject_type;
66 };
67 
68 extern const char *f2fs_fault_name[FAULT_MAX];
69 #define IS_FAULT_SET(fi, type) ((fi)->inject_type & (1 << (type)))
70 #endif
71 
72 /*
73  * For mount options
74  */
75 #define F2FS_MOUNT_DISABLE_ROLL_FORWARD	0x00000002
76 #define F2FS_MOUNT_DISCARD		0x00000004
77 #define F2FS_MOUNT_NOHEAP		0x00000008
78 #define F2FS_MOUNT_XATTR_USER		0x00000010
79 #define F2FS_MOUNT_POSIX_ACL		0x00000020
80 #define F2FS_MOUNT_DISABLE_EXT_IDENTIFY	0x00000040
81 #define F2FS_MOUNT_INLINE_XATTR		0x00000080
82 #define F2FS_MOUNT_INLINE_DATA		0x00000100
83 #define F2FS_MOUNT_INLINE_DENTRY	0x00000200
84 #define F2FS_MOUNT_FLUSH_MERGE		0x00000400
85 #define F2FS_MOUNT_NOBARRIER		0x00000800
86 #define F2FS_MOUNT_FASTBOOT		0x00001000
87 #define F2FS_MOUNT_EXTENT_CACHE		0x00002000
88 #define F2FS_MOUNT_DATA_FLUSH		0x00008000
89 #define F2FS_MOUNT_FAULT_INJECTION	0x00010000
90 #define F2FS_MOUNT_USRQUOTA		0x00080000
91 #define F2FS_MOUNT_GRPQUOTA		0x00100000
92 #define F2FS_MOUNT_PRJQUOTA		0x00200000
93 #define F2FS_MOUNT_QUOTA		0x00400000
94 #define F2FS_MOUNT_INLINE_XATTR_SIZE	0x00800000
95 #define F2FS_MOUNT_RESERVE_ROOT		0x01000000
96 #define F2FS_MOUNT_DISABLE_CHECKPOINT	0x02000000
97 #define F2FS_MOUNT_NORECOVERY		0x04000000
98 #define F2FS_MOUNT_ATGC			0x08000000
99 #define F2FS_MOUNT_MERGE_CHECKPOINT	0x10000000
100 
101 #define F2FS_OPTION(sbi)	((sbi)->mount_opt)
102 #define clear_opt(sbi, option)	(F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option)
103 #define set_opt(sbi, option)	(F2FS_OPTION(sbi).opt |= F2FS_MOUNT_##option)
104 #define test_opt(sbi, option)	(F2FS_OPTION(sbi).opt & F2FS_MOUNT_##option)
105 
106 #define ver_after(a, b)	(typecheck(unsigned long long, a) &&		\
107 		typecheck(unsigned long long, b) &&			\
108 		((long long)((a) - (b)) > 0))
109 
110 typedef u32 block_t;	/*
111 			 * should not change u32, since it is the on-disk block
112 			 * address format, __le32.
113 			 */
114 typedef u32 nid_t;
115 
116 #define COMPRESS_EXT_NUM		16
117 
118 struct f2fs_mount_info {
119 	unsigned int opt;
120 	int write_io_size_bits;		/* Write IO size bits */
121 	block_t root_reserved_blocks;	/* root reserved blocks */
122 	kuid_t s_resuid;		/* reserved blocks for uid */
123 	kgid_t s_resgid;		/* reserved blocks for gid */
124 	int active_logs;		/* # of active logs */
125 	int inline_xattr_size;		/* inline xattr size */
126 #ifdef CONFIG_F2FS_FAULT_INJECTION
127 	struct f2fs_fault_info fault_info;	/* For fault injection */
128 #endif
129 #ifdef CONFIG_QUOTA
130 	/* Names of quota files with journalled quota */
131 	char *s_qf_names[MAXQUOTAS];
132 	int s_jquota_fmt;			/* Format of quota to use */
133 #endif
134 	/* For which write hints are passed down to block layer */
135 	int whint_mode;
136 	int alloc_mode;			/* segment allocation policy */
137 	int fsync_mode;			/* fsync policy */
138 	int fs_mode;			/* fs mode: LFS or ADAPTIVE */
139 	int bggc_mode;			/* bggc mode: off, on or sync */
140 	struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
141 	block_t unusable_cap_perc;	/* percentage for cap */
142 	block_t unusable_cap;		/* Amount of space allowed to be
143 					 * unusable when disabling checkpoint
144 					 */
145 
146 	/* For compression */
147 	unsigned char compress_algorithm;	/* algorithm type */
148 	unsigned char compress_log_size;	/* cluster log size */
149 	unsigned char compress_level;		/* compress level */
150 	bool compress_chksum;			/* compressed data chksum */
151 	unsigned char compress_ext_cnt;		/* extension count */
152 	int compress_mode;			/* compression mode */
153 	unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN];	/* extensions */
154 };
155 
156 #define F2FS_FEATURE_ENCRYPT		0x0001
157 #define F2FS_FEATURE_BLKZONED		0x0002
158 #define F2FS_FEATURE_ATOMIC_WRITE	0x0004
159 #define F2FS_FEATURE_EXTRA_ATTR		0x0008
160 #define F2FS_FEATURE_PRJQUOTA		0x0010
161 #define F2FS_FEATURE_INODE_CHKSUM	0x0020
162 #define F2FS_FEATURE_FLEXIBLE_INLINE_XATTR	0x0040
163 #define F2FS_FEATURE_QUOTA_INO		0x0080
164 #define F2FS_FEATURE_INODE_CRTIME	0x0100
165 #define F2FS_FEATURE_LOST_FOUND		0x0200
166 #define F2FS_FEATURE_VERITY		0x0400
167 #define F2FS_FEATURE_SB_CHKSUM		0x0800
168 #define F2FS_FEATURE_CASEFOLD		0x1000
169 #define F2FS_FEATURE_COMPRESSION	0x2000
170 
171 #define __F2FS_HAS_FEATURE(raw_super, mask)				\
172 	((raw_super->feature & cpu_to_le32(mask)) != 0)
173 #define F2FS_HAS_FEATURE(sbi, mask)	__F2FS_HAS_FEATURE(sbi->raw_super, mask)
174 #define F2FS_SET_FEATURE(sbi, mask)					\
175 	(sbi->raw_super->feature |= cpu_to_le32(mask))
176 #define F2FS_CLEAR_FEATURE(sbi, mask)					\
177 	(sbi->raw_super->feature &= ~cpu_to_le32(mask))
178 
179 /*
180  * Default values for user and/or group using reserved blocks
181  */
182 #define	F2FS_DEF_RESUID		0
183 #define	F2FS_DEF_RESGID		0
184 
185 /*
186  * For checkpoint manager
187  */
188 enum {
189 	NAT_BITMAP,
190 	SIT_BITMAP
191 };
192 
193 #define	CP_UMOUNT	0x00000001
194 #define	CP_FASTBOOT	0x00000002
195 #define	CP_SYNC		0x00000004
196 #define	CP_RECOVERY	0x00000008
197 #define	CP_DISCARD	0x00000010
198 #define CP_TRIMMED	0x00000020
199 #define CP_PAUSE	0x00000040
200 #define CP_RESIZE 	0x00000080
201 
202 #define MAX_DISCARD_BLOCKS(sbi)		BLKS_PER_SEC(sbi)
203 #define DEF_MAX_DISCARD_REQUEST		8	/* issue 8 discards per round */
204 #define DEF_MIN_DISCARD_ISSUE_TIME	50	/* 50 ms, if exists */
205 #define DEF_MID_DISCARD_ISSUE_TIME	500	/* 500 ms, if device busy */
206 #define DEF_MAX_DISCARD_ISSUE_TIME	60000	/* 60 s, if no candidates */
207 #define DEF_DISCARD_URGENT_UTIL		80	/* do more discard over 80% */
208 #define DEF_CP_INTERVAL			60	/* 60 secs */
209 #define DEF_IDLE_INTERVAL		5	/* 5 secs */
210 #define DEF_DISABLE_INTERVAL		5	/* 5 secs */
211 #define DEF_DISABLE_QUICK_INTERVAL	1	/* 1 secs */
212 #define DEF_UMOUNT_DISCARD_TIMEOUT	5	/* 5 secs */
213 
214 struct cp_control {
215 	int reason;
216 	__u64 trim_start;
217 	__u64 trim_end;
218 	__u64 trim_minlen;
219 };
220 
221 /*
222  * indicate meta/data type
223  */
224 enum {
225 	META_CP,
226 	META_NAT,
227 	META_SIT,
228 	META_SSA,
229 	META_MAX,
230 	META_POR,
231 	DATA_GENERIC,		/* check range only */
232 	DATA_GENERIC_ENHANCE,	/* strong check on range and segment bitmap */
233 	DATA_GENERIC_ENHANCE_READ,	/*
234 					 * strong check on range and segment
235 					 * bitmap but no warning due to race
236 					 * condition of read on truncated area
237 					 * by extent_cache
238 					 */
239 	META_GENERIC,
240 };
241 
242 /* for the list of ino */
243 enum {
244 	ORPHAN_INO,		/* for orphan ino list */
245 	APPEND_INO,		/* for append ino list */
246 	UPDATE_INO,		/* for update ino list */
247 	TRANS_DIR_INO,		/* for trasactions dir ino list */
248 	FLUSH_INO,		/* for multiple device flushing */
249 	MAX_INO_ENTRY,		/* max. list */
250 };
251 
252 struct ino_entry {
253 	struct list_head list;		/* list head */
254 	nid_t ino;			/* inode number */
255 	unsigned int dirty_device;	/* dirty device bitmap */
256 };
257 
258 /* for the list of inodes to be GCed */
259 struct inode_entry {
260 	struct list_head list;	/* list head */
261 	struct inode *inode;	/* vfs inode pointer */
262 };
263 
264 struct fsync_node_entry {
265 	struct list_head list;	/* list head */
266 	struct page *page;	/* warm node page pointer */
267 	unsigned int seq_id;	/* sequence id */
268 };
269 
270 struct ckpt_req {
271 	struct completion wait;		/* completion for checkpoint done */
272 	struct llist_node llnode;	/* llist_node to be linked in wait queue */
273 	int ret;			/* return code of checkpoint */
274 	ktime_t queue_time;		/* request queued time */
275 };
276 
277 struct ckpt_req_control {
278 	struct task_struct *f2fs_issue_ckpt;	/* checkpoint task */
279 	int ckpt_thread_ioprio;			/* checkpoint merge thread ioprio */
280 	wait_queue_head_t ckpt_wait_queue;	/* waiting queue for wake-up */
281 	atomic_t issued_ckpt;		/* # of actually issued ckpts */
282 	atomic_t total_ckpt;		/* # of total ckpts */
283 	atomic_t queued_ckpt;		/* # of queued ckpts */
284 	struct llist_head issue_list;	/* list for command issue */
285 	spinlock_t stat_lock;		/* lock for below checkpoint time stats */
286 	unsigned int cur_time;		/* cur wait time in msec for currently issued checkpoint */
287 	unsigned int peak_time;		/* peak wait time in msec until now */
288 };
289 
290 /* for the bitmap indicate blocks to be discarded */
291 struct discard_entry {
292 	struct list_head list;	/* list head */
293 	block_t start_blkaddr;	/* start blockaddr of current segment */
294 	unsigned char discard_map[SIT_VBLOCK_MAP_SIZE];	/* segment discard bitmap */
295 };
296 
297 /* default discard granularity of inner discard thread, unit: block count */
298 #define DEFAULT_DISCARD_GRANULARITY		16
299 
300 /* max discard pend list number */
301 #define MAX_PLIST_NUM		512
302 #define plist_idx(blk_num)	((blk_num) >= MAX_PLIST_NUM ?		\
303 					(MAX_PLIST_NUM - 1) : ((blk_num) - 1))
304 
305 enum {
306 	D_PREP,			/* initial */
307 	D_PARTIAL,		/* partially submitted */
308 	D_SUBMIT,		/* all submitted */
309 	D_DONE,			/* finished */
310 };
311 
312 struct discard_info {
313 	block_t lstart;			/* logical start address */
314 	block_t len;			/* length */
315 	block_t start;			/* actual start address in dev */
316 };
317 
318 struct discard_cmd {
319 	struct rb_node rb_node;		/* rb node located in rb-tree */
320 	union {
321 		struct {
322 			block_t lstart;	/* logical start address */
323 			block_t len;	/* length */
324 			block_t start;	/* actual start address in dev */
325 		};
326 		struct discard_info di;	/* discard info */
327 
328 	};
329 	struct list_head list;		/* command list */
330 	struct completion wait;		/* compleation */
331 	struct block_device *bdev;	/* bdev */
332 	unsigned short ref;		/* reference count */
333 	unsigned char state;		/* state */
334 	unsigned char queued;		/* queued discard */
335 	int error;			/* bio error */
336 	spinlock_t lock;		/* for state/bio_ref updating */
337 	unsigned short bio_ref;		/* bio reference count */
338 };
339 
340 enum {
341 	DPOLICY_BG,
342 	DPOLICY_FORCE,
343 	DPOLICY_FSTRIM,
344 	DPOLICY_UMOUNT,
345 	MAX_DPOLICY,
346 };
347 
348 struct discard_policy {
349 	int type;			/* type of discard */
350 	unsigned int min_interval;	/* used for candidates exist */
351 	unsigned int mid_interval;	/* used for device busy */
352 	unsigned int max_interval;	/* used for candidates not exist */
353 	unsigned int max_requests;	/* # of discards issued per round */
354 	unsigned int io_aware_gran;	/* minimum granularity discard not be aware of I/O */
355 	bool io_aware;			/* issue discard in idle time */
356 	bool sync;			/* submit discard with REQ_SYNC flag */
357 	bool ordered;			/* issue discard by lba order */
358 	bool timeout;			/* discard timeout for put_super */
359 	unsigned int granularity;	/* discard granularity */
360 };
361 
362 struct discard_cmd_control {
363 	struct task_struct *f2fs_issue_discard;	/* discard thread */
364 	struct list_head entry_list;		/* 4KB discard entry list */
365 	struct list_head pend_list[MAX_PLIST_NUM];/* store pending entries */
366 	struct list_head wait_list;		/* store on-flushing entries */
367 	struct list_head fstrim_list;		/* in-flight discard from fstrim */
368 	wait_queue_head_t discard_wait_queue;	/* waiting queue for wake-up */
369 	unsigned int discard_wake;		/* to wake up discard thread */
370 	struct mutex cmd_lock;
371 	unsigned int nr_discards;		/* # of discards in the list */
372 	unsigned int max_discards;		/* max. discards to be issued */
373 	unsigned int discard_granularity;	/* discard granularity */
374 	unsigned int undiscard_blks;		/* # of undiscard blocks */
375 	unsigned int next_pos;			/* next discard position */
376 	atomic_t issued_discard;		/* # of issued discard */
377 	atomic_t queued_discard;		/* # of queued discard */
378 	atomic_t discard_cmd_cnt;		/* # of cached cmd count */
379 	struct rb_root_cached root;		/* root of discard rb-tree */
380 	bool rbtree_check;			/* config for consistence check */
381 };
382 
383 /* for the list of fsync inodes, used only during recovery */
384 struct fsync_inode_entry {
385 	struct list_head list;	/* list head */
386 	struct inode *inode;	/* vfs inode pointer */
387 	block_t blkaddr;	/* block address locating the last fsync */
388 	block_t last_dentry;	/* block address locating the last dentry */
389 };
390 
391 #define nats_in_cursum(jnl)		(le16_to_cpu((jnl)->n_nats))
392 #define sits_in_cursum(jnl)		(le16_to_cpu((jnl)->n_sits))
393 
394 #define nat_in_journal(jnl, i)		((jnl)->nat_j.entries[i].ne)
395 #define nid_in_journal(jnl, i)		((jnl)->nat_j.entries[i].nid)
396 #define sit_in_journal(jnl, i)		((jnl)->sit_j.entries[i].se)
397 #define segno_in_journal(jnl, i)	((jnl)->sit_j.entries[i].segno)
398 
399 #define MAX_NAT_JENTRIES(jnl)	(NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl))
400 #define MAX_SIT_JENTRIES(jnl)	(SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl))
401 
402 static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i)
403 {
404 	int before = nats_in_cursum(journal);
405 
406 	journal->n_nats = cpu_to_le16(before + i);
407 	return before;
408 }
409 
410 static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i)
411 {
412 	int before = sits_in_cursum(journal);
413 
414 	journal->n_sits = cpu_to_le16(before + i);
415 	return before;
416 }
417 
418 static inline bool __has_cursum_space(struct f2fs_journal *journal,
419 							int size, int type)
420 {
421 	if (type == NAT_JOURNAL)
422 		return size <= MAX_NAT_JENTRIES(journal);
423 	return size <= MAX_SIT_JENTRIES(journal);
424 }
425 
426 /* for inline stuff */
427 #define DEF_INLINE_RESERVED_SIZE	1
428 static inline int get_extra_isize(struct inode *inode);
429 static inline int get_inline_xattr_addrs(struct inode *inode);
430 #define MAX_INLINE_DATA(inode)	(sizeof(__le32) *			\
431 				(CUR_ADDRS_PER_INODE(inode) -		\
432 				get_inline_xattr_addrs(inode) -	\
433 				DEF_INLINE_RESERVED_SIZE))
434 
435 /* for inline dir */
436 #define NR_INLINE_DENTRY(inode)	(MAX_INLINE_DATA(inode) * BITS_PER_BYTE / \
437 				((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
438 				BITS_PER_BYTE + 1))
439 #define INLINE_DENTRY_BITMAP_SIZE(inode) \
440 	DIV_ROUND_UP(NR_INLINE_DENTRY(inode), BITS_PER_BYTE)
441 #define INLINE_RESERVED_SIZE(inode)	(MAX_INLINE_DATA(inode) - \
442 				((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \
443 				NR_INLINE_DENTRY(inode) + \
444 				INLINE_DENTRY_BITMAP_SIZE(inode)))
445 
446 /*
447  * For INODE and NODE manager
448  */
449 /* for directory operations */
450 
451 struct f2fs_filename {
452 	/*
453 	 * The filename the user specified.  This is NULL for some
454 	 * filesystem-internal operations, e.g. converting an inline directory
455 	 * to a non-inline one, or roll-forward recovering an encrypted dentry.
456 	 */
457 	const struct qstr *usr_fname;
458 
459 	/*
460 	 * The on-disk filename.  For encrypted directories, this is encrypted.
461 	 * This may be NULL for lookups in an encrypted dir without the key.
462 	 */
463 	struct fscrypt_str disk_name;
464 
465 	/* The dirhash of this filename */
466 	f2fs_hash_t hash;
467 
468 #ifdef CONFIG_FS_ENCRYPTION
469 	/*
470 	 * For lookups in encrypted directories: either the buffer backing
471 	 * disk_name, or a buffer that holds the decoded no-key name.
472 	 */
473 	struct fscrypt_str crypto_buf;
474 #endif
475 #ifdef CONFIG_UNICODE
476 	/*
477 	 * For casefolded directories: the casefolded name, but it's left NULL
478 	 * if the original name is not valid Unicode, if the directory is both
479 	 * casefolded and encrypted and its encryption key is unavailable, or if
480 	 * the filesystem is doing an internal operation where usr_fname is also
481 	 * NULL.  In all these cases we fall back to treating the name as an
482 	 * opaque byte sequence.
483 	 */
484 	struct fscrypt_str cf_name;
485 #endif
486 };
487 
488 struct f2fs_dentry_ptr {
489 	struct inode *inode;
490 	void *bitmap;
491 	struct f2fs_dir_entry *dentry;
492 	__u8 (*filename)[F2FS_SLOT_LEN];
493 	int max;
494 	int nr_bitmap;
495 };
496 
497 static inline void make_dentry_ptr_block(struct inode *inode,
498 		struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t)
499 {
500 	d->inode = inode;
501 	d->max = NR_DENTRY_IN_BLOCK;
502 	d->nr_bitmap = SIZE_OF_DENTRY_BITMAP;
503 	d->bitmap = t->dentry_bitmap;
504 	d->dentry = t->dentry;
505 	d->filename = t->filename;
506 }
507 
508 static inline void make_dentry_ptr_inline(struct inode *inode,
509 					struct f2fs_dentry_ptr *d, void *t)
510 {
511 	int entry_cnt = NR_INLINE_DENTRY(inode);
512 	int bitmap_size = INLINE_DENTRY_BITMAP_SIZE(inode);
513 	int reserved_size = INLINE_RESERVED_SIZE(inode);
514 
515 	d->inode = inode;
516 	d->max = entry_cnt;
517 	d->nr_bitmap = bitmap_size;
518 	d->bitmap = t;
519 	d->dentry = t + bitmap_size + reserved_size;
520 	d->filename = t + bitmap_size + reserved_size +
521 					SIZE_OF_DIR_ENTRY * entry_cnt;
522 }
523 
524 /*
525  * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1
526  * as its node offset to distinguish from index node blocks.
527  * But some bits are used to mark the node block.
528  */
529 #define XATTR_NODE_OFFSET	((((unsigned int)-1) << OFFSET_BIT_SHIFT) \
530 				>> OFFSET_BIT_SHIFT)
531 enum {
532 	ALLOC_NODE,			/* allocate a new node page if needed */
533 	LOOKUP_NODE,			/* look up a node without readahead */
534 	LOOKUP_NODE_RA,			/*
535 					 * look up a node with readahead called
536 					 * by get_data_block.
537 					 */
538 };
539 
540 #define DEFAULT_RETRY_IO_COUNT	8	/* maximum retry read IO count */
541 
542 /* congestion wait timeout value, default: 20ms */
543 #define	DEFAULT_IO_TIMEOUT	(msecs_to_jiffies(20))
544 
545 /* maximum retry quota flush count */
546 #define DEFAULT_RETRY_QUOTA_FLUSH_COUNT		8
547 
548 #define F2FS_LINK_MAX	0xffffffff	/* maximum link count per file */
549 
550 #define MAX_DIR_RA_PAGES	4	/* maximum ra pages of dir */
551 
552 /* for in-memory extent cache entry */
553 #define F2FS_MIN_EXTENT_LEN	64	/* minimum extent length */
554 
555 /* number of extent info in extent cache we try to shrink */
556 #define EXTENT_CACHE_SHRINK_NUMBER	128
557 
558 struct rb_entry {
559 	struct rb_node rb_node;		/* rb node located in rb-tree */
560 	union {
561 		struct {
562 			unsigned int ofs;	/* start offset of the entry */
563 			unsigned int len;	/* length of the entry */
564 		};
565 		unsigned long long key;		/* 64-bits key */
566 	} __packed;
567 };
568 
569 struct extent_info {
570 	unsigned int fofs;		/* start offset in a file */
571 	unsigned int len;		/* length of the extent */
572 	u32 blk;			/* start block address of the extent */
573 };
574 
575 struct extent_node {
576 	struct rb_node rb_node;		/* rb node located in rb-tree */
577 	struct extent_info ei;		/* extent info */
578 	struct list_head list;		/* node in global extent list of sbi */
579 	struct extent_tree *et;		/* extent tree pointer */
580 };
581 
582 struct extent_tree {
583 	nid_t ino;			/* inode number */
584 	struct rb_root_cached root;	/* root of extent info rb-tree */
585 	struct extent_node *cached_en;	/* recently accessed extent node */
586 	struct extent_info largest;	/* largested extent info */
587 	struct list_head list;		/* to be used by sbi->zombie_list */
588 	rwlock_t lock;			/* protect extent info rb-tree */
589 	atomic_t node_cnt;		/* # of extent node in rb-tree*/
590 	bool largest_updated;		/* largest extent updated */
591 };
592 
593 /*
594  * This structure is taken from ext4_map_blocks.
595  *
596  * Note that, however, f2fs uses NEW and MAPPED flags for f2fs_map_blocks().
597  */
598 #define F2FS_MAP_NEW		(1 << BH_New)
599 #define F2FS_MAP_MAPPED		(1 << BH_Mapped)
600 #define F2FS_MAP_UNWRITTEN	(1 << BH_Unwritten)
601 #define F2FS_MAP_FLAGS		(F2FS_MAP_NEW | F2FS_MAP_MAPPED |\
602 				F2FS_MAP_UNWRITTEN)
603 
604 struct f2fs_map_blocks {
605 	block_t m_pblk;
606 	block_t m_lblk;
607 	unsigned int m_len;
608 	unsigned int m_flags;
609 	pgoff_t *m_next_pgofs;		/* point next possible non-hole pgofs */
610 	pgoff_t *m_next_extent;		/* point to next possible extent */
611 	int m_seg_type;
612 	bool m_may_create;		/* indicate it is from write path */
613 };
614 
615 /* for flag in get_data_block */
616 enum {
617 	F2FS_GET_BLOCK_DEFAULT,
618 	F2FS_GET_BLOCK_FIEMAP,
619 	F2FS_GET_BLOCK_BMAP,
620 	F2FS_GET_BLOCK_DIO,
621 	F2FS_GET_BLOCK_PRE_DIO,
622 	F2FS_GET_BLOCK_PRE_AIO,
623 	F2FS_GET_BLOCK_PRECACHE,
624 };
625 
626 /*
627  * i_advise uses FADVISE_XXX_BIT. We can add additional hints later.
628  */
629 #define FADVISE_COLD_BIT	0x01
630 #define FADVISE_LOST_PINO_BIT	0x02
631 #define FADVISE_ENCRYPT_BIT	0x04
632 #define FADVISE_ENC_NAME_BIT	0x08
633 #define FADVISE_KEEP_SIZE_BIT	0x10
634 #define FADVISE_HOT_BIT		0x20
635 #define FADVISE_VERITY_BIT	0x40
636 
637 #define FADVISE_MODIFIABLE_BITS	(FADVISE_COLD_BIT | FADVISE_HOT_BIT)
638 
639 #define file_is_cold(inode)	is_file(inode, FADVISE_COLD_BIT)
640 #define file_wrong_pino(inode)	is_file(inode, FADVISE_LOST_PINO_BIT)
641 #define file_set_cold(inode)	set_file(inode, FADVISE_COLD_BIT)
642 #define file_lost_pino(inode)	set_file(inode, FADVISE_LOST_PINO_BIT)
643 #define file_clear_cold(inode)	clear_file(inode, FADVISE_COLD_BIT)
644 #define file_got_pino(inode)	clear_file(inode, FADVISE_LOST_PINO_BIT)
645 #define file_is_encrypt(inode)	is_file(inode, FADVISE_ENCRYPT_BIT)
646 #define file_set_encrypt(inode)	set_file(inode, FADVISE_ENCRYPT_BIT)
647 #define file_clear_encrypt(inode) clear_file(inode, FADVISE_ENCRYPT_BIT)
648 #define file_enc_name(inode)	is_file(inode, FADVISE_ENC_NAME_BIT)
649 #define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
650 #define file_keep_isize(inode)	is_file(inode, FADVISE_KEEP_SIZE_BIT)
651 #define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
652 #define file_is_hot(inode)	is_file(inode, FADVISE_HOT_BIT)
653 #define file_set_hot(inode)	set_file(inode, FADVISE_HOT_BIT)
654 #define file_clear_hot(inode)	clear_file(inode, FADVISE_HOT_BIT)
655 #define file_is_verity(inode)	is_file(inode, FADVISE_VERITY_BIT)
656 #define file_set_verity(inode)	set_file(inode, FADVISE_VERITY_BIT)
657 
658 #define DEF_DIR_LEVEL		0
659 
660 enum {
661 	GC_FAILURE_PIN,
662 	GC_FAILURE_ATOMIC,
663 	MAX_GC_FAILURE
664 };
665 
666 /* used for f2fs_inode_info->flags */
667 enum {
668 	FI_NEW_INODE,		/* indicate newly allocated inode */
669 	FI_DIRTY_INODE,		/* indicate inode is dirty or not */
670 	FI_AUTO_RECOVER,	/* indicate inode is recoverable */
671 	FI_DIRTY_DIR,		/* indicate directory has dirty pages */
672 	FI_INC_LINK,		/* need to increment i_nlink */
673 	FI_ACL_MODE,		/* indicate acl mode */
674 	FI_NO_ALLOC,		/* should not allocate any blocks */
675 	FI_FREE_NID,		/* free allocated nide */
676 	FI_NO_EXTENT,		/* not to use the extent cache */
677 	FI_INLINE_XATTR,	/* used for inline xattr */
678 	FI_INLINE_DATA,		/* used for inline data*/
679 	FI_INLINE_DENTRY,	/* used for inline dentry */
680 	FI_APPEND_WRITE,	/* inode has appended data */
681 	FI_UPDATE_WRITE,	/* inode has in-place-update data */
682 	FI_NEED_IPU,		/* used for ipu per file */
683 	FI_ATOMIC_FILE,		/* indicate atomic file */
684 	FI_ATOMIC_COMMIT,	/* indicate the state of atomical committing */
685 	FI_VOLATILE_FILE,	/* indicate volatile file */
686 	FI_FIRST_BLOCK_WRITTEN,	/* indicate #0 data block was written */
687 	FI_DROP_CACHE,		/* drop dirty page cache */
688 	FI_DATA_EXIST,		/* indicate data exists */
689 	FI_INLINE_DOTS,		/* indicate inline dot dentries */
690 	FI_DO_DEFRAG,		/* indicate defragment is running */
691 	FI_DIRTY_FILE,		/* indicate regular/symlink has dirty pages */
692 	FI_NO_PREALLOC,		/* indicate skipped preallocated blocks */
693 	FI_HOT_DATA,		/* indicate file is hot */
694 	FI_EXTRA_ATTR,		/* indicate file has extra attribute */
695 	FI_PROJ_INHERIT,	/* indicate file inherits projectid */
696 	FI_PIN_FILE,		/* indicate file should not be gced */
697 	FI_ATOMIC_REVOKE_REQUEST, /* request to drop atomic data */
698 	FI_VERITY_IN_PROGRESS,	/* building fs-verity Merkle tree */
699 	FI_COMPRESSED_FILE,	/* indicate file's data can be compressed */
700 	FI_COMPRESS_CORRUPT,	/* indicate compressed cluster is corrupted */
701 	FI_MMAP_FILE,		/* indicate file was mmapped */
702 	FI_ENABLE_COMPRESS,	/* enable compression in "user" compression mode */
703 	FI_MAX,			/* max flag, never be used */
704 };
705 
706 struct f2fs_inode_info {
707 	struct inode vfs_inode;		/* serve a vfs inode */
708 	unsigned long i_flags;		/* keep an inode flags for ioctl */
709 	unsigned char i_advise;		/* use to give file attribute hints */
710 	unsigned char i_dir_level;	/* use for dentry level for large dir */
711 	unsigned int i_current_depth;	/* only for directory depth */
712 	/* for gc failure statistic */
713 	unsigned int i_gc_failures[MAX_GC_FAILURE];
714 	unsigned int i_pino;		/* parent inode number */
715 	umode_t i_acl_mode;		/* keep file acl mode temporarily */
716 
717 	/* Use below internally in f2fs*/
718 	unsigned long flags[BITS_TO_LONGS(FI_MAX)];	/* use to pass per-file flags */
719 	struct rw_semaphore i_sem;	/* protect fi info */
720 	atomic_t dirty_pages;		/* # of dirty pages */
721 	f2fs_hash_t chash;		/* hash value of given file name */
722 	unsigned int clevel;		/* maximum level of given file name */
723 	struct task_struct *task;	/* lookup and create consistency */
724 	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
725 	nid_t i_xattr_nid;		/* node id that contains xattrs */
726 	loff_t	last_disk_size;		/* lastly written file size */
727 	spinlock_t i_size_lock;		/* protect last_disk_size */
728 
729 #ifdef CONFIG_QUOTA
730 	struct dquot *i_dquot[MAXQUOTAS];
731 
732 	/* quota space reservation, managed internally by quota code */
733 	qsize_t i_reserved_quota;
734 #endif
735 	struct list_head dirty_list;	/* dirty list for dirs and files */
736 	struct list_head gdirty_list;	/* linked in global dirty list */
737 	struct list_head inmem_ilist;	/* list for inmem inodes */
738 	struct list_head inmem_pages;	/* inmemory pages managed by f2fs */
739 	struct task_struct *inmem_task;	/* store inmemory task */
740 	struct mutex inmem_lock;	/* lock for inmemory pages */
741 	struct extent_tree *extent_tree;	/* cached extent_tree entry */
742 
743 	/* avoid racing between foreground op and gc */
744 	struct rw_semaphore i_gc_rwsem[2];
745 	struct rw_semaphore i_mmap_sem;
746 	struct rw_semaphore i_xattr_sem; /* avoid racing between reading and changing EAs */
747 
748 	int i_extra_isize;		/* size of extra space located in i_addr */
749 	kprojid_t i_projid;		/* id for project quota */
750 	int i_inline_xattr_size;	/* inline xattr size */
751 	struct timespec64 i_crtime;	/* inode creation time */
752 	struct timespec64 i_disk_time[4];/* inode disk times */
753 
754 	/* for file compress */
755 	atomic_t i_compr_blocks;		/* # of compressed blocks */
756 	unsigned char i_compress_algorithm;	/* algorithm type */
757 	unsigned char i_log_cluster_size;	/* log of cluster size */
758 	unsigned char i_compress_level;		/* compress level (lz4hc,zstd) */
759 	unsigned short i_compress_flag;		/* compress flag */
760 	unsigned int i_cluster_size;		/* cluster size */
761 };
762 
763 static inline void get_extent_info(struct extent_info *ext,
764 					struct f2fs_extent *i_ext)
765 {
766 	ext->fofs = le32_to_cpu(i_ext->fofs);
767 	ext->blk = le32_to_cpu(i_ext->blk);
768 	ext->len = le32_to_cpu(i_ext->len);
769 }
770 
771 static inline void set_raw_extent(struct extent_info *ext,
772 					struct f2fs_extent *i_ext)
773 {
774 	i_ext->fofs = cpu_to_le32(ext->fofs);
775 	i_ext->blk = cpu_to_le32(ext->blk);
776 	i_ext->len = cpu_to_le32(ext->len);
777 }
778 
779 static inline void set_extent_info(struct extent_info *ei, unsigned int fofs,
780 						u32 blk, unsigned int len)
781 {
782 	ei->fofs = fofs;
783 	ei->blk = blk;
784 	ei->len = len;
785 }
786 
787 static inline bool __is_discard_mergeable(struct discard_info *back,
788 			struct discard_info *front, unsigned int max_len)
789 {
790 	return (back->lstart + back->len == front->lstart) &&
791 		(back->len + front->len <= max_len);
792 }
793 
794 static inline bool __is_discard_back_mergeable(struct discard_info *cur,
795 			struct discard_info *back, unsigned int max_len)
796 {
797 	return __is_discard_mergeable(back, cur, max_len);
798 }
799 
800 static inline bool __is_discard_front_mergeable(struct discard_info *cur,
801 			struct discard_info *front, unsigned int max_len)
802 {
803 	return __is_discard_mergeable(cur, front, max_len);
804 }
805 
806 static inline bool __is_extent_mergeable(struct extent_info *back,
807 						struct extent_info *front)
808 {
809 	return (back->fofs + back->len == front->fofs &&
810 			back->blk + back->len == front->blk);
811 }
812 
813 static inline bool __is_back_mergeable(struct extent_info *cur,
814 						struct extent_info *back)
815 {
816 	return __is_extent_mergeable(back, cur);
817 }
818 
819 static inline bool __is_front_mergeable(struct extent_info *cur,
820 						struct extent_info *front)
821 {
822 	return __is_extent_mergeable(cur, front);
823 }
824 
825 extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
826 static inline void __try_update_largest_extent(struct extent_tree *et,
827 						struct extent_node *en)
828 {
829 	if (en->ei.len > et->largest.len) {
830 		et->largest = en->ei;
831 		et->largest_updated = true;
832 	}
833 }
834 
835 /*
836  * For free nid management
837  */
838 enum nid_state {
839 	FREE_NID,		/* newly added to free nid list */
840 	PREALLOC_NID,		/* it is preallocated */
841 	MAX_NID_STATE,
842 };
843 
844 enum nat_state {
845 	TOTAL_NAT,
846 	DIRTY_NAT,
847 	RECLAIMABLE_NAT,
848 	MAX_NAT_STATE,
849 };
850 
851 struct f2fs_nm_info {
852 	block_t nat_blkaddr;		/* base disk address of NAT */
853 	nid_t max_nid;			/* maximum possible node ids */
854 	nid_t available_nids;		/* # of available node ids */
855 	nid_t next_scan_nid;		/* the next nid to be scanned */
856 	unsigned int ram_thresh;	/* control the memory footprint */
857 	unsigned int ra_nid_pages;	/* # of nid pages to be readaheaded */
858 	unsigned int dirty_nats_ratio;	/* control dirty nats ratio threshold */
859 
860 	/* NAT cache management */
861 	struct radix_tree_root nat_root;/* root of the nat entry cache */
862 	struct radix_tree_root nat_set_root;/* root of the nat set cache */
863 	struct rw_semaphore nat_tree_lock;	/* protect nat_tree_lock */
864 	struct list_head nat_entries;	/* cached nat entry list (clean) */
865 	spinlock_t nat_list_lock;	/* protect clean nat entry list */
866 	unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */
867 	unsigned int nat_blocks;	/* # of nat blocks */
868 
869 	/* free node ids management */
870 	struct radix_tree_root free_nid_root;/* root of the free_nid cache */
871 	struct list_head free_nid_list;		/* list for free nids excluding preallocated nids */
872 	unsigned int nid_cnt[MAX_NID_STATE];	/* the number of free node id */
873 	spinlock_t nid_list_lock;	/* protect nid lists ops */
874 	struct mutex build_lock;	/* lock for build free nids */
875 	unsigned char **free_nid_bitmap;
876 	unsigned char *nat_block_bitmap;
877 	unsigned short *free_nid_count;	/* free nid count of NAT block */
878 
879 	/* for checkpoint */
880 	char *nat_bitmap;		/* NAT bitmap pointer */
881 
882 	unsigned int nat_bits_blocks;	/* # of nat bits blocks */
883 	unsigned char *nat_bits;	/* NAT bits blocks */
884 	unsigned char *full_nat_bits;	/* full NAT pages */
885 	unsigned char *empty_nat_bits;	/* empty NAT pages */
886 #ifdef CONFIG_F2FS_CHECK_FS
887 	char *nat_bitmap_mir;		/* NAT bitmap mirror */
888 #endif
889 	int bitmap_size;		/* bitmap size */
890 };
891 
892 /*
893  * this structure is used as one of function parameters.
894  * all the information are dedicated to a given direct node block determined
895  * by the data offset in a file.
896  */
897 struct dnode_of_data {
898 	struct inode *inode;		/* vfs inode pointer */
899 	struct page *inode_page;	/* its inode page, NULL is possible */
900 	struct page *node_page;		/* cached direct node page */
901 	nid_t nid;			/* node id of the direct node block */
902 	unsigned int ofs_in_node;	/* data offset in the node page */
903 	bool inode_page_locked;		/* inode page is locked or not */
904 	bool node_changed;		/* is node block changed */
905 	char cur_level;			/* level of hole node page */
906 	char max_level;			/* level of current page located */
907 	block_t	data_blkaddr;		/* block address of the node block */
908 };
909 
910 static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode,
911 		struct page *ipage, struct page *npage, nid_t nid)
912 {
913 	memset(dn, 0, sizeof(*dn));
914 	dn->inode = inode;
915 	dn->inode_page = ipage;
916 	dn->node_page = npage;
917 	dn->nid = nid;
918 }
919 
920 /*
921  * For SIT manager
922  *
923  * By default, there are 6 active log areas across the whole main area.
924  * When considering hot and cold data separation to reduce cleaning overhead,
925  * we split 3 for data logs and 3 for node logs as hot, warm, and cold types,
926  * respectively.
927  * In the current design, you should not change the numbers intentionally.
928  * Instead, as a mount option such as active_logs=x, you can use 2, 4, and 6
929  * logs individually according to the underlying devices. (default: 6)
930  * Just in case, on-disk layout covers maximum 16 logs that consist of 8 for
931  * data and 8 for node logs.
932  */
933 #define	NR_CURSEG_DATA_TYPE	(3)
934 #define NR_CURSEG_NODE_TYPE	(3)
935 #define NR_CURSEG_INMEM_TYPE	(2)
936 #define NR_CURSEG_PERSIST_TYPE	(NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
937 #define NR_CURSEG_TYPE		(NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE)
938 
939 enum {
940 	CURSEG_HOT_DATA	= 0,	/* directory entry blocks */
941 	CURSEG_WARM_DATA,	/* data blocks */
942 	CURSEG_COLD_DATA,	/* multimedia or GCed data blocks */
943 	CURSEG_HOT_NODE,	/* direct node blocks of directory files */
944 	CURSEG_WARM_NODE,	/* direct node blocks of normal files */
945 	CURSEG_COLD_NODE,	/* indirect node blocks */
946 	NR_PERSISTENT_LOG,	/* number of persistent log */
947 	CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG,
948 				/* pinned file that needs consecutive block address */
949 	CURSEG_ALL_DATA_ATGC,	/* SSR alloctor in hot/warm/cold data area */
950 	NO_CHECK_TYPE,		/* number of persistent & inmem log */
951 };
952 
953 struct flush_cmd {
954 	struct completion wait;
955 	struct llist_node llnode;
956 	nid_t ino;
957 	int ret;
958 };
959 
960 struct flush_cmd_control {
961 	struct task_struct *f2fs_issue_flush;	/* flush thread */
962 	wait_queue_head_t flush_wait_queue;	/* waiting queue for wake-up */
963 	atomic_t issued_flush;			/* # of issued flushes */
964 	atomic_t queued_flush;			/* # of queued flushes */
965 	struct llist_head issue_list;		/* list for command issue */
966 	struct llist_node *dispatch_list;	/* list for command dispatch */
967 };
968 
969 struct f2fs_sm_info {
970 	struct sit_info *sit_info;		/* whole segment information */
971 	struct free_segmap_info *free_info;	/* free segment information */
972 	struct dirty_seglist_info *dirty_info;	/* dirty segment information */
973 	struct curseg_info *curseg_array;	/* active segment information */
974 
975 	struct rw_semaphore curseg_lock;	/* for preventing curseg change */
976 
977 	block_t seg0_blkaddr;		/* block address of 0'th segment */
978 	block_t main_blkaddr;		/* start block address of main area */
979 	block_t ssa_blkaddr;		/* start block address of SSA area */
980 
981 	unsigned int segment_count;	/* total # of segments */
982 	unsigned int main_segments;	/* # of segments in main area */
983 	unsigned int reserved_segments;	/* # of reserved segments */
984 	unsigned int ovp_segments;	/* # of overprovision segments */
985 
986 	/* a threshold to reclaim prefree segments */
987 	unsigned int rec_prefree_segments;
988 
989 	/* for batched trimming */
990 	unsigned int trim_sections;		/* # of sections to trim */
991 
992 	struct list_head sit_entry_set;	/* sit entry set list */
993 
994 	unsigned int ipu_policy;	/* in-place-update policy */
995 	unsigned int min_ipu_util;	/* in-place-update threshold */
996 	unsigned int min_fsync_blocks;	/* threshold for fsync */
997 	unsigned int min_seq_blocks;	/* threshold for sequential blocks */
998 	unsigned int min_hot_blocks;	/* threshold for hot block allocation */
999 	unsigned int min_ssr_sections;	/* threshold to trigger SSR allocation */
1000 
1001 	/* for flush command control */
1002 	struct flush_cmd_control *fcc_info;
1003 
1004 	/* for discard command control */
1005 	struct discard_cmd_control *dcc_info;
1006 };
1007 
1008 /*
1009  * For superblock
1010  */
1011 /*
1012  * COUNT_TYPE for monitoring
1013  *
1014  * f2fs monitors the number of several block types such as on-writeback,
1015  * dirty dentry blocks, dirty node blocks, and dirty meta blocks.
1016  */
1017 #define WB_DATA_TYPE(p)	(__is_cp_guaranteed(p) ? F2FS_WB_CP_DATA : F2FS_WB_DATA)
1018 enum count_type {
1019 	F2FS_DIRTY_DENTS,
1020 	F2FS_DIRTY_DATA,
1021 	F2FS_DIRTY_QDATA,
1022 	F2FS_DIRTY_NODES,
1023 	F2FS_DIRTY_META,
1024 	F2FS_INMEM_PAGES,
1025 	F2FS_DIRTY_IMETA,
1026 	F2FS_WB_CP_DATA,
1027 	F2FS_WB_DATA,
1028 	F2FS_RD_DATA,
1029 	F2FS_RD_NODE,
1030 	F2FS_RD_META,
1031 	F2FS_DIO_WRITE,
1032 	F2FS_DIO_READ,
1033 	NR_COUNT_TYPE,
1034 };
1035 
1036 /*
1037  * The below are the page types of bios used in submit_bio().
1038  * The available types are:
1039  * DATA			User data pages. It operates as async mode.
1040  * NODE			Node pages. It operates as async mode.
1041  * META			FS metadata pages such as SIT, NAT, CP.
1042  * NR_PAGE_TYPE		The number of page types.
1043  * META_FLUSH		Make sure the previous pages are written
1044  *			with waiting the bio's completion
1045  * ...			Only can be used with META.
1046  */
1047 #define PAGE_TYPE_OF_BIO(type)	((type) > META ? META : (type))
1048 enum page_type {
1049 	DATA,
1050 	NODE,
1051 	META,
1052 	NR_PAGE_TYPE,
1053 	META_FLUSH,
1054 	INMEM,		/* the below types are used by tracepoints only. */
1055 	INMEM_DROP,
1056 	INMEM_INVALIDATE,
1057 	INMEM_REVOKE,
1058 	IPU,
1059 	OPU,
1060 };
1061 
1062 enum temp_type {
1063 	HOT = 0,	/* must be zero for meta bio */
1064 	WARM,
1065 	COLD,
1066 	NR_TEMP_TYPE,
1067 };
1068 
1069 enum need_lock_type {
1070 	LOCK_REQ = 0,
1071 	LOCK_DONE,
1072 	LOCK_RETRY,
1073 };
1074 
1075 enum cp_reason_type {
1076 	CP_NO_NEEDED,
1077 	CP_NON_REGULAR,
1078 	CP_COMPRESSED,
1079 	CP_HARDLINK,
1080 	CP_SB_NEED_CP,
1081 	CP_WRONG_PINO,
1082 	CP_NO_SPC_ROLL,
1083 	CP_NODE_NEED_CP,
1084 	CP_FASTBOOT_MODE,
1085 	CP_SPEC_LOG_NUM,
1086 	CP_RECOVER_DIR,
1087 };
1088 
1089 enum iostat_type {
1090 	/* WRITE IO */
1091 	APP_DIRECT_IO,			/* app direct write IOs */
1092 	APP_BUFFERED_IO,		/* app buffered write IOs */
1093 	APP_WRITE_IO,			/* app write IOs */
1094 	APP_MAPPED_IO,			/* app mapped IOs */
1095 	FS_DATA_IO,			/* data IOs from kworker/fsync/reclaimer */
1096 	FS_NODE_IO,			/* node IOs from kworker/fsync/reclaimer */
1097 	FS_META_IO,			/* meta IOs from kworker/reclaimer */
1098 	FS_GC_DATA_IO,			/* data IOs from forground gc */
1099 	FS_GC_NODE_IO,			/* node IOs from forground gc */
1100 	FS_CP_DATA_IO,			/* data IOs from checkpoint */
1101 	FS_CP_NODE_IO,			/* node IOs from checkpoint */
1102 	FS_CP_META_IO,			/* meta IOs from checkpoint */
1103 
1104 	/* READ IO */
1105 	APP_DIRECT_READ_IO,		/* app direct read IOs */
1106 	APP_BUFFERED_READ_IO,		/* app buffered read IOs */
1107 	APP_READ_IO,			/* app read IOs */
1108 	APP_MAPPED_READ_IO,		/* app mapped read IOs */
1109 	FS_DATA_READ_IO,		/* data read IOs */
1110 	FS_GDATA_READ_IO,		/* data read IOs from background gc */
1111 	FS_CDATA_READ_IO,		/* compressed data read IOs */
1112 	FS_NODE_READ_IO,		/* node read IOs */
1113 	FS_META_READ_IO,		/* meta read IOs */
1114 
1115 	/* other */
1116 	FS_DISCARD,			/* discard */
1117 	NR_IO_TYPE,
1118 };
1119 
1120 struct f2fs_io_info {
1121 	struct f2fs_sb_info *sbi;	/* f2fs_sb_info pointer */
1122 	nid_t ino;		/* inode number */
1123 	enum page_type type;	/* contains DATA/NODE/META/META_FLUSH */
1124 	enum temp_type temp;	/* contains HOT/WARM/COLD */
1125 	int op;			/* contains REQ_OP_ */
1126 	int op_flags;		/* req_flag_bits */
1127 	block_t new_blkaddr;	/* new block address to be written */
1128 	block_t old_blkaddr;	/* old block address before Cow */
1129 	struct page *page;	/* page to be written */
1130 	struct page *encrypted_page;	/* encrypted page */
1131 	struct page *compressed_page;	/* compressed page */
1132 	struct list_head list;		/* serialize IOs */
1133 	bool submitted;		/* indicate IO submission */
1134 	int need_lock;		/* indicate we need to lock cp_rwsem */
1135 	bool in_list;		/* indicate fio is in io_list */
1136 	bool is_por;		/* indicate IO is from recovery or not */
1137 	bool retry;		/* need to reallocate block address */
1138 	int compr_blocks;	/* # of compressed block addresses */
1139 	bool encrypted;		/* indicate file is encrypted */
1140 	enum iostat_type io_type;	/* io type */
1141 	struct writeback_control *io_wbc; /* writeback control */
1142 	struct bio **bio;		/* bio for ipu */
1143 	sector_t *last_block;		/* last block number in bio */
1144 	unsigned char version;		/* version of the node */
1145 };
1146 
1147 struct bio_entry {
1148 	struct bio *bio;
1149 	struct list_head list;
1150 };
1151 
1152 #define is_read_io(rw) ((rw) == READ)
1153 struct f2fs_bio_info {
1154 	struct f2fs_sb_info *sbi;	/* f2fs superblock */
1155 	struct bio *bio;		/* bios to merge */
1156 	sector_t last_block_in_bio;	/* last block number */
1157 	struct f2fs_io_info fio;	/* store buffered io info. */
1158 	struct rw_semaphore io_rwsem;	/* blocking op for bio */
1159 	spinlock_t io_lock;		/* serialize DATA/NODE IOs */
1160 	struct list_head io_list;	/* track fios */
1161 	struct list_head bio_list;	/* bio entry list head */
1162 	struct rw_semaphore bio_list_lock;	/* lock to protect bio entry list */
1163 };
1164 
1165 #define FDEV(i)				(sbi->devs[i])
1166 #define RDEV(i)				(raw_super->devs[i])
1167 struct f2fs_dev_info {
1168 	struct block_device *bdev;
1169 	char path[MAX_PATH_LEN];
1170 	unsigned int total_segments;
1171 	block_t start_blk;
1172 	block_t end_blk;
1173 #ifdef CONFIG_BLK_DEV_ZONED
1174 	unsigned int nr_blkz;		/* Total number of zones */
1175 	unsigned long *blkz_seq;	/* Bitmap indicating sequential zones */
1176 	block_t *zone_capacity_blocks;  /* Array of zone capacity in blks */
1177 #endif
1178 };
1179 
1180 enum inode_type {
1181 	DIR_INODE,			/* for dirty dir inode */
1182 	FILE_INODE,			/* for dirty regular/symlink inode */
1183 	DIRTY_META,			/* for all dirtied inode metadata */
1184 	ATOMIC_FILE,			/* for all atomic files */
1185 	NR_INODE_TYPE,
1186 };
1187 
1188 /* for inner inode cache management */
1189 struct inode_management {
1190 	struct radix_tree_root ino_root;	/* ino entry array */
1191 	spinlock_t ino_lock;			/* for ino entry lock */
1192 	struct list_head ino_list;		/* inode list head */
1193 	unsigned long ino_num;			/* number of entries */
1194 };
1195 
1196 /* for GC_AT */
1197 struct atgc_management {
1198 	bool atgc_enabled;			/* ATGC is enabled or not */
1199 	struct rb_root_cached root;		/* root of victim rb-tree */
1200 	struct list_head victim_list;		/* linked with all victim entries */
1201 	unsigned int victim_count;		/* victim count in rb-tree */
1202 	unsigned int candidate_ratio;		/* candidate ratio */
1203 	unsigned int max_candidate_count;	/* max candidate count */
1204 	unsigned int age_weight;		/* age weight, vblock_weight = 100 - age_weight */
1205 	unsigned long long age_threshold;	/* age threshold */
1206 };
1207 
1208 /* For s_flag in struct f2fs_sb_info */
1209 enum {
1210 	SBI_IS_DIRTY,				/* dirty flag for checkpoint */
1211 	SBI_IS_CLOSE,				/* specify unmounting */
1212 	SBI_NEED_FSCK,				/* need fsck.f2fs to fix */
1213 	SBI_POR_DOING,				/* recovery is doing or not */
1214 	SBI_NEED_SB_WRITE,			/* need to recover superblock */
1215 	SBI_NEED_CP,				/* need to checkpoint */
1216 	SBI_IS_SHUTDOWN,			/* shutdown by ioctl */
1217 	SBI_IS_RECOVERED,			/* recovered orphan/data */
1218 	SBI_CP_DISABLED,			/* CP was disabled last mount */
1219 	SBI_CP_DISABLED_QUICK,			/* CP was disabled quickly */
1220 	SBI_QUOTA_NEED_FLUSH,			/* need to flush quota info in CP */
1221 	SBI_QUOTA_SKIP_FLUSH,			/* skip flushing quota in current CP */
1222 	SBI_QUOTA_NEED_REPAIR,			/* quota file may be corrupted */
1223 	SBI_IS_RESIZEFS,			/* resizefs is in process */
1224 };
1225 
1226 enum {
1227 	CP_TIME,
1228 	REQ_TIME,
1229 	DISCARD_TIME,
1230 	GC_TIME,
1231 	DISABLE_TIME,
1232 	UMOUNT_DISCARD_TIMEOUT,
1233 	MAX_TIME,
1234 };
1235 
1236 enum {
1237 	GC_NORMAL,
1238 	GC_IDLE_CB,
1239 	GC_IDLE_GREEDY,
1240 	GC_IDLE_AT,
1241 	GC_URGENT_HIGH,
1242 	GC_URGENT_LOW,
1243 };
1244 
1245 enum {
1246 	BGGC_MODE_ON,		/* background gc is on */
1247 	BGGC_MODE_OFF,		/* background gc is off */
1248 	BGGC_MODE_SYNC,		/*
1249 				 * background gc is on, migrating blocks
1250 				 * like foreground gc
1251 				 */
1252 };
1253 
1254 enum {
1255 	FS_MODE_ADAPTIVE,	/* use both lfs/ssr allocation */
1256 	FS_MODE_LFS,		/* use lfs allocation only */
1257 };
1258 
1259 enum {
1260 	WHINT_MODE_OFF,		/* not pass down write hints */
1261 	WHINT_MODE_USER,	/* try to pass down hints given by users */
1262 	WHINT_MODE_FS,		/* pass down hints with F2FS policy */
1263 };
1264 
1265 enum {
1266 	ALLOC_MODE_DEFAULT,	/* stay default */
1267 	ALLOC_MODE_REUSE,	/* reuse segments as much as possible */
1268 };
1269 
1270 enum fsync_mode {
1271 	FSYNC_MODE_POSIX,	/* fsync follows posix semantics */
1272 	FSYNC_MODE_STRICT,	/* fsync behaves in line with ext4 */
1273 	FSYNC_MODE_NOBARRIER,	/* fsync behaves nobarrier based on posix */
1274 };
1275 
1276 enum {
1277 	COMPR_MODE_FS,		/*
1278 				 * automatically compress compression
1279 				 * enabled files
1280 				 */
1281 	COMPR_MODE_USER,	/*
1282 				 * automatical compression is disabled.
1283 				 * user can control the file compression
1284 				 * using ioctls
1285 				 */
1286 };
1287 
1288 /*
1289  * this value is set in page as a private data which indicate that
1290  * the page is atomically written, and it is in inmem_pages list.
1291  */
1292 #define ATOMIC_WRITTEN_PAGE		((unsigned long)-1)
1293 #define DUMMY_WRITTEN_PAGE		((unsigned long)-2)
1294 
1295 #define IS_ATOMIC_WRITTEN_PAGE(page)			\
1296 		(page_private(page) == ATOMIC_WRITTEN_PAGE)
1297 #define IS_DUMMY_WRITTEN_PAGE(page)			\
1298 		(page_private(page) == DUMMY_WRITTEN_PAGE)
1299 
1300 #ifdef CONFIG_F2FS_IO_TRACE
1301 #define IS_IO_TRACED_PAGE(page)			\
1302 		(page_private(page) > 0 &&		\
1303 		 page_private(page) < (unsigned long)PID_MAX_LIMIT)
1304 #else
1305 #define IS_IO_TRACED_PAGE(page) (0)
1306 #endif
1307 
1308 /* For compression */
1309 enum compress_algorithm_type {
1310 	COMPRESS_LZO,
1311 	COMPRESS_LZ4,
1312 	COMPRESS_ZSTD,
1313 	COMPRESS_LZORLE,
1314 	COMPRESS_MAX,
1315 };
1316 
1317 enum compress_flag {
1318 	COMPRESS_CHKSUM,
1319 	COMPRESS_MAX_FLAG,
1320 };
1321 
1322 #define COMPRESS_DATA_RESERVED_SIZE		4
1323 struct compress_data {
1324 	__le32 clen;			/* compressed data size */
1325 	__le32 chksum;			/* compressed data chksum */
1326 	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
1327 	u8 cdata[];			/* compressed data */
1328 };
1329 
1330 #define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
1331 
1332 #define F2FS_COMPRESSED_PAGE_MAGIC	0xF5F2C000
1333 
1334 #define	COMPRESS_LEVEL_OFFSET	8
1335 
1336 /* compress context */
1337 struct compress_ctx {
1338 	struct inode *inode;		/* inode the context belong to */
1339 	pgoff_t cluster_idx;		/* cluster index number */
1340 	unsigned int cluster_size;	/* page count in cluster */
1341 	unsigned int log_cluster_size;	/* log of cluster size */
1342 	struct page **rpages;		/* pages store raw data in cluster */
1343 	unsigned int nr_rpages;		/* total page number in rpages */
1344 	struct page **cpages;		/* pages store compressed data in cluster */
1345 	unsigned int nr_cpages;		/* total page number in cpages */
1346 	void *rbuf;			/* virtual mapped address on rpages */
1347 	struct compress_data *cbuf;	/* virtual mapped address on cpages */
1348 	size_t rlen;			/* valid data length in rbuf */
1349 	size_t clen;			/* valid data length in cbuf */
1350 	void *private;			/* payload buffer for specified compression algorithm */
1351 	void *private2;			/* extra payload buffer */
1352 };
1353 
1354 /* compress context for write IO path */
1355 struct compress_io_ctx {
1356 	u32 magic;			/* magic number to indicate page is compressed */
1357 	struct inode *inode;		/* inode the context belong to */
1358 	struct page **rpages;		/* pages store raw data in cluster */
1359 	unsigned int nr_rpages;		/* total page number in rpages */
1360 	atomic_t pending_pages;		/* in-flight compressed page count */
1361 };
1362 
1363 /* Context for decompressing one cluster on the read IO path */
1364 struct decompress_io_ctx {
1365 	u32 magic;			/* magic number to indicate page is compressed */
1366 	struct inode *inode;		/* inode the context belong to */
1367 	pgoff_t cluster_idx;		/* cluster index number */
1368 	unsigned int cluster_size;	/* page count in cluster */
1369 	unsigned int log_cluster_size;	/* log of cluster size */
1370 	struct page **rpages;		/* pages store raw data in cluster */
1371 	unsigned int nr_rpages;		/* total page number in rpages */
1372 	struct page **cpages;		/* pages store compressed data in cluster */
1373 	unsigned int nr_cpages;		/* total page number in cpages */
1374 	struct page **tpages;		/* temp pages to pad holes in cluster */
1375 	void *rbuf;			/* virtual mapped address on rpages */
1376 	struct compress_data *cbuf;	/* virtual mapped address on cpages */
1377 	size_t rlen;			/* valid data length in rbuf */
1378 	size_t clen;			/* valid data length in cbuf */
1379 
1380 	/*
1381 	 * The number of compressed pages remaining to be read in this cluster.
1382 	 * This is initially nr_cpages.  It is decremented by 1 each time a page
1383 	 * has been read (or failed to be read).  When it reaches 0, the cluster
1384 	 * is decompressed (or an error is reported).
1385 	 *
1386 	 * If an error occurs before all the pages have been submitted for I/O,
1387 	 * then this will never reach 0.  In this case the I/O submitter is
1388 	 * responsible for calling f2fs_decompress_end_io() instead.
1389 	 */
1390 	atomic_t remaining_pages;
1391 
1392 	/*
1393 	 * Number of references to this decompress_io_ctx.
1394 	 *
1395 	 * One reference is held for I/O completion.  This reference is dropped
1396 	 * after the pagecache pages are updated and unlocked -- either after
1397 	 * decompression (and verity if enabled), or after an error.
1398 	 *
1399 	 * In addition, each compressed page holds a reference while it is in a
1400 	 * bio.  These references are necessary prevent compressed pages from
1401 	 * being freed while they are still in a bio.
1402 	 */
1403 	refcount_t refcnt;
1404 
1405 	bool failed;			/* IO error occurred before decompression? */
1406 	bool need_verity;		/* need fs-verity verification after decompression? */
1407 	void *private;			/* payload buffer for specified decompression algorithm */
1408 	void *private2;			/* extra payload buffer */
1409 	struct work_struct verity_work;	/* work to verify the decompressed pages */
1410 };
1411 
1412 #define NULL_CLUSTER			((unsigned int)(~0))
1413 #define MIN_COMPRESS_LOG_SIZE		2
1414 #define MAX_COMPRESS_LOG_SIZE		8
1415 #define MAX_COMPRESS_WINDOW_SIZE(log_size)	((PAGE_SIZE) << (log_size))
1416 
1417 struct f2fs_sb_info {
1418 	struct super_block *sb;			/* pointer to VFS super block */
1419 	struct proc_dir_entry *s_proc;		/* proc entry */
1420 	struct f2fs_super_block *raw_super;	/* raw super block pointer */
1421 	struct rw_semaphore sb_lock;		/* lock for raw super block */
1422 	int valid_super_block;			/* valid super block no */
1423 	unsigned long s_flag;				/* flags for sbi */
1424 	struct mutex writepages;		/* mutex for writepages() */
1425 
1426 #ifdef CONFIG_BLK_DEV_ZONED
1427 	unsigned int blocks_per_blkz;		/* F2FS blocks per zone */
1428 	unsigned int log_blocks_per_blkz;	/* log2 F2FS blocks per zone */
1429 #endif
1430 
1431 	/* for node-related operations */
1432 	struct f2fs_nm_info *nm_info;		/* node manager */
1433 	struct inode *node_inode;		/* cache node blocks */
1434 
1435 	/* for segment-related operations */
1436 	struct f2fs_sm_info *sm_info;		/* segment manager */
1437 
1438 	/* for bio operations */
1439 	struct f2fs_bio_info *write_io[NR_PAGE_TYPE];	/* for write bios */
1440 	/* keep migration IO order for LFS mode */
1441 	struct rw_semaphore io_order_lock;
1442 	mempool_t *write_io_dummy;		/* Dummy pages */
1443 
1444 	/* for checkpoint */
1445 	struct f2fs_checkpoint *ckpt;		/* raw checkpoint pointer */
1446 	int cur_cp_pack;			/* remain current cp pack */
1447 	spinlock_t cp_lock;			/* for flag in ckpt */
1448 	struct inode *meta_inode;		/* cache meta blocks */
1449 	struct rw_semaphore cp_global_sem;	/* checkpoint procedure lock */
1450 	struct rw_semaphore cp_rwsem;		/* blocking FS operations */
1451 	struct rw_semaphore node_write;		/* locking node writes */
1452 	struct rw_semaphore node_change;	/* locking node change */
1453 	wait_queue_head_t cp_wait;
1454 	unsigned long last_time[MAX_TIME];	/* to store time in jiffies */
1455 	long interval_time[MAX_TIME];		/* to store thresholds */
1456 	struct ckpt_req_control cprc_info;	/* for checkpoint request control */
1457 
1458 	struct inode_management im[MAX_INO_ENTRY];	/* manage inode cache */
1459 
1460 	spinlock_t fsync_node_lock;		/* for node entry lock */
1461 	struct list_head fsync_node_list;	/* node list head */
1462 	unsigned int fsync_seg_id;		/* sequence id */
1463 	unsigned int fsync_node_num;		/* number of node entries */
1464 
1465 	/* for orphan inode, use 0'th array */
1466 	unsigned int max_orphans;		/* max orphan inodes */
1467 
1468 	/* for inode management */
1469 	struct list_head inode_list[NR_INODE_TYPE];	/* dirty inode list */
1470 	spinlock_t inode_lock[NR_INODE_TYPE];	/* for dirty inode list lock */
1471 	struct mutex flush_lock;		/* for flush exclusion */
1472 
1473 	/* for extent tree cache */
1474 	struct radix_tree_root extent_tree_root;/* cache extent cache entries */
1475 	struct mutex extent_tree_lock;	/* locking extent radix tree */
1476 	struct list_head extent_list;		/* lru list for shrinker */
1477 	spinlock_t extent_lock;			/* locking extent lru list */
1478 	atomic_t total_ext_tree;		/* extent tree count */
1479 	struct list_head zombie_list;		/* extent zombie tree list */
1480 	atomic_t total_zombie_tree;		/* extent zombie tree count */
1481 	atomic_t total_ext_node;		/* extent info count */
1482 
1483 	/* basic filesystem units */
1484 	unsigned int log_sectors_per_block;	/* log2 sectors per block */
1485 	unsigned int log_blocksize;		/* log2 block size */
1486 	unsigned int blocksize;			/* block size */
1487 	unsigned int root_ino_num;		/* root inode number*/
1488 	unsigned int node_ino_num;		/* node inode number*/
1489 	unsigned int meta_ino_num;		/* meta inode number*/
1490 	unsigned int log_blocks_per_seg;	/* log2 blocks per segment */
1491 	unsigned int blocks_per_seg;		/* blocks per segment */
1492 	unsigned int segs_per_sec;		/* segments per section */
1493 	unsigned int secs_per_zone;		/* sections per zone */
1494 	unsigned int total_sections;		/* total section count */
1495 	unsigned int total_node_count;		/* total node block count */
1496 	unsigned int total_valid_node_count;	/* valid node block count */
1497 	int dir_level;				/* directory level */
1498 	int readdir_ra;				/* readahead inode in readdir */
1499 	u64 max_io_bytes;			/* max io bytes to merge IOs */
1500 
1501 	block_t user_block_count;		/* # of user blocks */
1502 	block_t total_valid_block_count;	/* # of valid blocks */
1503 	block_t discard_blks;			/* discard command candidats */
1504 	block_t last_valid_block_count;		/* for recovery */
1505 	block_t reserved_blocks;		/* configurable reserved blocks */
1506 	block_t current_reserved_blocks;	/* current reserved blocks */
1507 
1508 	/* Additional tracking for no checkpoint mode */
1509 	block_t unusable_block_count;		/* # of blocks saved by last cp */
1510 
1511 	unsigned int nquota_files;		/* # of quota sysfile */
1512 	struct rw_semaphore quota_sem;		/* blocking cp for flags */
1513 
1514 	/* # of pages, see count_type */
1515 	atomic_t nr_pages[NR_COUNT_TYPE];
1516 	/* # of allocated blocks */
1517 	struct percpu_counter alloc_valid_block_count;
1518 
1519 	/* writeback control */
1520 	atomic_t wb_sync_req[META];	/* count # of WB_SYNC threads */
1521 
1522 	/* valid inode count */
1523 	struct percpu_counter total_valid_inode_count;
1524 
1525 	struct f2fs_mount_info mount_opt;	/* mount options */
1526 
1527 	/* for cleaning operations */
1528 	struct rw_semaphore gc_lock;		/*
1529 						 * semaphore for GC, avoid
1530 						 * race between GC and GC or CP
1531 						 */
1532 	struct f2fs_gc_kthread	*gc_thread;	/* GC thread */
1533 	struct atgc_management am;		/* atgc management */
1534 	unsigned int cur_victim_sec;		/* current victim section num */
1535 	unsigned int gc_mode;			/* current GC state */
1536 	unsigned int next_victim_seg[2];	/* next segment in victim section */
1537 
1538 	/* for skip statistic */
1539 	unsigned int atomic_files;		/* # of opened atomic file */
1540 	unsigned long long skipped_atomic_files[2];	/* FG_GC and BG_GC */
1541 	unsigned long long skipped_gc_rwsem;		/* FG_GC only */
1542 
1543 	/* threshold for gc trials on pinned files */
1544 	u64 gc_pin_file_threshold;
1545 	struct rw_semaphore pin_sem;
1546 
1547 	/* maximum # of trials to find a victim segment for SSR and GC */
1548 	unsigned int max_victim_search;
1549 	/* migration granularity of garbage collection, unit: segment */
1550 	unsigned int migration_granularity;
1551 
1552 	/*
1553 	 * for stat information.
1554 	 * one is for the LFS mode, and the other is for the SSR mode.
1555 	 */
1556 #ifdef CONFIG_F2FS_STAT_FS
1557 	struct f2fs_stat_info *stat_info;	/* FS status information */
1558 	atomic_t meta_count[META_MAX];		/* # of meta blocks */
1559 	unsigned int segment_count[2];		/* # of allocated segments */
1560 	unsigned int block_count[2];		/* # of allocated blocks */
1561 	atomic_t inplace_count;		/* # of inplace update */
1562 	atomic64_t total_hit_ext;		/* # of lookup extent cache */
1563 	atomic64_t read_hit_rbtree;		/* # of hit rbtree extent node */
1564 	atomic64_t read_hit_largest;		/* # of hit largest extent node */
1565 	atomic64_t read_hit_cached;		/* # of hit cached extent node */
1566 	atomic_t inline_xattr;			/* # of inline_xattr inodes */
1567 	atomic_t inline_inode;			/* # of inline_data inodes */
1568 	atomic_t inline_dir;			/* # of inline_dentry inodes */
1569 	atomic_t compr_inode;			/* # of compressed inodes */
1570 	atomic64_t compr_blocks;		/* # of compressed blocks */
1571 	atomic_t vw_cnt;			/* # of volatile writes */
1572 	atomic_t max_aw_cnt;			/* max # of atomic writes */
1573 	atomic_t max_vw_cnt;			/* max # of volatile writes */
1574 	unsigned int io_skip_bggc;		/* skip background gc for in-flight IO */
1575 	unsigned int other_skip_bggc;		/* skip background gc for other reasons */
1576 	unsigned int ndirty_inode[NR_INODE_TYPE];	/* # of dirty inodes */
1577 #endif
1578 	spinlock_t stat_lock;			/* lock for stat operations */
1579 
1580 	/* For app/fs IO statistics */
1581 	spinlock_t iostat_lock;
1582 	unsigned long long rw_iostat[NR_IO_TYPE];
1583 	unsigned long long prev_rw_iostat[NR_IO_TYPE];
1584 	bool iostat_enable;
1585 	unsigned long iostat_next_period;
1586 	unsigned int iostat_period_ms;
1587 
1588 	/* to attach REQ_META|REQ_FUA flags */
1589 	unsigned int data_io_flag;
1590 	unsigned int node_io_flag;
1591 
1592 	/* For sysfs suppport */
1593 	struct kobject s_kobj;			/* /sys/fs/f2fs/<devname> */
1594 	struct completion s_kobj_unregister;
1595 
1596 	struct kobject s_stat_kobj;		/* /sys/fs/f2fs/<devname>/stat */
1597 	struct completion s_stat_kobj_unregister;
1598 
1599 	/* For shrinker support */
1600 	struct list_head s_list;
1601 	int s_ndevs;				/* number of devices */
1602 	struct f2fs_dev_info *devs;		/* for device list */
1603 	unsigned int dirty_device;		/* for checkpoint data flush */
1604 	spinlock_t dev_lock;			/* protect dirty_device */
1605 	struct mutex umount_mutex;
1606 	unsigned int shrinker_run_no;
1607 
1608 	/* For write statistics */
1609 	u64 sectors_written_start;
1610 	u64 kbytes_written;
1611 
1612 	/* Reference to checksum algorithm driver via cryptoapi */
1613 	struct crypto_shash *s_chksum_driver;
1614 
1615 	/* Precomputed FS UUID checksum for seeding other checksums */
1616 	__u32 s_chksum_seed;
1617 
1618 	struct workqueue_struct *post_read_wq;	/* post read workqueue */
1619 
1620 	struct kmem_cache *inline_xattr_slab;	/* inline xattr entry */
1621 	unsigned int inline_xattr_slab_size;	/* default inline xattr slab size */
1622 
1623 #ifdef CONFIG_F2FS_FS_COMPRESSION
1624 	struct kmem_cache *page_array_slab;	/* page array entry */
1625 	unsigned int page_array_slab_size;	/* default page array slab size */
1626 #endif
1627 };
1628 
1629 struct f2fs_private_dio {
1630 	struct inode *inode;
1631 	void *orig_private;
1632 	bio_end_io_t *orig_end_io;
1633 	bool write;
1634 };
1635 
1636 #ifdef CONFIG_F2FS_FAULT_INJECTION
1637 #define f2fs_show_injection_info(sbi, type)					\
1638 	printk_ratelimited("%sF2FS-fs (%s) : inject %s in %s of %pS\n",	\
1639 		KERN_INFO, sbi->sb->s_id,				\
1640 		f2fs_fault_name[type],					\
1641 		__func__, __builtin_return_address(0))
1642 static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
1643 {
1644 	struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info;
1645 
1646 	if (!ffi->inject_rate)
1647 		return false;
1648 
1649 	if (!IS_FAULT_SET(ffi, type))
1650 		return false;
1651 
1652 	atomic_inc(&ffi->inject_ops);
1653 	if (atomic_read(&ffi->inject_ops) >= ffi->inject_rate) {
1654 		atomic_set(&ffi->inject_ops, 0);
1655 		return true;
1656 	}
1657 	return false;
1658 }
1659 #else
1660 #define f2fs_show_injection_info(sbi, type) do { } while (0)
1661 static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type)
1662 {
1663 	return false;
1664 }
1665 #endif
1666 
1667 /*
1668  * Test if the mounted volume is a multi-device volume.
1669  *   - For a single regular disk volume, sbi->s_ndevs is 0.
1670  *   - For a single zoned disk volume, sbi->s_ndevs is 1.
1671  *   - For a multi-device volume, sbi->s_ndevs is always 2 or more.
1672  */
1673 static inline bool f2fs_is_multi_device(struct f2fs_sb_info *sbi)
1674 {
1675 	return sbi->s_ndevs > 1;
1676 }
1677 
1678 static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
1679 {
1680 	unsigned long now = jiffies;
1681 
1682 	sbi->last_time[type] = now;
1683 
1684 	/* DISCARD_TIME and GC_TIME are based on REQ_TIME */
1685 	if (type == REQ_TIME) {
1686 		sbi->last_time[DISCARD_TIME] = now;
1687 		sbi->last_time[GC_TIME] = now;
1688 	}
1689 }
1690 
1691 static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type)
1692 {
1693 	unsigned long interval = sbi->interval_time[type] * HZ;
1694 
1695 	return time_after(jiffies, sbi->last_time[type] + interval);
1696 }
1697 
1698 static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi,
1699 						int type)
1700 {
1701 	unsigned long interval = sbi->interval_time[type] * HZ;
1702 	unsigned int wait_ms = 0;
1703 	long delta;
1704 
1705 	delta = (sbi->last_time[type] + interval) - jiffies;
1706 	if (delta > 0)
1707 		wait_ms = jiffies_to_msecs(delta);
1708 
1709 	return wait_ms;
1710 }
1711 
1712 /*
1713  * Inline functions
1714  */
1715 static inline u32 __f2fs_crc32(struct f2fs_sb_info *sbi, u32 crc,
1716 			      const void *address, unsigned int length)
1717 {
1718 	struct {
1719 		struct shash_desc shash;
1720 		char ctx[4];
1721 	} desc;
1722 	int err;
1723 
1724 	BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx));
1725 
1726 	desc.shash.tfm = sbi->s_chksum_driver;
1727 	*(u32 *)desc.ctx = crc;
1728 
1729 	err = crypto_shash_update(&desc.shash, address, length);
1730 	BUG_ON(err);
1731 
1732 	return *(u32 *)desc.ctx;
1733 }
1734 
1735 static inline u32 f2fs_crc32(struct f2fs_sb_info *sbi, const void *address,
1736 			   unsigned int length)
1737 {
1738 	return __f2fs_crc32(sbi, F2FS_SUPER_MAGIC, address, length);
1739 }
1740 
1741 static inline bool f2fs_crc_valid(struct f2fs_sb_info *sbi, __u32 blk_crc,
1742 				  void *buf, size_t buf_size)
1743 {
1744 	return f2fs_crc32(sbi, buf, buf_size) == blk_crc;
1745 }
1746 
1747 static inline u32 f2fs_chksum(struct f2fs_sb_info *sbi, u32 crc,
1748 			      const void *address, unsigned int length)
1749 {
1750 	return __f2fs_crc32(sbi, crc, address, length);
1751 }
1752 
1753 static inline struct f2fs_inode_info *F2FS_I(struct inode *inode)
1754 {
1755 	return container_of(inode, struct f2fs_inode_info, vfs_inode);
1756 }
1757 
1758 static inline struct f2fs_sb_info *F2FS_SB(struct super_block *sb)
1759 {
1760 	return sb->s_fs_info;
1761 }
1762 
1763 static inline struct f2fs_sb_info *F2FS_I_SB(struct inode *inode)
1764 {
1765 	return F2FS_SB(inode->i_sb);
1766 }
1767 
1768 static inline struct f2fs_sb_info *F2FS_M_SB(struct address_space *mapping)
1769 {
1770 	return F2FS_I_SB(mapping->host);
1771 }
1772 
1773 static inline struct f2fs_sb_info *F2FS_P_SB(struct page *page)
1774 {
1775 	return F2FS_M_SB(page_file_mapping(page));
1776 }
1777 
1778 static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi)
1779 {
1780 	return (struct f2fs_super_block *)(sbi->raw_super);
1781 }
1782 
1783 static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi)
1784 {
1785 	return (struct f2fs_checkpoint *)(sbi->ckpt);
1786 }
1787 
1788 static inline struct f2fs_node *F2FS_NODE(struct page *page)
1789 {
1790 	return (struct f2fs_node *)page_address(page);
1791 }
1792 
1793 static inline struct f2fs_inode *F2FS_INODE(struct page *page)
1794 {
1795 	return &((struct f2fs_node *)page_address(page))->i;
1796 }
1797 
1798 static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi)
1799 {
1800 	return (struct f2fs_nm_info *)(sbi->nm_info);
1801 }
1802 
1803 static inline struct f2fs_sm_info *SM_I(struct f2fs_sb_info *sbi)
1804 {
1805 	return (struct f2fs_sm_info *)(sbi->sm_info);
1806 }
1807 
1808 static inline struct sit_info *SIT_I(struct f2fs_sb_info *sbi)
1809 {
1810 	return (struct sit_info *)(SM_I(sbi)->sit_info);
1811 }
1812 
1813 static inline struct free_segmap_info *FREE_I(struct f2fs_sb_info *sbi)
1814 {
1815 	return (struct free_segmap_info *)(SM_I(sbi)->free_info);
1816 }
1817 
1818 static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi)
1819 {
1820 	return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info);
1821 }
1822 
1823 static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi)
1824 {
1825 	return sbi->meta_inode->i_mapping;
1826 }
1827 
1828 static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi)
1829 {
1830 	return sbi->node_inode->i_mapping;
1831 }
1832 
1833 static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type)
1834 {
1835 	return test_bit(type, &sbi->s_flag);
1836 }
1837 
1838 static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
1839 {
1840 	set_bit(type, &sbi->s_flag);
1841 }
1842 
1843 static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
1844 {
1845 	clear_bit(type, &sbi->s_flag);
1846 }
1847 
1848 static inline unsigned long long cur_cp_version(struct f2fs_checkpoint *cp)
1849 {
1850 	return le64_to_cpu(cp->checkpoint_ver);
1851 }
1852 
1853 static inline unsigned long f2fs_qf_ino(struct super_block *sb, int type)
1854 {
1855 	if (type < F2FS_MAX_QUOTAS)
1856 		return le32_to_cpu(F2FS_SB(sb)->raw_super->qf_ino[type]);
1857 	return 0;
1858 }
1859 
1860 static inline __u64 cur_cp_crc(struct f2fs_checkpoint *cp)
1861 {
1862 	size_t crc_offset = le32_to_cpu(cp->checksum_offset);
1863 	return le32_to_cpu(*((__le32 *)((unsigned char *)cp + crc_offset)));
1864 }
1865 
1866 static inline bool __is_set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
1867 {
1868 	unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags);
1869 
1870 	return ckpt_flags & f;
1871 }
1872 
1873 static inline bool is_set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
1874 {
1875 	return __is_set_ckpt_flags(F2FS_CKPT(sbi), f);
1876 }
1877 
1878 static inline void __set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
1879 {
1880 	unsigned int ckpt_flags;
1881 
1882 	ckpt_flags = le32_to_cpu(cp->ckpt_flags);
1883 	ckpt_flags |= f;
1884 	cp->ckpt_flags = cpu_to_le32(ckpt_flags);
1885 }
1886 
1887 static inline void set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
1888 {
1889 	unsigned long flags;
1890 
1891 	spin_lock_irqsave(&sbi->cp_lock, flags);
1892 	__set_ckpt_flags(F2FS_CKPT(sbi), f);
1893 	spin_unlock_irqrestore(&sbi->cp_lock, flags);
1894 }
1895 
1896 static inline void __clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
1897 {
1898 	unsigned int ckpt_flags;
1899 
1900 	ckpt_flags = le32_to_cpu(cp->ckpt_flags);
1901 	ckpt_flags &= (~f);
1902 	cp->ckpt_flags = cpu_to_le32(ckpt_flags);
1903 }
1904 
1905 static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
1906 {
1907 	unsigned long flags;
1908 
1909 	spin_lock_irqsave(&sbi->cp_lock, flags);
1910 	__clear_ckpt_flags(F2FS_CKPT(sbi), f);
1911 	spin_unlock_irqrestore(&sbi->cp_lock, flags);
1912 }
1913 
1914 static inline void disable_nat_bits(struct f2fs_sb_info *sbi, bool lock)
1915 {
1916 	unsigned long flags;
1917 	unsigned char *nat_bits;
1918 
1919 	/*
1920 	 * In order to re-enable nat_bits we need to call fsck.f2fs by
1921 	 * set_sbi_flag(sbi, SBI_NEED_FSCK). But it may give huge cost,
1922 	 * so let's rely on regular fsck or unclean shutdown.
1923 	 */
1924 
1925 	if (lock)
1926 		spin_lock_irqsave(&sbi->cp_lock, flags);
1927 	__clear_ckpt_flags(F2FS_CKPT(sbi), CP_NAT_BITS_FLAG);
1928 	nat_bits = NM_I(sbi)->nat_bits;
1929 	NM_I(sbi)->nat_bits = NULL;
1930 	if (lock)
1931 		spin_unlock_irqrestore(&sbi->cp_lock, flags);
1932 
1933 	kvfree(nat_bits);
1934 }
1935 
1936 static inline bool enabled_nat_bits(struct f2fs_sb_info *sbi,
1937 					struct cp_control *cpc)
1938 {
1939 	bool set = is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
1940 
1941 	return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set;
1942 }
1943 
1944 static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
1945 {
1946 	down_read(&sbi->cp_rwsem);
1947 }
1948 
1949 static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi)
1950 {
1951 	return down_read_trylock(&sbi->cp_rwsem);
1952 }
1953 
1954 static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi)
1955 {
1956 	up_read(&sbi->cp_rwsem);
1957 }
1958 
1959 static inline void f2fs_lock_all(struct f2fs_sb_info *sbi)
1960 {
1961 	down_write(&sbi->cp_rwsem);
1962 }
1963 
1964 static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi)
1965 {
1966 	up_write(&sbi->cp_rwsem);
1967 }
1968 
1969 static inline int __get_cp_reason(struct f2fs_sb_info *sbi)
1970 {
1971 	int reason = CP_SYNC;
1972 
1973 	if (test_opt(sbi, FASTBOOT))
1974 		reason = CP_FASTBOOT;
1975 	if (is_sbi_flag_set(sbi, SBI_IS_CLOSE))
1976 		reason = CP_UMOUNT;
1977 	return reason;
1978 }
1979 
1980 static inline bool __remain_node_summaries(int reason)
1981 {
1982 	return (reason & (CP_UMOUNT | CP_FASTBOOT));
1983 }
1984 
1985 static inline bool __exist_node_summaries(struct f2fs_sb_info *sbi)
1986 {
1987 	return (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG) ||
1988 			is_set_ckpt_flags(sbi, CP_FASTBOOT_FLAG));
1989 }
1990 
1991 /*
1992  * Check whether the inode has blocks or not
1993  */
1994 static inline int F2FS_HAS_BLOCKS(struct inode *inode)
1995 {
1996 	block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0;
1997 
1998 	return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block;
1999 }
2000 
2001 static inline bool f2fs_has_xattr_block(unsigned int ofs)
2002 {
2003 	return ofs == XATTR_NODE_OFFSET;
2004 }
2005 
2006 static inline bool __allow_reserved_blocks(struct f2fs_sb_info *sbi,
2007 					struct inode *inode, bool cap)
2008 {
2009 	if (!inode)
2010 		return true;
2011 	if (!test_opt(sbi, RESERVE_ROOT))
2012 		return false;
2013 	if (IS_NOQUOTA(inode))
2014 		return true;
2015 	if (uid_eq(F2FS_OPTION(sbi).s_resuid, current_fsuid()))
2016 		return true;
2017 	if (!gid_eq(F2FS_OPTION(sbi).s_resgid, GLOBAL_ROOT_GID) &&
2018 					in_group_p(F2FS_OPTION(sbi).s_resgid))
2019 		return true;
2020 	if (cap && capable(CAP_SYS_RESOURCE))
2021 		return true;
2022 	return false;
2023 }
2024 
2025 static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool);
2026 static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
2027 				 struct inode *inode, blkcnt_t *count)
2028 {
2029 	blkcnt_t diff = 0, release = 0;
2030 	block_t avail_user_block_count;
2031 	int ret;
2032 
2033 	ret = dquot_reserve_block(inode, *count);
2034 	if (ret)
2035 		return ret;
2036 
2037 	if (time_to_inject(sbi, FAULT_BLOCK)) {
2038 		f2fs_show_injection_info(sbi, FAULT_BLOCK);
2039 		release = *count;
2040 		goto release_quota;
2041 	}
2042 
2043 	/*
2044 	 * let's increase this in prior to actual block count change in order
2045 	 * for f2fs_sync_file to avoid data races when deciding checkpoint.
2046 	 */
2047 	percpu_counter_add(&sbi->alloc_valid_block_count, (*count));
2048 
2049 	spin_lock(&sbi->stat_lock);
2050 	sbi->total_valid_block_count += (block_t)(*count);
2051 	avail_user_block_count = sbi->user_block_count -
2052 					sbi->current_reserved_blocks;
2053 
2054 	if (!__allow_reserved_blocks(sbi, inode, true))
2055 		avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
2056 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2057 		if (avail_user_block_count > sbi->unusable_block_count)
2058 			avail_user_block_count -= sbi->unusable_block_count;
2059 		else
2060 			avail_user_block_count = 0;
2061 	}
2062 	if (unlikely(sbi->total_valid_block_count > avail_user_block_count)) {
2063 		diff = sbi->total_valid_block_count - avail_user_block_count;
2064 		if (diff > *count)
2065 			diff = *count;
2066 		*count -= diff;
2067 		release = diff;
2068 		sbi->total_valid_block_count -= diff;
2069 		if (!*count) {
2070 			spin_unlock(&sbi->stat_lock);
2071 			goto enospc;
2072 		}
2073 	}
2074 	spin_unlock(&sbi->stat_lock);
2075 
2076 	if (unlikely(release)) {
2077 		percpu_counter_sub(&sbi->alloc_valid_block_count, release);
2078 		dquot_release_reservation_block(inode, release);
2079 	}
2080 	f2fs_i_blocks_write(inode, *count, true, true);
2081 	return 0;
2082 
2083 enospc:
2084 	percpu_counter_sub(&sbi->alloc_valid_block_count, release);
2085 release_quota:
2086 	dquot_release_reservation_block(inode, release);
2087 	return -ENOSPC;
2088 }
2089 
2090 __printf(2, 3)
2091 void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...);
2092 
2093 #define f2fs_err(sbi, fmt, ...)						\
2094 	f2fs_printk(sbi, KERN_ERR fmt, ##__VA_ARGS__)
2095 #define f2fs_warn(sbi, fmt, ...)					\
2096 	f2fs_printk(sbi, KERN_WARNING fmt, ##__VA_ARGS__)
2097 #define f2fs_notice(sbi, fmt, ...)					\
2098 	f2fs_printk(sbi, KERN_NOTICE fmt, ##__VA_ARGS__)
2099 #define f2fs_info(sbi, fmt, ...)					\
2100 	f2fs_printk(sbi, KERN_INFO fmt, ##__VA_ARGS__)
2101 #define f2fs_debug(sbi, fmt, ...)					\
2102 	f2fs_printk(sbi, KERN_DEBUG fmt, ##__VA_ARGS__)
2103 
2104 static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
2105 						struct inode *inode,
2106 						block_t count)
2107 {
2108 	blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK;
2109 
2110 	spin_lock(&sbi->stat_lock);
2111 	f2fs_bug_on(sbi, sbi->total_valid_block_count < (block_t) count);
2112 	sbi->total_valid_block_count -= (block_t)count;
2113 	if (sbi->reserved_blocks &&
2114 		sbi->current_reserved_blocks < sbi->reserved_blocks)
2115 		sbi->current_reserved_blocks = min(sbi->reserved_blocks,
2116 					sbi->current_reserved_blocks + count);
2117 	spin_unlock(&sbi->stat_lock);
2118 	if (unlikely(inode->i_blocks < sectors)) {
2119 		f2fs_warn(sbi, "Inconsistent i_blocks, ino:%lu, iblocks:%llu, sectors:%llu",
2120 			  inode->i_ino,
2121 			  (unsigned long long)inode->i_blocks,
2122 			  (unsigned long long)sectors);
2123 		set_sbi_flag(sbi, SBI_NEED_FSCK);
2124 		return;
2125 	}
2126 	f2fs_i_blocks_write(inode, count, false, true);
2127 }
2128 
2129 static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
2130 {
2131 	atomic_inc(&sbi->nr_pages[count_type]);
2132 
2133 	if (count_type == F2FS_DIRTY_DENTS ||
2134 			count_type == F2FS_DIRTY_NODES ||
2135 			count_type == F2FS_DIRTY_META ||
2136 			count_type == F2FS_DIRTY_QDATA ||
2137 			count_type == F2FS_DIRTY_IMETA)
2138 		set_sbi_flag(sbi, SBI_IS_DIRTY);
2139 }
2140 
2141 static inline void inode_inc_dirty_pages(struct inode *inode)
2142 {
2143 	atomic_inc(&F2FS_I(inode)->dirty_pages);
2144 	inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2145 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2146 	if (IS_NOQUOTA(inode))
2147 		inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
2148 }
2149 
2150 static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
2151 {
2152 	atomic_dec(&sbi->nr_pages[count_type]);
2153 }
2154 
2155 static inline void inode_dec_dirty_pages(struct inode *inode)
2156 {
2157 	if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
2158 			!S_ISLNK(inode->i_mode))
2159 		return;
2160 
2161 	atomic_dec(&F2FS_I(inode)->dirty_pages);
2162 	dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
2163 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
2164 	if (IS_NOQUOTA(inode))
2165 		dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
2166 }
2167 
2168 static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
2169 {
2170 	return atomic_read(&sbi->nr_pages[count_type]);
2171 }
2172 
2173 static inline int get_dirty_pages(struct inode *inode)
2174 {
2175 	return atomic_read(&F2FS_I(inode)->dirty_pages);
2176 }
2177 
2178 static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type)
2179 {
2180 	unsigned int pages_per_sec = sbi->segs_per_sec * sbi->blocks_per_seg;
2181 	unsigned int segs = (get_pages(sbi, block_type) + pages_per_sec - 1) >>
2182 						sbi->log_blocks_per_seg;
2183 
2184 	return segs / sbi->segs_per_sec;
2185 }
2186 
2187 static inline block_t valid_user_blocks(struct f2fs_sb_info *sbi)
2188 {
2189 	return sbi->total_valid_block_count;
2190 }
2191 
2192 static inline block_t discard_blocks(struct f2fs_sb_info *sbi)
2193 {
2194 	return sbi->discard_blks;
2195 }
2196 
2197 static inline unsigned long __bitmap_size(struct f2fs_sb_info *sbi, int flag)
2198 {
2199 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2200 
2201 	/* return NAT or SIT bitmap */
2202 	if (flag == NAT_BITMAP)
2203 		return le32_to_cpu(ckpt->nat_ver_bitmap_bytesize);
2204 	else if (flag == SIT_BITMAP)
2205 		return le32_to_cpu(ckpt->sit_ver_bitmap_bytesize);
2206 
2207 	return 0;
2208 }
2209 
2210 static inline block_t __cp_payload(struct f2fs_sb_info *sbi)
2211 {
2212 	return le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
2213 }
2214 
2215 static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
2216 {
2217 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2218 	int offset;
2219 
2220 	if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) {
2221 		offset = (flag == SIT_BITMAP) ?
2222 			le32_to_cpu(ckpt->nat_ver_bitmap_bytesize) : 0;
2223 		/*
2224 		 * if large_nat_bitmap feature is enabled, leave checksum
2225 		 * protection for all nat/sit bitmaps.
2226 		 */
2227 		return &ckpt->sit_nat_version_bitmap + offset + sizeof(__le32);
2228 	}
2229 
2230 	if (__cp_payload(sbi) > 0) {
2231 		if (flag == NAT_BITMAP)
2232 			return &ckpt->sit_nat_version_bitmap;
2233 		else
2234 			return (unsigned char *)ckpt + F2FS_BLKSIZE;
2235 	} else {
2236 		offset = (flag == NAT_BITMAP) ?
2237 			le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
2238 		return &ckpt->sit_nat_version_bitmap + offset;
2239 	}
2240 }
2241 
2242 static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi)
2243 {
2244 	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2245 
2246 	if (sbi->cur_cp_pack == 2)
2247 		start_addr += sbi->blocks_per_seg;
2248 	return start_addr;
2249 }
2250 
2251 static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi)
2252 {
2253 	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
2254 
2255 	if (sbi->cur_cp_pack == 1)
2256 		start_addr += sbi->blocks_per_seg;
2257 	return start_addr;
2258 }
2259 
2260 static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi)
2261 {
2262 	sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1;
2263 }
2264 
2265 static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi)
2266 {
2267 	return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
2268 }
2269 
2270 static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
2271 					struct inode *inode, bool is_inode)
2272 {
2273 	block_t	valid_block_count;
2274 	unsigned int valid_node_count, user_block_count;
2275 	int err;
2276 
2277 	if (is_inode) {
2278 		if (inode) {
2279 			err = dquot_alloc_inode(inode);
2280 			if (err)
2281 				return err;
2282 		}
2283 	} else {
2284 		err = dquot_reserve_block(inode, 1);
2285 		if (err)
2286 			return err;
2287 	}
2288 
2289 	if (time_to_inject(sbi, FAULT_BLOCK)) {
2290 		f2fs_show_injection_info(sbi, FAULT_BLOCK);
2291 		goto enospc;
2292 	}
2293 
2294 	spin_lock(&sbi->stat_lock);
2295 
2296 	valid_block_count = sbi->total_valid_block_count +
2297 					sbi->current_reserved_blocks + 1;
2298 
2299 	if (!__allow_reserved_blocks(sbi, inode, false))
2300 		valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
2301 	user_block_count = sbi->user_block_count;
2302 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2303 		user_block_count -= sbi->unusable_block_count;
2304 
2305 	if (unlikely(valid_block_count > user_block_count)) {
2306 		spin_unlock(&sbi->stat_lock);
2307 		goto enospc;
2308 	}
2309 
2310 	valid_node_count = sbi->total_valid_node_count + 1;
2311 	if (unlikely(valid_node_count > sbi->total_node_count)) {
2312 		spin_unlock(&sbi->stat_lock);
2313 		goto enospc;
2314 	}
2315 
2316 	sbi->total_valid_node_count++;
2317 	sbi->total_valid_block_count++;
2318 	spin_unlock(&sbi->stat_lock);
2319 
2320 	if (inode) {
2321 		if (is_inode)
2322 			f2fs_mark_inode_dirty_sync(inode, true);
2323 		else
2324 			f2fs_i_blocks_write(inode, 1, true, true);
2325 	}
2326 
2327 	percpu_counter_inc(&sbi->alloc_valid_block_count);
2328 	return 0;
2329 
2330 enospc:
2331 	if (is_inode) {
2332 		if (inode)
2333 			dquot_free_inode(inode);
2334 	} else {
2335 		dquot_release_reservation_block(inode, 1);
2336 	}
2337 	return -ENOSPC;
2338 }
2339 
2340 static inline void dec_valid_node_count(struct f2fs_sb_info *sbi,
2341 					struct inode *inode, bool is_inode)
2342 {
2343 	spin_lock(&sbi->stat_lock);
2344 
2345 	f2fs_bug_on(sbi, !sbi->total_valid_block_count);
2346 	f2fs_bug_on(sbi, !sbi->total_valid_node_count);
2347 
2348 	sbi->total_valid_node_count--;
2349 	sbi->total_valid_block_count--;
2350 	if (sbi->reserved_blocks &&
2351 		sbi->current_reserved_blocks < sbi->reserved_blocks)
2352 		sbi->current_reserved_blocks++;
2353 
2354 	spin_unlock(&sbi->stat_lock);
2355 
2356 	if (is_inode) {
2357 		dquot_free_inode(inode);
2358 	} else {
2359 		if (unlikely(inode->i_blocks == 0)) {
2360 			f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%lu, iblocks:%llu",
2361 				  inode->i_ino,
2362 				  (unsigned long long)inode->i_blocks);
2363 			set_sbi_flag(sbi, SBI_NEED_FSCK);
2364 			return;
2365 		}
2366 		f2fs_i_blocks_write(inode, 1, false, true);
2367 	}
2368 }
2369 
2370 static inline unsigned int valid_node_count(struct f2fs_sb_info *sbi)
2371 {
2372 	return sbi->total_valid_node_count;
2373 }
2374 
2375 static inline void inc_valid_inode_count(struct f2fs_sb_info *sbi)
2376 {
2377 	percpu_counter_inc(&sbi->total_valid_inode_count);
2378 }
2379 
2380 static inline void dec_valid_inode_count(struct f2fs_sb_info *sbi)
2381 {
2382 	percpu_counter_dec(&sbi->total_valid_inode_count);
2383 }
2384 
2385 static inline s64 valid_inode_count(struct f2fs_sb_info *sbi)
2386 {
2387 	return percpu_counter_sum_positive(&sbi->total_valid_inode_count);
2388 }
2389 
2390 static inline struct page *f2fs_grab_cache_page(struct address_space *mapping,
2391 						pgoff_t index, bool for_write)
2392 {
2393 	struct page *page;
2394 
2395 	if (IS_ENABLED(CONFIG_F2FS_FAULT_INJECTION)) {
2396 		if (!for_write)
2397 			page = find_get_page_flags(mapping, index,
2398 							FGP_LOCK | FGP_ACCESSED);
2399 		else
2400 			page = find_lock_page(mapping, index);
2401 		if (page)
2402 			return page;
2403 
2404 		if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC)) {
2405 			f2fs_show_injection_info(F2FS_M_SB(mapping),
2406 							FAULT_PAGE_ALLOC);
2407 			return NULL;
2408 		}
2409 	}
2410 
2411 	if (!for_write)
2412 		return grab_cache_page(mapping, index);
2413 	return grab_cache_page_write_begin(mapping, index, AOP_FLAG_NOFS);
2414 }
2415 
2416 static inline struct page *f2fs_pagecache_get_page(
2417 				struct address_space *mapping, pgoff_t index,
2418 				int fgp_flags, gfp_t gfp_mask)
2419 {
2420 	if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET)) {
2421 		f2fs_show_injection_info(F2FS_M_SB(mapping), FAULT_PAGE_GET);
2422 		return NULL;
2423 	}
2424 
2425 	return pagecache_get_page(mapping, index, fgp_flags, gfp_mask);
2426 }
2427 
2428 static inline void f2fs_copy_page(struct page *src, struct page *dst)
2429 {
2430 	char *src_kaddr = kmap(src);
2431 	char *dst_kaddr = kmap(dst);
2432 
2433 	memcpy(dst_kaddr, src_kaddr, PAGE_SIZE);
2434 	kunmap(dst);
2435 	kunmap(src);
2436 }
2437 
2438 static inline void f2fs_put_page(struct page *page, int unlock)
2439 {
2440 	if (!page)
2441 		return;
2442 
2443 	if (unlock) {
2444 		f2fs_bug_on(F2FS_P_SB(page), !PageLocked(page));
2445 		unlock_page(page);
2446 	}
2447 	put_page(page);
2448 }
2449 
2450 static inline void f2fs_put_dnode(struct dnode_of_data *dn)
2451 {
2452 	if (dn->node_page)
2453 		f2fs_put_page(dn->node_page, 1);
2454 	if (dn->inode_page && dn->node_page != dn->inode_page)
2455 		f2fs_put_page(dn->inode_page, 0);
2456 	dn->node_page = NULL;
2457 	dn->inode_page = NULL;
2458 }
2459 
2460 static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name,
2461 					size_t size)
2462 {
2463 	return kmem_cache_create(name, size, 0, SLAB_RECLAIM_ACCOUNT, NULL);
2464 }
2465 
2466 static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep,
2467 						gfp_t flags)
2468 {
2469 	void *entry;
2470 
2471 	entry = kmem_cache_alloc(cachep, flags);
2472 	if (!entry)
2473 		entry = kmem_cache_alloc(cachep, flags | __GFP_NOFAIL);
2474 	return entry;
2475 }
2476 
2477 static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
2478 {
2479 	if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) ||
2480 		get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) ||
2481 		get_pages(sbi, F2FS_WB_CP_DATA) ||
2482 		get_pages(sbi, F2FS_DIO_READ) ||
2483 		get_pages(sbi, F2FS_DIO_WRITE))
2484 		return true;
2485 
2486 	if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
2487 			atomic_read(&SM_I(sbi)->dcc_info->queued_discard))
2488 		return true;
2489 
2490 	if (SM_I(sbi) && SM_I(sbi)->fcc_info &&
2491 			atomic_read(&SM_I(sbi)->fcc_info->queued_flush))
2492 		return true;
2493 	return false;
2494 }
2495 
2496 static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2497 {
2498 	if (sbi->gc_mode == GC_URGENT_HIGH)
2499 		return true;
2500 
2501 	if (is_inflight_io(sbi, type))
2502 		return false;
2503 
2504 	if (sbi->gc_mode == GC_URGENT_LOW &&
2505 			(type == DISCARD_TIME || type == GC_TIME))
2506 		return true;
2507 
2508 	return f2fs_time_over(sbi, type);
2509 }
2510 
2511 static inline void f2fs_radix_tree_insert(struct radix_tree_root *root,
2512 				unsigned long index, void *item)
2513 {
2514 	while (radix_tree_insert(root, index, item))
2515 		cond_resched();
2516 }
2517 
2518 #define RAW_IS_INODE(p)	((p)->footer.nid == (p)->footer.ino)
2519 
2520 static inline bool IS_INODE(struct page *page)
2521 {
2522 	struct f2fs_node *p = F2FS_NODE(page);
2523 
2524 	return RAW_IS_INODE(p);
2525 }
2526 
2527 static inline int offset_in_addr(struct f2fs_inode *i)
2528 {
2529 	return (i->i_inline & F2FS_EXTRA_ATTR) ?
2530 			(le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0;
2531 }
2532 
2533 static inline __le32 *blkaddr_in_node(struct f2fs_node *node)
2534 {
2535 	return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr;
2536 }
2537 
2538 static inline int f2fs_has_extra_attr(struct inode *inode);
2539 static inline block_t data_blkaddr(struct inode *inode,
2540 			struct page *node_page, unsigned int offset)
2541 {
2542 	struct f2fs_node *raw_node;
2543 	__le32 *addr_array;
2544 	int base = 0;
2545 	bool is_inode = IS_INODE(node_page);
2546 
2547 	raw_node = F2FS_NODE(node_page);
2548 
2549 	if (is_inode) {
2550 		if (!inode)
2551 			/* from GC path only */
2552 			base = offset_in_addr(&raw_node->i);
2553 		else if (f2fs_has_extra_attr(inode))
2554 			base = get_extra_isize(inode);
2555 	}
2556 
2557 	addr_array = blkaddr_in_node(raw_node);
2558 	return le32_to_cpu(addr_array[base + offset]);
2559 }
2560 
2561 static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn)
2562 {
2563 	return data_blkaddr(dn->inode, dn->node_page, dn->ofs_in_node);
2564 }
2565 
2566 static inline int f2fs_test_bit(unsigned int nr, char *addr)
2567 {
2568 	int mask;
2569 
2570 	addr += (nr >> 3);
2571 	mask = 1 << (7 - (nr & 0x07));
2572 	return mask & *addr;
2573 }
2574 
2575 static inline void f2fs_set_bit(unsigned int nr, char *addr)
2576 {
2577 	int mask;
2578 
2579 	addr += (nr >> 3);
2580 	mask = 1 << (7 - (nr & 0x07));
2581 	*addr |= mask;
2582 }
2583 
2584 static inline void f2fs_clear_bit(unsigned int nr, char *addr)
2585 {
2586 	int mask;
2587 
2588 	addr += (nr >> 3);
2589 	mask = 1 << (7 - (nr & 0x07));
2590 	*addr &= ~mask;
2591 }
2592 
2593 static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr)
2594 {
2595 	int mask;
2596 	int ret;
2597 
2598 	addr += (nr >> 3);
2599 	mask = 1 << (7 - (nr & 0x07));
2600 	ret = mask & *addr;
2601 	*addr |= mask;
2602 	return ret;
2603 }
2604 
2605 static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr)
2606 {
2607 	int mask;
2608 	int ret;
2609 
2610 	addr += (nr >> 3);
2611 	mask = 1 << (7 - (nr & 0x07));
2612 	ret = mask & *addr;
2613 	*addr &= ~mask;
2614 	return ret;
2615 }
2616 
2617 static inline void f2fs_change_bit(unsigned int nr, char *addr)
2618 {
2619 	int mask;
2620 
2621 	addr += (nr >> 3);
2622 	mask = 1 << (7 - (nr & 0x07));
2623 	*addr ^= mask;
2624 }
2625 
2626 /*
2627  * On-disk inode flags (f2fs_inode::i_flags)
2628  */
2629 #define F2FS_COMPR_FL			0x00000004 /* Compress file */
2630 #define F2FS_SYNC_FL			0x00000008 /* Synchronous updates */
2631 #define F2FS_IMMUTABLE_FL		0x00000010 /* Immutable file */
2632 #define F2FS_APPEND_FL			0x00000020 /* writes to file may only append */
2633 #define F2FS_NODUMP_FL			0x00000040 /* do not dump file */
2634 #define F2FS_NOATIME_FL			0x00000080 /* do not update atime */
2635 #define F2FS_NOCOMP_FL			0x00000400 /* Don't compress */
2636 #define F2FS_INDEX_FL			0x00001000 /* hash-indexed directory */
2637 #define F2FS_DIRSYNC_FL			0x00010000 /* dirsync behaviour (directories only) */
2638 #define F2FS_PROJINHERIT_FL		0x20000000 /* Create with parents projid */
2639 #define F2FS_CASEFOLD_FL		0x40000000 /* Casefolded file */
2640 
2641 /* Flags that should be inherited by new inodes from their parent. */
2642 #define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \
2643 			   F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
2644 			   F2FS_CASEFOLD_FL | F2FS_COMPR_FL | F2FS_NOCOMP_FL)
2645 
2646 /* Flags that are appropriate for regular files (all but dir-specific ones). */
2647 #define F2FS_REG_FLMASK		(~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \
2648 				F2FS_CASEFOLD_FL))
2649 
2650 /* Flags that are appropriate for non-directories/regular files. */
2651 #define F2FS_OTHER_FLMASK	(F2FS_NODUMP_FL | F2FS_NOATIME_FL)
2652 
2653 static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags)
2654 {
2655 	if (S_ISDIR(mode))
2656 		return flags;
2657 	else if (S_ISREG(mode))
2658 		return flags & F2FS_REG_FLMASK;
2659 	else
2660 		return flags & F2FS_OTHER_FLMASK;
2661 }
2662 
2663 static inline void __mark_inode_dirty_flag(struct inode *inode,
2664 						int flag, bool set)
2665 {
2666 	switch (flag) {
2667 	case FI_INLINE_XATTR:
2668 	case FI_INLINE_DATA:
2669 	case FI_INLINE_DENTRY:
2670 	case FI_NEW_INODE:
2671 		if (set)
2672 			return;
2673 		fallthrough;
2674 	case FI_DATA_EXIST:
2675 	case FI_INLINE_DOTS:
2676 	case FI_PIN_FILE:
2677 		f2fs_mark_inode_dirty_sync(inode, true);
2678 	}
2679 }
2680 
2681 static inline void set_inode_flag(struct inode *inode, int flag)
2682 {
2683 	set_bit(flag, F2FS_I(inode)->flags);
2684 	__mark_inode_dirty_flag(inode, flag, true);
2685 }
2686 
2687 static inline int is_inode_flag_set(struct inode *inode, int flag)
2688 {
2689 	return test_bit(flag, F2FS_I(inode)->flags);
2690 }
2691 
2692 static inline void clear_inode_flag(struct inode *inode, int flag)
2693 {
2694 	clear_bit(flag, F2FS_I(inode)->flags);
2695 	__mark_inode_dirty_flag(inode, flag, false);
2696 }
2697 
2698 static inline bool f2fs_verity_in_progress(struct inode *inode)
2699 {
2700 	return IS_ENABLED(CONFIG_FS_VERITY) &&
2701 	       is_inode_flag_set(inode, FI_VERITY_IN_PROGRESS);
2702 }
2703 
2704 static inline void set_acl_inode(struct inode *inode, umode_t mode)
2705 {
2706 	F2FS_I(inode)->i_acl_mode = mode;
2707 	set_inode_flag(inode, FI_ACL_MODE);
2708 	f2fs_mark_inode_dirty_sync(inode, false);
2709 }
2710 
2711 static inline void f2fs_i_links_write(struct inode *inode, bool inc)
2712 {
2713 	if (inc)
2714 		inc_nlink(inode);
2715 	else
2716 		drop_nlink(inode);
2717 	f2fs_mark_inode_dirty_sync(inode, true);
2718 }
2719 
2720 static inline void f2fs_i_blocks_write(struct inode *inode,
2721 					block_t diff, bool add, bool claim)
2722 {
2723 	bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
2724 	bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
2725 
2726 	/* add = 1, claim = 1 should be dquot_reserve_block in pair */
2727 	if (add) {
2728 		if (claim)
2729 			dquot_claim_block(inode, diff);
2730 		else
2731 			dquot_alloc_block_nofail(inode, diff);
2732 	} else {
2733 		dquot_free_block(inode, diff);
2734 	}
2735 
2736 	f2fs_mark_inode_dirty_sync(inode, true);
2737 	if (clean || recover)
2738 		set_inode_flag(inode, FI_AUTO_RECOVER);
2739 }
2740 
2741 static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size)
2742 {
2743 	bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE);
2744 	bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER);
2745 
2746 	if (i_size_read(inode) == i_size)
2747 		return;
2748 
2749 	i_size_write(inode, i_size);
2750 	f2fs_mark_inode_dirty_sync(inode, true);
2751 	if (clean || recover)
2752 		set_inode_flag(inode, FI_AUTO_RECOVER);
2753 }
2754 
2755 static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth)
2756 {
2757 	F2FS_I(inode)->i_current_depth = depth;
2758 	f2fs_mark_inode_dirty_sync(inode, true);
2759 }
2760 
2761 static inline void f2fs_i_gc_failures_write(struct inode *inode,
2762 					unsigned int count)
2763 {
2764 	F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN] = count;
2765 	f2fs_mark_inode_dirty_sync(inode, true);
2766 }
2767 
2768 static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid)
2769 {
2770 	F2FS_I(inode)->i_xattr_nid = xnid;
2771 	f2fs_mark_inode_dirty_sync(inode, true);
2772 }
2773 
2774 static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino)
2775 {
2776 	F2FS_I(inode)->i_pino = pino;
2777 	f2fs_mark_inode_dirty_sync(inode, true);
2778 }
2779 
2780 static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri)
2781 {
2782 	struct f2fs_inode_info *fi = F2FS_I(inode);
2783 
2784 	if (ri->i_inline & F2FS_INLINE_XATTR)
2785 		set_bit(FI_INLINE_XATTR, fi->flags);
2786 	if (ri->i_inline & F2FS_INLINE_DATA)
2787 		set_bit(FI_INLINE_DATA, fi->flags);
2788 	if (ri->i_inline & F2FS_INLINE_DENTRY)
2789 		set_bit(FI_INLINE_DENTRY, fi->flags);
2790 	if (ri->i_inline & F2FS_DATA_EXIST)
2791 		set_bit(FI_DATA_EXIST, fi->flags);
2792 	if (ri->i_inline & F2FS_INLINE_DOTS)
2793 		set_bit(FI_INLINE_DOTS, fi->flags);
2794 	if (ri->i_inline & F2FS_EXTRA_ATTR)
2795 		set_bit(FI_EXTRA_ATTR, fi->flags);
2796 	if (ri->i_inline & F2FS_PIN_FILE)
2797 		set_bit(FI_PIN_FILE, fi->flags);
2798 }
2799 
2800 static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri)
2801 {
2802 	ri->i_inline = 0;
2803 
2804 	if (is_inode_flag_set(inode, FI_INLINE_XATTR))
2805 		ri->i_inline |= F2FS_INLINE_XATTR;
2806 	if (is_inode_flag_set(inode, FI_INLINE_DATA))
2807 		ri->i_inline |= F2FS_INLINE_DATA;
2808 	if (is_inode_flag_set(inode, FI_INLINE_DENTRY))
2809 		ri->i_inline |= F2FS_INLINE_DENTRY;
2810 	if (is_inode_flag_set(inode, FI_DATA_EXIST))
2811 		ri->i_inline |= F2FS_DATA_EXIST;
2812 	if (is_inode_flag_set(inode, FI_INLINE_DOTS))
2813 		ri->i_inline |= F2FS_INLINE_DOTS;
2814 	if (is_inode_flag_set(inode, FI_EXTRA_ATTR))
2815 		ri->i_inline |= F2FS_EXTRA_ATTR;
2816 	if (is_inode_flag_set(inode, FI_PIN_FILE))
2817 		ri->i_inline |= F2FS_PIN_FILE;
2818 }
2819 
2820 static inline int f2fs_has_extra_attr(struct inode *inode)
2821 {
2822 	return is_inode_flag_set(inode, FI_EXTRA_ATTR);
2823 }
2824 
2825 static inline int f2fs_has_inline_xattr(struct inode *inode)
2826 {
2827 	return is_inode_flag_set(inode, FI_INLINE_XATTR);
2828 }
2829 
2830 static inline int f2fs_compressed_file(struct inode *inode)
2831 {
2832 	return S_ISREG(inode->i_mode) &&
2833 		is_inode_flag_set(inode, FI_COMPRESSED_FILE);
2834 }
2835 
2836 static inline bool f2fs_need_compress_data(struct inode *inode)
2837 {
2838 	int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode;
2839 
2840 	if (!f2fs_compressed_file(inode))
2841 		return false;
2842 
2843 	if (compress_mode == COMPR_MODE_FS)
2844 		return true;
2845 	else if (compress_mode == COMPR_MODE_USER &&
2846 			is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
2847 		return true;
2848 
2849 	return false;
2850 }
2851 
2852 static inline unsigned int addrs_per_inode(struct inode *inode)
2853 {
2854 	unsigned int addrs = CUR_ADDRS_PER_INODE(inode) -
2855 				get_inline_xattr_addrs(inode);
2856 
2857 	if (!f2fs_compressed_file(inode))
2858 		return addrs;
2859 	return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size);
2860 }
2861 
2862 static inline unsigned int addrs_per_block(struct inode *inode)
2863 {
2864 	if (!f2fs_compressed_file(inode))
2865 		return DEF_ADDRS_PER_BLOCK;
2866 	return ALIGN_DOWN(DEF_ADDRS_PER_BLOCK, F2FS_I(inode)->i_cluster_size);
2867 }
2868 
2869 static inline void *inline_xattr_addr(struct inode *inode, struct page *page)
2870 {
2871 	struct f2fs_inode *ri = F2FS_INODE(page);
2872 
2873 	return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE -
2874 					get_inline_xattr_addrs(inode)]);
2875 }
2876 
2877 static inline int inline_xattr_size(struct inode *inode)
2878 {
2879 	if (f2fs_has_inline_xattr(inode))
2880 		return get_inline_xattr_addrs(inode) * sizeof(__le32);
2881 	return 0;
2882 }
2883 
2884 static inline int f2fs_has_inline_data(struct inode *inode)
2885 {
2886 	return is_inode_flag_set(inode, FI_INLINE_DATA);
2887 }
2888 
2889 static inline int f2fs_exist_data(struct inode *inode)
2890 {
2891 	return is_inode_flag_set(inode, FI_DATA_EXIST);
2892 }
2893 
2894 static inline int f2fs_has_inline_dots(struct inode *inode)
2895 {
2896 	return is_inode_flag_set(inode, FI_INLINE_DOTS);
2897 }
2898 
2899 static inline int f2fs_is_mmap_file(struct inode *inode)
2900 {
2901 	return is_inode_flag_set(inode, FI_MMAP_FILE);
2902 }
2903 
2904 static inline bool f2fs_is_pinned_file(struct inode *inode)
2905 {
2906 	return is_inode_flag_set(inode, FI_PIN_FILE);
2907 }
2908 
2909 static inline bool f2fs_is_atomic_file(struct inode *inode)
2910 {
2911 	return is_inode_flag_set(inode, FI_ATOMIC_FILE);
2912 }
2913 
2914 static inline bool f2fs_is_commit_atomic_write(struct inode *inode)
2915 {
2916 	return is_inode_flag_set(inode, FI_ATOMIC_COMMIT);
2917 }
2918 
2919 static inline bool f2fs_is_volatile_file(struct inode *inode)
2920 {
2921 	return is_inode_flag_set(inode, FI_VOLATILE_FILE);
2922 }
2923 
2924 static inline bool f2fs_is_first_block_written(struct inode *inode)
2925 {
2926 	return is_inode_flag_set(inode, FI_FIRST_BLOCK_WRITTEN);
2927 }
2928 
2929 static inline bool f2fs_is_drop_cache(struct inode *inode)
2930 {
2931 	return is_inode_flag_set(inode, FI_DROP_CACHE);
2932 }
2933 
2934 static inline void *inline_data_addr(struct inode *inode, struct page *page)
2935 {
2936 	struct f2fs_inode *ri = F2FS_INODE(page);
2937 	int extra_size = get_extra_isize(inode);
2938 
2939 	return (void *)&(ri->i_addr[extra_size + DEF_INLINE_RESERVED_SIZE]);
2940 }
2941 
2942 static inline int f2fs_has_inline_dentry(struct inode *inode)
2943 {
2944 	return is_inode_flag_set(inode, FI_INLINE_DENTRY);
2945 }
2946 
2947 static inline int is_file(struct inode *inode, int type)
2948 {
2949 	return F2FS_I(inode)->i_advise & type;
2950 }
2951 
2952 static inline void set_file(struct inode *inode, int type)
2953 {
2954 	F2FS_I(inode)->i_advise |= type;
2955 	f2fs_mark_inode_dirty_sync(inode, true);
2956 }
2957 
2958 static inline void clear_file(struct inode *inode, int type)
2959 {
2960 	F2FS_I(inode)->i_advise &= ~type;
2961 	f2fs_mark_inode_dirty_sync(inode, true);
2962 }
2963 
2964 static inline bool f2fs_is_time_consistent(struct inode *inode)
2965 {
2966 	if (!timespec64_equal(F2FS_I(inode)->i_disk_time, &inode->i_atime))
2967 		return false;
2968 	if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 1, &inode->i_ctime))
2969 		return false;
2970 	if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 2, &inode->i_mtime))
2971 		return false;
2972 	if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 3,
2973 						&F2FS_I(inode)->i_crtime))
2974 		return false;
2975 	return true;
2976 }
2977 
2978 static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
2979 {
2980 	bool ret;
2981 
2982 	if (dsync) {
2983 		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2984 
2985 		spin_lock(&sbi->inode_lock[DIRTY_META]);
2986 		ret = list_empty(&F2FS_I(inode)->gdirty_list);
2987 		spin_unlock(&sbi->inode_lock[DIRTY_META]);
2988 		return ret;
2989 	}
2990 	if (!is_inode_flag_set(inode, FI_AUTO_RECOVER) ||
2991 			file_keep_isize(inode) ||
2992 			i_size_read(inode) & ~PAGE_MASK)
2993 		return false;
2994 
2995 	if (!f2fs_is_time_consistent(inode))
2996 		return false;
2997 
2998 	spin_lock(&F2FS_I(inode)->i_size_lock);
2999 	ret = F2FS_I(inode)->last_disk_size == i_size_read(inode);
3000 	spin_unlock(&F2FS_I(inode)->i_size_lock);
3001 
3002 	return ret;
3003 }
3004 
3005 static inline bool f2fs_readonly(struct super_block *sb)
3006 {
3007 	return sb_rdonly(sb);
3008 }
3009 
3010 static inline bool f2fs_cp_error(struct f2fs_sb_info *sbi)
3011 {
3012 	return is_set_ckpt_flags(sbi, CP_ERROR_FLAG);
3013 }
3014 
3015 static inline bool is_dot_dotdot(const u8 *name, size_t len)
3016 {
3017 	if (len == 1 && name[0] == '.')
3018 		return true;
3019 
3020 	if (len == 2 && name[0] == '.' && name[1] == '.')
3021 		return true;
3022 
3023 	return false;
3024 }
3025 
3026 static inline bool f2fs_may_extent_tree(struct inode *inode)
3027 {
3028 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3029 
3030 	if (!test_opt(sbi, EXTENT_CACHE) ||
3031 			is_inode_flag_set(inode, FI_NO_EXTENT) ||
3032 			is_inode_flag_set(inode, FI_COMPRESSED_FILE))
3033 		return false;
3034 
3035 	/*
3036 	 * for recovered files during mount do not create extents
3037 	 * if shrinker is not registered.
3038 	 */
3039 	if (list_empty(&sbi->s_list))
3040 		return false;
3041 
3042 	return S_ISREG(inode->i_mode);
3043 }
3044 
3045 static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi,
3046 					size_t size, gfp_t flags)
3047 {
3048 	if (time_to_inject(sbi, FAULT_KMALLOC)) {
3049 		f2fs_show_injection_info(sbi, FAULT_KMALLOC);
3050 		return NULL;
3051 	}
3052 
3053 	return kmalloc(size, flags);
3054 }
3055 
3056 static inline void *f2fs_kzalloc(struct f2fs_sb_info *sbi,
3057 					size_t size, gfp_t flags)
3058 {
3059 	return f2fs_kmalloc(sbi, size, flags | __GFP_ZERO);
3060 }
3061 
3062 static inline void *f2fs_kvmalloc(struct f2fs_sb_info *sbi,
3063 					size_t size, gfp_t flags)
3064 {
3065 	if (time_to_inject(sbi, FAULT_KVMALLOC)) {
3066 		f2fs_show_injection_info(sbi, FAULT_KVMALLOC);
3067 		return NULL;
3068 	}
3069 
3070 	return kvmalloc(size, flags);
3071 }
3072 
3073 static inline void *f2fs_kvzalloc(struct f2fs_sb_info *sbi,
3074 					size_t size, gfp_t flags)
3075 {
3076 	return f2fs_kvmalloc(sbi, size, flags | __GFP_ZERO);
3077 }
3078 
3079 static inline int get_extra_isize(struct inode *inode)
3080 {
3081 	return F2FS_I(inode)->i_extra_isize / sizeof(__le32);
3082 }
3083 
3084 static inline int get_inline_xattr_addrs(struct inode *inode)
3085 {
3086 	return F2FS_I(inode)->i_inline_xattr_size;
3087 }
3088 
3089 #define f2fs_get_inode_mode(i) \
3090 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
3091 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
3092 
3093 #define F2FS_TOTAL_EXTRA_ATTR_SIZE			\
3094 	(offsetof(struct f2fs_inode, i_extra_end) -	\
3095 	offsetof(struct f2fs_inode, i_extra_isize))	\
3096 
3097 #define F2FS_OLD_ATTRIBUTE_SIZE	(offsetof(struct f2fs_inode, i_addr))
3098 #define F2FS_FITS_IN_INODE(f2fs_inode, extra_isize, field)		\
3099 		((offsetof(typeof(*(f2fs_inode)), field) +	\
3100 		sizeof((f2fs_inode)->field))			\
3101 		<= (F2FS_OLD_ATTRIBUTE_SIZE + (extra_isize)))	\
3102 
3103 #define DEFAULT_IOSTAT_PERIOD_MS	3000
3104 #define MIN_IOSTAT_PERIOD_MS		100
3105 /* maximum period of iostat tracing is 1 day */
3106 #define MAX_IOSTAT_PERIOD_MS		8640000
3107 
3108 static inline void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
3109 {
3110 	int i;
3111 
3112 	spin_lock(&sbi->iostat_lock);
3113 	for (i = 0; i < NR_IO_TYPE; i++) {
3114 		sbi->rw_iostat[i] = 0;
3115 		sbi->prev_rw_iostat[i] = 0;
3116 	}
3117 	spin_unlock(&sbi->iostat_lock);
3118 }
3119 
3120 extern void f2fs_record_iostat(struct f2fs_sb_info *sbi);
3121 
3122 static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi,
3123 			enum iostat_type type, unsigned long long io_bytes)
3124 {
3125 	if (!sbi->iostat_enable)
3126 		return;
3127 	spin_lock(&sbi->iostat_lock);
3128 	sbi->rw_iostat[type] += io_bytes;
3129 
3130 	if (type == APP_WRITE_IO || type == APP_DIRECT_IO)
3131 		sbi->rw_iostat[APP_BUFFERED_IO] =
3132 			sbi->rw_iostat[APP_WRITE_IO] -
3133 			sbi->rw_iostat[APP_DIRECT_IO];
3134 
3135 	if (type == APP_READ_IO || type == APP_DIRECT_READ_IO)
3136 		sbi->rw_iostat[APP_BUFFERED_READ_IO] =
3137 			sbi->rw_iostat[APP_READ_IO] -
3138 			sbi->rw_iostat[APP_DIRECT_READ_IO];
3139 	spin_unlock(&sbi->iostat_lock);
3140 
3141 	f2fs_record_iostat(sbi);
3142 }
3143 
3144 #define __is_large_section(sbi)		((sbi)->segs_per_sec > 1)
3145 
3146 #define __is_meta_io(fio) (PAGE_TYPE_OF_BIO((fio)->type) == META)
3147 
3148 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3149 					block_t blkaddr, int type);
3150 static inline void verify_blkaddr(struct f2fs_sb_info *sbi,
3151 					block_t blkaddr, int type)
3152 {
3153 	if (!f2fs_is_valid_blkaddr(sbi, blkaddr, type)) {
3154 		f2fs_err(sbi, "invalid blkaddr: %u, type: %d, run fsck to fix.",
3155 			 blkaddr, type);
3156 		f2fs_bug_on(sbi, 1);
3157 	}
3158 }
3159 
3160 static inline bool __is_valid_data_blkaddr(block_t blkaddr)
3161 {
3162 	if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR ||
3163 			blkaddr == COMPRESS_ADDR)
3164 		return false;
3165 	return true;
3166 }
3167 
3168 static inline void f2fs_set_page_private(struct page *page,
3169 						unsigned long data)
3170 {
3171 	if (PagePrivate(page))
3172 		return;
3173 
3174 	attach_page_private(page, (void *)data);
3175 }
3176 
3177 static inline void f2fs_clear_page_private(struct page *page)
3178 {
3179 	detach_page_private(page);
3180 }
3181 
3182 /*
3183  * file.c
3184  */
3185 int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync);
3186 void f2fs_truncate_data_blocks(struct dnode_of_data *dn);
3187 int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock);
3188 int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock);
3189 int f2fs_truncate(struct inode *inode);
3190 int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path,
3191 		 struct kstat *stat, u32 request_mask, unsigned int flags);
3192 int f2fs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
3193 		 struct iattr *attr);
3194 int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end);
3195 void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count);
3196 int f2fs_precache_extents(struct inode *inode);
3197 long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
3198 long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
3199 int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid);
3200 int f2fs_pin_file_control(struct inode *inode, bool inc);
3201 
3202 /*
3203  * inode.c
3204  */
3205 void f2fs_set_inode_flags(struct inode *inode);
3206 bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct page *page);
3207 void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct page *page);
3208 struct inode *f2fs_iget(struct super_block *sb, unsigned long ino);
3209 struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino);
3210 int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink);
3211 void f2fs_update_inode(struct inode *inode, struct page *node_page);
3212 void f2fs_update_inode_page(struct inode *inode);
3213 int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc);
3214 void f2fs_evict_inode(struct inode *inode);
3215 void f2fs_handle_failed_inode(struct inode *inode);
3216 
3217 /*
3218  * namei.c
3219  */
3220 int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
3221 							bool hot, bool set);
3222 struct dentry *f2fs_get_parent(struct dentry *child);
3223 
3224 /*
3225  * dir.c
3226  */
3227 unsigned char f2fs_get_de_type(struct f2fs_dir_entry *de);
3228 int f2fs_init_casefolded_name(const struct inode *dir,
3229 			      struct f2fs_filename *fname);
3230 int f2fs_setup_filename(struct inode *dir, const struct qstr *iname,
3231 			int lookup, struct f2fs_filename *fname);
3232 int f2fs_prepare_lookup(struct inode *dir, struct dentry *dentry,
3233 			struct f2fs_filename *fname);
3234 void f2fs_free_filename(struct f2fs_filename *fname);
3235 struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d,
3236 			const struct f2fs_filename *fname, int *max_slots);
3237 int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
3238 			unsigned int start_pos, struct fscrypt_str *fstr);
3239 void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
3240 			struct f2fs_dentry_ptr *d);
3241 struct page *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
3242 			const struct f2fs_filename *fname, struct page *dpage);
3243 void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode,
3244 			unsigned int current_depth);
3245 int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots);
3246 void f2fs_drop_nlink(struct inode *dir, struct inode *inode);
3247 struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
3248 					 const struct f2fs_filename *fname,
3249 					 struct page **res_page);
3250 struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
3251 			const struct qstr *child, struct page **res_page);
3252 struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct page **p);
3253 ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr,
3254 			struct page **page);
3255 void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
3256 			struct page *page, struct inode *inode);
3257 bool f2fs_has_enough_room(struct inode *dir, struct page *ipage,
3258 			  const struct f2fs_filename *fname);
3259 void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d,
3260 			const struct fscrypt_str *name, f2fs_hash_t name_hash,
3261 			unsigned int bit_pos);
3262 int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
3263 			struct inode *inode, nid_t ino, umode_t mode);
3264 int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname,
3265 			struct inode *inode, nid_t ino, umode_t mode);
3266 int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
3267 			struct inode *inode, nid_t ino, umode_t mode);
3268 void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
3269 			struct inode *dir, struct inode *inode);
3270 int f2fs_do_tmpfile(struct inode *inode, struct inode *dir);
3271 bool f2fs_empty_dir(struct inode *dir);
3272 
3273 static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode)
3274 {
3275 	if (fscrypt_is_nokey_name(dentry))
3276 		return -ENOKEY;
3277 	return f2fs_do_add_link(d_inode(dentry->d_parent), &dentry->d_name,
3278 				inode, inode->i_ino, inode->i_mode);
3279 }
3280 
3281 /*
3282  * super.c
3283  */
3284 int f2fs_inode_dirtied(struct inode *inode, bool sync);
3285 void f2fs_inode_synced(struct inode *inode);
3286 int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
3287 int f2fs_quota_sync(struct super_block *sb, int type);
3288 loff_t max_file_blocks(struct inode *inode);
3289 void f2fs_quota_off_umount(struct super_block *sb);
3290 int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
3291 int f2fs_sync_fs(struct super_block *sb, int sync);
3292 int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
3293 
3294 /*
3295  * hash.c
3296  */
3297 void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname);
3298 
3299 /*
3300  * node.c
3301  */
3302 struct dnode_of_data;
3303 struct node_info;
3304 
3305 int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
3306 bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type);
3307 bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct page *page);
3308 void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi);
3309 void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct page *page);
3310 void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi);
3311 int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid);
3312 bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
3313 bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino);
3314 int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
3315 						struct node_info *ni);
3316 pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
3317 int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
3318 int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
3319 int f2fs_truncate_xattr_node(struct inode *inode);
3320 int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
3321 					unsigned int seq_id);
3322 int f2fs_remove_inode_page(struct inode *inode);
3323 struct page *f2fs_new_inode_page(struct inode *inode);
3324 struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs);
3325 void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid);
3326 struct page *f2fs_get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid);
3327 struct page *f2fs_get_node_page_ra(struct page *parent, int start);
3328 int f2fs_move_node_page(struct page *node_page, int gc_type);
3329 void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
3330 int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
3331 			struct writeback_control *wbc, bool atomic,
3332 			unsigned int *seq_id);
3333 int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
3334 			struct writeback_control *wbc,
3335 			bool do_balance, enum iostat_type io_type);
3336 int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount);
3337 bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid);
3338 void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid);
3339 void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid);
3340 int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink);
3341 int f2fs_recover_inline_xattr(struct inode *inode, struct page *page);
3342 int f2fs_recover_xattr_data(struct inode *inode, struct page *page);
3343 int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct page *page);
3344 int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
3345 			unsigned int segno, struct f2fs_summary_block *sum);
3346 int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3347 int f2fs_build_node_manager(struct f2fs_sb_info *sbi);
3348 void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi);
3349 int __init f2fs_create_node_manager_caches(void);
3350 void f2fs_destroy_node_manager_caches(void);
3351 
3352 /*
3353  * segment.c
3354  */
3355 bool f2fs_need_SSR(struct f2fs_sb_info *sbi);
3356 void f2fs_register_inmem_page(struct inode *inode, struct page *page);
3357 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure);
3358 void f2fs_drop_inmem_pages(struct inode *inode);
3359 void f2fs_drop_inmem_page(struct inode *inode, struct page *page);
3360 int f2fs_commit_inmem_pages(struct inode *inode);
3361 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need);
3362 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg);
3363 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino);
3364 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi);
3365 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi);
3366 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free);
3367 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr);
3368 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr);
3369 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi);
3370 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi);
3371 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi);
3372 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
3373 					struct cp_control *cpc);
3374 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi);
3375 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi);
3376 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
3377 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
3378 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
3379 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
3380 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi);
3381 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi);
3382 void f2fs_get_new_segment(struct f2fs_sb_info *sbi,
3383 			unsigned int *newseg, bool new_sec, int dir);
3384 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3385 					unsigned int start, unsigned int end);
3386 void f2fs_allocate_new_segment(struct f2fs_sb_info *sbi, int type);
3387 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
3388 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
3389 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3390 					struct cp_control *cpc);
3391 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno);
3392 void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src,
3393 					block_t blk_addr);
3394 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3395 						enum iostat_type io_type);
3396 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio);
3397 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3398 			struct f2fs_io_info *fio);
3399 int f2fs_inplace_write_data(struct f2fs_io_info *fio);
3400 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3401 			block_t old_blkaddr, block_t new_blkaddr,
3402 			bool recover_curseg, bool recover_newaddr,
3403 			bool from_gc);
3404 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3405 			block_t old_addr, block_t new_addr,
3406 			unsigned char version, bool recover_curseg,
3407 			bool recover_newaddr);
3408 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3409 			block_t old_blkaddr, block_t *new_blkaddr,
3410 			struct f2fs_summary *sum, int type,
3411 			struct f2fs_io_info *fio);
3412 void f2fs_wait_on_page_writeback(struct page *page,
3413 			enum page_type type, bool ordered, bool locked);
3414 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
3415 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3416 								block_t len);
3417 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3418 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk);
3419 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3420 			unsigned int val, int alloc);
3421 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3422 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi);
3423 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi);
3424 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi);
3425 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi);
3426 int __init f2fs_create_segment_manager_caches(void);
3427 void f2fs_destroy_segment_manager_caches(void);
3428 int f2fs_rw_hint_to_seg_type(enum rw_hint hint);
3429 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3430 			enum page_type type, enum temp_type temp);
3431 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
3432 			unsigned int segno);
3433 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
3434 			unsigned int segno);
3435 
3436 /*
3437  * checkpoint.c
3438  */
3439 void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io);
3440 struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3441 struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3442 struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index);
3443 struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index);
3444 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
3445 					block_t blkaddr, int type);
3446 int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
3447 			int type, bool sync);
3448 void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index);
3449 long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
3450 			long nr_to_write, enum iostat_type io_type);
3451 void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
3452 void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
3453 void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all);
3454 bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode);
3455 void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
3456 					unsigned int devidx, int type);
3457 bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
3458 					unsigned int devidx, int type);
3459 int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi);
3460 int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi);
3461 void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi);
3462 void f2fs_add_orphan_inode(struct inode *inode);
3463 void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino);
3464 int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi);
3465 int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
3466 void f2fs_update_dirty_page(struct inode *inode, struct page *page);
3467 void f2fs_remove_dirty_inode(struct inode *inode);
3468 int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type);
3469 void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
3470 u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
3471 int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3472 void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
3473 int __init f2fs_create_checkpoint_caches(void);
3474 void f2fs_destroy_checkpoint_caches(void);
3475 int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi);
3476 int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi);
3477 void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi);
3478 void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi);
3479 
3480 /*
3481  * data.c
3482  */
3483 int __init f2fs_init_bioset(void);
3484 void f2fs_destroy_bioset(void);
3485 int f2fs_init_bio_entry_cache(void);
3486 void f2fs_destroy_bio_entry_cache(void);
3487 void f2fs_submit_bio(struct f2fs_sb_info *sbi,
3488 				struct bio *bio, enum page_type type);
3489 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
3490 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
3491 				struct inode *inode, struct page *page,
3492 				nid_t ino, enum page_type type);
3493 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
3494 					struct bio **bio, struct page *page);
3495 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
3496 int f2fs_submit_page_bio(struct f2fs_io_info *fio);
3497 int f2fs_merge_page_bio(struct f2fs_io_info *fio);
3498 void f2fs_submit_page_write(struct f2fs_io_info *fio);
3499 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
3500 			block_t blk_addr, struct bio *bio);
3501 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr);
3502 void f2fs_set_data_blkaddr(struct dnode_of_data *dn);
3503 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr);
3504 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count);
3505 int f2fs_reserve_new_block(struct dnode_of_data *dn);
3506 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);
3507 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from);
3508 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
3509 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
3510 			int op_flags, bool for_write);
3511 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index);
3512 struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
3513 			bool for_write);
3514 struct page *f2fs_get_new_data_page(struct inode *inode,
3515 			struct page *ipage, pgoff_t index, bool new_i_size);
3516 int f2fs_do_write_data_page(struct f2fs_io_info *fio);
3517 void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock);
3518 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
3519 			int create, int flag);
3520 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3521 			u64 start, u64 len);
3522 int f2fs_encrypt_one_page(struct f2fs_io_info *fio);
3523 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio);
3524 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio);
3525 int f2fs_write_single_data_page(struct page *page, int *submitted,
3526 				struct bio **bio, sector_t *last_block,
3527 				struct writeback_control *wbc,
3528 				enum iostat_type io_type,
3529 				int compr_blocks, bool allow_balance);
3530 void f2fs_invalidate_page(struct page *page, unsigned int offset,
3531 			unsigned int length);
3532 int f2fs_release_page(struct page *page, gfp_t wait);
3533 #ifdef CONFIG_MIGRATION
3534 int f2fs_migrate_page(struct address_space *mapping, struct page *newpage,
3535 			struct page *page, enum migrate_mode mode);
3536 #endif
3537 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
3538 void f2fs_clear_page_cache_dirty_tag(struct page *page);
3539 int f2fs_init_post_read_processing(void);
3540 void f2fs_destroy_post_read_processing(void);
3541 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi);
3542 void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi);
3543 
3544 /*
3545  * gc.c
3546  */
3547 int f2fs_start_gc_thread(struct f2fs_sb_info *sbi);
3548 void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi);
3549 block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode);
3550 int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background,
3551 			unsigned int segno);
3552 void f2fs_build_gc_manager(struct f2fs_sb_info *sbi);
3553 int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count);
3554 int __init f2fs_create_garbage_collection_cache(void);
3555 void f2fs_destroy_garbage_collection_cache(void);
3556 
3557 /*
3558  * recovery.c
3559  */
3560 int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only);
3561 bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi);
3562 
3563 /*
3564  * debug.c
3565  */
3566 #ifdef CONFIG_F2FS_STAT_FS
3567 struct f2fs_stat_info {
3568 	struct list_head stat_list;
3569 	struct f2fs_sb_info *sbi;
3570 	int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs;
3571 	int main_area_segs, main_area_sections, main_area_zones;
3572 	unsigned long long hit_largest, hit_cached, hit_rbtree;
3573 	unsigned long long hit_total, total_ext;
3574 	int ext_tree, zombie_tree, ext_node;
3575 	int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta;
3576 	int ndirty_data, ndirty_qdata;
3577 	int inmem_pages;
3578 	unsigned int ndirty_dirs, ndirty_files, nquota_files, ndirty_all;
3579 	int nats, dirty_nats, sits, dirty_sits;
3580 	int free_nids, avail_nids, alloc_nids;
3581 	int total_count, utilization;
3582 	int bg_gc, nr_wb_cp_data, nr_wb_data;
3583 	int nr_rd_data, nr_rd_node, nr_rd_meta;
3584 	int nr_dio_read, nr_dio_write;
3585 	unsigned int io_skip_bggc, other_skip_bggc;
3586 	int nr_flushing, nr_flushed, flush_list_empty;
3587 	int nr_discarding, nr_discarded;
3588 	int nr_discard_cmd;
3589 	unsigned int undiscard_blks;
3590 	int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
3591 	unsigned int cur_ckpt_time, peak_ckpt_time;
3592 	int inline_xattr, inline_inode, inline_dir, append, update, orphans;
3593 	int compr_inode;
3594 	unsigned long long compr_blocks;
3595 	int aw_cnt, max_aw_cnt, vw_cnt, max_vw_cnt;
3596 	unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
3597 	unsigned int bimodal, avg_vblocks;
3598 	int util_free, util_valid, util_invalid;
3599 	int rsvd_segs, overp_segs;
3600 	int dirty_count, node_pages, meta_pages;
3601 	int prefree_count, call_count, cp_count, bg_cp_count;
3602 	int tot_segs, node_segs, data_segs, free_segs, free_secs;
3603 	int bg_node_segs, bg_data_segs;
3604 	int tot_blks, data_blks, node_blks;
3605 	int bg_data_blks, bg_node_blks;
3606 	unsigned long long skipped_atomic_files[2];
3607 	int curseg[NR_CURSEG_TYPE];
3608 	int cursec[NR_CURSEG_TYPE];
3609 	int curzone[NR_CURSEG_TYPE];
3610 	unsigned int dirty_seg[NR_CURSEG_TYPE];
3611 	unsigned int full_seg[NR_CURSEG_TYPE];
3612 	unsigned int valid_blks[NR_CURSEG_TYPE];
3613 
3614 	unsigned int meta_count[META_MAX];
3615 	unsigned int segment_count[2];
3616 	unsigned int block_count[2];
3617 	unsigned int inplace_count;
3618 	unsigned long long base_mem, cache_mem, page_mem;
3619 };
3620 
3621 static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi)
3622 {
3623 	return (struct f2fs_stat_info *)sbi->stat_info;
3624 }
3625 
3626 #define stat_inc_cp_count(si)		((si)->cp_count++)
3627 #define stat_inc_bg_cp_count(si)	((si)->bg_cp_count++)
3628 #define stat_inc_call_count(si)		((si)->call_count++)
3629 #define stat_inc_bggc_count(si)		((si)->bg_gc++)
3630 #define stat_io_skip_bggc_count(sbi)	((sbi)->io_skip_bggc++)
3631 #define stat_other_skip_bggc_count(sbi)	((sbi)->other_skip_bggc++)
3632 #define stat_inc_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]++)
3633 #define stat_dec_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]--)
3634 #define stat_inc_total_hit(sbi)		(atomic64_inc(&(sbi)->total_hit_ext))
3635 #define stat_inc_rbtree_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_rbtree))
3636 #define stat_inc_largest_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_largest))
3637 #define stat_inc_cached_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_cached))
3638 #define stat_inc_inline_xattr(inode)					\
3639 	do {								\
3640 		if (f2fs_has_inline_xattr(inode))			\
3641 			(atomic_inc(&F2FS_I_SB(inode)->inline_xattr));	\
3642 	} while (0)
3643 #define stat_dec_inline_xattr(inode)					\
3644 	do {								\
3645 		if (f2fs_has_inline_xattr(inode))			\
3646 			(atomic_dec(&F2FS_I_SB(inode)->inline_xattr));	\
3647 	} while (0)
3648 #define stat_inc_inline_inode(inode)					\
3649 	do {								\
3650 		if (f2fs_has_inline_data(inode))			\
3651 			(atomic_inc(&F2FS_I_SB(inode)->inline_inode));	\
3652 	} while (0)
3653 #define stat_dec_inline_inode(inode)					\
3654 	do {								\
3655 		if (f2fs_has_inline_data(inode))			\
3656 			(atomic_dec(&F2FS_I_SB(inode)->inline_inode));	\
3657 	} while (0)
3658 #define stat_inc_inline_dir(inode)					\
3659 	do {								\
3660 		if (f2fs_has_inline_dentry(inode))			\
3661 			(atomic_inc(&F2FS_I_SB(inode)->inline_dir));	\
3662 	} while (0)
3663 #define stat_dec_inline_dir(inode)					\
3664 	do {								\
3665 		if (f2fs_has_inline_dentry(inode))			\
3666 			(atomic_dec(&F2FS_I_SB(inode)->inline_dir));	\
3667 	} while (0)
3668 #define stat_inc_compr_inode(inode)					\
3669 	do {								\
3670 		if (f2fs_compressed_file(inode))			\
3671 			(atomic_inc(&F2FS_I_SB(inode)->compr_inode));	\
3672 	} while (0)
3673 #define stat_dec_compr_inode(inode)					\
3674 	do {								\
3675 		if (f2fs_compressed_file(inode))			\
3676 			(atomic_dec(&F2FS_I_SB(inode)->compr_inode));	\
3677 	} while (0)
3678 #define stat_add_compr_blocks(inode, blocks)				\
3679 		(atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
3680 #define stat_sub_compr_blocks(inode, blocks)				\
3681 		(atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
3682 #define stat_inc_meta_count(sbi, blkaddr)				\
3683 	do {								\
3684 		if (blkaddr < SIT_I(sbi)->sit_base_addr)		\
3685 			atomic_inc(&(sbi)->meta_count[META_CP]);	\
3686 		else if (blkaddr < NM_I(sbi)->nat_blkaddr)		\
3687 			atomic_inc(&(sbi)->meta_count[META_SIT]);	\
3688 		else if (blkaddr < SM_I(sbi)->ssa_blkaddr)		\
3689 			atomic_inc(&(sbi)->meta_count[META_NAT]);	\
3690 		else if (blkaddr < SM_I(sbi)->main_blkaddr)		\
3691 			atomic_inc(&(sbi)->meta_count[META_SSA]);	\
3692 	} while (0)
3693 #define stat_inc_seg_type(sbi, curseg)					\
3694 		((sbi)->segment_count[(curseg)->alloc_type]++)
3695 #define stat_inc_block_count(sbi, curseg)				\
3696 		((sbi)->block_count[(curseg)->alloc_type]++)
3697 #define stat_inc_inplace_blocks(sbi)					\
3698 		(atomic_inc(&(sbi)->inplace_count))
3699 #define stat_update_max_atomic_write(inode)				\
3700 	do {								\
3701 		int cur = F2FS_I_SB(inode)->atomic_files;	\
3702 		int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt);	\
3703 		if (cur > max)						\
3704 			atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur);	\
3705 	} while (0)
3706 #define stat_inc_volatile_write(inode)					\
3707 		(atomic_inc(&F2FS_I_SB(inode)->vw_cnt))
3708 #define stat_dec_volatile_write(inode)					\
3709 		(atomic_dec(&F2FS_I_SB(inode)->vw_cnt))
3710 #define stat_update_max_volatile_write(inode)				\
3711 	do {								\
3712 		int cur = atomic_read(&F2FS_I_SB(inode)->vw_cnt);	\
3713 		int max = atomic_read(&F2FS_I_SB(inode)->max_vw_cnt);	\
3714 		if (cur > max)						\
3715 			atomic_set(&F2FS_I_SB(inode)->max_vw_cnt, cur);	\
3716 	} while (0)
3717 #define stat_inc_seg_count(sbi, type, gc_type)				\
3718 	do {								\
3719 		struct f2fs_stat_info *si = F2FS_STAT(sbi);		\
3720 		si->tot_segs++;						\
3721 		if ((type) == SUM_TYPE_DATA) {				\
3722 			si->data_segs++;				\
3723 			si->bg_data_segs += (gc_type == BG_GC) ? 1 : 0;	\
3724 		} else {						\
3725 			si->node_segs++;				\
3726 			si->bg_node_segs += (gc_type == BG_GC) ? 1 : 0;	\
3727 		}							\
3728 	} while (0)
3729 
3730 #define stat_inc_tot_blk_count(si, blks)				\
3731 	((si)->tot_blks += (blks))
3732 
3733 #define stat_inc_data_blk_count(sbi, blks, gc_type)			\
3734 	do {								\
3735 		struct f2fs_stat_info *si = F2FS_STAT(sbi);		\
3736 		stat_inc_tot_blk_count(si, blks);			\
3737 		si->data_blks += (blks);				\
3738 		si->bg_data_blks += ((gc_type) == BG_GC) ? (blks) : 0;	\
3739 	} while (0)
3740 
3741 #define stat_inc_node_blk_count(sbi, blks, gc_type)			\
3742 	do {								\
3743 		struct f2fs_stat_info *si = F2FS_STAT(sbi);		\
3744 		stat_inc_tot_blk_count(si, blks);			\
3745 		si->node_blks += (blks);				\
3746 		si->bg_node_blks += ((gc_type) == BG_GC) ? (blks) : 0;	\
3747 	} while (0)
3748 
3749 int f2fs_build_stats(struct f2fs_sb_info *sbi);
3750 void f2fs_destroy_stats(struct f2fs_sb_info *sbi);
3751 void __init f2fs_create_root_stats(void);
3752 void f2fs_destroy_root_stats(void);
3753 void f2fs_update_sit_info(struct f2fs_sb_info *sbi);
3754 #else
3755 #define stat_inc_cp_count(si)				do { } while (0)
3756 #define stat_inc_bg_cp_count(si)			do { } while (0)
3757 #define stat_inc_call_count(si)				do { } while (0)
3758 #define stat_inc_bggc_count(si)				do { } while (0)
3759 #define stat_io_skip_bggc_count(sbi)			do { } while (0)
3760 #define stat_other_skip_bggc_count(sbi)			do { } while (0)
3761 #define stat_inc_dirty_inode(sbi, type)			do { } while (0)
3762 #define stat_dec_dirty_inode(sbi, type)			do { } while (0)
3763 #define stat_inc_total_hit(sbi)				do { } while (0)
3764 #define stat_inc_rbtree_node_hit(sbi)			do { } while (0)
3765 #define stat_inc_largest_node_hit(sbi)			do { } while (0)
3766 #define stat_inc_cached_node_hit(sbi)			do { } while (0)
3767 #define stat_inc_inline_xattr(inode)			do { } while (0)
3768 #define stat_dec_inline_xattr(inode)			do { } while (0)
3769 #define stat_inc_inline_inode(inode)			do { } while (0)
3770 #define stat_dec_inline_inode(inode)			do { } while (0)
3771 #define stat_inc_inline_dir(inode)			do { } while (0)
3772 #define stat_dec_inline_dir(inode)			do { } while (0)
3773 #define stat_inc_compr_inode(inode)			do { } while (0)
3774 #define stat_dec_compr_inode(inode)			do { } while (0)
3775 #define stat_add_compr_blocks(inode, blocks)		do { } while (0)
3776 #define stat_sub_compr_blocks(inode, blocks)		do { } while (0)
3777 #define stat_update_max_atomic_write(inode)		do { } while (0)
3778 #define stat_inc_volatile_write(inode)			do { } while (0)
3779 #define stat_dec_volatile_write(inode)			do { } while (0)
3780 #define stat_update_max_volatile_write(inode)		do { } while (0)
3781 #define stat_inc_meta_count(sbi, blkaddr)		do { } while (0)
3782 #define stat_inc_seg_type(sbi, curseg)			do { } while (0)
3783 #define stat_inc_block_count(sbi, curseg)		do { } while (0)
3784 #define stat_inc_inplace_blocks(sbi)			do { } while (0)
3785 #define stat_inc_seg_count(sbi, type, gc_type)		do { } while (0)
3786 #define stat_inc_tot_blk_count(si, blks)		do { } while (0)
3787 #define stat_inc_data_blk_count(sbi, blks, gc_type)	do { } while (0)
3788 #define stat_inc_node_blk_count(sbi, blks, gc_type)	do { } while (0)
3789 
3790 static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; }
3791 static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { }
3792 static inline void __init f2fs_create_root_stats(void) { }
3793 static inline void f2fs_destroy_root_stats(void) { }
3794 static inline void f2fs_update_sit_info(struct f2fs_sb_info *sbi) {}
3795 #endif
3796 
3797 extern const struct file_operations f2fs_dir_operations;
3798 extern const struct file_operations f2fs_file_operations;
3799 extern const struct inode_operations f2fs_file_inode_operations;
3800 extern const struct address_space_operations f2fs_dblock_aops;
3801 extern const struct address_space_operations f2fs_node_aops;
3802 extern const struct address_space_operations f2fs_meta_aops;
3803 extern const struct inode_operations f2fs_dir_inode_operations;
3804 extern const struct inode_operations f2fs_symlink_inode_operations;
3805 extern const struct inode_operations f2fs_encrypted_symlink_inode_operations;
3806 extern const struct inode_operations f2fs_special_inode_operations;
3807 extern struct kmem_cache *f2fs_inode_entry_slab;
3808 
3809 /*
3810  * inline.c
3811  */
3812 bool f2fs_may_inline_data(struct inode *inode);
3813 bool f2fs_may_inline_dentry(struct inode *inode);
3814 void f2fs_do_read_inline_data(struct page *page, struct page *ipage);
3815 void f2fs_truncate_inline_inode(struct inode *inode,
3816 						struct page *ipage, u64 from);
3817 int f2fs_read_inline_data(struct inode *inode, struct page *page);
3818 int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page);
3819 int f2fs_convert_inline_inode(struct inode *inode);
3820 int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry);
3821 int f2fs_write_inline_data(struct inode *inode, struct page *page);
3822 int f2fs_recover_inline_data(struct inode *inode, struct page *npage);
3823 struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
3824 					const struct f2fs_filename *fname,
3825 					struct page **res_page);
3826 int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
3827 			struct page *ipage);
3828 int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
3829 			struct inode *inode, nid_t ino, umode_t mode);
3830 void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
3831 				struct page *page, struct inode *dir,
3832 				struct inode *inode);
3833 bool f2fs_empty_inline_dir(struct inode *dir);
3834 int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
3835 			struct fscrypt_str *fstr);
3836 int f2fs_inline_data_fiemap(struct inode *inode,
3837 			struct fiemap_extent_info *fieinfo,
3838 			__u64 start, __u64 len);
3839 
3840 /*
3841  * shrinker.c
3842  */
3843 unsigned long f2fs_shrink_count(struct shrinker *shrink,
3844 			struct shrink_control *sc);
3845 unsigned long f2fs_shrink_scan(struct shrinker *shrink,
3846 			struct shrink_control *sc);
3847 void f2fs_join_shrinker(struct f2fs_sb_info *sbi);
3848 void f2fs_leave_shrinker(struct f2fs_sb_info *sbi);
3849 
3850 /*
3851  * extent_cache.c
3852  */
3853 struct rb_entry *f2fs_lookup_rb_tree(struct rb_root_cached *root,
3854 				struct rb_entry *cached_re, unsigned int ofs);
3855 struct rb_node **f2fs_lookup_rb_tree_ext(struct f2fs_sb_info *sbi,
3856 				struct rb_root_cached *root,
3857 				struct rb_node **parent,
3858 				unsigned long long key, bool *left_most);
3859 struct rb_node **f2fs_lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
3860 				struct rb_root_cached *root,
3861 				struct rb_node **parent,
3862 				unsigned int ofs, bool *leftmost);
3863 struct rb_entry *f2fs_lookup_rb_tree_ret(struct rb_root_cached *root,
3864 		struct rb_entry *cached_re, unsigned int ofs,
3865 		struct rb_entry **prev_entry, struct rb_entry **next_entry,
3866 		struct rb_node ***insert_p, struct rb_node **insert_parent,
3867 		bool force, bool *leftmost);
3868 bool f2fs_check_rb_tree_consistence(struct f2fs_sb_info *sbi,
3869 				struct rb_root_cached *root, bool check_key);
3870 unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink);
3871 void f2fs_init_extent_tree(struct inode *inode, struct page *ipage);
3872 void f2fs_drop_extent_tree(struct inode *inode);
3873 unsigned int f2fs_destroy_extent_node(struct inode *inode);
3874 void f2fs_destroy_extent_tree(struct inode *inode);
3875 bool f2fs_lookup_extent_cache(struct inode *inode, pgoff_t pgofs,
3876 			struct extent_info *ei);
3877 void f2fs_update_extent_cache(struct dnode_of_data *dn);
3878 void f2fs_update_extent_cache_range(struct dnode_of_data *dn,
3879 			pgoff_t fofs, block_t blkaddr, unsigned int len);
3880 void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi);
3881 int __init f2fs_create_extent_cache(void);
3882 void f2fs_destroy_extent_cache(void);
3883 
3884 /*
3885  * sysfs.c
3886  */
3887 int __init f2fs_init_sysfs(void);
3888 void f2fs_exit_sysfs(void);
3889 int f2fs_register_sysfs(struct f2fs_sb_info *sbi);
3890 void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi);
3891 
3892 /* verity.c */
3893 extern const struct fsverity_operations f2fs_verityops;
3894 
3895 /*
3896  * crypto support
3897  */
3898 static inline bool f2fs_encrypted_file(struct inode *inode)
3899 {
3900 	return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
3901 }
3902 
3903 static inline void f2fs_set_encrypted_inode(struct inode *inode)
3904 {
3905 #ifdef CONFIG_FS_ENCRYPTION
3906 	file_set_encrypt(inode);
3907 	f2fs_set_inode_flags(inode);
3908 #endif
3909 }
3910 
3911 /*
3912  * Returns true if the reads of the inode's data need to undergo some
3913  * postprocessing step, like decryption or authenticity verification.
3914  */
3915 static inline bool f2fs_post_read_required(struct inode *inode)
3916 {
3917 	return f2fs_encrypted_file(inode) || fsverity_active(inode) ||
3918 		f2fs_compressed_file(inode);
3919 }
3920 
3921 /*
3922  * compress.c
3923  */
3924 #ifdef CONFIG_F2FS_FS_COMPRESSION
3925 bool f2fs_is_compressed_page(struct page *page);
3926 struct page *f2fs_compress_control_page(struct page *page);
3927 int f2fs_prepare_compress_overwrite(struct inode *inode,
3928 			struct page **pagep, pgoff_t index, void **fsdata);
3929 bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
3930 					pgoff_t index, unsigned copied);
3931 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock);
3932 void f2fs_compress_write_end_io(struct bio *bio, struct page *page);
3933 bool f2fs_is_compress_backend_ready(struct inode *inode);
3934 int f2fs_init_compress_mempool(void);
3935 void f2fs_destroy_compress_mempool(void);
3936 void f2fs_end_read_compressed_page(struct page *page, bool failed);
3937 bool f2fs_cluster_is_empty(struct compress_ctx *cc);
3938 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
3939 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page);
3940 int f2fs_write_multi_pages(struct compress_ctx *cc,
3941 						int *submitted,
3942 						struct writeback_control *wbc,
3943 						enum iostat_type io_type);
3944 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index);
3945 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
3946 				unsigned nr_pages, sector_t *last_block_in_bio,
3947 				bool is_readahead, bool for_write);
3948 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc);
3949 void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed);
3950 void f2fs_put_page_dic(struct page *page);
3951 int f2fs_init_compress_ctx(struct compress_ctx *cc);
3952 void f2fs_destroy_compress_ctx(struct compress_ctx *cc);
3953 void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
3954 int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
3955 void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
3956 int __init f2fs_init_compress_cache(void);
3957 void f2fs_destroy_compress_cache(void);
3958 #else
3959 static inline bool f2fs_is_compressed_page(struct page *page) { return false; }
3960 static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
3961 {
3962 	if (!f2fs_compressed_file(inode))
3963 		return true;
3964 	/* not support compression */
3965 	return false;
3966 }
3967 static inline struct page *f2fs_compress_control_page(struct page *page)
3968 {
3969 	WARN_ON_ONCE(1);
3970 	return ERR_PTR(-EINVAL);
3971 }
3972 static inline int f2fs_init_compress_mempool(void) { return 0; }
3973 static inline void f2fs_destroy_compress_mempool(void) { }
3974 static inline void f2fs_end_read_compressed_page(struct page *page, bool failed)
3975 {
3976 	WARN_ON_ONCE(1);
3977 }
3978 static inline void f2fs_put_page_dic(struct page *page)
3979 {
3980 	WARN_ON_ONCE(1);
3981 }
3982 static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
3983 static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
3984 static inline int __init f2fs_init_compress_cache(void) { return 0; }
3985 static inline void f2fs_destroy_compress_cache(void) { }
3986 #endif
3987 
3988 static inline void set_compress_context(struct inode *inode)
3989 {
3990 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3991 
3992 	F2FS_I(inode)->i_compress_algorithm =
3993 			F2FS_OPTION(sbi).compress_algorithm;
3994 	F2FS_I(inode)->i_log_cluster_size =
3995 			F2FS_OPTION(sbi).compress_log_size;
3996 	F2FS_I(inode)->i_compress_flag =
3997 			F2FS_OPTION(sbi).compress_chksum ?
3998 				1 << COMPRESS_CHKSUM : 0;
3999 	F2FS_I(inode)->i_cluster_size =
4000 			1 << F2FS_I(inode)->i_log_cluster_size;
4001 	if (F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 &&
4002 			F2FS_OPTION(sbi).compress_level)
4003 		F2FS_I(inode)->i_compress_flag |=
4004 				F2FS_OPTION(sbi).compress_level <<
4005 				COMPRESS_LEVEL_OFFSET;
4006 	F2FS_I(inode)->i_flags |= F2FS_COMPR_FL;
4007 	set_inode_flag(inode, FI_COMPRESSED_FILE);
4008 	stat_inc_compr_inode(inode);
4009 	f2fs_mark_inode_dirty_sync(inode, true);
4010 }
4011 
4012 static inline bool f2fs_disable_compressed_file(struct inode *inode)
4013 {
4014 	struct f2fs_inode_info *fi = F2FS_I(inode);
4015 
4016 	if (!f2fs_compressed_file(inode))
4017 		return true;
4018 	if (S_ISREG(inode->i_mode) &&
4019 		(get_dirty_pages(inode) || atomic_read(&fi->i_compr_blocks)))
4020 		return false;
4021 
4022 	fi->i_flags &= ~F2FS_COMPR_FL;
4023 	stat_dec_compr_inode(inode);
4024 	clear_inode_flag(inode, FI_COMPRESSED_FILE);
4025 	f2fs_mark_inode_dirty_sync(inode, true);
4026 	return true;
4027 }
4028 
4029 #define F2FS_FEATURE_FUNCS(name, flagname) \
4030 static inline int f2fs_sb_has_##name(struct f2fs_sb_info *sbi) \
4031 { \
4032 	return F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_##flagname); \
4033 }
4034 
4035 F2FS_FEATURE_FUNCS(encrypt, ENCRYPT);
4036 F2FS_FEATURE_FUNCS(blkzoned, BLKZONED);
4037 F2FS_FEATURE_FUNCS(extra_attr, EXTRA_ATTR);
4038 F2FS_FEATURE_FUNCS(project_quota, PRJQUOTA);
4039 F2FS_FEATURE_FUNCS(inode_chksum, INODE_CHKSUM);
4040 F2FS_FEATURE_FUNCS(flexible_inline_xattr, FLEXIBLE_INLINE_XATTR);
4041 F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO);
4042 F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME);
4043 F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND);
4044 F2FS_FEATURE_FUNCS(verity, VERITY);
4045 F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
4046 F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
4047 F2FS_FEATURE_FUNCS(compression, COMPRESSION);
4048 
4049 #ifdef CONFIG_BLK_DEV_ZONED
4050 static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
4051 				    block_t blkaddr)
4052 {
4053 	unsigned int zno = blkaddr >> sbi->log_blocks_per_blkz;
4054 
4055 	return test_bit(zno, FDEV(devi).blkz_seq);
4056 }
4057 #endif
4058 
4059 static inline bool f2fs_hw_should_discard(struct f2fs_sb_info *sbi)
4060 {
4061 	return f2fs_sb_has_blkzoned(sbi);
4062 }
4063 
4064 static inline bool f2fs_bdev_support_discard(struct block_device *bdev)
4065 {
4066 	return blk_queue_discard(bdev_get_queue(bdev)) ||
4067 	       bdev_is_zoned(bdev);
4068 }
4069 
4070 static inline bool f2fs_hw_support_discard(struct f2fs_sb_info *sbi)
4071 {
4072 	int i;
4073 
4074 	if (!f2fs_is_multi_device(sbi))
4075 		return f2fs_bdev_support_discard(sbi->sb->s_bdev);
4076 
4077 	for (i = 0; i < sbi->s_ndevs; i++)
4078 		if (f2fs_bdev_support_discard(FDEV(i).bdev))
4079 			return true;
4080 	return false;
4081 }
4082 
4083 static inline bool f2fs_realtime_discard_enable(struct f2fs_sb_info *sbi)
4084 {
4085 	return (test_opt(sbi, DISCARD) && f2fs_hw_support_discard(sbi)) ||
4086 					f2fs_hw_should_discard(sbi);
4087 }
4088 
4089 static inline bool f2fs_hw_is_readonly(struct f2fs_sb_info *sbi)
4090 {
4091 	int i;
4092 
4093 	if (!f2fs_is_multi_device(sbi))
4094 		return bdev_read_only(sbi->sb->s_bdev);
4095 
4096 	for (i = 0; i < sbi->s_ndevs; i++)
4097 		if (bdev_read_only(FDEV(i).bdev))
4098 			return true;
4099 	return false;
4100 }
4101 
4102 static inline bool f2fs_lfs_mode(struct f2fs_sb_info *sbi)
4103 {
4104 	return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
4105 }
4106 
4107 static inline bool f2fs_may_compress(struct inode *inode)
4108 {
4109 	if (IS_SWAPFILE(inode) || f2fs_is_pinned_file(inode) ||
4110 				f2fs_is_atomic_file(inode) ||
4111 				f2fs_is_volatile_file(inode))
4112 		return false;
4113 	return S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode);
4114 }
4115 
4116 static inline void f2fs_i_compr_blocks_update(struct inode *inode,
4117 						u64 blocks, bool add)
4118 {
4119 	int diff = F2FS_I(inode)->i_cluster_size - blocks;
4120 	struct f2fs_inode_info *fi = F2FS_I(inode);
4121 
4122 	/* don't update i_compr_blocks if saved blocks were released */
4123 	if (!add && !atomic_read(&fi->i_compr_blocks))
4124 		return;
4125 
4126 	if (add) {
4127 		atomic_add(diff, &fi->i_compr_blocks);
4128 		stat_add_compr_blocks(inode, diff);
4129 	} else {
4130 		atomic_sub(diff, &fi->i_compr_blocks);
4131 		stat_sub_compr_blocks(inode, diff);
4132 	}
4133 	f2fs_mark_inode_dirty_sync(inode, true);
4134 }
4135 
4136 static inline int block_unaligned_IO(struct inode *inode,
4137 				struct kiocb *iocb, struct iov_iter *iter)
4138 {
4139 	unsigned int i_blkbits = READ_ONCE(inode->i_blkbits);
4140 	unsigned int blocksize_mask = (1 << i_blkbits) - 1;
4141 	loff_t offset = iocb->ki_pos;
4142 	unsigned long align = offset | iov_iter_alignment(iter);
4143 
4144 	return align & blocksize_mask;
4145 }
4146 
4147 static inline int allow_outplace_dio(struct inode *inode,
4148 				struct kiocb *iocb, struct iov_iter *iter)
4149 {
4150 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4151 	int rw = iov_iter_rw(iter);
4152 
4153 	return (f2fs_lfs_mode(sbi) && (rw == WRITE) &&
4154 				!block_unaligned_IO(inode, iocb, iter));
4155 }
4156 
4157 static inline bool f2fs_force_buffered_io(struct inode *inode,
4158 				struct kiocb *iocb, struct iov_iter *iter)
4159 {
4160 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4161 	int rw = iov_iter_rw(iter);
4162 
4163 	if (f2fs_post_read_required(inode))
4164 		return true;
4165 	if (f2fs_is_multi_device(sbi))
4166 		return true;
4167 	/*
4168 	 * for blkzoned device, fallback direct IO to buffered IO, so
4169 	 * all IOs can be serialized by log-structured write.
4170 	 */
4171 	if (f2fs_sb_has_blkzoned(sbi))
4172 		return true;
4173 	if (f2fs_lfs_mode(sbi) && (rw == WRITE)) {
4174 		if (block_unaligned_IO(inode, iocb, iter))
4175 			return true;
4176 		if (F2FS_IO_ALIGNED(sbi))
4177 			return true;
4178 	}
4179 	if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED) &&
4180 					!IS_SWAPFILE(inode))
4181 		return true;
4182 
4183 	return false;
4184 }
4185 
4186 static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
4187 {
4188 	return fsverity_active(inode) &&
4189 	       idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
4190 }
4191 
4192 #ifdef CONFIG_F2FS_FAULT_INJECTION
4193 extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate,
4194 							unsigned int type);
4195 #else
4196 #define f2fs_build_fault_attr(sbi, rate, type)		do { } while (0)
4197 #endif
4198 
4199 static inline bool is_journalled_quota(struct f2fs_sb_info *sbi)
4200 {
4201 #ifdef CONFIG_QUOTA
4202 	if (f2fs_sb_has_quota_ino(sbi))
4203 		return true;
4204 	if (F2FS_OPTION(sbi).s_qf_names[USRQUOTA] ||
4205 		F2FS_OPTION(sbi).s_qf_names[GRPQUOTA] ||
4206 		F2FS_OPTION(sbi).s_qf_names[PRJQUOTA])
4207 		return true;
4208 #endif
4209 	return false;
4210 }
4211 
4212 #define EFSBADCRC	EBADMSG		/* Bad CRC detected */
4213 #define EFSCORRUPTED	EUCLEAN		/* Filesystem is corrupted */
4214 
4215 #endif /* _LINUX_F2FS_H */
4216