xref: /openbmc/linux/fs/f2fs/segment.c (revision ae3473231e77a3f1909d48cd144cebe5e1d049b3)
1 /*
2  * fs/f2fs/segment.c
3  *
4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5  *             http://www.samsung.com/
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/bio.h>
14 #include <linux/blkdev.h>
15 #include <linux/prefetch.h>
16 #include <linux/kthread.h>
17 #include <linux/swap.h>
18 #include <linux/timer.h>
19 
20 #include "f2fs.h"
21 #include "segment.h"
22 #include "node.h"
23 #include "trace.h"
24 #include <trace/events/f2fs.h>
25 
26 #define __reverse_ffz(x) __reverse_ffs(~(x))
27 
28 static struct kmem_cache *discard_entry_slab;
29 static struct kmem_cache *bio_entry_slab;
30 static struct kmem_cache *sit_entry_set_slab;
31 static struct kmem_cache *inmem_entry_slab;
32 
33 static unsigned long __reverse_ulong(unsigned char *str)
34 {
35 	unsigned long tmp = 0;
36 	int shift = 24, idx = 0;
37 
38 #if BITS_PER_LONG == 64
39 	shift = 56;
40 #endif
41 	while (shift >= 0) {
42 		tmp |= (unsigned long)str[idx++] << shift;
43 		shift -= BITS_PER_BYTE;
44 	}
45 	return tmp;
46 }
47 
48 /*
49  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
50  * MSB and LSB are reversed in a byte by f2fs_set_bit.
51  */
52 static inline unsigned long __reverse_ffs(unsigned long word)
53 {
54 	int num = 0;
55 
56 #if BITS_PER_LONG == 64
57 	if ((word & 0xffffffff00000000UL) == 0)
58 		num += 32;
59 	else
60 		word >>= 32;
61 #endif
62 	if ((word & 0xffff0000) == 0)
63 		num += 16;
64 	else
65 		word >>= 16;
66 
67 	if ((word & 0xff00) == 0)
68 		num += 8;
69 	else
70 		word >>= 8;
71 
72 	if ((word & 0xf0) == 0)
73 		num += 4;
74 	else
75 		word >>= 4;
76 
77 	if ((word & 0xc) == 0)
78 		num += 2;
79 	else
80 		word >>= 2;
81 
82 	if ((word & 0x2) == 0)
83 		num += 1;
84 	return num;
85 }
86 
87 /*
88  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
89  * f2fs_set_bit makes MSB and LSB reversed in a byte.
90  * @size must be integral times of unsigned long.
91  * Example:
92  *                             MSB <--> LSB
93  *   f2fs_set_bit(0, bitmap) => 1000 0000
94  *   f2fs_set_bit(7, bitmap) => 0000 0001
95  */
96 static unsigned long __find_rev_next_bit(const unsigned long *addr,
97 			unsigned long size, unsigned long offset)
98 {
99 	const unsigned long *p = addr + BIT_WORD(offset);
100 	unsigned long result = size;
101 	unsigned long tmp;
102 
103 	if (offset >= size)
104 		return size;
105 
106 	size -= (offset & ~(BITS_PER_LONG - 1));
107 	offset %= BITS_PER_LONG;
108 
109 	while (1) {
110 		if (*p == 0)
111 			goto pass;
112 
113 		tmp = __reverse_ulong((unsigned char *)p);
114 
115 		tmp &= ~0UL >> offset;
116 		if (size < BITS_PER_LONG)
117 			tmp &= (~0UL << (BITS_PER_LONG - size));
118 		if (tmp)
119 			goto found;
120 pass:
121 		if (size <= BITS_PER_LONG)
122 			break;
123 		size -= BITS_PER_LONG;
124 		offset = 0;
125 		p++;
126 	}
127 	return result;
128 found:
129 	return result - size + __reverse_ffs(tmp);
130 }
131 
132 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
133 			unsigned long size, unsigned long offset)
134 {
135 	const unsigned long *p = addr + BIT_WORD(offset);
136 	unsigned long result = size;
137 	unsigned long tmp;
138 
139 	if (offset >= size)
140 		return size;
141 
142 	size -= (offset & ~(BITS_PER_LONG - 1));
143 	offset %= BITS_PER_LONG;
144 
145 	while (1) {
146 		if (*p == ~0UL)
147 			goto pass;
148 
149 		tmp = __reverse_ulong((unsigned char *)p);
150 
151 		if (offset)
152 			tmp |= ~0UL << (BITS_PER_LONG - offset);
153 		if (size < BITS_PER_LONG)
154 			tmp |= ~0UL >> size;
155 		if (tmp != ~0UL)
156 			goto found;
157 pass:
158 		if (size <= BITS_PER_LONG)
159 			break;
160 		size -= BITS_PER_LONG;
161 		offset = 0;
162 		p++;
163 	}
164 	return result;
165 found:
166 	return result - size + __reverse_ffz(tmp);
167 }
168 
169 void register_inmem_page(struct inode *inode, struct page *page)
170 {
171 	struct f2fs_inode_info *fi = F2FS_I(inode);
172 	struct inmem_pages *new;
173 
174 	f2fs_trace_pid(page);
175 
176 	set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
177 	SetPagePrivate(page);
178 
179 	new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
180 
181 	/* add atomic page indices to the list */
182 	new->page = page;
183 	INIT_LIST_HEAD(&new->list);
184 
185 	/* increase reference count with clean state */
186 	mutex_lock(&fi->inmem_lock);
187 	get_page(page);
188 	list_add_tail(&new->list, &fi->inmem_pages);
189 	inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
190 	mutex_unlock(&fi->inmem_lock);
191 
192 	trace_f2fs_register_inmem_page(page, INMEM);
193 }
194 
195 static int __revoke_inmem_pages(struct inode *inode,
196 				struct list_head *head, bool drop, bool recover)
197 {
198 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
199 	struct inmem_pages *cur, *tmp;
200 	int err = 0;
201 
202 	list_for_each_entry_safe(cur, tmp, head, list) {
203 		struct page *page = cur->page;
204 
205 		if (drop)
206 			trace_f2fs_commit_inmem_page(page, INMEM_DROP);
207 
208 		lock_page(page);
209 
210 		if (recover) {
211 			struct dnode_of_data dn;
212 			struct node_info ni;
213 
214 			trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
215 
216 			set_new_dnode(&dn, inode, NULL, NULL, 0);
217 			if (get_dnode_of_data(&dn, page->index, LOOKUP_NODE)) {
218 				err = -EAGAIN;
219 				goto next;
220 			}
221 			get_node_info(sbi, dn.nid, &ni);
222 			f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
223 					cur->old_addr, ni.version, true, true);
224 			f2fs_put_dnode(&dn);
225 		}
226 next:
227 		/* we don't need to invalidate this in the sccessful status */
228 		if (drop || recover)
229 			ClearPageUptodate(page);
230 		set_page_private(page, 0);
231 		ClearPagePrivate(page);
232 		f2fs_put_page(page, 1);
233 
234 		list_del(&cur->list);
235 		kmem_cache_free(inmem_entry_slab, cur);
236 		dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
237 	}
238 	return err;
239 }
240 
241 void drop_inmem_pages(struct inode *inode)
242 {
243 	struct f2fs_inode_info *fi = F2FS_I(inode);
244 
245 	clear_inode_flag(inode, FI_ATOMIC_FILE);
246 
247 	mutex_lock(&fi->inmem_lock);
248 	__revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
249 	mutex_unlock(&fi->inmem_lock);
250 }
251 
252 static int __commit_inmem_pages(struct inode *inode,
253 					struct list_head *revoke_list)
254 {
255 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
256 	struct f2fs_inode_info *fi = F2FS_I(inode);
257 	struct inmem_pages *cur, *tmp;
258 	struct f2fs_io_info fio = {
259 		.sbi = sbi,
260 		.type = DATA,
261 		.op = REQ_OP_WRITE,
262 		.op_flags = REQ_SYNC | REQ_PRIO,
263 		.encrypted_page = NULL,
264 	};
265 	bool submit_bio = false;
266 	int err = 0;
267 
268 	list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
269 		struct page *page = cur->page;
270 
271 		lock_page(page);
272 		if (page->mapping == inode->i_mapping) {
273 			trace_f2fs_commit_inmem_page(page, INMEM);
274 
275 			set_page_dirty(page);
276 			f2fs_wait_on_page_writeback(page, DATA, true);
277 			if (clear_page_dirty_for_io(page)) {
278 				inode_dec_dirty_pages(inode);
279 				remove_dirty_inode(inode);
280 			}
281 
282 			fio.page = page;
283 			err = do_write_data_page(&fio);
284 			if (err) {
285 				unlock_page(page);
286 				break;
287 			}
288 
289 			/* record old blkaddr for revoking */
290 			cur->old_addr = fio.old_blkaddr;
291 
292 			submit_bio = true;
293 		}
294 		unlock_page(page);
295 		list_move_tail(&cur->list, revoke_list);
296 	}
297 
298 	if (submit_bio)
299 		f2fs_submit_merged_bio_cond(sbi, inode, NULL, 0, DATA, WRITE);
300 
301 	if (!err)
302 		__revoke_inmem_pages(inode, revoke_list, false, false);
303 
304 	return err;
305 }
306 
307 int commit_inmem_pages(struct inode *inode)
308 {
309 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
310 	struct f2fs_inode_info *fi = F2FS_I(inode);
311 	struct list_head revoke_list;
312 	int err;
313 
314 	INIT_LIST_HEAD(&revoke_list);
315 	f2fs_balance_fs(sbi, true);
316 	f2fs_lock_op(sbi);
317 
318 	mutex_lock(&fi->inmem_lock);
319 	err = __commit_inmem_pages(inode, &revoke_list);
320 	if (err) {
321 		int ret;
322 		/*
323 		 * try to revoke all committed pages, but still we could fail
324 		 * due to no memory or other reason, if that happened, EAGAIN
325 		 * will be returned, which means in such case, transaction is
326 		 * already not integrity, caller should use journal to do the
327 		 * recovery or rewrite & commit last transaction. For other
328 		 * error number, revoking was done by filesystem itself.
329 		 */
330 		ret = __revoke_inmem_pages(inode, &revoke_list, false, true);
331 		if (ret)
332 			err = ret;
333 
334 		/* drop all uncommitted pages */
335 		__revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
336 	}
337 	mutex_unlock(&fi->inmem_lock);
338 
339 	f2fs_unlock_op(sbi);
340 	return err;
341 }
342 
343 /*
344  * This function balances dirty node and dentry pages.
345  * In addition, it controls garbage collection.
346  */
347 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
348 {
349 #ifdef CONFIG_F2FS_FAULT_INJECTION
350 	if (time_to_inject(sbi, FAULT_CHECKPOINT))
351 		f2fs_stop_checkpoint(sbi, false);
352 #endif
353 
354 	if (!need)
355 		return;
356 
357 	/* balance_fs_bg is able to be pending */
358 	if (excess_cached_nats(sbi))
359 		f2fs_balance_fs_bg(sbi);
360 
361 	/*
362 	 * We should do GC or end up with checkpoint, if there are so many dirty
363 	 * dir/node pages without enough free segments.
364 	 */
365 	if (has_not_enough_free_secs(sbi, 0, 0)) {
366 		mutex_lock(&sbi->gc_mutex);
367 		f2fs_gc(sbi, false, false);
368 	}
369 }
370 
371 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
372 {
373 	/* try to shrink extent cache when there is no enough memory */
374 	if (!available_free_memory(sbi, EXTENT_CACHE))
375 		f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
376 
377 	/* check the # of cached NAT entries */
378 	if (!available_free_memory(sbi, NAT_ENTRIES))
379 		try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
380 
381 	if (!available_free_memory(sbi, FREE_NIDS))
382 		try_to_free_nids(sbi, MAX_FREE_NIDS);
383 	else
384 		build_free_nids(sbi, false);
385 
386 	if (!is_idle(sbi))
387 		return;
388 
389 	/* checkpoint is the only way to shrink partial cached entries */
390 	if (!available_free_memory(sbi, NAT_ENTRIES) ||
391 			!available_free_memory(sbi, INO_ENTRIES) ||
392 			excess_prefree_segs(sbi) ||
393 			excess_dirty_nats(sbi) ||
394 			f2fs_time_over(sbi, CP_TIME)) {
395 		if (test_opt(sbi, DATA_FLUSH)) {
396 			struct blk_plug plug;
397 
398 			blk_start_plug(&plug);
399 			sync_dirty_inodes(sbi, FILE_INODE);
400 			blk_finish_plug(&plug);
401 		}
402 		f2fs_sync_fs(sbi->sb, true);
403 		stat_inc_bg_cp_count(sbi->stat_info);
404 	}
405 }
406 
407 static int __submit_flush_wait(struct block_device *bdev)
408 {
409 	struct bio *bio = f2fs_bio_alloc(0);
410 	int ret;
411 
412 	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
413 	bio->bi_bdev = bdev;
414 	ret = submit_bio_wait(bio);
415 	bio_put(bio);
416 	return ret;
417 }
418 
419 static int submit_flush_wait(struct f2fs_sb_info *sbi)
420 {
421 	int ret = __submit_flush_wait(sbi->sb->s_bdev);
422 	int i;
423 
424 	if (sbi->s_ndevs && !ret) {
425 		for (i = 1; i < sbi->s_ndevs; i++) {
426 			ret = __submit_flush_wait(FDEV(i).bdev);
427 			if (ret)
428 				break;
429 		}
430 	}
431 	return ret;
432 }
433 
434 static int issue_flush_thread(void *data)
435 {
436 	struct f2fs_sb_info *sbi = data;
437 	struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
438 	wait_queue_head_t *q = &fcc->flush_wait_queue;
439 repeat:
440 	if (kthread_should_stop())
441 		return 0;
442 
443 	if (!llist_empty(&fcc->issue_list)) {
444 		struct flush_cmd *cmd, *next;
445 		int ret;
446 
447 		fcc->dispatch_list = llist_del_all(&fcc->issue_list);
448 		fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
449 
450 		ret = submit_flush_wait(sbi);
451 		llist_for_each_entry_safe(cmd, next,
452 					  fcc->dispatch_list, llnode) {
453 			cmd->ret = ret;
454 			complete(&cmd->wait);
455 		}
456 		fcc->dispatch_list = NULL;
457 	}
458 
459 	wait_event_interruptible(*q,
460 		kthread_should_stop() || !llist_empty(&fcc->issue_list));
461 	goto repeat;
462 }
463 
464 int f2fs_issue_flush(struct f2fs_sb_info *sbi)
465 {
466 	struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
467 	struct flush_cmd cmd;
468 
469 	trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
470 					test_opt(sbi, FLUSH_MERGE));
471 
472 	if (test_opt(sbi, NOBARRIER))
473 		return 0;
474 
475 	if (!test_opt(sbi, FLUSH_MERGE) || !atomic_read(&fcc->submit_flush)) {
476 		int ret;
477 
478 		atomic_inc(&fcc->submit_flush);
479 		ret = submit_flush_wait(sbi);
480 		atomic_dec(&fcc->submit_flush);
481 		return ret;
482 	}
483 
484 	init_completion(&cmd.wait);
485 
486 	atomic_inc(&fcc->submit_flush);
487 	llist_add(&cmd.llnode, &fcc->issue_list);
488 
489 	if (!fcc->dispatch_list)
490 		wake_up(&fcc->flush_wait_queue);
491 
492 	if (fcc->f2fs_issue_flush) {
493 		wait_for_completion(&cmd.wait);
494 		atomic_dec(&fcc->submit_flush);
495 	} else {
496 		llist_del_all(&fcc->issue_list);
497 		atomic_set(&fcc->submit_flush, 0);
498 	}
499 
500 	return cmd.ret;
501 }
502 
503 int create_flush_cmd_control(struct f2fs_sb_info *sbi)
504 {
505 	dev_t dev = sbi->sb->s_bdev->bd_dev;
506 	struct flush_cmd_control *fcc;
507 	int err = 0;
508 
509 	if (SM_I(sbi)->cmd_control_info) {
510 		fcc = SM_I(sbi)->cmd_control_info;
511 		goto init_thread;
512 	}
513 
514 	fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
515 	if (!fcc)
516 		return -ENOMEM;
517 	atomic_set(&fcc->submit_flush, 0);
518 	init_waitqueue_head(&fcc->flush_wait_queue);
519 	init_llist_head(&fcc->issue_list);
520 	SM_I(sbi)->cmd_control_info = fcc;
521 init_thread:
522 	fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
523 				"f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
524 	if (IS_ERR(fcc->f2fs_issue_flush)) {
525 		err = PTR_ERR(fcc->f2fs_issue_flush);
526 		kfree(fcc);
527 		SM_I(sbi)->cmd_control_info = NULL;
528 		return err;
529 	}
530 
531 	return err;
532 }
533 
534 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
535 {
536 	struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
537 
538 	if (fcc && fcc->f2fs_issue_flush) {
539 		struct task_struct *flush_thread = fcc->f2fs_issue_flush;
540 
541 		fcc->f2fs_issue_flush = NULL;
542 		kthread_stop(flush_thread);
543 	}
544 	if (free) {
545 		kfree(fcc);
546 		SM_I(sbi)->cmd_control_info = NULL;
547 	}
548 }
549 
550 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
551 		enum dirty_type dirty_type)
552 {
553 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
554 
555 	/* need not be added */
556 	if (IS_CURSEG(sbi, segno))
557 		return;
558 
559 	if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
560 		dirty_i->nr_dirty[dirty_type]++;
561 
562 	if (dirty_type == DIRTY) {
563 		struct seg_entry *sentry = get_seg_entry(sbi, segno);
564 		enum dirty_type t = sentry->type;
565 
566 		if (unlikely(t >= DIRTY)) {
567 			f2fs_bug_on(sbi, 1);
568 			return;
569 		}
570 		if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
571 			dirty_i->nr_dirty[t]++;
572 	}
573 }
574 
575 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
576 		enum dirty_type dirty_type)
577 {
578 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
579 
580 	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
581 		dirty_i->nr_dirty[dirty_type]--;
582 
583 	if (dirty_type == DIRTY) {
584 		struct seg_entry *sentry = get_seg_entry(sbi, segno);
585 		enum dirty_type t = sentry->type;
586 
587 		if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
588 			dirty_i->nr_dirty[t]--;
589 
590 		if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
591 			clear_bit(GET_SECNO(sbi, segno),
592 						dirty_i->victim_secmap);
593 	}
594 }
595 
596 /*
597  * Should not occur error such as -ENOMEM.
598  * Adding dirty entry into seglist is not critical operation.
599  * If a given segment is one of current working segments, it won't be added.
600  */
601 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
602 {
603 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
604 	unsigned short valid_blocks;
605 
606 	if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
607 		return;
608 
609 	mutex_lock(&dirty_i->seglist_lock);
610 
611 	valid_blocks = get_valid_blocks(sbi, segno, 0);
612 
613 	if (valid_blocks == 0) {
614 		__locate_dirty_segment(sbi, segno, PRE);
615 		__remove_dirty_segment(sbi, segno, DIRTY);
616 	} else if (valid_blocks < sbi->blocks_per_seg) {
617 		__locate_dirty_segment(sbi, segno, DIRTY);
618 	} else {
619 		/* Recovery routine with SSR needs this */
620 		__remove_dirty_segment(sbi, segno, DIRTY);
621 	}
622 
623 	mutex_unlock(&dirty_i->seglist_lock);
624 }
625 
626 static struct bio_entry *__add_bio_entry(struct f2fs_sb_info *sbi,
627 							struct bio *bio)
628 {
629 	struct list_head *wait_list = &(SM_I(sbi)->wait_list);
630 	struct bio_entry *be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS);
631 
632 	INIT_LIST_HEAD(&be->list);
633 	be->bio = bio;
634 	init_completion(&be->event);
635 	list_add_tail(&be->list, wait_list);
636 
637 	return be;
638 }
639 
640 void f2fs_wait_all_discard_bio(struct f2fs_sb_info *sbi)
641 {
642 	struct list_head *wait_list = &(SM_I(sbi)->wait_list);
643 	struct bio_entry *be, *tmp;
644 
645 	list_for_each_entry_safe(be, tmp, wait_list, list) {
646 		struct bio *bio = be->bio;
647 		int err;
648 
649 		wait_for_completion_io(&be->event);
650 		err = be->error;
651 		if (err == -EOPNOTSUPP)
652 			err = 0;
653 
654 		if (err)
655 			f2fs_msg(sbi->sb, KERN_INFO,
656 				"Issue discard failed, ret: %d", err);
657 
658 		bio_put(bio);
659 		list_del(&be->list);
660 		kmem_cache_free(bio_entry_slab, be);
661 	}
662 }
663 
664 static void f2fs_submit_bio_wait_endio(struct bio *bio)
665 {
666 	struct bio_entry *be = (struct bio_entry *)bio->bi_private;
667 
668 	be->error = bio->bi_error;
669 	complete(&be->event);
670 }
671 
672 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
673 static int __f2fs_issue_discard_async(struct f2fs_sb_info *sbi,
674 		struct block_device *bdev, block_t blkstart, block_t blklen)
675 {
676 	struct bio *bio = NULL;
677 	int err;
678 
679 	trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
680 
681 	if (sbi->s_ndevs) {
682 		int devi = f2fs_target_device_index(sbi, blkstart);
683 
684 		blkstart -= FDEV(devi).start_blk;
685 	}
686 	err = __blkdev_issue_discard(bdev,
687 				SECTOR_FROM_BLOCK(blkstart),
688 				SECTOR_FROM_BLOCK(blklen),
689 				GFP_NOFS, 0, &bio);
690 	if (!err && bio) {
691 		struct bio_entry *be = __add_bio_entry(sbi, bio);
692 
693 		bio->bi_private = be;
694 		bio->bi_end_io = f2fs_submit_bio_wait_endio;
695 		bio->bi_opf |= REQ_SYNC;
696 		submit_bio(bio);
697 	}
698 
699 	return err;
700 }
701 
702 #ifdef CONFIG_BLK_DEV_ZONED
703 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
704 		struct block_device *bdev, block_t blkstart, block_t blklen)
705 {
706 	sector_t nr_sects = SECTOR_FROM_BLOCK(blklen);
707 	sector_t sector;
708 	int devi = 0;
709 
710 	if (sbi->s_ndevs) {
711 		devi = f2fs_target_device_index(sbi, blkstart);
712 		blkstart -= FDEV(devi).start_blk;
713 	}
714 	sector = SECTOR_FROM_BLOCK(blkstart);
715 
716 	if (sector & (bdev_zone_size(bdev) - 1) ||
717 				nr_sects != bdev_zone_size(bdev)) {
718 		f2fs_msg(sbi->sb, KERN_INFO,
719 			"(%d) %s: Unaligned discard attempted (block %x + %x)",
720 			devi, sbi->s_ndevs ? FDEV(devi).path: "",
721 			blkstart, blklen);
722 		return -EIO;
723 	}
724 
725 	/*
726 	 * We need to know the type of the zone: for conventional zones,
727 	 * use regular discard if the drive supports it. For sequential
728 	 * zones, reset the zone write pointer.
729 	 */
730 	switch (get_blkz_type(sbi, bdev, blkstart)) {
731 
732 	case BLK_ZONE_TYPE_CONVENTIONAL:
733 		if (!blk_queue_discard(bdev_get_queue(bdev)))
734 			return 0;
735 		return __f2fs_issue_discard_async(sbi, bdev, blkstart, blklen);
736 	case BLK_ZONE_TYPE_SEQWRITE_REQ:
737 	case BLK_ZONE_TYPE_SEQWRITE_PREF:
738 		trace_f2fs_issue_reset_zone(sbi->sb, blkstart);
739 		return blkdev_reset_zones(bdev, sector,
740 					  nr_sects, GFP_NOFS);
741 	default:
742 		/* Unknown zone type: broken device ? */
743 		return -EIO;
744 	}
745 }
746 #endif
747 
748 static int __issue_discard_async(struct f2fs_sb_info *sbi,
749 		struct block_device *bdev, block_t blkstart, block_t blklen)
750 {
751 #ifdef CONFIG_BLK_DEV_ZONED
752 	if (f2fs_sb_mounted_blkzoned(sbi->sb) &&
753 				bdev_zoned_model(bdev) != BLK_ZONED_NONE)
754 		return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
755 #endif
756 	return __f2fs_issue_discard_async(sbi, bdev, blkstart, blklen);
757 }
758 
759 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
760 				block_t blkstart, block_t blklen)
761 {
762 	sector_t start = blkstart, len = 0;
763 	struct block_device *bdev;
764 	struct seg_entry *se;
765 	unsigned int offset;
766 	block_t i;
767 	int err = 0;
768 
769 	bdev = f2fs_target_device(sbi, blkstart, NULL);
770 
771 	for (i = blkstart; i < blkstart + blklen; i++, len++) {
772 		if (i != start) {
773 			struct block_device *bdev2 =
774 				f2fs_target_device(sbi, i, NULL);
775 
776 			if (bdev2 != bdev) {
777 				err = __issue_discard_async(sbi, bdev,
778 						start, len);
779 				if (err)
780 					return err;
781 				bdev = bdev2;
782 				start = i;
783 				len = 0;
784 			}
785 		}
786 
787 		se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
788 		offset = GET_BLKOFF_FROM_SEG0(sbi, i);
789 
790 		if (!f2fs_test_and_set_bit(offset, se->discard_map))
791 			sbi->discard_blks--;
792 	}
793 
794 	if (len)
795 		err = __issue_discard_async(sbi, bdev, start, len);
796 	return err;
797 }
798 
799 static void __add_discard_entry(struct f2fs_sb_info *sbi,
800 		struct cp_control *cpc, struct seg_entry *se,
801 		unsigned int start, unsigned int end)
802 {
803 	struct list_head *head = &SM_I(sbi)->discard_list;
804 	struct discard_entry *new, *last;
805 
806 	if (!list_empty(head)) {
807 		last = list_last_entry(head, struct discard_entry, list);
808 		if (START_BLOCK(sbi, cpc->trim_start) + start ==
809 						last->blkaddr + last->len) {
810 			last->len += end - start;
811 			goto done;
812 		}
813 	}
814 
815 	new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
816 	INIT_LIST_HEAD(&new->list);
817 	new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
818 	new->len = end - start;
819 	list_add_tail(&new->list, head);
820 done:
821 	SM_I(sbi)->nr_discards += end - start;
822 }
823 
824 static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
825 {
826 	int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
827 	int max_blocks = sbi->blocks_per_seg;
828 	struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
829 	unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
830 	unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
831 	unsigned long *discard_map = (unsigned long *)se->discard_map;
832 	unsigned long *dmap = SIT_I(sbi)->tmp_map;
833 	unsigned int start = 0, end = -1;
834 	bool force = (cpc->reason == CP_DISCARD);
835 	int i;
836 
837 	if (se->valid_blocks == max_blocks || !f2fs_discard_en(sbi))
838 		return;
839 
840 	if (!force) {
841 		if (!test_opt(sbi, DISCARD) || !se->valid_blocks ||
842 		    SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards)
843 			return;
844 	}
845 
846 	/* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
847 	for (i = 0; i < entries; i++)
848 		dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
849 				(cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
850 
851 	while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
852 		start = __find_rev_next_bit(dmap, max_blocks, end + 1);
853 		if (start >= max_blocks)
854 			break;
855 
856 		end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
857 		if (force && start && end != max_blocks
858 					&& (end - start) < cpc->trim_minlen)
859 			continue;
860 
861 		__add_discard_entry(sbi, cpc, se, start, end);
862 	}
863 }
864 
865 void release_discard_addrs(struct f2fs_sb_info *sbi)
866 {
867 	struct list_head *head = &(SM_I(sbi)->discard_list);
868 	struct discard_entry *entry, *this;
869 
870 	/* drop caches */
871 	list_for_each_entry_safe(entry, this, head, list) {
872 		list_del(&entry->list);
873 		kmem_cache_free(discard_entry_slab, entry);
874 	}
875 }
876 
877 /*
878  * Should call clear_prefree_segments after checkpoint is done.
879  */
880 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
881 {
882 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
883 	unsigned int segno;
884 
885 	mutex_lock(&dirty_i->seglist_lock);
886 	for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
887 		__set_test_and_free(sbi, segno);
888 	mutex_unlock(&dirty_i->seglist_lock);
889 }
890 
891 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
892 {
893 	struct list_head *head = &(SM_I(sbi)->discard_list);
894 	struct discard_entry *entry, *this;
895 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
896 	struct blk_plug plug;
897 	unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
898 	unsigned int start = 0, end = -1;
899 	unsigned int secno, start_segno;
900 	bool force = (cpc->reason == CP_DISCARD);
901 
902 	blk_start_plug(&plug);
903 
904 	mutex_lock(&dirty_i->seglist_lock);
905 
906 	while (1) {
907 		int i;
908 		start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
909 		if (start >= MAIN_SEGS(sbi))
910 			break;
911 		end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
912 								start + 1);
913 
914 		for (i = start; i < end; i++)
915 			clear_bit(i, prefree_map);
916 
917 		dirty_i->nr_dirty[PRE] -= end - start;
918 
919 		if (force || !test_opt(sbi, DISCARD))
920 			continue;
921 
922 		if (!test_opt(sbi, LFS) || sbi->segs_per_sec == 1) {
923 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
924 				(end - start) << sbi->log_blocks_per_seg);
925 			continue;
926 		}
927 next:
928 		secno = GET_SECNO(sbi, start);
929 		start_segno = secno * sbi->segs_per_sec;
930 		if (!IS_CURSEC(sbi, secno) &&
931 			!get_valid_blocks(sbi, start, sbi->segs_per_sec))
932 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
933 				sbi->segs_per_sec << sbi->log_blocks_per_seg);
934 
935 		start = start_segno + sbi->segs_per_sec;
936 		if (start < end)
937 			goto next;
938 	}
939 	mutex_unlock(&dirty_i->seglist_lock);
940 
941 	/* send small discards */
942 	list_for_each_entry_safe(entry, this, head, list) {
943 		if (force && entry->len < cpc->trim_minlen)
944 			goto skip;
945 		f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
946 		cpc->trimmed += entry->len;
947 skip:
948 		list_del(&entry->list);
949 		SM_I(sbi)->nr_discards -= entry->len;
950 		kmem_cache_free(discard_entry_slab, entry);
951 	}
952 
953 	blk_finish_plug(&plug);
954 }
955 
956 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
957 {
958 	struct sit_info *sit_i = SIT_I(sbi);
959 
960 	if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
961 		sit_i->dirty_sentries++;
962 		return false;
963 	}
964 
965 	return true;
966 }
967 
968 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
969 					unsigned int segno, int modified)
970 {
971 	struct seg_entry *se = get_seg_entry(sbi, segno);
972 	se->type = type;
973 	if (modified)
974 		__mark_sit_entry_dirty(sbi, segno);
975 }
976 
977 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
978 {
979 	struct seg_entry *se;
980 	unsigned int segno, offset;
981 	long int new_vblocks;
982 
983 	segno = GET_SEGNO(sbi, blkaddr);
984 
985 	se = get_seg_entry(sbi, segno);
986 	new_vblocks = se->valid_blocks + del;
987 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
988 
989 	f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
990 				(new_vblocks > sbi->blocks_per_seg)));
991 
992 	se->valid_blocks = new_vblocks;
993 	se->mtime = get_mtime(sbi);
994 	SIT_I(sbi)->max_mtime = se->mtime;
995 
996 	/* Update valid block bitmap */
997 	if (del > 0) {
998 		if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
999 			f2fs_bug_on(sbi, 1);
1000 		if (f2fs_discard_en(sbi) &&
1001 			!f2fs_test_and_set_bit(offset, se->discard_map))
1002 			sbi->discard_blks--;
1003 	} else {
1004 		if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
1005 			f2fs_bug_on(sbi, 1);
1006 		if (f2fs_discard_en(sbi) &&
1007 			f2fs_test_and_clear_bit(offset, se->discard_map))
1008 			sbi->discard_blks++;
1009 	}
1010 	if (!f2fs_test_bit(offset, se->ckpt_valid_map))
1011 		se->ckpt_valid_blocks += del;
1012 
1013 	__mark_sit_entry_dirty(sbi, segno);
1014 
1015 	/* update total number of valid blocks to be written in ckpt area */
1016 	SIT_I(sbi)->written_valid_blocks += del;
1017 
1018 	if (sbi->segs_per_sec > 1)
1019 		get_sec_entry(sbi, segno)->valid_blocks += del;
1020 }
1021 
1022 void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
1023 {
1024 	update_sit_entry(sbi, new, 1);
1025 	if (GET_SEGNO(sbi, old) != NULL_SEGNO)
1026 		update_sit_entry(sbi, old, -1);
1027 
1028 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
1029 	locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
1030 }
1031 
1032 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
1033 {
1034 	unsigned int segno = GET_SEGNO(sbi, addr);
1035 	struct sit_info *sit_i = SIT_I(sbi);
1036 
1037 	f2fs_bug_on(sbi, addr == NULL_ADDR);
1038 	if (addr == NEW_ADDR)
1039 		return;
1040 
1041 	/* add it into sit main buffer */
1042 	mutex_lock(&sit_i->sentry_lock);
1043 
1044 	update_sit_entry(sbi, addr, -1);
1045 
1046 	/* add it into dirty seglist */
1047 	locate_dirty_segment(sbi, segno);
1048 
1049 	mutex_unlock(&sit_i->sentry_lock);
1050 }
1051 
1052 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
1053 {
1054 	struct sit_info *sit_i = SIT_I(sbi);
1055 	unsigned int segno, offset;
1056 	struct seg_entry *se;
1057 	bool is_cp = false;
1058 
1059 	if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
1060 		return true;
1061 
1062 	mutex_lock(&sit_i->sentry_lock);
1063 
1064 	segno = GET_SEGNO(sbi, blkaddr);
1065 	se = get_seg_entry(sbi, segno);
1066 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
1067 
1068 	if (f2fs_test_bit(offset, se->ckpt_valid_map))
1069 		is_cp = true;
1070 
1071 	mutex_unlock(&sit_i->sentry_lock);
1072 
1073 	return is_cp;
1074 }
1075 
1076 /*
1077  * This function should be resided under the curseg_mutex lock
1078  */
1079 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
1080 					struct f2fs_summary *sum)
1081 {
1082 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1083 	void *addr = curseg->sum_blk;
1084 	addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
1085 	memcpy(addr, sum, sizeof(struct f2fs_summary));
1086 }
1087 
1088 /*
1089  * Calculate the number of current summary pages for writing
1090  */
1091 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
1092 {
1093 	int valid_sum_count = 0;
1094 	int i, sum_in_page;
1095 
1096 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1097 		if (sbi->ckpt->alloc_type[i] == SSR)
1098 			valid_sum_count += sbi->blocks_per_seg;
1099 		else {
1100 			if (for_ra)
1101 				valid_sum_count += le16_to_cpu(
1102 					F2FS_CKPT(sbi)->cur_data_blkoff[i]);
1103 			else
1104 				valid_sum_count += curseg_blkoff(sbi, i);
1105 		}
1106 	}
1107 
1108 	sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
1109 			SUM_FOOTER_SIZE) / SUMMARY_SIZE;
1110 	if (valid_sum_count <= sum_in_page)
1111 		return 1;
1112 	else if ((valid_sum_count - sum_in_page) <=
1113 		(PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
1114 		return 2;
1115 	return 3;
1116 }
1117 
1118 /*
1119  * Caller should put this summary page
1120  */
1121 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
1122 {
1123 	return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
1124 }
1125 
1126 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr)
1127 {
1128 	struct page *page = grab_meta_page(sbi, blk_addr);
1129 	void *dst = page_address(page);
1130 
1131 	if (src)
1132 		memcpy(dst, src, PAGE_SIZE);
1133 	else
1134 		memset(dst, 0, PAGE_SIZE);
1135 	set_page_dirty(page);
1136 	f2fs_put_page(page, 1);
1137 }
1138 
1139 static void write_sum_page(struct f2fs_sb_info *sbi,
1140 			struct f2fs_summary_block *sum_blk, block_t blk_addr)
1141 {
1142 	update_meta_page(sbi, (void *)sum_blk, blk_addr);
1143 }
1144 
1145 static void write_current_sum_page(struct f2fs_sb_info *sbi,
1146 						int type, block_t blk_addr)
1147 {
1148 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1149 	struct page *page = grab_meta_page(sbi, blk_addr);
1150 	struct f2fs_summary_block *src = curseg->sum_blk;
1151 	struct f2fs_summary_block *dst;
1152 
1153 	dst = (struct f2fs_summary_block *)page_address(page);
1154 
1155 	mutex_lock(&curseg->curseg_mutex);
1156 
1157 	down_read(&curseg->journal_rwsem);
1158 	memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
1159 	up_read(&curseg->journal_rwsem);
1160 
1161 	memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
1162 	memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
1163 
1164 	mutex_unlock(&curseg->curseg_mutex);
1165 
1166 	set_page_dirty(page);
1167 	f2fs_put_page(page, 1);
1168 }
1169 
1170 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
1171 {
1172 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1173 	unsigned int segno = curseg->segno + 1;
1174 	struct free_segmap_info *free_i = FREE_I(sbi);
1175 
1176 	if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
1177 		return !test_bit(segno, free_i->free_segmap);
1178 	return 0;
1179 }
1180 
1181 /*
1182  * Find a new segment from the free segments bitmap to right order
1183  * This function should be returned with success, otherwise BUG
1184  */
1185 static void get_new_segment(struct f2fs_sb_info *sbi,
1186 			unsigned int *newseg, bool new_sec, int dir)
1187 {
1188 	struct free_segmap_info *free_i = FREE_I(sbi);
1189 	unsigned int segno, secno, zoneno;
1190 	unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
1191 	unsigned int hint = *newseg / sbi->segs_per_sec;
1192 	unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
1193 	unsigned int left_start = hint;
1194 	bool init = true;
1195 	int go_left = 0;
1196 	int i;
1197 
1198 	spin_lock(&free_i->segmap_lock);
1199 
1200 	if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
1201 		segno = find_next_zero_bit(free_i->free_segmap,
1202 				(hint + 1) * sbi->segs_per_sec, *newseg + 1);
1203 		if (segno < (hint + 1) * sbi->segs_per_sec)
1204 			goto got_it;
1205 	}
1206 find_other_zone:
1207 	secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
1208 	if (secno >= MAIN_SECS(sbi)) {
1209 		if (dir == ALLOC_RIGHT) {
1210 			secno = find_next_zero_bit(free_i->free_secmap,
1211 							MAIN_SECS(sbi), 0);
1212 			f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
1213 		} else {
1214 			go_left = 1;
1215 			left_start = hint - 1;
1216 		}
1217 	}
1218 	if (go_left == 0)
1219 		goto skip_left;
1220 
1221 	while (test_bit(left_start, free_i->free_secmap)) {
1222 		if (left_start > 0) {
1223 			left_start--;
1224 			continue;
1225 		}
1226 		left_start = find_next_zero_bit(free_i->free_secmap,
1227 							MAIN_SECS(sbi), 0);
1228 		f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
1229 		break;
1230 	}
1231 	secno = left_start;
1232 skip_left:
1233 	hint = secno;
1234 	segno = secno * sbi->segs_per_sec;
1235 	zoneno = secno / sbi->secs_per_zone;
1236 
1237 	/* give up on finding another zone */
1238 	if (!init)
1239 		goto got_it;
1240 	if (sbi->secs_per_zone == 1)
1241 		goto got_it;
1242 	if (zoneno == old_zoneno)
1243 		goto got_it;
1244 	if (dir == ALLOC_LEFT) {
1245 		if (!go_left && zoneno + 1 >= total_zones)
1246 			goto got_it;
1247 		if (go_left && zoneno == 0)
1248 			goto got_it;
1249 	}
1250 	for (i = 0; i < NR_CURSEG_TYPE; i++)
1251 		if (CURSEG_I(sbi, i)->zone == zoneno)
1252 			break;
1253 
1254 	if (i < NR_CURSEG_TYPE) {
1255 		/* zone is in user, try another */
1256 		if (go_left)
1257 			hint = zoneno * sbi->secs_per_zone - 1;
1258 		else if (zoneno + 1 >= total_zones)
1259 			hint = 0;
1260 		else
1261 			hint = (zoneno + 1) * sbi->secs_per_zone;
1262 		init = false;
1263 		goto find_other_zone;
1264 	}
1265 got_it:
1266 	/* set it as dirty segment in free segmap */
1267 	f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
1268 	__set_inuse(sbi, segno);
1269 	*newseg = segno;
1270 	spin_unlock(&free_i->segmap_lock);
1271 }
1272 
1273 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
1274 {
1275 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1276 	struct summary_footer *sum_footer;
1277 
1278 	curseg->segno = curseg->next_segno;
1279 	curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
1280 	curseg->next_blkoff = 0;
1281 	curseg->next_segno = NULL_SEGNO;
1282 
1283 	sum_footer = &(curseg->sum_blk->footer);
1284 	memset(sum_footer, 0, sizeof(struct summary_footer));
1285 	if (IS_DATASEG(type))
1286 		SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
1287 	if (IS_NODESEG(type))
1288 		SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
1289 	__set_sit_entry_type(sbi, type, curseg->segno, modified);
1290 }
1291 
1292 /*
1293  * Allocate a current working segment.
1294  * This function always allocates a free segment in LFS manner.
1295  */
1296 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
1297 {
1298 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1299 	unsigned int segno = curseg->segno;
1300 	int dir = ALLOC_LEFT;
1301 
1302 	write_sum_page(sbi, curseg->sum_blk,
1303 				GET_SUM_BLOCK(sbi, segno));
1304 	if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
1305 		dir = ALLOC_RIGHT;
1306 
1307 	if (test_opt(sbi, NOHEAP))
1308 		dir = ALLOC_RIGHT;
1309 
1310 	get_new_segment(sbi, &segno, new_sec, dir);
1311 	curseg->next_segno = segno;
1312 	reset_curseg(sbi, type, 1);
1313 	curseg->alloc_type = LFS;
1314 }
1315 
1316 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
1317 			struct curseg_info *seg, block_t start)
1318 {
1319 	struct seg_entry *se = get_seg_entry(sbi, seg->segno);
1320 	int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1321 	unsigned long *target_map = SIT_I(sbi)->tmp_map;
1322 	unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1323 	unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1324 	int i, pos;
1325 
1326 	for (i = 0; i < entries; i++)
1327 		target_map[i] = ckpt_map[i] | cur_map[i];
1328 
1329 	pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
1330 
1331 	seg->next_blkoff = pos;
1332 }
1333 
1334 /*
1335  * If a segment is written by LFS manner, next block offset is just obtained
1336  * by increasing the current block offset. However, if a segment is written by
1337  * SSR manner, next block offset obtained by calling __next_free_blkoff
1338  */
1339 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
1340 				struct curseg_info *seg)
1341 {
1342 	if (seg->alloc_type == SSR)
1343 		__next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
1344 	else
1345 		seg->next_blkoff++;
1346 }
1347 
1348 /*
1349  * This function always allocates a used segment(from dirty seglist) by SSR
1350  * manner, so it should recover the existing segment information of valid blocks
1351  */
1352 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
1353 {
1354 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1355 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1356 	unsigned int new_segno = curseg->next_segno;
1357 	struct f2fs_summary_block *sum_node;
1358 	struct page *sum_page;
1359 
1360 	write_sum_page(sbi, curseg->sum_blk,
1361 				GET_SUM_BLOCK(sbi, curseg->segno));
1362 	__set_test_and_inuse(sbi, new_segno);
1363 
1364 	mutex_lock(&dirty_i->seglist_lock);
1365 	__remove_dirty_segment(sbi, new_segno, PRE);
1366 	__remove_dirty_segment(sbi, new_segno, DIRTY);
1367 	mutex_unlock(&dirty_i->seglist_lock);
1368 
1369 	reset_curseg(sbi, type, 1);
1370 	curseg->alloc_type = SSR;
1371 	__next_free_blkoff(sbi, curseg, 0);
1372 
1373 	if (reuse) {
1374 		sum_page = get_sum_page(sbi, new_segno);
1375 		sum_node = (struct f2fs_summary_block *)page_address(sum_page);
1376 		memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
1377 		f2fs_put_page(sum_page, 1);
1378 	}
1379 }
1380 
1381 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
1382 {
1383 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1384 	const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
1385 
1386 	if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0, 0))
1387 		return v_ops->get_victim(sbi,
1388 				&(curseg)->next_segno, BG_GC, type, SSR);
1389 
1390 	/* For data segments, let's do SSR more intensively */
1391 	for (; type >= CURSEG_HOT_DATA; type--)
1392 		if (v_ops->get_victim(sbi, &(curseg)->next_segno,
1393 						BG_GC, type, SSR))
1394 			return 1;
1395 	return 0;
1396 }
1397 
1398 /*
1399  * flush out current segment and replace it with new segment
1400  * This function should be returned with success, otherwise BUG
1401  */
1402 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
1403 						int type, bool force)
1404 {
1405 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1406 
1407 	if (force)
1408 		new_curseg(sbi, type, true);
1409 	else if (type == CURSEG_WARM_NODE)
1410 		new_curseg(sbi, type, false);
1411 	else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
1412 		new_curseg(sbi, type, false);
1413 	else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
1414 		change_curseg(sbi, type, true);
1415 	else
1416 		new_curseg(sbi, type, false);
1417 
1418 	stat_inc_seg_type(sbi, curseg);
1419 }
1420 
1421 void allocate_new_segments(struct f2fs_sb_info *sbi)
1422 {
1423 	struct curseg_info *curseg;
1424 	unsigned int old_segno;
1425 	int i;
1426 
1427 	if (test_opt(sbi, LFS))
1428 		return;
1429 
1430 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1431 		curseg = CURSEG_I(sbi, i);
1432 		old_segno = curseg->segno;
1433 		SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
1434 		locate_dirty_segment(sbi, old_segno);
1435 	}
1436 }
1437 
1438 static const struct segment_allocation default_salloc_ops = {
1439 	.allocate_segment = allocate_segment_by_default,
1440 };
1441 
1442 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1443 {
1444 	__u64 start = F2FS_BYTES_TO_BLK(range->start);
1445 	__u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
1446 	unsigned int start_segno, end_segno;
1447 	struct cp_control cpc;
1448 	int err = 0;
1449 
1450 	if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
1451 		return -EINVAL;
1452 
1453 	cpc.trimmed = 0;
1454 	if (end <= MAIN_BLKADDR(sbi))
1455 		goto out;
1456 
1457 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1458 		f2fs_msg(sbi->sb, KERN_WARNING,
1459 			"Found FS corruption, run fsck to fix.");
1460 		goto out;
1461 	}
1462 
1463 	/* start/end segment number in main_area */
1464 	start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
1465 	end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
1466 						GET_SEGNO(sbi, end);
1467 	cpc.reason = CP_DISCARD;
1468 	cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
1469 
1470 	/* do checkpoint to issue discard commands safely */
1471 	for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) {
1472 		cpc.trim_start = start_segno;
1473 
1474 		if (sbi->discard_blks == 0)
1475 			break;
1476 		else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi))
1477 			cpc.trim_end = end_segno;
1478 		else
1479 			cpc.trim_end = min_t(unsigned int,
1480 				rounddown(start_segno +
1481 				BATCHED_TRIM_SEGMENTS(sbi),
1482 				sbi->segs_per_sec) - 1, end_segno);
1483 
1484 		mutex_lock(&sbi->gc_mutex);
1485 		err = write_checkpoint(sbi, &cpc);
1486 		mutex_unlock(&sbi->gc_mutex);
1487 		if (err)
1488 			break;
1489 
1490 		schedule();
1491 	}
1492 out:
1493 	range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);
1494 	return err;
1495 }
1496 
1497 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
1498 {
1499 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1500 	if (curseg->next_blkoff < sbi->blocks_per_seg)
1501 		return true;
1502 	return false;
1503 }
1504 
1505 static int __get_segment_type_2(struct page *page, enum page_type p_type)
1506 {
1507 	if (p_type == DATA)
1508 		return CURSEG_HOT_DATA;
1509 	else
1510 		return CURSEG_HOT_NODE;
1511 }
1512 
1513 static int __get_segment_type_4(struct page *page, enum page_type p_type)
1514 {
1515 	if (p_type == DATA) {
1516 		struct inode *inode = page->mapping->host;
1517 
1518 		if (S_ISDIR(inode->i_mode))
1519 			return CURSEG_HOT_DATA;
1520 		else
1521 			return CURSEG_COLD_DATA;
1522 	} else {
1523 		if (IS_DNODE(page) && is_cold_node(page))
1524 			return CURSEG_WARM_NODE;
1525 		else
1526 			return CURSEG_COLD_NODE;
1527 	}
1528 }
1529 
1530 static int __get_segment_type_6(struct page *page, enum page_type p_type)
1531 {
1532 	if (p_type == DATA) {
1533 		struct inode *inode = page->mapping->host;
1534 
1535 		if (S_ISDIR(inode->i_mode))
1536 			return CURSEG_HOT_DATA;
1537 		else if (is_cold_data(page) || file_is_cold(inode))
1538 			return CURSEG_COLD_DATA;
1539 		else
1540 			return CURSEG_WARM_DATA;
1541 	} else {
1542 		if (IS_DNODE(page))
1543 			return is_cold_node(page) ? CURSEG_WARM_NODE :
1544 						CURSEG_HOT_NODE;
1545 		else
1546 			return CURSEG_COLD_NODE;
1547 	}
1548 }
1549 
1550 static int __get_segment_type(struct page *page, enum page_type p_type)
1551 {
1552 	switch (F2FS_P_SB(page)->active_logs) {
1553 	case 2:
1554 		return __get_segment_type_2(page, p_type);
1555 	case 4:
1556 		return __get_segment_type_4(page, p_type);
1557 	}
1558 	/* NR_CURSEG_TYPE(6) logs by default */
1559 	f2fs_bug_on(F2FS_P_SB(page),
1560 		F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE);
1561 	return __get_segment_type_6(page, p_type);
1562 }
1563 
1564 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
1565 		block_t old_blkaddr, block_t *new_blkaddr,
1566 		struct f2fs_summary *sum, int type)
1567 {
1568 	struct sit_info *sit_i = SIT_I(sbi);
1569 	struct curseg_info *curseg = CURSEG_I(sbi, type);
1570 
1571 	mutex_lock(&curseg->curseg_mutex);
1572 	mutex_lock(&sit_i->sentry_lock);
1573 
1574 	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
1575 
1576 	/*
1577 	 * __add_sum_entry should be resided under the curseg_mutex
1578 	 * because, this function updates a summary entry in the
1579 	 * current summary block.
1580 	 */
1581 	__add_sum_entry(sbi, type, sum);
1582 
1583 	__refresh_next_blkoff(sbi, curseg);
1584 
1585 	stat_inc_block_count(sbi, curseg);
1586 
1587 	if (!__has_curseg_space(sbi, type))
1588 		sit_i->s_ops->allocate_segment(sbi, type, false);
1589 	/*
1590 	 * SIT information should be updated before segment allocation,
1591 	 * since SSR needs latest valid block information.
1592 	 */
1593 	refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
1594 
1595 	mutex_unlock(&sit_i->sentry_lock);
1596 
1597 	if (page && IS_NODESEG(type))
1598 		fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1599 
1600 	mutex_unlock(&curseg->curseg_mutex);
1601 }
1602 
1603 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
1604 {
1605 	int type = __get_segment_type(fio->page, fio->type);
1606 
1607 	if (fio->type == NODE || fio->type == DATA)
1608 		mutex_lock(&fio->sbi->wio_mutex[fio->type]);
1609 
1610 	allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
1611 					&fio->new_blkaddr, sum, type);
1612 
1613 	/* writeout dirty page into bdev */
1614 	f2fs_submit_page_mbio(fio);
1615 
1616 	if (fio->type == NODE || fio->type == DATA)
1617 		mutex_unlock(&fio->sbi->wio_mutex[fio->type]);
1618 }
1619 
1620 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
1621 {
1622 	struct f2fs_io_info fio = {
1623 		.sbi = sbi,
1624 		.type = META,
1625 		.op = REQ_OP_WRITE,
1626 		.op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
1627 		.old_blkaddr = page->index,
1628 		.new_blkaddr = page->index,
1629 		.page = page,
1630 		.encrypted_page = NULL,
1631 	};
1632 
1633 	if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
1634 		fio.op_flags &= ~REQ_META;
1635 
1636 	set_page_writeback(page);
1637 	f2fs_submit_page_mbio(&fio);
1638 }
1639 
1640 void write_node_page(unsigned int nid, struct f2fs_io_info *fio)
1641 {
1642 	struct f2fs_summary sum;
1643 
1644 	set_summary(&sum, nid, 0, 0);
1645 	do_write_page(&sum, fio);
1646 }
1647 
1648 void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)
1649 {
1650 	struct f2fs_sb_info *sbi = fio->sbi;
1651 	struct f2fs_summary sum;
1652 	struct node_info ni;
1653 
1654 	f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
1655 	get_node_info(sbi, dn->nid, &ni);
1656 	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1657 	do_write_page(&sum, fio);
1658 	f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
1659 }
1660 
1661 void rewrite_data_page(struct f2fs_io_info *fio)
1662 {
1663 	fio->new_blkaddr = fio->old_blkaddr;
1664 	stat_inc_inplace_blocks(fio->sbi);
1665 	f2fs_submit_page_mbio(fio);
1666 }
1667 
1668 void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
1669 				block_t old_blkaddr, block_t new_blkaddr,
1670 				bool recover_curseg, bool recover_newaddr)
1671 {
1672 	struct sit_info *sit_i = SIT_I(sbi);
1673 	struct curseg_info *curseg;
1674 	unsigned int segno, old_cursegno;
1675 	struct seg_entry *se;
1676 	int type;
1677 	unsigned short old_blkoff;
1678 
1679 	segno = GET_SEGNO(sbi, new_blkaddr);
1680 	se = get_seg_entry(sbi, segno);
1681 	type = se->type;
1682 
1683 	if (!recover_curseg) {
1684 		/* for recovery flow */
1685 		if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1686 			if (old_blkaddr == NULL_ADDR)
1687 				type = CURSEG_COLD_DATA;
1688 			else
1689 				type = CURSEG_WARM_DATA;
1690 		}
1691 	} else {
1692 		if (!IS_CURSEG(sbi, segno))
1693 			type = CURSEG_WARM_DATA;
1694 	}
1695 
1696 	curseg = CURSEG_I(sbi, type);
1697 
1698 	mutex_lock(&curseg->curseg_mutex);
1699 	mutex_lock(&sit_i->sentry_lock);
1700 
1701 	old_cursegno = curseg->segno;
1702 	old_blkoff = curseg->next_blkoff;
1703 
1704 	/* change the current segment */
1705 	if (segno != curseg->segno) {
1706 		curseg->next_segno = segno;
1707 		change_curseg(sbi, type, true);
1708 	}
1709 
1710 	curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
1711 	__add_sum_entry(sbi, type, sum);
1712 
1713 	if (!recover_curseg || recover_newaddr)
1714 		update_sit_entry(sbi, new_blkaddr, 1);
1715 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1716 		update_sit_entry(sbi, old_blkaddr, -1);
1717 
1718 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1719 	locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
1720 
1721 	locate_dirty_segment(sbi, old_cursegno);
1722 
1723 	if (recover_curseg) {
1724 		if (old_cursegno != curseg->segno) {
1725 			curseg->next_segno = old_cursegno;
1726 			change_curseg(sbi, type, true);
1727 		}
1728 		curseg->next_blkoff = old_blkoff;
1729 	}
1730 
1731 	mutex_unlock(&sit_i->sentry_lock);
1732 	mutex_unlock(&curseg->curseg_mutex);
1733 }
1734 
1735 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
1736 				block_t old_addr, block_t new_addr,
1737 				unsigned char version, bool recover_curseg,
1738 				bool recover_newaddr)
1739 {
1740 	struct f2fs_summary sum;
1741 
1742 	set_summary(&sum, dn->nid, dn->ofs_in_node, version);
1743 
1744 	__f2fs_replace_block(sbi, &sum, old_addr, new_addr,
1745 					recover_curseg, recover_newaddr);
1746 
1747 	f2fs_update_data_blkaddr(dn, new_addr);
1748 }
1749 
1750 void f2fs_wait_on_page_writeback(struct page *page,
1751 				enum page_type type, bool ordered)
1752 {
1753 	if (PageWriteback(page)) {
1754 		struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1755 
1756 		f2fs_submit_merged_bio_cond(sbi, NULL, page, 0, type, WRITE);
1757 		if (ordered)
1758 			wait_on_page_writeback(page);
1759 		else
1760 			wait_for_stable_page(page);
1761 	}
1762 }
1763 
1764 void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,
1765 							block_t blkaddr)
1766 {
1767 	struct page *cpage;
1768 
1769 	if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
1770 		return;
1771 
1772 	cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
1773 	if (cpage) {
1774 		f2fs_wait_on_page_writeback(cpage, DATA, true);
1775 		f2fs_put_page(cpage, 1);
1776 	}
1777 }
1778 
1779 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1780 {
1781 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1782 	struct curseg_info *seg_i;
1783 	unsigned char *kaddr;
1784 	struct page *page;
1785 	block_t start;
1786 	int i, j, offset;
1787 
1788 	start = start_sum_block(sbi);
1789 
1790 	page = get_meta_page(sbi, start++);
1791 	kaddr = (unsigned char *)page_address(page);
1792 
1793 	/* Step 1: restore nat cache */
1794 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1795 	memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
1796 
1797 	/* Step 2: restore sit cache */
1798 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1799 	memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
1800 	offset = 2 * SUM_JOURNAL_SIZE;
1801 
1802 	/* Step 3: restore summary entries */
1803 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1804 		unsigned short blk_off;
1805 		unsigned int segno;
1806 
1807 		seg_i = CURSEG_I(sbi, i);
1808 		segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1809 		blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1810 		seg_i->next_segno = segno;
1811 		reset_curseg(sbi, i, 0);
1812 		seg_i->alloc_type = ckpt->alloc_type[i];
1813 		seg_i->next_blkoff = blk_off;
1814 
1815 		if (seg_i->alloc_type == SSR)
1816 			blk_off = sbi->blocks_per_seg;
1817 
1818 		for (j = 0; j < blk_off; j++) {
1819 			struct f2fs_summary *s;
1820 			s = (struct f2fs_summary *)(kaddr + offset);
1821 			seg_i->sum_blk->entries[j] = *s;
1822 			offset += SUMMARY_SIZE;
1823 			if (offset + SUMMARY_SIZE <= PAGE_SIZE -
1824 						SUM_FOOTER_SIZE)
1825 				continue;
1826 
1827 			f2fs_put_page(page, 1);
1828 			page = NULL;
1829 
1830 			page = get_meta_page(sbi, start++);
1831 			kaddr = (unsigned char *)page_address(page);
1832 			offset = 0;
1833 		}
1834 	}
1835 	f2fs_put_page(page, 1);
1836 	return 0;
1837 }
1838 
1839 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1840 {
1841 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1842 	struct f2fs_summary_block *sum;
1843 	struct curseg_info *curseg;
1844 	struct page *new;
1845 	unsigned short blk_off;
1846 	unsigned int segno = 0;
1847 	block_t blk_addr = 0;
1848 
1849 	/* get segment number and block addr */
1850 	if (IS_DATASEG(type)) {
1851 		segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1852 		blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1853 							CURSEG_HOT_DATA]);
1854 		if (__exist_node_summaries(sbi))
1855 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1856 		else
1857 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1858 	} else {
1859 		segno = le32_to_cpu(ckpt->cur_node_segno[type -
1860 							CURSEG_HOT_NODE]);
1861 		blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1862 							CURSEG_HOT_NODE]);
1863 		if (__exist_node_summaries(sbi))
1864 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1865 							type - CURSEG_HOT_NODE);
1866 		else
1867 			blk_addr = GET_SUM_BLOCK(sbi, segno);
1868 	}
1869 
1870 	new = get_meta_page(sbi, blk_addr);
1871 	sum = (struct f2fs_summary_block *)page_address(new);
1872 
1873 	if (IS_NODESEG(type)) {
1874 		if (__exist_node_summaries(sbi)) {
1875 			struct f2fs_summary *ns = &sum->entries[0];
1876 			int i;
1877 			for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1878 				ns->version = 0;
1879 				ns->ofs_in_node = 0;
1880 			}
1881 		} else {
1882 			int err;
1883 
1884 			err = restore_node_summary(sbi, segno, sum);
1885 			if (err) {
1886 				f2fs_put_page(new, 1);
1887 				return err;
1888 			}
1889 		}
1890 	}
1891 
1892 	/* set uncompleted segment to curseg */
1893 	curseg = CURSEG_I(sbi, type);
1894 	mutex_lock(&curseg->curseg_mutex);
1895 
1896 	/* update journal info */
1897 	down_write(&curseg->journal_rwsem);
1898 	memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
1899 	up_write(&curseg->journal_rwsem);
1900 
1901 	memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
1902 	memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
1903 	curseg->next_segno = segno;
1904 	reset_curseg(sbi, type, 0);
1905 	curseg->alloc_type = ckpt->alloc_type[type];
1906 	curseg->next_blkoff = blk_off;
1907 	mutex_unlock(&curseg->curseg_mutex);
1908 	f2fs_put_page(new, 1);
1909 	return 0;
1910 }
1911 
1912 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1913 {
1914 	int type = CURSEG_HOT_DATA;
1915 	int err;
1916 
1917 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
1918 		int npages = npages_for_summary_flush(sbi, true);
1919 
1920 		if (npages >= 2)
1921 			ra_meta_pages(sbi, start_sum_block(sbi), npages,
1922 							META_CP, true);
1923 
1924 		/* restore for compacted data summary */
1925 		if (read_compacted_summaries(sbi))
1926 			return -EINVAL;
1927 		type = CURSEG_HOT_NODE;
1928 	}
1929 
1930 	if (__exist_node_summaries(sbi))
1931 		ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
1932 					NR_CURSEG_TYPE - type, META_CP, true);
1933 
1934 	for (; type <= CURSEG_COLD_NODE; type++) {
1935 		err = read_normal_summaries(sbi, type);
1936 		if (err)
1937 			return err;
1938 	}
1939 
1940 	return 0;
1941 }
1942 
1943 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1944 {
1945 	struct page *page;
1946 	unsigned char *kaddr;
1947 	struct f2fs_summary *summary;
1948 	struct curseg_info *seg_i;
1949 	int written_size = 0;
1950 	int i, j;
1951 
1952 	page = grab_meta_page(sbi, blkaddr++);
1953 	kaddr = (unsigned char *)page_address(page);
1954 
1955 	/* Step 1: write nat cache */
1956 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1957 	memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
1958 	written_size += SUM_JOURNAL_SIZE;
1959 
1960 	/* Step 2: write sit cache */
1961 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1962 	memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
1963 	written_size += SUM_JOURNAL_SIZE;
1964 
1965 	/* Step 3: write summary entries */
1966 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1967 		unsigned short blkoff;
1968 		seg_i = CURSEG_I(sbi, i);
1969 		if (sbi->ckpt->alloc_type[i] == SSR)
1970 			blkoff = sbi->blocks_per_seg;
1971 		else
1972 			blkoff = curseg_blkoff(sbi, i);
1973 
1974 		for (j = 0; j < blkoff; j++) {
1975 			if (!page) {
1976 				page = grab_meta_page(sbi, blkaddr++);
1977 				kaddr = (unsigned char *)page_address(page);
1978 				written_size = 0;
1979 			}
1980 			summary = (struct f2fs_summary *)(kaddr + written_size);
1981 			*summary = seg_i->sum_blk->entries[j];
1982 			written_size += SUMMARY_SIZE;
1983 
1984 			if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
1985 							SUM_FOOTER_SIZE)
1986 				continue;
1987 
1988 			set_page_dirty(page);
1989 			f2fs_put_page(page, 1);
1990 			page = NULL;
1991 		}
1992 	}
1993 	if (page) {
1994 		set_page_dirty(page);
1995 		f2fs_put_page(page, 1);
1996 	}
1997 }
1998 
1999 static void write_normal_summaries(struct f2fs_sb_info *sbi,
2000 					block_t blkaddr, int type)
2001 {
2002 	int i, end;
2003 	if (IS_DATASEG(type))
2004 		end = type + NR_CURSEG_DATA_TYPE;
2005 	else
2006 		end = type + NR_CURSEG_NODE_TYPE;
2007 
2008 	for (i = type; i < end; i++)
2009 		write_current_sum_page(sbi, i, blkaddr + (i - type));
2010 }
2011 
2012 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
2013 {
2014 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
2015 		write_compacted_summaries(sbi, start_blk);
2016 	else
2017 		write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
2018 }
2019 
2020 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
2021 {
2022 	write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
2023 }
2024 
2025 int lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
2026 					unsigned int val, int alloc)
2027 {
2028 	int i;
2029 
2030 	if (type == NAT_JOURNAL) {
2031 		for (i = 0; i < nats_in_cursum(journal); i++) {
2032 			if (le32_to_cpu(nid_in_journal(journal, i)) == val)
2033 				return i;
2034 		}
2035 		if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
2036 			return update_nats_in_cursum(journal, 1);
2037 	} else if (type == SIT_JOURNAL) {
2038 		for (i = 0; i < sits_in_cursum(journal); i++)
2039 			if (le32_to_cpu(segno_in_journal(journal, i)) == val)
2040 				return i;
2041 		if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
2042 			return update_sits_in_cursum(journal, 1);
2043 	}
2044 	return -1;
2045 }
2046 
2047 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
2048 					unsigned int segno)
2049 {
2050 	return get_meta_page(sbi, current_sit_addr(sbi, segno));
2051 }
2052 
2053 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
2054 					unsigned int start)
2055 {
2056 	struct sit_info *sit_i = SIT_I(sbi);
2057 	struct page *src_page, *dst_page;
2058 	pgoff_t src_off, dst_off;
2059 	void *src_addr, *dst_addr;
2060 
2061 	src_off = current_sit_addr(sbi, start);
2062 	dst_off = next_sit_addr(sbi, src_off);
2063 
2064 	/* get current sit block page without lock */
2065 	src_page = get_meta_page(sbi, src_off);
2066 	dst_page = grab_meta_page(sbi, dst_off);
2067 	f2fs_bug_on(sbi, PageDirty(src_page));
2068 
2069 	src_addr = page_address(src_page);
2070 	dst_addr = page_address(dst_page);
2071 	memcpy(dst_addr, src_addr, PAGE_SIZE);
2072 
2073 	set_page_dirty(dst_page);
2074 	f2fs_put_page(src_page, 1);
2075 
2076 	set_to_next_sit(sit_i, start);
2077 
2078 	return dst_page;
2079 }
2080 
2081 static struct sit_entry_set *grab_sit_entry_set(void)
2082 {
2083 	struct sit_entry_set *ses =
2084 			f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
2085 
2086 	ses->entry_cnt = 0;
2087 	INIT_LIST_HEAD(&ses->set_list);
2088 	return ses;
2089 }
2090 
2091 static void release_sit_entry_set(struct sit_entry_set *ses)
2092 {
2093 	list_del(&ses->set_list);
2094 	kmem_cache_free(sit_entry_set_slab, ses);
2095 }
2096 
2097 static void adjust_sit_entry_set(struct sit_entry_set *ses,
2098 						struct list_head *head)
2099 {
2100 	struct sit_entry_set *next = ses;
2101 
2102 	if (list_is_last(&ses->set_list, head))
2103 		return;
2104 
2105 	list_for_each_entry_continue(next, head, set_list)
2106 		if (ses->entry_cnt <= next->entry_cnt)
2107 			break;
2108 
2109 	list_move_tail(&ses->set_list, &next->set_list);
2110 }
2111 
2112 static void add_sit_entry(unsigned int segno, struct list_head *head)
2113 {
2114 	struct sit_entry_set *ses;
2115 	unsigned int start_segno = START_SEGNO(segno);
2116 
2117 	list_for_each_entry(ses, head, set_list) {
2118 		if (ses->start_segno == start_segno) {
2119 			ses->entry_cnt++;
2120 			adjust_sit_entry_set(ses, head);
2121 			return;
2122 		}
2123 	}
2124 
2125 	ses = grab_sit_entry_set();
2126 
2127 	ses->start_segno = start_segno;
2128 	ses->entry_cnt++;
2129 	list_add(&ses->set_list, head);
2130 }
2131 
2132 static void add_sits_in_set(struct f2fs_sb_info *sbi)
2133 {
2134 	struct f2fs_sm_info *sm_info = SM_I(sbi);
2135 	struct list_head *set_list = &sm_info->sit_entry_set;
2136 	unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
2137 	unsigned int segno;
2138 
2139 	for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
2140 		add_sit_entry(segno, set_list);
2141 }
2142 
2143 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
2144 {
2145 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2146 	struct f2fs_journal *journal = curseg->journal;
2147 	int i;
2148 
2149 	down_write(&curseg->journal_rwsem);
2150 	for (i = 0; i < sits_in_cursum(journal); i++) {
2151 		unsigned int segno;
2152 		bool dirtied;
2153 
2154 		segno = le32_to_cpu(segno_in_journal(journal, i));
2155 		dirtied = __mark_sit_entry_dirty(sbi, segno);
2156 
2157 		if (!dirtied)
2158 			add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
2159 	}
2160 	update_sits_in_cursum(journal, -i);
2161 	up_write(&curseg->journal_rwsem);
2162 }
2163 
2164 /*
2165  * CP calls this function, which flushes SIT entries including sit_journal,
2166  * and moves prefree segs to free segs.
2167  */
2168 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
2169 {
2170 	struct sit_info *sit_i = SIT_I(sbi);
2171 	unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
2172 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2173 	struct f2fs_journal *journal = curseg->journal;
2174 	struct sit_entry_set *ses, *tmp;
2175 	struct list_head *head = &SM_I(sbi)->sit_entry_set;
2176 	bool to_journal = true;
2177 	struct seg_entry *se;
2178 
2179 	mutex_lock(&sit_i->sentry_lock);
2180 
2181 	if (!sit_i->dirty_sentries)
2182 		goto out;
2183 
2184 	/*
2185 	 * add and account sit entries of dirty bitmap in sit entry
2186 	 * set temporarily
2187 	 */
2188 	add_sits_in_set(sbi);
2189 
2190 	/*
2191 	 * if there are no enough space in journal to store dirty sit
2192 	 * entries, remove all entries from journal and add and account
2193 	 * them in sit entry set.
2194 	 */
2195 	if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL))
2196 		remove_sits_in_journal(sbi);
2197 
2198 	/*
2199 	 * there are two steps to flush sit entries:
2200 	 * #1, flush sit entries to journal in current cold data summary block.
2201 	 * #2, flush sit entries to sit page.
2202 	 */
2203 	list_for_each_entry_safe(ses, tmp, head, set_list) {
2204 		struct page *page = NULL;
2205 		struct f2fs_sit_block *raw_sit = NULL;
2206 		unsigned int start_segno = ses->start_segno;
2207 		unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
2208 						(unsigned long)MAIN_SEGS(sbi));
2209 		unsigned int segno = start_segno;
2210 
2211 		if (to_journal &&
2212 			!__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
2213 			to_journal = false;
2214 
2215 		if (to_journal) {
2216 			down_write(&curseg->journal_rwsem);
2217 		} else {
2218 			page = get_next_sit_page(sbi, start_segno);
2219 			raw_sit = page_address(page);
2220 		}
2221 
2222 		/* flush dirty sit entries in region of current sit set */
2223 		for_each_set_bit_from(segno, bitmap, end) {
2224 			int offset, sit_offset;
2225 
2226 			se = get_seg_entry(sbi, segno);
2227 
2228 			/* add discard candidates */
2229 			if (cpc->reason != CP_DISCARD) {
2230 				cpc->trim_start = segno;
2231 				add_discard_addrs(sbi, cpc);
2232 			}
2233 
2234 			if (to_journal) {
2235 				offset = lookup_journal_in_cursum(journal,
2236 							SIT_JOURNAL, segno, 1);
2237 				f2fs_bug_on(sbi, offset < 0);
2238 				segno_in_journal(journal, offset) =
2239 							cpu_to_le32(segno);
2240 				seg_info_to_raw_sit(se,
2241 					&sit_in_journal(journal, offset));
2242 			} else {
2243 				sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
2244 				seg_info_to_raw_sit(se,
2245 						&raw_sit->entries[sit_offset]);
2246 			}
2247 
2248 			__clear_bit(segno, bitmap);
2249 			sit_i->dirty_sentries--;
2250 			ses->entry_cnt--;
2251 		}
2252 
2253 		if (to_journal)
2254 			up_write(&curseg->journal_rwsem);
2255 		else
2256 			f2fs_put_page(page, 1);
2257 
2258 		f2fs_bug_on(sbi, ses->entry_cnt);
2259 		release_sit_entry_set(ses);
2260 	}
2261 
2262 	f2fs_bug_on(sbi, !list_empty(head));
2263 	f2fs_bug_on(sbi, sit_i->dirty_sentries);
2264 out:
2265 	if (cpc->reason == CP_DISCARD) {
2266 		for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
2267 			add_discard_addrs(sbi, cpc);
2268 	}
2269 	mutex_unlock(&sit_i->sentry_lock);
2270 
2271 	set_prefree_as_free_segments(sbi);
2272 }
2273 
2274 static int build_sit_info(struct f2fs_sb_info *sbi)
2275 {
2276 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2277 	struct sit_info *sit_i;
2278 	unsigned int sit_segs, start;
2279 	char *src_bitmap, *dst_bitmap;
2280 	unsigned int bitmap_size;
2281 
2282 	/* allocate memory for SIT information */
2283 	sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
2284 	if (!sit_i)
2285 		return -ENOMEM;
2286 
2287 	SM_I(sbi)->sit_info = sit_i;
2288 
2289 	sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) *
2290 					sizeof(struct seg_entry), GFP_KERNEL);
2291 	if (!sit_i->sentries)
2292 		return -ENOMEM;
2293 
2294 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2295 	sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2296 	if (!sit_i->dirty_sentries_bitmap)
2297 		return -ENOMEM;
2298 
2299 	for (start = 0; start < MAIN_SEGS(sbi); start++) {
2300 		sit_i->sentries[start].cur_valid_map
2301 			= kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2302 		sit_i->sentries[start].ckpt_valid_map
2303 			= kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2304 		if (!sit_i->sentries[start].cur_valid_map ||
2305 				!sit_i->sentries[start].ckpt_valid_map)
2306 			return -ENOMEM;
2307 
2308 		if (f2fs_discard_en(sbi)) {
2309 			sit_i->sentries[start].discard_map
2310 				= kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2311 			if (!sit_i->sentries[start].discard_map)
2312 				return -ENOMEM;
2313 		}
2314 	}
2315 
2316 	sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2317 	if (!sit_i->tmp_map)
2318 		return -ENOMEM;
2319 
2320 	if (sbi->segs_per_sec > 1) {
2321 		sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) *
2322 					sizeof(struct sec_entry), GFP_KERNEL);
2323 		if (!sit_i->sec_entries)
2324 			return -ENOMEM;
2325 	}
2326 
2327 	/* get information related with SIT */
2328 	sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
2329 
2330 	/* setup SIT bitmap from ckeckpoint pack */
2331 	bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
2332 	src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
2333 
2334 	dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
2335 	if (!dst_bitmap)
2336 		return -ENOMEM;
2337 
2338 	/* init SIT information */
2339 	sit_i->s_ops = &default_salloc_ops;
2340 
2341 	sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
2342 	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
2343 	sit_i->written_valid_blocks = 0;
2344 	sit_i->sit_bitmap = dst_bitmap;
2345 	sit_i->bitmap_size = bitmap_size;
2346 	sit_i->dirty_sentries = 0;
2347 	sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
2348 	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
2349 	sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
2350 	mutex_init(&sit_i->sentry_lock);
2351 	return 0;
2352 }
2353 
2354 static int build_free_segmap(struct f2fs_sb_info *sbi)
2355 {
2356 	struct free_segmap_info *free_i;
2357 	unsigned int bitmap_size, sec_bitmap_size;
2358 
2359 	/* allocate memory for free segmap information */
2360 	free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
2361 	if (!free_i)
2362 		return -ENOMEM;
2363 
2364 	SM_I(sbi)->free_info = free_i;
2365 
2366 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2367 	free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL);
2368 	if (!free_i->free_segmap)
2369 		return -ENOMEM;
2370 
2371 	sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2372 	free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL);
2373 	if (!free_i->free_secmap)
2374 		return -ENOMEM;
2375 
2376 	/* set all segments as dirty temporarily */
2377 	memset(free_i->free_segmap, 0xff, bitmap_size);
2378 	memset(free_i->free_secmap, 0xff, sec_bitmap_size);
2379 
2380 	/* init free segmap information */
2381 	free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
2382 	free_i->free_segments = 0;
2383 	free_i->free_sections = 0;
2384 	spin_lock_init(&free_i->segmap_lock);
2385 	return 0;
2386 }
2387 
2388 static int build_curseg(struct f2fs_sb_info *sbi)
2389 {
2390 	struct curseg_info *array;
2391 	int i;
2392 
2393 	array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
2394 	if (!array)
2395 		return -ENOMEM;
2396 
2397 	SM_I(sbi)->curseg_array = array;
2398 
2399 	for (i = 0; i < NR_CURSEG_TYPE; i++) {
2400 		mutex_init(&array[i].curseg_mutex);
2401 		array[i].sum_blk = kzalloc(PAGE_SIZE, GFP_KERNEL);
2402 		if (!array[i].sum_blk)
2403 			return -ENOMEM;
2404 		init_rwsem(&array[i].journal_rwsem);
2405 		array[i].journal = kzalloc(sizeof(struct f2fs_journal),
2406 							GFP_KERNEL);
2407 		if (!array[i].journal)
2408 			return -ENOMEM;
2409 		array[i].segno = NULL_SEGNO;
2410 		array[i].next_blkoff = 0;
2411 	}
2412 	return restore_curseg_summaries(sbi);
2413 }
2414 
2415 static void build_sit_entries(struct f2fs_sb_info *sbi)
2416 {
2417 	struct sit_info *sit_i = SIT_I(sbi);
2418 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2419 	struct f2fs_journal *journal = curseg->journal;
2420 	struct seg_entry *se;
2421 	struct f2fs_sit_entry sit;
2422 	int sit_blk_cnt = SIT_BLK_CNT(sbi);
2423 	unsigned int i, start, end;
2424 	unsigned int readed, start_blk = 0;
2425 
2426 	do {
2427 		readed = ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
2428 							META_SIT, true);
2429 
2430 		start = start_blk * sit_i->sents_per_block;
2431 		end = (start_blk + readed) * sit_i->sents_per_block;
2432 
2433 		for (; start < end && start < MAIN_SEGS(sbi); start++) {
2434 			struct f2fs_sit_block *sit_blk;
2435 			struct page *page;
2436 
2437 			se = &sit_i->sentries[start];
2438 			page = get_current_sit_page(sbi, start);
2439 			sit_blk = (struct f2fs_sit_block *)page_address(page);
2440 			sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
2441 			f2fs_put_page(page, 1);
2442 
2443 			check_block_count(sbi, start, &sit);
2444 			seg_info_from_raw_sit(se, &sit);
2445 
2446 			/* build discard map only one time */
2447 			if (f2fs_discard_en(sbi)) {
2448 				memcpy(se->discard_map, se->cur_valid_map,
2449 							SIT_VBLOCK_MAP_SIZE);
2450 				sbi->discard_blks += sbi->blocks_per_seg -
2451 							se->valid_blocks;
2452 			}
2453 
2454 			if (sbi->segs_per_sec > 1)
2455 				get_sec_entry(sbi, start)->valid_blocks +=
2456 							se->valid_blocks;
2457 		}
2458 		start_blk += readed;
2459 	} while (start_blk < sit_blk_cnt);
2460 
2461 	down_read(&curseg->journal_rwsem);
2462 	for (i = 0; i < sits_in_cursum(journal); i++) {
2463 		unsigned int old_valid_blocks;
2464 
2465 		start = le32_to_cpu(segno_in_journal(journal, i));
2466 		se = &sit_i->sentries[start];
2467 		sit = sit_in_journal(journal, i);
2468 
2469 		old_valid_blocks = se->valid_blocks;
2470 
2471 		check_block_count(sbi, start, &sit);
2472 		seg_info_from_raw_sit(se, &sit);
2473 
2474 		if (f2fs_discard_en(sbi)) {
2475 			memcpy(se->discard_map, se->cur_valid_map,
2476 						SIT_VBLOCK_MAP_SIZE);
2477 			sbi->discard_blks += old_valid_blocks -
2478 						se->valid_blocks;
2479 		}
2480 
2481 		if (sbi->segs_per_sec > 1)
2482 			get_sec_entry(sbi, start)->valid_blocks +=
2483 				se->valid_blocks - old_valid_blocks;
2484 	}
2485 	up_read(&curseg->journal_rwsem);
2486 }
2487 
2488 static void init_free_segmap(struct f2fs_sb_info *sbi)
2489 {
2490 	unsigned int start;
2491 	int type;
2492 
2493 	for (start = 0; start < MAIN_SEGS(sbi); start++) {
2494 		struct seg_entry *sentry = get_seg_entry(sbi, start);
2495 		if (!sentry->valid_blocks)
2496 			__set_free(sbi, start);
2497 		else
2498 			SIT_I(sbi)->written_valid_blocks +=
2499 						sentry->valid_blocks;
2500 	}
2501 
2502 	/* set use the current segments */
2503 	for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
2504 		struct curseg_info *curseg_t = CURSEG_I(sbi, type);
2505 		__set_test_and_inuse(sbi, curseg_t->segno);
2506 	}
2507 }
2508 
2509 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
2510 {
2511 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2512 	struct free_segmap_info *free_i = FREE_I(sbi);
2513 	unsigned int segno = 0, offset = 0;
2514 	unsigned short valid_blocks;
2515 
2516 	while (1) {
2517 		/* find dirty segment based on free segmap */
2518 		segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
2519 		if (segno >= MAIN_SEGS(sbi))
2520 			break;
2521 		offset = segno + 1;
2522 		valid_blocks = get_valid_blocks(sbi, segno, 0);
2523 		if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
2524 			continue;
2525 		if (valid_blocks > sbi->blocks_per_seg) {
2526 			f2fs_bug_on(sbi, 1);
2527 			continue;
2528 		}
2529 		mutex_lock(&dirty_i->seglist_lock);
2530 		__locate_dirty_segment(sbi, segno, DIRTY);
2531 		mutex_unlock(&dirty_i->seglist_lock);
2532 	}
2533 }
2534 
2535 static int init_victim_secmap(struct f2fs_sb_info *sbi)
2536 {
2537 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2538 	unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2539 
2540 	dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2541 	if (!dirty_i->victim_secmap)
2542 		return -ENOMEM;
2543 	return 0;
2544 }
2545 
2546 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
2547 {
2548 	struct dirty_seglist_info *dirty_i;
2549 	unsigned int bitmap_size, i;
2550 
2551 	/* allocate memory for dirty segments list information */
2552 	dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
2553 	if (!dirty_i)
2554 		return -ENOMEM;
2555 
2556 	SM_I(sbi)->dirty_info = dirty_i;
2557 	mutex_init(&dirty_i->seglist_lock);
2558 
2559 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2560 
2561 	for (i = 0; i < NR_DIRTY_TYPE; i++) {
2562 		dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2563 		if (!dirty_i->dirty_segmap[i])
2564 			return -ENOMEM;
2565 	}
2566 
2567 	init_dirty_segmap(sbi);
2568 	return init_victim_secmap(sbi);
2569 }
2570 
2571 /*
2572  * Update min, max modified time for cost-benefit GC algorithm
2573  */
2574 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
2575 {
2576 	struct sit_info *sit_i = SIT_I(sbi);
2577 	unsigned int segno;
2578 
2579 	mutex_lock(&sit_i->sentry_lock);
2580 
2581 	sit_i->min_mtime = LLONG_MAX;
2582 
2583 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
2584 		unsigned int i;
2585 		unsigned long long mtime = 0;
2586 
2587 		for (i = 0; i < sbi->segs_per_sec; i++)
2588 			mtime += get_seg_entry(sbi, segno + i)->mtime;
2589 
2590 		mtime = div_u64(mtime, sbi->segs_per_sec);
2591 
2592 		if (sit_i->min_mtime > mtime)
2593 			sit_i->min_mtime = mtime;
2594 	}
2595 	sit_i->max_mtime = get_mtime(sbi);
2596 	mutex_unlock(&sit_i->sentry_lock);
2597 }
2598 
2599 int build_segment_manager(struct f2fs_sb_info *sbi)
2600 {
2601 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2602 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2603 	struct f2fs_sm_info *sm_info;
2604 	int err;
2605 
2606 	sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
2607 	if (!sm_info)
2608 		return -ENOMEM;
2609 
2610 	/* init sm info */
2611 	sbi->sm_info = sm_info;
2612 	sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
2613 	sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
2614 	sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
2615 	sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
2616 	sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
2617 	sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
2618 	sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
2619 	sm_info->rec_prefree_segments = sm_info->main_segments *
2620 					DEF_RECLAIM_PREFREE_SEGMENTS / 100;
2621 	if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
2622 		sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
2623 
2624 	if (!test_opt(sbi, LFS))
2625 		sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
2626 	sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
2627 	sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
2628 
2629 	INIT_LIST_HEAD(&sm_info->discard_list);
2630 	INIT_LIST_HEAD(&sm_info->wait_list);
2631 	sm_info->nr_discards = 0;
2632 	sm_info->max_discards = 0;
2633 
2634 	sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS;
2635 
2636 	INIT_LIST_HEAD(&sm_info->sit_entry_set);
2637 
2638 	if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
2639 		err = create_flush_cmd_control(sbi);
2640 		if (err)
2641 			return err;
2642 	}
2643 
2644 	err = build_sit_info(sbi);
2645 	if (err)
2646 		return err;
2647 	err = build_free_segmap(sbi);
2648 	if (err)
2649 		return err;
2650 	err = build_curseg(sbi);
2651 	if (err)
2652 		return err;
2653 
2654 	/* reinit free segmap based on SIT */
2655 	build_sit_entries(sbi);
2656 
2657 	init_free_segmap(sbi);
2658 	err = build_dirty_segmap(sbi);
2659 	if (err)
2660 		return err;
2661 
2662 	init_min_max_mtime(sbi);
2663 	return 0;
2664 }
2665 
2666 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
2667 		enum dirty_type dirty_type)
2668 {
2669 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2670 
2671 	mutex_lock(&dirty_i->seglist_lock);
2672 	kvfree(dirty_i->dirty_segmap[dirty_type]);
2673 	dirty_i->nr_dirty[dirty_type] = 0;
2674 	mutex_unlock(&dirty_i->seglist_lock);
2675 }
2676 
2677 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
2678 {
2679 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2680 	kvfree(dirty_i->victim_secmap);
2681 }
2682 
2683 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
2684 {
2685 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2686 	int i;
2687 
2688 	if (!dirty_i)
2689 		return;
2690 
2691 	/* discard pre-free/dirty segments list */
2692 	for (i = 0; i < NR_DIRTY_TYPE; i++)
2693 		discard_dirty_segmap(sbi, i);
2694 
2695 	destroy_victim_secmap(sbi);
2696 	SM_I(sbi)->dirty_info = NULL;
2697 	kfree(dirty_i);
2698 }
2699 
2700 static void destroy_curseg(struct f2fs_sb_info *sbi)
2701 {
2702 	struct curseg_info *array = SM_I(sbi)->curseg_array;
2703 	int i;
2704 
2705 	if (!array)
2706 		return;
2707 	SM_I(sbi)->curseg_array = NULL;
2708 	for (i = 0; i < NR_CURSEG_TYPE; i++) {
2709 		kfree(array[i].sum_blk);
2710 		kfree(array[i].journal);
2711 	}
2712 	kfree(array);
2713 }
2714 
2715 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
2716 {
2717 	struct free_segmap_info *free_i = SM_I(sbi)->free_info;
2718 	if (!free_i)
2719 		return;
2720 	SM_I(sbi)->free_info = NULL;
2721 	kvfree(free_i->free_segmap);
2722 	kvfree(free_i->free_secmap);
2723 	kfree(free_i);
2724 }
2725 
2726 static void destroy_sit_info(struct f2fs_sb_info *sbi)
2727 {
2728 	struct sit_info *sit_i = SIT_I(sbi);
2729 	unsigned int start;
2730 
2731 	if (!sit_i)
2732 		return;
2733 
2734 	if (sit_i->sentries) {
2735 		for (start = 0; start < MAIN_SEGS(sbi); start++) {
2736 			kfree(sit_i->sentries[start].cur_valid_map);
2737 			kfree(sit_i->sentries[start].ckpt_valid_map);
2738 			kfree(sit_i->sentries[start].discard_map);
2739 		}
2740 	}
2741 	kfree(sit_i->tmp_map);
2742 
2743 	kvfree(sit_i->sentries);
2744 	kvfree(sit_i->sec_entries);
2745 	kvfree(sit_i->dirty_sentries_bitmap);
2746 
2747 	SM_I(sbi)->sit_info = NULL;
2748 	kfree(sit_i->sit_bitmap);
2749 	kfree(sit_i);
2750 }
2751 
2752 void destroy_segment_manager(struct f2fs_sb_info *sbi)
2753 {
2754 	struct f2fs_sm_info *sm_info = SM_I(sbi);
2755 
2756 	if (!sm_info)
2757 		return;
2758 	destroy_flush_cmd_control(sbi, true);
2759 	destroy_dirty_segmap(sbi);
2760 	destroy_curseg(sbi);
2761 	destroy_free_segmap(sbi);
2762 	destroy_sit_info(sbi);
2763 	sbi->sm_info = NULL;
2764 	kfree(sm_info);
2765 }
2766 
2767 int __init create_segment_manager_caches(void)
2768 {
2769 	discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
2770 			sizeof(struct discard_entry));
2771 	if (!discard_entry_slab)
2772 		goto fail;
2773 
2774 	bio_entry_slab = f2fs_kmem_cache_create("bio_entry",
2775 			sizeof(struct bio_entry));
2776 	if (!bio_entry_slab)
2777 		goto destroy_discard_entry;
2778 
2779 	sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
2780 			sizeof(struct sit_entry_set));
2781 	if (!sit_entry_set_slab)
2782 		goto destroy_bio_entry;
2783 
2784 	inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
2785 			sizeof(struct inmem_pages));
2786 	if (!inmem_entry_slab)
2787 		goto destroy_sit_entry_set;
2788 	return 0;
2789 
2790 destroy_sit_entry_set:
2791 	kmem_cache_destroy(sit_entry_set_slab);
2792 destroy_bio_entry:
2793 	kmem_cache_destroy(bio_entry_slab);
2794 destroy_discard_entry:
2795 	kmem_cache_destroy(discard_entry_slab);
2796 fail:
2797 	return -ENOMEM;
2798 }
2799 
2800 void destroy_segment_manager_caches(void)
2801 {
2802 	kmem_cache_destroy(sit_entry_set_slab);
2803 	kmem_cache_destroy(bio_entry_slab);
2804 	kmem_cache_destroy(discard_entry_slab);
2805 	kmem_cache_destroy(inmem_entry_slab);
2806 }
2807