xref: /openbmc/linux/fs/f2fs/segment.c (revision 364afd8a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * fs/f2fs/segment.c
4  *
5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6  *             http://www.samsung.com/
7  */
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/sched/mm.h>
13 #include <linux/prefetch.h>
14 #include <linux/kthread.h>
15 #include <linux/swap.h>
16 #include <linux/timer.h>
17 #include <linux/freezer.h>
18 #include <linux/sched/signal.h>
19 #include <linux/random.h>
20 
21 #include "f2fs.h"
22 #include "segment.h"
23 #include "node.h"
24 #include "gc.h"
25 #include "iostat.h"
26 #include <trace/events/f2fs.h>
27 
28 #define __reverse_ffz(x) __reverse_ffs(~(x))
29 
30 static struct kmem_cache *discard_entry_slab;
31 static struct kmem_cache *discard_cmd_slab;
32 static struct kmem_cache *sit_entry_set_slab;
33 static struct kmem_cache *revoke_entry_slab;
34 
__reverse_ulong(unsigned char * str)35 static unsigned long __reverse_ulong(unsigned char *str)
36 {
37 	unsigned long tmp = 0;
38 	int shift = 24, idx = 0;
39 
40 #if BITS_PER_LONG == 64
41 	shift = 56;
42 #endif
43 	while (shift >= 0) {
44 		tmp |= (unsigned long)str[idx++] << shift;
45 		shift -= BITS_PER_BYTE;
46 	}
47 	return tmp;
48 }
49 
50 /*
51  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
52  * MSB and LSB are reversed in a byte by f2fs_set_bit.
53  */
__reverse_ffs(unsigned long word)54 static inline unsigned long __reverse_ffs(unsigned long word)
55 {
56 	int num = 0;
57 
58 #if BITS_PER_LONG == 64
59 	if ((word & 0xffffffff00000000UL) == 0)
60 		num += 32;
61 	else
62 		word >>= 32;
63 #endif
64 	if ((word & 0xffff0000) == 0)
65 		num += 16;
66 	else
67 		word >>= 16;
68 
69 	if ((word & 0xff00) == 0)
70 		num += 8;
71 	else
72 		word >>= 8;
73 
74 	if ((word & 0xf0) == 0)
75 		num += 4;
76 	else
77 		word >>= 4;
78 
79 	if ((word & 0xc) == 0)
80 		num += 2;
81 	else
82 		word >>= 2;
83 
84 	if ((word & 0x2) == 0)
85 		num += 1;
86 	return num;
87 }
88 
89 /*
90  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
91  * f2fs_set_bit makes MSB and LSB reversed in a byte.
92  * @size must be integral times of unsigned long.
93  * Example:
94  *                             MSB <--> LSB
95  *   f2fs_set_bit(0, bitmap) => 1000 0000
96  *   f2fs_set_bit(7, bitmap) => 0000 0001
97  */
__find_rev_next_bit(const unsigned long * addr,unsigned long size,unsigned long offset)98 static unsigned long __find_rev_next_bit(const unsigned long *addr,
99 			unsigned long size, unsigned long offset)
100 {
101 	const unsigned long *p = addr + BIT_WORD(offset);
102 	unsigned long result = size;
103 	unsigned long tmp;
104 
105 	if (offset >= size)
106 		return size;
107 
108 	size -= (offset & ~(BITS_PER_LONG - 1));
109 	offset %= BITS_PER_LONG;
110 
111 	while (1) {
112 		if (*p == 0)
113 			goto pass;
114 
115 		tmp = __reverse_ulong((unsigned char *)p);
116 
117 		tmp &= ~0UL >> offset;
118 		if (size < BITS_PER_LONG)
119 			tmp &= (~0UL << (BITS_PER_LONG - size));
120 		if (tmp)
121 			goto found;
122 pass:
123 		if (size <= BITS_PER_LONG)
124 			break;
125 		size -= BITS_PER_LONG;
126 		offset = 0;
127 		p++;
128 	}
129 	return result;
130 found:
131 	return result - size + __reverse_ffs(tmp);
132 }
133 
__find_rev_next_zero_bit(const unsigned long * addr,unsigned long size,unsigned long offset)134 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
135 			unsigned long size, unsigned long offset)
136 {
137 	const unsigned long *p = addr + BIT_WORD(offset);
138 	unsigned long result = size;
139 	unsigned long tmp;
140 
141 	if (offset >= size)
142 		return size;
143 
144 	size -= (offset & ~(BITS_PER_LONG - 1));
145 	offset %= BITS_PER_LONG;
146 
147 	while (1) {
148 		if (*p == ~0UL)
149 			goto pass;
150 
151 		tmp = __reverse_ulong((unsigned char *)p);
152 
153 		if (offset)
154 			tmp |= ~0UL << (BITS_PER_LONG - offset);
155 		if (size < BITS_PER_LONG)
156 			tmp |= ~0UL >> size;
157 		if (tmp != ~0UL)
158 			goto found;
159 pass:
160 		if (size <= BITS_PER_LONG)
161 			break;
162 		size -= BITS_PER_LONG;
163 		offset = 0;
164 		p++;
165 	}
166 	return result;
167 found:
168 	return result - size + __reverse_ffz(tmp);
169 }
170 
f2fs_need_SSR(struct f2fs_sb_info * sbi)171 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
172 {
173 	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
174 	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
175 	int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
176 
177 	if (f2fs_lfs_mode(sbi))
178 		return false;
179 	if (sbi->gc_mode == GC_URGENT_HIGH)
180 		return true;
181 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
182 		return true;
183 
184 	return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
185 			SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
186 }
187 
f2fs_abort_atomic_write(struct inode * inode,bool clean)188 void f2fs_abort_atomic_write(struct inode *inode, bool clean)
189 {
190 	struct f2fs_inode_info *fi = F2FS_I(inode);
191 
192 	if (!f2fs_is_atomic_file(inode))
193 		return;
194 
195 	if (clean)
196 		truncate_inode_pages_final(inode->i_mapping);
197 
198 	release_atomic_write_cnt(inode);
199 	clear_inode_flag(inode, FI_ATOMIC_COMMITTED);
200 	clear_inode_flag(inode, FI_ATOMIC_REPLACE);
201 	clear_inode_flag(inode, FI_ATOMIC_FILE);
202 	if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) {
203 		clear_inode_flag(inode, FI_ATOMIC_DIRTIED);
204 		f2fs_mark_inode_dirty_sync(inode, true);
205 	}
206 	stat_dec_atomic_inode(inode);
207 
208 	F2FS_I(inode)->atomic_write_task = NULL;
209 
210 	if (clean) {
211 		f2fs_i_size_write(inode, fi->original_i_size);
212 		fi->original_i_size = 0;
213 	}
214 	/* avoid stale dirty inode during eviction */
215 	sync_inode_metadata(inode, 0);
216 }
217 
__replace_atomic_write_block(struct inode * inode,pgoff_t index,block_t new_addr,block_t * old_addr,bool recover)218 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
219 			block_t new_addr, block_t *old_addr, bool recover)
220 {
221 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
222 	struct dnode_of_data dn;
223 	struct node_info ni;
224 	int err;
225 
226 retry:
227 	set_new_dnode(&dn, inode, NULL, NULL, 0);
228 	err = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE);
229 	if (err) {
230 		if (err == -ENOMEM) {
231 			f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
232 			goto retry;
233 		}
234 		return err;
235 	}
236 
237 	err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
238 	if (err) {
239 		f2fs_put_dnode(&dn);
240 		return err;
241 	}
242 
243 	if (recover) {
244 		/* dn.data_blkaddr is always valid */
245 		if (!__is_valid_data_blkaddr(new_addr)) {
246 			if (new_addr == NULL_ADDR)
247 				dec_valid_block_count(sbi, inode, 1);
248 			f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
249 			f2fs_update_data_blkaddr(&dn, new_addr);
250 		} else {
251 			f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
252 				new_addr, ni.version, true, true);
253 		}
254 	} else {
255 		blkcnt_t count = 1;
256 
257 		err = inc_valid_block_count(sbi, inode, &count, true);
258 		if (err) {
259 			f2fs_put_dnode(&dn);
260 			return err;
261 		}
262 
263 		*old_addr = dn.data_blkaddr;
264 		f2fs_truncate_data_blocks_range(&dn, 1);
265 		dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count);
266 
267 		f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr,
268 					ni.version, true, false);
269 	}
270 
271 	f2fs_put_dnode(&dn);
272 
273 	trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode,
274 			index, old_addr ? *old_addr : 0, new_addr, recover);
275 	return 0;
276 }
277 
__complete_revoke_list(struct inode * inode,struct list_head * head,bool revoke)278 static void __complete_revoke_list(struct inode *inode, struct list_head *head,
279 					bool revoke)
280 {
281 	struct revoke_entry *cur, *tmp;
282 	pgoff_t start_index = 0;
283 	bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE);
284 
285 	list_for_each_entry_safe(cur, tmp, head, list) {
286 		if (revoke) {
287 			__replace_atomic_write_block(inode, cur->index,
288 						cur->old_addr, NULL, true);
289 		} else if (truncate) {
290 			f2fs_truncate_hole(inode, start_index, cur->index);
291 			start_index = cur->index + 1;
292 		}
293 
294 		list_del(&cur->list);
295 		kmem_cache_free(revoke_entry_slab, cur);
296 	}
297 
298 	if (!revoke && truncate)
299 		f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false);
300 }
301 
__f2fs_commit_atomic_write(struct inode * inode)302 static int __f2fs_commit_atomic_write(struct inode *inode)
303 {
304 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
305 	struct f2fs_inode_info *fi = F2FS_I(inode);
306 	struct inode *cow_inode = fi->cow_inode;
307 	struct revoke_entry *new;
308 	struct list_head revoke_list;
309 	block_t blkaddr;
310 	struct dnode_of_data dn;
311 	pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
312 	pgoff_t off = 0, blen, index;
313 	int ret = 0, i;
314 
315 	INIT_LIST_HEAD(&revoke_list);
316 
317 	while (len) {
318 		blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len);
319 
320 		set_new_dnode(&dn, cow_inode, NULL, NULL, 0);
321 		ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA);
322 		if (ret && ret != -ENOENT) {
323 			goto out;
324 		} else if (ret == -ENOENT) {
325 			ret = 0;
326 			if (dn.max_level == 0)
327 				goto out;
328 			goto next;
329 		}
330 
331 		blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_page, cow_inode),
332 				len);
333 		index = off;
334 		for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) {
335 			blkaddr = f2fs_data_blkaddr(&dn);
336 
337 			if (!__is_valid_data_blkaddr(blkaddr)) {
338 				continue;
339 			} else if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
340 					DATA_GENERIC_ENHANCE)) {
341 				f2fs_put_dnode(&dn);
342 				ret = -EFSCORRUPTED;
343 				f2fs_handle_error(sbi,
344 						ERROR_INVALID_BLKADDR);
345 				goto out;
346 			}
347 
348 			new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS,
349 							true, NULL);
350 
351 			ret = __replace_atomic_write_block(inode, index, blkaddr,
352 							&new->old_addr, false);
353 			if (ret) {
354 				f2fs_put_dnode(&dn);
355 				kmem_cache_free(revoke_entry_slab, new);
356 				goto out;
357 			}
358 
359 			f2fs_update_data_blkaddr(&dn, NULL_ADDR);
360 			new->index = index;
361 			list_add_tail(&new->list, &revoke_list);
362 		}
363 		f2fs_put_dnode(&dn);
364 next:
365 		off += blen;
366 		len -= blen;
367 	}
368 
369 out:
370 	if (ret) {
371 		sbi->revoked_atomic_block += fi->atomic_write_cnt;
372 	} else {
373 		sbi->committed_atomic_block += fi->atomic_write_cnt;
374 		set_inode_flag(inode, FI_ATOMIC_COMMITTED);
375 		if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) {
376 			clear_inode_flag(inode, FI_ATOMIC_DIRTIED);
377 			f2fs_mark_inode_dirty_sync(inode, true);
378 		}
379 	}
380 
381 	__complete_revoke_list(inode, &revoke_list, ret ? true : false);
382 
383 	return ret;
384 }
385 
f2fs_commit_atomic_write(struct inode * inode)386 int f2fs_commit_atomic_write(struct inode *inode)
387 {
388 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
389 	struct f2fs_inode_info *fi = F2FS_I(inode);
390 	int err;
391 
392 	err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
393 	if (err)
394 		return err;
395 
396 	f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
397 	f2fs_lock_op(sbi);
398 
399 	err = __f2fs_commit_atomic_write(inode);
400 
401 	f2fs_unlock_op(sbi);
402 	f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
403 
404 	return err;
405 }
406 
407 /*
408  * This function balances dirty node and dentry pages.
409  * In addition, it controls garbage collection.
410  */
f2fs_balance_fs(struct f2fs_sb_info * sbi,bool need)411 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
412 {
413 	if (time_to_inject(sbi, FAULT_CHECKPOINT))
414 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
415 
416 	/* balance_fs_bg is able to be pending */
417 	if (need && excess_cached_nats(sbi))
418 		f2fs_balance_fs_bg(sbi, false);
419 
420 	if (!f2fs_is_checkpoint_ready(sbi))
421 		return;
422 
423 	/*
424 	 * We should do GC or end up with checkpoint, if there are so many dirty
425 	 * dir/node pages without enough free segments.
426 	 */
427 	if (has_enough_free_secs(sbi, 0, 0))
428 		return;
429 
430 	if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
431 				sbi->gc_thread->f2fs_gc_task) {
432 		DEFINE_WAIT(wait);
433 
434 		prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
435 					TASK_UNINTERRUPTIBLE);
436 		wake_up(&sbi->gc_thread->gc_wait_queue_head);
437 		io_schedule();
438 		finish_wait(&sbi->gc_thread->fggc_wq, &wait);
439 	} else {
440 		struct f2fs_gc_control gc_control = {
441 			.victim_segno = NULL_SEGNO,
442 			.init_gc_type = BG_GC,
443 			.no_bg_gc = true,
444 			.should_migrate_blocks = false,
445 			.err_gc_skipped = false,
446 			.nr_free_secs = 1 };
447 		f2fs_down_write(&sbi->gc_lock);
448 		stat_inc_gc_call_count(sbi, FOREGROUND);
449 		f2fs_gc(sbi, &gc_control);
450 	}
451 }
452 
excess_dirty_threshold(struct f2fs_sb_info * sbi)453 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
454 {
455 	int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
456 	unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
457 	unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
458 	unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
459 	unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
460 	unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
461 	unsigned int threshold = (factor * DEFAULT_DIRTY_THRESHOLD) <<
462 				sbi->log_blocks_per_seg;
463 	unsigned int global_threshold = threshold * 3 / 2;
464 
465 	if (dents >= threshold || qdata >= threshold ||
466 		nodes >= threshold || meta >= threshold ||
467 		imeta >= threshold)
468 		return true;
469 	return dents + qdata + nodes + meta + imeta >  global_threshold;
470 }
471 
f2fs_balance_fs_bg(struct f2fs_sb_info * sbi,bool from_bg)472 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
473 {
474 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
475 		return;
476 
477 	/* try to shrink extent cache when there is no enough memory */
478 	if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))
479 		f2fs_shrink_read_extent_tree(sbi,
480 				READ_EXTENT_CACHE_SHRINK_NUMBER);
481 
482 	/* try to shrink age extent cache when there is no enough memory */
483 	if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE))
484 		f2fs_shrink_age_extent_tree(sbi,
485 				AGE_EXTENT_CACHE_SHRINK_NUMBER);
486 
487 	/* check the # of cached NAT entries */
488 	if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
489 		f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
490 
491 	if (!f2fs_available_free_memory(sbi, FREE_NIDS))
492 		f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
493 	else
494 		f2fs_build_free_nids(sbi, false, false);
495 
496 	if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
497 		excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
498 		goto do_sync;
499 
500 	/* there is background inflight IO or foreground operation recently */
501 	if (is_inflight_io(sbi, REQ_TIME) ||
502 		(!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
503 		return;
504 
505 	/* exceed periodical checkpoint timeout threshold */
506 	if (f2fs_time_over(sbi, CP_TIME))
507 		goto do_sync;
508 
509 	/* checkpoint is the only way to shrink partial cached entries */
510 	if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
511 		f2fs_available_free_memory(sbi, INO_ENTRIES))
512 		return;
513 
514 do_sync:
515 	if (test_opt(sbi, DATA_FLUSH) && from_bg) {
516 		struct blk_plug plug;
517 
518 		mutex_lock(&sbi->flush_lock);
519 
520 		blk_start_plug(&plug);
521 		f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);
522 		blk_finish_plug(&plug);
523 
524 		mutex_unlock(&sbi->flush_lock);
525 	}
526 	stat_inc_cp_call_count(sbi, BACKGROUND);
527 	f2fs_sync_fs(sbi->sb, 1);
528 }
529 
__submit_flush_wait(struct f2fs_sb_info * sbi,struct block_device * bdev)530 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
531 				struct block_device *bdev)
532 {
533 	int ret = blkdev_issue_flush(bdev);
534 
535 	trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
536 				test_opt(sbi, FLUSH_MERGE), ret);
537 	if (!ret)
538 		f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0);
539 	return ret;
540 }
541 
submit_flush_wait(struct f2fs_sb_info * sbi,nid_t ino)542 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
543 {
544 	int ret = 0;
545 	int i;
546 
547 	if (!f2fs_is_multi_device(sbi))
548 		return __submit_flush_wait(sbi, sbi->sb->s_bdev);
549 
550 	for (i = 0; i < sbi->s_ndevs; i++) {
551 		if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
552 			continue;
553 		ret = __submit_flush_wait(sbi, FDEV(i).bdev);
554 		if (ret)
555 			break;
556 	}
557 	return ret;
558 }
559 
issue_flush_thread(void * data)560 static int issue_flush_thread(void *data)
561 {
562 	struct f2fs_sb_info *sbi = data;
563 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
564 	wait_queue_head_t *q = &fcc->flush_wait_queue;
565 repeat:
566 	if (kthread_should_stop())
567 		return 0;
568 
569 	if (!llist_empty(&fcc->issue_list)) {
570 		struct flush_cmd *cmd, *next;
571 		int ret;
572 
573 		fcc->dispatch_list = llist_del_all(&fcc->issue_list);
574 		fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
575 
576 		cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
577 
578 		ret = submit_flush_wait(sbi, cmd->ino);
579 		atomic_inc(&fcc->issued_flush);
580 
581 		llist_for_each_entry_safe(cmd, next,
582 					  fcc->dispatch_list, llnode) {
583 			cmd->ret = ret;
584 			complete(&cmd->wait);
585 		}
586 		fcc->dispatch_list = NULL;
587 	}
588 
589 	wait_event_interruptible(*q,
590 		kthread_should_stop() || !llist_empty(&fcc->issue_list));
591 	goto repeat;
592 }
593 
f2fs_issue_flush(struct f2fs_sb_info * sbi,nid_t ino)594 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
595 {
596 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
597 	struct flush_cmd cmd;
598 	int ret;
599 
600 	if (test_opt(sbi, NOBARRIER))
601 		return 0;
602 
603 	if (!test_opt(sbi, FLUSH_MERGE)) {
604 		atomic_inc(&fcc->queued_flush);
605 		ret = submit_flush_wait(sbi, ino);
606 		atomic_dec(&fcc->queued_flush);
607 		atomic_inc(&fcc->issued_flush);
608 		return ret;
609 	}
610 
611 	if (atomic_inc_return(&fcc->queued_flush) == 1 ||
612 	    f2fs_is_multi_device(sbi)) {
613 		ret = submit_flush_wait(sbi, ino);
614 		atomic_dec(&fcc->queued_flush);
615 
616 		atomic_inc(&fcc->issued_flush);
617 		return ret;
618 	}
619 
620 	cmd.ino = ino;
621 	init_completion(&cmd.wait);
622 
623 	llist_add(&cmd.llnode, &fcc->issue_list);
624 
625 	/*
626 	 * update issue_list before we wake up issue_flush thread, this
627 	 * smp_mb() pairs with another barrier in ___wait_event(), see
628 	 * more details in comments of waitqueue_active().
629 	 */
630 	smp_mb();
631 
632 	if (waitqueue_active(&fcc->flush_wait_queue))
633 		wake_up(&fcc->flush_wait_queue);
634 
635 	if (fcc->f2fs_issue_flush) {
636 		wait_for_completion(&cmd.wait);
637 		atomic_dec(&fcc->queued_flush);
638 	} else {
639 		struct llist_node *list;
640 
641 		list = llist_del_all(&fcc->issue_list);
642 		if (!list) {
643 			wait_for_completion(&cmd.wait);
644 			atomic_dec(&fcc->queued_flush);
645 		} else {
646 			struct flush_cmd *tmp, *next;
647 
648 			ret = submit_flush_wait(sbi, ino);
649 
650 			llist_for_each_entry_safe(tmp, next, list, llnode) {
651 				if (tmp == &cmd) {
652 					cmd.ret = ret;
653 					atomic_dec(&fcc->queued_flush);
654 					continue;
655 				}
656 				tmp->ret = ret;
657 				complete(&tmp->wait);
658 			}
659 		}
660 	}
661 
662 	return cmd.ret;
663 }
664 
f2fs_create_flush_cmd_control(struct f2fs_sb_info * sbi)665 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
666 {
667 	dev_t dev = sbi->sb->s_bdev->bd_dev;
668 	struct flush_cmd_control *fcc;
669 
670 	if (SM_I(sbi)->fcc_info) {
671 		fcc = SM_I(sbi)->fcc_info;
672 		if (fcc->f2fs_issue_flush)
673 			return 0;
674 		goto init_thread;
675 	}
676 
677 	fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
678 	if (!fcc)
679 		return -ENOMEM;
680 	atomic_set(&fcc->issued_flush, 0);
681 	atomic_set(&fcc->queued_flush, 0);
682 	init_waitqueue_head(&fcc->flush_wait_queue);
683 	init_llist_head(&fcc->issue_list);
684 	SM_I(sbi)->fcc_info = fcc;
685 	if (!test_opt(sbi, FLUSH_MERGE))
686 		return 0;
687 
688 init_thread:
689 	fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
690 				"f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
691 	if (IS_ERR(fcc->f2fs_issue_flush)) {
692 		int err = PTR_ERR(fcc->f2fs_issue_flush);
693 
694 		fcc->f2fs_issue_flush = NULL;
695 		return err;
696 	}
697 
698 	return 0;
699 }
700 
f2fs_destroy_flush_cmd_control(struct f2fs_sb_info * sbi,bool free)701 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
702 {
703 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
704 
705 	if (fcc && fcc->f2fs_issue_flush) {
706 		struct task_struct *flush_thread = fcc->f2fs_issue_flush;
707 
708 		fcc->f2fs_issue_flush = NULL;
709 		kthread_stop(flush_thread);
710 	}
711 	if (free) {
712 		kfree(fcc);
713 		SM_I(sbi)->fcc_info = NULL;
714 	}
715 }
716 
f2fs_flush_device_cache(struct f2fs_sb_info * sbi)717 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
718 {
719 	int ret = 0, i;
720 
721 	if (!f2fs_is_multi_device(sbi))
722 		return 0;
723 
724 	if (test_opt(sbi, NOBARRIER))
725 		return 0;
726 
727 	for (i = 1; i < sbi->s_ndevs; i++) {
728 		int count = DEFAULT_RETRY_IO_COUNT;
729 
730 		if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
731 			continue;
732 
733 		do {
734 			ret = __submit_flush_wait(sbi, FDEV(i).bdev);
735 			if (ret)
736 				f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
737 		} while (ret && --count);
738 
739 		if (ret) {
740 			f2fs_stop_checkpoint(sbi, false,
741 					STOP_CP_REASON_FLUSH_FAIL);
742 			break;
743 		}
744 
745 		spin_lock(&sbi->dev_lock);
746 		f2fs_clear_bit(i, (char *)&sbi->dirty_device);
747 		spin_unlock(&sbi->dev_lock);
748 	}
749 
750 	return ret;
751 }
752 
__locate_dirty_segment(struct f2fs_sb_info * sbi,unsigned int segno,enum dirty_type dirty_type)753 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
754 		enum dirty_type dirty_type)
755 {
756 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
757 
758 	/* need not be added */
759 	if (IS_CURSEG(sbi, segno))
760 		return;
761 
762 	if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
763 		dirty_i->nr_dirty[dirty_type]++;
764 
765 	if (dirty_type == DIRTY) {
766 		struct seg_entry *sentry = get_seg_entry(sbi, segno);
767 		enum dirty_type t = sentry->type;
768 
769 		if (unlikely(t >= DIRTY)) {
770 			f2fs_bug_on(sbi, 1);
771 			return;
772 		}
773 		if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
774 			dirty_i->nr_dirty[t]++;
775 
776 		if (__is_large_section(sbi)) {
777 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
778 			block_t valid_blocks =
779 				get_valid_blocks(sbi, segno, true);
780 
781 			f2fs_bug_on(sbi, unlikely(!valid_blocks ||
782 					valid_blocks == CAP_BLKS_PER_SEC(sbi)));
783 
784 			if (!IS_CURSEC(sbi, secno))
785 				set_bit(secno, dirty_i->dirty_secmap);
786 		}
787 	}
788 }
789 
__remove_dirty_segment(struct f2fs_sb_info * sbi,unsigned int segno,enum dirty_type dirty_type)790 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
791 		enum dirty_type dirty_type)
792 {
793 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
794 	block_t valid_blocks;
795 
796 	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
797 		dirty_i->nr_dirty[dirty_type]--;
798 
799 	if (dirty_type == DIRTY) {
800 		struct seg_entry *sentry = get_seg_entry(sbi, segno);
801 		enum dirty_type t = sentry->type;
802 
803 		if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
804 			dirty_i->nr_dirty[t]--;
805 
806 		valid_blocks = get_valid_blocks(sbi, segno, true);
807 		if (valid_blocks == 0) {
808 			clear_bit(GET_SEC_FROM_SEG(sbi, segno),
809 						dirty_i->victim_secmap);
810 #ifdef CONFIG_F2FS_CHECK_FS
811 			clear_bit(segno, SIT_I(sbi)->invalid_segmap);
812 #endif
813 		}
814 		if (__is_large_section(sbi)) {
815 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
816 
817 			if (!valid_blocks ||
818 					valid_blocks == CAP_BLKS_PER_SEC(sbi)) {
819 				clear_bit(secno, dirty_i->dirty_secmap);
820 				return;
821 			}
822 
823 			if (!IS_CURSEC(sbi, secno))
824 				set_bit(secno, dirty_i->dirty_secmap);
825 		}
826 	}
827 }
828 
829 /*
830  * Should not occur error such as -ENOMEM.
831  * Adding dirty entry into seglist is not critical operation.
832  * If a given segment is one of current working segments, it won't be added.
833  */
locate_dirty_segment(struct f2fs_sb_info * sbi,unsigned int segno)834 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
835 {
836 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
837 	unsigned short valid_blocks, ckpt_valid_blocks;
838 	unsigned int usable_blocks;
839 
840 	if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
841 		return;
842 
843 	usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
844 	mutex_lock(&dirty_i->seglist_lock);
845 
846 	valid_blocks = get_valid_blocks(sbi, segno, false);
847 	ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
848 
849 	if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
850 		ckpt_valid_blocks == usable_blocks)) {
851 		__locate_dirty_segment(sbi, segno, PRE);
852 		__remove_dirty_segment(sbi, segno, DIRTY);
853 	} else if (valid_blocks < usable_blocks) {
854 		__locate_dirty_segment(sbi, segno, DIRTY);
855 	} else {
856 		/* Recovery routine with SSR needs this */
857 		__remove_dirty_segment(sbi, segno, DIRTY);
858 	}
859 
860 	mutex_unlock(&dirty_i->seglist_lock);
861 }
862 
863 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
f2fs_dirty_to_prefree(struct f2fs_sb_info * sbi)864 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
865 {
866 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
867 	unsigned int segno;
868 
869 	mutex_lock(&dirty_i->seglist_lock);
870 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
871 		if (get_valid_blocks(sbi, segno, false))
872 			continue;
873 		if (IS_CURSEG(sbi, segno))
874 			continue;
875 		__locate_dirty_segment(sbi, segno, PRE);
876 		__remove_dirty_segment(sbi, segno, DIRTY);
877 	}
878 	mutex_unlock(&dirty_i->seglist_lock);
879 }
880 
f2fs_get_unusable_blocks(struct f2fs_sb_info * sbi)881 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
882 {
883 	int ovp_hole_segs =
884 		(overprovision_segments(sbi) - reserved_segments(sbi));
885 	block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
886 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
887 	block_t holes[2] = {0, 0};	/* DATA and NODE */
888 	block_t unusable;
889 	struct seg_entry *se;
890 	unsigned int segno;
891 
892 	mutex_lock(&dirty_i->seglist_lock);
893 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
894 		se = get_seg_entry(sbi, segno);
895 		if (IS_NODESEG(se->type))
896 			holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
897 							se->valid_blocks;
898 		else
899 			holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
900 							se->valid_blocks;
901 	}
902 	mutex_unlock(&dirty_i->seglist_lock);
903 
904 	unusable = max(holes[DATA], holes[NODE]);
905 	if (unusable > ovp_holes)
906 		return unusable - ovp_holes;
907 	return 0;
908 }
909 
f2fs_disable_cp_again(struct f2fs_sb_info * sbi,block_t unusable)910 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
911 {
912 	int ovp_hole_segs =
913 		(overprovision_segments(sbi) - reserved_segments(sbi));
914 	if (unusable > F2FS_OPTION(sbi).unusable_cap)
915 		return -EAGAIN;
916 	if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
917 		dirty_segments(sbi) > ovp_hole_segs)
918 		return -EAGAIN;
919 	return 0;
920 }
921 
922 /* This is only used by SBI_CP_DISABLED */
get_free_segment(struct f2fs_sb_info * sbi)923 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
924 {
925 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
926 	unsigned int segno = 0;
927 
928 	mutex_lock(&dirty_i->seglist_lock);
929 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
930 		if (get_valid_blocks(sbi, segno, false))
931 			continue;
932 		if (get_ckpt_valid_blocks(sbi, segno, false))
933 			continue;
934 		mutex_unlock(&dirty_i->seglist_lock);
935 		return segno;
936 	}
937 	mutex_unlock(&dirty_i->seglist_lock);
938 	return NULL_SEGNO;
939 }
940 
__create_discard_cmd(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t lstart,block_t start,block_t len)941 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
942 		struct block_device *bdev, block_t lstart,
943 		block_t start, block_t len)
944 {
945 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
946 	struct list_head *pend_list;
947 	struct discard_cmd *dc;
948 
949 	f2fs_bug_on(sbi, !len);
950 
951 	pend_list = &dcc->pend_list[plist_idx(len)];
952 
953 	dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
954 	INIT_LIST_HEAD(&dc->list);
955 	dc->bdev = bdev;
956 	dc->di.lstart = lstart;
957 	dc->di.start = start;
958 	dc->di.len = len;
959 	dc->ref = 0;
960 	dc->state = D_PREP;
961 	dc->queued = 0;
962 	dc->error = 0;
963 	init_completion(&dc->wait);
964 	list_add_tail(&dc->list, pend_list);
965 	spin_lock_init(&dc->lock);
966 	dc->bio_ref = 0;
967 	atomic_inc(&dcc->discard_cmd_cnt);
968 	dcc->undiscard_blks += len;
969 
970 	return dc;
971 }
972 
f2fs_check_discard_tree(struct f2fs_sb_info * sbi)973 static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi)
974 {
975 #ifdef CONFIG_F2FS_CHECK_FS
976 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
977 	struct rb_node *cur = rb_first_cached(&dcc->root), *next;
978 	struct discard_cmd *cur_dc, *next_dc;
979 
980 	while (cur) {
981 		next = rb_next(cur);
982 		if (!next)
983 			return true;
984 
985 		cur_dc = rb_entry(cur, struct discard_cmd, rb_node);
986 		next_dc = rb_entry(next, struct discard_cmd, rb_node);
987 
988 		if (cur_dc->di.lstart + cur_dc->di.len > next_dc->di.lstart) {
989 			f2fs_info(sbi, "broken discard_rbtree, "
990 				"cur(%u, %u) next(%u, %u)",
991 				cur_dc->di.lstart, cur_dc->di.len,
992 				next_dc->di.lstart, next_dc->di.len);
993 			return false;
994 		}
995 		cur = next;
996 	}
997 #endif
998 	return true;
999 }
1000 
__lookup_discard_cmd(struct f2fs_sb_info * sbi,block_t blkaddr)1001 static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi,
1002 						block_t blkaddr)
1003 {
1004 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1005 	struct rb_node *node = dcc->root.rb_root.rb_node;
1006 	struct discard_cmd *dc;
1007 
1008 	while (node) {
1009 		dc = rb_entry(node, struct discard_cmd, rb_node);
1010 
1011 		if (blkaddr < dc->di.lstart)
1012 			node = node->rb_left;
1013 		else if (blkaddr >= dc->di.lstart + dc->di.len)
1014 			node = node->rb_right;
1015 		else
1016 			return dc;
1017 	}
1018 	return NULL;
1019 }
1020 
__lookup_discard_cmd_ret(struct rb_root_cached * root,block_t blkaddr,struct discard_cmd ** prev_entry,struct discard_cmd ** next_entry,struct rb_node *** insert_p,struct rb_node ** insert_parent)1021 static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root,
1022 				block_t blkaddr,
1023 				struct discard_cmd **prev_entry,
1024 				struct discard_cmd **next_entry,
1025 				struct rb_node ***insert_p,
1026 				struct rb_node **insert_parent)
1027 {
1028 	struct rb_node **pnode = &root->rb_root.rb_node;
1029 	struct rb_node *parent = NULL, *tmp_node;
1030 	struct discard_cmd *dc;
1031 
1032 	*insert_p = NULL;
1033 	*insert_parent = NULL;
1034 	*prev_entry = NULL;
1035 	*next_entry = NULL;
1036 
1037 	if (RB_EMPTY_ROOT(&root->rb_root))
1038 		return NULL;
1039 
1040 	while (*pnode) {
1041 		parent = *pnode;
1042 		dc = rb_entry(*pnode, struct discard_cmd, rb_node);
1043 
1044 		if (blkaddr < dc->di.lstart)
1045 			pnode = &(*pnode)->rb_left;
1046 		else if (blkaddr >= dc->di.lstart + dc->di.len)
1047 			pnode = &(*pnode)->rb_right;
1048 		else
1049 			goto lookup_neighbors;
1050 	}
1051 
1052 	*insert_p = pnode;
1053 	*insert_parent = parent;
1054 
1055 	dc = rb_entry(parent, struct discard_cmd, rb_node);
1056 	tmp_node = parent;
1057 	if (parent && blkaddr > dc->di.lstart)
1058 		tmp_node = rb_next(parent);
1059 	*next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1060 
1061 	tmp_node = parent;
1062 	if (parent && blkaddr < dc->di.lstart)
1063 		tmp_node = rb_prev(parent);
1064 	*prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1065 	return NULL;
1066 
1067 lookup_neighbors:
1068 	/* lookup prev node for merging backward later */
1069 	tmp_node = rb_prev(&dc->rb_node);
1070 	*prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1071 
1072 	/* lookup next node for merging frontward later */
1073 	tmp_node = rb_next(&dc->rb_node);
1074 	*next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1075 	return dc;
1076 }
1077 
__detach_discard_cmd(struct discard_cmd_control * dcc,struct discard_cmd * dc)1078 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1079 							struct discard_cmd *dc)
1080 {
1081 	if (dc->state == D_DONE)
1082 		atomic_sub(dc->queued, &dcc->queued_discard);
1083 
1084 	list_del(&dc->list);
1085 	rb_erase_cached(&dc->rb_node, &dcc->root);
1086 	dcc->undiscard_blks -= dc->di.len;
1087 
1088 	kmem_cache_free(discard_cmd_slab, dc);
1089 
1090 	atomic_dec(&dcc->discard_cmd_cnt);
1091 }
1092 
__remove_discard_cmd(struct f2fs_sb_info * sbi,struct discard_cmd * dc)1093 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1094 							struct discard_cmd *dc)
1095 {
1096 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1097 	unsigned long flags;
1098 
1099 	trace_f2fs_remove_discard(dc->bdev, dc->di.start, dc->di.len);
1100 
1101 	spin_lock_irqsave(&dc->lock, flags);
1102 	if (dc->bio_ref) {
1103 		spin_unlock_irqrestore(&dc->lock, flags);
1104 		return;
1105 	}
1106 	spin_unlock_irqrestore(&dc->lock, flags);
1107 
1108 	f2fs_bug_on(sbi, dc->ref);
1109 
1110 	if (dc->error == -EOPNOTSUPP)
1111 		dc->error = 0;
1112 
1113 	if (dc->error)
1114 		f2fs_info_ratelimited(sbi,
1115 			"Issue discard(%u, %u, %u) failed, ret: %d",
1116 			dc->di.lstart, dc->di.start, dc->di.len, dc->error);
1117 	__detach_discard_cmd(dcc, dc);
1118 }
1119 
f2fs_submit_discard_endio(struct bio * bio)1120 static void f2fs_submit_discard_endio(struct bio *bio)
1121 {
1122 	struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1123 	unsigned long flags;
1124 
1125 	spin_lock_irqsave(&dc->lock, flags);
1126 	if (!dc->error)
1127 		dc->error = blk_status_to_errno(bio->bi_status);
1128 	dc->bio_ref--;
1129 	if (!dc->bio_ref && dc->state == D_SUBMIT) {
1130 		dc->state = D_DONE;
1131 		complete_all(&dc->wait);
1132 	}
1133 	spin_unlock_irqrestore(&dc->lock, flags);
1134 	bio_put(bio);
1135 }
1136 
__check_sit_bitmap(struct f2fs_sb_info * sbi,block_t start,block_t end)1137 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1138 				block_t start, block_t end)
1139 {
1140 #ifdef CONFIG_F2FS_CHECK_FS
1141 	struct seg_entry *sentry;
1142 	unsigned int segno;
1143 	block_t blk = start;
1144 	unsigned long offset, size, *map;
1145 
1146 	while (blk < end) {
1147 		segno = GET_SEGNO(sbi, blk);
1148 		sentry = get_seg_entry(sbi, segno);
1149 		offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1150 
1151 		if (end < START_BLOCK(sbi, segno + 1))
1152 			size = GET_BLKOFF_FROM_SEG0(sbi, end);
1153 		else
1154 			size = BLKS_PER_SEG(sbi);
1155 		map = (unsigned long *)(sentry->cur_valid_map);
1156 		offset = __find_rev_next_bit(map, size, offset);
1157 		f2fs_bug_on(sbi, offset != size);
1158 		blk = START_BLOCK(sbi, segno + 1);
1159 	}
1160 #endif
1161 }
1162 
__init_discard_policy(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,int discard_type,unsigned int granularity)1163 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1164 				struct discard_policy *dpolicy,
1165 				int discard_type, unsigned int granularity)
1166 {
1167 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1168 
1169 	/* common policy */
1170 	dpolicy->type = discard_type;
1171 	dpolicy->sync = true;
1172 	dpolicy->ordered = false;
1173 	dpolicy->granularity = granularity;
1174 
1175 	dpolicy->max_requests = dcc->max_discard_request;
1176 	dpolicy->io_aware_gran = dcc->discard_io_aware_gran;
1177 	dpolicy->timeout = false;
1178 
1179 	if (discard_type == DPOLICY_BG) {
1180 		dpolicy->min_interval = dcc->min_discard_issue_time;
1181 		dpolicy->mid_interval = dcc->mid_discard_issue_time;
1182 		dpolicy->max_interval = dcc->max_discard_issue_time;
1183 		dpolicy->io_aware = true;
1184 		dpolicy->sync = false;
1185 		dpolicy->ordered = true;
1186 		if (utilization(sbi) > dcc->discard_urgent_util) {
1187 			dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1188 			if (atomic_read(&dcc->discard_cmd_cnt))
1189 				dpolicy->max_interval =
1190 					dcc->min_discard_issue_time;
1191 		}
1192 	} else if (discard_type == DPOLICY_FORCE) {
1193 		dpolicy->min_interval = dcc->min_discard_issue_time;
1194 		dpolicy->mid_interval = dcc->mid_discard_issue_time;
1195 		dpolicy->max_interval = dcc->max_discard_issue_time;
1196 		dpolicy->io_aware = false;
1197 	} else if (discard_type == DPOLICY_FSTRIM) {
1198 		dpolicy->io_aware = false;
1199 	} else if (discard_type == DPOLICY_UMOUNT) {
1200 		dpolicy->io_aware = false;
1201 		/* we need to issue all to keep CP_TRIMMED_FLAG */
1202 		dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1203 		dpolicy->timeout = true;
1204 	}
1205 }
1206 
1207 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1208 				struct block_device *bdev, block_t lstart,
1209 				block_t start, block_t len);
1210 
1211 #ifdef CONFIG_BLK_DEV_ZONED
__submit_zone_reset_cmd(struct f2fs_sb_info * sbi,struct discard_cmd * dc,blk_opf_t flag,struct list_head * wait_list,unsigned int * issued)1212 static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi,
1213 				   struct discard_cmd *dc, blk_opf_t flag,
1214 				   struct list_head *wait_list,
1215 				   unsigned int *issued)
1216 {
1217 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1218 	struct block_device *bdev = dc->bdev;
1219 	struct bio *bio = bio_alloc(bdev, 0, REQ_OP_ZONE_RESET | flag, GFP_NOFS);
1220 	unsigned long flags;
1221 
1222 	trace_f2fs_issue_reset_zone(bdev, dc->di.start);
1223 
1224 	spin_lock_irqsave(&dc->lock, flags);
1225 	dc->state = D_SUBMIT;
1226 	dc->bio_ref++;
1227 	spin_unlock_irqrestore(&dc->lock, flags);
1228 
1229 	if (issued)
1230 		(*issued)++;
1231 
1232 	atomic_inc(&dcc->queued_discard);
1233 	dc->queued++;
1234 	list_move_tail(&dc->list, wait_list);
1235 
1236 	/* sanity check on discard range */
1237 	__check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len);
1238 
1239 	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(dc->di.start);
1240 	bio->bi_private = dc;
1241 	bio->bi_end_io = f2fs_submit_discard_endio;
1242 	submit_bio(bio);
1243 
1244 	atomic_inc(&dcc->issued_discard);
1245 	f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE);
1246 }
1247 #endif
1248 
1249 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
__submit_discard_cmd(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,struct discard_cmd * dc,int * issued)1250 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1251 				struct discard_policy *dpolicy,
1252 				struct discard_cmd *dc, int *issued)
1253 {
1254 	struct block_device *bdev = dc->bdev;
1255 	unsigned int max_discard_blocks =
1256 			SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1257 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1258 	struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1259 					&(dcc->fstrim_list) : &(dcc->wait_list);
1260 	blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0;
1261 	block_t lstart, start, len, total_len;
1262 	int err = 0;
1263 
1264 	if (dc->state != D_PREP)
1265 		return 0;
1266 
1267 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1268 		return 0;
1269 
1270 #ifdef CONFIG_BLK_DEV_ZONED
1271 	if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) {
1272 		int devi = f2fs_bdev_index(sbi, bdev);
1273 
1274 		if (devi < 0)
1275 			return -EINVAL;
1276 
1277 		if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {
1278 			__submit_zone_reset_cmd(sbi, dc, flag,
1279 						wait_list, issued);
1280 			return 0;
1281 		}
1282 	}
1283 #endif
1284 
1285 	trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len);
1286 
1287 	lstart = dc->di.lstart;
1288 	start = dc->di.start;
1289 	len = dc->di.len;
1290 	total_len = len;
1291 
1292 	dc->di.len = 0;
1293 
1294 	while (total_len && *issued < dpolicy->max_requests && !err) {
1295 		struct bio *bio = NULL;
1296 		unsigned long flags;
1297 		bool last = true;
1298 
1299 		if (len > max_discard_blocks) {
1300 			len = max_discard_blocks;
1301 			last = false;
1302 		}
1303 
1304 		(*issued)++;
1305 		if (*issued == dpolicy->max_requests)
1306 			last = true;
1307 
1308 		dc->di.len += len;
1309 
1310 		if (time_to_inject(sbi, FAULT_DISCARD)) {
1311 			err = -EIO;
1312 		} else {
1313 			err = __blkdev_issue_discard(bdev,
1314 					SECTOR_FROM_BLOCK(start),
1315 					SECTOR_FROM_BLOCK(len),
1316 					GFP_NOFS, &bio);
1317 		}
1318 		if (err) {
1319 			spin_lock_irqsave(&dc->lock, flags);
1320 			if (dc->state == D_PARTIAL)
1321 				dc->state = D_SUBMIT;
1322 			spin_unlock_irqrestore(&dc->lock, flags);
1323 
1324 			break;
1325 		}
1326 
1327 		f2fs_bug_on(sbi, !bio);
1328 
1329 		/*
1330 		 * should keep before submission to avoid D_DONE
1331 		 * right away
1332 		 */
1333 		spin_lock_irqsave(&dc->lock, flags);
1334 		if (last)
1335 			dc->state = D_SUBMIT;
1336 		else
1337 			dc->state = D_PARTIAL;
1338 		dc->bio_ref++;
1339 		spin_unlock_irqrestore(&dc->lock, flags);
1340 
1341 		atomic_inc(&dcc->queued_discard);
1342 		dc->queued++;
1343 		list_move_tail(&dc->list, wait_list);
1344 
1345 		/* sanity check on discard range */
1346 		__check_sit_bitmap(sbi, lstart, lstart + len);
1347 
1348 		bio->bi_private = dc;
1349 		bio->bi_end_io = f2fs_submit_discard_endio;
1350 		bio->bi_opf |= flag;
1351 		submit_bio(bio);
1352 
1353 		atomic_inc(&dcc->issued_discard);
1354 
1355 		f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE);
1356 
1357 		lstart += len;
1358 		start += len;
1359 		total_len -= len;
1360 		len = total_len;
1361 	}
1362 
1363 	if (!err && len) {
1364 		dcc->undiscard_blks -= len;
1365 		__update_discard_tree_range(sbi, bdev, lstart, start, len);
1366 	}
1367 	return err;
1368 }
1369 
__insert_discard_cmd(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t lstart,block_t start,block_t len)1370 static void __insert_discard_cmd(struct f2fs_sb_info *sbi,
1371 				struct block_device *bdev, block_t lstart,
1372 				block_t start, block_t len)
1373 {
1374 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1375 	struct rb_node **p = &dcc->root.rb_root.rb_node;
1376 	struct rb_node *parent = NULL;
1377 	struct discard_cmd *dc;
1378 	bool leftmost = true;
1379 
1380 	/* look up rb tree to find parent node */
1381 	while (*p) {
1382 		parent = *p;
1383 		dc = rb_entry(parent, struct discard_cmd, rb_node);
1384 
1385 		if (lstart < dc->di.lstart) {
1386 			p = &(*p)->rb_left;
1387 		} else if (lstart >= dc->di.lstart + dc->di.len) {
1388 			p = &(*p)->rb_right;
1389 			leftmost = false;
1390 		} else {
1391 			f2fs_bug_on(sbi, 1);
1392 		}
1393 	}
1394 
1395 	dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1396 
1397 	rb_link_node(&dc->rb_node, parent, p);
1398 	rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1399 }
1400 
__relocate_discard_cmd(struct discard_cmd_control * dcc,struct discard_cmd * dc)1401 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1402 						struct discard_cmd *dc)
1403 {
1404 	list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->di.len)]);
1405 }
1406 
__punch_discard_cmd(struct f2fs_sb_info * sbi,struct discard_cmd * dc,block_t blkaddr)1407 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1408 				struct discard_cmd *dc, block_t blkaddr)
1409 {
1410 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1411 	struct discard_info di = dc->di;
1412 	bool modified = false;
1413 
1414 	if (dc->state == D_DONE || dc->di.len == 1) {
1415 		__remove_discard_cmd(sbi, dc);
1416 		return;
1417 	}
1418 
1419 	dcc->undiscard_blks -= di.len;
1420 
1421 	if (blkaddr > di.lstart) {
1422 		dc->di.len = blkaddr - dc->di.lstart;
1423 		dcc->undiscard_blks += dc->di.len;
1424 		__relocate_discard_cmd(dcc, dc);
1425 		modified = true;
1426 	}
1427 
1428 	if (blkaddr < di.lstart + di.len - 1) {
1429 		if (modified) {
1430 			__insert_discard_cmd(sbi, dc->bdev, blkaddr + 1,
1431 					di.start + blkaddr + 1 - di.lstart,
1432 					di.lstart + di.len - 1 - blkaddr);
1433 		} else {
1434 			dc->di.lstart++;
1435 			dc->di.len--;
1436 			dc->di.start++;
1437 			dcc->undiscard_blks += dc->di.len;
1438 			__relocate_discard_cmd(dcc, dc);
1439 		}
1440 	}
1441 }
1442 
__update_discard_tree_range(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t lstart,block_t start,block_t len)1443 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1444 				struct block_device *bdev, block_t lstart,
1445 				block_t start, block_t len)
1446 {
1447 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1448 	struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1449 	struct discard_cmd *dc;
1450 	struct discard_info di = {0};
1451 	struct rb_node **insert_p = NULL, *insert_parent = NULL;
1452 	unsigned int max_discard_blocks =
1453 			SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1454 	block_t end = lstart + len;
1455 
1456 	dc = __lookup_discard_cmd_ret(&dcc->root, lstart,
1457 				&prev_dc, &next_dc, &insert_p, &insert_parent);
1458 	if (dc)
1459 		prev_dc = dc;
1460 
1461 	if (!prev_dc) {
1462 		di.lstart = lstart;
1463 		di.len = next_dc ? next_dc->di.lstart - lstart : len;
1464 		di.len = min(di.len, len);
1465 		di.start = start;
1466 	}
1467 
1468 	while (1) {
1469 		struct rb_node *node;
1470 		bool merged = false;
1471 		struct discard_cmd *tdc = NULL;
1472 
1473 		if (prev_dc) {
1474 			di.lstart = prev_dc->di.lstart + prev_dc->di.len;
1475 			if (di.lstart < lstart)
1476 				di.lstart = lstart;
1477 			if (di.lstart >= end)
1478 				break;
1479 
1480 			if (!next_dc || next_dc->di.lstart > end)
1481 				di.len = end - di.lstart;
1482 			else
1483 				di.len = next_dc->di.lstart - di.lstart;
1484 			di.start = start + di.lstart - lstart;
1485 		}
1486 
1487 		if (!di.len)
1488 			goto next;
1489 
1490 		if (prev_dc && prev_dc->state == D_PREP &&
1491 			prev_dc->bdev == bdev &&
1492 			__is_discard_back_mergeable(&di, &prev_dc->di,
1493 							max_discard_blocks)) {
1494 			prev_dc->di.len += di.len;
1495 			dcc->undiscard_blks += di.len;
1496 			__relocate_discard_cmd(dcc, prev_dc);
1497 			di = prev_dc->di;
1498 			tdc = prev_dc;
1499 			merged = true;
1500 		}
1501 
1502 		if (next_dc && next_dc->state == D_PREP &&
1503 			next_dc->bdev == bdev &&
1504 			__is_discard_front_mergeable(&di, &next_dc->di,
1505 							max_discard_blocks)) {
1506 			next_dc->di.lstart = di.lstart;
1507 			next_dc->di.len += di.len;
1508 			next_dc->di.start = di.start;
1509 			dcc->undiscard_blks += di.len;
1510 			__relocate_discard_cmd(dcc, next_dc);
1511 			if (tdc)
1512 				__remove_discard_cmd(sbi, tdc);
1513 			merged = true;
1514 		}
1515 
1516 		if (!merged)
1517 			__insert_discard_cmd(sbi, bdev,
1518 						di.lstart, di.start, di.len);
1519  next:
1520 		prev_dc = next_dc;
1521 		if (!prev_dc)
1522 			break;
1523 
1524 		node = rb_next(&prev_dc->rb_node);
1525 		next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1526 	}
1527 }
1528 
1529 #ifdef CONFIG_BLK_DEV_ZONED
__queue_zone_reset_cmd(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t blkstart,block_t lblkstart,block_t blklen)1530 static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi,
1531 		struct block_device *bdev, block_t blkstart, block_t lblkstart,
1532 		block_t blklen)
1533 {
1534 	trace_f2fs_queue_reset_zone(bdev, blkstart);
1535 
1536 	mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1537 	__insert_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen);
1538 	mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1539 }
1540 #endif
1541 
__queue_discard_cmd(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t blkstart,block_t blklen)1542 static void __queue_discard_cmd(struct f2fs_sb_info *sbi,
1543 		struct block_device *bdev, block_t blkstart, block_t blklen)
1544 {
1545 	block_t lblkstart = blkstart;
1546 
1547 	if (!f2fs_bdev_support_discard(bdev))
1548 		return;
1549 
1550 	trace_f2fs_queue_discard(bdev, blkstart, blklen);
1551 
1552 	if (f2fs_is_multi_device(sbi)) {
1553 		int devi = f2fs_target_device_index(sbi, blkstart);
1554 
1555 		blkstart -= FDEV(devi).start_blk;
1556 	}
1557 	mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1558 	__update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1559 	mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1560 }
1561 
__issue_discard_cmd_orderly(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,int * issued)1562 static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1563 		struct discard_policy *dpolicy, int *issued)
1564 {
1565 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1566 	struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1567 	struct rb_node **insert_p = NULL, *insert_parent = NULL;
1568 	struct discard_cmd *dc;
1569 	struct blk_plug plug;
1570 	bool io_interrupted = false;
1571 
1572 	mutex_lock(&dcc->cmd_lock);
1573 	dc = __lookup_discard_cmd_ret(&dcc->root, dcc->next_pos,
1574 				&prev_dc, &next_dc, &insert_p, &insert_parent);
1575 	if (!dc)
1576 		dc = next_dc;
1577 
1578 	blk_start_plug(&plug);
1579 
1580 	while (dc) {
1581 		struct rb_node *node;
1582 		int err = 0;
1583 
1584 		if (dc->state != D_PREP)
1585 			goto next;
1586 
1587 		if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1588 			io_interrupted = true;
1589 			break;
1590 		}
1591 
1592 		dcc->next_pos = dc->di.lstart + dc->di.len;
1593 		err = __submit_discard_cmd(sbi, dpolicy, dc, issued);
1594 
1595 		if (*issued >= dpolicy->max_requests)
1596 			break;
1597 next:
1598 		node = rb_next(&dc->rb_node);
1599 		if (err)
1600 			__remove_discard_cmd(sbi, dc);
1601 		dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1602 	}
1603 
1604 	blk_finish_plug(&plug);
1605 
1606 	if (!dc)
1607 		dcc->next_pos = 0;
1608 
1609 	mutex_unlock(&dcc->cmd_lock);
1610 
1611 	if (!(*issued) && io_interrupted)
1612 		*issued = -1;
1613 }
1614 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1615 					struct discard_policy *dpolicy);
1616 
__issue_discard_cmd(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy)1617 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1618 					struct discard_policy *dpolicy)
1619 {
1620 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1621 	struct list_head *pend_list;
1622 	struct discard_cmd *dc, *tmp;
1623 	struct blk_plug plug;
1624 	int i, issued;
1625 	bool io_interrupted = false;
1626 
1627 	if (dpolicy->timeout)
1628 		f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1629 
1630 retry:
1631 	issued = 0;
1632 	for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1633 		if (dpolicy->timeout &&
1634 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1635 			break;
1636 
1637 		if (i + 1 < dpolicy->granularity)
1638 			break;
1639 
1640 		if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) {
1641 			__issue_discard_cmd_orderly(sbi, dpolicy, &issued);
1642 			return issued;
1643 		}
1644 
1645 		pend_list = &dcc->pend_list[i];
1646 
1647 		mutex_lock(&dcc->cmd_lock);
1648 		if (list_empty(pend_list))
1649 			goto next;
1650 		if (unlikely(dcc->rbtree_check))
1651 			f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
1652 		blk_start_plug(&plug);
1653 		list_for_each_entry_safe(dc, tmp, pend_list, list) {
1654 			f2fs_bug_on(sbi, dc->state != D_PREP);
1655 
1656 			if (dpolicy->timeout &&
1657 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1658 				break;
1659 
1660 			if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1661 						!is_idle(sbi, DISCARD_TIME)) {
1662 				io_interrupted = true;
1663 				break;
1664 			}
1665 
1666 			__submit_discard_cmd(sbi, dpolicy, dc, &issued);
1667 
1668 			if (issued >= dpolicy->max_requests)
1669 				break;
1670 		}
1671 		blk_finish_plug(&plug);
1672 next:
1673 		mutex_unlock(&dcc->cmd_lock);
1674 
1675 		if (issued >= dpolicy->max_requests || io_interrupted)
1676 			break;
1677 	}
1678 
1679 	if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1680 		__wait_all_discard_cmd(sbi, dpolicy);
1681 		goto retry;
1682 	}
1683 
1684 	if (!issued && io_interrupted)
1685 		issued = -1;
1686 
1687 	return issued;
1688 }
1689 
__drop_discard_cmd(struct f2fs_sb_info * sbi)1690 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1691 {
1692 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1693 	struct list_head *pend_list;
1694 	struct discard_cmd *dc, *tmp;
1695 	int i;
1696 	bool dropped = false;
1697 
1698 	mutex_lock(&dcc->cmd_lock);
1699 	for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1700 		pend_list = &dcc->pend_list[i];
1701 		list_for_each_entry_safe(dc, tmp, pend_list, list) {
1702 			f2fs_bug_on(sbi, dc->state != D_PREP);
1703 			__remove_discard_cmd(sbi, dc);
1704 			dropped = true;
1705 		}
1706 	}
1707 	mutex_unlock(&dcc->cmd_lock);
1708 
1709 	return dropped;
1710 }
1711 
f2fs_drop_discard_cmd(struct f2fs_sb_info * sbi)1712 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1713 {
1714 	__drop_discard_cmd(sbi);
1715 }
1716 
__wait_one_discard_bio(struct f2fs_sb_info * sbi,struct discard_cmd * dc)1717 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1718 							struct discard_cmd *dc)
1719 {
1720 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1721 	unsigned int len = 0;
1722 
1723 	wait_for_completion_io(&dc->wait);
1724 	mutex_lock(&dcc->cmd_lock);
1725 	f2fs_bug_on(sbi, dc->state != D_DONE);
1726 	dc->ref--;
1727 	if (!dc->ref) {
1728 		if (!dc->error)
1729 			len = dc->di.len;
1730 		__remove_discard_cmd(sbi, dc);
1731 	}
1732 	mutex_unlock(&dcc->cmd_lock);
1733 
1734 	return len;
1735 }
1736 
__wait_discard_cmd_range(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,block_t start,block_t end)1737 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1738 						struct discard_policy *dpolicy,
1739 						block_t start, block_t end)
1740 {
1741 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1742 	struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1743 					&(dcc->fstrim_list) : &(dcc->wait_list);
1744 	struct discard_cmd *dc = NULL, *iter, *tmp;
1745 	unsigned int trimmed = 0;
1746 
1747 next:
1748 	dc = NULL;
1749 
1750 	mutex_lock(&dcc->cmd_lock);
1751 	list_for_each_entry_safe(iter, tmp, wait_list, list) {
1752 		if (iter->di.lstart + iter->di.len <= start ||
1753 					end <= iter->di.lstart)
1754 			continue;
1755 		if (iter->di.len < dpolicy->granularity)
1756 			continue;
1757 		if (iter->state == D_DONE && !iter->ref) {
1758 			wait_for_completion_io(&iter->wait);
1759 			if (!iter->error)
1760 				trimmed += iter->di.len;
1761 			__remove_discard_cmd(sbi, iter);
1762 		} else {
1763 			iter->ref++;
1764 			dc = iter;
1765 			break;
1766 		}
1767 	}
1768 	mutex_unlock(&dcc->cmd_lock);
1769 
1770 	if (dc) {
1771 		trimmed += __wait_one_discard_bio(sbi, dc);
1772 		goto next;
1773 	}
1774 
1775 	return trimmed;
1776 }
1777 
__wait_all_discard_cmd(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy)1778 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1779 						struct discard_policy *dpolicy)
1780 {
1781 	struct discard_policy dp;
1782 	unsigned int discard_blks;
1783 
1784 	if (dpolicy)
1785 		return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1786 
1787 	/* wait all */
1788 	__init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY);
1789 	discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1790 	__init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY);
1791 	discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1792 
1793 	return discard_blks;
1794 }
1795 
1796 /* This should be covered by global mutex, &sit_i->sentry_lock */
f2fs_wait_discard_bio(struct f2fs_sb_info * sbi,block_t blkaddr)1797 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1798 {
1799 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1800 	struct discard_cmd *dc;
1801 	bool need_wait = false;
1802 
1803 	mutex_lock(&dcc->cmd_lock);
1804 	dc = __lookup_discard_cmd(sbi, blkaddr);
1805 #ifdef CONFIG_BLK_DEV_ZONED
1806 	if (dc && f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(dc->bdev)) {
1807 		int devi = f2fs_bdev_index(sbi, dc->bdev);
1808 
1809 		if (devi < 0) {
1810 			mutex_unlock(&dcc->cmd_lock);
1811 			return;
1812 		}
1813 
1814 		if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {
1815 			/* force submit zone reset */
1816 			if (dc->state == D_PREP)
1817 				__submit_zone_reset_cmd(sbi, dc, REQ_SYNC,
1818 							&dcc->wait_list, NULL);
1819 			dc->ref++;
1820 			mutex_unlock(&dcc->cmd_lock);
1821 			/* wait zone reset */
1822 			__wait_one_discard_bio(sbi, dc);
1823 			return;
1824 		}
1825 	}
1826 #endif
1827 	if (dc) {
1828 		if (dc->state == D_PREP) {
1829 			__punch_discard_cmd(sbi, dc, blkaddr);
1830 		} else {
1831 			dc->ref++;
1832 			need_wait = true;
1833 		}
1834 	}
1835 	mutex_unlock(&dcc->cmd_lock);
1836 
1837 	if (need_wait)
1838 		__wait_one_discard_bio(sbi, dc);
1839 }
1840 
f2fs_stop_discard_thread(struct f2fs_sb_info * sbi)1841 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1842 {
1843 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1844 
1845 	if (dcc && dcc->f2fs_issue_discard) {
1846 		struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1847 
1848 		dcc->f2fs_issue_discard = NULL;
1849 		kthread_stop(discard_thread);
1850 	}
1851 }
1852 
1853 /**
1854  * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT
1855  * @sbi: the f2fs_sb_info data for discard cmd to issue
1856  *
1857  * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped
1858  *
1859  * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false.
1860  */
f2fs_issue_discard_timeout(struct f2fs_sb_info * sbi)1861 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1862 {
1863 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1864 	struct discard_policy dpolicy;
1865 	bool dropped;
1866 
1867 	if (!atomic_read(&dcc->discard_cmd_cnt))
1868 		return true;
1869 
1870 	__init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1871 					dcc->discard_granularity);
1872 	__issue_discard_cmd(sbi, &dpolicy);
1873 	dropped = __drop_discard_cmd(sbi);
1874 
1875 	/* just to make sure there is no pending discard commands */
1876 	__wait_all_discard_cmd(sbi, NULL);
1877 
1878 	f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1879 	return !dropped;
1880 }
1881 
issue_discard_thread(void * data)1882 static int issue_discard_thread(void *data)
1883 {
1884 	struct f2fs_sb_info *sbi = data;
1885 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1886 	wait_queue_head_t *q = &dcc->discard_wait_queue;
1887 	struct discard_policy dpolicy;
1888 	unsigned int wait_ms = dcc->min_discard_issue_time;
1889 	int issued;
1890 
1891 	set_freezable();
1892 
1893 	do {
1894 		wait_event_interruptible_timeout(*q,
1895 				kthread_should_stop() || freezing(current) ||
1896 				dcc->discard_wake,
1897 				msecs_to_jiffies(wait_ms));
1898 
1899 		if (sbi->gc_mode == GC_URGENT_HIGH ||
1900 			!f2fs_available_free_memory(sbi, DISCARD_CACHE))
1901 			__init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE,
1902 						MIN_DISCARD_GRANULARITY);
1903 		else
1904 			__init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1905 						dcc->discard_granularity);
1906 
1907 		if (dcc->discard_wake)
1908 			dcc->discard_wake = false;
1909 
1910 		/* clean up pending candidates before going to sleep */
1911 		if (atomic_read(&dcc->queued_discard))
1912 			__wait_all_discard_cmd(sbi, NULL);
1913 
1914 		if (try_to_freeze())
1915 			continue;
1916 		if (f2fs_readonly(sbi->sb))
1917 			continue;
1918 		if (kthread_should_stop())
1919 			return 0;
1920 		if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) ||
1921 			!atomic_read(&dcc->discard_cmd_cnt)) {
1922 			wait_ms = dpolicy.max_interval;
1923 			continue;
1924 		}
1925 
1926 		sb_start_intwrite(sbi->sb);
1927 
1928 		issued = __issue_discard_cmd(sbi, &dpolicy);
1929 		if (issued > 0) {
1930 			__wait_all_discard_cmd(sbi, &dpolicy);
1931 			wait_ms = dpolicy.min_interval;
1932 		} else if (issued == -1) {
1933 			wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1934 			if (!wait_ms)
1935 				wait_ms = dpolicy.mid_interval;
1936 		} else {
1937 			wait_ms = dpolicy.max_interval;
1938 		}
1939 		if (!atomic_read(&dcc->discard_cmd_cnt))
1940 			wait_ms = dpolicy.max_interval;
1941 
1942 		sb_end_intwrite(sbi->sb);
1943 
1944 	} while (!kthread_should_stop());
1945 	return 0;
1946 }
1947 
1948 #ifdef CONFIG_BLK_DEV_ZONED
__f2fs_issue_discard_zone(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t blkstart,block_t blklen)1949 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1950 		struct block_device *bdev, block_t blkstart, block_t blklen)
1951 {
1952 	sector_t sector, nr_sects;
1953 	block_t lblkstart = blkstart;
1954 	int devi = 0;
1955 	u64 remainder = 0;
1956 
1957 	if (f2fs_is_multi_device(sbi)) {
1958 		devi = f2fs_target_device_index(sbi, blkstart);
1959 		if (blkstart < FDEV(devi).start_blk ||
1960 		    blkstart > FDEV(devi).end_blk) {
1961 			f2fs_err(sbi, "Invalid block %x", blkstart);
1962 			return -EIO;
1963 		}
1964 		blkstart -= FDEV(devi).start_blk;
1965 	}
1966 
1967 	/* For sequential zones, reset the zone write pointer */
1968 	if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1969 		sector = SECTOR_FROM_BLOCK(blkstart);
1970 		nr_sects = SECTOR_FROM_BLOCK(blklen);
1971 		div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder);
1972 
1973 		if (remainder || nr_sects != bdev_zone_sectors(bdev)) {
1974 			f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1975 				 devi, sbi->s_ndevs ? FDEV(devi).path : "",
1976 				 blkstart, blklen);
1977 			return -EIO;
1978 		}
1979 
1980 		if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) {
1981 			trace_f2fs_issue_reset_zone(bdev, blkstart);
1982 			return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1983 						sector, nr_sects, GFP_NOFS);
1984 		}
1985 
1986 		__queue_zone_reset_cmd(sbi, bdev, blkstart, lblkstart, blklen);
1987 		return 0;
1988 	}
1989 
1990 	/* For conventional zones, use regular discard if supported */
1991 	__queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1992 	return 0;
1993 }
1994 #endif
1995 
__issue_discard_async(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t blkstart,block_t blklen)1996 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1997 		struct block_device *bdev, block_t blkstart, block_t blklen)
1998 {
1999 #ifdef CONFIG_BLK_DEV_ZONED
2000 	if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
2001 		return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
2002 #endif
2003 	__queue_discard_cmd(sbi, bdev, blkstart, blklen);
2004 	return 0;
2005 }
2006 
f2fs_issue_discard(struct f2fs_sb_info * sbi,block_t blkstart,block_t blklen)2007 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
2008 				block_t blkstart, block_t blklen)
2009 {
2010 	sector_t start = blkstart, len = 0;
2011 	struct block_device *bdev;
2012 	struct seg_entry *se;
2013 	unsigned int offset;
2014 	block_t i;
2015 	int err = 0;
2016 
2017 	bdev = f2fs_target_device(sbi, blkstart, NULL);
2018 
2019 	for (i = blkstart; i < blkstart + blklen; i++, len++) {
2020 		if (i != start) {
2021 			struct block_device *bdev2 =
2022 				f2fs_target_device(sbi, i, NULL);
2023 
2024 			if (bdev2 != bdev) {
2025 				err = __issue_discard_async(sbi, bdev,
2026 						start, len);
2027 				if (err)
2028 					return err;
2029 				bdev = bdev2;
2030 				start = i;
2031 				len = 0;
2032 			}
2033 		}
2034 
2035 		se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
2036 		offset = GET_BLKOFF_FROM_SEG0(sbi, i);
2037 
2038 		if (f2fs_block_unit_discard(sbi) &&
2039 				!f2fs_test_and_set_bit(offset, se->discard_map))
2040 			sbi->discard_blks--;
2041 	}
2042 
2043 	if (len)
2044 		err = __issue_discard_async(sbi, bdev, start, len);
2045 	return err;
2046 }
2047 
add_discard_addrs(struct f2fs_sb_info * sbi,struct cp_control * cpc,bool check_only)2048 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
2049 							bool check_only)
2050 {
2051 	int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2052 	struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
2053 	unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2054 	unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2055 	unsigned long *discard_map = (unsigned long *)se->discard_map;
2056 	unsigned long *dmap = SIT_I(sbi)->tmp_map;
2057 	unsigned int start = 0, end = -1;
2058 	bool force = (cpc->reason & CP_DISCARD);
2059 	struct discard_entry *de = NULL;
2060 	struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
2061 	int i;
2062 
2063 	if (se->valid_blocks == BLKS_PER_SEG(sbi) ||
2064 	    !f2fs_hw_support_discard(sbi) ||
2065 	    !f2fs_block_unit_discard(sbi))
2066 		return false;
2067 
2068 	if (!force) {
2069 		if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
2070 			SM_I(sbi)->dcc_info->nr_discards >=
2071 				SM_I(sbi)->dcc_info->max_discards)
2072 			return false;
2073 	}
2074 
2075 	/* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
2076 	for (i = 0; i < entries; i++)
2077 		dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
2078 				(cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
2079 
2080 	while (force || SM_I(sbi)->dcc_info->nr_discards <=
2081 				SM_I(sbi)->dcc_info->max_discards) {
2082 		start = __find_rev_next_bit(dmap, BLKS_PER_SEG(sbi), end + 1);
2083 		if (start >= BLKS_PER_SEG(sbi))
2084 			break;
2085 
2086 		end = __find_rev_next_zero_bit(dmap,
2087 						BLKS_PER_SEG(sbi), start + 1);
2088 		if (force && start && end != BLKS_PER_SEG(sbi) &&
2089 		    (end - start) < cpc->trim_minlen)
2090 			continue;
2091 
2092 		if (check_only)
2093 			return true;
2094 
2095 		if (!de) {
2096 			de = f2fs_kmem_cache_alloc(discard_entry_slab,
2097 						GFP_F2FS_ZERO, true, NULL);
2098 			de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
2099 			list_add_tail(&de->list, head);
2100 		}
2101 
2102 		for (i = start; i < end; i++)
2103 			__set_bit_le(i, (void *)de->discard_map);
2104 
2105 		SM_I(sbi)->dcc_info->nr_discards += end - start;
2106 	}
2107 	return false;
2108 }
2109 
release_discard_addr(struct discard_entry * entry)2110 static void release_discard_addr(struct discard_entry *entry)
2111 {
2112 	list_del(&entry->list);
2113 	kmem_cache_free(discard_entry_slab, entry);
2114 }
2115 
f2fs_release_discard_addrs(struct f2fs_sb_info * sbi)2116 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
2117 {
2118 	struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
2119 	struct discard_entry *entry, *this;
2120 
2121 	/* drop caches */
2122 	list_for_each_entry_safe(entry, this, head, list)
2123 		release_discard_addr(entry);
2124 }
2125 
2126 /*
2127  * Should call f2fs_clear_prefree_segments after checkpoint is done.
2128  */
set_prefree_as_free_segments(struct f2fs_sb_info * sbi)2129 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
2130 {
2131 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2132 	unsigned int segno;
2133 
2134 	mutex_lock(&dirty_i->seglist_lock);
2135 	for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
2136 		__set_test_and_free(sbi, segno, false);
2137 	mutex_unlock(&dirty_i->seglist_lock);
2138 }
2139 
f2fs_clear_prefree_segments(struct f2fs_sb_info * sbi,struct cp_control * cpc)2140 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
2141 						struct cp_control *cpc)
2142 {
2143 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2144 	struct list_head *head = &dcc->entry_list;
2145 	struct discard_entry *entry, *this;
2146 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2147 	unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2148 	unsigned int start = 0, end = -1;
2149 	unsigned int secno, start_segno;
2150 	bool force = (cpc->reason & CP_DISCARD);
2151 	bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2152 						DISCARD_UNIT_SECTION;
2153 
2154 	if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2155 		section_alignment = true;
2156 
2157 	mutex_lock(&dirty_i->seglist_lock);
2158 
2159 	while (1) {
2160 		int i;
2161 
2162 		if (section_alignment && end != -1)
2163 			end--;
2164 		start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2165 		if (start >= MAIN_SEGS(sbi))
2166 			break;
2167 		end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2168 								start + 1);
2169 
2170 		if (section_alignment) {
2171 			start = rounddown(start, SEGS_PER_SEC(sbi));
2172 			end = roundup(end, SEGS_PER_SEC(sbi));
2173 		}
2174 
2175 		for (i = start; i < end; i++) {
2176 			if (test_and_clear_bit(i, prefree_map))
2177 				dirty_i->nr_dirty[PRE]--;
2178 		}
2179 
2180 		if (!f2fs_realtime_discard_enable(sbi))
2181 			continue;
2182 
2183 		if (force && start >= cpc->trim_start &&
2184 					(end - 1) <= cpc->trim_end)
2185 			continue;
2186 
2187 		/* Should cover 2MB zoned device for zone-based reset */
2188 		if (!f2fs_sb_has_blkzoned(sbi) &&
2189 		    (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) {
2190 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2191 				(end - start) << sbi->log_blocks_per_seg);
2192 			continue;
2193 		}
2194 next:
2195 		secno = GET_SEC_FROM_SEG(sbi, start);
2196 		start_segno = GET_SEG_FROM_SEC(sbi, secno);
2197 		if (!IS_CURSEC(sbi, secno) &&
2198 			!get_valid_blocks(sbi, start, true))
2199 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2200 						BLKS_PER_SEC(sbi));
2201 
2202 		start = start_segno + SEGS_PER_SEC(sbi);
2203 		if (start < end)
2204 			goto next;
2205 		else
2206 			end = start - 1;
2207 	}
2208 	mutex_unlock(&dirty_i->seglist_lock);
2209 
2210 	if (!f2fs_block_unit_discard(sbi))
2211 		goto wakeup;
2212 
2213 	/* send small discards */
2214 	list_for_each_entry_safe(entry, this, head, list) {
2215 		unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2216 		bool is_valid = test_bit_le(0, entry->discard_map);
2217 
2218 find_next:
2219 		if (is_valid) {
2220 			next_pos = find_next_zero_bit_le(entry->discard_map,
2221 						BLKS_PER_SEG(sbi), cur_pos);
2222 			len = next_pos - cur_pos;
2223 
2224 			if (f2fs_sb_has_blkzoned(sbi) ||
2225 			    (force && len < cpc->trim_minlen))
2226 				goto skip;
2227 
2228 			f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2229 									len);
2230 			total_len += len;
2231 		} else {
2232 			next_pos = find_next_bit_le(entry->discard_map,
2233 						BLKS_PER_SEG(sbi), cur_pos);
2234 		}
2235 skip:
2236 		cur_pos = next_pos;
2237 		is_valid = !is_valid;
2238 
2239 		if (cur_pos < BLKS_PER_SEG(sbi))
2240 			goto find_next;
2241 
2242 		release_discard_addr(entry);
2243 		dcc->nr_discards -= total_len;
2244 	}
2245 
2246 wakeup:
2247 	wake_up_discard_thread(sbi, false);
2248 }
2249 
f2fs_start_discard_thread(struct f2fs_sb_info * sbi)2250 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2251 {
2252 	dev_t dev = sbi->sb->s_bdev->bd_dev;
2253 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2254 	int err = 0;
2255 
2256 	if (!f2fs_realtime_discard_enable(sbi))
2257 		return 0;
2258 
2259 	dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2260 				"f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2261 	if (IS_ERR(dcc->f2fs_issue_discard)) {
2262 		err = PTR_ERR(dcc->f2fs_issue_discard);
2263 		dcc->f2fs_issue_discard = NULL;
2264 	}
2265 
2266 	return err;
2267 }
2268 
create_discard_cmd_control(struct f2fs_sb_info * sbi)2269 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2270 {
2271 	struct discard_cmd_control *dcc;
2272 	int err = 0, i;
2273 
2274 	if (SM_I(sbi)->dcc_info) {
2275 		dcc = SM_I(sbi)->dcc_info;
2276 		goto init_thread;
2277 	}
2278 
2279 	dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2280 	if (!dcc)
2281 		return -ENOMEM;
2282 
2283 	dcc->discard_io_aware_gran = MAX_PLIST_NUM;
2284 	dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2285 	dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY;
2286 	if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2287 		dcc->discard_granularity = BLKS_PER_SEG(sbi);
2288 	else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2289 		dcc->discard_granularity = BLKS_PER_SEC(sbi);
2290 
2291 	INIT_LIST_HEAD(&dcc->entry_list);
2292 	for (i = 0; i < MAX_PLIST_NUM; i++)
2293 		INIT_LIST_HEAD(&dcc->pend_list[i]);
2294 	INIT_LIST_HEAD(&dcc->wait_list);
2295 	INIT_LIST_HEAD(&dcc->fstrim_list);
2296 	mutex_init(&dcc->cmd_lock);
2297 	atomic_set(&dcc->issued_discard, 0);
2298 	atomic_set(&dcc->queued_discard, 0);
2299 	atomic_set(&dcc->discard_cmd_cnt, 0);
2300 	dcc->nr_discards = 0;
2301 	dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2302 	dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;
2303 	dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;
2304 	dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;
2305 	dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;
2306 	dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL;
2307 	dcc->undiscard_blks = 0;
2308 	dcc->next_pos = 0;
2309 	dcc->root = RB_ROOT_CACHED;
2310 	dcc->rbtree_check = false;
2311 
2312 	init_waitqueue_head(&dcc->discard_wait_queue);
2313 	SM_I(sbi)->dcc_info = dcc;
2314 init_thread:
2315 	err = f2fs_start_discard_thread(sbi);
2316 	if (err) {
2317 		kfree(dcc);
2318 		SM_I(sbi)->dcc_info = NULL;
2319 	}
2320 
2321 	return err;
2322 }
2323 
destroy_discard_cmd_control(struct f2fs_sb_info * sbi)2324 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2325 {
2326 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2327 
2328 	if (!dcc)
2329 		return;
2330 
2331 	f2fs_stop_discard_thread(sbi);
2332 
2333 	/*
2334 	 * Recovery can cache discard commands, so in error path of
2335 	 * fill_super(), it needs to give a chance to handle them.
2336 	 */
2337 	f2fs_issue_discard_timeout(sbi);
2338 
2339 	kfree(dcc);
2340 	SM_I(sbi)->dcc_info = NULL;
2341 }
2342 
__mark_sit_entry_dirty(struct f2fs_sb_info * sbi,unsigned int segno)2343 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2344 {
2345 	struct sit_info *sit_i = SIT_I(sbi);
2346 
2347 	if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2348 		sit_i->dirty_sentries++;
2349 		return false;
2350 	}
2351 
2352 	return true;
2353 }
2354 
__set_sit_entry_type(struct f2fs_sb_info * sbi,int type,unsigned int segno,int modified)2355 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2356 					unsigned int segno, int modified)
2357 {
2358 	struct seg_entry *se = get_seg_entry(sbi, segno);
2359 
2360 	se->type = type;
2361 	if (modified)
2362 		__mark_sit_entry_dirty(sbi, segno);
2363 }
2364 
get_segment_mtime(struct f2fs_sb_info * sbi,block_t blkaddr)2365 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2366 								block_t blkaddr)
2367 {
2368 	unsigned int segno = GET_SEGNO(sbi, blkaddr);
2369 
2370 	if (segno == NULL_SEGNO)
2371 		return 0;
2372 	return get_seg_entry(sbi, segno)->mtime;
2373 }
2374 
update_segment_mtime(struct f2fs_sb_info * sbi,block_t blkaddr,unsigned long long old_mtime)2375 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2376 						unsigned long long old_mtime)
2377 {
2378 	struct seg_entry *se;
2379 	unsigned int segno = GET_SEGNO(sbi, blkaddr);
2380 	unsigned long long ctime = get_mtime(sbi, false);
2381 	unsigned long long mtime = old_mtime ? old_mtime : ctime;
2382 
2383 	if (segno == NULL_SEGNO)
2384 		return;
2385 
2386 	se = get_seg_entry(sbi, segno);
2387 
2388 	if (!se->mtime)
2389 		se->mtime = mtime;
2390 	else
2391 		se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2392 						se->valid_blocks + 1);
2393 
2394 	if (ctime > SIT_I(sbi)->max_mtime)
2395 		SIT_I(sbi)->max_mtime = ctime;
2396 }
2397 
update_sit_entry(struct f2fs_sb_info * sbi,block_t blkaddr,int del)2398 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2399 {
2400 	struct seg_entry *se;
2401 	unsigned int segno, offset;
2402 	long int new_vblocks;
2403 	bool exist;
2404 #ifdef CONFIG_F2FS_CHECK_FS
2405 	bool mir_exist;
2406 #endif
2407 
2408 	segno = GET_SEGNO(sbi, blkaddr);
2409 	if (segno == NULL_SEGNO)
2410 		return;
2411 
2412 	se = get_seg_entry(sbi, segno);
2413 	new_vblocks = se->valid_blocks + del;
2414 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2415 
2416 	f2fs_bug_on(sbi, (new_vblocks < 0 ||
2417 			(new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2418 
2419 	se->valid_blocks = new_vblocks;
2420 
2421 	/* Update valid block bitmap */
2422 	if (del > 0) {
2423 		exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2424 #ifdef CONFIG_F2FS_CHECK_FS
2425 		mir_exist = f2fs_test_and_set_bit(offset,
2426 						se->cur_valid_map_mir);
2427 		if (unlikely(exist != mir_exist)) {
2428 			f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2429 				 blkaddr, exist);
2430 			f2fs_bug_on(sbi, 1);
2431 		}
2432 #endif
2433 		if (unlikely(exist)) {
2434 			f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2435 				 blkaddr);
2436 			f2fs_bug_on(sbi, 1);
2437 			se->valid_blocks--;
2438 			del = 0;
2439 		}
2440 
2441 		if (f2fs_block_unit_discard(sbi) &&
2442 				!f2fs_test_and_set_bit(offset, se->discard_map))
2443 			sbi->discard_blks--;
2444 
2445 		/*
2446 		 * SSR should never reuse block which is checkpointed
2447 		 * or newly invalidated.
2448 		 */
2449 		if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2450 			if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2451 				se->ckpt_valid_blocks++;
2452 		}
2453 	} else {
2454 		exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2455 #ifdef CONFIG_F2FS_CHECK_FS
2456 		mir_exist = f2fs_test_and_clear_bit(offset,
2457 						se->cur_valid_map_mir);
2458 		if (unlikely(exist != mir_exist)) {
2459 			f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2460 				 blkaddr, exist);
2461 			f2fs_bug_on(sbi, 1);
2462 		}
2463 #endif
2464 		if (unlikely(!exist)) {
2465 			f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2466 				 blkaddr);
2467 			f2fs_bug_on(sbi, 1);
2468 			se->valid_blocks++;
2469 			del = 0;
2470 		} else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2471 			/*
2472 			 * If checkpoints are off, we must not reuse data that
2473 			 * was used in the previous checkpoint. If it was used
2474 			 * before, we must track that to know how much space we
2475 			 * really have.
2476 			 */
2477 			if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2478 				spin_lock(&sbi->stat_lock);
2479 				sbi->unusable_block_count++;
2480 				spin_unlock(&sbi->stat_lock);
2481 			}
2482 		}
2483 
2484 		if (f2fs_block_unit_discard(sbi) &&
2485 			f2fs_test_and_clear_bit(offset, se->discard_map))
2486 			sbi->discard_blks++;
2487 	}
2488 	if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2489 		se->ckpt_valid_blocks += del;
2490 
2491 	__mark_sit_entry_dirty(sbi, segno);
2492 
2493 	/* update total number of valid blocks to be written in ckpt area */
2494 	SIT_I(sbi)->written_valid_blocks += del;
2495 
2496 	if (__is_large_section(sbi))
2497 		get_sec_entry(sbi, segno)->valid_blocks += del;
2498 }
2499 
f2fs_invalidate_blocks(struct f2fs_sb_info * sbi,block_t addr)2500 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2501 {
2502 	unsigned int segno = GET_SEGNO(sbi, addr);
2503 	struct sit_info *sit_i = SIT_I(sbi);
2504 
2505 	f2fs_bug_on(sbi, addr == NULL_ADDR);
2506 	if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2507 		return;
2508 
2509 	f2fs_invalidate_internal_cache(sbi, addr);
2510 
2511 	/* add it into sit main buffer */
2512 	down_write(&sit_i->sentry_lock);
2513 
2514 	update_segment_mtime(sbi, addr, 0);
2515 	update_sit_entry(sbi, addr, -1);
2516 
2517 	/* add it into dirty seglist */
2518 	locate_dirty_segment(sbi, segno);
2519 
2520 	up_write(&sit_i->sentry_lock);
2521 }
2522 
f2fs_is_checkpointed_data(struct f2fs_sb_info * sbi,block_t blkaddr)2523 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2524 {
2525 	struct sit_info *sit_i = SIT_I(sbi);
2526 	unsigned int segno, offset;
2527 	struct seg_entry *se;
2528 	bool is_cp = false;
2529 
2530 	if (!__is_valid_data_blkaddr(blkaddr))
2531 		return true;
2532 
2533 	down_read(&sit_i->sentry_lock);
2534 
2535 	segno = GET_SEGNO(sbi, blkaddr);
2536 	se = get_seg_entry(sbi, segno);
2537 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2538 
2539 	if (f2fs_test_bit(offset, se->ckpt_valid_map))
2540 		is_cp = true;
2541 
2542 	up_read(&sit_i->sentry_lock);
2543 
2544 	return is_cp;
2545 }
2546 
f2fs_curseg_valid_blocks(struct f2fs_sb_info * sbi,int type)2547 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type)
2548 {
2549 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2550 
2551 	if (sbi->ckpt->alloc_type[type] == SSR)
2552 		return BLKS_PER_SEG(sbi);
2553 	return curseg->next_blkoff;
2554 }
2555 
2556 /*
2557  * Calculate the number of current summary pages for writing
2558  */
f2fs_npages_for_summary_flush(struct f2fs_sb_info * sbi,bool for_ra)2559 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2560 {
2561 	int valid_sum_count = 0;
2562 	int i, sum_in_page;
2563 
2564 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2565 		if (sbi->ckpt->alloc_type[i] != SSR && for_ra)
2566 			valid_sum_count +=
2567 				le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2568 		else
2569 			valid_sum_count += f2fs_curseg_valid_blocks(sbi, i);
2570 	}
2571 
2572 	sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2573 			SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2574 	if (valid_sum_count <= sum_in_page)
2575 		return 1;
2576 	else if ((valid_sum_count - sum_in_page) <=
2577 		(PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2578 		return 2;
2579 	return 3;
2580 }
2581 
2582 /*
2583  * Caller should put this summary page
2584  */
f2fs_get_sum_page(struct f2fs_sb_info * sbi,unsigned int segno)2585 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2586 {
2587 	if (unlikely(f2fs_cp_error(sbi)))
2588 		return ERR_PTR(-EIO);
2589 	return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2590 }
2591 
f2fs_update_meta_page(struct f2fs_sb_info * sbi,void * src,block_t blk_addr)2592 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2593 					void *src, block_t blk_addr)
2594 {
2595 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2596 
2597 	memcpy(page_address(page), src, PAGE_SIZE);
2598 	set_page_dirty(page);
2599 	f2fs_put_page(page, 1);
2600 }
2601 
write_sum_page(struct f2fs_sb_info * sbi,struct f2fs_summary_block * sum_blk,block_t blk_addr)2602 static void write_sum_page(struct f2fs_sb_info *sbi,
2603 			struct f2fs_summary_block *sum_blk, block_t blk_addr)
2604 {
2605 	f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2606 }
2607 
write_current_sum_page(struct f2fs_sb_info * sbi,int type,block_t blk_addr)2608 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2609 						int type, block_t blk_addr)
2610 {
2611 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2612 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2613 	struct f2fs_summary_block *src = curseg->sum_blk;
2614 	struct f2fs_summary_block *dst;
2615 
2616 	dst = (struct f2fs_summary_block *)page_address(page);
2617 	memset(dst, 0, PAGE_SIZE);
2618 
2619 	mutex_lock(&curseg->curseg_mutex);
2620 
2621 	down_read(&curseg->journal_rwsem);
2622 	memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2623 	up_read(&curseg->journal_rwsem);
2624 
2625 	memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2626 	memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2627 
2628 	mutex_unlock(&curseg->curseg_mutex);
2629 
2630 	set_page_dirty(page);
2631 	f2fs_put_page(page, 1);
2632 }
2633 
is_next_segment_free(struct f2fs_sb_info * sbi,struct curseg_info * curseg,int type)2634 static int is_next_segment_free(struct f2fs_sb_info *sbi,
2635 				struct curseg_info *curseg, int type)
2636 {
2637 	unsigned int segno = curseg->segno + 1;
2638 	struct free_segmap_info *free_i = FREE_I(sbi);
2639 
2640 	if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi))
2641 		return !test_bit(segno, free_i->free_segmap);
2642 	return 0;
2643 }
2644 
2645 /*
2646  * Find a new segment from the free segments bitmap to right order
2647  * This function should be returned with success, otherwise BUG
2648  */
get_new_segment(struct f2fs_sb_info * sbi,unsigned int * newseg,bool new_sec,bool pinning)2649 static void get_new_segment(struct f2fs_sb_info *sbi,
2650 			unsigned int *newseg, bool new_sec, bool pinning)
2651 {
2652 	struct free_segmap_info *free_i = FREE_I(sbi);
2653 	unsigned int segno, secno, zoneno;
2654 	unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2655 	unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2656 	unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2657 	bool init = true;
2658 	int i;
2659 	int ret = 0;
2660 
2661 	spin_lock(&free_i->segmap_lock);
2662 
2663 	if (!new_sec && ((*newseg + 1) % SEGS_PER_SEC(sbi))) {
2664 		segno = find_next_zero_bit(free_i->free_segmap,
2665 			GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2666 		if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2667 			goto got_it;
2668 	}
2669 
2670 	/*
2671 	 * If we format f2fs on zoned storage, let's try to get pinned sections
2672 	 * from beginning of the storage, which should be a conventional one.
2673 	 */
2674 	if (f2fs_sb_has_blkzoned(sbi)) {
2675 		segno = pinning ? 0 : max(first_zoned_segno(sbi), *newseg);
2676 		hint = GET_SEC_FROM_SEG(sbi, segno);
2677 	}
2678 
2679 find_other_zone:
2680 	secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2681 	if (secno >= MAIN_SECS(sbi)) {
2682 		secno = find_first_zero_bit(free_i->free_secmap,
2683 							MAIN_SECS(sbi));
2684 		if (secno >= MAIN_SECS(sbi)) {
2685 			ret = -ENOSPC;
2686 			goto out_unlock;
2687 		}
2688 	}
2689 	segno = GET_SEG_FROM_SEC(sbi, secno);
2690 	zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2691 
2692 	/* give up on finding another zone */
2693 	if (!init)
2694 		goto got_it;
2695 	if (sbi->secs_per_zone == 1)
2696 		goto got_it;
2697 	if (zoneno == old_zoneno)
2698 		goto got_it;
2699 	for (i = 0; i < NR_CURSEG_TYPE; i++)
2700 		if (CURSEG_I(sbi, i)->zone == zoneno)
2701 			break;
2702 
2703 	if (i < NR_CURSEG_TYPE) {
2704 		/* zone is in user, try another */
2705 		if (zoneno + 1 >= total_zones)
2706 			hint = 0;
2707 		else
2708 			hint = (zoneno + 1) * sbi->secs_per_zone;
2709 		init = false;
2710 		goto find_other_zone;
2711 	}
2712 got_it:
2713 	/* set it as dirty segment in free segmap */
2714 	f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2715 	__set_inuse(sbi, segno);
2716 	*newseg = segno;
2717 out_unlock:
2718 	spin_unlock(&free_i->segmap_lock);
2719 
2720 	if (ret) {
2721 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_NO_SEGMENT);
2722 		f2fs_bug_on(sbi, 1);
2723 	}
2724 }
2725 
reset_curseg(struct f2fs_sb_info * sbi,int type,int modified)2726 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2727 {
2728 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2729 	struct summary_footer *sum_footer;
2730 	unsigned short seg_type = curseg->seg_type;
2731 
2732 	curseg->inited = true;
2733 	curseg->segno = curseg->next_segno;
2734 	curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2735 	curseg->next_blkoff = 0;
2736 	curseg->next_segno = NULL_SEGNO;
2737 
2738 	sum_footer = &(curseg->sum_blk->footer);
2739 	memset(sum_footer, 0, sizeof(struct summary_footer));
2740 
2741 	sanity_check_seg_type(sbi, seg_type);
2742 
2743 	if (IS_DATASEG(seg_type))
2744 		SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2745 	if (IS_NODESEG(seg_type))
2746 		SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2747 	__set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2748 }
2749 
__get_next_segno(struct f2fs_sb_info * sbi,int type)2750 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2751 {
2752 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2753 	unsigned short seg_type = curseg->seg_type;
2754 
2755 	sanity_check_seg_type(sbi, seg_type);
2756 	if (f2fs_need_rand_seg(sbi))
2757 		return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi));
2758 
2759 	if (__is_large_section(sbi))
2760 		return curseg->segno;
2761 
2762 	/* inmem log may not locate on any segment after mount */
2763 	if (!curseg->inited)
2764 		return 0;
2765 
2766 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2767 		return 0;
2768 
2769 	if (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type))
2770 		return 0;
2771 
2772 	if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2773 		return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2774 
2775 	/* find segments from 0 to reuse freed segments */
2776 	if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2777 		return 0;
2778 
2779 	return curseg->segno;
2780 }
2781 
2782 /*
2783  * Allocate a current working segment.
2784  * This function always allocates a free segment in LFS manner.
2785  */
new_curseg(struct f2fs_sb_info * sbi,int type,bool new_sec)2786 static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2787 {
2788 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2789 	unsigned int segno = curseg->segno;
2790 	bool pinning = type == CURSEG_COLD_DATA_PINNED;
2791 
2792 	if (curseg->inited)
2793 		write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, segno));
2794 
2795 	segno = __get_next_segno(sbi, type);
2796 	get_new_segment(sbi, &segno, new_sec, pinning);
2797 	if (new_sec && pinning &&
2798 	    !f2fs_valid_pinned_area(sbi, START_BLOCK(sbi, segno))) {
2799 		__set_free(sbi, segno);
2800 		return -EAGAIN;
2801 	}
2802 
2803 	curseg->next_segno = segno;
2804 	reset_curseg(sbi, type, 1);
2805 	curseg->alloc_type = LFS;
2806 	if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
2807 		curseg->fragment_remained_chunk =
2808 				get_random_u32_inclusive(1, sbi->max_fragment_chunk);
2809 	return 0;
2810 }
2811 
__next_free_blkoff(struct f2fs_sb_info * sbi,int segno,block_t start)2812 static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2813 					int segno, block_t start)
2814 {
2815 	struct seg_entry *se = get_seg_entry(sbi, segno);
2816 	int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2817 	unsigned long *target_map = SIT_I(sbi)->tmp_map;
2818 	unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2819 	unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2820 	int i;
2821 
2822 	for (i = 0; i < entries; i++)
2823 		target_map[i] = ckpt_map[i] | cur_map[i];
2824 
2825 	return __find_rev_next_zero_bit(target_map, BLKS_PER_SEG(sbi), start);
2826 }
2827 
f2fs_find_next_ssr_block(struct f2fs_sb_info * sbi,struct curseg_info * seg)2828 static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi,
2829 		struct curseg_info *seg)
2830 {
2831 	return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1);
2832 }
2833 
f2fs_segment_has_free_slot(struct f2fs_sb_info * sbi,int segno)2834 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2835 {
2836 	return __next_free_blkoff(sbi, segno, 0) < BLKS_PER_SEG(sbi);
2837 }
2838 
2839 /*
2840  * This function always allocates a used segment(from dirty seglist) by SSR
2841  * manner, so it should recover the existing segment information of valid blocks
2842  */
change_curseg(struct f2fs_sb_info * sbi,int type)2843 static void change_curseg(struct f2fs_sb_info *sbi, int type)
2844 {
2845 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2846 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2847 	unsigned int new_segno = curseg->next_segno;
2848 	struct f2fs_summary_block *sum_node;
2849 	struct page *sum_page;
2850 
2851 	write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno));
2852 
2853 	__set_test_and_inuse(sbi, new_segno);
2854 
2855 	mutex_lock(&dirty_i->seglist_lock);
2856 	__remove_dirty_segment(sbi, new_segno, PRE);
2857 	__remove_dirty_segment(sbi, new_segno, DIRTY);
2858 	mutex_unlock(&dirty_i->seglist_lock);
2859 
2860 	reset_curseg(sbi, type, 1);
2861 	curseg->alloc_type = SSR;
2862 	curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2863 
2864 	sum_page = f2fs_get_sum_page(sbi, new_segno);
2865 	if (IS_ERR(sum_page)) {
2866 		/* GC won't be able to use stale summary pages by cp_error */
2867 		memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2868 		return;
2869 	}
2870 	sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2871 	memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2872 	f2fs_put_page(sum_page, 1);
2873 }
2874 
2875 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2876 				int alloc_mode, unsigned long long age);
2877 
get_atssr_segment(struct f2fs_sb_info * sbi,int type,int target_type,int alloc_mode,unsigned long long age)2878 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2879 					int target_type, int alloc_mode,
2880 					unsigned long long age)
2881 {
2882 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2883 
2884 	curseg->seg_type = target_type;
2885 
2886 	if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2887 		struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2888 
2889 		curseg->seg_type = se->type;
2890 		change_curseg(sbi, type);
2891 	} else {
2892 		/* allocate cold segment by default */
2893 		curseg->seg_type = CURSEG_COLD_DATA;
2894 		new_curseg(sbi, type, true);
2895 	}
2896 	stat_inc_seg_type(sbi, curseg);
2897 }
2898 
__f2fs_init_atgc_curseg(struct f2fs_sb_info * sbi)2899 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2900 {
2901 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2902 
2903 	if (!sbi->am.atgc_enabled)
2904 		return;
2905 
2906 	f2fs_down_read(&SM_I(sbi)->curseg_lock);
2907 
2908 	mutex_lock(&curseg->curseg_mutex);
2909 	down_write(&SIT_I(sbi)->sentry_lock);
2910 
2911 	get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2912 
2913 	up_write(&SIT_I(sbi)->sentry_lock);
2914 	mutex_unlock(&curseg->curseg_mutex);
2915 
2916 	f2fs_up_read(&SM_I(sbi)->curseg_lock);
2917 
2918 }
f2fs_init_inmem_curseg(struct f2fs_sb_info * sbi)2919 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2920 {
2921 	__f2fs_init_atgc_curseg(sbi);
2922 }
2923 
__f2fs_save_inmem_curseg(struct f2fs_sb_info * sbi,int type)2924 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2925 {
2926 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2927 
2928 	mutex_lock(&curseg->curseg_mutex);
2929 	if (!curseg->inited)
2930 		goto out;
2931 
2932 	if (get_valid_blocks(sbi, curseg->segno, false)) {
2933 		write_sum_page(sbi, curseg->sum_blk,
2934 				GET_SUM_BLOCK(sbi, curseg->segno));
2935 	} else {
2936 		mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2937 		__set_test_and_free(sbi, curseg->segno, true);
2938 		mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2939 	}
2940 out:
2941 	mutex_unlock(&curseg->curseg_mutex);
2942 }
2943 
f2fs_save_inmem_curseg(struct f2fs_sb_info * sbi)2944 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2945 {
2946 	__f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2947 
2948 	if (sbi->am.atgc_enabled)
2949 		__f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2950 }
2951 
__f2fs_restore_inmem_curseg(struct f2fs_sb_info * sbi,int type)2952 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2953 {
2954 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2955 
2956 	mutex_lock(&curseg->curseg_mutex);
2957 	if (!curseg->inited)
2958 		goto out;
2959 	if (get_valid_blocks(sbi, curseg->segno, false))
2960 		goto out;
2961 
2962 	mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2963 	__set_test_and_inuse(sbi, curseg->segno);
2964 	mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2965 out:
2966 	mutex_unlock(&curseg->curseg_mutex);
2967 }
2968 
f2fs_restore_inmem_curseg(struct f2fs_sb_info * sbi)2969 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2970 {
2971 	__f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2972 
2973 	if (sbi->am.atgc_enabled)
2974 		__f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2975 }
2976 
get_ssr_segment(struct f2fs_sb_info * sbi,int type,int alloc_mode,unsigned long long age)2977 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2978 				int alloc_mode, unsigned long long age)
2979 {
2980 	struct curseg_info *curseg = CURSEG_I(sbi, type);
2981 	unsigned segno = NULL_SEGNO;
2982 	unsigned short seg_type = curseg->seg_type;
2983 	int i, cnt;
2984 	bool reversed = false;
2985 
2986 	sanity_check_seg_type(sbi, seg_type);
2987 
2988 	/* f2fs_need_SSR() already forces to do this */
2989 	if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2990 		curseg->next_segno = segno;
2991 		return 1;
2992 	}
2993 
2994 	/* For node segments, let's do SSR more intensively */
2995 	if (IS_NODESEG(seg_type)) {
2996 		if (seg_type >= CURSEG_WARM_NODE) {
2997 			reversed = true;
2998 			i = CURSEG_COLD_NODE;
2999 		} else {
3000 			i = CURSEG_HOT_NODE;
3001 		}
3002 		cnt = NR_CURSEG_NODE_TYPE;
3003 	} else {
3004 		if (seg_type >= CURSEG_WARM_DATA) {
3005 			reversed = true;
3006 			i = CURSEG_COLD_DATA;
3007 		} else {
3008 			i = CURSEG_HOT_DATA;
3009 		}
3010 		cnt = NR_CURSEG_DATA_TYPE;
3011 	}
3012 
3013 	for (; cnt-- > 0; reversed ? i-- : i++) {
3014 		if (i == seg_type)
3015 			continue;
3016 		if (!f2fs_get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
3017 			curseg->next_segno = segno;
3018 			return 1;
3019 		}
3020 	}
3021 
3022 	/* find valid_blocks=0 in dirty list */
3023 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
3024 		segno = get_free_segment(sbi);
3025 		if (segno != NULL_SEGNO) {
3026 			curseg->next_segno = segno;
3027 			return 1;
3028 		}
3029 	}
3030 	return 0;
3031 }
3032 
need_new_seg(struct f2fs_sb_info * sbi,int type)3033 static bool need_new_seg(struct f2fs_sb_info *sbi, int type)
3034 {
3035 	struct curseg_info *curseg = CURSEG_I(sbi, type);
3036 
3037 	if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
3038 	    curseg->seg_type == CURSEG_WARM_NODE)
3039 		return true;
3040 	if (curseg->alloc_type == LFS &&
3041 	    is_next_segment_free(sbi, curseg, type) &&
3042 	    likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
3043 		return true;
3044 	if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0))
3045 		return true;
3046 	return false;
3047 }
3048 
f2fs_allocate_segment_for_resize(struct f2fs_sb_info * sbi,int type,unsigned int start,unsigned int end)3049 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3050 					unsigned int start, unsigned int end)
3051 {
3052 	struct curseg_info *curseg = CURSEG_I(sbi, type);
3053 	unsigned int segno;
3054 
3055 	f2fs_down_read(&SM_I(sbi)->curseg_lock);
3056 	mutex_lock(&curseg->curseg_mutex);
3057 	down_write(&SIT_I(sbi)->sentry_lock);
3058 
3059 	segno = CURSEG_I(sbi, type)->segno;
3060 	if (segno < start || segno > end)
3061 		goto unlock;
3062 
3063 	if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
3064 		change_curseg(sbi, type);
3065 	else
3066 		new_curseg(sbi, type, true);
3067 
3068 	stat_inc_seg_type(sbi, curseg);
3069 
3070 	locate_dirty_segment(sbi, segno);
3071 unlock:
3072 	up_write(&SIT_I(sbi)->sentry_lock);
3073 
3074 	if (segno != curseg->segno)
3075 		f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
3076 			    type, segno, curseg->segno);
3077 
3078 	mutex_unlock(&curseg->curseg_mutex);
3079 	f2fs_up_read(&SM_I(sbi)->curseg_lock);
3080 }
3081 
__allocate_new_segment(struct f2fs_sb_info * sbi,int type,bool new_sec,bool force)3082 static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
3083 						bool new_sec, bool force)
3084 {
3085 	struct curseg_info *curseg = CURSEG_I(sbi, type);
3086 	unsigned int old_segno;
3087 
3088 	if (!force && curseg->inited &&
3089 	    !curseg->next_blkoff &&
3090 	    !get_valid_blocks(sbi, curseg->segno, new_sec) &&
3091 	    !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
3092 		return 0;
3093 
3094 	old_segno = curseg->segno;
3095 	if (new_curseg(sbi, type, true))
3096 		return -EAGAIN;
3097 	stat_inc_seg_type(sbi, curseg);
3098 	locate_dirty_segment(sbi, old_segno);
3099 	return 0;
3100 }
3101 
f2fs_allocate_new_section(struct f2fs_sb_info * sbi,int type,bool force)3102 int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
3103 {
3104 	int ret;
3105 
3106 	f2fs_down_read(&SM_I(sbi)->curseg_lock);
3107 	down_write(&SIT_I(sbi)->sentry_lock);
3108 	ret = __allocate_new_segment(sbi, type, true, force);
3109 	up_write(&SIT_I(sbi)->sentry_lock);
3110 	f2fs_up_read(&SM_I(sbi)->curseg_lock);
3111 
3112 	return ret;
3113 }
3114 
f2fs_allocate_pinning_section(struct f2fs_sb_info * sbi)3115 int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi)
3116 {
3117 	int err;
3118 	bool gc_required = true;
3119 
3120 retry:
3121 	f2fs_lock_op(sbi);
3122 	err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false);
3123 	f2fs_unlock_op(sbi);
3124 
3125 	if (f2fs_sb_has_blkzoned(sbi) && err && gc_required) {
3126 		f2fs_down_write(&sbi->gc_lock);
3127 		f2fs_gc_range(sbi, 0, GET_SEGNO(sbi, FDEV(0).end_blk), true, 1);
3128 		f2fs_up_write(&sbi->gc_lock);
3129 
3130 		gc_required = false;
3131 		goto retry;
3132 	}
3133 
3134 	return err;
3135 }
3136 
f2fs_allocate_new_segments(struct f2fs_sb_info * sbi)3137 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
3138 {
3139 	int i;
3140 
3141 	f2fs_down_read(&SM_I(sbi)->curseg_lock);
3142 	down_write(&SIT_I(sbi)->sentry_lock);
3143 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
3144 		__allocate_new_segment(sbi, i, false, false);
3145 	up_write(&SIT_I(sbi)->sentry_lock);
3146 	f2fs_up_read(&SM_I(sbi)->curseg_lock);
3147 }
3148 
f2fs_exist_trim_candidates(struct f2fs_sb_info * sbi,struct cp_control * cpc)3149 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3150 						struct cp_control *cpc)
3151 {
3152 	__u64 trim_start = cpc->trim_start;
3153 	bool has_candidate = false;
3154 
3155 	down_write(&SIT_I(sbi)->sentry_lock);
3156 	for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
3157 		if (add_discard_addrs(sbi, cpc, true)) {
3158 			has_candidate = true;
3159 			break;
3160 		}
3161 	}
3162 	up_write(&SIT_I(sbi)->sentry_lock);
3163 
3164 	cpc->trim_start = trim_start;
3165 	return has_candidate;
3166 }
3167 
__issue_discard_cmd_range(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,unsigned int start,unsigned int end)3168 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3169 					struct discard_policy *dpolicy,
3170 					unsigned int start, unsigned int end)
3171 {
3172 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3173 	struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3174 	struct rb_node **insert_p = NULL, *insert_parent = NULL;
3175 	struct discard_cmd *dc;
3176 	struct blk_plug plug;
3177 	int issued;
3178 	unsigned int trimmed = 0;
3179 
3180 next:
3181 	issued = 0;
3182 
3183 	mutex_lock(&dcc->cmd_lock);
3184 	if (unlikely(dcc->rbtree_check))
3185 		f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
3186 
3187 	dc = __lookup_discard_cmd_ret(&dcc->root, start,
3188 				&prev_dc, &next_dc, &insert_p, &insert_parent);
3189 	if (!dc)
3190 		dc = next_dc;
3191 
3192 	blk_start_plug(&plug);
3193 
3194 	while (dc && dc->di.lstart <= end) {
3195 		struct rb_node *node;
3196 		int err = 0;
3197 
3198 		if (dc->di.len < dpolicy->granularity)
3199 			goto skip;
3200 
3201 		if (dc->state != D_PREP) {
3202 			list_move_tail(&dc->list, &dcc->fstrim_list);
3203 			goto skip;
3204 		}
3205 
3206 		err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3207 
3208 		if (issued >= dpolicy->max_requests) {
3209 			start = dc->di.lstart + dc->di.len;
3210 
3211 			if (err)
3212 				__remove_discard_cmd(sbi, dc);
3213 
3214 			blk_finish_plug(&plug);
3215 			mutex_unlock(&dcc->cmd_lock);
3216 			trimmed += __wait_all_discard_cmd(sbi, NULL);
3217 			f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
3218 			goto next;
3219 		}
3220 skip:
3221 		node = rb_next(&dc->rb_node);
3222 		if (err)
3223 			__remove_discard_cmd(sbi, dc);
3224 		dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3225 
3226 		if (fatal_signal_pending(current))
3227 			break;
3228 	}
3229 
3230 	blk_finish_plug(&plug);
3231 	mutex_unlock(&dcc->cmd_lock);
3232 
3233 	return trimmed;
3234 }
3235 
f2fs_trim_fs(struct f2fs_sb_info * sbi,struct fstrim_range * range)3236 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3237 {
3238 	__u64 start = F2FS_BYTES_TO_BLK(range->start);
3239 	__u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3240 	unsigned int start_segno, end_segno;
3241 	block_t start_block, end_block;
3242 	struct cp_control cpc;
3243 	struct discard_policy dpolicy;
3244 	unsigned long long trimmed = 0;
3245 	int err = 0;
3246 	bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3247 
3248 	if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3249 		return -EINVAL;
3250 
3251 	if (end < MAIN_BLKADDR(sbi))
3252 		goto out;
3253 
3254 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3255 		f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3256 		return -EFSCORRUPTED;
3257 	}
3258 
3259 	/* start/end segment number in main_area */
3260 	start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3261 	end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3262 						GET_SEGNO(sbi, end);
3263 	if (need_align) {
3264 		start_segno = rounddown(start_segno, SEGS_PER_SEC(sbi));
3265 		end_segno = roundup(end_segno + 1, SEGS_PER_SEC(sbi)) - 1;
3266 	}
3267 
3268 	cpc.reason = CP_DISCARD;
3269 	cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3270 	cpc.trim_start = start_segno;
3271 	cpc.trim_end = end_segno;
3272 
3273 	if (sbi->discard_blks == 0)
3274 		goto out;
3275 
3276 	f2fs_down_write(&sbi->gc_lock);
3277 	stat_inc_cp_call_count(sbi, TOTAL_CALL);
3278 	err = f2fs_write_checkpoint(sbi, &cpc);
3279 	f2fs_up_write(&sbi->gc_lock);
3280 	if (err)
3281 		goto out;
3282 
3283 	/*
3284 	 * We filed discard candidates, but actually we don't need to wait for
3285 	 * all of them, since they'll be issued in idle time along with runtime
3286 	 * discard option. User configuration looks like using runtime discard
3287 	 * or periodic fstrim instead of it.
3288 	 */
3289 	if (f2fs_realtime_discard_enable(sbi))
3290 		goto out;
3291 
3292 	start_block = START_BLOCK(sbi, start_segno);
3293 	end_block = START_BLOCK(sbi, end_segno + 1);
3294 
3295 	__init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3296 	trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3297 					start_block, end_block);
3298 
3299 	trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3300 					start_block, end_block);
3301 out:
3302 	if (!err)
3303 		range->len = F2FS_BLK_TO_BYTES(trimmed);
3304 	return err;
3305 }
3306 
f2fs_rw_hint_to_seg_type(enum rw_hint hint)3307 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3308 {
3309 	switch (hint) {
3310 	case WRITE_LIFE_SHORT:
3311 		return CURSEG_HOT_DATA;
3312 	case WRITE_LIFE_EXTREME:
3313 		return CURSEG_COLD_DATA;
3314 	default:
3315 		return CURSEG_WARM_DATA;
3316 	}
3317 }
3318 
__get_segment_type_2(struct f2fs_io_info * fio)3319 static int __get_segment_type_2(struct f2fs_io_info *fio)
3320 {
3321 	if (fio->type == DATA)
3322 		return CURSEG_HOT_DATA;
3323 	else
3324 		return CURSEG_HOT_NODE;
3325 }
3326 
__get_segment_type_4(struct f2fs_io_info * fio)3327 static int __get_segment_type_4(struct f2fs_io_info *fio)
3328 {
3329 	if (fio->type == DATA) {
3330 		struct inode *inode = fio->page->mapping->host;
3331 
3332 		if (S_ISDIR(inode->i_mode))
3333 			return CURSEG_HOT_DATA;
3334 		else
3335 			return CURSEG_COLD_DATA;
3336 	} else {
3337 		if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3338 			return CURSEG_WARM_NODE;
3339 		else
3340 			return CURSEG_COLD_NODE;
3341 	}
3342 }
3343 
__get_age_segment_type(struct inode * inode,pgoff_t pgofs)3344 static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs)
3345 {
3346 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3347 	struct extent_info ei = {};
3348 
3349 	if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) {
3350 		if (!ei.age)
3351 			return NO_CHECK_TYPE;
3352 		if (ei.age <= sbi->hot_data_age_threshold)
3353 			return CURSEG_HOT_DATA;
3354 		if (ei.age <= sbi->warm_data_age_threshold)
3355 			return CURSEG_WARM_DATA;
3356 		return CURSEG_COLD_DATA;
3357 	}
3358 	return NO_CHECK_TYPE;
3359 }
3360 
__get_segment_type_6(struct f2fs_io_info * fio)3361 static int __get_segment_type_6(struct f2fs_io_info *fio)
3362 {
3363 	if (fio->type == DATA) {
3364 		struct inode *inode = fio->page->mapping->host;
3365 		int type;
3366 
3367 		if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3368 			return CURSEG_COLD_DATA_PINNED;
3369 
3370 		if (page_private_gcing(fio->page)) {
3371 			if (fio->sbi->am.atgc_enabled &&
3372 				(fio->io_type == FS_DATA_IO) &&
3373 				(fio->sbi->gc_mode != GC_URGENT_HIGH) &&
3374 				__is_valid_data_blkaddr(fio->old_blkaddr) &&
3375 				!is_inode_flag_set(inode, FI_OPU_WRITE))
3376 				return CURSEG_ALL_DATA_ATGC;
3377 			else
3378 				return CURSEG_COLD_DATA;
3379 		}
3380 		if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3381 			return CURSEG_COLD_DATA;
3382 
3383 		type = __get_age_segment_type(inode, fio->page->index);
3384 		if (type != NO_CHECK_TYPE)
3385 			return type;
3386 
3387 		if (file_is_hot(inode) ||
3388 				is_inode_flag_set(inode, FI_HOT_DATA) ||
3389 				f2fs_is_cow_file(inode))
3390 			return CURSEG_HOT_DATA;
3391 		return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3392 	} else {
3393 		if (IS_DNODE(fio->page))
3394 			return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3395 						CURSEG_HOT_NODE;
3396 		return CURSEG_COLD_NODE;
3397 	}
3398 }
3399 
__get_segment_type(struct f2fs_io_info * fio)3400 static int __get_segment_type(struct f2fs_io_info *fio)
3401 {
3402 	int type = 0;
3403 
3404 	switch (F2FS_OPTION(fio->sbi).active_logs) {
3405 	case 2:
3406 		type = __get_segment_type_2(fio);
3407 		break;
3408 	case 4:
3409 		type = __get_segment_type_4(fio);
3410 		break;
3411 	case 6:
3412 		type = __get_segment_type_6(fio);
3413 		break;
3414 	default:
3415 		f2fs_bug_on(fio->sbi, true);
3416 	}
3417 
3418 	if (IS_HOT(type))
3419 		fio->temp = HOT;
3420 	else if (IS_WARM(type))
3421 		fio->temp = WARM;
3422 	else
3423 		fio->temp = COLD;
3424 	return type;
3425 }
3426 
f2fs_randomize_chunk(struct f2fs_sb_info * sbi,struct curseg_info * seg)3427 static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,
3428 		struct curseg_info *seg)
3429 {
3430 	/* To allocate block chunks in different sizes, use random number */
3431 	if (--seg->fragment_remained_chunk > 0)
3432 		return;
3433 
3434 	seg->fragment_remained_chunk =
3435 		get_random_u32_inclusive(1, sbi->max_fragment_chunk);
3436 	seg->next_blkoff +=
3437 		get_random_u32_inclusive(1, sbi->max_fragment_hole);
3438 }
3439 
f2fs_allocate_data_block(struct f2fs_sb_info * sbi,struct page * page,block_t old_blkaddr,block_t * new_blkaddr,struct f2fs_summary * sum,int type,struct f2fs_io_info * fio)3440 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3441 		block_t old_blkaddr, block_t *new_blkaddr,
3442 		struct f2fs_summary *sum, int type,
3443 		struct f2fs_io_info *fio)
3444 {
3445 	struct sit_info *sit_i = SIT_I(sbi);
3446 	struct curseg_info *curseg = CURSEG_I(sbi, type);
3447 	unsigned long long old_mtime;
3448 	bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3449 	struct seg_entry *se = NULL;
3450 	bool segment_full = false;
3451 
3452 	f2fs_down_read(&SM_I(sbi)->curseg_lock);
3453 
3454 	mutex_lock(&curseg->curseg_mutex);
3455 	down_write(&sit_i->sentry_lock);
3456 
3457 	if (from_gc) {
3458 		f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3459 		se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3460 		sanity_check_seg_type(sbi, se->type);
3461 		f2fs_bug_on(sbi, IS_NODESEG(se->type));
3462 	}
3463 	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3464 
3465 	f2fs_bug_on(sbi, curseg->next_blkoff >= BLKS_PER_SEG(sbi));
3466 
3467 	f2fs_wait_discard_bio(sbi, *new_blkaddr);
3468 
3469 	curseg->sum_blk->entries[curseg->next_blkoff] = *sum;
3470 	if (curseg->alloc_type == SSR) {
3471 		curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg);
3472 	} else {
3473 		curseg->next_blkoff++;
3474 		if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
3475 			f2fs_randomize_chunk(sbi, curseg);
3476 	}
3477 	if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno))
3478 		segment_full = true;
3479 	stat_inc_block_count(sbi, curseg);
3480 
3481 	if (from_gc) {
3482 		old_mtime = get_segment_mtime(sbi, old_blkaddr);
3483 	} else {
3484 		update_segment_mtime(sbi, old_blkaddr, 0);
3485 		old_mtime = 0;
3486 	}
3487 	update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3488 
3489 	/*
3490 	 * SIT information should be updated before segment allocation,
3491 	 * since SSR needs latest valid block information.
3492 	 */
3493 	update_sit_entry(sbi, *new_blkaddr, 1);
3494 	update_sit_entry(sbi, old_blkaddr, -1);
3495 
3496 	/*
3497 	 * If the current segment is full, flush it out and replace it with a
3498 	 * new segment.
3499 	 */
3500 	if (segment_full) {
3501 		if (type == CURSEG_COLD_DATA_PINNED &&
3502 		    !((curseg->segno + 1) % sbi->segs_per_sec)) {
3503 			write_sum_page(sbi, curseg->sum_blk,
3504 					GET_SUM_BLOCK(sbi, curseg->segno));
3505 			goto skip_new_segment;
3506 		}
3507 
3508 		if (from_gc) {
3509 			get_atssr_segment(sbi, type, se->type,
3510 						AT_SSR, se->mtime);
3511 		} else {
3512 			if (need_new_seg(sbi, type))
3513 				new_curseg(sbi, type, false);
3514 			else
3515 				change_curseg(sbi, type);
3516 			stat_inc_seg_type(sbi, curseg);
3517 		}
3518 	}
3519 
3520 skip_new_segment:
3521 	/*
3522 	 * segment dirty status should be updated after segment allocation,
3523 	 * so we just need to update status only one time after previous
3524 	 * segment being closed.
3525 	 */
3526 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3527 	locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3528 
3529 	if (IS_DATASEG(curseg->seg_type))
3530 		atomic64_inc(&sbi->allocated_data_blocks);
3531 
3532 	up_write(&sit_i->sentry_lock);
3533 
3534 	if (page && IS_NODESEG(curseg->seg_type)) {
3535 		fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3536 
3537 		f2fs_inode_chksum_set(sbi, page);
3538 	}
3539 
3540 	if (fio) {
3541 		struct f2fs_bio_info *io;
3542 
3543 		INIT_LIST_HEAD(&fio->list);
3544 		fio->in_list = 1;
3545 		io = sbi->write_io[fio->type] + fio->temp;
3546 		spin_lock(&io->io_lock);
3547 		list_add_tail(&fio->list, &io->io_list);
3548 		spin_unlock(&io->io_lock);
3549 	}
3550 
3551 	mutex_unlock(&curseg->curseg_mutex);
3552 
3553 	f2fs_up_read(&SM_I(sbi)->curseg_lock);
3554 }
3555 
f2fs_update_device_state(struct f2fs_sb_info * sbi,nid_t ino,block_t blkaddr,unsigned int blkcnt)3556 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3557 					block_t blkaddr, unsigned int blkcnt)
3558 {
3559 	if (!f2fs_is_multi_device(sbi))
3560 		return;
3561 
3562 	while (1) {
3563 		unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
3564 		unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
3565 
3566 		/* update device state for fsync */
3567 		f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
3568 
3569 		/* update device state for checkpoint */
3570 		if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3571 			spin_lock(&sbi->dev_lock);
3572 			f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3573 			spin_unlock(&sbi->dev_lock);
3574 		}
3575 
3576 		if (blkcnt <= blks)
3577 			break;
3578 		blkcnt -= blks;
3579 		blkaddr += blks;
3580 	}
3581 }
3582 
do_write_page(struct f2fs_summary * sum,struct f2fs_io_info * fio)3583 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3584 {
3585 	int type = __get_segment_type(fio);
3586 	bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3587 
3588 	if (keep_order)
3589 		f2fs_down_read(&fio->sbi->io_order_lock);
3590 
3591 	f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3592 			&fio->new_blkaddr, sum, type, fio);
3593 	if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)
3594 		f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr);
3595 
3596 	/* writeout dirty page into bdev */
3597 	f2fs_submit_page_write(fio);
3598 
3599 	f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
3600 
3601 	if (keep_order)
3602 		f2fs_up_read(&fio->sbi->io_order_lock);
3603 }
3604 
f2fs_do_write_meta_page(struct f2fs_sb_info * sbi,struct page * page,enum iostat_type io_type)3605 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3606 					enum iostat_type io_type)
3607 {
3608 	struct f2fs_io_info fio = {
3609 		.sbi = sbi,
3610 		.type = META,
3611 		.temp = HOT,
3612 		.op = REQ_OP_WRITE,
3613 		.op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3614 		.old_blkaddr = page->index,
3615 		.new_blkaddr = page->index,
3616 		.page = page,
3617 		.encrypted_page = NULL,
3618 		.in_list = 0,
3619 	};
3620 
3621 	if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3622 		fio.op_flags &= ~REQ_META;
3623 
3624 	set_page_writeback(page);
3625 	f2fs_submit_page_write(&fio);
3626 
3627 	stat_inc_meta_count(sbi, page->index);
3628 	f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);
3629 }
3630 
f2fs_do_write_node_page(unsigned int nid,struct f2fs_io_info * fio)3631 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3632 {
3633 	struct f2fs_summary sum;
3634 
3635 	set_summary(&sum, nid, 0, 0);
3636 	do_write_page(&sum, fio);
3637 
3638 	f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);
3639 }
3640 
f2fs_outplace_write_data(struct dnode_of_data * dn,struct f2fs_io_info * fio)3641 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3642 					struct f2fs_io_info *fio)
3643 {
3644 	struct f2fs_sb_info *sbi = fio->sbi;
3645 	struct f2fs_summary sum;
3646 
3647 	f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3648 	if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO)
3649 		f2fs_update_age_extent_cache(dn);
3650 	set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3651 	do_write_page(&sum, fio);
3652 	f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3653 
3654 	f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);
3655 }
3656 
f2fs_inplace_write_data(struct f2fs_io_info * fio)3657 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3658 {
3659 	int err;
3660 	struct f2fs_sb_info *sbi = fio->sbi;
3661 	unsigned int segno;
3662 
3663 	fio->new_blkaddr = fio->old_blkaddr;
3664 	/* i/o temperature is needed for passing down write hints */
3665 	__get_segment_type(fio);
3666 
3667 	segno = GET_SEGNO(sbi, fio->new_blkaddr);
3668 
3669 	if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3670 		set_sbi_flag(sbi, SBI_NEED_FSCK);
3671 		f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3672 			  __func__, segno);
3673 		err = -EFSCORRUPTED;
3674 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
3675 		goto drop_bio;
3676 	}
3677 
3678 	if (f2fs_cp_error(sbi)) {
3679 		err = -EIO;
3680 		goto drop_bio;
3681 	}
3682 
3683 	if (fio->meta_gc)
3684 		f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1);
3685 
3686 	stat_inc_inplace_blocks(fio->sbi);
3687 
3688 	if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi))
3689 		err = f2fs_merge_page_bio(fio);
3690 	else
3691 		err = f2fs_submit_page_bio(fio);
3692 	if (!err) {
3693 		f2fs_update_device_state(fio->sbi, fio->ino,
3694 						fio->new_blkaddr, 1);
3695 		f2fs_update_iostat(fio->sbi, fio->page->mapping->host,
3696 						fio->io_type, F2FS_BLKSIZE);
3697 	}
3698 
3699 	return err;
3700 drop_bio:
3701 	if (fio->bio && *(fio->bio)) {
3702 		struct bio *bio = *(fio->bio);
3703 
3704 		bio->bi_status = BLK_STS_IOERR;
3705 		bio_endio(bio);
3706 		*(fio->bio) = NULL;
3707 	}
3708 	return err;
3709 }
3710 
__f2fs_get_curseg(struct f2fs_sb_info * sbi,unsigned int segno)3711 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3712 						unsigned int segno)
3713 {
3714 	int i;
3715 
3716 	for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3717 		if (CURSEG_I(sbi, i)->segno == segno)
3718 			break;
3719 	}
3720 	return i;
3721 }
3722 
f2fs_do_replace_block(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,block_t old_blkaddr,block_t new_blkaddr,bool recover_curseg,bool recover_newaddr,bool from_gc)3723 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3724 				block_t old_blkaddr, block_t new_blkaddr,
3725 				bool recover_curseg, bool recover_newaddr,
3726 				bool from_gc)
3727 {
3728 	struct sit_info *sit_i = SIT_I(sbi);
3729 	struct curseg_info *curseg;
3730 	unsigned int segno, old_cursegno;
3731 	struct seg_entry *se;
3732 	int type;
3733 	unsigned short old_blkoff;
3734 	unsigned char old_alloc_type;
3735 
3736 	segno = GET_SEGNO(sbi, new_blkaddr);
3737 	se = get_seg_entry(sbi, segno);
3738 	type = se->type;
3739 
3740 	f2fs_down_write(&SM_I(sbi)->curseg_lock);
3741 
3742 	if (!recover_curseg) {
3743 		/* for recovery flow */
3744 		if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3745 			if (old_blkaddr == NULL_ADDR)
3746 				type = CURSEG_COLD_DATA;
3747 			else
3748 				type = CURSEG_WARM_DATA;
3749 		}
3750 	} else {
3751 		if (IS_CURSEG(sbi, segno)) {
3752 			/* se->type is volatile as SSR allocation */
3753 			type = __f2fs_get_curseg(sbi, segno);
3754 			f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3755 		} else {
3756 			type = CURSEG_WARM_DATA;
3757 		}
3758 	}
3759 
3760 	f2fs_bug_on(sbi, !IS_DATASEG(type));
3761 	curseg = CURSEG_I(sbi, type);
3762 
3763 	mutex_lock(&curseg->curseg_mutex);
3764 	down_write(&sit_i->sentry_lock);
3765 
3766 	old_cursegno = curseg->segno;
3767 	old_blkoff = curseg->next_blkoff;
3768 	old_alloc_type = curseg->alloc_type;
3769 
3770 	/* change the current segment */
3771 	if (segno != curseg->segno) {
3772 		curseg->next_segno = segno;
3773 		change_curseg(sbi, type);
3774 	}
3775 
3776 	curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3777 	curseg->sum_blk->entries[curseg->next_blkoff] = *sum;
3778 
3779 	if (!recover_curseg || recover_newaddr) {
3780 		if (!from_gc)
3781 			update_segment_mtime(sbi, new_blkaddr, 0);
3782 		update_sit_entry(sbi, new_blkaddr, 1);
3783 	}
3784 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3785 		f2fs_invalidate_internal_cache(sbi, old_blkaddr);
3786 		if (!from_gc)
3787 			update_segment_mtime(sbi, old_blkaddr, 0);
3788 		update_sit_entry(sbi, old_blkaddr, -1);
3789 	}
3790 
3791 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3792 	locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3793 
3794 	locate_dirty_segment(sbi, old_cursegno);
3795 
3796 	if (recover_curseg) {
3797 		if (old_cursegno != curseg->segno) {
3798 			curseg->next_segno = old_cursegno;
3799 			change_curseg(sbi, type);
3800 		}
3801 		curseg->next_blkoff = old_blkoff;
3802 		curseg->alloc_type = old_alloc_type;
3803 	}
3804 
3805 	up_write(&sit_i->sentry_lock);
3806 	mutex_unlock(&curseg->curseg_mutex);
3807 	f2fs_up_write(&SM_I(sbi)->curseg_lock);
3808 }
3809 
f2fs_replace_block(struct f2fs_sb_info * sbi,struct dnode_of_data * dn,block_t old_addr,block_t new_addr,unsigned char version,bool recover_curseg,bool recover_newaddr)3810 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3811 				block_t old_addr, block_t new_addr,
3812 				unsigned char version, bool recover_curseg,
3813 				bool recover_newaddr)
3814 {
3815 	struct f2fs_summary sum;
3816 
3817 	set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3818 
3819 	f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3820 					recover_curseg, recover_newaddr, false);
3821 
3822 	f2fs_update_data_blkaddr(dn, new_addr);
3823 }
3824 
f2fs_wait_on_page_writeback(struct page * page,enum page_type type,bool ordered,bool locked)3825 void f2fs_wait_on_page_writeback(struct page *page,
3826 				enum page_type type, bool ordered, bool locked)
3827 {
3828 	if (PageWriteback(page)) {
3829 		struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3830 
3831 		/* submit cached LFS IO */
3832 		f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3833 		/* submit cached IPU IO */
3834 		f2fs_submit_merged_ipu_write(sbi, NULL, page);
3835 		if (ordered) {
3836 			wait_on_page_writeback(page);
3837 			f2fs_bug_on(sbi, locked && PageWriteback(page));
3838 		} else {
3839 			wait_for_stable_page(page);
3840 		}
3841 	}
3842 }
3843 
f2fs_wait_on_block_writeback(struct inode * inode,block_t blkaddr)3844 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3845 {
3846 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3847 	struct page *cpage;
3848 
3849 	if (!f2fs_meta_inode_gc_required(inode))
3850 		return;
3851 
3852 	if (!__is_valid_data_blkaddr(blkaddr))
3853 		return;
3854 
3855 	cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3856 	if (cpage) {
3857 		f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3858 		f2fs_put_page(cpage, 1);
3859 	}
3860 }
3861 
f2fs_wait_on_block_writeback_range(struct inode * inode,block_t blkaddr,block_t len)3862 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3863 								block_t len)
3864 {
3865 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3866 	block_t i;
3867 
3868 	if (!f2fs_meta_inode_gc_required(inode))
3869 		return;
3870 
3871 	for (i = 0; i < len; i++)
3872 		f2fs_wait_on_block_writeback(inode, blkaddr + i);
3873 
3874 	f2fs_truncate_meta_inode_pages(sbi, blkaddr, len);
3875 }
3876 
read_compacted_summaries(struct f2fs_sb_info * sbi)3877 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3878 {
3879 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3880 	struct curseg_info *seg_i;
3881 	unsigned char *kaddr;
3882 	struct page *page;
3883 	block_t start;
3884 	int i, j, offset;
3885 
3886 	start = start_sum_block(sbi);
3887 
3888 	page = f2fs_get_meta_page(sbi, start++);
3889 	if (IS_ERR(page))
3890 		return PTR_ERR(page);
3891 	kaddr = (unsigned char *)page_address(page);
3892 
3893 	/* Step 1: restore nat cache */
3894 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3895 	memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3896 
3897 	/* Step 2: restore sit cache */
3898 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3899 	memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3900 	offset = 2 * SUM_JOURNAL_SIZE;
3901 
3902 	/* Step 3: restore summary entries */
3903 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3904 		unsigned short blk_off;
3905 		unsigned int segno;
3906 
3907 		seg_i = CURSEG_I(sbi, i);
3908 		segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3909 		blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3910 		seg_i->next_segno = segno;
3911 		reset_curseg(sbi, i, 0);
3912 		seg_i->alloc_type = ckpt->alloc_type[i];
3913 		seg_i->next_blkoff = blk_off;
3914 
3915 		if (seg_i->alloc_type == SSR)
3916 			blk_off = BLKS_PER_SEG(sbi);
3917 
3918 		for (j = 0; j < blk_off; j++) {
3919 			struct f2fs_summary *s;
3920 
3921 			s = (struct f2fs_summary *)(kaddr + offset);
3922 			seg_i->sum_blk->entries[j] = *s;
3923 			offset += SUMMARY_SIZE;
3924 			if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3925 						SUM_FOOTER_SIZE)
3926 				continue;
3927 
3928 			f2fs_put_page(page, 1);
3929 			page = NULL;
3930 
3931 			page = f2fs_get_meta_page(sbi, start++);
3932 			if (IS_ERR(page))
3933 				return PTR_ERR(page);
3934 			kaddr = (unsigned char *)page_address(page);
3935 			offset = 0;
3936 		}
3937 	}
3938 	f2fs_put_page(page, 1);
3939 	return 0;
3940 }
3941 
read_normal_summaries(struct f2fs_sb_info * sbi,int type)3942 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3943 {
3944 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3945 	struct f2fs_summary_block *sum;
3946 	struct curseg_info *curseg;
3947 	struct page *new;
3948 	unsigned short blk_off;
3949 	unsigned int segno = 0;
3950 	block_t blk_addr = 0;
3951 	int err = 0;
3952 
3953 	/* get segment number and block addr */
3954 	if (IS_DATASEG(type)) {
3955 		segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3956 		blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3957 							CURSEG_HOT_DATA]);
3958 		if (__exist_node_summaries(sbi))
3959 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3960 		else
3961 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3962 	} else {
3963 		segno = le32_to_cpu(ckpt->cur_node_segno[type -
3964 							CURSEG_HOT_NODE]);
3965 		blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3966 							CURSEG_HOT_NODE]);
3967 		if (__exist_node_summaries(sbi))
3968 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3969 							type - CURSEG_HOT_NODE);
3970 		else
3971 			blk_addr = GET_SUM_BLOCK(sbi, segno);
3972 	}
3973 
3974 	new = f2fs_get_meta_page(sbi, blk_addr);
3975 	if (IS_ERR(new))
3976 		return PTR_ERR(new);
3977 	sum = (struct f2fs_summary_block *)page_address(new);
3978 
3979 	if (IS_NODESEG(type)) {
3980 		if (__exist_node_summaries(sbi)) {
3981 			struct f2fs_summary *ns = &sum->entries[0];
3982 			int i;
3983 
3984 			for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) {
3985 				ns->version = 0;
3986 				ns->ofs_in_node = 0;
3987 			}
3988 		} else {
3989 			err = f2fs_restore_node_summary(sbi, segno, sum);
3990 			if (err)
3991 				goto out;
3992 		}
3993 	}
3994 
3995 	/* set uncompleted segment to curseg */
3996 	curseg = CURSEG_I(sbi, type);
3997 	mutex_lock(&curseg->curseg_mutex);
3998 
3999 	/* update journal info */
4000 	down_write(&curseg->journal_rwsem);
4001 	memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
4002 	up_write(&curseg->journal_rwsem);
4003 
4004 	memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
4005 	memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
4006 	curseg->next_segno = segno;
4007 	reset_curseg(sbi, type, 0);
4008 	curseg->alloc_type = ckpt->alloc_type[type];
4009 	curseg->next_blkoff = blk_off;
4010 	mutex_unlock(&curseg->curseg_mutex);
4011 out:
4012 	f2fs_put_page(new, 1);
4013 	return err;
4014 }
4015 
restore_curseg_summaries(struct f2fs_sb_info * sbi)4016 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
4017 {
4018 	struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
4019 	struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
4020 	int type = CURSEG_HOT_DATA;
4021 	int err;
4022 
4023 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
4024 		int npages = f2fs_npages_for_summary_flush(sbi, true);
4025 
4026 		if (npages >= 2)
4027 			f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
4028 							META_CP, true);
4029 
4030 		/* restore for compacted data summary */
4031 		err = read_compacted_summaries(sbi);
4032 		if (err)
4033 			return err;
4034 		type = CURSEG_HOT_NODE;
4035 	}
4036 
4037 	if (__exist_node_summaries(sbi))
4038 		f2fs_ra_meta_pages(sbi,
4039 				sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
4040 				NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
4041 
4042 	for (; type <= CURSEG_COLD_NODE; type++) {
4043 		err = read_normal_summaries(sbi, type);
4044 		if (err)
4045 			return err;
4046 	}
4047 
4048 	/* sanity check for summary blocks */
4049 	if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
4050 			sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
4051 		f2fs_err(sbi, "invalid journal entries nats %u sits %u",
4052 			 nats_in_cursum(nat_j), sits_in_cursum(sit_j));
4053 		return -EINVAL;
4054 	}
4055 
4056 	return 0;
4057 }
4058 
write_compacted_summaries(struct f2fs_sb_info * sbi,block_t blkaddr)4059 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
4060 {
4061 	struct page *page;
4062 	unsigned char *kaddr;
4063 	struct f2fs_summary *summary;
4064 	struct curseg_info *seg_i;
4065 	int written_size = 0;
4066 	int i, j;
4067 
4068 	page = f2fs_grab_meta_page(sbi, blkaddr++);
4069 	kaddr = (unsigned char *)page_address(page);
4070 	memset(kaddr, 0, PAGE_SIZE);
4071 
4072 	/* Step 1: write nat cache */
4073 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
4074 	memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
4075 	written_size += SUM_JOURNAL_SIZE;
4076 
4077 	/* Step 2: write sit cache */
4078 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
4079 	memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
4080 	written_size += SUM_JOURNAL_SIZE;
4081 
4082 	/* Step 3: write summary entries */
4083 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
4084 		seg_i = CURSEG_I(sbi, i);
4085 		for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) {
4086 			if (!page) {
4087 				page = f2fs_grab_meta_page(sbi, blkaddr++);
4088 				kaddr = (unsigned char *)page_address(page);
4089 				memset(kaddr, 0, PAGE_SIZE);
4090 				written_size = 0;
4091 			}
4092 			summary = (struct f2fs_summary *)(kaddr + written_size);
4093 			*summary = seg_i->sum_blk->entries[j];
4094 			written_size += SUMMARY_SIZE;
4095 
4096 			if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
4097 							SUM_FOOTER_SIZE)
4098 				continue;
4099 
4100 			set_page_dirty(page);
4101 			f2fs_put_page(page, 1);
4102 			page = NULL;
4103 		}
4104 	}
4105 	if (page) {
4106 		set_page_dirty(page);
4107 		f2fs_put_page(page, 1);
4108 	}
4109 }
4110 
write_normal_summaries(struct f2fs_sb_info * sbi,block_t blkaddr,int type)4111 static void write_normal_summaries(struct f2fs_sb_info *sbi,
4112 					block_t blkaddr, int type)
4113 {
4114 	int i, end;
4115 
4116 	if (IS_DATASEG(type))
4117 		end = type + NR_CURSEG_DATA_TYPE;
4118 	else
4119 		end = type + NR_CURSEG_NODE_TYPE;
4120 
4121 	for (i = type; i < end; i++)
4122 		write_current_sum_page(sbi, i, blkaddr + (i - type));
4123 }
4124 
f2fs_write_data_summaries(struct f2fs_sb_info * sbi,block_t start_blk)4125 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4126 {
4127 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
4128 		write_compacted_summaries(sbi, start_blk);
4129 	else
4130 		write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4131 }
4132 
f2fs_write_node_summaries(struct f2fs_sb_info * sbi,block_t start_blk)4133 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4134 {
4135 	write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4136 }
4137 
f2fs_lookup_journal_in_cursum(struct f2fs_journal * journal,int type,unsigned int val,int alloc)4138 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
4139 					unsigned int val, int alloc)
4140 {
4141 	int i;
4142 
4143 	if (type == NAT_JOURNAL) {
4144 		for (i = 0; i < nats_in_cursum(journal); i++) {
4145 			if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4146 				return i;
4147 		}
4148 		if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
4149 			return update_nats_in_cursum(journal, 1);
4150 	} else if (type == SIT_JOURNAL) {
4151 		for (i = 0; i < sits_in_cursum(journal); i++)
4152 			if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4153 				return i;
4154 		if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
4155 			return update_sits_in_cursum(journal, 1);
4156 	}
4157 	return -1;
4158 }
4159 
get_current_sit_page(struct f2fs_sb_info * sbi,unsigned int segno)4160 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
4161 					unsigned int segno)
4162 {
4163 	return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
4164 }
4165 
get_next_sit_page(struct f2fs_sb_info * sbi,unsigned int start)4166 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
4167 					unsigned int start)
4168 {
4169 	struct sit_info *sit_i = SIT_I(sbi);
4170 	struct page *page;
4171 	pgoff_t src_off, dst_off;
4172 
4173 	src_off = current_sit_addr(sbi, start);
4174 	dst_off = next_sit_addr(sbi, src_off);
4175 
4176 	page = f2fs_grab_meta_page(sbi, dst_off);
4177 	seg_info_to_sit_page(sbi, page, start);
4178 
4179 	set_page_dirty(page);
4180 	set_to_next_sit(sit_i, start);
4181 
4182 	return page;
4183 }
4184 
grab_sit_entry_set(void)4185 static struct sit_entry_set *grab_sit_entry_set(void)
4186 {
4187 	struct sit_entry_set *ses =
4188 			f2fs_kmem_cache_alloc(sit_entry_set_slab,
4189 						GFP_NOFS, true, NULL);
4190 
4191 	ses->entry_cnt = 0;
4192 	INIT_LIST_HEAD(&ses->set_list);
4193 	return ses;
4194 }
4195 
release_sit_entry_set(struct sit_entry_set * ses)4196 static void release_sit_entry_set(struct sit_entry_set *ses)
4197 {
4198 	list_del(&ses->set_list);
4199 	kmem_cache_free(sit_entry_set_slab, ses);
4200 }
4201 
adjust_sit_entry_set(struct sit_entry_set * ses,struct list_head * head)4202 static void adjust_sit_entry_set(struct sit_entry_set *ses,
4203 						struct list_head *head)
4204 {
4205 	struct sit_entry_set *next = ses;
4206 
4207 	if (list_is_last(&ses->set_list, head))
4208 		return;
4209 
4210 	list_for_each_entry_continue(next, head, set_list)
4211 		if (ses->entry_cnt <= next->entry_cnt) {
4212 			list_move_tail(&ses->set_list, &next->set_list);
4213 			return;
4214 		}
4215 
4216 	list_move_tail(&ses->set_list, head);
4217 }
4218 
add_sit_entry(unsigned int segno,struct list_head * head)4219 static void add_sit_entry(unsigned int segno, struct list_head *head)
4220 {
4221 	struct sit_entry_set *ses;
4222 	unsigned int start_segno = START_SEGNO(segno);
4223 
4224 	list_for_each_entry(ses, head, set_list) {
4225 		if (ses->start_segno == start_segno) {
4226 			ses->entry_cnt++;
4227 			adjust_sit_entry_set(ses, head);
4228 			return;
4229 		}
4230 	}
4231 
4232 	ses = grab_sit_entry_set();
4233 
4234 	ses->start_segno = start_segno;
4235 	ses->entry_cnt++;
4236 	list_add(&ses->set_list, head);
4237 }
4238 
add_sits_in_set(struct f2fs_sb_info * sbi)4239 static void add_sits_in_set(struct f2fs_sb_info *sbi)
4240 {
4241 	struct f2fs_sm_info *sm_info = SM_I(sbi);
4242 	struct list_head *set_list = &sm_info->sit_entry_set;
4243 	unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4244 	unsigned int segno;
4245 
4246 	for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4247 		add_sit_entry(segno, set_list);
4248 }
4249 
remove_sits_in_journal(struct f2fs_sb_info * sbi)4250 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4251 {
4252 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4253 	struct f2fs_journal *journal = curseg->journal;
4254 	int i;
4255 
4256 	down_write(&curseg->journal_rwsem);
4257 	for (i = 0; i < sits_in_cursum(journal); i++) {
4258 		unsigned int segno;
4259 		bool dirtied;
4260 
4261 		segno = le32_to_cpu(segno_in_journal(journal, i));
4262 		dirtied = __mark_sit_entry_dirty(sbi, segno);
4263 
4264 		if (!dirtied)
4265 			add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4266 	}
4267 	update_sits_in_cursum(journal, -i);
4268 	up_write(&curseg->journal_rwsem);
4269 }
4270 
4271 /*
4272  * CP calls this function, which flushes SIT entries including sit_journal,
4273  * and moves prefree segs to free segs.
4274  */
f2fs_flush_sit_entries(struct f2fs_sb_info * sbi,struct cp_control * cpc)4275 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4276 {
4277 	struct sit_info *sit_i = SIT_I(sbi);
4278 	unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4279 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4280 	struct f2fs_journal *journal = curseg->journal;
4281 	struct sit_entry_set *ses, *tmp;
4282 	struct list_head *head = &SM_I(sbi)->sit_entry_set;
4283 	bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4284 	struct seg_entry *se;
4285 
4286 	down_write(&sit_i->sentry_lock);
4287 
4288 	if (!sit_i->dirty_sentries)
4289 		goto out;
4290 
4291 	/*
4292 	 * add and account sit entries of dirty bitmap in sit entry
4293 	 * set temporarily
4294 	 */
4295 	add_sits_in_set(sbi);
4296 
4297 	/*
4298 	 * if there are no enough space in journal to store dirty sit
4299 	 * entries, remove all entries from journal and add and account
4300 	 * them in sit entry set.
4301 	 */
4302 	if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4303 								!to_journal)
4304 		remove_sits_in_journal(sbi);
4305 
4306 	/*
4307 	 * there are two steps to flush sit entries:
4308 	 * #1, flush sit entries to journal in current cold data summary block.
4309 	 * #2, flush sit entries to sit page.
4310 	 */
4311 	list_for_each_entry_safe(ses, tmp, head, set_list) {
4312 		struct page *page = NULL;
4313 		struct f2fs_sit_block *raw_sit = NULL;
4314 		unsigned int start_segno = ses->start_segno;
4315 		unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4316 						(unsigned long)MAIN_SEGS(sbi));
4317 		unsigned int segno = start_segno;
4318 
4319 		if (to_journal &&
4320 			!__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4321 			to_journal = false;
4322 
4323 		if (to_journal) {
4324 			down_write(&curseg->journal_rwsem);
4325 		} else {
4326 			page = get_next_sit_page(sbi, start_segno);
4327 			raw_sit = page_address(page);
4328 		}
4329 
4330 		/* flush dirty sit entries in region of current sit set */
4331 		for_each_set_bit_from(segno, bitmap, end) {
4332 			int offset, sit_offset;
4333 
4334 			se = get_seg_entry(sbi, segno);
4335 #ifdef CONFIG_F2FS_CHECK_FS
4336 			if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4337 						SIT_VBLOCK_MAP_SIZE))
4338 				f2fs_bug_on(sbi, 1);
4339 #endif
4340 
4341 			/* add discard candidates */
4342 			if (!(cpc->reason & CP_DISCARD)) {
4343 				cpc->trim_start = segno;
4344 				add_discard_addrs(sbi, cpc, false);
4345 			}
4346 
4347 			if (to_journal) {
4348 				offset = f2fs_lookup_journal_in_cursum(journal,
4349 							SIT_JOURNAL, segno, 1);
4350 				f2fs_bug_on(sbi, offset < 0);
4351 				segno_in_journal(journal, offset) =
4352 							cpu_to_le32(segno);
4353 				seg_info_to_raw_sit(se,
4354 					&sit_in_journal(journal, offset));
4355 				check_block_count(sbi, segno,
4356 					&sit_in_journal(journal, offset));
4357 			} else {
4358 				sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4359 				seg_info_to_raw_sit(se,
4360 						&raw_sit->entries[sit_offset]);
4361 				check_block_count(sbi, segno,
4362 						&raw_sit->entries[sit_offset]);
4363 			}
4364 
4365 			__clear_bit(segno, bitmap);
4366 			sit_i->dirty_sentries--;
4367 			ses->entry_cnt--;
4368 		}
4369 
4370 		if (to_journal)
4371 			up_write(&curseg->journal_rwsem);
4372 		else
4373 			f2fs_put_page(page, 1);
4374 
4375 		f2fs_bug_on(sbi, ses->entry_cnt);
4376 		release_sit_entry_set(ses);
4377 	}
4378 
4379 	f2fs_bug_on(sbi, !list_empty(head));
4380 	f2fs_bug_on(sbi, sit_i->dirty_sentries);
4381 out:
4382 	if (cpc->reason & CP_DISCARD) {
4383 		__u64 trim_start = cpc->trim_start;
4384 
4385 		for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4386 			add_discard_addrs(sbi, cpc, false);
4387 
4388 		cpc->trim_start = trim_start;
4389 	}
4390 	up_write(&sit_i->sentry_lock);
4391 
4392 	set_prefree_as_free_segments(sbi);
4393 }
4394 
build_sit_info(struct f2fs_sb_info * sbi)4395 static int build_sit_info(struct f2fs_sb_info *sbi)
4396 {
4397 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4398 	struct sit_info *sit_i;
4399 	unsigned int sit_segs, start;
4400 	char *src_bitmap, *bitmap;
4401 	unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4402 	unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4403 
4404 	/* allocate memory for SIT information */
4405 	sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4406 	if (!sit_i)
4407 		return -ENOMEM;
4408 
4409 	SM_I(sbi)->sit_info = sit_i;
4410 
4411 	sit_i->sentries =
4412 		f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4413 					      MAIN_SEGS(sbi)),
4414 			      GFP_KERNEL);
4415 	if (!sit_i->sentries)
4416 		return -ENOMEM;
4417 
4418 	main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4419 	sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4420 								GFP_KERNEL);
4421 	if (!sit_i->dirty_sentries_bitmap)
4422 		return -ENOMEM;
4423 
4424 #ifdef CONFIG_F2FS_CHECK_FS
4425 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4426 #else
4427 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4428 #endif
4429 	sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4430 	if (!sit_i->bitmap)
4431 		return -ENOMEM;
4432 
4433 	bitmap = sit_i->bitmap;
4434 
4435 	for (start = 0; start < MAIN_SEGS(sbi); start++) {
4436 		sit_i->sentries[start].cur_valid_map = bitmap;
4437 		bitmap += SIT_VBLOCK_MAP_SIZE;
4438 
4439 		sit_i->sentries[start].ckpt_valid_map = bitmap;
4440 		bitmap += SIT_VBLOCK_MAP_SIZE;
4441 
4442 #ifdef CONFIG_F2FS_CHECK_FS
4443 		sit_i->sentries[start].cur_valid_map_mir = bitmap;
4444 		bitmap += SIT_VBLOCK_MAP_SIZE;
4445 #endif
4446 
4447 		if (discard_map) {
4448 			sit_i->sentries[start].discard_map = bitmap;
4449 			bitmap += SIT_VBLOCK_MAP_SIZE;
4450 		}
4451 	}
4452 
4453 	sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4454 	if (!sit_i->tmp_map)
4455 		return -ENOMEM;
4456 
4457 	if (__is_large_section(sbi)) {
4458 		sit_i->sec_entries =
4459 			f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4460 						      MAIN_SECS(sbi)),
4461 				      GFP_KERNEL);
4462 		if (!sit_i->sec_entries)
4463 			return -ENOMEM;
4464 	}
4465 
4466 	/* get information related with SIT */
4467 	sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4468 
4469 	/* setup SIT bitmap from ckeckpoint pack */
4470 	sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4471 	src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4472 
4473 	sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4474 	if (!sit_i->sit_bitmap)
4475 		return -ENOMEM;
4476 
4477 #ifdef CONFIG_F2FS_CHECK_FS
4478 	sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4479 					sit_bitmap_size, GFP_KERNEL);
4480 	if (!sit_i->sit_bitmap_mir)
4481 		return -ENOMEM;
4482 
4483 	sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4484 					main_bitmap_size, GFP_KERNEL);
4485 	if (!sit_i->invalid_segmap)
4486 		return -ENOMEM;
4487 #endif
4488 
4489 	sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4490 	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4491 	sit_i->written_valid_blocks = 0;
4492 	sit_i->bitmap_size = sit_bitmap_size;
4493 	sit_i->dirty_sentries = 0;
4494 	sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4495 	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4496 	sit_i->mounted_time = ktime_get_boottime_seconds();
4497 	init_rwsem(&sit_i->sentry_lock);
4498 	return 0;
4499 }
4500 
build_free_segmap(struct f2fs_sb_info * sbi)4501 static int build_free_segmap(struct f2fs_sb_info *sbi)
4502 {
4503 	struct free_segmap_info *free_i;
4504 	unsigned int bitmap_size, sec_bitmap_size;
4505 
4506 	/* allocate memory for free segmap information */
4507 	free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4508 	if (!free_i)
4509 		return -ENOMEM;
4510 
4511 	SM_I(sbi)->free_info = free_i;
4512 
4513 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4514 	free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4515 	if (!free_i->free_segmap)
4516 		return -ENOMEM;
4517 
4518 	sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4519 	free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4520 	if (!free_i->free_secmap)
4521 		return -ENOMEM;
4522 
4523 	/* set all segments as dirty temporarily */
4524 	memset(free_i->free_segmap, 0xff, bitmap_size);
4525 	memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4526 
4527 	/* init free segmap information */
4528 	free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4529 	free_i->free_segments = 0;
4530 	free_i->free_sections = 0;
4531 	spin_lock_init(&free_i->segmap_lock);
4532 	return 0;
4533 }
4534 
build_curseg(struct f2fs_sb_info * sbi)4535 static int build_curseg(struct f2fs_sb_info *sbi)
4536 {
4537 	struct curseg_info *array;
4538 	int i;
4539 
4540 	array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4541 					sizeof(*array)), GFP_KERNEL);
4542 	if (!array)
4543 		return -ENOMEM;
4544 
4545 	SM_I(sbi)->curseg_array = array;
4546 
4547 	for (i = 0; i < NO_CHECK_TYPE; i++) {
4548 		mutex_init(&array[i].curseg_mutex);
4549 		array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4550 		if (!array[i].sum_blk)
4551 			return -ENOMEM;
4552 		init_rwsem(&array[i].journal_rwsem);
4553 		array[i].journal = f2fs_kzalloc(sbi,
4554 				sizeof(struct f2fs_journal), GFP_KERNEL);
4555 		if (!array[i].journal)
4556 			return -ENOMEM;
4557 		if (i < NR_PERSISTENT_LOG)
4558 			array[i].seg_type = CURSEG_HOT_DATA + i;
4559 		else if (i == CURSEG_COLD_DATA_PINNED)
4560 			array[i].seg_type = CURSEG_COLD_DATA;
4561 		else if (i == CURSEG_ALL_DATA_ATGC)
4562 			array[i].seg_type = CURSEG_COLD_DATA;
4563 		array[i].segno = NULL_SEGNO;
4564 		array[i].next_blkoff = 0;
4565 		array[i].inited = false;
4566 	}
4567 	return restore_curseg_summaries(sbi);
4568 }
4569 
build_sit_entries(struct f2fs_sb_info * sbi)4570 static int build_sit_entries(struct f2fs_sb_info *sbi)
4571 {
4572 	struct sit_info *sit_i = SIT_I(sbi);
4573 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4574 	struct f2fs_journal *journal = curseg->journal;
4575 	struct seg_entry *se;
4576 	struct f2fs_sit_entry sit;
4577 	int sit_blk_cnt = SIT_BLK_CNT(sbi);
4578 	unsigned int i, start, end;
4579 	unsigned int readed, start_blk = 0;
4580 	int err = 0;
4581 	block_t sit_valid_blocks[2] = {0, 0};
4582 
4583 	do {
4584 		readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4585 							META_SIT, true);
4586 
4587 		start = start_blk * sit_i->sents_per_block;
4588 		end = (start_blk + readed) * sit_i->sents_per_block;
4589 
4590 		for (; start < end && start < MAIN_SEGS(sbi); start++) {
4591 			struct f2fs_sit_block *sit_blk;
4592 			struct page *page;
4593 
4594 			se = &sit_i->sentries[start];
4595 			page = get_current_sit_page(sbi, start);
4596 			if (IS_ERR(page))
4597 				return PTR_ERR(page);
4598 			sit_blk = (struct f2fs_sit_block *)page_address(page);
4599 			sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4600 			f2fs_put_page(page, 1);
4601 
4602 			err = check_block_count(sbi, start, &sit);
4603 			if (err)
4604 				return err;
4605 			seg_info_from_raw_sit(se, &sit);
4606 
4607 			if (se->type >= NR_PERSISTENT_LOG) {
4608 				f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4609 							se->type, start);
4610 				f2fs_handle_error(sbi,
4611 						ERROR_INCONSISTENT_SUM_TYPE);
4612 				return -EFSCORRUPTED;
4613 			}
4614 
4615 			sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4616 
4617 			if (!f2fs_block_unit_discard(sbi))
4618 				goto init_discard_map_done;
4619 
4620 			/* build discard map only one time */
4621 			if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4622 				memset(se->discard_map, 0xff,
4623 						SIT_VBLOCK_MAP_SIZE);
4624 				goto init_discard_map_done;
4625 			}
4626 			memcpy(se->discard_map, se->cur_valid_map,
4627 						SIT_VBLOCK_MAP_SIZE);
4628 			sbi->discard_blks += BLKS_PER_SEG(sbi) -
4629 						se->valid_blocks;
4630 init_discard_map_done:
4631 			if (__is_large_section(sbi))
4632 				get_sec_entry(sbi, start)->valid_blocks +=
4633 							se->valid_blocks;
4634 		}
4635 		start_blk += readed;
4636 	} while (start_blk < sit_blk_cnt);
4637 
4638 	down_read(&curseg->journal_rwsem);
4639 	for (i = 0; i < sits_in_cursum(journal); i++) {
4640 		unsigned int old_valid_blocks;
4641 
4642 		start = le32_to_cpu(segno_in_journal(journal, i));
4643 		if (start >= MAIN_SEGS(sbi)) {
4644 			f2fs_err(sbi, "Wrong journal entry on segno %u",
4645 				 start);
4646 			err = -EFSCORRUPTED;
4647 			f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL);
4648 			break;
4649 		}
4650 
4651 		se = &sit_i->sentries[start];
4652 		sit = sit_in_journal(journal, i);
4653 
4654 		old_valid_blocks = se->valid_blocks;
4655 
4656 		sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
4657 
4658 		err = check_block_count(sbi, start, &sit);
4659 		if (err)
4660 			break;
4661 		seg_info_from_raw_sit(se, &sit);
4662 
4663 		if (se->type >= NR_PERSISTENT_LOG) {
4664 			f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4665 							se->type, start);
4666 			err = -EFSCORRUPTED;
4667 			f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
4668 			break;
4669 		}
4670 
4671 		sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4672 
4673 		if (f2fs_block_unit_discard(sbi)) {
4674 			if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4675 				memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4676 			} else {
4677 				memcpy(se->discard_map, se->cur_valid_map,
4678 							SIT_VBLOCK_MAP_SIZE);
4679 				sbi->discard_blks += old_valid_blocks;
4680 				sbi->discard_blks -= se->valid_blocks;
4681 			}
4682 		}
4683 
4684 		if (__is_large_section(sbi)) {
4685 			get_sec_entry(sbi, start)->valid_blocks +=
4686 							se->valid_blocks;
4687 			get_sec_entry(sbi, start)->valid_blocks -=
4688 							old_valid_blocks;
4689 		}
4690 	}
4691 	up_read(&curseg->journal_rwsem);
4692 
4693 	if (err)
4694 		return err;
4695 
4696 	if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
4697 		f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4698 			 sit_valid_blocks[NODE], valid_node_count(sbi));
4699 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT);
4700 		return -EFSCORRUPTED;
4701 	}
4702 
4703 	if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
4704 				valid_user_blocks(sbi)) {
4705 		f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
4706 			 sit_valid_blocks[DATA], sit_valid_blocks[NODE],
4707 			 valid_user_blocks(sbi));
4708 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT);
4709 		return -EFSCORRUPTED;
4710 	}
4711 
4712 	return 0;
4713 }
4714 
init_free_segmap(struct f2fs_sb_info * sbi)4715 static void init_free_segmap(struct f2fs_sb_info *sbi)
4716 {
4717 	unsigned int start;
4718 	int type;
4719 	struct seg_entry *sentry;
4720 
4721 	for (start = 0; start < MAIN_SEGS(sbi); start++) {
4722 		if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4723 			continue;
4724 		sentry = get_seg_entry(sbi, start);
4725 		if (!sentry->valid_blocks)
4726 			__set_free(sbi, start);
4727 		else
4728 			SIT_I(sbi)->written_valid_blocks +=
4729 						sentry->valid_blocks;
4730 	}
4731 
4732 	/* set use the current segments */
4733 	for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4734 		struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4735 
4736 		__set_test_and_inuse(sbi, curseg_t->segno);
4737 	}
4738 }
4739 
init_dirty_segmap(struct f2fs_sb_info * sbi)4740 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4741 {
4742 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4743 	struct free_segmap_info *free_i = FREE_I(sbi);
4744 	unsigned int segno = 0, offset = 0, secno;
4745 	block_t valid_blocks, usable_blks_in_seg;
4746 
4747 	while (1) {
4748 		/* find dirty segment based on free segmap */
4749 		segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4750 		if (segno >= MAIN_SEGS(sbi))
4751 			break;
4752 		offset = segno + 1;
4753 		valid_blocks = get_valid_blocks(sbi, segno, false);
4754 		usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4755 		if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4756 			continue;
4757 		if (valid_blocks > usable_blks_in_seg) {
4758 			f2fs_bug_on(sbi, 1);
4759 			continue;
4760 		}
4761 		mutex_lock(&dirty_i->seglist_lock);
4762 		__locate_dirty_segment(sbi, segno, DIRTY);
4763 		mutex_unlock(&dirty_i->seglist_lock);
4764 	}
4765 
4766 	if (!__is_large_section(sbi))
4767 		return;
4768 
4769 	mutex_lock(&dirty_i->seglist_lock);
4770 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) {
4771 		valid_blocks = get_valid_blocks(sbi, segno, true);
4772 		secno = GET_SEC_FROM_SEG(sbi, segno);
4773 
4774 		if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi))
4775 			continue;
4776 		if (IS_CURSEC(sbi, secno))
4777 			continue;
4778 		set_bit(secno, dirty_i->dirty_secmap);
4779 	}
4780 	mutex_unlock(&dirty_i->seglist_lock);
4781 }
4782 
init_victim_secmap(struct f2fs_sb_info * sbi)4783 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4784 {
4785 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4786 	unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4787 
4788 	dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4789 	if (!dirty_i->victim_secmap)
4790 		return -ENOMEM;
4791 
4792 	dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4793 	if (!dirty_i->pinned_secmap)
4794 		return -ENOMEM;
4795 
4796 	dirty_i->pinned_secmap_cnt = 0;
4797 	dirty_i->enable_pin_section = true;
4798 	return 0;
4799 }
4800 
build_dirty_segmap(struct f2fs_sb_info * sbi)4801 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4802 {
4803 	struct dirty_seglist_info *dirty_i;
4804 	unsigned int bitmap_size, i;
4805 
4806 	/* allocate memory for dirty segments list information */
4807 	dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4808 								GFP_KERNEL);
4809 	if (!dirty_i)
4810 		return -ENOMEM;
4811 
4812 	SM_I(sbi)->dirty_info = dirty_i;
4813 	mutex_init(&dirty_i->seglist_lock);
4814 
4815 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4816 
4817 	for (i = 0; i < NR_DIRTY_TYPE; i++) {
4818 		dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4819 								GFP_KERNEL);
4820 		if (!dirty_i->dirty_segmap[i])
4821 			return -ENOMEM;
4822 	}
4823 
4824 	if (__is_large_section(sbi)) {
4825 		bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4826 		dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4827 						bitmap_size, GFP_KERNEL);
4828 		if (!dirty_i->dirty_secmap)
4829 			return -ENOMEM;
4830 	}
4831 
4832 	init_dirty_segmap(sbi);
4833 	return init_victim_secmap(sbi);
4834 }
4835 
sanity_check_curseg(struct f2fs_sb_info * sbi)4836 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4837 {
4838 	int i;
4839 
4840 	/*
4841 	 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
4842 	 * In LFS curseg, all blkaddr after .next_blkoff should be unused.
4843 	 */
4844 	for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4845 		struct curseg_info *curseg = CURSEG_I(sbi, i);
4846 		struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4847 		unsigned int blkofs = curseg->next_blkoff;
4848 
4849 		if (f2fs_sb_has_readonly(sbi) &&
4850 			i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
4851 			continue;
4852 
4853 		sanity_check_seg_type(sbi, curseg->seg_type);
4854 
4855 		if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
4856 			f2fs_err(sbi,
4857 				 "Current segment has invalid alloc_type:%d",
4858 				 curseg->alloc_type);
4859 			f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
4860 			return -EFSCORRUPTED;
4861 		}
4862 
4863 		if (f2fs_test_bit(blkofs, se->cur_valid_map))
4864 			goto out;
4865 
4866 		if (curseg->alloc_type == SSR)
4867 			continue;
4868 
4869 		for (blkofs += 1; blkofs < BLKS_PER_SEG(sbi); blkofs++) {
4870 			if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4871 				continue;
4872 out:
4873 			f2fs_err(sbi,
4874 				 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4875 				 i, curseg->segno, curseg->alloc_type,
4876 				 curseg->next_blkoff, blkofs);
4877 			f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
4878 			return -EFSCORRUPTED;
4879 		}
4880 	}
4881 	return 0;
4882 }
4883 
4884 #ifdef CONFIG_BLK_DEV_ZONED
4885 
check_zone_write_pointer(struct f2fs_sb_info * sbi,struct f2fs_dev_info * fdev,struct blk_zone * zone)4886 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4887 				    struct f2fs_dev_info *fdev,
4888 				    struct blk_zone *zone)
4889 {
4890 	unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4891 	block_t zone_block, wp_block, last_valid_block;
4892 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4893 	int i, s, b, ret;
4894 	struct seg_entry *se;
4895 
4896 	if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4897 		return 0;
4898 
4899 	wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4900 	wp_segno = GET_SEGNO(sbi, wp_block);
4901 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4902 	zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4903 	zone_segno = GET_SEGNO(sbi, zone_block);
4904 	zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4905 
4906 	if (zone_segno >= MAIN_SEGS(sbi))
4907 		return 0;
4908 
4909 	/*
4910 	 * Skip check of zones cursegs point to, since
4911 	 * fix_curseg_write_pointer() checks them.
4912 	 */
4913 	for (i = 0; i < NO_CHECK_TYPE; i++)
4914 		if (zone_secno == GET_SEC_FROM_SEG(sbi,
4915 						   CURSEG_I(sbi, i)->segno))
4916 			return 0;
4917 
4918 	/*
4919 	 * Get last valid block of the zone.
4920 	 */
4921 	last_valid_block = zone_block - 1;
4922 	for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4923 		segno = zone_segno + s;
4924 		se = get_seg_entry(sbi, segno);
4925 		for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4926 			if (f2fs_test_bit(b, se->cur_valid_map)) {
4927 				last_valid_block = START_BLOCK(sbi, segno) + b;
4928 				break;
4929 			}
4930 		if (last_valid_block >= zone_block)
4931 			break;
4932 	}
4933 
4934 	/*
4935 	 * The write pointer matches with the valid blocks or
4936 	 * already points to the end of the zone.
4937 	 */
4938 	if ((last_valid_block + 1 == wp_block) ||
4939 			(zone->wp == zone->start + zone->len))
4940 		return 0;
4941 
4942 	if (last_valid_block + 1 == zone_block) {
4943 		/*
4944 		 * If there is no valid block in the zone and if write pointer
4945 		 * is not at zone start, reset the write pointer.
4946 		 */
4947 		f2fs_notice(sbi,
4948 			    "Zone without valid block has non-zero write "
4949 			    "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4950 			    wp_segno, wp_blkoff);
4951 		ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4952 					zone->len >> log_sectors_per_block);
4953 		if (ret)
4954 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4955 				 fdev->path, ret);
4956 
4957 		return ret;
4958 	}
4959 
4960 	/*
4961 	 * If there are valid blocks and the write pointer doesn't
4962 	 * match with them, we need to report the inconsistency and
4963 	 * fill the zone till the end to close the zone. This inconsistency
4964 	 * does not cause write error because the zone will not be selected
4965 	 * for write operation until it get discarded.
4966 	 */
4967 	f2fs_notice(sbi, "Valid blocks are not aligned with write pointer: "
4968 		    "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4969 		    GET_SEGNO(sbi, last_valid_block),
4970 		    GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4971 		    wp_segno, wp_blkoff);
4972 
4973 	ret = blkdev_zone_mgmt(fdev->bdev, REQ_OP_ZONE_FINISH,
4974 				zone->start, zone->len, GFP_NOFS);
4975 	if (ret == -EOPNOTSUPP) {
4976 		ret = blkdev_issue_zeroout(fdev->bdev, zone->wp,
4977 					zone->len - (zone->wp - zone->start),
4978 					GFP_NOFS, 0);
4979 		if (ret)
4980 			f2fs_err(sbi, "Fill up zone failed: %s (errno=%d)",
4981 					fdev->path, ret);
4982 	} else if (ret) {
4983 		f2fs_err(sbi, "Finishing zone failed: %s (errno=%d)",
4984 				fdev->path, ret);
4985 	}
4986 
4987 	return ret;
4988 }
4989 
get_target_zoned_dev(struct f2fs_sb_info * sbi,block_t zone_blkaddr)4990 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4991 						  block_t zone_blkaddr)
4992 {
4993 	int i;
4994 
4995 	for (i = 0; i < sbi->s_ndevs; i++) {
4996 		if (!bdev_is_zoned(FDEV(i).bdev))
4997 			continue;
4998 		if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4999 				zone_blkaddr <= FDEV(i).end_blk))
5000 			return &FDEV(i);
5001 	}
5002 
5003 	return NULL;
5004 }
5005 
report_one_zone_cb(struct blk_zone * zone,unsigned int idx,void * data)5006 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
5007 			      void *data)
5008 {
5009 	memcpy(data, zone, sizeof(struct blk_zone));
5010 	return 0;
5011 }
5012 
fix_curseg_write_pointer(struct f2fs_sb_info * sbi,int type)5013 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
5014 {
5015 	struct curseg_info *cs = CURSEG_I(sbi, type);
5016 	struct f2fs_dev_info *zbd;
5017 	struct blk_zone zone;
5018 	unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
5019 	block_t cs_zone_block, wp_block;
5020 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
5021 	sector_t zone_sector;
5022 	int err;
5023 
5024 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
5025 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
5026 
5027 	zbd = get_target_zoned_dev(sbi, cs_zone_block);
5028 	if (!zbd)
5029 		return 0;
5030 
5031 	/* report zone for the sector the curseg points to */
5032 	zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
5033 		<< log_sectors_per_block;
5034 	err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
5035 				  report_one_zone_cb, &zone);
5036 	if (err != 1) {
5037 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
5038 			 zbd->path, err);
5039 		return err;
5040 	}
5041 
5042 	if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5043 		return 0;
5044 
5045 	wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
5046 	wp_segno = GET_SEGNO(sbi, wp_block);
5047 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
5048 	wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
5049 
5050 	if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
5051 		wp_sector_off == 0)
5052 		return 0;
5053 
5054 	f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
5055 		    "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
5056 		    type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
5057 
5058 	f2fs_notice(sbi, "Assign new section to curseg[%d]: "
5059 		    "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
5060 
5061 	f2fs_allocate_new_section(sbi, type, true);
5062 
5063 	/* check consistency of the zone curseg pointed to */
5064 	if (check_zone_write_pointer(sbi, zbd, &zone))
5065 		return -EIO;
5066 
5067 	/* check newly assigned zone */
5068 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
5069 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
5070 
5071 	zbd = get_target_zoned_dev(sbi, cs_zone_block);
5072 	if (!zbd)
5073 		return 0;
5074 
5075 	zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
5076 		<< log_sectors_per_block;
5077 	err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
5078 				  report_one_zone_cb, &zone);
5079 	if (err != 1) {
5080 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
5081 			 zbd->path, err);
5082 		return err;
5083 	}
5084 
5085 	if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5086 		return 0;
5087 
5088 	if (zone.wp != zone.start) {
5089 		f2fs_notice(sbi,
5090 			    "New zone for curseg[%d] is not yet discarded. "
5091 			    "Reset the zone: curseg[0x%x,0x%x]",
5092 			    type, cs->segno, cs->next_blkoff);
5093 		err = __f2fs_issue_discard_zone(sbi, zbd->bdev,	cs_zone_block,
5094 					zone.len >> log_sectors_per_block);
5095 		if (err) {
5096 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
5097 				 zbd->path, err);
5098 			return err;
5099 		}
5100 	}
5101 
5102 	return 0;
5103 }
5104 
f2fs_fix_curseg_write_pointer(struct f2fs_sb_info * sbi)5105 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5106 {
5107 	int i, ret;
5108 
5109 	for (i = 0; i < NR_PERSISTENT_LOG; i++) {
5110 		ret = fix_curseg_write_pointer(sbi, i);
5111 		if (ret)
5112 			return ret;
5113 	}
5114 
5115 	return 0;
5116 }
5117 
5118 struct check_zone_write_pointer_args {
5119 	struct f2fs_sb_info *sbi;
5120 	struct f2fs_dev_info *fdev;
5121 };
5122 
check_zone_write_pointer_cb(struct blk_zone * zone,unsigned int idx,void * data)5123 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
5124 				      void *data)
5125 {
5126 	struct check_zone_write_pointer_args *args;
5127 
5128 	args = (struct check_zone_write_pointer_args *)data;
5129 
5130 	return check_zone_write_pointer(args->sbi, args->fdev, zone);
5131 }
5132 
f2fs_check_write_pointer(struct f2fs_sb_info * sbi)5133 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5134 {
5135 	int i, ret;
5136 	struct check_zone_write_pointer_args args;
5137 
5138 	for (i = 0; i < sbi->s_ndevs; i++) {
5139 		if (!bdev_is_zoned(FDEV(i).bdev))
5140 			continue;
5141 
5142 		args.sbi = sbi;
5143 		args.fdev = &FDEV(i);
5144 		ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
5145 					  check_zone_write_pointer_cb, &args);
5146 		if (ret < 0)
5147 			return ret;
5148 	}
5149 
5150 	return 0;
5151 }
5152 
5153 /*
5154  * Return the number of usable blocks in a segment. The number of blocks
5155  * returned is always equal to the number of blocks in a segment for
5156  * segments fully contained within a sequential zone capacity or a
5157  * conventional zone. For segments partially contained in a sequential
5158  * zone capacity, the number of usable blocks up to the zone capacity
5159  * is returned. 0 is returned in all other cases.
5160  */
f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info * sbi,unsigned int segno)5161 static inline unsigned int f2fs_usable_zone_blks_in_seg(
5162 			struct f2fs_sb_info *sbi, unsigned int segno)
5163 {
5164 	block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5165 	unsigned int secno;
5166 
5167 	if (!sbi->unusable_blocks_per_sec)
5168 		return BLKS_PER_SEG(sbi);
5169 
5170 	secno = GET_SEC_FROM_SEG(sbi, segno);
5171 	seg_start = START_BLOCK(sbi, segno);
5172 	sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5173 	sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
5174 
5175 	/*
5176 	 * If segment starts before zone capacity and spans beyond
5177 	 * zone capacity, then usable blocks are from seg start to
5178 	 * zone capacity. If the segment starts after the zone capacity,
5179 	 * then there are no usable blocks.
5180 	 */
5181 	if (seg_start >= sec_cap_blkaddr)
5182 		return 0;
5183 	if (seg_start + BLKS_PER_SEG(sbi) > sec_cap_blkaddr)
5184 		return sec_cap_blkaddr - seg_start;
5185 
5186 	return BLKS_PER_SEG(sbi);
5187 }
5188 #else
f2fs_fix_curseg_write_pointer(struct f2fs_sb_info * sbi)5189 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5190 {
5191 	return 0;
5192 }
5193 
f2fs_check_write_pointer(struct f2fs_sb_info * sbi)5194 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5195 {
5196 	return 0;
5197 }
5198 
f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info * sbi,unsigned int segno)5199 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5200 							unsigned int segno)
5201 {
5202 	return 0;
5203 }
5204 
5205 #endif
f2fs_usable_blks_in_seg(struct f2fs_sb_info * sbi,unsigned int segno)5206 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5207 					unsigned int segno)
5208 {
5209 	if (f2fs_sb_has_blkzoned(sbi))
5210 		return f2fs_usable_zone_blks_in_seg(sbi, segno);
5211 
5212 	return BLKS_PER_SEG(sbi);
5213 }
5214 
f2fs_usable_segs_in_sec(struct f2fs_sb_info * sbi,unsigned int segno)5215 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
5216 					unsigned int segno)
5217 {
5218 	if (f2fs_sb_has_blkzoned(sbi))
5219 		return CAP_SEGS_PER_SEC(sbi);
5220 
5221 	return SEGS_PER_SEC(sbi);
5222 }
5223 
5224 /*
5225  * Update min, max modified time for cost-benefit GC algorithm
5226  */
init_min_max_mtime(struct f2fs_sb_info * sbi)5227 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5228 {
5229 	struct sit_info *sit_i = SIT_I(sbi);
5230 	unsigned int segno;
5231 
5232 	down_write(&sit_i->sentry_lock);
5233 
5234 	sit_i->min_mtime = ULLONG_MAX;
5235 
5236 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) {
5237 		unsigned int i;
5238 		unsigned long long mtime = 0;
5239 
5240 		for (i = 0; i < SEGS_PER_SEC(sbi); i++)
5241 			mtime += get_seg_entry(sbi, segno + i)->mtime;
5242 
5243 		mtime = div_u64(mtime, SEGS_PER_SEC(sbi));
5244 
5245 		if (sit_i->min_mtime > mtime)
5246 			sit_i->min_mtime = mtime;
5247 	}
5248 	sit_i->max_mtime = get_mtime(sbi, false);
5249 	sit_i->dirty_max_mtime = 0;
5250 	up_write(&sit_i->sentry_lock);
5251 }
5252 
f2fs_build_segment_manager(struct f2fs_sb_info * sbi)5253 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5254 {
5255 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5256 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5257 	struct f2fs_sm_info *sm_info;
5258 	int err;
5259 
5260 	sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5261 	if (!sm_info)
5262 		return -ENOMEM;
5263 
5264 	/* init sm info */
5265 	sbi->sm_info = sm_info;
5266 	sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5267 	sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5268 	sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5269 	sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5270 	sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5271 	sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5272 	sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5273 	sm_info->rec_prefree_segments = sm_info->main_segments *
5274 					DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5275 	if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5276 		sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5277 
5278 	if (!f2fs_lfs_mode(sbi))
5279 		sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC);
5280 	sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5281 	sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5282 	sm_info->min_seq_blocks = BLKS_PER_SEG(sbi);
5283 	sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5284 	sm_info->min_ssr_sections = reserved_sections(sbi);
5285 
5286 	INIT_LIST_HEAD(&sm_info->sit_entry_set);
5287 
5288 	init_f2fs_rwsem(&sm_info->curseg_lock);
5289 
5290 	err = f2fs_create_flush_cmd_control(sbi);
5291 	if (err)
5292 		return err;
5293 
5294 	err = create_discard_cmd_control(sbi);
5295 	if (err)
5296 		return err;
5297 
5298 	err = build_sit_info(sbi);
5299 	if (err)
5300 		return err;
5301 	err = build_free_segmap(sbi);
5302 	if (err)
5303 		return err;
5304 	err = build_curseg(sbi);
5305 	if (err)
5306 		return err;
5307 
5308 	/* reinit free segmap based on SIT */
5309 	err = build_sit_entries(sbi);
5310 	if (err)
5311 		return err;
5312 
5313 	init_free_segmap(sbi);
5314 	err = build_dirty_segmap(sbi);
5315 	if (err)
5316 		return err;
5317 
5318 	err = sanity_check_curseg(sbi);
5319 	if (err)
5320 		return err;
5321 
5322 	init_min_max_mtime(sbi);
5323 	return 0;
5324 }
5325 
discard_dirty_segmap(struct f2fs_sb_info * sbi,enum dirty_type dirty_type)5326 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5327 		enum dirty_type dirty_type)
5328 {
5329 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5330 
5331 	mutex_lock(&dirty_i->seglist_lock);
5332 	kvfree(dirty_i->dirty_segmap[dirty_type]);
5333 	dirty_i->nr_dirty[dirty_type] = 0;
5334 	mutex_unlock(&dirty_i->seglist_lock);
5335 }
5336 
destroy_victim_secmap(struct f2fs_sb_info * sbi)5337 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5338 {
5339 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5340 
5341 	kvfree(dirty_i->pinned_secmap);
5342 	kvfree(dirty_i->victim_secmap);
5343 }
5344 
destroy_dirty_segmap(struct f2fs_sb_info * sbi)5345 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5346 {
5347 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5348 	int i;
5349 
5350 	if (!dirty_i)
5351 		return;
5352 
5353 	/* discard pre-free/dirty segments list */
5354 	for (i = 0; i < NR_DIRTY_TYPE; i++)
5355 		discard_dirty_segmap(sbi, i);
5356 
5357 	if (__is_large_section(sbi)) {
5358 		mutex_lock(&dirty_i->seglist_lock);
5359 		kvfree(dirty_i->dirty_secmap);
5360 		mutex_unlock(&dirty_i->seglist_lock);
5361 	}
5362 
5363 	destroy_victim_secmap(sbi);
5364 	SM_I(sbi)->dirty_info = NULL;
5365 	kfree(dirty_i);
5366 }
5367 
destroy_curseg(struct f2fs_sb_info * sbi)5368 static void destroy_curseg(struct f2fs_sb_info *sbi)
5369 {
5370 	struct curseg_info *array = SM_I(sbi)->curseg_array;
5371 	int i;
5372 
5373 	if (!array)
5374 		return;
5375 	SM_I(sbi)->curseg_array = NULL;
5376 	for (i = 0; i < NR_CURSEG_TYPE; i++) {
5377 		kfree(array[i].sum_blk);
5378 		kfree(array[i].journal);
5379 	}
5380 	kfree(array);
5381 }
5382 
destroy_free_segmap(struct f2fs_sb_info * sbi)5383 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5384 {
5385 	struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5386 
5387 	if (!free_i)
5388 		return;
5389 	SM_I(sbi)->free_info = NULL;
5390 	kvfree(free_i->free_segmap);
5391 	kvfree(free_i->free_secmap);
5392 	kfree(free_i);
5393 }
5394 
destroy_sit_info(struct f2fs_sb_info * sbi)5395 static void destroy_sit_info(struct f2fs_sb_info *sbi)
5396 {
5397 	struct sit_info *sit_i = SIT_I(sbi);
5398 
5399 	if (!sit_i)
5400 		return;
5401 
5402 	if (sit_i->sentries)
5403 		kvfree(sit_i->bitmap);
5404 	kfree(sit_i->tmp_map);
5405 
5406 	kvfree(sit_i->sentries);
5407 	kvfree(sit_i->sec_entries);
5408 	kvfree(sit_i->dirty_sentries_bitmap);
5409 
5410 	SM_I(sbi)->sit_info = NULL;
5411 	kvfree(sit_i->sit_bitmap);
5412 #ifdef CONFIG_F2FS_CHECK_FS
5413 	kvfree(sit_i->sit_bitmap_mir);
5414 	kvfree(sit_i->invalid_segmap);
5415 #endif
5416 	kfree(sit_i);
5417 }
5418 
f2fs_destroy_segment_manager(struct f2fs_sb_info * sbi)5419 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5420 {
5421 	struct f2fs_sm_info *sm_info = SM_I(sbi);
5422 
5423 	if (!sm_info)
5424 		return;
5425 	f2fs_destroy_flush_cmd_control(sbi, true);
5426 	destroy_discard_cmd_control(sbi);
5427 	destroy_dirty_segmap(sbi);
5428 	destroy_curseg(sbi);
5429 	destroy_free_segmap(sbi);
5430 	destroy_sit_info(sbi);
5431 	sbi->sm_info = NULL;
5432 	kfree(sm_info);
5433 }
5434 
f2fs_create_segment_manager_caches(void)5435 int __init f2fs_create_segment_manager_caches(void)
5436 {
5437 	discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5438 			sizeof(struct discard_entry));
5439 	if (!discard_entry_slab)
5440 		goto fail;
5441 
5442 	discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5443 			sizeof(struct discard_cmd));
5444 	if (!discard_cmd_slab)
5445 		goto destroy_discard_entry;
5446 
5447 	sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5448 			sizeof(struct sit_entry_set));
5449 	if (!sit_entry_set_slab)
5450 		goto destroy_discard_cmd;
5451 
5452 	revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry",
5453 			sizeof(struct revoke_entry));
5454 	if (!revoke_entry_slab)
5455 		goto destroy_sit_entry_set;
5456 	return 0;
5457 
5458 destroy_sit_entry_set:
5459 	kmem_cache_destroy(sit_entry_set_slab);
5460 destroy_discard_cmd:
5461 	kmem_cache_destroy(discard_cmd_slab);
5462 destroy_discard_entry:
5463 	kmem_cache_destroy(discard_entry_slab);
5464 fail:
5465 	return -ENOMEM;
5466 }
5467 
f2fs_destroy_segment_manager_caches(void)5468 void f2fs_destroy_segment_manager_caches(void)
5469 {
5470 	kmem_cache_destroy(sit_entry_set_slab);
5471 	kmem_cache_destroy(discard_cmd_slab);
5472 	kmem_cache_destroy(discard_entry_slab);
5473 	kmem_cache_destroy(revoke_entry_slab);
5474 }
5475