1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/segment.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/sched/mm.h> 13 #include <linux/prefetch.h> 14 #include <linux/kthread.h> 15 #include <linux/swap.h> 16 #include <linux/timer.h> 17 #include <linux/freezer.h> 18 #include <linux/sched/signal.h> 19 #include <linux/random.h> 20 21 #include "f2fs.h" 22 #include "segment.h" 23 #include "node.h" 24 #include "gc.h" 25 #include "iostat.h" 26 #include <trace/events/f2fs.h> 27 28 #define __reverse_ffz(x) __reverse_ffs(~(x)) 29 30 static struct kmem_cache *discard_entry_slab; 31 static struct kmem_cache *discard_cmd_slab; 32 static struct kmem_cache *sit_entry_set_slab; 33 static struct kmem_cache *revoke_entry_slab; 34 35 static unsigned long __reverse_ulong(unsigned char *str) 36 { 37 unsigned long tmp = 0; 38 int shift = 24, idx = 0; 39 40 #if BITS_PER_LONG == 64 41 shift = 56; 42 #endif 43 while (shift >= 0) { 44 tmp |= (unsigned long)str[idx++] << shift; 45 shift -= BITS_PER_BYTE; 46 } 47 return tmp; 48 } 49 50 /* 51 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 52 * MSB and LSB are reversed in a byte by f2fs_set_bit. 53 */ 54 static inline unsigned long __reverse_ffs(unsigned long word) 55 { 56 int num = 0; 57 58 #if BITS_PER_LONG == 64 59 if ((word & 0xffffffff00000000UL) == 0) 60 num += 32; 61 else 62 word >>= 32; 63 #endif 64 if ((word & 0xffff0000) == 0) 65 num += 16; 66 else 67 word >>= 16; 68 69 if ((word & 0xff00) == 0) 70 num += 8; 71 else 72 word >>= 8; 73 74 if ((word & 0xf0) == 0) 75 num += 4; 76 else 77 word >>= 4; 78 79 if ((word & 0xc) == 0) 80 num += 2; 81 else 82 word >>= 2; 83 84 if ((word & 0x2) == 0) 85 num += 1; 86 return num; 87 } 88 89 /* 90 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because 91 * f2fs_set_bit makes MSB and LSB reversed in a byte. 92 * @size must be integral times of unsigned long. 93 * Example: 94 * MSB <--> LSB 95 * f2fs_set_bit(0, bitmap) => 1000 0000 96 * f2fs_set_bit(7, bitmap) => 0000 0001 97 */ 98 static unsigned long __find_rev_next_bit(const unsigned long *addr, 99 unsigned long size, unsigned long offset) 100 { 101 const unsigned long *p = addr + BIT_WORD(offset); 102 unsigned long result = size; 103 unsigned long tmp; 104 105 if (offset >= size) 106 return size; 107 108 size -= (offset & ~(BITS_PER_LONG - 1)); 109 offset %= BITS_PER_LONG; 110 111 while (1) { 112 if (*p == 0) 113 goto pass; 114 115 tmp = __reverse_ulong((unsigned char *)p); 116 117 tmp &= ~0UL >> offset; 118 if (size < BITS_PER_LONG) 119 tmp &= (~0UL << (BITS_PER_LONG - size)); 120 if (tmp) 121 goto found; 122 pass: 123 if (size <= BITS_PER_LONG) 124 break; 125 size -= BITS_PER_LONG; 126 offset = 0; 127 p++; 128 } 129 return result; 130 found: 131 return result - size + __reverse_ffs(tmp); 132 } 133 134 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 135 unsigned long size, unsigned long offset) 136 { 137 const unsigned long *p = addr + BIT_WORD(offset); 138 unsigned long result = size; 139 unsigned long tmp; 140 141 if (offset >= size) 142 return size; 143 144 size -= (offset & ~(BITS_PER_LONG - 1)); 145 offset %= BITS_PER_LONG; 146 147 while (1) { 148 if (*p == ~0UL) 149 goto pass; 150 151 tmp = __reverse_ulong((unsigned char *)p); 152 153 if (offset) 154 tmp |= ~0UL << (BITS_PER_LONG - offset); 155 if (size < BITS_PER_LONG) 156 tmp |= ~0UL >> size; 157 if (tmp != ~0UL) 158 goto found; 159 pass: 160 if (size <= BITS_PER_LONG) 161 break; 162 size -= BITS_PER_LONG; 163 offset = 0; 164 p++; 165 } 166 return result; 167 found: 168 return result - size + __reverse_ffz(tmp); 169 } 170 171 bool f2fs_need_SSR(struct f2fs_sb_info *sbi) 172 { 173 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); 174 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); 175 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); 176 177 if (f2fs_lfs_mode(sbi)) 178 return false; 179 if (sbi->gc_mode == GC_URGENT_HIGH) 180 return true; 181 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 182 return true; 183 184 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + 185 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); 186 } 187 188 void f2fs_abort_atomic_write(struct inode *inode, bool clean) 189 { 190 struct f2fs_inode_info *fi = F2FS_I(inode); 191 192 if (!f2fs_is_atomic_file(inode)) 193 return; 194 195 release_atomic_write_cnt(inode); 196 clear_inode_flag(inode, FI_ATOMIC_COMMITTED); 197 clear_inode_flag(inode, FI_ATOMIC_REPLACE); 198 clear_inode_flag(inode, FI_ATOMIC_FILE); 199 stat_dec_atomic_inode(inode); 200 201 F2FS_I(inode)->atomic_write_task = NULL; 202 203 if (clean) { 204 truncate_inode_pages_final(inode->i_mapping); 205 f2fs_i_size_write(inode, fi->original_i_size); 206 fi->original_i_size = 0; 207 } 208 } 209 210 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index, 211 block_t new_addr, block_t *old_addr, bool recover) 212 { 213 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 214 struct dnode_of_data dn; 215 struct node_info ni; 216 int err; 217 218 retry: 219 set_new_dnode(&dn, inode, NULL, NULL, 0); 220 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE_RA); 221 if (err) { 222 if (err == -ENOMEM) { 223 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); 224 goto retry; 225 } 226 return err; 227 } 228 229 err = f2fs_get_node_info(sbi, dn.nid, &ni, false); 230 if (err) { 231 f2fs_put_dnode(&dn); 232 return err; 233 } 234 235 if (recover) { 236 /* dn.data_blkaddr is always valid */ 237 if (!__is_valid_data_blkaddr(new_addr)) { 238 if (new_addr == NULL_ADDR) 239 dec_valid_block_count(sbi, inode, 1); 240 f2fs_invalidate_blocks(sbi, dn.data_blkaddr); 241 f2fs_update_data_blkaddr(&dn, new_addr); 242 } else { 243 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, 244 new_addr, ni.version, true, true); 245 } 246 } else { 247 blkcnt_t count = 1; 248 249 *old_addr = dn.data_blkaddr; 250 f2fs_truncate_data_blocks_range(&dn, 1); 251 dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count); 252 inc_valid_block_count(sbi, inode, &count); 253 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr, 254 ni.version, true, false); 255 } 256 257 f2fs_put_dnode(&dn); 258 259 trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode, 260 index, *old_addr, new_addr, recover); 261 return 0; 262 } 263 264 static void __complete_revoke_list(struct inode *inode, struct list_head *head, 265 bool revoke) 266 { 267 struct revoke_entry *cur, *tmp; 268 pgoff_t start_index = 0; 269 bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE); 270 271 list_for_each_entry_safe(cur, tmp, head, list) { 272 if (revoke) { 273 __replace_atomic_write_block(inode, cur->index, 274 cur->old_addr, NULL, true); 275 } else if (truncate) { 276 f2fs_truncate_hole(inode, start_index, cur->index); 277 start_index = cur->index + 1; 278 } 279 280 list_del(&cur->list); 281 kmem_cache_free(revoke_entry_slab, cur); 282 } 283 284 if (!revoke && truncate) 285 f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false); 286 } 287 288 static int __f2fs_commit_atomic_write(struct inode *inode) 289 { 290 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 291 struct f2fs_inode_info *fi = F2FS_I(inode); 292 struct inode *cow_inode = fi->cow_inode; 293 struct revoke_entry *new; 294 struct list_head revoke_list; 295 block_t blkaddr; 296 struct dnode_of_data dn; 297 pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 298 pgoff_t off = 0, blen, index; 299 int ret = 0, i; 300 301 INIT_LIST_HEAD(&revoke_list); 302 303 while (len) { 304 blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len); 305 306 set_new_dnode(&dn, cow_inode, NULL, NULL, 0); 307 ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA); 308 if (ret && ret != -ENOENT) { 309 goto out; 310 } else if (ret == -ENOENT) { 311 ret = 0; 312 if (dn.max_level == 0) 313 goto out; 314 goto next; 315 } 316 317 blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_page, cow_inode), 318 len); 319 index = off; 320 for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) { 321 blkaddr = f2fs_data_blkaddr(&dn); 322 323 if (!__is_valid_data_blkaddr(blkaddr)) { 324 continue; 325 } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr, 326 DATA_GENERIC_ENHANCE)) { 327 f2fs_put_dnode(&dn); 328 ret = -EFSCORRUPTED; 329 f2fs_handle_error(sbi, 330 ERROR_INVALID_BLKADDR); 331 goto out; 332 } 333 334 new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS, 335 true, NULL); 336 337 ret = __replace_atomic_write_block(inode, index, blkaddr, 338 &new->old_addr, false); 339 if (ret) { 340 f2fs_put_dnode(&dn); 341 kmem_cache_free(revoke_entry_slab, new); 342 goto out; 343 } 344 345 f2fs_update_data_blkaddr(&dn, NULL_ADDR); 346 new->index = index; 347 list_add_tail(&new->list, &revoke_list); 348 } 349 f2fs_put_dnode(&dn); 350 next: 351 off += blen; 352 len -= blen; 353 } 354 355 out: 356 if (ret) { 357 sbi->revoked_atomic_block += fi->atomic_write_cnt; 358 } else { 359 sbi->committed_atomic_block += fi->atomic_write_cnt; 360 set_inode_flag(inode, FI_ATOMIC_COMMITTED); 361 } 362 363 __complete_revoke_list(inode, &revoke_list, ret ? true : false); 364 365 return ret; 366 } 367 368 int f2fs_commit_atomic_write(struct inode *inode) 369 { 370 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 371 struct f2fs_inode_info *fi = F2FS_I(inode); 372 int err; 373 374 err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); 375 if (err) 376 return err; 377 378 f2fs_down_write(&fi->i_gc_rwsem[WRITE]); 379 f2fs_lock_op(sbi); 380 381 err = __f2fs_commit_atomic_write(inode); 382 383 f2fs_unlock_op(sbi); 384 f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 385 386 return err; 387 } 388 389 /* 390 * This function balances dirty node and dentry pages. 391 * In addition, it controls garbage collection. 392 */ 393 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) 394 { 395 if (time_to_inject(sbi, FAULT_CHECKPOINT)) 396 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT); 397 398 /* balance_fs_bg is able to be pending */ 399 if (need && excess_cached_nats(sbi)) 400 f2fs_balance_fs_bg(sbi, false); 401 402 if (!f2fs_is_checkpoint_ready(sbi)) 403 return; 404 405 /* 406 * We should do GC or end up with checkpoint, if there are so many dirty 407 * dir/node pages without enough free segments. 408 */ 409 if (has_not_enough_free_secs(sbi, 0, 0)) { 410 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && 411 sbi->gc_thread->f2fs_gc_task) { 412 DEFINE_WAIT(wait); 413 414 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait, 415 TASK_UNINTERRUPTIBLE); 416 wake_up(&sbi->gc_thread->gc_wait_queue_head); 417 io_schedule(); 418 finish_wait(&sbi->gc_thread->fggc_wq, &wait); 419 } else { 420 struct f2fs_gc_control gc_control = { 421 .victim_segno = NULL_SEGNO, 422 .init_gc_type = BG_GC, 423 .no_bg_gc = true, 424 .should_migrate_blocks = false, 425 .err_gc_skipped = false, 426 .nr_free_secs = 1 }; 427 f2fs_down_write(&sbi->gc_lock); 428 f2fs_gc(sbi, &gc_control); 429 } 430 } 431 } 432 433 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi) 434 { 435 int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2; 436 unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS); 437 unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA); 438 unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES); 439 unsigned int meta = get_pages(sbi, F2FS_DIRTY_META); 440 unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA); 441 unsigned int threshold = sbi->blocks_per_seg * factor * 442 DEFAULT_DIRTY_THRESHOLD; 443 unsigned int global_threshold = threshold * 3 / 2; 444 445 if (dents >= threshold || qdata >= threshold || 446 nodes >= threshold || meta >= threshold || 447 imeta >= threshold) 448 return true; 449 return dents + qdata + nodes + meta + imeta > global_threshold; 450 } 451 452 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) 453 { 454 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 455 return; 456 457 /* try to shrink extent cache when there is no enough memory */ 458 if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE)) 459 f2fs_shrink_read_extent_tree(sbi, 460 READ_EXTENT_CACHE_SHRINK_NUMBER); 461 462 /* try to shrink age extent cache when there is no enough memory */ 463 if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE)) 464 f2fs_shrink_age_extent_tree(sbi, 465 AGE_EXTENT_CACHE_SHRINK_NUMBER); 466 467 /* check the # of cached NAT entries */ 468 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) 469 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); 470 471 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) 472 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); 473 else 474 f2fs_build_free_nids(sbi, false, false); 475 476 if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) || 477 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi)) 478 goto do_sync; 479 480 /* there is background inflight IO or foreground operation recently */ 481 if (is_inflight_io(sbi, REQ_TIME) || 482 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem))) 483 return; 484 485 /* exceed periodical checkpoint timeout threshold */ 486 if (f2fs_time_over(sbi, CP_TIME)) 487 goto do_sync; 488 489 /* checkpoint is the only way to shrink partial cached entries */ 490 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) && 491 f2fs_available_free_memory(sbi, INO_ENTRIES)) 492 return; 493 494 do_sync: 495 if (test_opt(sbi, DATA_FLUSH) && from_bg) { 496 struct blk_plug plug; 497 498 mutex_lock(&sbi->flush_lock); 499 500 blk_start_plug(&plug); 501 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false); 502 blk_finish_plug(&plug); 503 504 mutex_unlock(&sbi->flush_lock); 505 } 506 f2fs_sync_fs(sbi->sb, 1); 507 stat_inc_bg_cp_count(sbi->stat_info); 508 } 509 510 static int __submit_flush_wait(struct f2fs_sb_info *sbi, 511 struct block_device *bdev) 512 { 513 int ret = blkdev_issue_flush(bdev); 514 515 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), 516 test_opt(sbi, FLUSH_MERGE), ret); 517 if (!ret) 518 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); 519 return ret; 520 } 521 522 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) 523 { 524 int ret = 0; 525 int i; 526 527 if (!f2fs_is_multi_device(sbi)) 528 return __submit_flush_wait(sbi, sbi->sb->s_bdev); 529 530 for (i = 0; i < sbi->s_ndevs; i++) { 531 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) 532 continue; 533 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 534 if (ret) 535 break; 536 } 537 return ret; 538 } 539 540 static int issue_flush_thread(void *data) 541 { 542 struct f2fs_sb_info *sbi = data; 543 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 544 wait_queue_head_t *q = &fcc->flush_wait_queue; 545 repeat: 546 if (kthread_should_stop()) 547 return 0; 548 549 if (!llist_empty(&fcc->issue_list)) { 550 struct flush_cmd *cmd, *next; 551 int ret; 552 553 fcc->dispatch_list = llist_del_all(&fcc->issue_list); 554 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); 555 556 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode); 557 558 ret = submit_flush_wait(sbi, cmd->ino); 559 atomic_inc(&fcc->issued_flush); 560 561 llist_for_each_entry_safe(cmd, next, 562 fcc->dispatch_list, llnode) { 563 cmd->ret = ret; 564 complete(&cmd->wait); 565 } 566 fcc->dispatch_list = NULL; 567 } 568 569 wait_event_interruptible(*q, 570 kthread_should_stop() || !llist_empty(&fcc->issue_list)); 571 goto repeat; 572 } 573 574 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) 575 { 576 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 577 struct flush_cmd cmd; 578 int ret; 579 580 if (test_opt(sbi, NOBARRIER)) 581 return 0; 582 583 if (!test_opt(sbi, FLUSH_MERGE)) { 584 atomic_inc(&fcc->queued_flush); 585 ret = submit_flush_wait(sbi, ino); 586 atomic_dec(&fcc->queued_flush); 587 atomic_inc(&fcc->issued_flush); 588 return ret; 589 } 590 591 if (atomic_inc_return(&fcc->queued_flush) == 1 || 592 f2fs_is_multi_device(sbi)) { 593 ret = submit_flush_wait(sbi, ino); 594 atomic_dec(&fcc->queued_flush); 595 596 atomic_inc(&fcc->issued_flush); 597 return ret; 598 } 599 600 cmd.ino = ino; 601 init_completion(&cmd.wait); 602 603 llist_add(&cmd.llnode, &fcc->issue_list); 604 605 /* 606 * update issue_list before we wake up issue_flush thread, this 607 * smp_mb() pairs with another barrier in ___wait_event(), see 608 * more details in comments of waitqueue_active(). 609 */ 610 smp_mb(); 611 612 if (waitqueue_active(&fcc->flush_wait_queue)) 613 wake_up(&fcc->flush_wait_queue); 614 615 if (fcc->f2fs_issue_flush) { 616 wait_for_completion(&cmd.wait); 617 atomic_dec(&fcc->queued_flush); 618 } else { 619 struct llist_node *list; 620 621 list = llist_del_all(&fcc->issue_list); 622 if (!list) { 623 wait_for_completion(&cmd.wait); 624 atomic_dec(&fcc->queued_flush); 625 } else { 626 struct flush_cmd *tmp, *next; 627 628 ret = submit_flush_wait(sbi, ino); 629 630 llist_for_each_entry_safe(tmp, next, list, llnode) { 631 if (tmp == &cmd) { 632 cmd.ret = ret; 633 atomic_dec(&fcc->queued_flush); 634 continue; 635 } 636 tmp->ret = ret; 637 complete(&tmp->wait); 638 } 639 } 640 } 641 642 return cmd.ret; 643 } 644 645 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) 646 { 647 dev_t dev = sbi->sb->s_bdev->bd_dev; 648 struct flush_cmd_control *fcc; 649 650 if (SM_I(sbi)->fcc_info) { 651 fcc = SM_I(sbi)->fcc_info; 652 if (fcc->f2fs_issue_flush) 653 return 0; 654 goto init_thread; 655 } 656 657 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); 658 if (!fcc) 659 return -ENOMEM; 660 atomic_set(&fcc->issued_flush, 0); 661 atomic_set(&fcc->queued_flush, 0); 662 init_waitqueue_head(&fcc->flush_wait_queue); 663 init_llist_head(&fcc->issue_list); 664 SM_I(sbi)->fcc_info = fcc; 665 if (!test_opt(sbi, FLUSH_MERGE)) 666 return 0; 667 668 init_thread: 669 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, 670 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); 671 if (IS_ERR(fcc->f2fs_issue_flush)) { 672 int err = PTR_ERR(fcc->f2fs_issue_flush); 673 674 fcc->f2fs_issue_flush = NULL; 675 return err; 676 } 677 678 return 0; 679 } 680 681 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) 682 { 683 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 684 685 if (fcc && fcc->f2fs_issue_flush) { 686 struct task_struct *flush_thread = fcc->f2fs_issue_flush; 687 688 fcc->f2fs_issue_flush = NULL; 689 kthread_stop(flush_thread); 690 } 691 if (free) { 692 kfree(fcc); 693 SM_I(sbi)->fcc_info = NULL; 694 } 695 } 696 697 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) 698 { 699 int ret = 0, i; 700 701 if (!f2fs_is_multi_device(sbi)) 702 return 0; 703 704 if (test_opt(sbi, NOBARRIER)) 705 return 0; 706 707 for (i = 1; i < sbi->s_ndevs; i++) { 708 int count = DEFAULT_RETRY_IO_COUNT; 709 710 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) 711 continue; 712 713 do { 714 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 715 if (ret) 716 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); 717 } while (ret && --count); 718 719 if (ret) { 720 f2fs_stop_checkpoint(sbi, false, 721 STOP_CP_REASON_FLUSH_FAIL); 722 break; 723 } 724 725 spin_lock(&sbi->dev_lock); 726 f2fs_clear_bit(i, (char *)&sbi->dirty_device); 727 spin_unlock(&sbi->dev_lock); 728 } 729 730 return ret; 731 } 732 733 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 734 enum dirty_type dirty_type) 735 { 736 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 737 738 /* need not be added */ 739 if (IS_CURSEG(sbi, segno)) 740 return; 741 742 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 743 dirty_i->nr_dirty[dirty_type]++; 744 745 if (dirty_type == DIRTY) { 746 struct seg_entry *sentry = get_seg_entry(sbi, segno); 747 enum dirty_type t = sentry->type; 748 749 if (unlikely(t >= DIRTY)) { 750 f2fs_bug_on(sbi, 1); 751 return; 752 } 753 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 754 dirty_i->nr_dirty[t]++; 755 756 if (__is_large_section(sbi)) { 757 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 758 block_t valid_blocks = 759 get_valid_blocks(sbi, segno, true); 760 761 f2fs_bug_on(sbi, unlikely(!valid_blocks || 762 valid_blocks == CAP_BLKS_PER_SEC(sbi))); 763 764 if (!IS_CURSEC(sbi, secno)) 765 set_bit(secno, dirty_i->dirty_secmap); 766 } 767 } 768 } 769 770 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 771 enum dirty_type dirty_type) 772 { 773 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 774 block_t valid_blocks; 775 776 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 777 dirty_i->nr_dirty[dirty_type]--; 778 779 if (dirty_type == DIRTY) { 780 struct seg_entry *sentry = get_seg_entry(sbi, segno); 781 enum dirty_type t = sentry->type; 782 783 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 784 dirty_i->nr_dirty[t]--; 785 786 valid_blocks = get_valid_blocks(sbi, segno, true); 787 if (valid_blocks == 0) { 788 clear_bit(GET_SEC_FROM_SEG(sbi, segno), 789 dirty_i->victim_secmap); 790 #ifdef CONFIG_F2FS_CHECK_FS 791 clear_bit(segno, SIT_I(sbi)->invalid_segmap); 792 #endif 793 } 794 if (__is_large_section(sbi)) { 795 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 796 797 if (!valid_blocks || 798 valid_blocks == CAP_BLKS_PER_SEC(sbi)) { 799 clear_bit(secno, dirty_i->dirty_secmap); 800 return; 801 } 802 803 if (!IS_CURSEC(sbi, secno)) 804 set_bit(secno, dirty_i->dirty_secmap); 805 } 806 } 807 } 808 809 /* 810 * Should not occur error such as -ENOMEM. 811 * Adding dirty entry into seglist is not critical operation. 812 * If a given segment is one of current working segments, it won't be added. 813 */ 814 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 815 { 816 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 817 unsigned short valid_blocks, ckpt_valid_blocks; 818 unsigned int usable_blocks; 819 820 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) 821 return; 822 823 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno); 824 mutex_lock(&dirty_i->seglist_lock); 825 826 valid_blocks = get_valid_blocks(sbi, segno, false); 827 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false); 828 829 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || 830 ckpt_valid_blocks == usable_blocks)) { 831 __locate_dirty_segment(sbi, segno, PRE); 832 __remove_dirty_segment(sbi, segno, DIRTY); 833 } else if (valid_blocks < usable_blocks) { 834 __locate_dirty_segment(sbi, segno, DIRTY); 835 } else { 836 /* Recovery routine with SSR needs this */ 837 __remove_dirty_segment(sbi, segno, DIRTY); 838 } 839 840 mutex_unlock(&dirty_i->seglist_lock); 841 } 842 843 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */ 844 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) 845 { 846 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 847 unsigned int segno; 848 849 mutex_lock(&dirty_i->seglist_lock); 850 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 851 if (get_valid_blocks(sbi, segno, false)) 852 continue; 853 if (IS_CURSEG(sbi, segno)) 854 continue; 855 __locate_dirty_segment(sbi, segno, PRE); 856 __remove_dirty_segment(sbi, segno, DIRTY); 857 } 858 mutex_unlock(&dirty_i->seglist_lock); 859 } 860 861 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi) 862 { 863 int ovp_hole_segs = 864 (overprovision_segments(sbi) - reserved_segments(sbi)); 865 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg; 866 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 867 block_t holes[2] = {0, 0}; /* DATA and NODE */ 868 block_t unusable; 869 struct seg_entry *se; 870 unsigned int segno; 871 872 mutex_lock(&dirty_i->seglist_lock); 873 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 874 se = get_seg_entry(sbi, segno); 875 if (IS_NODESEG(se->type)) 876 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) - 877 se->valid_blocks; 878 else 879 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) - 880 se->valid_blocks; 881 } 882 mutex_unlock(&dirty_i->seglist_lock); 883 884 unusable = max(holes[DATA], holes[NODE]); 885 if (unusable > ovp_holes) 886 return unusable - ovp_holes; 887 return 0; 888 } 889 890 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable) 891 { 892 int ovp_hole_segs = 893 (overprovision_segments(sbi) - reserved_segments(sbi)); 894 if (unusable > F2FS_OPTION(sbi).unusable_cap) 895 return -EAGAIN; 896 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && 897 dirty_segments(sbi) > ovp_hole_segs) 898 return -EAGAIN; 899 return 0; 900 } 901 902 /* This is only used by SBI_CP_DISABLED */ 903 static unsigned int get_free_segment(struct f2fs_sb_info *sbi) 904 { 905 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 906 unsigned int segno = 0; 907 908 mutex_lock(&dirty_i->seglist_lock); 909 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 910 if (get_valid_blocks(sbi, segno, false)) 911 continue; 912 if (get_ckpt_valid_blocks(sbi, segno, false)) 913 continue; 914 mutex_unlock(&dirty_i->seglist_lock); 915 return segno; 916 } 917 mutex_unlock(&dirty_i->seglist_lock); 918 return NULL_SEGNO; 919 } 920 921 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, 922 struct block_device *bdev, block_t lstart, 923 block_t start, block_t len) 924 { 925 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 926 struct list_head *pend_list; 927 struct discard_cmd *dc; 928 929 f2fs_bug_on(sbi, !len); 930 931 pend_list = &dcc->pend_list[plist_idx(len)]; 932 933 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL); 934 INIT_LIST_HEAD(&dc->list); 935 dc->bdev = bdev; 936 dc->lstart = lstart; 937 dc->start = start; 938 dc->len = len; 939 dc->ref = 0; 940 dc->state = D_PREP; 941 dc->queued = 0; 942 dc->error = 0; 943 init_completion(&dc->wait); 944 list_add_tail(&dc->list, pend_list); 945 spin_lock_init(&dc->lock); 946 dc->bio_ref = 0; 947 atomic_inc(&dcc->discard_cmd_cnt); 948 dcc->undiscard_blks += len; 949 950 return dc; 951 } 952 953 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, 954 struct block_device *bdev, block_t lstart, 955 block_t start, block_t len, 956 struct rb_node *parent, struct rb_node **p, 957 bool leftmost) 958 { 959 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 960 struct discard_cmd *dc; 961 962 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); 963 964 rb_link_node(&dc->rb_node, parent, p); 965 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); 966 967 return dc; 968 } 969 970 static void __detach_discard_cmd(struct discard_cmd_control *dcc, 971 struct discard_cmd *dc) 972 { 973 if (dc->state == D_DONE) 974 atomic_sub(dc->queued, &dcc->queued_discard); 975 976 list_del(&dc->list); 977 rb_erase_cached(&dc->rb_node, &dcc->root); 978 dcc->undiscard_blks -= dc->len; 979 980 kmem_cache_free(discard_cmd_slab, dc); 981 982 atomic_dec(&dcc->discard_cmd_cnt); 983 } 984 985 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, 986 struct discard_cmd *dc) 987 { 988 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 989 unsigned long flags; 990 991 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len); 992 993 spin_lock_irqsave(&dc->lock, flags); 994 if (dc->bio_ref) { 995 spin_unlock_irqrestore(&dc->lock, flags); 996 return; 997 } 998 spin_unlock_irqrestore(&dc->lock, flags); 999 1000 f2fs_bug_on(sbi, dc->ref); 1001 1002 if (dc->error == -EOPNOTSUPP) 1003 dc->error = 0; 1004 1005 if (dc->error) 1006 printk_ratelimited( 1007 "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d", 1008 KERN_INFO, sbi->sb->s_id, 1009 dc->lstart, dc->start, dc->len, dc->error); 1010 __detach_discard_cmd(dcc, dc); 1011 } 1012 1013 static void f2fs_submit_discard_endio(struct bio *bio) 1014 { 1015 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; 1016 unsigned long flags; 1017 1018 spin_lock_irqsave(&dc->lock, flags); 1019 if (!dc->error) 1020 dc->error = blk_status_to_errno(bio->bi_status); 1021 dc->bio_ref--; 1022 if (!dc->bio_ref && dc->state == D_SUBMIT) { 1023 dc->state = D_DONE; 1024 complete_all(&dc->wait); 1025 } 1026 spin_unlock_irqrestore(&dc->lock, flags); 1027 bio_put(bio); 1028 } 1029 1030 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, 1031 block_t start, block_t end) 1032 { 1033 #ifdef CONFIG_F2FS_CHECK_FS 1034 struct seg_entry *sentry; 1035 unsigned int segno; 1036 block_t blk = start; 1037 unsigned long offset, size, max_blocks = sbi->blocks_per_seg; 1038 unsigned long *map; 1039 1040 while (blk < end) { 1041 segno = GET_SEGNO(sbi, blk); 1042 sentry = get_seg_entry(sbi, segno); 1043 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); 1044 1045 if (end < START_BLOCK(sbi, segno + 1)) 1046 size = GET_BLKOFF_FROM_SEG0(sbi, end); 1047 else 1048 size = max_blocks; 1049 map = (unsigned long *)(sentry->cur_valid_map); 1050 offset = __find_rev_next_bit(map, size, offset); 1051 f2fs_bug_on(sbi, offset != size); 1052 blk = START_BLOCK(sbi, segno + 1); 1053 } 1054 #endif 1055 } 1056 1057 static void __init_discard_policy(struct f2fs_sb_info *sbi, 1058 struct discard_policy *dpolicy, 1059 int discard_type, unsigned int granularity) 1060 { 1061 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1062 1063 /* common policy */ 1064 dpolicy->type = discard_type; 1065 dpolicy->sync = true; 1066 dpolicy->ordered = false; 1067 dpolicy->granularity = granularity; 1068 1069 dpolicy->max_requests = dcc->max_discard_request; 1070 dpolicy->io_aware_gran = dcc->discard_io_aware_gran; 1071 dpolicy->timeout = false; 1072 1073 if (discard_type == DPOLICY_BG) { 1074 dpolicy->min_interval = dcc->min_discard_issue_time; 1075 dpolicy->mid_interval = dcc->mid_discard_issue_time; 1076 dpolicy->max_interval = dcc->max_discard_issue_time; 1077 dpolicy->io_aware = true; 1078 dpolicy->sync = false; 1079 dpolicy->ordered = true; 1080 if (utilization(sbi) > dcc->discard_urgent_util) { 1081 dpolicy->granularity = MIN_DISCARD_GRANULARITY; 1082 if (atomic_read(&dcc->discard_cmd_cnt)) 1083 dpolicy->max_interval = 1084 dcc->min_discard_issue_time; 1085 } 1086 } else if (discard_type == DPOLICY_FORCE) { 1087 dpolicy->min_interval = dcc->min_discard_issue_time; 1088 dpolicy->mid_interval = dcc->mid_discard_issue_time; 1089 dpolicy->max_interval = dcc->max_discard_issue_time; 1090 dpolicy->io_aware = false; 1091 } else if (discard_type == DPOLICY_FSTRIM) { 1092 dpolicy->io_aware = false; 1093 } else if (discard_type == DPOLICY_UMOUNT) { 1094 dpolicy->io_aware = false; 1095 /* we need to issue all to keep CP_TRIMMED_FLAG */ 1096 dpolicy->granularity = MIN_DISCARD_GRANULARITY; 1097 dpolicy->timeout = true; 1098 } 1099 } 1100 1101 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1102 struct block_device *bdev, block_t lstart, 1103 block_t start, block_t len); 1104 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ 1105 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, 1106 struct discard_policy *dpolicy, 1107 struct discard_cmd *dc, int *issued) 1108 { 1109 struct block_device *bdev = dc->bdev; 1110 unsigned int max_discard_blocks = 1111 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); 1112 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1113 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1114 &(dcc->fstrim_list) : &(dcc->wait_list); 1115 blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0; 1116 block_t lstart, start, len, total_len; 1117 int err = 0; 1118 1119 if (dc->state != D_PREP) 1120 return 0; 1121 1122 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 1123 return 0; 1124 1125 trace_f2fs_issue_discard(bdev, dc->start, dc->len); 1126 1127 lstart = dc->lstart; 1128 start = dc->start; 1129 len = dc->len; 1130 total_len = len; 1131 1132 dc->len = 0; 1133 1134 while (total_len && *issued < dpolicy->max_requests && !err) { 1135 struct bio *bio = NULL; 1136 unsigned long flags; 1137 bool last = true; 1138 1139 if (len > max_discard_blocks) { 1140 len = max_discard_blocks; 1141 last = false; 1142 } 1143 1144 (*issued)++; 1145 if (*issued == dpolicy->max_requests) 1146 last = true; 1147 1148 dc->len += len; 1149 1150 if (time_to_inject(sbi, FAULT_DISCARD)) { 1151 err = -EIO; 1152 } else { 1153 err = __blkdev_issue_discard(bdev, 1154 SECTOR_FROM_BLOCK(start), 1155 SECTOR_FROM_BLOCK(len), 1156 GFP_NOFS, &bio); 1157 } 1158 if (err) { 1159 spin_lock_irqsave(&dc->lock, flags); 1160 if (dc->state == D_PARTIAL) 1161 dc->state = D_SUBMIT; 1162 spin_unlock_irqrestore(&dc->lock, flags); 1163 1164 break; 1165 } 1166 1167 f2fs_bug_on(sbi, !bio); 1168 1169 /* 1170 * should keep before submission to avoid D_DONE 1171 * right away 1172 */ 1173 spin_lock_irqsave(&dc->lock, flags); 1174 if (last) 1175 dc->state = D_SUBMIT; 1176 else 1177 dc->state = D_PARTIAL; 1178 dc->bio_ref++; 1179 spin_unlock_irqrestore(&dc->lock, flags); 1180 1181 atomic_inc(&dcc->queued_discard); 1182 dc->queued++; 1183 list_move_tail(&dc->list, wait_list); 1184 1185 /* sanity check on discard range */ 1186 __check_sit_bitmap(sbi, lstart, lstart + len); 1187 1188 bio->bi_private = dc; 1189 bio->bi_end_io = f2fs_submit_discard_endio; 1190 bio->bi_opf |= flag; 1191 submit_bio(bio); 1192 1193 atomic_inc(&dcc->issued_discard); 1194 1195 f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE); 1196 1197 lstart += len; 1198 start += len; 1199 total_len -= len; 1200 len = total_len; 1201 } 1202 1203 if (!err && len) { 1204 dcc->undiscard_blks -= len; 1205 __update_discard_tree_range(sbi, bdev, lstart, start, len); 1206 } 1207 return err; 1208 } 1209 1210 static void __insert_discard_tree(struct f2fs_sb_info *sbi, 1211 struct block_device *bdev, block_t lstart, 1212 block_t start, block_t len, 1213 struct rb_node **insert_p, 1214 struct rb_node *insert_parent) 1215 { 1216 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1217 struct rb_node **p; 1218 struct rb_node *parent = NULL; 1219 bool leftmost = true; 1220 1221 if (insert_p && insert_parent) { 1222 parent = insert_parent; 1223 p = insert_p; 1224 goto do_insert; 1225 } 1226 1227 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, 1228 lstart, &leftmost); 1229 do_insert: 1230 __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, 1231 p, leftmost); 1232 } 1233 1234 static void __relocate_discard_cmd(struct discard_cmd_control *dcc, 1235 struct discard_cmd *dc) 1236 { 1237 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]); 1238 } 1239 1240 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, 1241 struct discard_cmd *dc, block_t blkaddr) 1242 { 1243 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1244 struct discard_info di = dc->di; 1245 bool modified = false; 1246 1247 if (dc->state == D_DONE || dc->len == 1) { 1248 __remove_discard_cmd(sbi, dc); 1249 return; 1250 } 1251 1252 dcc->undiscard_blks -= di.len; 1253 1254 if (blkaddr > di.lstart) { 1255 dc->len = blkaddr - dc->lstart; 1256 dcc->undiscard_blks += dc->len; 1257 __relocate_discard_cmd(dcc, dc); 1258 modified = true; 1259 } 1260 1261 if (blkaddr < di.lstart + di.len - 1) { 1262 if (modified) { 1263 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, 1264 di.start + blkaddr + 1 - di.lstart, 1265 di.lstart + di.len - 1 - blkaddr, 1266 NULL, NULL); 1267 } else { 1268 dc->lstart++; 1269 dc->len--; 1270 dc->start++; 1271 dcc->undiscard_blks += dc->len; 1272 __relocate_discard_cmd(dcc, dc); 1273 } 1274 } 1275 } 1276 1277 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1278 struct block_device *bdev, block_t lstart, 1279 block_t start, block_t len) 1280 { 1281 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1282 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1283 struct discard_cmd *dc; 1284 struct discard_info di = {0}; 1285 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1286 unsigned int max_discard_blocks = 1287 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); 1288 block_t end = lstart + len; 1289 1290 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 1291 NULL, lstart, 1292 (struct rb_entry **)&prev_dc, 1293 (struct rb_entry **)&next_dc, 1294 &insert_p, &insert_parent, true, NULL); 1295 if (dc) 1296 prev_dc = dc; 1297 1298 if (!prev_dc) { 1299 di.lstart = lstart; 1300 di.len = next_dc ? next_dc->lstart - lstart : len; 1301 di.len = min(di.len, len); 1302 di.start = start; 1303 } 1304 1305 while (1) { 1306 struct rb_node *node; 1307 bool merged = false; 1308 struct discard_cmd *tdc = NULL; 1309 1310 if (prev_dc) { 1311 di.lstart = prev_dc->lstart + prev_dc->len; 1312 if (di.lstart < lstart) 1313 di.lstart = lstart; 1314 if (di.lstart >= end) 1315 break; 1316 1317 if (!next_dc || next_dc->lstart > end) 1318 di.len = end - di.lstart; 1319 else 1320 di.len = next_dc->lstart - di.lstart; 1321 di.start = start + di.lstart - lstart; 1322 } 1323 1324 if (!di.len) 1325 goto next; 1326 1327 if (prev_dc && prev_dc->state == D_PREP && 1328 prev_dc->bdev == bdev && 1329 __is_discard_back_mergeable(&di, &prev_dc->di, 1330 max_discard_blocks)) { 1331 prev_dc->di.len += di.len; 1332 dcc->undiscard_blks += di.len; 1333 __relocate_discard_cmd(dcc, prev_dc); 1334 di = prev_dc->di; 1335 tdc = prev_dc; 1336 merged = true; 1337 } 1338 1339 if (next_dc && next_dc->state == D_PREP && 1340 next_dc->bdev == bdev && 1341 __is_discard_front_mergeable(&di, &next_dc->di, 1342 max_discard_blocks)) { 1343 next_dc->di.lstart = di.lstart; 1344 next_dc->di.len += di.len; 1345 next_dc->di.start = di.start; 1346 dcc->undiscard_blks += di.len; 1347 __relocate_discard_cmd(dcc, next_dc); 1348 if (tdc) 1349 __remove_discard_cmd(sbi, tdc); 1350 merged = true; 1351 } 1352 1353 if (!merged) { 1354 __insert_discard_tree(sbi, bdev, di.lstart, di.start, 1355 di.len, NULL, NULL); 1356 } 1357 next: 1358 prev_dc = next_dc; 1359 if (!prev_dc) 1360 break; 1361 1362 node = rb_next(&prev_dc->rb_node); 1363 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1364 } 1365 } 1366 1367 static void __queue_discard_cmd(struct f2fs_sb_info *sbi, 1368 struct block_device *bdev, block_t blkstart, block_t blklen) 1369 { 1370 block_t lblkstart = blkstart; 1371 1372 if (!f2fs_bdev_support_discard(bdev)) 1373 return; 1374 1375 trace_f2fs_queue_discard(bdev, blkstart, blklen); 1376 1377 if (f2fs_is_multi_device(sbi)) { 1378 int devi = f2fs_target_device_index(sbi, blkstart); 1379 1380 blkstart -= FDEV(devi).start_blk; 1381 } 1382 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); 1383 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); 1384 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1385 } 1386 1387 static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, 1388 struct discard_policy *dpolicy, int *issued) 1389 { 1390 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1391 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1392 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1393 struct discard_cmd *dc; 1394 struct blk_plug plug; 1395 unsigned int pos = dcc->next_pos; 1396 bool io_interrupted = false; 1397 1398 mutex_lock(&dcc->cmd_lock); 1399 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 1400 NULL, pos, 1401 (struct rb_entry **)&prev_dc, 1402 (struct rb_entry **)&next_dc, 1403 &insert_p, &insert_parent, true, NULL); 1404 if (!dc) 1405 dc = next_dc; 1406 1407 blk_start_plug(&plug); 1408 1409 while (dc) { 1410 struct rb_node *node; 1411 int err = 0; 1412 1413 if (dc->state != D_PREP) 1414 goto next; 1415 1416 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { 1417 io_interrupted = true; 1418 break; 1419 } 1420 1421 dcc->next_pos = dc->lstart + dc->len; 1422 err = __submit_discard_cmd(sbi, dpolicy, dc, issued); 1423 1424 if (*issued >= dpolicy->max_requests) 1425 break; 1426 next: 1427 node = rb_next(&dc->rb_node); 1428 if (err) 1429 __remove_discard_cmd(sbi, dc); 1430 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1431 } 1432 1433 blk_finish_plug(&plug); 1434 1435 if (!dc) 1436 dcc->next_pos = 0; 1437 1438 mutex_unlock(&dcc->cmd_lock); 1439 1440 if (!(*issued) && io_interrupted) 1441 *issued = -1; 1442 } 1443 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1444 struct discard_policy *dpolicy); 1445 1446 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, 1447 struct discard_policy *dpolicy) 1448 { 1449 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1450 struct list_head *pend_list; 1451 struct discard_cmd *dc, *tmp; 1452 struct blk_plug plug; 1453 int i, issued; 1454 bool io_interrupted = false; 1455 1456 if (dpolicy->timeout) 1457 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); 1458 1459 retry: 1460 issued = 0; 1461 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1462 if (dpolicy->timeout && 1463 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1464 break; 1465 1466 if (i + 1 < dpolicy->granularity) 1467 break; 1468 1469 if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) { 1470 __issue_discard_cmd_orderly(sbi, dpolicy, &issued); 1471 return issued; 1472 } 1473 1474 pend_list = &dcc->pend_list[i]; 1475 1476 mutex_lock(&dcc->cmd_lock); 1477 if (list_empty(pend_list)) 1478 goto next; 1479 if (unlikely(dcc->rbtree_check)) 1480 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, 1481 &dcc->root, false)); 1482 blk_start_plug(&plug); 1483 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1484 f2fs_bug_on(sbi, dc->state != D_PREP); 1485 1486 if (dpolicy->timeout && 1487 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1488 break; 1489 1490 if (dpolicy->io_aware && i < dpolicy->io_aware_gran && 1491 !is_idle(sbi, DISCARD_TIME)) { 1492 io_interrupted = true; 1493 break; 1494 } 1495 1496 __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1497 1498 if (issued >= dpolicy->max_requests) 1499 break; 1500 } 1501 blk_finish_plug(&plug); 1502 next: 1503 mutex_unlock(&dcc->cmd_lock); 1504 1505 if (issued >= dpolicy->max_requests || io_interrupted) 1506 break; 1507 } 1508 1509 if (dpolicy->type == DPOLICY_UMOUNT && issued) { 1510 __wait_all_discard_cmd(sbi, dpolicy); 1511 goto retry; 1512 } 1513 1514 if (!issued && io_interrupted) 1515 issued = -1; 1516 1517 return issued; 1518 } 1519 1520 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) 1521 { 1522 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1523 struct list_head *pend_list; 1524 struct discard_cmd *dc, *tmp; 1525 int i; 1526 bool dropped = false; 1527 1528 mutex_lock(&dcc->cmd_lock); 1529 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1530 pend_list = &dcc->pend_list[i]; 1531 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1532 f2fs_bug_on(sbi, dc->state != D_PREP); 1533 __remove_discard_cmd(sbi, dc); 1534 dropped = true; 1535 } 1536 } 1537 mutex_unlock(&dcc->cmd_lock); 1538 1539 return dropped; 1540 } 1541 1542 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) 1543 { 1544 __drop_discard_cmd(sbi); 1545 } 1546 1547 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, 1548 struct discard_cmd *dc) 1549 { 1550 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1551 unsigned int len = 0; 1552 1553 wait_for_completion_io(&dc->wait); 1554 mutex_lock(&dcc->cmd_lock); 1555 f2fs_bug_on(sbi, dc->state != D_DONE); 1556 dc->ref--; 1557 if (!dc->ref) { 1558 if (!dc->error) 1559 len = dc->len; 1560 __remove_discard_cmd(sbi, dc); 1561 } 1562 mutex_unlock(&dcc->cmd_lock); 1563 1564 return len; 1565 } 1566 1567 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, 1568 struct discard_policy *dpolicy, 1569 block_t start, block_t end) 1570 { 1571 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1572 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1573 &(dcc->fstrim_list) : &(dcc->wait_list); 1574 struct discard_cmd *dc = NULL, *iter, *tmp; 1575 unsigned int trimmed = 0; 1576 1577 next: 1578 dc = NULL; 1579 1580 mutex_lock(&dcc->cmd_lock); 1581 list_for_each_entry_safe(iter, tmp, wait_list, list) { 1582 if (iter->lstart + iter->len <= start || end <= iter->lstart) 1583 continue; 1584 if (iter->len < dpolicy->granularity) 1585 continue; 1586 if (iter->state == D_DONE && !iter->ref) { 1587 wait_for_completion_io(&iter->wait); 1588 if (!iter->error) 1589 trimmed += iter->len; 1590 __remove_discard_cmd(sbi, iter); 1591 } else { 1592 iter->ref++; 1593 dc = iter; 1594 break; 1595 } 1596 } 1597 mutex_unlock(&dcc->cmd_lock); 1598 1599 if (dc) { 1600 trimmed += __wait_one_discard_bio(sbi, dc); 1601 goto next; 1602 } 1603 1604 return trimmed; 1605 } 1606 1607 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1608 struct discard_policy *dpolicy) 1609 { 1610 struct discard_policy dp; 1611 unsigned int discard_blks; 1612 1613 if (dpolicy) 1614 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); 1615 1616 /* wait all */ 1617 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY); 1618 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1619 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY); 1620 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1621 1622 return discard_blks; 1623 } 1624 1625 /* This should be covered by global mutex, &sit_i->sentry_lock */ 1626 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) 1627 { 1628 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1629 struct discard_cmd *dc; 1630 bool need_wait = false; 1631 1632 mutex_lock(&dcc->cmd_lock); 1633 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root, 1634 NULL, blkaddr); 1635 if (dc) { 1636 if (dc->state == D_PREP) { 1637 __punch_discard_cmd(sbi, dc, blkaddr); 1638 } else { 1639 dc->ref++; 1640 need_wait = true; 1641 } 1642 } 1643 mutex_unlock(&dcc->cmd_lock); 1644 1645 if (need_wait) 1646 __wait_one_discard_bio(sbi, dc); 1647 } 1648 1649 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) 1650 { 1651 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1652 1653 if (dcc && dcc->f2fs_issue_discard) { 1654 struct task_struct *discard_thread = dcc->f2fs_issue_discard; 1655 1656 dcc->f2fs_issue_discard = NULL; 1657 kthread_stop(discard_thread); 1658 } 1659 } 1660 1661 /** 1662 * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT 1663 * @sbi: the f2fs_sb_info data for discard cmd to issue 1664 * 1665 * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped 1666 * 1667 * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false. 1668 */ 1669 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) 1670 { 1671 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1672 struct discard_policy dpolicy; 1673 bool dropped; 1674 1675 if (!atomic_read(&dcc->discard_cmd_cnt)) 1676 return true; 1677 1678 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, 1679 dcc->discard_granularity); 1680 __issue_discard_cmd(sbi, &dpolicy); 1681 dropped = __drop_discard_cmd(sbi); 1682 1683 /* just to make sure there is no pending discard commands */ 1684 __wait_all_discard_cmd(sbi, NULL); 1685 1686 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); 1687 return !dropped; 1688 } 1689 1690 static int issue_discard_thread(void *data) 1691 { 1692 struct f2fs_sb_info *sbi = data; 1693 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1694 wait_queue_head_t *q = &dcc->discard_wait_queue; 1695 struct discard_policy dpolicy; 1696 unsigned int wait_ms = dcc->min_discard_issue_time; 1697 int issued; 1698 1699 set_freezable(); 1700 1701 do { 1702 wait_event_interruptible_timeout(*q, 1703 kthread_should_stop() || freezing(current) || 1704 dcc->discard_wake, 1705 msecs_to_jiffies(wait_ms)); 1706 1707 if (sbi->gc_mode == GC_URGENT_HIGH || 1708 !f2fs_available_free_memory(sbi, DISCARD_CACHE)) 1709 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1710 MIN_DISCARD_GRANULARITY); 1711 else 1712 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, 1713 dcc->discard_granularity); 1714 1715 if (dcc->discard_wake) 1716 dcc->discard_wake = false; 1717 1718 /* clean up pending candidates before going to sleep */ 1719 if (atomic_read(&dcc->queued_discard)) 1720 __wait_all_discard_cmd(sbi, NULL); 1721 1722 if (try_to_freeze()) 1723 continue; 1724 if (f2fs_readonly(sbi->sb)) 1725 continue; 1726 if (kthread_should_stop()) 1727 return 0; 1728 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || 1729 !atomic_read(&dcc->discard_cmd_cnt)) { 1730 wait_ms = dpolicy.max_interval; 1731 continue; 1732 } 1733 1734 sb_start_intwrite(sbi->sb); 1735 1736 issued = __issue_discard_cmd(sbi, &dpolicy); 1737 if (issued > 0) { 1738 __wait_all_discard_cmd(sbi, &dpolicy); 1739 wait_ms = dpolicy.min_interval; 1740 } else if (issued == -1) { 1741 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); 1742 if (!wait_ms) 1743 wait_ms = dpolicy.mid_interval; 1744 } else { 1745 wait_ms = dpolicy.max_interval; 1746 } 1747 if (!atomic_read(&dcc->discard_cmd_cnt)) 1748 wait_ms = dpolicy.max_interval; 1749 1750 sb_end_intwrite(sbi->sb); 1751 1752 } while (!kthread_should_stop()); 1753 return 0; 1754 } 1755 1756 #ifdef CONFIG_BLK_DEV_ZONED 1757 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, 1758 struct block_device *bdev, block_t blkstart, block_t blklen) 1759 { 1760 sector_t sector, nr_sects; 1761 block_t lblkstart = blkstart; 1762 int devi = 0; 1763 1764 if (f2fs_is_multi_device(sbi)) { 1765 devi = f2fs_target_device_index(sbi, blkstart); 1766 if (blkstart < FDEV(devi).start_blk || 1767 blkstart > FDEV(devi).end_blk) { 1768 f2fs_err(sbi, "Invalid block %x", blkstart); 1769 return -EIO; 1770 } 1771 blkstart -= FDEV(devi).start_blk; 1772 } 1773 1774 /* For sequential zones, reset the zone write pointer */ 1775 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { 1776 sector = SECTOR_FROM_BLOCK(blkstart); 1777 nr_sects = SECTOR_FROM_BLOCK(blklen); 1778 1779 if (sector & (bdev_zone_sectors(bdev) - 1) || 1780 nr_sects != bdev_zone_sectors(bdev)) { 1781 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", 1782 devi, sbi->s_ndevs ? FDEV(devi).path : "", 1783 blkstart, blklen); 1784 return -EIO; 1785 } 1786 trace_f2fs_issue_reset_zone(bdev, blkstart); 1787 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 1788 sector, nr_sects, GFP_NOFS); 1789 } 1790 1791 /* For conventional zones, use regular discard if supported */ 1792 __queue_discard_cmd(sbi, bdev, lblkstart, blklen); 1793 return 0; 1794 } 1795 #endif 1796 1797 static int __issue_discard_async(struct f2fs_sb_info *sbi, 1798 struct block_device *bdev, block_t blkstart, block_t blklen) 1799 { 1800 #ifdef CONFIG_BLK_DEV_ZONED 1801 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) 1802 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); 1803 #endif 1804 __queue_discard_cmd(sbi, bdev, blkstart, blklen); 1805 return 0; 1806 } 1807 1808 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, 1809 block_t blkstart, block_t blklen) 1810 { 1811 sector_t start = blkstart, len = 0; 1812 struct block_device *bdev; 1813 struct seg_entry *se; 1814 unsigned int offset; 1815 block_t i; 1816 int err = 0; 1817 1818 bdev = f2fs_target_device(sbi, blkstart, NULL); 1819 1820 for (i = blkstart; i < blkstart + blklen; i++, len++) { 1821 if (i != start) { 1822 struct block_device *bdev2 = 1823 f2fs_target_device(sbi, i, NULL); 1824 1825 if (bdev2 != bdev) { 1826 err = __issue_discard_async(sbi, bdev, 1827 start, len); 1828 if (err) 1829 return err; 1830 bdev = bdev2; 1831 start = i; 1832 len = 0; 1833 } 1834 } 1835 1836 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); 1837 offset = GET_BLKOFF_FROM_SEG0(sbi, i); 1838 1839 if (f2fs_block_unit_discard(sbi) && 1840 !f2fs_test_and_set_bit(offset, se->discard_map)) 1841 sbi->discard_blks--; 1842 } 1843 1844 if (len) 1845 err = __issue_discard_async(sbi, bdev, start, len); 1846 return err; 1847 } 1848 1849 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, 1850 bool check_only) 1851 { 1852 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 1853 int max_blocks = sbi->blocks_per_seg; 1854 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); 1855 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 1856 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 1857 unsigned long *discard_map = (unsigned long *)se->discard_map; 1858 unsigned long *dmap = SIT_I(sbi)->tmp_map; 1859 unsigned int start = 0, end = -1; 1860 bool force = (cpc->reason & CP_DISCARD); 1861 struct discard_entry *de = NULL; 1862 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; 1863 int i; 1864 1865 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) || 1866 !f2fs_block_unit_discard(sbi)) 1867 return false; 1868 1869 if (!force) { 1870 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks || 1871 SM_I(sbi)->dcc_info->nr_discards >= 1872 SM_I(sbi)->dcc_info->max_discards) 1873 return false; 1874 } 1875 1876 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 1877 for (i = 0; i < entries; i++) 1878 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : 1879 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 1880 1881 while (force || SM_I(sbi)->dcc_info->nr_discards <= 1882 SM_I(sbi)->dcc_info->max_discards) { 1883 start = __find_rev_next_bit(dmap, max_blocks, end + 1); 1884 if (start >= max_blocks) 1885 break; 1886 1887 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); 1888 if (force && start && end != max_blocks 1889 && (end - start) < cpc->trim_minlen) 1890 continue; 1891 1892 if (check_only) 1893 return true; 1894 1895 if (!de) { 1896 de = f2fs_kmem_cache_alloc(discard_entry_slab, 1897 GFP_F2FS_ZERO, true, NULL); 1898 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); 1899 list_add_tail(&de->list, head); 1900 } 1901 1902 for (i = start; i < end; i++) 1903 __set_bit_le(i, (void *)de->discard_map); 1904 1905 SM_I(sbi)->dcc_info->nr_discards += end - start; 1906 } 1907 return false; 1908 } 1909 1910 static void release_discard_addr(struct discard_entry *entry) 1911 { 1912 list_del(&entry->list); 1913 kmem_cache_free(discard_entry_slab, entry); 1914 } 1915 1916 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) 1917 { 1918 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); 1919 struct discard_entry *entry, *this; 1920 1921 /* drop caches */ 1922 list_for_each_entry_safe(entry, this, head, list) 1923 release_discard_addr(entry); 1924 } 1925 1926 /* 1927 * Should call f2fs_clear_prefree_segments after checkpoint is done. 1928 */ 1929 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 1930 { 1931 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1932 unsigned int segno; 1933 1934 mutex_lock(&dirty_i->seglist_lock); 1935 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) 1936 __set_test_and_free(sbi, segno, false); 1937 mutex_unlock(&dirty_i->seglist_lock); 1938 } 1939 1940 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, 1941 struct cp_control *cpc) 1942 { 1943 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1944 struct list_head *head = &dcc->entry_list; 1945 struct discard_entry *entry, *this; 1946 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1947 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 1948 unsigned int start = 0, end = -1; 1949 unsigned int secno, start_segno; 1950 bool force = (cpc->reason & CP_DISCARD); 1951 bool section_alignment = F2FS_OPTION(sbi).discard_unit == 1952 DISCARD_UNIT_SECTION; 1953 1954 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi)) 1955 section_alignment = true; 1956 1957 mutex_lock(&dirty_i->seglist_lock); 1958 1959 while (1) { 1960 int i; 1961 1962 if (section_alignment && end != -1) 1963 end--; 1964 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); 1965 if (start >= MAIN_SEGS(sbi)) 1966 break; 1967 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), 1968 start + 1); 1969 1970 if (section_alignment) { 1971 start = rounddown(start, sbi->segs_per_sec); 1972 end = roundup(end, sbi->segs_per_sec); 1973 } 1974 1975 for (i = start; i < end; i++) { 1976 if (test_and_clear_bit(i, prefree_map)) 1977 dirty_i->nr_dirty[PRE]--; 1978 } 1979 1980 if (!f2fs_realtime_discard_enable(sbi)) 1981 continue; 1982 1983 if (force && start >= cpc->trim_start && 1984 (end - 1) <= cpc->trim_end) 1985 continue; 1986 1987 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) { 1988 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 1989 (end - start) << sbi->log_blocks_per_seg); 1990 continue; 1991 } 1992 next: 1993 secno = GET_SEC_FROM_SEG(sbi, start); 1994 start_segno = GET_SEG_FROM_SEC(sbi, secno); 1995 if (!IS_CURSEC(sbi, secno) && 1996 !get_valid_blocks(sbi, start, true)) 1997 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), 1998 sbi->segs_per_sec << sbi->log_blocks_per_seg); 1999 2000 start = start_segno + sbi->segs_per_sec; 2001 if (start < end) 2002 goto next; 2003 else 2004 end = start - 1; 2005 } 2006 mutex_unlock(&dirty_i->seglist_lock); 2007 2008 if (!f2fs_block_unit_discard(sbi)) 2009 goto wakeup; 2010 2011 /* send small discards */ 2012 list_for_each_entry_safe(entry, this, head, list) { 2013 unsigned int cur_pos = 0, next_pos, len, total_len = 0; 2014 bool is_valid = test_bit_le(0, entry->discard_map); 2015 2016 find_next: 2017 if (is_valid) { 2018 next_pos = find_next_zero_bit_le(entry->discard_map, 2019 sbi->blocks_per_seg, cur_pos); 2020 len = next_pos - cur_pos; 2021 2022 if (f2fs_sb_has_blkzoned(sbi) || 2023 (force && len < cpc->trim_minlen)) 2024 goto skip; 2025 2026 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, 2027 len); 2028 total_len += len; 2029 } else { 2030 next_pos = find_next_bit_le(entry->discard_map, 2031 sbi->blocks_per_seg, cur_pos); 2032 } 2033 skip: 2034 cur_pos = next_pos; 2035 is_valid = !is_valid; 2036 2037 if (cur_pos < sbi->blocks_per_seg) 2038 goto find_next; 2039 2040 release_discard_addr(entry); 2041 dcc->nr_discards -= total_len; 2042 } 2043 2044 wakeup: 2045 wake_up_discard_thread(sbi, false); 2046 } 2047 2048 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi) 2049 { 2050 dev_t dev = sbi->sb->s_bdev->bd_dev; 2051 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2052 int err = 0; 2053 2054 if (!f2fs_realtime_discard_enable(sbi)) 2055 return 0; 2056 2057 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, 2058 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); 2059 if (IS_ERR(dcc->f2fs_issue_discard)) { 2060 err = PTR_ERR(dcc->f2fs_issue_discard); 2061 dcc->f2fs_issue_discard = NULL; 2062 } 2063 2064 return err; 2065 } 2066 2067 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) 2068 { 2069 struct discard_cmd_control *dcc; 2070 int err = 0, i; 2071 2072 if (SM_I(sbi)->dcc_info) { 2073 dcc = SM_I(sbi)->dcc_info; 2074 goto init_thread; 2075 } 2076 2077 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); 2078 if (!dcc) 2079 return -ENOMEM; 2080 2081 dcc->discard_io_aware_gran = MAX_PLIST_NUM; 2082 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; 2083 dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY; 2084 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) 2085 dcc->discard_granularity = sbi->blocks_per_seg; 2086 else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) 2087 dcc->discard_granularity = BLKS_PER_SEC(sbi); 2088 2089 INIT_LIST_HEAD(&dcc->entry_list); 2090 for (i = 0; i < MAX_PLIST_NUM; i++) 2091 INIT_LIST_HEAD(&dcc->pend_list[i]); 2092 INIT_LIST_HEAD(&dcc->wait_list); 2093 INIT_LIST_HEAD(&dcc->fstrim_list); 2094 mutex_init(&dcc->cmd_lock); 2095 atomic_set(&dcc->issued_discard, 0); 2096 atomic_set(&dcc->queued_discard, 0); 2097 atomic_set(&dcc->discard_cmd_cnt, 0); 2098 dcc->nr_discards = 0; 2099 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; 2100 dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST; 2101 dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME; 2102 dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME; 2103 dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME; 2104 dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL; 2105 dcc->undiscard_blks = 0; 2106 dcc->next_pos = 0; 2107 dcc->root = RB_ROOT_CACHED; 2108 dcc->rbtree_check = false; 2109 2110 init_waitqueue_head(&dcc->discard_wait_queue); 2111 SM_I(sbi)->dcc_info = dcc; 2112 init_thread: 2113 err = f2fs_start_discard_thread(sbi); 2114 if (err) { 2115 kfree(dcc); 2116 SM_I(sbi)->dcc_info = NULL; 2117 } 2118 2119 return err; 2120 } 2121 2122 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) 2123 { 2124 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2125 2126 if (!dcc) 2127 return; 2128 2129 f2fs_stop_discard_thread(sbi); 2130 2131 /* 2132 * Recovery can cache discard commands, so in error path of 2133 * fill_super(), it needs to give a chance to handle them. 2134 */ 2135 f2fs_issue_discard_timeout(sbi); 2136 2137 kfree(dcc); 2138 SM_I(sbi)->dcc_info = NULL; 2139 } 2140 2141 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 2142 { 2143 struct sit_info *sit_i = SIT_I(sbi); 2144 2145 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { 2146 sit_i->dirty_sentries++; 2147 return false; 2148 } 2149 2150 return true; 2151 } 2152 2153 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 2154 unsigned int segno, int modified) 2155 { 2156 struct seg_entry *se = get_seg_entry(sbi, segno); 2157 2158 se->type = type; 2159 if (modified) 2160 __mark_sit_entry_dirty(sbi, segno); 2161 } 2162 2163 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi, 2164 block_t blkaddr) 2165 { 2166 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2167 2168 if (segno == NULL_SEGNO) 2169 return 0; 2170 return get_seg_entry(sbi, segno)->mtime; 2171 } 2172 2173 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr, 2174 unsigned long long old_mtime) 2175 { 2176 struct seg_entry *se; 2177 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2178 unsigned long long ctime = get_mtime(sbi, false); 2179 unsigned long long mtime = old_mtime ? old_mtime : ctime; 2180 2181 if (segno == NULL_SEGNO) 2182 return; 2183 2184 se = get_seg_entry(sbi, segno); 2185 2186 if (!se->mtime) 2187 se->mtime = mtime; 2188 else 2189 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime, 2190 se->valid_blocks + 1); 2191 2192 if (ctime > SIT_I(sbi)->max_mtime) 2193 SIT_I(sbi)->max_mtime = ctime; 2194 } 2195 2196 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 2197 { 2198 struct seg_entry *se; 2199 unsigned int segno, offset; 2200 long int new_vblocks; 2201 bool exist; 2202 #ifdef CONFIG_F2FS_CHECK_FS 2203 bool mir_exist; 2204 #endif 2205 2206 segno = GET_SEGNO(sbi, blkaddr); 2207 2208 se = get_seg_entry(sbi, segno); 2209 new_vblocks = se->valid_blocks + del; 2210 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2211 2212 f2fs_bug_on(sbi, (new_vblocks < 0 || 2213 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno)))); 2214 2215 se->valid_blocks = new_vblocks; 2216 2217 /* Update valid block bitmap */ 2218 if (del > 0) { 2219 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); 2220 #ifdef CONFIG_F2FS_CHECK_FS 2221 mir_exist = f2fs_test_and_set_bit(offset, 2222 se->cur_valid_map_mir); 2223 if (unlikely(exist != mir_exist)) { 2224 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d", 2225 blkaddr, exist); 2226 f2fs_bug_on(sbi, 1); 2227 } 2228 #endif 2229 if (unlikely(exist)) { 2230 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", 2231 blkaddr); 2232 f2fs_bug_on(sbi, 1); 2233 se->valid_blocks--; 2234 del = 0; 2235 } 2236 2237 if (f2fs_block_unit_discard(sbi) && 2238 !f2fs_test_and_set_bit(offset, se->discard_map)) 2239 sbi->discard_blks--; 2240 2241 /* 2242 * SSR should never reuse block which is checkpointed 2243 * or newly invalidated. 2244 */ 2245 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { 2246 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map)) 2247 se->ckpt_valid_blocks++; 2248 } 2249 } else { 2250 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map); 2251 #ifdef CONFIG_F2FS_CHECK_FS 2252 mir_exist = f2fs_test_and_clear_bit(offset, 2253 se->cur_valid_map_mir); 2254 if (unlikely(exist != mir_exist)) { 2255 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d", 2256 blkaddr, exist); 2257 f2fs_bug_on(sbi, 1); 2258 } 2259 #endif 2260 if (unlikely(!exist)) { 2261 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", 2262 blkaddr); 2263 f2fs_bug_on(sbi, 1); 2264 se->valid_blocks++; 2265 del = 0; 2266 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2267 /* 2268 * If checkpoints are off, we must not reuse data that 2269 * was used in the previous checkpoint. If it was used 2270 * before, we must track that to know how much space we 2271 * really have. 2272 */ 2273 if (f2fs_test_bit(offset, se->ckpt_valid_map)) { 2274 spin_lock(&sbi->stat_lock); 2275 sbi->unusable_block_count++; 2276 spin_unlock(&sbi->stat_lock); 2277 } 2278 } 2279 2280 if (f2fs_block_unit_discard(sbi) && 2281 f2fs_test_and_clear_bit(offset, se->discard_map)) 2282 sbi->discard_blks++; 2283 } 2284 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 2285 se->ckpt_valid_blocks += del; 2286 2287 __mark_sit_entry_dirty(sbi, segno); 2288 2289 /* update total number of valid blocks to be written in ckpt area */ 2290 SIT_I(sbi)->written_valid_blocks += del; 2291 2292 if (__is_large_section(sbi)) 2293 get_sec_entry(sbi, segno)->valid_blocks += del; 2294 } 2295 2296 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) 2297 { 2298 unsigned int segno = GET_SEGNO(sbi, addr); 2299 struct sit_info *sit_i = SIT_I(sbi); 2300 2301 f2fs_bug_on(sbi, addr == NULL_ADDR); 2302 if (addr == NEW_ADDR || addr == COMPRESS_ADDR) 2303 return; 2304 2305 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr); 2306 f2fs_invalidate_compress_page(sbi, addr); 2307 2308 /* add it into sit main buffer */ 2309 down_write(&sit_i->sentry_lock); 2310 2311 update_segment_mtime(sbi, addr, 0); 2312 update_sit_entry(sbi, addr, -1); 2313 2314 /* add it into dirty seglist */ 2315 locate_dirty_segment(sbi, segno); 2316 2317 up_write(&sit_i->sentry_lock); 2318 } 2319 2320 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) 2321 { 2322 struct sit_info *sit_i = SIT_I(sbi); 2323 unsigned int segno, offset; 2324 struct seg_entry *se; 2325 bool is_cp = false; 2326 2327 if (!__is_valid_data_blkaddr(blkaddr)) 2328 return true; 2329 2330 down_read(&sit_i->sentry_lock); 2331 2332 segno = GET_SEGNO(sbi, blkaddr); 2333 se = get_seg_entry(sbi, segno); 2334 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2335 2336 if (f2fs_test_bit(offset, se->ckpt_valid_map)) 2337 is_cp = true; 2338 2339 up_read(&sit_i->sentry_lock); 2340 2341 return is_cp; 2342 } 2343 2344 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type) 2345 { 2346 struct curseg_info *curseg = CURSEG_I(sbi, type); 2347 2348 if (sbi->ckpt->alloc_type[type] == SSR) 2349 return sbi->blocks_per_seg; 2350 return curseg->next_blkoff; 2351 } 2352 2353 /* 2354 * Calculate the number of current summary pages for writing 2355 */ 2356 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) 2357 { 2358 int valid_sum_count = 0; 2359 int i, sum_in_page; 2360 2361 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2362 if (sbi->ckpt->alloc_type[i] != SSR && for_ra) 2363 valid_sum_count += 2364 le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]); 2365 else 2366 valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); 2367 } 2368 2369 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - 2370 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 2371 if (valid_sum_count <= sum_in_page) 2372 return 1; 2373 else if ((valid_sum_count - sum_in_page) <= 2374 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 2375 return 2; 2376 return 3; 2377 } 2378 2379 /* 2380 * Caller should put this summary page 2381 */ 2382 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) 2383 { 2384 if (unlikely(f2fs_cp_error(sbi))) 2385 return ERR_PTR(-EIO); 2386 return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno)); 2387 } 2388 2389 void f2fs_update_meta_page(struct f2fs_sb_info *sbi, 2390 void *src, block_t blk_addr) 2391 { 2392 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); 2393 2394 memcpy(page_address(page), src, PAGE_SIZE); 2395 set_page_dirty(page); 2396 f2fs_put_page(page, 1); 2397 } 2398 2399 static void write_sum_page(struct f2fs_sb_info *sbi, 2400 struct f2fs_summary_block *sum_blk, block_t blk_addr) 2401 { 2402 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr); 2403 } 2404 2405 static void write_current_sum_page(struct f2fs_sb_info *sbi, 2406 int type, block_t blk_addr) 2407 { 2408 struct curseg_info *curseg = CURSEG_I(sbi, type); 2409 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); 2410 struct f2fs_summary_block *src = curseg->sum_blk; 2411 struct f2fs_summary_block *dst; 2412 2413 dst = (struct f2fs_summary_block *)page_address(page); 2414 memset(dst, 0, PAGE_SIZE); 2415 2416 mutex_lock(&curseg->curseg_mutex); 2417 2418 down_read(&curseg->journal_rwsem); 2419 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); 2420 up_read(&curseg->journal_rwsem); 2421 2422 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); 2423 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); 2424 2425 mutex_unlock(&curseg->curseg_mutex); 2426 2427 set_page_dirty(page); 2428 f2fs_put_page(page, 1); 2429 } 2430 2431 static int is_next_segment_free(struct f2fs_sb_info *sbi, 2432 struct curseg_info *curseg, int type) 2433 { 2434 unsigned int segno = curseg->segno + 1; 2435 struct free_segmap_info *free_i = FREE_I(sbi); 2436 2437 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) 2438 return !test_bit(segno, free_i->free_segmap); 2439 return 0; 2440 } 2441 2442 /* 2443 * Find a new segment from the free segments bitmap to right order 2444 * This function should be returned with success, otherwise BUG 2445 */ 2446 static void get_new_segment(struct f2fs_sb_info *sbi, 2447 unsigned int *newseg, bool new_sec, int dir) 2448 { 2449 struct free_segmap_info *free_i = FREE_I(sbi); 2450 unsigned int segno, secno, zoneno; 2451 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; 2452 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); 2453 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); 2454 unsigned int left_start = hint; 2455 bool init = true; 2456 int go_left = 0; 2457 int i; 2458 2459 spin_lock(&free_i->segmap_lock); 2460 2461 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { 2462 segno = find_next_zero_bit(free_i->free_segmap, 2463 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); 2464 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) 2465 goto got_it; 2466 } 2467 find_other_zone: 2468 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); 2469 if (secno >= MAIN_SECS(sbi)) { 2470 if (dir == ALLOC_RIGHT) { 2471 secno = find_first_zero_bit(free_i->free_secmap, 2472 MAIN_SECS(sbi)); 2473 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); 2474 } else { 2475 go_left = 1; 2476 left_start = hint - 1; 2477 } 2478 } 2479 if (go_left == 0) 2480 goto skip_left; 2481 2482 while (test_bit(left_start, free_i->free_secmap)) { 2483 if (left_start > 0) { 2484 left_start--; 2485 continue; 2486 } 2487 left_start = find_first_zero_bit(free_i->free_secmap, 2488 MAIN_SECS(sbi)); 2489 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); 2490 break; 2491 } 2492 secno = left_start; 2493 skip_left: 2494 segno = GET_SEG_FROM_SEC(sbi, secno); 2495 zoneno = GET_ZONE_FROM_SEC(sbi, secno); 2496 2497 /* give up on finding another zone */ 2498 if (!init) 2499 goto got_it; 2500 if (sbi->secs_per_zone == 1) 2501 goto got_it; 2502 if (zoneno == old_zoneno) 2503 goto got_it; 2504 if (dir == ALLOC_LEFT) { 2505 if (!go_left && zoneno + 1 >= total_zones) 2506 goto got_it; 2507 if (go_left && zoneno == 0) 2508 goto got_it; 2509 } 2510 for (i = 0; i < NR_CURSEG_TYPE; i++) 2511 if (CURSEG_I(sbi, i)->zone == zoneno) 2512 break; 2513 2514 if (i < NR_CURSEG_TYPE) { 2515 /* zone is in user, try another */ 2516 if (go_left) 2517 hint = zoneno * sbi->secs_per_zone - 1; 2518 else if (zoneno + 1 >= total_zones) 2519 hint = 0; 2520 else 2521 hint = (zoneno + 1) * sbi->secs_per_zone; 2522 init = false; 2523 goto find_other_zone; 2524 } 2525 got_it: 2526 /* set it as dirty segment in free segmap */ 2527 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); 2528 __set_inuse(sbi, segno); 2529 *newseg = segno; 2530 spin_unlock(&free_i->segmap_lock); 2531 } 2532 2533 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 2534 { 2535 struct curseg_info *curseg = CURSEG_I(sbi, type); 2536 struct summary_footer *sum_footer; 2537 unsigned short seg_type = curseg->seg_type; 2538 2539 curseg->inited = true; 2540 curseg->segno = curseg->next_segno; 2541 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); 2542 curseg->next_blkoff = 0; 2543 curseg->next_segno = NULL_SEGNO; 2544 2545 sum_footer = &(curseg->sum_blk->footer); 2546 memset(sum_footer, 0, sizeof(struct summary_footer)); 2547 2548 sanity_check_seg_type(sbi, seg_type); 2549 2550 if (IS_DATASEG(seg_type)) 2551 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 2552 if (IS_NODESEG(seg_type)) 2553 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 2554 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified); 2555 } 2556 2557 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) 2558 { 2559 struct curseg_info *curseg = CURSEG_I(sbi, type); 2560 unsigned short seg_type = curseg->seg_type; 2561 2562 sanity_check_seg_type(sbi, seg_type); 2563 if (f2fs_need_rand_seg(sbi)) 2564 return get_random_u32_below(MAIN_SECS(sbi) * sbi->segs_per_sec); 2565 2566 /* if segs_per_sec is large than 1, we need to keep original policy. */ 2567 if (__is_large_section(sbi)) 2568 return curseg->segno; 2569 2570 /* inmem log may not locate on any segment after mount */ 2571 if (!curseg->inited) 2572 return 0; 2573 2574 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 2575 return 0; 2576 2577 if (test_opt(sbi, NOHEAP) && 2578 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type))) 2579 return 0; 2580 2581 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) 2582 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; 2583 2584 /* find segments from 0 to reuse freed segments */ 2585 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) 2586 return 0; 2587 2588 return curseg->segno; 2589 } 2590 2591 /* 2592 * Allocate a current working segment. 2593 * This function always allocates a free segment in LFS manner. 2594 */ 2595 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 2596 { 2597 struct curseg_info *curseg = CURSEG_I(sbi, type); 2598 unsigned short seg_type = curseg->seg_type; 2599 unsigned int segno = curseg->segno; 2600 int dir = ALLOC_LEFT; 2601 2602 if (curseg->inited) 2603 write_sum_page(sbi, curseg->sum_blk, 2604 GET_SUM_BLOCK(sbi, segno)); 2605 if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA) 2606 dir = ALLOC_RIGHT; 2607 2608 if (test_opt(sbi, NOHEAP)) 2609 dir = ALLOC_RIGHT; 2610 2611 segno = __get_next_segno(sbi, type); 2612 get_new_segment(sbi, &segno, new_sec, dir); 2613 curseg->next_segno = segno; 2614 reset_curseg(sbi, type, 1); 2615 curseg->alloc_type = LFS; 2616 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) 2617 curseg->fragment_remained_chunk = 2618 get_random_u32_inclusive(1, sbi->max_fragment_chunk); 2619 } 2620 2621 static int __next_free_blkoff(struct f2fs_sb_info *sbi, 2622 int segno, block_t start) 2623 { 2624 struct seg_entry *se = get_seg_entry(sbi, segno); 2625 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 2626 unsigned long *target_map = SIT_I(sbi)->tmp_map; 2627 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 2628 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 2629 int i; 2630 2631 for (i = 0; i < entries; i++) 2632 target_map[i] = ckpt_map[i] | cur_map[i]; 2633 2634 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); 2635 } 2636 2637 static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi, 2638 struct curseg_info *seg) 2639 { 2640 return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1); 2641 } 2642 2643 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) 2644 { 2645 return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg; 2646 } 2647 2648 /* 2649 * This function always allocates a used segment(from dirty seglist) by SSR 2650 * manner, so it should recover the existing segment information of valid blocks 2651 */ 2652 static void change_curseg(struct f2fs_sb_info *sbi, int type) 2653 { 2654 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2655 struct curseg_info *curseg = CURSEG_I(sbi, type); 2656 unsigned int new_segno = curseg->next_segno; 2657 struct f2fs_summary_block *sum_node; 2658 struct page *sum_page; 2659 2660 write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno)); 2661 2662 __set_test_and_inuse(sbi, new_segno); 2663 2664 mutex_lock(&dirty_i->seglist_lock); 2665 __remove_dirty_segment(sbi, new_segno, PRE); 2666 __remove_dirty_segment(sbi, new_segno, DIRTY); 2667 mutex_unlock(&dirty_i->seglist_lock); 2668 2669 reset_curseg(sbi, type, 1); 2670 curseg->alloc_type = SSR; 2671 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0); 2672 2673 sum_page = f2fs_get_sum_page(sbi, new_segno); 2674 if (IS_ERR(sum_page)) { 2675 /* GC won't be able to use stale summary pages by cp_error */ 2676 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE); 2677 return; 2678 } 2679 sum_node = (struct f2fs_summary_block *)page_address(sum_page); 2680 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); 2681 f2fs_put_page(sum_page, 1); 2682 } 2683 2684 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 2685 int alloc_mode, unsigned long long age); 2686 2687 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type, 2688 int target_type, int alloc_mode, 2689 unsigned long long age) 2690 { 2691 struct curseg_info *curseg = CURSEG_I(sbi, type); 2692 2693 curseg->seg_type = target_type; 2694 2695 if (get_ssr_segment(sbi, type, alloc_mode, age)) { 2696 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno); 2697 2698 curseg->seg_type = se->type; 2699 change_curseg(sbi, type); 2700 } else { 2701 /* allocate cold segment by default */ 2702 curseg->seg_type = CURSEG_COLD_DATA; 2703 new_curseg(sbi, type, true); 2704 } 2705 stat_inc_seg_type(sbi, curseg); 2706 } 2707 2708 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi) 2709 { 2710 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC); 2711 2712 if (!sbi->am.atgc_enabled) 2713 return; 2714 2715 f2fs_down_read(&SM_I(sbi)->curseg_lock); 2716 2717 mutex_lock(&curseg->curseg_mutex); 2718 down_write(&SIT_I(sbi)->sentry_lock); 2719 2720 get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0); 2721 2722 up_write(&SIT_I(sbi)->sentry_lock); 2723 mutex_unlock(&curseg->curseg_mutex); 2724 2725 f2fs_up_read(&SM_I(sbi)->curseg_lock); 2726 2727 } 2728 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi) 2729 { 2730 __f2fs_init_atgc_curseg(sbi); 2731 } 2732 2733 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) 2734 { 2735 struct curseg_info *curseg = CURSEG_I(sbi, type); 2736 2737 mutex_lock(&curseg->curseg_mutex); 2738 if (!curseg->inited) 2739 goto out; 2740 2741 if (get_valid_blocks(sbi, curseg->segno, false)) { 2742 write_sum_page(sbi, curseg->sum_blk, 2743 GET_SUM_BLOCK(sbi, curseg->segno)); 2744 } else { 2745 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 2746 __set_test_and_free(sbi, curseg->segno, true); 2747 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 2748 } 2749 out: 2750 mutex_unlock(&curseg->curseg_mutex); 2751 } 2752 2753 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi) 2754 { 2755 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 2756 2757 if (sbi->am.atgc_enabled) 2758 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 2759 } 2760 2761 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type) 2762 { 2763 struct curseg_info *curseg = CURSEG_I(sbi, type); 2764 2765 mutex_lock(&curseg->curseg_mutex); 2766 if (!curseg->inited) 2767 goto out; 2768 if (get_valid_blocks(sbi, curseg->segno, false)) 2769 goto out; 2770 2771 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 2772 __set_test_and_inuse(sbi, curseg->segno); 2773 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 2774 out: 2775 mutex_unlock(&curseg->curseg_mutex); 2776 } 2777 2778 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi) 2779 { 2780 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 2781 2782 if (sbi->am.atgc_enabled) 2783 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 2784 } 2785 2786 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 2787 int alloc_mode, unsigned long long age) 2788 { 2789 struct curseg_info *curseg = CURSEG_I(sbi, type); 2790 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; 2791 unsigned segno = NULL_SEGNO; 2792 unsigned short seg_type = curseg->seg_type; 2793 int i, cnt; 2794 bool reversed = false; 2795 2796 sanity_check_seg_type(sbi, seg_type); 2797 2798 /* f2fs_need_SSR() already forces to do this */ 2799 if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) { 2800 curseg->next_segno = segno; 2801 return 1; 2802 } 2803 2804 /* For node segments, let's do SSR more intensively */ 2805 if (IS_NODESEG(seg_type)) { 2806 if (seg_type >= CURSEG_WARM_NODE) { 2807 reversed = true; 2808 i = CURSEG_COLD_NODE; 2809 } else { 2810 i = CURSEG_HOT_NODE; 2811 } 2812 cnt = NR_CURSEG_NODE_TYPE; 2813 } else { 2814 if (seg_type >= CURSEG_WARM_DATA) { 2815 reversed = true; 2816 i = CURSEG_COLD_DATA; 2817 } else { 2818 i = CURSEG_HOT_DATA; 2819 } 2820 cnt = NR_CURSEG_DATA_TYPE; 2821 } 2822 2823 for (; cnt-- > 0; reversed ? i-- : i++) { 2824 if (i == seg_type) 2825 continue; 2826 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) { 2827 curseg->next_segno = segno; 2828 return 1; 2829 } 2830 } 2831 2832 /* find valid_blocks=0 in dirty list */ 2833 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2834 segno = get_free_segment(sbi); 2835 if (segno != NULL_SEGNO) { 2836 curseg->next_segno = segno; 2837 return 1; 2838 } 2839 } 2840 return 0; 2841 } 2842 2843 static bool need_new_seg(struct f2fs_sb_info *sbi, int type) 2844 { 2845 struct curseg_info *curseg = CURSEG_I(sbi, type); 2846 2847 if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && 2848 curseg->seg_type == CURSEG_WARM_NODE) 2849 return true; 2850 if (curseg->alloc_type == LFS && 2851 is_next_segment_free(sbi, curseg, type) && 2852 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 2853 return true; 2854 if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0)) 2855 return true; 2856 return false; 2857 } 2858 2859 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, 2860 unsigned int start, unsigned int end) 2861 { 2862 struct curseg_info *curseg = CURSEG_I(sbi, type); 2863 unsigned int segno; 2864 2865 f2fs_down_read(&SM_I(sbi)->curseg_lock); 2866 mutex_lock(&curseg->curseg_mutex); 2867 down_write(&SIT_I(sbi)->sentry_lock); 2868 2869 segno = CURSEG_I(sbi, type)->segno; 2870 if (segno < start || segno > end) 2871 goto unlock; 2872 2873 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0)) 2874 change_curseg(sbi, type); 2875 else 2876 new_curseg(sbi, type, true); 2877 2878 stat_inc_seg_type(sbi, curseg); 2879 2880 locate_dirty_segment(sbi, segno); 2881 unlock: 2882 up_write(&SIT_I(sbi)->sentry_lock); 2883 2884 if (segno != curseg->segno) 2885 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u", 2886 type, segno, curseg->segno); 2887 2888 mutex_unlock(&curseg->curseg_mutex); 2889 f2fs_up_read(&SM_I(sbi)->curseg_lock); 2890 } 2891 2892 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type, 2893 bool new_sec, bool force) 2894 { 2895 struct curseg_info *curseg = CURSEG_I(sbi, type); 2896 unsigned int old_segno; 2897 2898 if (!force && curseg->inited && 2899 !curseg->next_blkoff && 2900 !get_valid_blocks(sbi, curseg->segno, new_sec) && 2901 !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) 2902 return; 2903 2904 old_segno = curseg->segno; 2905 new_curseg(sbi, type, true); 2906 stat_inc_seg_type(sbi, curseg); 2907 locate_dirty_segment(sbi, old_segno); 2908 } 2909 2910 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) 2911 { 2912 f2fs_down_read(&SM_I(sbi)->curseg_lock); 2913 down_write(&SIT_I(sbi)->sentry_lock); 2914 __allocate_new_segment(sbi, type, true, force); 2915 up_write(&SIT_I(sbi)->sentry_lock); 2916 f2fs_up_read(&SM_I(sbi)->curseg_lock); 2917 } 2918 2919 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) 2920 { 2921 int i; 2922 2923 f2fs_down_read(&SM_I(sbi)->curseg_lock); 2924 down_write(&SIT_I(sbi)->sentry_lock); 2925 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) 2926 __allocate_new_segment(sbi, i, false, false); 2927 up_write(&SIT_I(sbi)->sentry_lock); 2928 f2fs_up_read(&SM_I(sbi)->curseg_lock); 2929 } 2930 2931 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, 2932 struct cp_control *cpc) 2933 { 2934 __u64 trim_start = cpc->trim_start; 2935 bool has_candidate = false; 2936 2937 down_write(&SIT_I(sbi)->sentry_lock); 2938 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { 2939 if (add_discard_addrs(sbi, cpc, true)) { 2940 has_candidate = true; 2941 break; 2942 } 2943 } 2944 up_write(&SIT_I(sbi)->sentry_lock); 2945 2946 cpc->trim_start = trim_start; 2947 return has_candidate; 2948 } 2949 2950 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, 2951 struct discard_policy *dpolicy, 2952 unsigned int start, unsigned int end) 2953 { 2954 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2955 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 2956 struct rb_node **insert_p = NULL, *insert_parent = NULL; 2957 struct discard_cmd *dc; 2958 struct blk_plug plug; 2959 int issued; 2960 unsigned int trimmed = 0; 2961 2962 next: 2963 issued = 0; 2964 2965 mutex_lock(&dcc->cmd_lock); 2966 if (unlikely(dcc->rbtree_check)) 2967 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, 2968 &dcc->root, false)); 2969 2970 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 2971 NULL, start, 2972 (struct rb_entry **)&prev_dc, 2973 (struct rb_entry **)&next_dc, 2974 &insert_p, &insert_parent, true, NULL); 2975 if (!dc) 2976 dc = next_dc; 2977 2978 blk_start_plug(&plug); 2979 2980 while (dc && dc->lstart <= end) { 2981 struct rb_node *node; 2982 int err = 0; 2983 2984 if (dc->len < dpolicy->granularity) 2985 goto skip; 2986 2987 if (dc->state != D_PREP) { 2988 list_move_tail(&dc->list, &dcc->fstrim_list); 2989 goto skip; 2990 } 2991 2992 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 2993 2994 if (issued >= dpolicy->max_requests) { 2995 start = dc->lstart + dc->len; 2996 2997 if (err) 2998 __remove_discard_cmd(sbi, dc); 2999 3000 blk_finish_plug(&plug); 3001 mutex_unlock(&dcc->cmd_lock); 3002 trimmed += __wait_all_discard_cmd(sbi, NULL); 3003 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); 3004 goto next; 3005 } 3006 skip: 3007 node = rb_next(&dc->rb_node); 3008 if (err) 3009 __remove_discard_cmd(sbi, dc); 3010 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 3011 3012 if (fatal_signal_pending(current)) 3013 break; 3014 } 3015 3016 blk_finish_plug(&plug); 3017 mutex_unlock(&dcc->cmd_lock); 3018 3019 return trimmed; 3020 } 3021 3022 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) 3023 { 3024 __u64 start = F2FS_BYTES_TO_BLK(range->start); 3025 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; 3026 unsigned int start_segno, end_segno; 3027 block_t start_block, end_block; 3028 struct cp_control cpc; 3029 struct discard_policy dpolicy; 3030 unsigned long long trimmed = 0; 3031 int err = 0; 3032 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); 3033 3034 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) 3035 return -EINVAL; 3036 3037 if (end < MAIN_BLKADDR(sbi)) 3038 goto out; 3039 3040 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 3041 f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); 3042 return -EFSCORRUPTED; 3043 } 3044 3045 /* start/end segment number in main_area */ 3046 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); 3047 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : 3048 GET_SEGNO(sbi, end); 3049 if (need_align) { 3050 start_segno = rounddown(start_segno, sbi->segs_per_sec); 3051 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1; 3052 } 3053 3054 cpc.reason = CP_DISCARD; 3055 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); 3056 cpc.trim_start = start_segno; 3057 cpc.trim_end = end_segno; 3058 3059 if (sbi->discard_blks == 0) 3060 goto out; 3061 3062 f2fs_down_write(&sbi->gc_lock); 3063 err = f2fs_write_checkpoint(sbi, &cpc); 3064 f2fs_up_write(&sbi->gc_lock); 3065 if (err) 3066 goto out; 3067 3068 /* 3069 * We filed discard candidates, but actually we don't need to wait for 3070 * all of them, since they'll be issued in idle time along with runtime 3071 * discard option. User configuration looks like using runtime discard 3072 * or periodic fstrim instead of it. 3073 */ 3074 if (f2fs_realtime_discard_enable(sbi)) 3075 goto out; 3076 3077 start_block = START_BLOCK(sbi, start_segno); 3078 end_block = START_BLOCK(sbi, end_segno + 1); 3079 3080 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); 3081 trimmed = __issue_discard_cmd_range(sbi, &dpolicy, 3082 start_block, end_block); 3083 3084 trimmed += __wait_discard_cmd_range(sbi, &dpolicy, 3085 start_block, end_block); 3086 out: 3087 if (!err) 3088 range->len = F2FS_BLK_TO_BYTES(trimmed); 3089 return err; 3090 } 3091 3092 int f2fs_rw_hint_to_seg_type(enum rw_hint hint) 3093 { 3094 switch (hint) { 3095 case WRITE_LIFE_SHORT: 3096 return CURSEG_HOT_DATA; 3097 case WRITE_LIFE_EXTREME: 3098 return CURSEG_COLD_DATA; 3099 default: 3100 return CURSEG_WARM_DATA; 3101 } 3102 } 3103 3104 static int __get_segment_type_2(struct f2fs_io_info *fio) 3105 { 3106 if (fio->type == DATA) 3107 return CURSEG_HOT_DATA; 3108 else 3109 return CURSEG_HOT_NODE; 3110 } 3111 3112 static int __get_segment_type_4(struct f2fs_io_info *fio) 3113 { 3114 if (fio->type == DATA) { 3115 struct inode *inode = fio->page->mapping->host; 3116 3117 if (S_ISDIR(inode->i_mode)) 3118 return CURSEG_HOT_DATA; 3119 else 3120 return CURSEG_COLD_DATA; 3121 } else { 3122 if (IS_DNODE(fio->page) && is_cold_node(fio->page)) 3123 return CURSEG_WARM_NODE; 3124 else 3125 return CURSEG_COLD_NODE; 3126 } 3127 } 3128 3129 static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs) 3130 { 3131 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3132 struct extent_info ei = {}; 3133 3134 if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) { 3135 if (!ei.age) 3136 return NO_CHECK_TYPE; 3137 if (ei.age <= sbi->hot_data_age_threshold) 3138 return CURSEG_HOT_DATA; 3139 if (ei.age <= sbi->warm_data_age_threshold) 3140 return CURSEG_WARM_DATA; 3141 return CURSEG_COLD_DATA; 3142 } 3143 return NO_CHECK_TYPE; 3144 } 3145 3146 static int __get_segment_type_6(struct f2fs_io_info *fio) 3147 { 3148 if (fio->type == DATA) { 3149 struct inode *inode = fio->page->mapping->host; 3150 int type; 3151 3152 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 3153 return CURSEG_COLD_DATA_PINNED; 3154 3155 if (page_private_gcing(fio->page)) { 3156 if (fio->sbi->am.atgc_enabled && 3157 (fio->io_type == FS_DATA_IO) && 3158 (fio->sbi->gc_mode != GC_URGENT_HIGH)) 3159 return CURSEG_ALL_DATA_ATGC; 3160 else 3161 return CURSEG_COLD_DATA; 3162 } 3163 if (file_is_cold(inode) || f2fs_need_compress_data(inode)) 3164 return CURSEG_COLD_DATA; 3165 3166 type = __get_age_segment_type(inode, fio->page->index); 3167 if (type != NO_CHECK_TYPE) 3168 return type; 3169 3170 if (file_is_hot(inode) || 3171 is_inode_flag_set(inode, FI_HOT_DATA) || 3172 f2fs_is_cow_file(inode)) 3173 return CURSEG_HOT_DATA; 3174 return f2fs_rw_hint_to_seg_type(inode->i_write_hint); 3175 } else { 3176 if (IS_DNODE(fio->page)) 3177 return is_cold_node(fio->page) ? CURSEG_WARM_NODE : 3178 CURSEG_HOT_NODE; 3179 return CURSEG_COLD_NODE; 3180 } 3181 } 3182 3183 static int __get_segment_type(struct f2fs_io_info *fio) 3184 { 3185 int type = 0; 3186 3187 switch (F2FS_OPTION(fio->sbi).active_logs) { 3188 case 2: 3189 type = __get_segment_type_2(fio); 3190 break; 3191 case 4: 3192 type = __get_segment_type_4(fio); 3193 break; 3194 case 6: 3195 type = __get_segment_type_6(fio); 3196 break; 3197 default: 3198 f2fs_bug_on(fio->sbi, true); 3199 } 3200 3201 if (IS_HOT(type)) 3202 fio->temp = HOT; 3203 else if (IS_WARM(type)) 3204 fio->temp = WARM; 3205 else 3206 fio->temp = COLD; 3207 return type; 3208 } 3209 3210 static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi, 3211 struct curseg_info *seg) 3212 { 3213 /* To allocate block chunks in different sizes, use random number */ 3214 if (--seg->fragment_remained_chunk > 0) 3215 return; 3216 3217 seg->fragment_remained_chunk = 3218 get_random_u32_inclusive(1, sbi->max_fragment_chunk); 3219 seg->next_blkoff += 3220 get_random_u32_inclusive(1, sbi->max_fragment_hole); 3221 } 3222 3223 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, 3224 block_t old_blkaddr, block_t *new_blkaddr, 3225 struct f2fs_summary *sum, int type, 3226 struct f2fs_io_info *fio) 3227 { 3228 struct sit_info *sit_i = SIT_I(sbi); 3229 struct curseg_info *curseg = CURSEG_I(sbi, type); 3230 unsigned long long old_mtime; 3231 bool from_gc = (type == CURSEG_ALL_DATA_ATGC); 3232 struct seg_entry *se = NULL; 3233 bool segment_full = false; 3234 3235 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3236 3237 mutex_lock(&curseg->curseg_mutex); 3238 down_write(&sit_i->sentry_lock); 3239 3240 if (from_gc) { 3241 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO); 3242 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr)); 3243 sanity_check_seg_type(sbi, se->type); 3244 f2fs_bug_on(sbi, IS_NODESEG(se->type)); 3245 } 3246 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 3247 3248 f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg); 3249 3250 f2fs_wait_discard_bio(sbi, *new_blkaddr); 3251 3252 curseg->sum_blk->entries[curseg->next_blkoff] = *sum; 3253 if (curseg->alloc_type == SSR) { 3254 curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); 3255 } else { 3256 curseg->next_blkoff++; 3257 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) 3258 f2fs_randomize_chunk(sbi, curseg); 3259 } 3260 if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) 3261 segment_full = true; 3262 stat_inc_block_count(sbi, curseg); 3263 3264 if (from_gc) { 3265 old_mtime = get_segment_mtime(sbi, old_blkaddr); 3266 } else { 3267 update_segment_mtime(sbi, old_blkaddr, 0); 3268 old_mtime = 0; 3269 } 3270 update_segment_mtime(sbi, *new_blkaddr, old_mtime); 3271 3272 /* 3273 * SIT information should be updated before segment allocation, 3274 * since SSR needs latest valid block information. 3275 */ 3276 update_sit_entry(sbi, *new_blkaddr, 1); 3277 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 3278 update_sit_entry(sbi, old_blkaddr, -1); 3279 3280 /* 3281 * If the current segment is full, flush it out and replace it with a 3282 * new segment. 3283 */ 3284 if (segment_full) { 3285 if (from_gc) { 3286 get_atssr_segment(sbi, type, se->type, 3287 AT_SSR, se->mtime); 3288 } else { 3289 if (need_new_seg(sbi, type)) 3290 new_curseg(sbi, type, false); 3291 else 3292 change_curseg(sbi, type); 3293 stat_inc_seg_type(sbi, curseg); 3294 } 3295 } 3296 /* 3297 * segment dirty status should be updated after segment allocation, 3298 * so we just need to update status only one time after previous 3299 * segment being closed. 3300 */ 3301 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 3302 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); 3303 3304 if (IS_DATASEG(type)) 3305 atomic64_inc(&sbi->allocated_data_blocks); 3306 3307 up_write(&sit_i->sentry_lock); 3308 3309 if (page && IS_NODESEG(type)) { 3310 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); 3311 3312 f2fs_inode_chksum_set(sbi, page); 3313 } 3314 3315 if (fio) { 3316 struct f2fs_bio_info *io; 3317 3318 if (F2FS_IO_ALIGNED(sbi)) 3319 fio->retry = 0; 3320 3321 INIT_LIST_HEAD(&fio->list); 3322 fio->in_list = 1; 3323 io = sbi->write_io[fio->type] + fio->temp; 3324 spin_lock(&io->io_lock); 3325 list_add_tail(&fio->list, &io->io_list); 3326 spin_unlock(&io->io_lock); 3327 } 3328 3329 mutex_unlock(&curseg->curseg_mutex); 3330 3331 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3332 } 3333 3334 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, 3335 block_t blkaddr, unsigned int blkcnt) 3336 { 3337 if (!f2fs_is_multi_device(sbi)) 3338 return; 3339 3340 while (1) { 3341 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr); 3342 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1; 3343 3344 /* update device state for fsync */ 3345 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO); 3346 3347 /* update device state for checkpoint */ 3348 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { 3349 spin_lock(&sbi->dev_lock); 3350 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); 3351 spin_unlock(&sbi->dev_lock); 3352 } 3353 3354 if (blkcnt <= blks) 3355 break; 3356 blkcnt -= blks; 3357 blkaddr += blks; 3358 } 3359 } 3360 3361 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) 3362 { 3363 int type = __get_segment_type(fio); 3364 bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA); 3365 3366 if (keep_order) 3367 f2fs_down_read(&fio->sbi->io_order_lock); 3368 reallocate: 3369 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, 3370 &fio->new_blkaddr, sum, type, fio); 3371 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) { 3372 invalidate_mapping_pages(META_MAPPING(fio->sbi), 3373 fio->old_blkaddr, fio->old_blkaddr); 3374 f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr); 3375 } 3376 3377 /* writeout dirty page into bdev */ 3378 f2fs_submit_page_write(fio); 3379 if (fio->retry) { 3380 fio->old_blkaddr = fio->new_blkaddr; 3381 goto reallocate; 3382 } 3383 3384 f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); 3385 3386 if (keep_order) 3387 f2fs_up_read(&fio->sbi->io_order_lock); 3388 } 3389 3390 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page, 3391 enum iostat_type io_type) 3392 { 3393 struct f2fs_io_info fio = { 3394 .sbi = sbi, 3395 .type = META, 3396 .temp = HOT, 3397 .op = REQ_OP_WRITE, 3398 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, 3399 .old_blkaddr = page->index, 3400 .new_blkaddr = page->index, 3401 .page = page, 3402 .encrypted_page = NULL, 3403 .in_list = 0, 3404 }; 3405 3406 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) 3407 fio.op_flags &= ~REQ_META; 3408 3409 set_page_writeback(page); 3410 f2fs_submit_page_write(&fio); 3411 3412 stat_inc_meta_count(sbi, page->index); 3413 f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE); 3414 } 3415 3416 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) 3417 { 3418 struct f2fs_summary sum; 3419 3420 set_summary(&sum, nid, 0, 0); 3421 do_write_page(&sum, fio); 3422 3423 f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE); 3424 } 3425 3426 void f2fs_outplace_write_data(struct dnode_of_data *dn, 3427 struct f2fs_io_info *fio) 3428 { 3429 struct f2fs_sb_info *sbi = fio->sbi; 3430 struct f2fs_summary sum; 3431 3432 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); 3433 if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO) 3434 f2fs_update_age_extent_cache(dn); 3435 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); 3436 do_write_page(&sum, fio); 3437 f2fs_update_data_blkaddr(dn, fio->new_blkaddr); 3438 3439 f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE); 3440 } 3441 3442 int f2fs_inplace_write_data(struct f2fs_io_info *fio) 3443 { 3444 int err; 3445 struct f2fs_sb_info *sbi = fio->sbi; 3446 unsigned int segno; 3447 3448 fio->new_blkaddr = fio->old_blkaddr; 3449 /* i/o temperature is needed for passing down write hints */ 3450 __get_segment_type(fio); 3451 3452 segno = GET_SEGNO(sbi, fio->new_blkaddr); 3453 3454 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) { 3455 set_sbi_flag(sbi, SBI_NEED_FSCK); 3456 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", 3457 __func__, segno); 3458 err = -EFSCORRUPTED; 3459 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); 3460 goto drop_bio; 3461 } 3462 3463 if (f2fs_cp_error(sbi)) { 3464 err = -EIO; 3465 goto drop_bio; 3466 } 3467 3468 if (fio->post_read) 3469 invalidate_mapping_pages(META_MAPPING(sbi), 3470 fio->new_blkaddr, fio->new_blkaddr); 3471 3472 stat_inc_inplace_blocks(fio->sbi); 3473 3474 if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi)) 3475 err = f2fs_merge_page_bio(fio); 3476 else 3477 err = f2fs_submit_page_bio(fio); 3478 if (!err) { 3479 f2fs_update_device_state(fio->sbi, fio->ino, 3480 fio->new_blkaddr, 1); 3481 f2fs_update_iostat(fio->sbi, fio->page->mapping->host, 3482 fio->io_type, F2FS_BLKSIZE); 3483 } 3484 3485 return err; 3486 drop_bio: 3487 if (fio->bio && *(fio->bio)) { 3488 struct bio *bio = *(fio->bio); 3489 3490 bio->bi_status = BLK_STS_IOERR; 3491 bio_endio(bio); 3492 *(fio->bio) = NULL; 3493 } 3494 return err; 3495 } 3496 3497 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, 3498 unsigned int segno) 3499 { 3500 int i; 3501 3502 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { 3503 if (CURSEG_I(sbi, i)->segno == segno) 3504 break; 3505 } 3506 return i; 3507 } 3508 3509 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, 3510 block_t old_blkaddr, block_t new_blkaddr, 3511 bool recover_curseg, bool recover_newaddr, 3512 bool from_gc) 3513 { 3514 struct sit_info *sit_i = SIT_I(sbi); 3515 struct curseg_info *curseg; 3516 unsigned int segno, old_cursegno; 3517 struct seg_entry *se; 3518 int type; 3519 unsigned short old_blkoff; 3520 unsigned char old_alloc_type; 3521 3522 segno = GET_SEGNO(sbi, new_blkaddr); 3523 se = get_seg_entry(sbi, segno); 3524 type = se->type; 3525 3526 f2fs_down_write(&SM_I(sbi)->curseg_lock); 3527 3528 if (!recover_curseg) { 3529 /* for recovery flow */ 3530 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { 3531 if (old_blkaddr == NULL_ADDR) 3532 type = CURSEG_COLD_DATA; 3533 else 3534 type = CURSEG_WARM_DATA; 3535 } 3536 } else { 3537 if (IS_CURSEG(sbi, segno)) { 3538 /* se->type is volatile as SSR allocation */ 3539 type = __f2fs_get_curseg(sbi, segno); 3540 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); 3541 } else { 3542 type = CURSEG_WARM_DATA; 3543 } 3544 } 3545 3546 f2fs_bug_on(sbi, !IS_DATASEG(type)); 3547 curseg = CURSEG_I(sbi, type); 3548 3549 mutex_lock(&curseg->curseg_mutex); 3550 down_write(&sit_i->sentry_lock); 3551 3552 old_cursegno = curseg->segno; 3553 old_blkoff = curseg->next_blkoff; 3554 old_alloc_type = curseg->alloc_type; 3555 3556 /* change the current segment */ 3557 if (segno != curseg->segno) { 3558 curseg->next_segno = segno; 3559 change_curseg(sbi, type); 3560 } 3561 3562 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 3563 curseg->sum_blk->entries[curseg->next_blkoff] = *sum; 3564 3565 if (!recover_curseg || recover_newaddr) { 3566 if (!from_gc) 3567 update_segment_mtime(sbi, new_blkaddr, 0); 3568 update_sit_entry(sbi, new_blkaddr, 1); 3569 } 3570 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { 3571 invalidate_mapping_pages(META_MAPPING(sbi), 3572 old_blkaddr, old_blkaddr); 3573 f2fs_invalidate_compress_page(sbi, old_blkaddr); 3574 if (!from_gc) 3575 update_segment_mtime(sbi, old_blkaddr, 0); 3576 update_sit_entry(sbi, old_blkaddr, -1); 3577 } 3578 3579 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 3580 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); 3581 3582 locate_dirty_segment(sbi, old_cursegno); 3583 3584 if (recover_curseg) { 3585 if (old_cursegno != curseg->segno) { 3586 curseg->next_segno = old_cursegno; 3587 change_curseg(sbi, type); 3588 } 3589 curseg->next_blkoff = old_blkoff; 3590 curseg->alloc_type = old_alloc_type; 3591 } 3592 3593 up_write(&sit_i->sentry_lock); 3594 mutex_unlock(&curseg->curseg_mutex); 3595 f2fs_up_write(&SM_I(sbi)->curseg_lock); 3596 } 3597 3598 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, 3599 block_t old_addr, block_t new_addr, 3600 unsigned char version, bool recover_curseg, 3601 bool recover_newaddr) 3602 { 3603 struct f2fs_summary sum; 3604 3605 set_summary(&sum, dn->nid, dn->ofs_in_node, version); 3606 3607 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, 3608 recover_curseg, recover_newaddr, false); 3609 3610 f2fs_update_data_blkaddr(dn, new_addr); 3611 } 3612 3613 void f2fs_wait_on_page_writeback(struct page *page, 3614 enum page_type type, bool ordered, bool locked) 3615 { 3616 if (PageWriteback(page)) { 3617 struct f2fs_sb_info *sbi = F2FS_P_SB(page); 3618 3619 /* submit cached LFS IO */ 3620 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type); 3621 /* submit cached IPU IO */ 3622 f2fs_submit_merged_ipu_write(sbi, NULL, page); 3623 if (ordered) { 3624 wait_on_page_writeback(page); 3625 f2fs_bug_on(sbi, locked && PageWriteback(page)); 3626 } else { 3627 wait_for_stable_page(page); 3628 } 3629 } 3630 } 3631 3632 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) 3633 { 3634 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3635 struct page *cpage; 3636 3637 if (!f2fs_post_read_required(inode)) 3638 return; 3639 3640 if (!__is_valid_data_blkaddr(blkaddr)) 3641 return; 3642 3643 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); 3644 if (cpage) { 3645 f2fs_wait_on_page_writeback(cpage, DATA, true, true); 3646 f2fs_put_page(cpage, 1); 3647 } 3648 } 3649 3650 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, 3651 block_t len) 3652 { 3653 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3654 block_t i; 3655 3656 if (!f2fs_post_read_required(inode)) 3657 return; 3658 3659 for (i = 0; i < len; i++) 3660 f2fs_wait_on_block_writeback(inode, blkaddr + i); 3661 3662 invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1); 3663 } 3664 3665 static int read_compacted_summaries(struct f2fs_sb_info *sbi) 3666 { 3667 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3668 struct curseg_info *seg_i; 3669 unsigned char *kaddr; 3670 struct page *page; 3671 block_t start; 3672 int i, j, offset; 3673 3674 start = start_sum_block(sbi); 3675 3676 page = f2fs_get_meta_page(sbi, start++); 3677 if (IS_ERR(page)) 3678 return PTR_ERR(page); 3679 kaddr = (unsigned char *)page_address(page); 3680 3681 /* Step 1: restore nat cache */ 3682 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3683 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); 3684 3685 /* Step 2: restore sit cache */ 3686 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3687 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); 3688 offset = 2 * SUM_JOURNAL_SIZE; 3689 3690 /* Step 3: restore summary entries */ 3691 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3692 unsigned short blk_off; 3693 unsigned int segno; 3694 3695 seg_i = CURSEG_I(sbi, i); 3696 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 3697 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 3698 seg_i->next_segno = segno; 3699 reset_curseg(sbi, i, 0); 3700 seg_i->alloc_type = ckpt->alloc_type[i]; 3701 seg_i->next_blkoff = blk_off; 3702 3703 if (seg_i->alloc_type == SSR) 3704 blk_off = sbi->blocks_per_seg; 3705 3706 for (j = 0; j < blk_off; j++) { 3707 struct f2fs_summary *s; 3708 3709 s = (struct f2fs_summary *)(kaddr + offset); 3710 seg_i->sum_blk->entries[j] = *s; 3711 offset += SUMMARY_SIZE; 3712 if (offset + SUMMARY_SIZE <= PAGE_SIZE - 3713 SUM_FOOTER_SIZE) 3714 continue; 3715 3716 f2fs_put_page(page, 1); 3717 page = NULL; 3718 3719 page = f2fs_get_meta_page(sbi, start++); 3720 if (IS_ERR(page)) 3721 return PTR_ERR(page); 3722 kaddr = (unsigned char *)page_address(page); 3723 offset = 0; 3724 } 3725 } 3726 f2fs_put_page(page, 1); 3727 return 0; 3728 } 3729 3730 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 3731 { 3732 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3733 struct f2fs_summary_block *sum; 3734 struct curseg_info *curseg; 3735 struct page *new; 3736 unsigned short blk_off; 3737 unsigned int segno = 0; 3738 block_t blk_addr = 0; 3739 int err = 0; 3740 3741 /* get segment number and block addr */ 3742 if (IS_DATASEG(type)) { 3743 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 3744 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 3745 CURSEG_HOT_DATA]); 3746 if (__exist_node_summaries(sbi)) 3747 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type); 3748 else 3749 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 3750 } else { 3751 segno = le32_to_cpu(ckpt->cur_node_segno[type - 3752 CURSEG_HOT_NODE]); 3753 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 3754 CURSEG_HOT_NODE]); 3755 if (__exist_node_summaries(sbi)) 3756 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 3757 type - CURSEG_HOT_NODE); 3758 else 3759 blk_addr = GET_SUM_BLOCK(sbi, segno); 3760 } 3761 3762 new = f2fs_get_meta_page(sbi, blk_addr); 3763 if (IS_ERR(new)) 3764 return PTR_ERR(new); 3765 sum = (struct f2fs_summary_block *)page_address(new); 3766 3767 if (IS_NODESEG(type)) { 3768 if (__exist_node_summaries(sbi)) { 3769 struct f2fs_summary *ns = &sum->entries[0]; 3770 int i; 3771 3772 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { 3773 ns->version = 0; 3774 ns->ofs_in_node = 0; 3775 } 3776 } else { 3777 err = f2fs_restore_node_summary(sbi, segno, sum); 3778 if (err) 3779 goto out; 3780 } 3781 } 3782 3783 /* set uncompleted segment to curseg */ 3784 curseg = CURSEG_I(sbi, type); 3785 mutex_lock(&curseg->curseg_mutex); 3786 3787 /* update journal info */ 3788 down_write(&curseg->journal_rwsem); 3789 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); 3790 up_write(&curseg->journal_rwsem); 3791 3792 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); 3793 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); 3794 curseg->next_segno = segno; 3795 reset_curseg(sbi, type, 0); 3796 curseg->alloc_type = ckpt->alloc_type[type]; 3797 curseg->next_blkoff = blk_off; 3798 mutex_unlock(&curseg->curseg_mutex); 3799 out: 3800 f2fs_put_page(new, 1); 3801 return err; 3802 } 3803 3804 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 3805 { 3806 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; 3807 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; 3808 int type = CURSEG_HOT_DATA; 3809 int err; 3810 3811 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { 3812 int npages = f2fs_npages_for_summary_flush(sbi, true); 3813 3814 if (npages >= 2) 3815 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, 3816 META_CP, true); 3817 3818 /* restore for compacted data summary */ 3819 err = read_compacted_summaries(sbi); 3820 if (err) 3821 return err; 3822 type = CURSEG_HOT_NODE; 3823 } 3824 3825 if (__exist_node_summaries(sbi)) 3826 f2fs_ra_meta_pages(sbi, 3827 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type), 3828 NR_CURSEG_PERSIST_TYPE - type, META_CP, true); 3829 3830 for (; type <= CURSEG_COLD_NODE; type++) { 3831 err = read_normal_summaries(sbi, type); 3832 if (err) 3833 return err; 3834 } 3835 3836 /* sanity check for summary blocks */ 3837 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || 3838 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) { 3839 f2fs_err(sbi, "invalid journal entries nats %u sits %u", 3840 nats_in_cursum(nat_j), sits_in_cursum(sit_j)); 3841 return -EINVAL; 3842 } 3843 3844 return 0; 3845 } 3846 3847 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 3848 { 3849 struct page *page; 3850 unsigned char *kaddr; 3851 struct f2fs_summary *summary; 3852 struct curseg_info *seg_i; 3853 int written_size = 0; 3854 int i, j; 3855 3856 page = f2fs_grab_meta_page(sbi, blkaddr++); 3857 kaddr = (unsigned char *)page_address(page); 3858 memset(kaddr, 0, PAGE_SIZE); 3859 3860 /* Step 1: write nat cache */ 3861 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3862 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); 3863 written_size += SUM_JOURNAL_SIZE; 3864 3865 /* Step 2: write sit cache */ 3866 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3867 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); 3868 written_size += SUM_JOURNAL_SIZE; 3869 3870 /* Step 3: write summary entries */ 3871 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3872 seg_i = CURSEG_I(sbi, i); 3873 for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) { 3874 if (!page) { 3875 page = f2fs_grab_meta_page(sbi, blkaddr++); 3876 kaddr = (unsigned char *)page_address(page); 3877 memset(kaddr, 0, PAGE_SIZE); 3878 written_size = 0; 3879 } 3880 summary = (struct f2fs_summary *)(kaddr + written_size); 3881 *summary = seg_i->sum_blk->entries[j]; 3882 written_size += SUMMARY_SIZE; 3883 3884 if (written_size + SUMMARY_SIZE <= PAGE_SIZE - 3885 SUM_FOOTER_SIZE) 3886 continue; 3887 3888 set_page_dirty(page); 3889 f2fs_put_page(page, 1); 3890 page = NULL; 3891 } 3892 } 3893 if (page) { 3894 set_page_dirty(page); 3895 f2fs_put_page(page, 1); 3896 } 3897 } 3898 3899 static void write_normal_summaries(struct f2fs_sb_info *sbi, 3900 block_t blkaddr, int type) 3901 { 3902 int i, end; 3903 3904 if (IS_DATASEG(type)) 3905 end = type + NR_CURSEG_DATA_TYPE; 3906 else 3907 end = type + NR_CURSEG_NODE_TYPE; 3908 3909 for (i = type; i < end; i++) 3910 write_current_sum_page(sbi, i, blkaddr + (i - type)); 3911 } 3912 3913 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 3914 { 3915 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) 3916 write_compacted_summaries(sbi, start_blk); 3917 else 3918 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 3919 } 3920 3921 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 3922 { 3923 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 3924 } 3925 3926 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, 3927 unsigned int val, int alloc) 3928 { 3929 int i; 3930 3931 if (type == NAT_JOURNAL) { 3932 for (i = 0; i < nats_in_cursum(journal); i++) { 3933 if (le32_to_cpu(nid_in_journal(journal, i)) == val) 3934 return i; 3935 } 3936 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) 3937 return update_nats_in_cursum(journal, 1); 3938 } else if (type == SIT_JOURNAL) { 3939 for (i = 0; i < sits_in_cursum(journal); i++) 3940 if (le32_to_cpu(segno_in_journal(journal, i)) == val) 3941 return i; 3942 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) 3943 return update_sits_in_cursum(journal, 1); 3944 } 3945 return -1; 3946 } 3947 3948 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 3949 unsigned int segno) 3950 { 3951 return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno)); 3952 } 3953 3954 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 3955 unsigned int start) 3956 { 3957 struct sit_info *sit_i = SIT_I(sbi); 3958 struct page *page; 3959 pgoff_t src_off, dst_off; 3960 3961 src_off = current_sit_addr(sbi, start); 3962 dst_off = next_sit_addr(sbi, src_off); 3963 3964 page = f2fs_grab_meta_page(sbi, dst_off); 3965 seg_info_to_sit_page(sbi, page, start); 3966 3967 set_page_dirty(page); 3968 set_to_next_sit(sit_i, start); 3969 3970 return page; 3971 } 3972 3973 static struct sit_entry_set *grab_sit_entry_set(void) 3974 { 3975 struct sit_entry_set *ses = 3976 f2fs_kmem_cache_alloc(sit_entry_set_slab, 3977 GFP_NOFS, true, NULL); 3978 3979 ses->entry_cnt = 0; 3980 INIT_LIST_HEAD(&ses->set_list); 3981 return ses; 3982 } 3983 3984 static void release_sit_entry_set(struct sit_entry_set *ses) 3985 { 3986 list_del(&ses->set_list); 3987 kmem_cache_free(sit_entry_set_slab, ses); 3988 } 3989 3990 static void adjust_sit_entry_set(struct sit_entry_set *ses, 3991 struct list_head *head) 3992 { 3993 struct sit_entry_set *next = ses; 3994 3995 if (list_is_last(&ses->set_list, head)) 3996 return; 3997 3998 list_for_each_entry_continue(next, head, set_list) 3999 if (ses->entry_cnt <= next->entry_cnt) { 4000 list_move_tail(&ses->set_list, &next->set_list); 4001 return; 4002 } 4003 4004 list_move_tail(&ses->set_list, head); 4005 } 4006 4007 static void add_sit_entry(unsigned int segno, struct list_head *head) 4008 { 4009 struct sit_entry_set *ses; 4010 unsigned int start_segno = START_SEGNO(segno); 4011 4012 list_for_each_entry(ses, head, set_list) { 4013 if (ses->start_segno == start_segno) { 4014 ses->entry_cnt++; 4015 adjust_sit_entry_set(ses, head); 4016 return; 4017 } 4018 } 4019 4020 ses = grab_sit_entry_set(); 4021 4022 ses->start_segno = start_segno; 4023 ses->entry_cnt++; 4024 list_add(&ses->set_list, head); 4025 } 4026 4027 static void add_sits_in_set(struct f2fs_sb_info *sbi) 4028 { 4029 struct f2fs_sm_info *sm_info = SM_I(sbi); 4030 struct list_head *set_list = &sm_info->sit_entry_set; 4031 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; 4032 unsigned int segno; 4033 4034 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) 4035 add_sit_entry(segno, set_list); 4036 } 4037 4038 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) 4039 { 4040 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4041 struct f2fs_journal *journal = curseg->journal; 4042 int i; 4043 4044 down_write(&curseg->journal_rwsem); 4045 for (i = 0; i < sits_in_cursum(journal); i++) { 4046 unsigned int segno; 4047 bool dirtied; 4048 4049 segno = le32_to_cpu(segno_in_journal(journal, i)); 4050 dirtied = __mark_sit_entry_dirty(sbi, segno); 4051 4052 if (!dirtied) 4053 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); 4054 } 4055 update_sits_in_cursum(journal, -i); 4056 up_write(&curseg->journal_rwsem); 4057 } 4058 4059 /* 4060 * CP calls this function, which flushes SIT entries including sit_journal, 4061 * and moves prefree segs to free segs. 4062 */ 4063 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) 4064 { 4065 struct sit_info *sit_i = SIT_I(sbi); 4066 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 4067 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4068 struct f2fs_journal *journal = curseg->journal; 4069 struct sit_entry_set *ses, *tmp; 4070 struct list_head *head = &SM_I(sbi)->sit_entry_set; 4071 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS); 4072 struct seg_entry *se; 4073 4074 down_write(&sit_i->sentry_lock); 4075 4076 if (!sit_i->dirty_sentries) 4077 goto out; 4078 4079 /* 4080 * add and account sit entries of dirty bitmap in sit entry 4081 * set temporarily 4082 */ 4083 add_sits_in_set(sbi); 4084 4085 /* 4086 * if there are no enough space in journal to store dirty sit 4087 * entries, remove all entries from journal and add and account 4088 * them in sit entry set. 4089 */ 4090 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) || 4091 !to_journal) 4092 remove_sits_in_journal(sbi); 4093 4094 /* 4095 * there are two steps to flush sit entries: 4096 * #1, flush sit entries to journal in current cold data summary block. 4097 * #2, flush sit entries to sit page. 4098 */ 4099 list_for_each_entry_safe(ses, tmp, head, set_list) { 4100 struct page *page = NULL; 4101 struct f2fs_sit_block *raw_sit = NULL; 4102 unsigned int start_segno = ses->start_segno; 4103 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, 4104 (unsigned long)MAIN_SEGS(sbi)); 4105 unsigned int segno = start_segno; 4106 4107 if (to_journal && 4108 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) 4109 to_journal = false; 4110 4111 if (to_journal) { 4112 down_write(&curseg->journal_rwsem); 4113 } else { 4114 page = get_next_sit_page(sbi, start_segno); 4115 raw_sit = page_address(page); 4116 } 4117 4118 /* flush dirty sit entries in region of current sit set */ 4119 for_each_set_bit_from(segno, bitmap, end) { 4120 int offset, sit_offset; 4121 4122 se = get_seg_entry(sbi, segno); 4123 #ifdef CONFIG_F2FS_CHECK_FS 4124 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir, 4125 SIT_VBLOCK_MAP_SIZE)) 4126 f2fs_bug_on(sbi, 1); 4127 #endif 4128 4129 /* add discard candidates */ 4130 if (!(cpc->reason & CP_DISCARD)) { 4131 cpc->trim_start = segno; 4132 add_discard_addrs(sbi, cpc, false); 4133 } 4134 4135 if (to_journal) { 4136 offset = f2fs_lookup_journal_in_cursum(journal, 4137 SIT_JOURNAL, segno, 1); 4138 f2fs_bug_on(sbi, offset < 0); 4139 segno_in_journal(journal, offset) = 4140 cpu_to_le32(segno); 4141 seg_info_to_raw_sit(se, 4142 &sit_in_journal(journal, offset)); 4143 check_block_count(sbi, segno, 4144 &sit_in_journal(journal, offset)); 4145 } else { 4146 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 4147 seg_info_to_raw_sit(se, 4148 &raw_sit->entries[sit_offset]); 4149 check_block_count(sbi, segno, 4150 &raw_sit->entries[sit_offset]); 4151 } 4152 4153 __clear_bit(segno, bitmap); 4154 sit_i->dirty_sentries--; 4155 ses->entry_cnt--; 4156 } 4157 4158 if (to_journal) 4159 up_write(&curseg->journal_rwsem); 4160 else 4161 f2fs_put_page(page, 1); 4162 4163 f2fs_bug_on(sbi, ses->entry_cnt); 4164 release_sit_entry_set(ses); 4165 } 4166 4167 f2fs_bug_on(sbi, !list_empty(head)); 4168 f2fs_bug_on(sbi, sit_i->dirty_sentries); 4169 out: 4170 if (cpc->reason & CP_DISCARD) { 4171 __u64 trim_start = cpc->trim_start; 4172 4173 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) 4174 add_discard_addrs(sbi, cpc, false); 4175 4176 cpc->trim_start = trim_start; 4177 } 4178 up_write(&sit_i->sentry_lock); 4179 4180 set_prefree_as_free_segments(sbi); 4181 } 4182 4183 static int build_sit_info(struct f2fs_sb_info *sbi) 4184 { 4185 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 4186 struct sit_info *sit_i; 4187 unsigned int sit_segs, start; 4188 char *src_bitmap, *bitmap; 4189 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size; 4190 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0; 4191 4192 /* allocate memory for SIT information */ 4193 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); 4194 if (!sit_i) 4195 return -ENOMEM; 4196 4197 SM_I(sbi)->sit_info = sit_i; 4198 4199 sit_i->sentries = 4200 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), 4201 MAIN_SEGS(sbi)), 4202 GFP_KERNEL); 4203 if (!sit_i->sentries) 4204 return -ENOMEM; 4205 4206 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4207 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size, 4208 GFP_KERNEL); 4209 if (!sit_i->dirty_sentries_bitmap) 4210 return -ENOMEM; 4211 4212 #ifdef CONFIG_F2FS_CHECK_FS 4213 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map); 4214 #else 4215 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map); 4216 #endif 4217 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 4218 if (!sit_i->bitmap) 4219 return -ENOMEM; 4220 4221 bitmap = sit_i->bitmap; 4222 4223 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4224 sit_i->sentries[start].cur_valid_map = bitmap; 4225 bitmap += SIT_VBLOCK_MAP_SIZE; 4226 4227 sit_i->sentries[start].ckpt_valid_map = bitmap; 4228 bitmap += SIT_VBLOCK_MAP_SIZE; 4229 4230 #ifdef CONFIG_F2FS_CHECK_FS 4231 sit_i->sentries[start].cur_valid_map_mir = bitmap; 4232 bitmap += SIT_VBLOCK_MAP_SIZE; 4233 #endif 4234 4235 if (discard_map) { 4236 sit_i->sentries[start].discard_map = bitmap; 4237 bitmap += SIT_VBLOCK_MAP_SIZE; 4238 } 4239 } 4240 4241 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 4242 if (!sit_i->tmp_map) 4243 return -ENOMEM; 4244 4245 if (__is_large_section(sbi)) { 4246 sit_i->sec_entries = 4247 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), 4248 MAIN_SECS(sbi)), 4249 GFP_KERNEL); 4250 if (!sit_i->sec_entries) 4251 return -ENOMEM; 4252 } 4253 4254 /* get information related with SIT */ 4255 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 4256 4257 /* setup SIT bitmap from ckeckpoint pack */ 4258 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 4259 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 4260 4261 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL); 4262 if (!sit_i->sit_bitmap) 4263 return -ENOMEM; 4264 4265 #ifdef CONFIG_F2FS_CHECK_FS 4266 sit_i->sit_bitmap_mir = kmemdup(src_bitmap, 4267 sit_bitmap_size, GFP_KERNEL); 4268 if (!sit_i->sit_bitmap_mir) 4269 return -ENOMEM; 4270 4271 sit_i->invalid_segmap = f2fs_kvzalloc(sbi, 4272 main_bitmap_size, GFP_KERNEL); 4273 if (!sit_i->invalid_segmap) 4274 return -ENOMEM; 4275 #endif 4276 4277 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 4278 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; 4279 sit_i->written_valid_blocks = 0; 4280 sit_i->bitmap_size = sit_bitmap_size; 4281 sit_i->dirty_sentries = 0; 4282 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 4283 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 4284 sit_i->mounted_time = ktime_get_boottime_seconds(); 4285 init_rwsem(&sit_i->sentry_lock); 4286 return 0; 4287 } 4288 4289 static int build_free_segmap(struct f2fs_sb_info *sbi) 4290 { 4291 struct free_segmap_info *free_i; 4292 unsigned int bitmap_size, sec_bitmap_size; 4293 4294 /* allocate memory for free segmap information */ 4295 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); 4296 if (!free_i) 4297 return -ENOMEM; 4298 4299 SM_I(sbi)->free_info = free_i; 4300 4301 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4302 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); 4303 if (!free_i->free_segmap) 4304 return -ENOMEM; 4305 4306 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 4307 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); 4308 if (!free_i->free_secmap) 4309 return -ENOMEM; 4310 4311 /* set all segments as dirty temporarily */ 4312 memset(free_i->free_segmap, 0xff, bitmap_size); 4313 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 4314 4315 /* init free segmap information */ 4316 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); 4317 free_i->free_segments = 0; 4318 free_i->free_sections = 0; 4319 spin_lock_init(&free_i->segmap_lock); 4320 return 0; 4321 } 4322 4323 static int build_curseg(struct f2fs_sb_info *sbi) 4324 { 4325 struct curseg_info *array; 4326 int i; 4327 4328 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, 4329 sizeof(*array)), GFP_KERNEL); 4330 if (!array) 4331 return -ENOMEM; 4332 4333 SM_I(sbi)->curseg_array = array; 4334 4335 for (i = 0; i < NO_CHECK_TYPE; i++) { 4336 mutex_init(&array[i].curseg_mutex); 4337 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); 4338 if (!array[i].sum_blk) 4339 return -ENOMEM; 4340 init_rwsem(&array[i].journal_rwsem); 4341 array[i].journal = f2fs_kzalloc(sbi, 4342 sizeof(struct f2fs_journal), GFP_KERNEL); 4343 if (!array[i].journal) 4344 return -ENOMEM; 4345 if (i < NR_PERSISTENT_LOG) 4346 array[i].seg_type = CURSEG_HOT_DATA + i; 4347 else if (i == CURSEG_COLD_DATA_PINNED) 4348 array[i].seg_type = CURSEG_COLD_DATA; 4349 else if (i == CURSEG_ALL_DATA_ATGC) 4350 array[i].seg_type = CURSEG_COLD_DATA; 4351 array[i].segno = NULL_SEGNO; 4352 array[i].next_blkoff = 0; 4353 array[i].inited = false; 4354 } 4355 return restore_curseg_summaries(sbi); 4356 } 4357 4358 static int build_sit_entries(struct f2fs_sb_info *sbi) 4359 { 4360 struct sit_info *sit_i = SIT_I(sbi); 4361 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4362 struct f2fs_journal *journal = curseg->journal; 4363 struct seg_entry *se; 4364 struct f2fs_sit_entry sit; 4365 int sit_blk_cnt = SIT_BLK_CNT(sbi); 4366 unsigned int i, start, end; 4367 unsigned int readed, start_blk = 0; 4368 int err = 0; 4369 block_t sit_valid_blocks[2] = {0, 0}; 4370 4371 do { 4372 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS, 4373 META_SIT, true); 4374 4375 start = start_blk * sit_i->sents_per_block; 4376 end = (start_blk + readed) * sit_i->sents_per_block; 4377 4378 for (; start < end && start < MAIN_SEGS(sbi); start++) { 4379 struct f2fs_sit_block *sit_blk; 4380 struct page *page; 4381 4382 se = &sit_i->sentries[start]; 4383 page = get_current_sit_page(sbi, start); 4384 if (IS_ERR(page)) 4385 return PTR_ERR(page); 4386 sit_blk = (struct f2fs_sit_block *)page_address(page); 4387 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 4388 f2fs_put_page(page, 1); 4389 4390 err = check_block_count(sbi, start, &sit); 4391 if (err) 4392 return err; 4393 seg_info_from_raw_sit(se, &sit); 4394 4395 if (se->type >= NR_PERSISTENT_LOG) { 4396 f2fs_err(sbi, "Invalid segment type: %u, segno: %u", 4397 se->type, start); 4398 f2fs_handle_error(sbi, 4399 ERROR_INCONSISTENT_SUM_TYPE); 4400 return -EFSCORRUPTED; 4401 } 4402 4403 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; 4404 4405 if (f2fs_block_unit_discard(sbi)) { 4406 /* build discard map only one time */ 4407 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 4408 memset(se->discard_map, 0xff, 4409 SIT_VBLOCK_MAP_SIZE); 4410 } else { 4411 memcpy(se->discard_map, 4412 se->cur_valid_map, 4413 SIT_VBLOCK_MAP_SIZE); 4414 sbi->discard_blks += 4415 sbi->blocks_per_seg - 4416 se->valid_blocks; 4417 } 4418 } 4419 4420 if (__is_large_section(sbi)) 4421 get_sec_entry(sbi, start)->valid_blocks += 4422 se->valid_blocks; 4423 } 4424 start_blk += readed; 4425 } while (start_blk < sit_blk_cnt); 4426 4427 down_read(&curseg->journal_rwsem); 4428 for (i = 0; i < sits_in_cursum(journal); i++) { 4429 unsigned int old_valid_blocks; 4430 4431 start = le32_to_cpu(segno_in_journal(journal, i)); 4432 if (start >= MAIN_SEGS(sbi)) { 4433 f2fs_err(sbi, "Wrong journal entry on segno %u", 4434 start); 4435 err = -EFSCORRUPTED; 4436 f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL); 4437 break; 4438 } 4439 4440 se = &sit_i->sentries[start]; 4441 sit = sit_in_journal(journal, i); 4442 4443 old_valid_blocks = se->valid_blocks; 4444 4445 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks; 4446 4447 err = check_block_count(sbi, start, &sit); 4448 if (err) 4449 break; 4450 seg_info_from_raw_sit(se, &sit); 4451 4452 if (se->type >= NR_PERSISTENT_LOG) { 4453 f2fs_err(sbi, "Invalid segment type: %u, segno: %u", 4454 se->type, start); 4455 err = -EFSCORRUPTED; 4456 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); 4457 break; 4458 } 4459 4460 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; 4461 4462 if (f2fs_block_unit_discard(sbi)) { 4463 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 4464 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE); 4465 } else { 4466 memcpy(se->discard_map, se->cur_valid_map, 4467 SIT_VBLOCK_MAP_SIZE); 4468 sbi->discard_blks += old_valid_blocks; 4469 sbi->discard_blks -= se->valid_blocks; 4470 } 4471 } 4472 4473 if (__is_large_section(sbi)) { 4474 get_sec_entry(sbi, start)->valid_blocks += 4475 se->valid_blocks; 4476 get_sec_entry(sbi, start)->valid_blocks -= 4477 old_valid_blocks; 4478 } 4479 } 4480 up_read(&curseg->journal_rwsem); 4481 4482 if (err) 4483 return err; 4484 4485 if (sit_valid_blocks[NODE] != valid_node_count(sbi)) { 4486 f2fs_err(sbi, "SIT is corrupted node# %u vs %u", 4487 sit_valid_blocks[NODE], valid_node_count(sbi)); 4488 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT); 4489 return -EFSCORRUPTED; 4490 } 4491 4492 if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] > 4493 valid_user_blocks(sbi)) { 4494 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u", 4495 sit_valid_blocks[DATA], sit_valid_blocks[NODE], 4496 valid_user_blocks(sbi)); 4497 f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT); 4498 return -EFSCORRUPTED; 4499 } 4500 4501 return 0; 4502 } 4503 4504 static void init_free_segmap(struct f2fs_sb_info *sbi) 4505 { 4506 unsigned int start; 4507 int type; 4508 struct seg_entry *sentry; 4509 4510 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4511 if (f2fs_usable_blks_in_seg(sbi, start) == 0) 4512 continue; 4513 sentry = get_seg_entry(sbi, start); 4514 if (!sentry->valid_blocks) 4515 __set_free(sbi, start); 4516 else 4517 SIT_I(sbi)->written_valid_blocks += 4518 sentry->valid_blocks; 4519 } 4520 4521 /* set use the current segments */ 4522 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 4523 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 4524 4525 __set_test_and_inuse(sbi, curseg_t->segno); 4526 } 4527 } 4528 4529 static void init_dirty_segmap(struct f2fs_sb_info *sbi) 4530 { 4531 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4532 struct free_segmap_info *free_i = FREE_I(sbi); 4533 unsigned int segno = 0, offset = 0, secno; 4534 block_t valid_blocks, usable_blks_in_seg; 4535 4536 while (1) { 4537 /* find dirty segment based on free segmap */ 4538 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); 4539 if (segno >= MAIN_SEGS(sbi)) 4540 break; 4541 offset = segno + 1; 4542 valid_blocks = get_valid_blocks(sbi, segno, false); 4543 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); 4544 if (valid_blocks == usable_blks_in_seg || !valid_blocks) 4545 continue; 4546 if (valid_blocks > usable_blks_in_seg) { 4547 f2fs_bug_on(sbi, 1); 4548 continue; 4549 } 4550 mutex_lock(&dirty_i->seglist_lock); 4551 __locate_dirty_segment(sbi, segno, DIRTY); 4552 mutex_unlock(&dirty_i->seglist_lock); 4553 } 4554 4555 if (!__is_large_section(sbi)) 4556 return; 4557 4558 mutex_lock(&dirty_i->seglist_lock); 4559 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { 4560 valid_blocks = get_valid_blocks(sbi, segno, true); 4561 secno = GET_SEC_FROM_SEG(sbi, segno); 4562 4563 if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi)) 4564 continue; 4565 if (IS_CURSEC(sbi, secno)) 4566 continue; 4567 set_bit(secno, dirty_i->dirty_secmap); 4568 } 4569 mutex_unlock(&dirty_i->seglist_lock); 4570 } 4571 4572 static int init_victim_secmap(struct f2fs_sb_info *sbi) 4573 { 4574 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4575 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 4576 4577 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 4578 if (!dirty_i->victim_secmap) 4579 return -ENOMEM; 4580 4581 dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 4582 if (!dirty_i->pinned_secmap) 4583 return -ENOMEM; 4584 4585 dirty_i->pinned_secmap_cnt = 0; 4586 dirty_i->enable_pin_section = true; 4587 return 0; 4588 } 4589 4590 static int build_dirty_segmap(struct f2fs_sb_info *sbi) 4591 { 4592 struct dirty_seglist_info *dirty_i; 4593 unsigned int bitmap_size, i; 4594 4595 /* allocate memory for dirty segments list information */ 4596 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), 4597 GFP_KERNEL); 4598 if (!dirty_i) 4599 return -ENOMEM; 4600 4601 SM_I(sbi)->dirty_info = dirty_i; 4602 mutex_init(&dirty_i->seglist_lock); 4603 4604 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4605 4606 for (i = 0; i < NR_DIRTY_TYPE; i++) { 4607 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, 4608 GFP_KERNEL); 4609 if (!dirty_i->dirty_segmap[i]) 4610 return -ENOMEM; 4611 } 4612 4613 if (__is_large_section(sbi)) { 4614 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 4615 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi, 4616 bitmap_size, GFP_KERNEL); 4617 if (!dirty_i->dirty_secmap) 4618 return -ENOMEM; 4619 } 4620 4621 init_dirty_segmap(sbi); 4622 return init_victim_secmap(sbi); 4623 } 4624 4625 static int sanity_check_curseg(struct f2fs_sb_info *sbi) 4626 { 4627 int i; 4628 4629 /* 4630 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr; 4631 * In LFS curseg, all blkaddr after .next_blkoff should be unused. 4632 */ 4633 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 4634 struct curseg_info *curseg = CURSEG_I(sbi, i); 4635 struct seg_entry *se = get_seg_entry(sbi, curseg->segno); 4636 unsigned int blkofs = curseg->next_blkoff; 4637 4638 if (f2fs_sb_has_readonly(sbi) && 4639 i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE) 4640 continue; 4641 4642 sanity_check_seg_type(sbi, curseg->seg_type); 4643 4644 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) { 4645 f2fs_err(sbi, 4646 "Current segment has invalid alloc_type:%d", 4647 curseg->alloc_type); 4648 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); 4649 return -EFSCORRUPTED; 4650 } 4651 4652 if (f2fs_test_bit(blkofs, se->cur_valid_map)) 4653 goto out; 4654 4655 if (curseg->alloc_type == SSR) 4656 continue; 4657 4658 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) { 4659 if (!f2fs_test_bit(blkofs, se->cur_valid_map)) 4660 continue; 4661 out: 4662 f2fs_err(sbi, 4663 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u", 4664 i, curseg->segno, curseg->alloc_type, 4665 curseg->next_blkoff, blkofs); 4666 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); 4667 return -EFSCORRUPTED; 4668 } 4669 } 4670 return 0; 4671 } 4672 4673 #ifdef CONFIG_BLK_DEV_ZONED 4674 4675 static int check_zone_write_pointer(struct f2fs_sb_info *sbi, 4676 struct f2fs_dev_info *fdev, 4677 struct blk_zone *zone) 4678 { 4679 unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno; 4680 block_t zone_block, wp_block, last_valid_block; 4681 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 4682 int i, s, b, ret; 4683 struct seg_entry *se; 4684 4685 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ) 4686 return 0; 4687 4688 wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block); 4689 wp_segno = GET_SEGNO(sbi, wp_block); 4690 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); 4691 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block); 4692 zone_segno = GET_SEGNO(sbi, zone_block); 4693 zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno); 4694 4695 if (zone_segno >= MAIN_SEGS(sbi)) 4696 return 0; 4697 4698 /* 4699 * Skip check of zones cursegs point to, since 4700 * fix_curseg_write_pointer() checks them. 4701 */ 4702 for (i = 0; i < NO_CHECK_TYPE; i++) 4703 if (zone_secno == GET_SEC_FROM_SEG(sbi, 4704 CURSEG_I(sbi, i)->segno)) 4705 return 0; 4706 4707 /* 4708 * Get last valid block of the zone. 4709 */ 4710 last_valid_block = zone_block - 1; 4711 for (s = sbi->segs_per_sec - 1; s >= 0; s--) { 4712 segno = zone_segno + s; 4713 se = get_seg_entry(sbi, segno); 4714 for (b = sbi->blocks_per_seg - 1; b >= 0; b--) 4715 if (f2fs_test_bit(b, se->cur_valid_map)) { 4716 last_valid_block = START_BLOCK(sbi, segno) + b; 4717 break; 4718 } 4719 if (last_valid_block >= zone_block) 4720 break; 4721 } 4722 4723 /* 4724 * If last valid block is beyond the write pointer, report the 4725 * inconsistency. This inconsistency does not cause write error 4726 * because the zone will not be selected for write operation until 4727 * it get discarded. Just report it. 4728 */ 4729 if (last_valid_block >= wp_block) { 4730 f2fs_notice(sbi, "Valid block beyond write pointer: " 4731 "valid block[0x%x,0x%x] wp[0x%x,0x%x]", 4732 GET_SEGNO(sbi, last_valid_block), 4733 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block), 4734 wp_segno, wp_blkoff); 4735 return 0; 4736 } 4737 4738 /* 4739 * If there is no valid block in the zone and if write pointer is 4740 * not at zone start, reset the write pointer. 4741 */ 4742 if (last_valid_block + 1 == zone_block && zone->wp != zone->start) { 4743 f2fs_notice(sbi, 4744 "Zone without valid block has non-zero write " 4745 "pointer. Reset the write pointer: wp[0x%x,0x%x]", 4746 wp_segno, wp_blkoff); 4747 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block, 4748 zone->len >> log_sectors_per_block); 4749 if (ret) { 4750 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 4751 fdev->path, ret); 4752 return ret; 4753 } 4754 } 4755 4756 return 0; 4757 } 4758 4759 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi, 4760 block_t zone_blkaddr) 4761 { 4762 int i; 4763 4764 for (i = 0; i < sbi->s_ndevs; i++) { 4765 if (!bdev_is_zoned(FDEV(i).bdev)) 4766 continue; 4767 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr && 4768 zone_blkaddr <= FDEV(i).end_blk)) 4769 return &FDEV(i); 4770 } 4771 4772 return NULL; 4773 } 4774 4775 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx, 4776 void *data) 4777 { 4778 memcpy(data, zone, sizeof(struct blk_zone)); 4779 return 0; 4780 } 4781 4782 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) 4783 { 4784 struct curseg_info *cs = CURSEG_I(sbi, type); 4785 struct f2fs_dev_info *zbd; 4786 struct blk_zone zone; 4787 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off; 4788 block_t cs_zone_block, wp_block; 4789 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 4790 sector_t zone_sector; 4791 int err; 4792 4793 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 4794 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 4795 4796 zbd = get_target_zoned_dev(sbi, cs_zone_block); 4797 if (!zbd) 4798 return 0; 4799 4800 /* report zone for the sector the curseg points to */ 4801 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 4802 << log_sectors_per_block; 4803 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 4804 report_one_zone_cb, &zone); 4805 if (err != 1) { 4806 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 4807 zbd->path, err); 4808 return err; 4809 } 4810 4811 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 4812 return 0; 4813 4814 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block); 4815 wp_segno = GET_SEGNO(sbi, wp_block); 4816 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); 4817 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0); 4818 4819 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff && 4820 wp_sector_off == 0) 4821 return 0; 4822 4823 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: " 4824 "curseg[0x%x,0x%x] wp[0x%x,0x%x]", 4825 type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff); 4826 4827 f2fs_notice(sbi, "Assign new section to curseg[%d]: " 4828 "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff); 4829 4830 f2fs_allocate_new_section(sbi, type, true); 4831 4832 /* check consistency of the zone curseg pointed to */ 4833 if (check_zone_write_pointer(sbi, zbd, &zone)) 4834 return -EIO; 4835 4836 /* check newly assigned zone */ 4837 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 4838 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 4839 4840 zbd = get_target_zoned_dev(sbi, cs_zone_block); 4841 if (!zbd) 4842 return 0; 4843 4844 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 4845 << log_sectors_per_block; 4846 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 4847 report_one_zone_cb, &zone); 4848 if (err != 1) { 4849 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 4850 zbd->path, err); 4851 return err; 4852 } 4853 4854 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 4855 return 0; 4856 4857 if (zone.wp != zone.start) { 4858 f2fs_notice(sbi, 4859 "New zone for curseg[%d] is not yet discarded. " 4860 "Reset the zone: curseg[0x%x,0x%x]", 4861 type, cs->segno, cs->next_blkoff); 4862 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, 4863 zone_sector >> log_sectors_per_block, 4864 zone.len >> log_sectors_per_block); 4865 if (err) { 4866 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 4867 zbd->path, err); 4868 return err; 4869 } 4870 } 4871 4872 return 0; 4873 } 4874 4875 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) 4876 { 4877 int i, ret; 4878 4879 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 4880 ret = fix_curseg_write_pointer(sbi, i); 4881 if (ret) 4882 return ret; 4883 } 4884 4885 return 0; 4886 } 4887 4888 struct check_zone_write_pointer_args { 4889 struct f2fs_sb_info *sbi; 4890 struct f2fs_dev_info *fdev; 4891 }; 4892 4893 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx, 4894 void *data) 4895 { 4896 struct check_zone_write_pointer_args *args; 4897 4898 args = (struct check_zone_write_pointer_args *)data; 4899 4900 return check_zone_write_pointer(args->sbi, args->fdev, zone); 4901 } 4902 4903 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) 4904 { 4905 int i, ret; 4906 struct check_zone_write_pointer_args args; 4907 4908 for (i = 0; i < sbi->s_ndevs; i++) { 4909 if (!bdev_is_zoned(FDEV(i).bdev)) 4910 continue; 4911 4912 args.sbi = sbi; 4913 args.fdev = &FDEV(i); 4914 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES, 4915 check_zone_write_pointer_cb, &args); 4916 if (ret < 0) 4917 return ret; 4918 } 4919 4920 return 0; 4921 } 4922 4923 static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx, 4924 unsigned int dev_idx) 4925 { 4926 if (!bdev_is_zoned(FDEV(dev_idx).bdev)) 4927 return true; 4928 return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq); 4929 } 4930 4931 /* Return the zone index in the given device */ 4932 static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno, 4933 int dev_idx) 4934 { 4935 block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); 4936 4937 return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >> 4938 sbi->log_blocks_per_blkz; 4939 } 4940 4941 /* 4942 * Return the usable segments in a section based on the zone's 4943 * corresponding zone capacity. Zone is equal to a section. 4944 */ 4945 static inline unsigned int f2fs_usable_zone_segs_in_sec( 4946 struct f2fs_sb_info *sbi, unsigned int segno) 4947 { 4948 unsigned int dev_idx, zone_idx; 4949 4950 dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 4951 zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx); 4952 4953 /* Conventional zone's capacity is always equal to zone size */ 4954 if (is_conv_zone(sbi, zone_idx, dev_idx)) 4955 return sbi->segs_per_sec; 4956 4957 if (!sbi->unusable_blocks_per_sec) 4958 return sbi->segs_per_sec; 4959 4960 /* Get the segment count beyond zone capacity block */ 4961 return sbi->segs_per_sec - (sbi->unusable_blocks_per_sec >> 4962 sbi->log_blocks_per_seg); 4963 } 4964 4965 /* 4966 * Return the number of usable blocks in a segment. The number of blocks 4967 * returned is always equal to the number of blocks in a segment for 4968 * segments fully contained within a sequential zone capacity or a 4969 * conventional zone. For segments partially contained in a sequential 4970 * zone capacity, the number of usable blocks up to the zone capacity 4971 * is returned. 0 is returned in all other cases. 4972 */ 4973 static inline unsigned int f2fs_usable_zone_blks_in_seg( 4974 struct f2fs_sb_info *sbi, unsigned int segno) 4975 { 4976 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr; 4977 unsigned int zone_idx, dev_idx, secno; 4978 4979 secno = GET_SEC_FROM_SEG(sbi, segno); 4980 seg_start = START_BLOCK(sbi, segno); 4981 dev_idx = f2fs_target_device_index(sbi, seg_start); 4982 zone_idx = get_zone_idx(sbi, secno, dev_idx); 4983 4984 /* 4985 * Conventional zone's capacity is always equal to zone size, 4986 * so, blocks per segment is unchanged. 4987 */ 4988 if (is_conv_zone(sbi, zone_idx, dev_idx)) 4989 return sbi->blocks_per_seg; 4990 4991 if (!sbi->unusable_blocks_per_sec) 4992 return sbi->blocks_per_seg; 4993 4994 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); 4995 sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi); 4996 4997 /* 4998 * If segment starts before zone capacity and spans beyond 4999 * zone capacity, then usable blocks are from seg start to 5000 * zone capacity. If the segment starts after the zone capacity, 5001 * then there are no usable blocks. 5002 */ 5003 if (seg_start >= sec_cap_blkaddr) 5004 return 0; 5005 if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr) 5006 return sec_cap_blkaddr - seg_start; 5007 5008 return sbi->blocks_per_seg; 5009 } 5010 #else 5011 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) 5012 { 5013 return 0; 5014 } 5015 5016 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) 5017 { 5018 return 0; 5019 } 5020 5021 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi, 5022 unsigned int segno) 5023 { 5024 return 0; 5025 } 5026 5027 static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi, 5028 unsigned int segno) 5029 { 5030 return 0; 5031 } 5032 #endif 5033 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, 5034 unsigned int segno) 5035 { 5036 if (f2fs_sb_has_blkzoned(sbi)) 5037 return f2fs_usable_zone_blks_in_seg(sbi, segno); 5038 5039 return sbi->blocks_per_seg; 5040 } 5041 5042 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi, 5043 unsigned int segno) 5044 { 5045 if (f2fs_sb_has_blkzoned(sbi)) 5046 return f2fs_usable_zone_segs_in_sec(sbi, segno); 5047 5048 return sbi->segs_per_sec; 5049 } 5050 5051 /* 5052 * Update min, max modified time for cost-benefit GC algorithm 5053 */ 5054 static void init_min_max_mtime(struct f2fs_sb_info *sbi) 5055 { 5056 struct sit_info *sit_i = SIT_I(sbi); 5057 unsigned int segno; 5058 5059 down_write(&sit_i->sentry_lock); 5060 5061 sit_i->min_mtime = ULLONG_MAX; 5062 5063 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { 5064 unsigned int i; 5065 unsigned long long mtime = 0; 5066 5067 for (i = 0; i < sbi->segs_per_sec; i++) 5068 mtime += get_seg_entry(sbi, segno + i)->mtime; 5069 5070 mtime = div_u64(mtime, sbi->segs_per_sec); 5071 5072 if (sit_i->min_mtime > mtime) 5073 sit_i->min_mtime = mtime; 5074 } 5075 sit_i->max_mtime = get_mtime(sbi, false); 5076 sit_i->dirty_max_mtime = 0; 5077 up_write(&sit_i->sentry_lock); 5078 } 5079 5080 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) 5081 { 5082 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 5083 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 5084 struct f2fs_sm_info *sm_info; 5085 int err; 5086 5087 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); 5088 if (!sm_info) 5089 return -ENOMEM; 5090 5091 /* init sm info */ 5092 sbi->sm_info = sm_info; 5093 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 5094 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 5095 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 5096 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 5097 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 5098 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 5099 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 5100 sm_info->rec_prefree_segments = sm_info->main_segments * 5101 DEF_RECLAIM_PREFREE_SEGMENTS / 100; 5102 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS) 5103 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; 5104 5105 if (!f2fs_lfs_mode(sbi)) 5106 sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC); 5107 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 5108 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; 5109 sm_info->min_seq_blocks = sbi->blocks_per_seg; 5110 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS; 5111 sm_info->min_ssr_sections = reserved_sections(sbi); 5112 5113 INIT_LIST_HEAD(&sm_info->sit_entry_set); 5114 5115 init_f2fs_rwsem(&sm_info->curseg_lock); 5116 5117 err = f2fs_create_flush_cmd_control(sbi); 5118 if (err) 5119 return err; 5120 5121 err = create_discard_cmd_control(sbi); 5122 if (err) 5123 return err; 5124 5125 err = build_sit_info(sbi); 5126 if (err) 5127 return err; 5128 err = build_free_segmap(sbi); 5129 if (err) 5130 return err; 5131 err = build_curseg(sbi); 5132 if (err) 5133 return err; 5134 5135 /* reinit free segmap based on SIT */ 5136 err = build_sit_entries(sbi); 5137 if (err) 5138 return err; 5139 5140 init_free_segmap(sbi); 5141 err = build_dirty_segmap(sbi); 5142 if (err) 5143 return err; 5144 5145 err = sanity_check_curseg(sbi); 5146 if (err) 5147 return err; 5148 5149 init_min_max_mtime(sbi); 5150 return 0; 5151 } 5152 5153 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 5154 enum dirty_type dirty_type) 5155 { 5156 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5157 5158 mutex_lock(&dirty_i->seglist_lock); 5159 kvfree(dirty_i->dirty_segmap[dirty_type]); 5160 dirty_i->nr_dirty[dirty_type] = 0; 5161 mutex_unlock(&dirty_i->seglist_lock); 5162 } 5163 5164 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 5165 { 5166 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5167 5168 kvfree(dirty_i->pinned_secmap); 5169 kvfree(dirty_i->victim_secmap); 5170 } 5171 5172 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 5173 { 5174 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5175 int i; 5176 5177 if (!dirty_i) 5178 return; 5179 5180 /* discard pre-free/dirty segments list */ 5181 for (i = 0; i < NR_DIRTY_TYPE; i++) 5182 discard_dirty_segmap(sbi, i); 5183 5184 if (__is_large_section(sbi)) { 5185 mutex_lock(&dirty_i->seglist_lock); 5186 kvfree(dirty_i->dirty_secmap); 5187 mutex_unlock(&dirty_i->seglist_lock); 5188 } 5189 5190 destroy_victim_secmap(sbi); 5191 SM_I(sbi)->dirty_info = NULL; 5192 kfree(dirty_i); 5193 } 5194 5195 static void destroy_curseg(struct f2fs_sb_info *sbi) 5196 { 5197 struct curseg_info *array = SM_I(sbi)->curseg_array; 5198 int i; 5199 5200 if (!array) 5201 return; 5202 SM_I(sbi)->curseg_array = NULL; 5203 for (i = 0; i < NR_CURSEG_TYPE; i++) { 5204 kfree(array[i].sum_blk); 5205 kfree(array[i].journal); 5206 } 5207 kfree(array); 5208 } 5209 5210 static void destroy_free_segmap(struct f2fs_sb_info *sbi) 5211 { 5212 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 5213 5214 if (!free_i) 5215 return; 5216 SM_I(sbi)->free_info = NULL; 5217 kvfree(free_i->free_segmap); 5218 kvfree(free_i->free_secmap); 5219 kfree(free_i); 5220 } 5221 5222 static void destroy_sit_info(struct f2fs_sb_info *sbi) 5223 { 5224 struct sit_info *sit_i = SIT_I(sbi); 5225 5226 if (!sit_i) 5227 return; 5228 5229 if (sit_i->sentries) 5230 kvfree(sit_i->bitmap); 5231 kfree(sit_i->tmp_map); 5232 5233 kvfree(sit_i->sentries); 5234 kvfree(sit_i->sec_entries); 5235 kvfree(sit_i->dirty_sentries_bitmap); 5236 5237 SM_I(sbi)->sit_info = NULL; 5238 kvfree(sit_i->sit_bitmap); 5239 #ifdef CONFIG_F2FS_CHECK_FS 5240 kvfree(sit_i->sit_bitmap_mir); 5241 kvfree(sit_i->invalid_segmap); 5242 #endif 5243 kfree(sit_i); 5244 } 5245 5246 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) 5247 { 5248 struct f2fs_sm_info *sm_info = SM_I(sbi); 5249 5250 if (!sm_info) 5251 return; 5252 f2fs_destroy_flush_cmd_control(sbi, true); 5253 destroy_discard_cmd_control(sbi); 5254 destroy_dirty_segmap(sbi); 5255 destroy_curseg(sbi); 5256 destroy_free_segmap(sbi); 5257 destroy_sit_info(sbi); 5258 sbi->sm_info = NULL; 5259 kfree(sm_info); 5260 } 5261 5262 int __init f2fs_create_segment_manager_caches(void) 5263 { 5264 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry", 5265 sizeof(struct discard_entry)); 5266 if (!discard_entry_slab) 5267 goto fail; 5268 5269 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd", 5270 sizeof(struct discard_cmd)); 5271 if (!discard_cmd_slab) 5272 goto destroy_discard_entry; 5273 5274 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set", 5275 sizeof(struct sit_entry_set)); 5276 if (!sit_entry_set_slab) 5277 goto destroy_discard_cmd; 5278 5279 revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry", 5280 sizeof(struct revoke_entry)); 5281 if (!revoke_entry_slab) 5282 goto destroy_sit_entry_set; 5283 return 0; 5284 5285 destroy_sit_entry_set: 5286 kmem_cache_destroy(sit_entry_set_slab); 5287 destroy_discard_cmd: 5288 kmem_cache_destroy(discard_cmd_slab); 5289 destroy_discard_entry: 5290 kmem_cache_destroy(discard_entry_slab); 5291 fail: 5292 return -ENOMEM; 5293 } 5294 5295 void f2fs_destroy_segment_manager_caches(void) 5296 { 5297 kmem_cache_destroy(sit_entry_set_slab); 5298 kmem_cache_destroy(discard_cmd_slab); 5299 kmem_cache_destroy(discard_entry_slab); 5300 kmem_cache_destroy(revoke_entry_slab); 5301 } 5302