1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/segment.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/prefetch.h> 13 #include <linux/kthread.h> 14 #include <linux/swap.h> 15 #include <linux/timer.h> 16 #include <linux/freezer.h> 17 #include <linux/sched/signal.h> 18 19 #include "f2fs.h" 20 #include "segment.h" 21 #include "node.h" 22 #include "gc.h" 23 #include <trace/events/f2fs.h> 24 25 #define __reverse_ffz(x) __reverse_ffs(~(x)) 26 27 static struct kmem_cache *discard_entry_slab; 28 static struct kmem_cache *discard_cmd_slab; 29 static struct kmem_cache *sit_entry_set_slab; 30 static struct kmem_cache *inmem_entry_slab; 31 32 static unsigned long __reverse_ulong(unsigned char *str) 33 { 34 unsigned long tmp = 0; 35 int shift = 24, idx = 0; 36 37 #if BITS_PER_LONG == 64 38 shift = 56; 39 #endif 40 while (shift >= 0) { 41 tmp |= (unsigned long)str[idx++] << shift; 42 shift -= BITS_PER_BYTE; 43 } 44 return tmp; 45 } 46 47 /* 48 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 49 * MSB and LSB are reversed in a byte by f2fs_set_bit. 50 */ 51 static inline unsigned long __reverse_ffs(unsigned long word) 52 { 53 int num = 0; 54 55 #if BITS_PER_LONG == 64 56 if ((word & 0xffffffff00000000UL) == 0) 57 num += 32; 58 else 59 word >>= 32; 60 #endif 61 if ((word & 0xffff0000) == 0) 62 num += 16; 63 else 64 word >>= 16; 65 66 if ((word & 0xff00) == 0) 67 num += 8; 68 else 69 word >>= 8; 70 71 if ((word & 0xf0) == 0) 72 num += 4; 73 else 74 word >>= 4; 75 76 if ((word & 0xc) == 0) 77 num += 2; 78 else 79 word >>= 2; 80 81 if ((word & 0x2) == 0) 82 num += 1; 83 return num; 84 } 85 86 /* 87 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because 88 * f2fs_set_bit makes MSB and LSB reversed in a byte. 89 * @size must be integral times of unsigned long. 90 * Example: 91 * MSB <--> LSB 92 * f2fs_set_bit(0, bitmap) => 1000 0000 93 * f2fs_set_bit(7, bitmap) => 0000 0001 94 */ 95 static unsigned long __find_rev_next_bit(const unsigned long *addr, 96 unsigned long size, unsigned long offset) 97 { 98 const unsigned long *p = addr + BIT_WORD(offset); 99 unsigned long result = size; 100 unsigned long tmp; 101 102 if (offset >= size) 103 return size; 104 105 size -= (offset & ~(BITS_PER_LONG - 1)); 106 offset %= BITS_PER_LONG; 107 108 while (1) { 109 if (*p == 0) 110 goto pass; 111 112 tmp = __reverse_ulong((unsigned char *)p); 113 114 tmp &= ~0UL >> offset; 115 if (size < BITS_PER_LONG) 116 tmp &= (~0UL << (BITS_PER_LONG - size)); 117 if (tmp) 118 goto found; 119 pass: 120 if (size <= BITS_PER_LONG) 121 break; 122 size -= BITS_PER_LONG; 123 offset = 0; 124 p++; 125 } 126 return result; 127 found: 128 return result - size + __reverse_ffs(tmp); 129 } 130 131 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 132 unsigned long size, unsigned long offset) 133 { 134 const unsigned long *p = addr + BIT_WORD(offset); 135 unsigned long result = size; 136 unsigned long tmp; 137 138 if (offset >= size) 139 return size; 140 141 size -= (offset & ~(BITS_PER_LONG - 1)); 142 offset %= BITS_PER_LONG; 143 144 while (1) { 145 if (*p == ~0UL) 146 goto pass; 147 148 tmp = __reverse_ulong((unsigned char *)p); 149 150 if (offset) 151 tmp |= ~0UL << (BITS_PER_LONG - offset); 152 if (size < BITS_PER_LONG) 153 tmp |= ~0UL >> size; 154 if (tmp != ~0UL) 155 goto found; 156 pass: 157 if (size <= BITS_PER_LONG) 158 break; 159 size -= BITS_PER_LONG; 160 offset = 0; 161 p++; 162 } 163 return result; 164 found: 165 return result - size + __reverse_ffz(tmp); 166 } 167 168 bool f2fs_need_SSR(struct f2fs_sb_info *sbi) 169 { 170 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); 171 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); 172 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); 173 174 if (f2fs_lfs_mode(sbi)) 175 return false; 176 if (sbi->gc_mode == GC_URGENT_HIGH) 177 return true; 178 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 179 return true; 180 181 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + 182 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); 183 } 184 185 void f2fs_register_inmem_page(struct inode *inode, struct page *page) 186 { 187 struct inmem_pages *new; 188 189 if (PagePrivate(page)) 190 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); 191 else 192 f2fs_set_page_private(page, ATOMIC_WRITTEN_PAGE); 193 194 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); 195 196 /* add atomic page indices to the list */ 197 new->page = page; 198 INIT_LIST_HEAD(&new->list); 199 200 /* increase reference count with clean state */ 201 get_page(page); 202 mutex_lock(&F2FS_I(inode)->inmem_lock); 203 list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages); 204 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); 205 mutex_unlock(&F2FS_I(inode)->inmem_lock); 206 207 trace_f2fs_register_inmem_page(page, INMEM); 208 } 209 210 static int __revoke_inmem_pages(struct inode *inode, 211 struct list_head *head, bool drop, bool recover, 212 bool trylock) 213 { 214 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 215 struct inmem_pages *cur, *tmp; 216 int err = 0; 217 218 list_for_each_entry_safe(cur, tmp, head, list) { 219 struct page *page = cur->page; 220 221 if (drop) 222 trace_f2fs_commit_inmem_page(page, INMEM_DROP); 223 224 if (trylock) { 225 /* 226 * to avoid deadlock in between page lock and 227 * inmem_lock. 228 */ 229 if (!trylock_page(page)) 230 continue; 231 } else { 232 lock_page(page); 233 } 234 235 f2fs_wait_on_page_writeback(page, DATA, true, true); 236 237 if (recover) { 238 struct dnode_of_data dn; 239 struct node_info ni; 240 241 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE); 242 retry: 243 set_new_dnode(&dn, inode, NULL, NULL, 0); 244 err = f2fs_get_dnode_of_data(&dn, page->index, 245 LOOKUP_NODE); 246 if (err) { 247 if (err == -ENOMEM) { 248 congestion_wait(BLK_RW_ASYNC, 249 DEFAULT_IO_TIMEOUT); 250 cond_resched(); 251 goto retry; 252 } 253 err = -EAGAIN; 254 goto next; 255 } 256 257 err = f2fs_get_node_info(sbi, dn.nid, &ni); 258 if (err) { 259 f2fs_put_dnode(&dn); 260 return err; 261 } 262 263 if (cur->old_addr == NEW_ADDR) { 264 f2fs_invalidate_blocks(sbi, dn.data_blkaddr); 265 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 266 } else 267 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, 268 cur->old_addr, ni.version, true, true); 269 f2fs_put_dnode(&dn); 270 } 271 next: 272 /* we don't need to invalidate this in the sccessful status */ 273 if (drop || recover) { 274 ClearPageUptodate(page); 275 clear_cold_data(page); 276 } 277 f2fs_clear_page_private(page); 278 f2fs_put_page(page, 1); 279 280 list_del(&cur->list); 281 kmem_cache_free(inmem_entry_slab, cur); 282 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); 283 } 284 return err; 285 } 286 287 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) 288 { 289 struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; 290 struct inode *inode; 291 struct f2fs_inode_info *fi; 292 unsigned int count = sbi->atomic_files; 293 unsigned int looped = 0; 294 next: 295 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 296 if (list_empty(head)) { 297 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 298 return; 299 } 300 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); 301 inode = igrab(&fi->vfs_inode); 302 if (inode) 303 list_move_tail(&fi->inmem_ilist, head); 304 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 305 306 if (inode) { 307 if (gc_failure) { 308 if (!fi->i_gc_failures[GC_FAILURE_ATOMIC]) 309 goto skip; 310 } 311 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST); 312 f2fs_drop_inmem_pages(inode); 313 skip: 314 iput(inode); 315 } 316 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); 317 cond_resched(); 318 if (gc_failure) { 319 if (++looped >= count) 320 return; 321 } 322 goto next; 323 } 324 325 void f2fs_drop_inmem_pages(struct inode *inode) 326 { 327 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 328 struct f2fs_inode_info *fi = F2FS_I(inode); 329 330 do { 331 mutex_lock(&fi->inmem_lock); 332 if (list_empty(&fi->inmem_pages)) { 333 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; 334 335 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 336 if (!list_empty(&fi->inmem_ilist)) 337 list_del_init(&fi->inmem_ilist); 338 if (f2fs_is_atomic_file(inode)) { 339 clear_inode_flag(inode, FI_ATOMIC_FILE); 340 sbi->atomic_files--; 341 } 342 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 343 344 mutex_unlock(&fi->inmem_lock); 345 break; 346 } 347 __revoke_inmem_pages(inode, &fi->inmem_pages, 348 true, false, true); 349 mutex_unlock(&fi->inmem_lock); 350 } while (1); 351 } 352 353 void f2fs_drop_inmem_page(struct inode *inode, struct page *page) 354 { 355 struct f2fs_inode_info *fi = F2FS_I(inode); 356 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 357 struct list_head *head = &fi->inmem_pages; 358 struct inmem_pages *cur = NULL; 359 360 f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); 361 362 mutex_lock(&fi->inmem_lock); 363 list_for_each_entry(cur, head, list) { 364 if (cur->page == page) 365 break; 366 } 367 368 f2fs_bug_on(sbi, list_empty(head) || cur->page != page); 369 list_del(&cur->list); 370 mutex_unlock(&fi->inmem_lock); 371 372 dec_page_count(sbi, F2FS_INMEM_PAGES); 373 kmem_cache_free(inmem_entry_slab, cur); 374 375 ClearPageUptodate(page); 376 f2fs_clear_page_private(page); 377 f2fs_put_page(page, 0); 378 379 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE); 380 } 381 382 static int __f2fs_commit_inmem_pages(struct inode *inode) 383 { 384 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 385 struct f2fs_inode_info *fi = F2FS_I(inode); 386 struct inmem_pages *cur, *tmp; 387 struct f2fs_io_info fio = { 388 .sbi = sbi, 389 .ino = inode->i_ino, 390 .type = DATA, 391 .op = REQ_OP_WRITE, 392 .op_flags = REQ_SYNC | REQ_PRIO, 393 .io_type = FS_DATA_IO, 394 }; 395 struct list_head revoke_list; 396 bool submit_bio = false; 397 int err = 0; 398 399 INIT_LIST_HEAD(&revoke_list); 400 401 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) { 402 struct page *page = cur->page; 403 404 lock_page(page); 405 if (page->mapping == inode->i_mapping) { 406 trace_f2fs_commit_inmem_page(page, INMEM); 407 408 f2fs_wait_on_page_writeback(page, DATA, true, true); 409 410 set_page_dirty(page); 411 if (clear_page_dirty_for_io(page)) { 412 inode_dec_dirty_pages(inode); 413 f2fs_remove_dirty_inode(inode); 414 } 415 retry: 416 fio.page = page; 417 fio.old_blkaddr = NULL_ADDR; 418 fio.encrypted_page = NULL; 419 fio.need_lock = LOCK_DONE; 420 err = f2fs_do_write_data_page(&fio); 421 if (err) { 422 if (err == -ENOMEM) { 423 congestion_wait(BLK_RW_ASYNC, 424 DEFAULT_IO_TIMEOUT); 425 cond_resched(); 426 goto retry; 427 } 428 unlock_page(page); 429 break; 430 } 431 /* record old blkaddr for revoking */ 432 cur->old_addr = fio.old_blkaddr; 433 submit_bio = true; 434 } 435 unlock_page(page); 436 list_move_tail(&cur->list, &revoke_list); 437 } 438 439 if (submit_bio) 440 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA); 441 442 if (err) { 443 /* 444 * try to revoke all committed pages, but still we could fail 445 * due to no memory or other reason, if that happened, EAGAIN 446 * will be returned, which means in such case, transaction is 447 * already not integrity, caller should use journal to do the 448 * recovery or rewrite & commit last transaction. For other 449 * error number, revoking was done by filesystem itself. 450 */ 451 err = __revoke_inmem_pages(inode, &revoke_list, 452 false, true, false); 453 454 /* drop all uncommitted pages */ 455 __revoke_inmem_pages(inode, &fi->inmem_pages, 456 true, false, false); 457 } else { 458 __revoke_inmem_pages(inode, &revoke_list, 459 false, false, false); 460 } 461 462 return err; 463 } 464 465 int f2fs_commit_inmem_pages(struct inode *inode) 466 { 467 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 468 struct f2fs_inode_info *fi = F2FS_I(inode); 469 int err; 470 471 f2fs_balance_fs(sbi, true); 472 473 down_write(&fi->i_gc_rwsem[WRITE]); 474 475 f2fs_lock_op(sbi); 476 set_inode_flag(inode, FI_ATOMIC_COMMIT); 477 478 mutex_lock(&fi->inmem_lock); 479 err = __f2fs_commit_inmem_pages(inode); 480 mutex_unlock(&fi->inmem_lock); 481 482 clear_inode_flag(inode, FI_ATOMIC_COMMIT); 483 484 f2fs_unlock_op(sbi); 485 up_write(&fi->i_gc_rwsem[WRITE]); 486 487 return err; 488 } 489 490 /* 491 * This function balances dirty node and dentry pages. 492 * In addition, it controls garbage collection. 493 */ 494 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) 495 { 496 if (time_to_inject(sbi, FAULT_CHECKPOINT)) { 497 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); 498 f2fs_stop_checkpoint(sbi, false); 499 } 500 501 /* balance_fs_bg is able to be pending */ 502 if (need && excess_cached_nats(sbi)) 503 f2fs_balance_fs_bg(sbi, false); 504 505 if (!f2fs_is_checkpoint_ready(sbi)) 506 return; 507 508 /* 509 * We should do GC or end up with checkpoint, if there are so many dirty 510 * dir/node pages without enough free segments. 511 */ 512 if (has_not_enough_free_secs(sbi, 0, 0)) { 513 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && 514 sbi->gc_thread->f2fs_gc_task) { 515 DEFINE_WAIT(wait); 516 517 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait, 518 TASK_UNINTERRUPTIBLE); 519 wake_up(&sbi->gc_thread->gc_wait_queue_head); 520 io_schedule(); 521 finish_wait(&sbi->gc_thread->fggc_wq, &wait); 522 } else { 523 down_write(&sbi->gc_lock); 524 f2fs_gc(sbi, false, false, false, NULL_SEGNO); 525 } 526 } 527 } 528 529 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) 530 { 531 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 532 return; 533 534 /* try to shrink extent cache when there is no enough memory */ 535 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE)) 536 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); 537 538 /* check the # of cached NAT entries */ 539 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) 540 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); 541 542 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) 543 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); 544 else 545 f2fs_build_free_nids(sbi, false, false); 546 547 if (excess_dirty_nats(sbi) || excess_dirty_nodes(sbi) || 548 excess_prefree_segs(sbi)) 549 goto do_sync; 550 551 /* there is background inflight IO or foreground operation recently */ 552 if (is_inflight_io(sbi, REQ_TIME) || 553 (!f2fs_time_over(sbi, REQ_TIME) && rwsem_is_locked(&sbi->cp_rwsem))) 554 return; 555 556 /* exceed periodical checkpoint timeout threshold */ 557 if (f2fs_time_over(sbi, CP_TIME)) 558 goto do_sync; 559 560 /* checkpoint is the only way to shrink partial cached entries */ 561 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) || 562 f2fs_available_free_memory(sbi, INO_ENTRIES)) 563 return; 564 565 do_sync: 566 if (test_opt(sbi, DATA_FLUSH) && from_bg) { 567 struct blk_plug plug; 568 569 mutex_lock(&sbi->flush_lock); 570 571 blk_start_plug(&plug); 572 f2fs_sync_dirty_inodes(sbi, FILE_INODE); 573 blk_finish_plug(&plug); 574 575 mutex_unlock(&sbi->flush_lock); 576 } 577 f2fs_sync_fs(sbi->sb, true); 578 stat_inc_bg_cp_count(sbi->stat_info); 579 } 580 581 static int __submit_flush_wait(struct f2fs_sb_info *sbi, 582 struct block_device *bdev) 583 { 584 int ret = blkdev_issue_flush(bdev); 585 586 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), 587 test_opt(sbi, FLUSH_MERGE), ret); 588 return ret; 589 } 590 591 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) 592 { 593 int ret = 0; 594 int i; 595 596 if (!f2fs_is_multi_device(sbi)) 597 return __submit_flush_wait(sbi, sbi->sb->s_bdev); 598 599 for (i = 0; i < sbi->s_ndevs; i++) { 600 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) 601 continue; 602 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 603 if (ret) 604 break; 605 } 606 return ret; 607 } 608 609 static int issue_flush_thread(void *data) 610 { 611 struct f2fs_sb_info *sbi = data; 612 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 613 wait_queue_head_t *q = &fcc->flush_wait_queue; 614 repeat: 615 if (kthread_should_stop()) 616 return 0; 617 618 if (!llist_empty(&fcc->issue_list)) { 619 struct flush_cmd *cmd, *next; 620 int ret; 621 622 fcc->dispatch_list = llist_del_all(&fcc->issue_list); 623 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); 624 625 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode); 626 627 ret = submit_flush_wait(sbi, cmd->ino); 628 atomic_inc(&fcc->issued_flush); 629 630 llist_for_each_entry_safe(cmd, next, 631 fcc->dispatch_list, llnode) { 632 cmd->ret = ret; 633 complete(&cmd->wait); 634 } 635 fcc->dispatch_list = NULL; 636 } 637 638 wait_event_interruptible(*q, 639 kthread_should_stop() || !llist_empty(&fcc->issue_list)); 640 goto repeat; 641 } 642 643 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) 644 { 645 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 646 struct flush_cmd cmd; 647 int ret; 648 649 if (test_opt(sbi, NOBARRIER)) 650 return 0; 651 652 if (!test_opt(sbi, FLUSH_MERGE)) { 653 atomic_inc(&fcc->queued_flush); 654 ret = submit_flush_wait(sbi, ino); 655 atomic_dec(&fcc->queued_flush); 656 atomic_inc(&fcc->issued_flush); 657 return ret; 658 } 659 660 if (atomic_inc_return(&fcc->queued_flush) == 1 || 661 f2fs_is_multi_device(sbi)) { 662 ret = submit_flush_wait(sbi, ino); 663 atomic_dec(&fcc->queued_flush); 664 665 atomic_inc(&fcc->issued_flush); 666 return ret; 667 } 668 669 cmd.ino = ino; 670 init_completion(&cmd.wait); 671 672 llist_add(&cmd.llnode, &fcc->issue_list); 673 674 /* 675 * update issue_list before we wake up issue_flush thread, this 676 * smp_mb() pairs with another barrier in ___wait_event(), see 677 * more details in comments of waitqueue_active(). 678 */ 679 smp_mb(); 680 681 if (waitqueue_active(&fcc->flush_wait_queue)) 682 wake_up(&fcc->flush_wait_queue); 683 684 if (fcc->f2fs_issue_flush) { 685 wait_for_completion(&cmd.wait); 686 atomic_dec(&fcc->queued_flush); 687 } else { 688 struct llist_node *list; 689 690 list = llist_del_all(&fcc->issue_list); 691 if (!list) { 692 wait_for_completion(&cmd.wait); 693 atomic_dec(&fcc->queued_flush); 694 } else { 695 struct flush_cmd *tmp, *next; 696 697 ret = submit_flush_wait(sbi, ino); 698 699 llist_for_each_entry_safe(tmp, next, list, llnode) { 700 if (tmp == &cmd) { 701 cmd.ret = ret; 702 atomic_dec(&fcc->queued_flush); 703 continue; 704 } 705 tmp->ret = ret; 706 complete(&tmp->wait); 707 } 708 } 709 } 710 711 return cmd.ret; 712 } 713 714 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) 715 { 716 dev_t dev = sbi->sb->s_bdev->bd_dev; 717 struct flush_cmd_control *fcc; 718 int err = 0; 719 720 if (SM_I(sbi)->fcc_info) { 721 fcc = SM_I(sbi)->fcc_info; 722 if (fcc->f2fs_issue_flush) 723 return err; 724 goto init_thread; 725 } 726 727 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); 728 if (!fcc) 729 return -ENOMEM; 730 atomic_set(&fcc->issued_flush, 0); 731 atomic_set(&fcc->queued_flush, 0); 732 init_waitqueue_head(&fcc->flush_wait_queue); 733 init_llist_head(&fcc->issue_list); 734 SM_I(sbi)->fcc_info = fcc; 735 if (!test_opt(sbi, FLUSH_MERGE)) 736 return err; 737 738 init_thread: 739 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, 740 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); 741 if (IS_ERR(fcc->f2fs_issue_flush)) { 742 err = PTR_ERR(fcc->f2fs_issue_flush); 743 kfree(fcc); 744 SM_I(sbi)->fcc_info = NULL; 745 return err; 746 } 747 748 return err; 749 } 750 751 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) 752 { 753 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 754 755 if (fcc && fcc->f2fs_issue_flush) { 756 struct task_struct *flush_thread = fcc->f2fs_issue_flush; 757 758 fcc->f2fs_issue_flush = NULL; 759 kthread_stop(flush_thread); 760 } 761 if (free) { 762 kfree(fcc); 763 SM_I(sbi)->fcc_info = NULL; 764 } 765 } 766 767 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) 768 { 769 int ret = 0, i; 770 771 if (!f2fs_is_multi_device(sbi)) 772 return 0; 773 774 if (test_opt(sbi, NOBARRIER)) 775 return 0; 776 777 for (i = 1; i < sbi->s_ndevs; i++) { 778 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) 779 continue; 780 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 781 if (ret) 782 break; 783 784 spin_lock(&sbi->dev_lock); 785 f2fs_clear_bit(i, (char *)&sbi->dirty_device); 786 spin_unlock(&sbi->dev_lock); 787 } 788 789 return ret; 790 } 791 792 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 793 enum dirty_type dirty_type) 794 { 795 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 796 797 /* need not be added */ 798 if (IS_CURSEG(sbi, segno)) 799 return; 800 801 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 802 dirty_i->nr_dirty[dirty_type]++; 803 804 if (dirty_type == DIRTY) { 805 struct seg_entry *sentry = get_seg_entry(sbi, segno); 806 enum dirty_type t = sentry->type; 807 808 if (unlikely(t >= DIRTY)) { 809 f2fs_bug_on(sbi, 1); 810 return; 811 } 812 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 813 dirty_i->nr_dirty[t]++; 814 815 if (__is_large_section(sbi)) { 816 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 817 block_t valid_blocks = 818 get_valid_blocks(sbi, segno, true); 819 820 f2fs_bug_on(sbi, unlikely(!valid_blocks || 821 valid_blocks == BLKS_PER_SEC(sbi))); 822 823 if (!IS_CURSEC(sbi, secno)) 824 set_bit(secno, dirty_i->dirty_secmap); 825 } 826 } 827 } 828 829 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 830 enum dirty_type dirty_type) 831 { 832 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 833 block_t valid_blocks; 834 835 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 836 dirty_i->nr_dirty[dirty_type]--; 837 838 if (dirty_type == DIRTY) { 839 struct seg_entry *sentry = get_seg_entry(sbi, segno); 840 enum dirty_type t = sentry->type; 841 842 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 843 dirty_i->nr_dirty[t]--; 844 845 valid_blocks = get_valid_blocks(sbi, segno, true); 846 if (valid_blocks == 0) { 847 clear_bit(GET_SEC_FROM_SEG(sbi, segno), 848 dirty_i->victim_secmap); 849 #ifdef CONFIG_F2FS_CHECK_FS 850 clear_bit(segno, SIT_I(sbi)->invalid_segmap); 851 #endif 852 } 853 if (__is_large_section(sbi)) { 854 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 855 856 if (!valid_blocks || 857 valid_blocks == BLKS_PER_SEC(sbi)) { 858 clear_bit(secno, dirty_i->dirty_secmap); 859 return; 860 } 861 862 if (!IS_CURSEC(sbi, secno)) 863 set_bit(secno, dirty_i->dirty_secmap); 864 } 865 } 866 } 867 868 /* 869 * Should not occur error such as -ENOMEM. 870 * Adding dirty entry into seglist is not critical operation. 871 * If a given segment is one of current working segments, it won't be added. 872 */ 873 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 874 { 875 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 876 unsigned short valid_blocks, ckpt_valid_blocks; 877 unsigned int usable_blocks; 878 879 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) 880 return; 881 882 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno); 883 mutex_lock(&dirty_i->seglist_lock); 884 885 valid_blocks = get_valid_blocks(sbi, segno, false); 886 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false); 887 888 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || 889 ckpt_valid_blocks == usable_blocks)) { 890 __locate_dirty_segment(sbi, segno, PRE); 891 __remove_dirty_segment(sbi, segno, DIRTY); 892 } else if (valid_blocks < usable_blocks) { 893 __locate_dirty_segment(sbi, segno, DIRTY); 894 } else { 895 /* Recovery routine with SSR needs this */ 896 __remove_dirty_segment(sbi, segno, DIRTY); 897 } 898 899 mutex_unlock(&dirty_i->seglist_lock); 900 } 901 902 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */ 903 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) 904 { 905 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 906 unsigned int segno; 907 908 mutex_lock(&dirty_i->seglist_lock); 909 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 910 if (get_valid_blocks(sbi, segno, false)) 911 continue; 912 if (IS_CURSEG(sbi, segno)) 913 continue; 914 __locate_dirty_segment(sbi, segno, PRE); 915 __remove_dirty_segment(sbi, segno, DIRTY); 916 } 917 mutex_unlock(&dirty_i->seglist_lock); 918 } 919 920 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi) 921 { 922 int ovp_hole_segs = 923 (overprovision_segments(sbi) - reserved_segments(sbi)); 924 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg; 925 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 926 block_t holes[2] = {0, 0}; /* DATA and NODE */ 927 block_t unusable; 928 struct seg_entry *se; 929 unsigned int segno; 930 931 mutex_lock(&dirty_i->seglist_lock); 932 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 933 se = get_seg_entry(sbi, segno); 934 if (IS_NODESEG(se->type)) 935 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) - 936 se->valid_blocks; 937 else 938 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) - 939 se->valid_blocks; 940 } 941 mutex_unlock(&dirty_i->seglist_lock); 942 943 unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE]; 944 if (unusable > ovp_holes) 945 return unusable - ovp_holes; 946 return 0; 947 } 948 949 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable) 950 { 951 int ovp_hole_segs = 952 (overprovision_segments(sbi) - reserved_segments(sbi)); 953 if (unusable > F2FS_OPTION(sbi).unusable_cap) 954 return -EAGAIN; 955 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && 956 dirty_segments(sbi) > ovp_hole_segs) 957 return -EAGAIN; 958 return 0; 959 } 960 961 /* This is only used by SBI_CP_DISABLED */ 962 static unsigned int get_free_segment(struct f2fs_sb_info *sbi) 963 { 964 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 965 unsigned int segno = 0; 966 967 mutex_lock(&dirty_i->seglist_lock); 968 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 969 if (get_valid_blocks(sbi, segno, false)) 970 continue; 971 if (get_ckpt_valid_blocks(sbi, segno, false)) 972 continue; 973 mutex_unlock(&dirty_i->seglist_lock); 974 return segno; 975 } 976 mutex_unlock(&dirty_i->seglist_lock); 977 return NULL_SEGNO; 978 } 979 980 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, 981 struct block_device *bdev, block_t lstart, 982 block_t start, block_t len) 983 { 984 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 985 struct list_head *pend_list; 986 struct discard_cmd *dc; 987 988 f2fs_bug_on(sbi, !len); 989 990 pend_list = &dcc->pend_list[plist_idx(len)]; 991 992 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS); 993 INIT_LIST_HEAD(&dc->list); 994 dc->bdev = bdev; 995 dc->lstart = lstart; 996 dc->start = start; 997 dc->len = len; 998 dc->ref = 0; 999 dc->state = D_PREP; 1000 dc->queued = 0; 1001 dc->error = 0; 1002 init_completion(&dc->wait); 1003 list_add_tail(&dc->list, pend_list); 1004 spin_lock_init(&dc->lock); 1005 dc->bio_ref = 0; 1006 atomic_inc(&dcc->discard_cmd_cnt); 1007 dcc->undiscard_blks += len; 1008 1009 return dc; 1010 } 1011 1012 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, 1013 struct block_device *bdev, block_t lstart, 1014 block_t start, block_t len, 1015 struct rb_node *parent, struct rb_node **p, 1016 bool leftmost) 1017 { 1018 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1019 struct discard_cmd *dc; 1020 1021 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); 1022 1023 rb_link_node(&dc->rb_node, parent, p); 1024 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); 1025 1026 return dc; 1027 } 1028 1029 static void __detach_discard_cmd(struct discard_cmd_control *dcc, 1030 struct discard_cmd *dc) 1031 { 1032 if (dc->state == D_DONE) 1033 atomic_sub(dc->queued, &dcc->queued_discard); 1034 1035 list_del(&dc->list); 1036 rb_erase_cached(&dc->rb_node, &dcc->root); 1037 dcc->undiscard_blks -= dc->len; 1038 1039 kmem_cache_free(discard_cmd_slab, dc); 1040 1041 atomic_dec(&dcc->discard_cmd_cnt); 1042 } 1043 1044 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, 1045 struct discard_cmd *dc) 1046 { 1047 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1048 unsigned long flags; 1049 1050 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len); 1051 1052 spin_lock_irqsave(&dc->lock, flags); 1053 if (dc->bio_ref) { 1054 spin_unlock_irqrestore(&dc->lock, flags); 1055 return; 1056 } 1057 spin_unlock_irqrestore(&dc->lock, flags); 1058 1059 f2fs_bug_on(sbi, dc->ref); 1060 1061 if (dc->error == -EOPNOTSUPP) 1062 dc->error = 0; 1063 1064 if (dc->error) 1065 printk_ratelimited( 1066 "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d", 1067 KERN_INFO, sbi->sb->s_id, 1068 dc->lstart, dc->start, dc->len, dc->error); 1069 __detach_discard_cmd(dcc, dc); 1070 } 1071 1072 static void f2fs_submit_discard_endio(struct bio *bio) 1073 { 1074 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; 1075 unsigned long flags; 1076 1077 spin_lock_irqsave(&dc->lock, flags); 1078 if (!dc->error) 1079 dc->error = blk_status_to_errno(bio->bi_status); 1080 dc->bio_ref--; 1081 if (!dc->bio_ref && dc->state == D_SUBMIT) { 1082 dc->state = D_DONE; 1083 complete_all(&dc->wait); 1084 } 1085 spin_unlock_irqrestore(&dc->lock, flags); 1086 bio_put(bio); 1087 } 1088 1089 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, 1090 block_t start, block_t end) 1091 { 1092 #ifdef CONFIG_F2FS_CHECK_FS 1093 struct seg_entry *sentry; 1094 unsigned int segno; 1095 block_t blk = start; 1096 unsigned long offset, size, max_blocks = sbi->blocks_per_seg; 1097 unsigned long *map; 1098 1099 while (blk < end) { 1100 segno = GET_SEGNO(sbi, blk); 1101 sentry = get_seg_entry(sbi, segno); 1102 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); 1103 1104 if (end < START_BLOCK(sbi, segno + 1)) 1105 size = GET_BLKOFF_FROM_SEG0(sbi, end); 1106 else 1107 size = max_blocks; 1108 map = (unsigned long *)(sentry->cur_valid_map); 1109 offset = __find_rev_next_bit(map, size, offset); 1110 f2fs_bug_on(sbi, offset != size); 1111 blk = START_BLOCK(sbi, segno + 1); 1112 } 1113 #endif 1114 } 1115 1116 static void __init_discard_policy(struct f2fs_sb_info *sbi, 1117 struct discard_policy *dpolicy, 1118 int discard_type, unsigned int granularity) 1119 { 1120 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1121 1122 /* common policy */ 1123 dpolicy->type = discard_type; 1124 dpolicy->sync = true; 1125 dpolicy->ordered = false; 1126 dpolicy->granularity = granularity; 1127 1128 dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST; 1129 dpolicy->io_aware_gran = MAX_PLIST_NUM; 1130 dpolicy->timeout = false; 1131 1132 if (discard_type == DPOLICY_BG) { 1133 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME; 1134 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME; 1135 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME; 1136 dpolicy->io_aware = true; 1137 dpolicy->sync = false; 1138 dpolicy->ordered = true; 1139 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) { 1140 dpolicy->granularity = 1; 1141 if (atomic_read(&dcc->discard_cmd_cnt)) 1142 dpolicy->max_interval = 1143 DEF_MIN_DISCARD_ISSUE_TIME; 1144 } 1145 } else if (discard_type == DPOLICY_FORCE) { 1146 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME; 1147 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME; 1148 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME; 1149 dpolicy->io_aware = false; 1150 } else if (discard_type == DPOLICY_FSTRIM) { 1151 dpolicy->io_aware = false; 1152 } else if (discard_type == DPOLICY_UMOUNT) { 1153 dpolicy->io_aware = false; 1154 /* we need to issue all to keep CP_TRIMMED_FLAG */ 1155 dpolicy->granularity = 1; 1156 dpolicy->timeout = true; 1157 } 1158 } 1159 1160 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1161 struct block_device *bdev, block_t lstart, 1162 block_t start, block_t len); 1163 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ 1164 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, 1165 struct discard_policy *dpolicy, 1166 struct discard_cmd *dc, 1167 unsigned int *issued) 1168 { 1169 struct block_device *bdev = dc->bdev; 1170 struct request_queue *q = bdev_get_queue(bdev); 1171 unsigned int max_discard_blocks = 1172 SECTOR_TO_BLOCK(q->limits.max_discard_sectors); 1173 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1174 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1175 &(dcc->fstrim_list) : &(dcc->wait_list); 1176 int flag = dpolicy->sync ? REQ_SYNC : 0; 1177 block_t lstart, start, len, total_len; 1178 int err = 0; 1179 1180 if (dc->state != D_PREP) 1181 return 0; 1182 1183 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 1184 return 0; 1185 1186 trace_f2fs_issue_discard(bdev, dc->start, dc->len); 1187 1188 lstart = dc->lstart; 1189 start = dc->start; 1190 len = dc->len; 1191 total_len = len; 1192 1193 dc->len = 0; 1194 1195 while (total_len && *issued < dpolicy->max_requests && !err) { 1196 struct bio *bio = NULL; 1197 unsigned long flags; 1198 bool last = true; 1199 1200 if (len > max_discard_blocks) { 1201 len = max_discard_blocks; 1202 last = false; 1203 } 1204 1205 (*issued)++; 1206 if (*issued == dpolicy->max_requests) 1207 last = true; 1208 1209 dc->len += len; 1210 1211 if (time_to_inject(sbi, FAULT_DISCARD)) { 1212 f2fs_show_injection_info(sbi, FAULT_DISCARD); 1213 err = -EIO; 1214 goto submit; 1215 } 1216 err = __blkdev_issue_discard(bdev, 1217 SECTOR_FROM_BLOCK(start), 1218 SECTOR_FROM_BLOCK(len), 1219 GFP_NOFS, 0, &bio); 1220 submit: 1221 if (err) { 1222 spin_lock_irqsave(&dc->lock, flags); 1223 if (dc->state == D_PARTIAL) 1224 dc->state = D_SUBMIT; 1225 spin_unlock_irqrestore(&dc->lock, flags); 1226 1227 break; 1228 } 1229 1230 f2fs_bug_on(sbi, !bio); 1231 1232 /* 1233 * should keep before submission to avoid D_DONE 1234 * right away 1235 */ 1236 spin_lock_irqsave(&dc->lock, flags); 1237 if (last) 1238 dc->state = D_SUBMIT; 1239 else 1240 dc->state = D_PARTIAL; 1241 dc->bio_ref++; 1242 spin_unlock_irqrestore(&dc->lock, flags); 1243 1244 atomic_inc(&dcc->queued_discard); 1245 dc->queued++; 1246 list_move_tail(&dc->list, wait_list); 1247 1248 /* sanity check on discard range */ 1249 __check_sit_bitmap(sbi, lstart, lstart + len); 1250 1251 bio->bi_private = dc; 1252 bio->bi_end_io = f2fs_submit_discard_endio; 1253 bio->bi_opf |= flag; 1254 submit_bio(bio); 1255 1256 atomic_inc(&dcc->issued_discard); 1257 1258 f2fs_update_iostat(sbi, FS_DISCARD, 1); 1259 1260 lstart += len; 1261 start += len; 1262 total_len -= len; 1263 len = total_len; 1264 } 1265 1266 if (!err && len) { 1267 dcc->undiscard_blks -= len; 1268 __update_discard_tree_range(sbi, bdev, lstart, start, len); 1269 } 1270 return err; 1271 } 1272 1273 static void __insert_discard_tree(struct f2fs_sb_info *sbi, 1274 struct block_device *bdev, block_t lstart, 1275 block_t start, block_t len, 1276 struct rb_node **insert_p, 1277 struct rb_node *insert_parent) 1278 { 1279 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1280 struct rb_node **p; 1281 struct rb_node *parent = NULL; 1282 bool leftmost = true; 1283 1284 if (insert_p && insert_parent) { 1285 parent = insert_parent; 1286 p = insert_p; 1287 goto do_insert; 1288 } 1289 1290 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, 1291 lstart, &leftmost); 1292 do_insert: 1293 __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, 1294 p, leftmost); 1295 } 1296 1297 static void __relocate_discard_cmd(struct discard_cmd_control *dcc, 1298 struct discard_cmd *dc) 1299 { 1300 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]); 1301 } 1302 1303 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, 1304 struct discard_cmd *dc, block_t blkaddr) 1305 { 1306 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1307 struct discard_info di = dc->di; 1308 bool modified = false; 1309 1310 if (dc->state == D_DONE || dc->len == 1) { 1311 __remove_discard_cmd(sbi, dc); 1312 return; 1313 } 1314 1315 dcc->undiscard_blks -= di.len; 1316 1317 if (blkaddr > di.lstart) { 1318 dc->len = blkaddr - dc->lstart; 1319 dcc->undiscard_blks += dc->len; 1320 __relocate_discard_cmd(dcc, dc); 1321 modified = true; 1322 } 1323 1324 if (blkaddr < di.lstart + di.len - 1) { 1325 if (modified) { 1326 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, 1327 di.start + blkaddr + 1 - di.lstart, 1328 di.lstart + di.len - 1 - blkaddr, 1329 NULL, NULL); 1330 } else { 1331 dc->lstart++; 1332 dc->len--; 1333 dc->start++; 1334 dcc->undiscard_blks += dc->len; 1335 __relocate_discard_cmd(dcc, dc); 1336 } 1337 } 1338 } 1339 1340 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1341 struct block_device *bdev, block_t lstart, 1342 block_t start, block_t len) 1343 { 1344 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1345 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1346 struct discard_cmd *dc; 1347 struct discard_info di = {0}; 1348 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1349 struct request_queue *q = bdev_get_queue(bdev); 1350 unsigned int max_discard_blocks = 1351 SECTOR_TO_BLOCK(q->limits.max_discard_sectors); 1352 block_t end = lstart + len; 1353 1354 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 1355 NULL, lstart, 1356 (struct rb_entry **)&prev_dc, 1357 (struct rb_entry **)&next_dc, 1358 &insert_p, &insert_parent, true, NULL); 1359 if (dc) 1360 prev_dc = dc; 1361 1362 if (!prev_dc) { 1363 di.lstart = lstart; 1364 di.len = next_dc ? next_dc->lstart - lstart : len; 1365 di.len = min(di.len, len); 1366 di.start = start; 1367 } 1368 1369 while (1) { 1370 struct rb_node *node; 1371 bool merged = false; 1372 struct discard_cmd *tdc = NULL; 1373 1374 if (prev_dc) { 1375 di.lstart = prev_dc->lstart + prev_dc->len; 1376 if (di.lstart < lstart) 1377 di.lstart = lstart; 1378 if (di.lstart >= end) 1379 break; 1380 1381 if (!next_dc || next_dc->lstart > end) 1382 di.len = end - di.lstart; 1383 else 1384 di.len = next_dc->lstart - di.lstart; 1385 di.start = start + di.lstart - lstart; 1386 } 1387 1388 if (!di.len) 1389 goto next; 1390 1391 if (prev_dc && prev_dc->state == D_PREP && 1392 prev_dc->bdev == bdev && 1393 __is_discard_back_mergeable(&di, &prev_dc->di, 1394 max_discard_blocks)) { 1395 prev_dc->di.len += di.len; 1396 dcc->undiscard_blks += di.len; 1397 __relocate_discard_cmd(dcc, prev_dc); 1398 di = prev_dc->di; 1399 tdc = prev_dc; 1400 merged = true; 1401 } 1402 1403 if (next_dc && next_dc->state == D_PREP && 1404 next_dc->bdev == bdev && 1405 __is_discard_front_mergeable(&di, &next_dc->di, 1406 max_discard_blocks)) { 1407 next_dc->di.lstart = di.lstart; 1408 next_dc->di.len += di.len; 1409 next_dc->di.start = di.start; 1410 dcc->undiscard_blks += di.len; 1411 __relocate_discard_cmd(dcc, next_dc); 1412 if (tdc) 1413 __remove_discard_cmd(sbi, tdc); 1414 merged = true; 1415 } 1416 1417 if (!merged) { 1418 __insert_discard_tree(sbi, bdev, di.lstart, di.start, 1419 di.len, NULL, NULL); 1420 } 1421 next: 1422 prev_dc = next_dc; 1423 if (!prev_dc) 1424 break; 1425 1426 node = rb_next(&prev_dc->rb_node); 1427 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1428 } 1429 } 1430 1431 static int __queue_discard_cmd(struct f2fs_sb_info *sbi, 1432 struct block_device *bdev, block_t blkstart, block_t blklen) 1433 { 1434 block_t lblkstart = blkstart; 1435 1436 if (!f2fs_bdev_support_discard(bdev)) 1437 return 0; 1438 1439 trace_f2fs_queue_discard(bdev, blkstart, blklen); 1440 1441 if (f2fs_is_multi_device(sbi)) { 1442 int devi = f2fs_target_device_index(sbi, blkstart); 1443 1444 blkstart -= FDEV(devi).start_blk; 1445 } 1446 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); 1447 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); 1448 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1449 return 0; 1450 } 1451 1452 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, 1453 struct discard_policy *dpolicy) 1454 { 1455 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1456 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1457 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1458 struct discard_cmd *dc; 1459 struct blk_plug plug; 1460 unsigned int pos = dcc->next_pos; 1461 unsigned int issued = 0; 1462 bool io_interrupted = false; 1463 1464 mutex_lock(&dcc->cmd_lock); 1465 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 1466 NULL, pos, 1467 (struct rb_entry **)&prev_dc, 1468 (struct rb_entry **)&next_dc, 1469 &insert_p, &insert_parent, true, NULL); 1470 if (!dc) 1471 dc = next_dc; 1472 1473 blk_start_plug(&plug); 1474 1475 while (dc) { 1476 struct rb_node *node; 1477 int err = 0; 1478 1479 if (dc->state != D_PREP) 1480 goto next; 1481 1482 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { 1483 io_interrupted = true; 1484 break; 1485 } 1486 1487 dcc->next_pos = dc->lstart + dc->len; 1488 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1489 1490 if (issued >= dpolicy->max_requests) 1491 break; 1492 next: 1493 node = rb_next(&dc->rb_node); 1494 if (err) 1495 __remove_discard_cmd(sbi, dc); 1496 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1497 } 1498 1499 blk_finish_plug(&plug); 1500 1501 if (!dc) 1502 dcc->next_pos = 0; 1503 1504 mutex_unlock(&dcc->cmd_lock); 1505 1506 if (!issued && io_interrupted) 1507 issued = -1; 1508 1509 return issued; 1510 } 1511 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1512 struct discard_policy *dpolicy); 1513 1514 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, 1515 struct discard_policy *dpolicy) 1516 { 1517 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1518 struct list_head *pend_list; 1519 struct discard_cmd *dc, *tmp; 1520 struct blk_plug plug; 1521 int i, issued; 1522 bool io_interrupted = false; 1523 1524 if (dpolicy->timeout) 1525 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); 1526 1527 retry: 1528 issued = 0; 1529 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1530 if (dpolicy->timeout && 1531 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1532 break; 1533 1534 if (i + 1 < dpolicy->granularity) 1535 break; 1536 1537 if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered) 1538 return __issue_discard_cmd_orderly(sbi, dpolicy); 1539 1540 pend_list = &dcc->pend_list[i]; 1541 1542 mutex_lock(&dcc->cmd_lock); 1543 if (list_empty(pend_list)) 1544 goto next; 1545 if (unlikely(dcc->rbtree_check)) 1546 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, 1547 &dcc->root, false)); 1548 blk_start_plug(&plug); 1549 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1550 f2fs_bug_on(sbi, dc->state != D_PREP); 1551 1552 if (dpolicy->timeout && 1553 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1554 break; 1555 1556 if (dpolicy->io_aware && i < dpolicy->io_aware_gran && 1557 !is_idle(sbi, DISCARD_TIME)) { 1558 io_interrupted = true; 1559 break; 1560 } 1561 1562 __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1563 1564 if (issued >= dpolicy->max_requests) 1565 break; 1566 } 1567 blk_finish_plug(&plug); 1568 next: 1569 mutex_unlock(&dcc->cmd_lock); 1570 1571 if (issued >= dpolicy->max_requests || io_interrupted) 1572 break; 1573 } 1574 1575 if (dpolicy->type == DPOLICY_UMOUNT && issued) { 1576 __wait_all_discard_cmd(sbi, dpolicy); 1577 goto retry; 1578 } 1579 1580 if (!issued && io_interrupted) 1581 issued = -1; 1582 1583 return issued; 1584 } 1585 1586 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) 1587 { 1588 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1589 struct list_head *pend_list; 1590 struct discard_cmd *dc, *tmp; 1591 int i; 1592 bool dropped = false; 1593 1594 mutex_lock(&dcc->cmd_lock); 1595 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1596 pend_list = &dcc->pend_list[i]; 1597 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1598 f2fs_bug_on(sbi, dc->state != D_PREP); 1599 __remove_discard_cmd(sbi, dc); 1600 dropped = true; 1601 } 1602 } 1603 mutex_unlock(&dcc->cmd_lock); 1604 1605 return dropped; 1606 } 1607 1608 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) 1609 { 1610 __drop_discard_cmd(sbi); 1611 } 1612 1613 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, 1614 struct discard_cmd *dc) 1615 { 1616 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1617 unsigned int len = 0; 1618 1619 wait_for_completion_io(&dc->wait); 1620 mutex_lock(&dcc->cmd_lock); 1621 f2fs_bug_on(sbi, dc->state != D_DONE); 1622 dc->ref--; 1623 if (!dc->ref) { 1624 if (!dc->error) 1625 len = dc->len; 1626 __remove_discard_cmd(sbi, dc); 1627 } 1628 mutex_unlock(&dcc->cmd_lock); 1629 1630 return len; 1631 } 1632 1633 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, 1634 struct discard_policy *dpolicy, 1635 block_t start, block_t end) 1636 { 1637 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1638 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1639 &(dcc->fstrim_list) : &(dcc->wait_list); 1640 struct discard_cmd *dc, *tmp; 1641 bool need_wait; 1642 unsigned int trimmed = 0; 1643 1644 next: 1645 need_wait = false; 1646 1647 mutex_lock(&dcc->cmd_lock); 1648 list_for_each_entry_safe(dc, tmp, wait_list, list) { 1649 if (dc->lstart + dc->len <= start || end <= dc->lstart) 1650 continue; 1651 if (dc->len < dpolicy->granularity) 1652 continue; 1653 if (dc->state == D_DONE && !dc->ref) { 1654 wait_for_completion_io(&dc->wait); 1655 if (!dc->error) 1656 trimmed += dc->len; 1657 __remove_discard_cmd(sbi, dc); 1658 } else { 1659 dc->ref++; 1660 need_wait = true; 1661 break; 1662 } 1663 } 1664 mutex_unlock(&dcc->cmd_lock); 1665 1666 if (need_wait) { 1667 trimmed += __wait_one_discard_bio(sbi, dc); 1668 goto next; 1669 } 1670 1671 return trimmed; 1672 } 1673 1674 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1675 struct discard_policy *dpolicy) 1676 { 1677 struct discard_policy dp; 1678 unsigned int discard_blks; 1679 1680 if (dpolicy) 1681 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); 1682 1683 /* wait all */ 1684 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); 1685 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1686 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); 1687 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1688 1689 return discard_blks; 1690 } 1691 1692 /* This should be covered by global mutex, &sit_i->sentry_lock */ 1693 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) 1694 { 1695 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1696 struct discard_cmd *dc; 1697 bool need_wait = false; 1698 1699 mutex_lock(&dcc->cmd_lock); 1700 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root, 1701 NULL, blkaddr); 1702 if (dc) { 1703 if (dc->state == D_PREP) { 1704 __punch_discard_cmd(sbi, dc, blkaddr); 1705 } else { 1706 dc->ref++; 1707 need_wait = true; 1708 } 1709 } 1710 mutex_unlock(&dcc->cmd_lock); 1711 1712 if (need_wait) 1713 __wait_one_discard_bio(sbi, dc); 1714 } 1715 1716 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) 1717 { 1718 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1719 1720 if (dcc && dcc->f2fs_issue_discard) { 1721 struct task_struct *discard_thread = dcc->f2fs_issue_discard; 1722 1723 dcc->f2fs_issue_discard = NULL; 1724 kthread_stop(discard_thread); 1725 } 1726 } 1727 1728 /* This comes from f2fs_put_super */ 1729 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) 1730 { 1731 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1732 struct discard_policy dpolicy; 1733 bool dropped; 1734 1735 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, 1736 dcc->discard_granularity); 1737 __issue_discard_cmd(sbi, &dpolicy); 1738 dropped = __drop_discard_cmd(sbi); 1739 1740 /* just to make sure there is no pending discard commands */ 1741 __wait_all_discard_cmd(sbi, NULL); 1742 1743 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); 1744 return dropped; 1745 } 1746 1747 static int issue_discard_thread(void *data) 1748 { 1749 struct f2fs_sb_info *sbi = data; 1750 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1751 wait_queue_head_t *q = &dcc->discard_wait_queue; 1752 struct discard_policy dpolicy; 1753 unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME; 1754 int issued; 1755 1756 set_freezable(); 1757 1758 do { 1759 if (sbi->gc_mode == GC_URGENT_HIGH || 1760 !f2fs_available_free_memory(sbi, DISCARD_CACHE)) 1761 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); 1762 else 1763 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, 1764 dcc->discard_granularity); 1765 1766 if (!atomic_read(&dcc->discard_cmd_cnt)) 1767 wait_ms = dpolicy.max_interval; 1768 1769 wait_event_interruptible_timeout(*q, 1770 kthread_should_stop() || freezing(current) || 1771 dcc->discard_wake, 1772 msecs_to_jiffies(wait_ms)); 1773 1774 if (dcc->discard_wake) 1775 dcc->discard_wake = 0; 1776 1777 /* clean up pending candidates before going to sleep */ 1778 if (atomic_read(&dcc->queued_discard)) 1779 __wait_all_discard_cmd(sbi, NULL); 1780 1781 if (try_to_freeze()) 1782 continue; 1783 if (f2fs_readonly(sbi->sb)) 1784 continue; 1785 if (kthread_should_stop()) 1786 return 0; 1787 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 1788 wait_ms = dpolicy.max_interval; 1789 continue; 1790 } 1791 if (!atomic_read(&dcc->discard_cmd_cnt)) 1792 continue; 1793 1794 sb_start_intwrite(sbi->sb); 1795 1796 issued = __issue_discard_cmd(sbi, &dpolicy); 1797 if (issued > 0) { 1798 __wait_all_discard_cmd(sbi, &dpolicy); 1799 wait_ms = dpolicy.min_interval; 1800 } else if (issued == -1) { 1801 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); 1802 if (!wait_ms) 1803 wait_ms = dpolicy.mid_interval; 1804 } else { 1805 wait_ms = dpolicy.max_interval; 1806 } 1807 1808 sb_end_intwrite(sbi->sb); 1809 1810 } while (!kthread_should_stop()); 1811 return 0; 1812 } 1813 1814 #ifdef CONFIG_BLK_DEV_ZONED 1815 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, 1816 struct block_device *bdev, block_t blkstart, block_t blklen) 1817 { 1818 sector_t sector, nr_sects; 1819 block_t lblkstart = blkstart; 1820 int devi = 0; 1821 1822 if (f2fs_is_multi_device(sbi)) { 1823 devi = f2fs_target_device_index(sbi, blkstart); 1824 if (blkstart < FDEV(devi).start_blk || 1825 blkstart > FDEV(devi).end_blk) { 1826 f2fs_err(sbi, "Invalid block %x", blkstart); 1827 return -EIO; 1828 } 1829 blkstart -= FDEV(devi).start_blk; 1830 } 1831 1832 /* For sequential zones, reset the zone write pointer */ 1833 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { 1834 sector = SECTOR_FROM_BLOCK(blkstart); 1835 nr_sects = SECTOR_FROM_BLOCK(blklen); 1836 1837 if (sector & (bdev_zone_sectors(bdev) - 1) || 1838 nr_sects != bdev_zone_sectors(bdev)) { 1839 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", 1840 devi, sbi->s_ndevs ? FDEV(devi).path : "", 1841 blkstart, blklen); 1842 return -EIO; 1843 } 1844 trace_f2fs_issue_reset_zone(bdev, blkstart); 1845 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 1846 sector, nr_sects, GFP_NOFS); 1847 } 1848 1849 /* For conventional zones, use regular discard if supported */ 1850 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen); 1851 } 1852 #endif 1853 1854 static int __issue_discard_async(struct f2fs_sb_info *sbi, 1855 struct block_device *bdev, block_t blkstart, block_t blklen) 1856 { 1857 #ifdef CONFIG_BLK_DEV_ZONED 1858 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) 1859 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); 1860 #endif 1861 return __queue_discard_cmd(sbi, bdev, blkstart, blklen); 1862 } 1863 1864 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, 1865 block_t blkstart, block_t blklen) 1866 { 1867 sector_t start = blkstart, len = 0; 1868 struct block_device *bdev; 1869 struct seg_entry *se; 1870 unsigned int offset; 1871 block_t i; 1872 int err = 0; 1873 1874 bdev = f2fs_target_device(sbi, blkstart, NULL); 1875 1876 for (i = blkstart; i < blkstart + blklen; i++, len++) { 1877 if (i != start) { 1878 struct block_device *bdev2 = 1879 f2fs_target_device(sbi, i, NULL); 1880 1881 if (bdev2 != bdev) { 1882 err = __issue_discard_async(sbi, bdev, 1883 start, len); 1884 if (err) 1885 return err; 1886 bdev = bdev2; 1887 start = i; 1888 len = 0; 1889 } 1890 } 1891 1892 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); 1893 offset = GET_BLKOFF_FROM_SEG0(sbi, i); 1894 1895 if (!f2fs_test_and_set_bit(offset, se->discard_map)) 1896 sbi->discard_blks--; 1897 } 1898 1899 if (len) 1900 err = __issue_discard_async(sbi, bdev, start, len); 1901 return err; 1902 } 1903 1904 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, 1905 bool check_only) 1906 { 1907 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 1908 int max_blocks = sbi->blocks_per_seg; 1909 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); 1910 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 1911 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 1912 unsigned long *discard_map = (unsigned long *)se->discard_map; 1913 unsigned long *dmap = SIT_I(sbi)->tmp_map; 1914 unsigned int start = 0, end = -1; 1915 bool force = (cpc->reason & CP_DISCARD); 1916 struct discard_entry *de = NULL; 1917 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; 1918 int i; 1919 1920 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi)) 1921 return false; 1922 1923 if (!force) { 1924 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks || 1925 SM_I(sbi)->dcc_info->nr_discards >= 1926 SM_I(sbi)->dcc_info->max_discards) 1927 return false; 1928 } 1929 1930 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 1931 for (i = 0; i < entries; i++) 1932 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : 1933 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 1934 1935 while (force || SM_I(sbi)->dcc_info->nr_discards <= 1936 SM_I(sbi)->dcc_info->max_discards) { 1937 start = __find_rev_next_bit(dmap, max_blocks, end + 1); 1938 if (start >= max_blocks) 1939 break; 1940 1941 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); 1942 if (force && start && end != max_blocks 1943 && (end - start) < cpc->trim_minlen) 1944 continue; 1945 1946 if (check_only) 1947 return true; 1948 1949 if (!de) { 1950 de = f2fs_kmem_cache_alloc(discard_entry_slab, 1951 GFP_F2FS_ZERO); 1952 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); 1953 list_add_tail(&de->list, head); 1954 } 1955 1956 for (i = start; i < end; i++) 1957 __set_bit_le(i, (void *)de->discard_map); 1958 1959 SM_I(sbi)->dcc_info->nr_discards += end - start; 1960 } 1961 return false; 1962 } 1963 1964 static void release_discard_addr(struct discard_entry *entry) 1965 { 1966 list_del(&entry->list); 1967 kmem_cache_free(discard_entry_slab, entry); 1968 } 1969 1970 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) 1971 { 1972 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); 1973 struct discard_entry *entry, *this; 1974 1975 /* drop caches */ 1976 list_for_each_entry_safe(entry, this, head, list) 1977 release_discard_addr(entry); 1978 } 1979 1980 /* 1981 * Should call f2fs_clear_prefree_segments after checkpoint is done. 1982 */ 1983 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 1984 { 1985 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1986 unsigned int segno; 1987 1988 mutex_lock(&dirty_i->seglist_lock); 1989 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) 1990 __set_test_and_free(sbi, segno, false); 1991 mutex_unlock(&dirty_i->seglist_lock); 1992 } 1993 1994 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, 1995 struct cp_control *cpc) 1996 { 1997 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1998 struct list_head *head = &dcc->entry_list; 1999 struct discard_entry *entry, *this; 2000 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2001 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 2002 unsigned int start = 0, end = -1; 2003 unsigned int secno, start_segno; 2004 bool force = (cpc->reason & CP_DISCARD); 2005 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); 2006 2007 mutex_lock(&dirty_i->seglist_lock); 2008 2009 while (1) { 2010 int i; 2011 2012 if (need_align && end != -1) 2013 end--; 2014 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); 2015 if (start >= MAIN_SEGS(sbi)) 2016 break; 2017 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), 2018 start + 1); 2019 2020 if (need_align) { 2021 start = rounddown(start, sbi->segs_per_sec); 2022 end = roundup(end, sbi->segs_per_sec); 2023 } 2024 2025 for (i = start; i < end; i++) { 2026 if (test_and_clear_bit(i, prefree_map)) 2027 dirty_i->nr_dirty[PRE]--; 2028 } 2029 2030 if (!f2fs_realtime_discard_enable(sbi)) 2031 continue; 2032 2033 if (force && start >= cpc->trim_start && 2034 (end - 1) <= cpc->trim_end) 2035 continue; 2036 2037 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) { 2038 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 2039 (end - start) << sbi->log_blocks_per_seg); 2040 continue; 2041 } 2042 next: 2043 secno = GET_SEC_FROM_SEG(sbi, start); 2044 start_segno = GET_SEG_FROM_SEC(sbi, secno); 2045 if (!IS_CURSEC(sbi, secno) && 2046 !get_valid_blocks(sbi, start, true)) 2047 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), 2048 sbi->segs_per_sec << sbi->log_blocks_per_seg); 2049 2050 start = start_segno + sbi->segs_per_sec; 2051 if (start < end) 2052 goto next; 2053 else 2054 end = start - 1; 2055 } 2056 mutex_unlock(&dirty_i->seglist_lock); 2057 2058 /* send small discards */ 2059 list_for_each_entry_safe(entry, this, head, list) { 2060 unsigned int cur_pos = 0, next_pos, len, total_len = 0; 2061 bool is_valid = test_bit_le(0, entry->discard_map); 2062 2063 find_next: 2064 if (is_valid) { 2065 next_pos = find_next_zero_bit_le(entry->discard_map, 2066 sbi->blocks_per_seg, cur_pos); 2067 len = next_pos - cur_pos; 2068 2069 if (f2fs_sb_has_blkzoned(sbi) || 2070 (force && len < cpc->trim_minlen)) 2071 goto skip; 2072 2073 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, 2074 len); 2075 total_len += len; 2076 } else { 2077 next_pos = find_next_bit_le(entry->discard_map, 2078 sbi->blocks_per_seg, cur_pos); 2079 } 2080 skip: 2081 cur_pos = next_pos; 2082 is_valid = !is_valid; 2083 2084 if (cur_pos < sbi->blocks_per_seg) 2085 goto find_next; 2086 2087 release_discard_addr(entry); 2088 dcc->nr_discards -= total_len; 2089 } 2090 2091 wake_up_discard_thread(sbi, false); 2092 } 2093 2094 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) 2095 { 2096 dev_t dev = sbi->sb->s_bdev->bd_dev; 2097 struct discard_cmd_control *dcc; 2098 int err = 0, i; 2099 2100 if (SM_I(sbi)->dcc_info) { 2101 dcc = SM_I(sbi)->dcc_info; 2102 goto init_thread; 2103 } 2104 2105 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); 2106 if (!dcc) 2107 return -ENOMEM; 2108 2109 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; 2110 INIT_LIST_HEAD(&dcc->entry_list); 2111 for (i = 0; i < MAX_PLIST_NUM; i++) 2112 INIT_LIST_HEAD(&dcc->pend_list[i]); 2113 INIT_LIST_HEAD(&dcc->wait_list); 2114 INIT_LIST_HEAD(&dcc->fstrim_list); 2115 mutex_init(&dcc->cmd_lock); 2116 atomic_set(&dcc->issued_discard, 0); 2117 atomic_set(&dcc->queued_discard, 0); 2118 atomic_set(&dcc->discard_cmd_cnt, 0); 2119 dcc->nr_discards = 0; 2120 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; 2121 dcc->undiscard_blks = 0; 2122 dcc->next_pos = 0; 2123 dcc->root = RB_ROOT_CACHED; 2124 dcc->rbtree_check = false; 2125 2126 init_waitqueue_head(&dcc->discard_wait_queue); 2127 SM_I(sbi)->dcc_info = dcc; 2128 init_thread: 2129 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, 2130 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); 2131 if (IS_ERR(dcc->f2fs_issue_discard)) { 2132 err = PTR_ERR(dcc->f2fs_issue_discard); 2133 kfree(dcc); 2134 SM_I(sbi)->dcc_info = NULL; 2135 return err; 2136 } 2137 2138 return err; 2139 } 2140 2141 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) 2142 { 2143 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2144 2145 if (!dcc) 2146 return; 2147 2148 f2fs_stop_discard_thread(sbi); 2149 2150 /* 2151 * Recovery can cache discard commands, so in error path of 2152 * fill_super(), it needs to give a chance to handle them. 2153 */ 2154 if (unlikely(atomic_read(&dcc->discard_cmd_cnt))) 2155 f2fs_issue_discard_timeout(sbi); 2156 2157 kfree(dcc); 2158 SM_I(sbi)->dcc_info = NULL; 2159 } 2160 2161 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 2162 { 2163 struct sit_info *sit_i = SIT_I(sbi); 2164 2165 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { 2166 sit_i->dirty_sentries++; 2167 return false; 2168 } 2169 2170 return true; 2171 } 2172 2173 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 2174 unsigned int segno, int modified) 2175 { 2176 struct seg_entry *se = get_seg_entry(sbi, segno); 2177 2178 se->type = type; 2179 if (modified) 2180 __mark_sit_entry_dirty(sbi, segno); 2181 } 2182 2183 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi, 2184 block_t blkaddr) 2185 { 2186 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2187 2188 if (segno == NULL_SEGNO) 2189 return 0; 2190 return get_seg_entry(sbi, segno)->mtime; 2191 } 2192 2193 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr, 2194 unsigned long long old_mtime) 2195 { 2196 struct seg_entry *se; 2197 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2198 unsigned long long ctime = get_mtime(sbi, false); 2199 unsigned long long mtime = old_mtime ? old_mtime : ctime; 2200 2201 if (segno == NULL_SEGNO) 2202 return; 2203 2204 se = get_seg_entry(sbi, segno); 2205 2206 if (!se->mtime) 2207 se->mtime = mtime; 2208 else 2209 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime, 2210 se->valid_blocks + 1); 2211 2212 if (ctime > SIT_I(sbi)->max_mtime) 2213 SIT_I(sbi)->max_mtime = ctime; 2214 } 2215 2216 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 2217 { 2218 struct seg_entry *se; 2219 unsigned int segno, offset; 2220 long int new_vblocks; 2221 bool exist; 2222 #ifdef CONFIG_F2FS_CHECK_FS 2223 bool mir_exist; 2224 #endif 2225 2226 segno = GET_SEGNO(sbi, blkaddr); 2227 2228 se = get_seg_entry(sbi, segno); 2229 new_vblocks = se->valid_blocks + del; 2230 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2231 2232 f2fs_bug_on(sbi, (new_vblocks < 0 || 2233 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno)))); 2234 2235 se->valid_blocks = new_vblocks; 2236 2237 /* Update valid block bitmap */ 2238 if (del > 0) { 2239 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); 2240 #ifdef CONFIG_F2FS_CHECK_FS 2241 mir_exist = f2fs_test_and_set_bit(offset, 2242 se->cur_valid_map_mir); 2243 if (unlikely(exist != mir_exist)) { 2244 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d", 2245 blkaddr, exist); 2246 f2fs_bug_on(sbi, 1); 2247 } 2248 #endif 2249 if (unlikely(exist)) { 2250 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", 2251 blkaddr); 2252 f2fs_bug_on(sbi, 1); 2253 se->valid_blocks--; 2254 del = 0; 2255 } 2256 2257 if (!f2fs_test_and_set_bit(offset, se->discard_map)) 2258 sbi->discard_blks--; 2259 2260 /* 2261 * SSR should never reuse block which is checkpointed 2262 * or newly invalidated. 2263 */ 2264 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { 2265 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map)) 2266 se->ckpt_valid_blocks++; 2267 } 2268 } else { 2269 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map); 2270 #ifdef CONFIG_F2FS_CHECK_FS 2271 mir_exist = f2fs_test_and_clear_bit(offset, 2272 se->cur_valid_map_mir); 2273 if (unlikely(exist != mir_exist)) { 2274 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d", 2275 blkaddr, exist); 2276 f2fs_bug_on(sbi, 1); 2277 } 2278 #endif 2279 if (unlikely(!exist)) { 2280 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", 2281 blkaddr); 2282 f2fs_bug_on(sbi, 1); 2283 se->valid_blocks++; 2284 del = 0; 2285 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2286 /* 2287 * If checkpoints are off, we must not reuse data that 2288 * was used in the previous checkpoint. If it was used 2289 * before, we must track that to know how much space we 2290 * really have. 2291 */ 2292 if (f2fs_test_bit(offset, se->ckpt_valid_map)) { 2293 spin_lock(&sbi->stat_lock); 2294 sbi->unusable_block_count++; 2295 spin_unlock(&sbi->stat_lock); 2296 } 2297 } 2298 2299 if (f2fs_test_and_clear_bit(offset, se->discard_map)) 2300 sbi->discard_blks++; 2301 } 2302 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 2303 se->ckpt_valid_blocks += del; 2304 2305 __mark_sit_entry_dirty(sbi, segno); 2306 2307 /* update total number of valid blocks to be written in ckpt area */ 2308 SIT_I(sbi)->written_valid_blocks += del; 2309 2310 if (__is_large_section(sbi)) 2311 get_sec_entry(sbi, segno)->valid_blocks += del; 2312 } 2313 2314 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) 2315 { 2316 unsigned int segno = GET_SEGNO(sbi, addr); 2317 struct sit_info *sit_i = SIT_I(sbi); 2318 2319 f2fs_bug_on(sbi, addr == NULL_ADDR); 2320 if (addr == NEW_ADDR || addr == COMPRESS_ADDR) 2321 return; 2322 2323 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr); 2324 2325 /* add it into sit main buffer */ 2326 down_write(&sit_i->sentry_lock); 2327 2328 update_segment_mtime(sbi, addr, 0); 2329 update_sit_entry(sbi, addr, -1); 2330 2331 /* add it into dirty seglist */ 2332 locate_dirty_segment(sbi, segno); 2333 2334 up_write(&sit_i->sentry_lock); 2335 } 2336 2337 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) 2338 { 2339 struct sit_info *sit_i = SIT_I(sbi); 2340 unsigned int segno, offset; 2341 struct seg_entry *se; 2342 bool is_cp = false; 2343 2344 if (!__is_valid_data_blkaddr(blkaddr)) 2345 return true; 2346 2347 down_read(&sit_i->sentry_lock); 2348 2349 segno = GET_SEGNO(sbi, blkaddr); 2350 se = get_seg_entry(sbi, segno); 2351 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2352 2353 if (f2fs_test_bit(offset, se->ckpt_valid_map)) 2354 is_cp = true; 2355 2356 up_read(&sit_i->sentry_lock); 2357 2358 return is_cp; 2359 } 2360 2361 /* 2362 * This function should be resided under the curseg_mutex lock 2363 */ 2364 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, 2365 struct f2fs_summary *sum) 2366 { 2367 struct curseg_info *curseg = CURSEG_I(sbi, type); 2368 void *addr = curseg->sum_blk; 2369 2370 addr += curseg->next_blkoff * sizeof(struct f2fs_summary); 2371 memcpy(addr, sum, sizeof(struct f2fs_summary)); 2372 } 2373 2374 /* 2375 * Calculate the number of current summary pages for writing 2376 */ 2377 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) 2378 { 2379 int valid_sum_count = 0; 2380 int i, sum_in_page; 2381 2382 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2383 if (sbi->ckpt->alloc_type[i] == SSR) 2384 valid_sum_count += sbi->blocks_per_seg; 2385 else { 2386 if (for_ra) 2387 valid_sum_count += le16_to_cpu( 2388 F2FS_CKPT(sbi)->cur_data_blkoff[i]); 2389 else 2390 valid_sum_count += curseg_blkoff(sbi, i); 2391 } 2392 } 2393 2394 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - 2395 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 2396 if (valid_sum_count <= sum_in_page) 2397 return 1; 2398 else if ((valid_sum_count - sum_in_page) <= 2399 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 2400 return 2; 2401 return 3; 2402 } 2403 2404 /* 2405 * Caller should put this summary page 2406 */ 2407 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) 2408 { 2409 if (unlikely(f2fs_cp_error(sbi))) 2410 return ERR_PTR(-EIO); 2411 return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno)); 2412 } 2413 2414 void f2fs_update_meta_page(struct f2fs_sb_info *sbi, 2415 void *src, block_t blk_addr) 2416 { 2417 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); 2418 2419 memcpy(page_address(page), src, PAGE_SIZE); 2420 set_page_dirty(page); 2421 f2fs_put_page(page, 1); 2422 } 2423 2424 static void write_sum_page(struct f2fs_sb_info *sbi, 2425 struct f2fs_summary_block *sum_blk, block_t blk_addr) 2426 { 2427 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr); 2428 } 2429 2430 static void write_current_sum_page(struct f2fs_sb_info *sbi, 2431 int type, block_t blk_addr) 2432 { 2433 struct curseg_info *curseg = CURSEG_I(sbi, type); 2434 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); 2435 struct f2fs_summary_block *src = curseg->sum_blk; 2436 struct f2fs_summary_block *dst; 2437 2438 dst = (struct f2fs_summary_block *)page_address(page); 2439 memset(dst, 0, PAGE_SIZE); 2440 2441 mutex_lock(&curseg->curseg_mutex); 2442 2443 down_read(&curseg->journal_rwsem); 2444 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); 2445 up_read(&curseg->journal_rwsem); 2446 2447 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); 2448 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); 2449 2450 mutex_unlock(&curseg->curseg_mutex); 2451 2452 set_page_dirty(page); 2453 f2fs_put_page(page, 1); 2454 } 2455 2456 static int is_next_segment_free(struct f2fs_sb_info *sbi, 2457 struct curseg_info *curseg, int type) 2458 { 2459 unsigned int segno = curseg->segno + 1; 2460 struct free_segmap_info *free_i = FREE_I(sbi); 2461 2462 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) 2463 return !test_bit(segno, free_i->free_segmap); 2464 return 0; 2465 } 2466 2467 /* 2468 * Find a new segment from the free segments bitmap to right order 2469 * This function should be returned with success, otherwise BUG 2470 */ 2471 static void get_new_segment(struct f2fs_sb_info *sbi, 2472 unsigned int *newseg, bool new_sec, int dir) 2473 { 2474 struct free_segmap_info *free_i = FREE_I(sbi); 2475 unsigned int segno, secno, zoneno; 2476 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; 2477 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); 2478 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); 2479 unsigned int left_start = hint; 2480 bool init = true; 2481 int go_left = 0; 2482 int i; 2483 2484 spin_lock(&free_i->segmap_lock); 2485 2486 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { 2487 segno = find_next_zero_bit(free_i->free_segmap, 2488 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); 2489 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) 2490 goto got_it; 2491 } 2492 find_other_zone: 2493 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); 2494 if (secno >= MAIN_SECS(sbi)) { 2495 if (dir == ALLOC_RIGHT) { 2496 secno = find_next_zero_bit(free_i->free_secmap, 2497 MAIN_SECS(sbi), 0); 2498 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); 2499 } else { 2500 go_left = 1; 2501 left_start = hint - 1; 2502 } 2503 } 2504 if (go_left == 0) 2505 goto skip_left; 2506 2507 while (test_bit(left_start, free_i->free_secmap)) { 2508 if (left_start > 0) { 2509 left_start--; 2510 continue; 2511 } 2512 left_start = find_next_zero_bit(free_i->free_secmap, 2513 MAIN_SECS(sbi), 0); 2514 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); 2515 break; 2516 } 2517 secno = left_start; 2518 skip_left: 2519 segno = GET_SEG_FROM_SEC(sbi, secno); 2520 zoneno = GET_ZONE_FROM_SEC(sbi, secno); 2521 2522 /* give up on finding another zone */ 2523 if (!init) 2524 goto got_it; 2525 if (sbi->secs_per_zone == 1) 2526 goto got_it; 2527 if (zoneno == old_zoneno) 2528 goto got_it; 2529 if (dir == ALLOC_LEFT) { 2530 if (!go_left && zoneno + 1 >= total_zones) 2531 goto got_it; 2532 if (go_left && zoneno == 0) 2533 goto got_it; 2534 } 2535 for (i = 0; i < NR_CURSEG_TYPE; i++) 2536 if (CURSEG_I(sbi, i)->zone == zoneno) 2537 break; 2538 2539 if (i < NR_CURSEG_TYPE) { 2540 /* zone is in user, try another */ 2541 if (go_left) 2542 hint = zoneno * sbi->secs_per_zone - 1; 2543 else if (zoneno + 1 >= total_zones) 2544 hint = 0; 2545 else 2546 hint = (zoneno + 1) * sbi->secs_per_zone; 2547 init = false; 2548 goto find_other_zone; 2549 } 2550 got_it: 2551 /* set it as dirty segment in free segmap */ 2552 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); 2553 __set_inuse(sbi, segno); 2554 *newseg = segno; 2555 spin_unlock(&free_i->segmap_lock); 2556 } 2557 2558 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 2559 { 2560 struct curseg_info *curseg = CURSEG_I(sbi, type); 2561 struct summary_footer *sum_footer; 2562 unsigned short seg_type = curseg->seg_type; 2563 2564 curseg->inited = true; 2565 curseg->segno = curseg->next_segno; 2566 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); 2567 curseg->next_blkoff = 0; 2568 curseg->next_segno = NULL_SEGNO; 2569 2570 sum_footer = &(curseg->sum_blk->footer); 2571 memset(sum_footer, 0, sizeof(struct summary_footer)); 2572 2573 sanity_check_seg_type(sbi, seg_type); 2574 2575 if (IS_DATASEG(seg_type)) 2576 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 2577 if (IS_NODESEG(seg_type)) 2578 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 2579 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified); 2580 } 2581 2582 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) 2583 { 2584 struct curseg_info *curseg = CURSEG_I(sbi, type); 2585 unsigned short seg_type = curseg->seg_type; 2586 2587 sanity_check_seg_type(sbi, seg_type); 2588 2589 /* if segs_per_sec is large than 1, we need to keep original policy. */ 2590 if (__is_large_section(sbi)) 2591 return curseg->segno; 2592 2593 /* inmem log may not locate on any segment after mount */ 2594 if (!curseg->inited) 2595 return 0; 2596 2597 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 2598 return 0; 2599 2600 if (test_opt(sbi, NOHEAP) && 2601 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type))) 2602 return 0; 2603 2604 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) 2605 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; 2606 2607 /* find segments from 0 to reuse freed segments */ 2608 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) 2609 return 0; 2610 2611 return curseg->segno; 2612 } 2613 2614 /* 2615 * Allocate a current working segment. 2616 * This function always allocates a free segment in LFS manner. 2617 */ 2618 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 2619 { 2620 struct curseg_info *curseg = CURSEG_I(sbi, type); 2621 unsigned short seg_type = curseg->seg_type; 2622 unsigned int segno = curseg->segno; 2623 int dir = ALLOC_LEFT; 2624 2625 if (curseg->inited) 2626 write_sum_page(sbi, curseg->sum_blk, 2627 GET_SUM_BLOCK(sbi, segno)); 2628 if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA) 2629 dir = ALLOC_RIGHT; 2630 2631 if (test_opt(sbi, NOHEAP)) 2632 dir = ALLOC_RIGHT; 2633 2634 segno = __get_next_segno(sbi, type); 2635 get_new_segment(sbi, &segno, new_sec, dir); 2636 curseg->next_segno = segno; 2637 reset_curseg(sbi, type, 1); 2638 curseg->alloc_type = LFS; 2639 } 2640 2641 static int __next_free_blkoff(struct f2fs_sb_info *sbi, 2642 int segno, block_t start) 2643 { 2644 struct seg_entry *se = get_seg_entry(sbi, segno); 2645 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 2646 unsigned long *target_map = SIT_I(sbi)->tmp_map; 2647 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 2648 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 2649 int i; 2650 2651 for (i = 0; i < entries; i++) 2652 target_map[i] = ckpt_map[i] | cur_map[i]; 2653 2654 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); 2655 } 2656 2657 /* 2658 * If a segment is written by LFS manner, next block offset is just obtained 2659 * by increasing the current block offset. However, if a segment is written by 2660 * SSR manner, next block offset obtained by calling __next_free_blkoff 2661 */ 2662 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, 2663 struct curseg_info *seg) 2664 { 2665 if (seg->alloc_type == SSR) 2666 seg->next_blkoff = 2667 __next_free_blkoff(sbi, seg->segno, 2668 seg->next_blkoff + 1); 2669 else 2670 seg->next_blkoff++; 2671 } 2672 2673 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) 2674 { 2675 return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg; 2676 } 2677 2678 /* 2679 * This function always allocates a used segment(from dirty seglist) by SSR 2680 * manner, so it should recover the existing segment information of valid blocks 2681 */ 2682 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush) 2683 { 2684 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2685 struct curseg_info *curseg = CURSEG_I(sbi, type); 2686 unsigned int new_segno = curseg->next_segno; 2687 struct f2fs_summary_block *sum_node; 2688 struct page *sum_page; 2689 2690 if (flush) 2691 write_sum_page(sbi, curseg->sum_blk, 2692 GET_SUM_BLOCK(sbi, curseg->segno)); 2693 2694 __set_test_and_inuse(sbi, new_segno); 2695 2696 mutex_lock(&dirty_i->seglist_lock); 2697 __remove_dirty_segment(sbi, new_segno, PRE); 2698 __remove_dirty_segment(sbi, new_segno, DIRTY); 2699 mutex_unlock(&dirty_i->seglist_lock); 2700 2701 reset_curseg(sbi, type, 1); 2702 curseg->alloc_type = SSR; 2703 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0); 2704 2705 sum_page = f2fs_get_sum_page(sbi, new_segno); 2706 if (IS_ERR(sum_page)) { 2707 /* GC won't be able to use stale summary pages by cp_error */ 2708 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE); 2709 return; 2710 } 2711 sum_node = (struct f2fs_summary_block *)page_address(sum_page); 2712 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); 2713 f2fs_put_page(sum_page, 1); 2714 } 2715 2716 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 2717 int alloc_mode, unsigned long long age); 2718 2719 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type, 2720 int target_type, int alloc_mode, 2721 unsigned long long age) 2722 { 2723 struct curseg_info *curseg = CURSEG_I(sbi, type); 2724 2725 curseg->seg_type = target_type; 2726 2727 if (get_ssr_segment(sbi, type, alloc_mode, age)) { 2728 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno); 2729 2730 curseg->seg_type = se->type; 2731 change_curseg(sbi, type, true); 2732 } else { 2733 /* allocate cold segment by default */ 2734 curseg->seg_type = CURSEG_COLD_DATA; 2735 new_curseg(sbi, type, true); 2736 } 2737 stat_inc_seg_type(sbi, curseg); 2738 } 2739 2740 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi) 2741 { 2742 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC); 2743 2744 if (!sbi->am.atgc_enabled) 2745 return; 2746 2747 down_read(&SM_I(sbi)->curseg_lock); 2748 2749 mutex_lock(&curseg->curseg_mutex); 2750 down_write(&SIT_I(sbi)->sentry_lock); 2751 2752 get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0); 2753 2754 up_write(&SIT_I(sbi)->sentry_lock); 2755 mutex_unlock(&curseg->curseg_mutex); 2756 2757 up_read(&SM_I(sbi)->curseg_lock); 2758 2759 } 2760 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi) 2761 { 2762 __f2fs_init_atgc_curseg(sbi); 2763 } 2764 2765 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) 2766 { 2767 struct curseg_info *curseg = CURSEG_I(sbi, type); 2768 2769 mutex_lock(&curseg->curseg_mutex); 2770 if (!curseg->inited) 2771 goto out; 2772 2773 if (get_valid_blocks(sbi, curseg->segno, false)) { 2774 write_sum_page(sbi, curseg->sum_blk, 2775 GET_SUM_BLOCK(sbi, curseg->segno)); 2776 } else { 2777 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 2778 __set_test_and_free(sbi, curseg->segno, true); 2779 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 2780 } 2781 out: 2782 mutex_unlock(&curseg->curseg_mutex); 2783 } 2784 2785 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi) 2786 { 2787 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 2788 2789 if (sbi->am.atgc_enabled) 2790 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 2791 } 2792 2793 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type) 2794 { 2795 struct curseg_info *curseg = CURSEG_I(sbi, type); 2796 2797 mutex_lock(&curseg->curseg_mutex); 2798 if (!curseg->inited) 2799 goto out; 2800 if (get_valid_blocks(sbi, curseg->segno, false)) 2801 goto out; 2802 2803 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 2804 __set_test_and_inuse(sbi, curseg->segno); 2805 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 2806 out: 2807 mutex_unlock(&curseg->curseg_mutex); 2808 } 2809 2810 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi) 2811 { 2812 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 2813 2814 if (sbi->am.atgc_enabled) 2815 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 2816 } 2817 2818 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 2819 int alloc_mode, unsigned long long age) 2820 { 2821 struct curseg_info *curseg = CURSEG_I(sbi, type); 2822 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; 2823 unsigned segno = NULL_SEGNO; 2824 unsigned short seg_type = curseg->seg_type; 2825 int i, cnt; 2826 bool reversed = false; 2827 2828 sanity_check_seg_type(sbi, seg_type); 2829 2830 /* f2fs_need_SSR() already forces to do this */ 2831 if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) { 2832 curseg->next_segno = segno; 2833 return 1; 2834 } 2835 2836 /* For node segments, let's do SSR more intensively */ 2837 if (IS_NODESEG(seg_type)) { 2838 if (seg_type >= CURSEG_WARM_NODE) { 2839 reversed = true; 2840 i = CURSEG_COLD_NODE; 2841 } else { 2842 i = CURSEG_HOT_NODE; 2843 } 2844 cnt = NR_CURSEG_NODE_TYPE; 2845 } else { 2846 if (seg_type >= CURSEG_WARM_DATA) { 2847 reversed = true; 2848 i = CURSEG_COLD_DATA; 2849 } else { 2850 i = CURSEG_HOT_DATA; 2851 } 2852 cnt = NR_CURSEG_DATA_TYPE; 2853 } 2854 2855 for (; cnt-- > 0; reversed ? i-- : i++) { 2856 if (i == seg_type) 2857 continue; 2858 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) { 2859 curseg->next_segno = segno; 2860 return 1; 2861 } 2862 } 2863 2864 /* find valid_blocks=0 in dirty list */ 2865 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2866 segno = get_free_segment(sbi); 2867 if (segno != NULL_SEGNO) { 2868 curseg->next_segno = segno; 2869 return 1; 2870 } 2871 } 2872 return 0; 2873 } 2874 2875 /* 2876 * flush out current segment and replace it with new segment 2877 * This function should be returned with success, otherwise BUG 2878 */ 2879 static void allocate_segment_by_default(struct f2fs_sb_info *sbi, 2880 int type, bool force) 2881 { 2882 struct curseg_info *curseg = CURSEG_I(sbi, type); 2883 2884 if (force) 2885 new_curseg(sbi, type, true); 2886 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && 2887 curseg->seg_type == CURSEG_WARM_NODE) 2888 new_curseg(sbi, type, false); 2889 else if (curseg->alloc_type == LFS && 2890 is_next_segment_free(sbi, curseg, type) && 2891 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 2892 new_curseg(sbi, type, false); 2893 else if (f2fs_need_SSR(sbi) && 2894 get_ssr_segment(sbi, type, SSR, 0)) 2895 change_curseg(sbi, type, true); 2896 else 2897 new_curseg(sbi, type, false); 2898 2899 stat_inc_seg_type(sbi, curseg); 2900 } 2901 2902 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, 2903 unsigned int start, unsigned int end) 2904 { 2905 struct curseg_info *curseg = CURSEG_I(sbi, type); 2906 unsigned int segno; 2907 2908 down_read(&SM_I(sbi)->curseg_lock); 2909 mutex_lock(&curseg->curseg_mutex); 2910 down_write(&SIT_I(sbi)->sentry_lock); 2911 2912 segno = CURSEG_I(sbi, type)->segno; 2913 if (segno < start || segno > end) 2914 goto unlock; 2915 2916 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0)) 2917 change_curseg(sbi, type, true); 2918 else 2919 new_curseg(sbi, type, true); 2920 2921 stat_inc_seg_type(sbi, curseg); 2922 2923 locate_dirty_segment(sbi, segno); 2924 unlock: 2925 up_write(&SIT_I(sbi)->sentry_lock); 2926 2927 if (segno != curseg->segno) 2928 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u", 2929 type, segno, curseg->segno); 2930 2931 mutex_unlock(&curseg->curseg_mutex); 2932 up_read(&SM_I(sbi)->curseg_lock); 2933 } 2934 2935 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type, 2936 bool new_sec, bool force) 2937 { 2938 struct curseg_info *curseg = CURSEG_I(sbi, type); 2939 unsigned int old_segno; 2940 2941 if (!curseg->inited) 2942 goto alloc; 2943 2944 if (force || curseg->next_blkoff || 2945 get_valid_blocks(sbi, curseg->segno, new_sec)) 2946 goto alloc; 2947 2948 if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) 2949 return; 2950 alloc: 2951 old_segno = curseg->segno; 2952 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true); 2953 locate_dirty_segment(sbi, old_segno); 2954 } 2955 2956 static void __allocate_new_section(struct f2fs_sb_info *sbi, 2957 int type, bool force) 2958 { 2959 __allocate_new_segment(sbi, type, true, force); 2960 } 2961 2962 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) 2963 { 2964 down_read(&SM_I(sbi)->curseg_lock); 2965 down_write(&SIT_I(sbi)->sentry_lock); 2966 __allocate_new_section(sbi, type, force); 2967 up_write(&SIT_I(sbi)->sentry_lock); 2968 up_read(&SM_I(sbi)->curseg_lock); 2969 } 2970 2971 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) 2972 { 2973 int i; 2974 2975 down_read(&SM_I(sbi)->curseg_lock); 2976 down_write(&SIT_I(sbi)->sentry_lock); 2977 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) 2978 __allocate_new_segment(sbi, i, false, false); 2979 up_write(&SIT_I(sbi)->sentry_lock); 2980 up_read(&SM_I(sbi)->curseg_lock); 2981 } 2982 2983 static const struct segment_allocation default_salloc_ops = { 2984 .allocate_segment = allocate_segment_by_default, 2985 }; 2986 2987 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, 2988 struct cp_control *cpc) 2989 { 2990 __u64 trim_start = cpc->trim_start; 2991 bool has_candidate = false; 2992 2993 down_write(&SIT_I(sbi)->sentry_lock); 2994 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { 2995 if (add_discard_addrs(sbi, cpc, true)) { 2996 has_candidate = true; 2997 break; 2998 } 2999 } 3000 up_write(&SIT_I(sbi)->sentry_lock); 3001 3002 cpc->trim_start = trim_start; 3003 return has_candidate; 3004 } 3005 3006 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, 3007 struct discard_policy *dpolicy, 3008 unsigned int start, unsigned int end) 3009 { 3010 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 3011 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 3012 struct rb_node **insert_p = NULL, *insert_parent = NULL; 3013 struct discard_cmd *dc; 3014 struct blk_plug plug; 3015 int issued; 3016 unsigned int trimmed = 0; 3017 3018 next: 3019 issued = 0; 3020 3021 mutex_lock(&dcc->cmd_lock); 3022 if (unlikely(dcc->rbtree_check)) 3023 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, 3024 &dcc->root, false)); 3025 3026 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 3027 NULL, start, 3028 (struct rb_entry **)&prev_dc, 3029 (struct rb_entry **)&next_dc, 3030 &insert_p, &insert_parent, true, NULL); 3031 if (!dc) 3032 dc = next_dc; 3033 3034 blk_start_plug(&plug); 3035 3036 while (dc && dc->lstart <= end) { 3037 struct rb_node *node; 3038 int err = 0; 3039 3040 if (dc->len < dpolicy->granularity) 3041 goto skip; 3042 3043 if (dc->state != D_PREP) { 3044 list_move_tail(&dc->list, &dcc->fstrim_list); 3045 goto skip; 3046 } 3047 3048 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 3049 3050 if (issued >= dpolicy->max_requests) { 3051 start = dc->lstart + dc->len; 3052 3053 if (err) 3054 __remove_discard_cmd(sbi, dc); 3055 3056 blk_finish_plug(&plug); 3057 mutex_unlock(&dcc->cmd_lock); 3058 trimmed += __wait_all_discard_cmd(sbi, NULL); 3059 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); 3060 goto next; 3061 } 3062 skip: 3063 node = rb_next(&dc->rb_node); 3064 if (err) 3065 __remove_discard_cmd(sbi, dc); 3066 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 3067 3068 if (fatal_signal_pending(current)) 3069 break; 3070 } 3071 3072 blk_finish_plug(&plug); 3073 mutex_unlock(&dcc->cmd_lock); 3074 3075 return trimmed; 3076 } 3077 3078 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) 3079 { 3080 __u64 start = F2FS_BYTES_TO_BLK(range->start); 3081 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; 3082 unsigned int start_segno, end_segno; 3083 block_t start_block, end_block; 3084 struct cp_control cpc; 3085 struct discard_policy dpolicy; 3086 unsigned long long trimmed = 0; 3087 int err = 0; 3088 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); 3089 3090 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) 3091 return -EINVAL; 3092 3093 if (end < MAIN_BLKADDR(sbi)) 3094 goto out; 3095 3096 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 3097 f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); 3098 return -EFSCORRUPTED; 3099 } 3100 3101 /* start/end segment number in main_area */ 3102 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); 3103 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : 3104 GET_SEGNO(sbi, end); 3105 if (need_align) { 3106 start_segno = rounddown(start_segno, sbi->segs_per_sec); 3107 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1; 3108 } 3109 3110 cpc.reason = CP_DISCARD; 3111 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); 3112 cpc.trim_start = start_segno; 3113 cpc.trim_end = end_segno; 3114 3115 if (sbi->discard_blks == 0) 3116 goto out; 3117 3118 down_write(&sbi->gc_lock); 3119 err = f2fs_write_checkpoint(sbi, &cpc); 3120 up_write(&sbi->gc_lock); 3121 if (err) 3122 goto out; 3123 3124 /* 3125 * We filed discard candidates, but actually we don't need to wait for 3126 * all of them, since they'll be issued in idle time along with runtime 3127 * discard option. User configuration looks like using runtime discard 3128 * or periodic fstrim instead of it. 3129 */ 3130 if (f2fs_realtime_discard_enable(sbi)) 3131 goto out; 3132 3133 start_block = START_BLOCK(sbi, start_segno); 3134 end_block = START_BLOCK(sbi, end_segno + 1); 3135 3136 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); 3137 trimmed = __issue_discard_cmd_range(sbi, &dpolicy, 3138 start_block, end_block); 3139 3140 trimmed += __wait_discard_cmd_range(sbi, &dpolicy, 3141 start_block, end_block); 3142 out: 3143 if (!err) 3144 range->len = F2FS_BLK_TO_BYTES(trimmed); 3145 return err; 3146 } 3147 3148 static bool __has_curseg_space(struct f2fs_sb_info *sbi, 3149 struct curseg_info *curseg) 3150 { 3151 return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi, 3152 curseg->segno); 3153 } 3154 3155 int f2fs_rw_hint_to_seg_type(enum rw_hint hint) 3156 { 3157 switch (hint) { 3158 case WRITE_LIFE_SHORT: 3159 return CURSEG_HOT_DATA; 3160 case WRITE_LIFE_EXTREME: 3161 return CURSEG_COLD_DATA; 3162 default: 3163 return CURSEG_WARM_DATA; 3164 } 3165 } 3166 3167 /* This returns write hints for each segment type. This hints will be 3168 * passed down to block layer. There are mapping tables which depend on 3169 * the mount option 'whint_mode'. 3170 * 3171 * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET. 3172 * 3173 * 2) whint_mode=user-based. F2FS tries to pass down hints given by users. 3174 * 3175 * User F2FS Block 3176 * ---- ---- ----- 3177 * META WRITE_LIFE_NOT_SET 3178 * HOT_NODE " 3179 * WARM_NODE " 3180 * COLD_NODE " 3181 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 3182 * extension list " " 3183 * 3184 * -- buffered io 3185 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 3186 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 3187 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 3188 * WRITE_LIFE_NONE " " 3189 * WRITE_LIFE_MEDIUM " " 3190 * WRITE_LIFE_LONG " " 3191 * 3192 * -- direct io 3193 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 3194 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 3195 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 3196 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 3197 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 3198 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 3199 * 3200 * 3) whint_mode=fs-based. F2FS passes down hints with its policy. 3201 * 3202 * User F2FS Block 3203 * ---- ---- ----- 3204 * META WRITE_LIFE_MEDIUM; 3205 * HOT_NODE WRITE_LIFE_NOT_SET 3206 * WARM_NODE " 3207 * COLD_NODE WRITE_LIFE_NONE 3208 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 3209 * extension list " " 3210 * 3211 * -- buffered io 3212 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 3213 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 3214 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_LONG 3215 * WRITE_LIFE_NONE " " 3216 * WRITE_LIFE_MEDIUM " " 3217 * WRITE_LIFE_LONG " " 3218 * 3219 * -- direct io 3220 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 3221 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 3222 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 3223 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 3224 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 3225 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 3226 */ 3227 3228 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, 3229 enum page_type type, enum temp_type temp) 3230 { 3231 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) { 3232 if (type == DATA) { 3233 if (temp == WARM) 3234 return WRITE_LIFE_NOT_SET; 3235 else if (temp == HOT) 3236 return WRITE_LIFE_SHORT; 3237 else if (temp == COLD) 3238 return WRITE_LIFE_EXTREME; 3239 } else { 3240 return WRITE_LIFE_NOT_SET; 3241 } 3242 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) { 3243 if (type == DATA) { 3244 if (temp == WARM) 3245 return WRITE_LIFE_LONG; 3246 else if (temp == HOT) 3247 return WRITE_LIFE_SHORT; 3248 else if (temp == COLD) 3249 return WRITE_LIFE_EXTREME; 3250 } else if (type == NODE) { 3251 if (temp == WARM || temp == HOT) 3252 return WRITE_LIFE_NOT_SET; 3253 else if (temp == COLD) 3254 return WRITE_LIFE_NONE; 3255 } else if (type == META) { 3256 return WRITE_LIFE_MEDIUM; 3257 } 3258 } 3259 return WRITE_LIFE_NOT_SET; 3260 } 3261 3262 static int __get_segment_type_2(struct f2fs_io_info *fio) 3263 { 3264 if (fio->type == DATA) 3265 return CURSEG_HOT_DATA; 3266 else 3267 return CURSEG_HOT_NODE; 3268 } 3269 3270 static int __get_segment_type_4(struct f2fs_io_info *fio) 3271 { 3272 if (fio->type == DATA) { 3273 struct inode *inode = fio->page->mapping->host; 3274 3275 if (S_ISDIR(inode->i_mode)) 3276 return CURSEG_HOT_DATA; 3277 else 3278 return CURSEG_COLD_DATA; 3279 } else { 3280 if (IS_DNODE(fio->page) && is_cold_node(fio->page)) 3281 return CURSEG_WARM_NODE; 3282 else 3283 return CURSEG_COLD_NODE; 3284 } 3285 } 3286 3287 static int __get_segment_type_6(struct f2fs_io_info *fio) 3288 { 3289 if (fio->type == DATA) { 3290 struct inode *inode = fio->page->mapping->host; 3291 3292 if (is_cold_data(fio->page)) { 3293 if (fio->sbi->am.atgc_enabled && 3294 (fio->io_type == FS_DATA_IO) && 3295 (fio->sbi->gc_mode != GC_URGENT_HIGH)) 3296 return CURSEG_ALL_DATA_ATGC; 3297 else 3298 return CURSEG_COLD_DATA; 3299 } 3300 if (file_is_cold(inode) || f2fs_need_compress_data(inode)) 3301 return CURSEG_COLD_DATA; 3302 if (file_is_hot(inode) || 3303 is_inode_flag_set(inode, FI_HOT_DATA) || 3304 f2fs_is_atomic_file(inode) || 3305 f2fs_is_volatile_file(inode)) 3306 return CURSEG_HOT_DATA; 3307 return f2fs_rw_hint_to_seg_type(inode->i_write_hint); 3308 } else { 3309 if (IS_DNODE(fio->page)) 3310 return is_cold_node(fio->page) ? CURSEG_WARM_NODE : 3311 CURSEG_HOT_NODE; 3312 return CURSEG_COLD_NODE; 3313 } 3314 } 3315 3316 static int __get_segment_type(struct f2fs_io_info *fio) 3317 { 3318 int type = 0; 3319 3320 switch (F2FS_OPTION(fio->sbi).active_logs) { 3321 case 2: 3322 type = __get_segment_type_2(fio); 3323 break; 3324 case 4: 3325 type = __get_segment_type_4(fio); 3326 break; 3327 case 6: 3328 type = __get_segment_type_6(fio); 3329 break; 3330 default: 3331 f2fs_bug_on(fio->sbi, true); 3332 } 3333 3334 if (IS_HOT(type)) 3335 fio->temp = HOT; 3336 else if (IS_WARM(type)) 3337 fio->temp = WARM; 3338 else 3339 fio->temp = COLD; 3340 return type; 3341 } 3342 3343 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, 3344 block_t old_blkaddr, block_t *new_blkaddr, 3345 struct f2fs_summary *sum, int type, 3346 struct f2fs_io_info *fio) 3347 { 3348 struct sit_info *sit_i = SIT_I(sbi); 3349 struct curseg_info *curseg = CURSEG_I(sbi, type); 3350 unsigned long long old_mtime; 3351 bool from_gc = (type == CURSEG_ALL_DATA_ATGC); 3352 struct seg_entry *se = NULL; 3353 3354 down_read(&SM_I(sbi)->curseg_lock); 3355 3356 mutex_lock(&curseg->curseg_mutex); 3357 down_write(&sit_i->sentry_lock); 3358 3359 if (from_gc) { 3360 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO); 3361 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr)); 3362 sanity_check_seg_type(sbi, se->type); 3363 f2fs_bug_on(sbi, IS_NODESEG(se->type)); 3364 } 3365 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 3366 3367 f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg); 3368 3369 f2fs_wait_discard_bio(sbi, *new_blkaddr); 3370 3371 /* 3372 * __add_sum_entry should be resided under the curseg_mutex 3373 * because, this function updates a summary entry in the 3374 * current summary block. 3375 */ 3376 __add_sum_entry(sbi, type, sum); 3377 3378 __refresh_next_blkoff(sbi, curseg); 3379 3380 stat_inc_block_count(sbi, curseg); 3381 3382 if (from_gc) { 3383 old_mtime = get_segment_mtime(sbi, old_blkaddr); 3384 } else { 3385 update_segment_mtime(sbi, old_blkaddr, 0); 3386 old_mtime = 0; 3387 } 3388 update_segment_mtime(sbi, *new_blkaddr, old_mtime); 3389 3390 /* 3391 * SIT information should be updated before segment allocation, 3392 * since SSR needs latest valid block information. 3393 */ 3394 update_sit_entry(sbi, *new_blkaddr, 1); 3395 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 3396 update_sit_entry(sbi, old_blkaddr, -1); 3397 3398 if (!__has_curseg_space(sbi, curseg)) { 3399 if (from_gc) 3400 get_atssr_segment(sbi, type, se->type, 3401 AT_SSR, se->mtime); 3402 else 3403 sit_i->s_ops->allocate_segment(sbi, type, false); 3404 } 3405 /* 3406 * segment dirty status should be updated after segment allocation, 3407 * so we just need to update status only one time after previous 3408 * segment being closed. 3409 */ 3410 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 3411 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); 3412 3413 up_write(&sit_i->sentry_lock); 3414 3415 if (page && IS_NODESEG(type)) { 3416 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); 3417 3418 f2fs_inode_chksum_set(sbi, page); 3419 } 3420 3421 if (fio) { 3422 struct f2fs_bio_info *io; 3423 3424 if (F2FS_IO_ALIGNED(sbi)) 3425 fio->retry = false; 3426 3427 INIT_LIST_HEAD(&fio->list); 3428 fio->in_list = true; 3429 io = sbi->write_io[fio->type] + fio->temp; 3430 spin_lock(&io->io_lock); 3431 list_add_tail(&fio->list, &io->io_list); 3432 spin_unlock(&io->io_lock); 3433 } 3434 3435 mutex_unlock(&curseg->curseg_mutex); 3436 3437 up_read(&SM_I(sbi)->curseg_lock); 3438 } 3439 3440 static void update_device_state(struct f2fs_io_info *fio) 3441 { 3442 struct f2fs_sb_info *sbi = fio->sbi; 3443 unsigned int devidx; 3444 3445 if (!f2fs_is_multi_device(sbi)) 3446 return; 3447 3448 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr); 3449 3450 /* update device state for fsync */ 3451 f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO); 3452 3453 /* update device state for checkpoint */ 3454 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { 3455 spin_lock(&sbi->dev_lock); 3456 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); 3457 spin_unlock(&sbi->dev_lock); 3458 } 3459 } 3460 3461 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) 3462 { 3463 int type = __get_segment_type(fio); 3464 bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA); 3465 3466 if (keep_order) 3467 down_read(&fio->sbi->io_order_lock); 3468 reallocate: 3469 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, 3470 &fio->new_blkaddr, sum, type, fio); 3471 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) 3472 invalidate_mapping_pages(META_MAPPING(fio->sbi), 3473 fio->old_blkaddr, fio->old_blkaddr); 3474 3475 /* writeout dirty page into bdev */ 3476 f2fs_submit_page_write(fio); 3477 if (fio->retry) { 3478 fio->old_blkaddr = fio->new_blkaddr; 3479 goto reallocate; 3480 } 3481 3482 update_device_state(fio); 3483 3484 if (keep_order) 3485 up_read(&fio->sbi->io_order_lock); 3486 } 3487 3488 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page, 3489 enum iostat_type io_type) 3490 { 3491 struct f2fs_io_info fio = { 3492 .sbi = sbi, 3493 .type = META, 3494 .temp = HOT, 3495 .op = REQ_OP_WRITE, 3496 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, 3497 .old_blkaddr = page->index, 3498 .new_blkaddr = page->index, 3499 .page = page, 3500 .encrypted_page = NULL, 3501 .in_list = false, 3502 }; 3503 3504 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) 3505 fio.op_flags &= ~REQ_META; 3506 3507 set_page_writeback(page); 3508 ClearPageError(page); 3509 f2fs_submit_page_write(&fio); 3510 3511 stat_inc_meta_count(sbi, page->index); 3512 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE); 3513 } 3514 3515 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) 3516 { 3517 struct f2fs_summary sum; 3518 3519 set_summary(&sum, nid, 0, 0); 3520 do_write_page(&sum, fio); 3521 3522 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); 3523 } 3524 3525 void f2fs_outplace_write_data(struct dnode_of_data *dn, 3526 struct f2fs_io_info *fio) 3527 { 3528 struct f2fs_sb_info *sbi = fio->sbi; 3529 struct f2fs_summary sum; 3530 3531 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); 3532 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); 3533 do_write_page(&sum, fio); 3534 f2fs_update_data_blkaddr(dn, fio->new_blkaddr); 3535 3536 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE); 3537 } 3538 3539 int f2fs_inplace_write_data(struct f2fs_io_info *fio) 3540 { 3541 int err; 3542 struct f2fs_sb_info *sbi = fio->sbi; 3543 unsigned int segno; 3544 3545 fio->new_blkaddr = fio->old_blkaddr; 3546 /* i/o temperature is needed for passing down write hints */ 3547 __get_segment_type(fio); 3548 3549 segno = GET_SEGNO(sbi, fio->new_blkaddr); 3550 3551 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) { 3552 set_sbi_flag(sbi, SBI_NEED_FSCK); 3553 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", 3554 __func__, segno); 3555 err = -EFSCORRUPTED; 3556 goto drop_bio; 3557 } 3558 3559 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || f2fs_cp_error(sbi)) { 3560 err = -EIO; 3561 goto drop_bio; 3562 } 3563 3564 stat_inc_inplace_blocks(fio->sbi); 3565 3566 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE))) 3567 err = f2fs_merge_page_bio(fio); 3568 else 3569 err = f2fs_submit_page_bio(fio); 3570 if (!err) { 3571 update_device_state(fio); 3572 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); 3573 } 3574 3575 return err; 3576 drop_bio: 3577 if (fio->bio && *(fio->bio)) { 3578 struct bio *bio = *(fio->bio); 3579 3580 bio->bi_status = BLK_STS_IOERR; 3581 bio_endio(bio); 3582 *(fio->bio) = NULL; 3583 } 3584 return err; 3585 } 3586 3587 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, 3588 unsigned int segno) 3589 { 3590 int i; 3591 3592 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { 3593 if (CURSEG_I(sbi, i)->segno == segno) 3594 break; 3595 } 3596 return i; 3597 } 3598 3599 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, 3600 block_t old_blkaddr, block_t new_blkaddr, 3601 bool recover_curseg, bool recover_newaddr, 3602 bool from_gc) 3603 { 3604 struct sit_info *sit_i = SIT_I(sbi); 3605 struct curseg_info *curseg; 3606 unsigned int segno, old_cursegno; 3607 struct seg_entry *se; 3608 int type; 3609 unsigned short old_blkoff; 3610 unsigned char old_alloc_type; 3611 3612 segno = GET_SEGNO(sbi, new_blkaddr); 3613 se = get_seg_entry(sbi, segno); 3614 type = se->type; 3615 3616 down_write(&SM_I(sbi)->curseg_lock); 3617 3618 if (!recover_curseg) { 3619 /* for recovery flow */ 3620 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { 3621 if (old_blkaddr == NULL_ADDR) 3622 type = CURSEG_COLD_DATA; 3623 else 3624 type = CURSEG_WARM_DATA; 3625 } 3626 } else { 3627 if (IS_CURSEG(sbi, segno)) { 3628 /* se->type is volatile as SSR allocation */ 3629 type = __f2fs_get_curseg(sbi, segno); 3630 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); 3631 } else { 3632 type = CURSEG_WARM_DATA; 3633 } 3634 } 3635 3636 f2fs_bug_on(sbi, !IS_DATASEG(type)); 3637 curseg = CURSEG_I(sbi, type); 3638 3639 mutex_lock(&curseg->curseg_mutex); 3640 down_write(&sit_i->sentry_lock); 3641 3642 old_cursegno = curseg->segno; 3643 old_blkoff = curseg->next_blkoff; 3644 old_alloc_type = curseg->alloc_type; 3645 3646 /* change the current segment */ 3647 if (segno != curseg->segno) { 3648 curseg->next_segno = segno; 3649 change_curseg(sbi, type, true); 3650 } 3651 3652 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 3653 __add_sum_entry(sbi, type, sum); 3654 3655 if (!recover_curseg || recover_newaddr) { 3656 if (!from_gc) 3657 update_segment_mtime(sbi, new_blkaddr, 0); 3658 update_sit_entry(sbi, new_blkaddr, 1); 3659 } 3660 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { 3661 invalidate_mapping_pages(META_MAPPING(sbi), 3662 old_blkaddr, old_blkaddr); 3663 if (!from_gc) 3664 update_segment_mtime(sbi, old_blkaddr, 0); 3665 update_sit_entry(sbi, old_blkaddr, -1); 3666 } 3667 3668 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 3669 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); 3670 3671 locate_dirty_segment(sbi, old_cursegno); 3672 3673 if (recover_curseg) { 3674 if (old_cursegno != curseg->segno) { 3675 curseg->next_segno = old_cursegno; 3676 change_curseg(sbi, type, true); 3677 } 3678 curseg->next_blkoff = old_blkoff; 3679 curseg->alloc_type = old_alloc_type; 3680 } 3681 3682 up_write(&sit_i->sentry_lock); 3683 mutex_unlock(&curseg->curseg_mutex); 3684 up_write(&SM_I(sbi)->curseg_lock); 3685 } 3686 3687 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, 3688 block_t old_addr, block_t new_addr, 3689 unsigned char version, bool recover_curseg, 3690 bool recover_newaddr) 3691 { 3692 struct f2fs_summary sum; 3693 3694 set_summary(&sum, dn->nid, dn->ofs_in_node, version); 3695 3696 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, 3697 recover_curseg, recover_newaddr, false); 3698 3699 f2fs_update_data_blkaddr(dn, new_addr); 3700 } 3701 3702 void f2fs_wait_on_page_writeback(struct page *page, 3703 enum page_type type, bool ordered, bool locked) 3704 { 3705 if (PageWriteback(page)) { 3706 struct f2fs_sb_info *sbi = F2FS_P_SB(page); 3707 3708 /* submit cached LFS IO */ 3709 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type); 3710 /* sbumit cached IPU IO */ 3711 f2fs_submit_merged_ipu_write(sbi, NULL, page); 3712 if (ordered) { 3713 wait_on_page_writeback(page); 3714 f2fs_bug_on(sbi, locked && PageWriteback(page)); 3715 } else { 3716 wait_for_stable_page(page); 3717 } 3718 } 3719 } 3720 3721 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) 3722 { 3723 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3724 struct page *cpage; 3725 3726 if (!f2fs_post_read_required(inode)) 3727 return; 3728 3729 if (!__is_valid_data_blkaddr(blkaddr)) 3730 return; 3731 3732 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); 3733 if (cpage) { 3734 f2fs_wait_on_page_writeback(cpage, DATA, true, true); 3735 f2fs_put_page(cpage, 1); 3736 } 3737 } 3738 3739 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, 3740 block_t len) 3741 { 3742 block_t i; 3743 3744 for (i = 0; i < len; i++) 3745 f2fs_wait_on_block_writeback(inode, blkaddr + i); 3746 } 3747 3748 static int read_compacted_summaries(struct f2fs_sb_info *sbi) 3749 { 3750 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3751 struct curseg_info *seg_i; 3752 unsigned char *kaddr; 3753 struct page *page; 3754 block_t start; 3755 int i, j, offset; 3756 3757 start = start_sum_block(sbi); 3758 3759 page = f2fs_get_meta_page(sbi, start++); 3760 if (IS_ERR(page)) 3761 return PTR_ERR(page); 3762 kaddr = (unsigned char *)page_address(page); 3763 3764 /* Step 1: restore nat cache */ 3765 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3766 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); 3767 3768 /* Step 2: restore sit cache */ 3769 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3770 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); 3771 offset = 2 * SUM_JOURNAL_SIZE; 3772 3773 /* Step 3: restore summary entries */ 3774 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3775 unsigned short blk_off; 3776 unsigned int segno; 3777 3778 seg_i = CURSEG_I(sbi, i); 3779 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 3780 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 3781 seg_i->next_segno = segno; 3782 reset_curseg(sbi, i, 0); 3783 seg_i->alloc_type = ckpt->alloc_type[i]; 3784 seg_i->next_blkoff = blk_off; 3785 3786 if (seg_i->alloc_type == SSR) 3787 blk_off = sbi->blocks_per_seg; 3788 3789 for (j = 0; j < blk_off; j++) { 3790 struct f2fs_summary *s; 3791 3792 s = (struct f2fs_summary *)(kaddr + offset); 3793 seg_i->sum_blk->entries[j] = *s; 3794 offset += SUMMARY_SIZE; 3795 if (offset + SUMMARY_SIZE <= PAGE_SIZE - 3796 SUM_FOOTER_SIZE) 3797 continue; 3798 3799 f2fs_put_page(page, 1); 3800 page = NULL; 3801 3802 page = f2fs_get_meta_page(sbi, start++); 3803 if (IS_ERR(page)) 3804 return PTR_ERR(page); 3805 kaddr = (unsigned char *)page_address(page); 3806 offset = 0; 3807 } 3808 } 3809 f2fs_put_page(page, 1); 3810 return 0; 3811 } 3812 3813 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 3814 { 3815 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3816 struct f2fs_summary_block *sum; 3817 struct curseg_info *curseg; 3818 struct page *new; 3819 unsigned short blk_off; 3820 unsigned int segno = 0; 3821 block_t blk_addr = 0; 3822 int err = 0; 3823 3824 /* get segment number and block addr */ 3825 if (IS_DATASEG(type)) { 3826 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 3827 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 3828 CURSEG_HOT_DATA]); 3829 if (__exist_node_summaries(sbi)) 3830 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type); 3831 else 3832 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 3833 } else { 3834 segno = le32_to_cpu(ckpt->cur_node_segno[type - 3835 CURSEG_HOT_NODE]); 3836 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 3837 CURSEG_HOT_NODE]); 3838 if (__exist_node_summaries(sbi)) 3839 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 3840 type - CURSEG_HOT_NODE); 3841 else 3842 blk_addr = GET_SUM_BLOCK(sbi, segno); 3843 } 3844 3845 new = f2fs_get_meta_page(sbi, blk_addr); 3846 if (IS_ERR(new)) 3847 return PTR_ERR(new); 3848 sum = (struct f2fs_summary_block *)page_address(new); 3849 3850 if (IS_NODESEG(type)) { 3851 if (__exist_node_summaries(sbi)) { 3852 struct f2fs_summary *ns = &sum->entries[0]; 3853 int i; 3854 3855 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { 3856 ns->version = 0; 3857 ns->ofs_in_node = 0; 3858 } 3859 } else { 3860 err = f2fs_restore_node_summary(sbi, segno, sum); 3861 if (err) 3862 goto out; 3863 } 3864 } 3865 3866 /* set uncompleted segment to curseg */ 3867 curseg = CURSEG_I(sbi, type); 3868 mutex_lock(&curseg->curseg_mutex); 3869 3870 /* update journal info */ 3871 down_write(&curseg->journal_rwsem); 3872 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); 3873 up_write(&curseg->journal_rwsem); 3874 3875 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); 3876 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); 3877 curseg->next_segno = segno; 3878 reset_curseg(sbi, type, 0); 3879 curseg->alloc_type = ckpt->alloc_type[type]; 3880 curseg->next_blkoff = blk_off; 3881 mutex_unlock(&curseg->curseg_mutex); 3882 out: 3883 f2fs_put_page(new, 1); 3884 return err; 3885 } 3886 3887 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 3888 { 3889 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; 3890 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; 3891 int type = CURSEG_HOT_DATA; 3892 int err; 3893 3894 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { 3895 int npages = f2fs_npages_for_summary_flush(sbi, true); 3896 3897 if (npages >= 2) 3898 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, 3899 META_CP, true); 3900 3901 /* restore for compacted data summary */ 3902 err = read_compacted_summaries(sbi); 3903 if (err) 3904 return err; 3905 type = CURSEG_HOT_NODE; 3906 } 3907 3908 if (__exist_node_summaries(sbi)) 3909 f2fs_ra_meta_pages(sbi, 3910 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type), 3911 NR_CURSEG_PERSIST_TYPE - type, META_CP, true); 3912 3913 for (; type <= CURSEG_COLD_NODE; type++) { 3914 err = read_normal_summaries(sbi, type); 3915 if (err) 3916 return err; 3917 } 3918 3919 /* sanity check for summary blocks */ 3920 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || 3921 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) { 3922 f2fs_err(sbi, "invalid journal entries nats %u sits %u\n", 3923 nats_in_cursum(nat_j), sits_in_cursum(sit_j)); 3924 return -EINVAL; 3925 } 3926 3927 return 0; 3928 } 3929 3930 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 3931 { 3932 struct page *page; 3933 unsigned char *kaddr; 3934 struct f2fs_summary *summary; 3935 struct curseg_info *seg_i; 3936 int written_size = 0; 3937 int i, j; 3938 3939 page = f2fs_grab_meta_page(sbi, blkaddr++); 3940 kaddr = (unsigned char *)page_address(page); 3941 memset(kaddr, 0, PAGE_SIZE); 3942 3943 /* Step 1: write nat cache */ 3944 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3945 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); 3946 written_size += SUM_JOURNAL_SIZE; 3947 3948 /* Step 2: write sit cache */ 3949 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3950 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); 3951 written_size += SUM_JOURNAL_SIZE; 3952 3953 /* Step 3: write summary entries */ 3954 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3955 unsigned short blkoff; 3956 3957 seg_i = CURSEG_I(sbi, i); 3958 if (sbi->ckpt->alloc_type[i] == SSR) 3959 blkoff = sbi->blocks_per_seg; 3960 else 3961 blkoff = curseg_blkoff(sbi, i); 3962 3963 for (j = 0; j < blkoff; j++) { 3964 if (!page) { 3965 page = f2fs_grab_meta_page(sbi, blkaddr++); 3966 kaddr = (unsigned char *)page_address(page); 3967 memset(kaddr, 0, PAGE_SIZE); 3968 written_size = 0; 3969 } 3970 summary = (struct f2fs_summary *)(kaddr + written_size); 3971 *summary = seg_i->sum_blk->entries[j]; 3972 written_size += SUMMARY_SIZE; 3973 3974 if (written_size + SUMMARY_SIZE <= PAGE_SIZE - 3975 SUM_FOOTER_SIZE) 3976 continue; 3977 3978 set_page_dirty(page); 3979 f2fs_put_page(page, 1); 3980 page = NULL; 3981 } 3982 } 3983 if (page) { 3984 set_page_dirty(page); 3985 f2fs_put_page(page, 1); 3986 } 3987 } 3988 3989 static void write_normal_summaries(struct f2fs_sb_info *sbi, 3990 block_t blkaddr, int type) 3991 { 3992 int i, end; 3993 3994 if (IS_DATASEG(type)) 3995 end = type + NR_CURSEG_DATA_TYPE; 3996 else 3997 end = type + NR_CURSEG_NODE_TYPE; 3998 3999 for (i = type; i < end; i++) 4000 write_current_sum_page(sbi, i, blkaddr + (i - type)); 4001 } 4002 4003 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 4004 { 4005 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) 4006 write_compacted_summaries(sbi, start_blk); 4007 else 4008 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 4009 } 4010 4011 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 4012 { 4013 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 4014 } 4015 4016 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, 4017 unsigned int val, int alloc) 4018 { 4019 int i; 4020 4021 if (type == NAT_JOURNAL) { 4022 for (i = 0; i < nats_in_cursum(journal); i++) { 4023 if (le32_to_cpu(nid_in_journal(journal, i)) == val) 4024 return i; 4025 } 4026 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) 4027 return update_nats_in_cursum(journal, 1); 4028 } else if (type == SIT_JOURNAL) { 4029 for (i = 0; i < sits_in_cursum(journal); i++) 4030 if (le32_to_cpu(segno_in_journal(journal, i)) == val) 4031 return i; 4032 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) 4033 return update_sits_in_cursum(journal, 1); 4034 } 4035 return -1; 4036 } 4037 4038 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 4039 unsigned int segno) 4040 { 4041 return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno)); 4042 } 4043 4044 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 4045 unsigned int start) 4046 { 4047 struct sit_info *sit_i = SIT_I(sbi); 4048 struct page *page; 4049 pgoff_t src_off, dst_off; 4050 4051 src_off = current_sit_addr(sbi, start); 4052 dst_off = next_sit_addr(sbi, src_off); 4053 4054 page = f2fs_grab_meta_page(sbi, dst_off); 4055 seg_info_to_sit_page(sbi, page, start); 4056 4057 set_page_dirty(page); 4058 set_to_next_sit(sit_i, start); 4059 4060 return page; 4061 } 4062 4063 static struct sit_entry_set *grab_sit_entry_set(void) 4064 { 4065 struct sit_entry_set *ses = 4066 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS); 4067 4068 ses->entry_cnt = 0; 4069 INIT_LIST_HEAD(&ses->set_list); 4070 return ses; 4071 } 4072 4073 static void release_sit_entry_set(struct sit_entry_set *ses) 4074 { 4075 list_del(&ses->set_list); 4076 kmem_cache_free(sit_entry_set_slab, ses); 4077 } 4078 4079 static void adjust_sit_entry_set(struct sit_entry_set *ses, 4080 struct list_head *head) 4081 { 4082 struct sit_entry_set *next = ses; 4083 4084 if (list_is_last(&ses->set_list, head)) 4085 return; 4086 4087 list_for_each_entry_continue(next, head, set_list) 4088 if (ses->entry_cnt <= next->entry_cnt) 4089 break; 4090 4091 list_move_tail(&ses->set_list, &next->set_list); 4092 } 4093 4094 static void add_sit_entry(unsigned int segno, struct list_head *head) 4095 { 4096 struct sit_entry_set *ses; 4097 unsigned int start_segno = START_SEGNO(segno); 4098 4099 list_for_each_entry(ses, head, set_list) { 4100 if (ses->start_segno == start_segno) { 4101 ses->entry_cnt++; 4102 adjust_sit_entry_set(ses, head); 4103 return; 4104 } 4105 } 4106 4107 ses = grab_sit_entry_set(); 4108 4109 ses->start_segno = start_segno; 4110 ses->entry_cnt++; 4111 list_add(&ses->set_list, head); 4112 } 4113 4114 static void add_sits_in_set(struct f2fs_sb_info *sbi) 4115 { 4116 struct f2fs_sm_info *sm_info = SM_I(sbi); 4117 struct list_head *set_list = &sm_info->sit_entry_set; 4118 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; 4119 unsigned int segno; 4120 4121 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) 4122 add_sit_entry(segno, set_list); 4123 } 4124 4125 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) 4126 { 4127 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4128 struct f2fs_journal *journal = curseg->journal; 4129 int i; 4130 4131 down_write(&curseg->journal_rwsem); 4132 for (i = 0; i < sits_in_cursum(journal); i++) { 4133 unsigned int segno; 4134 bool dirtied; 4135 4136 segno = le32_to_cpu(segno_in_journal(journal, i)); 4137 dirtied = __mark_sit_entry_dirty(sbi, segno); 4138 4139 if (!dirtied) 4140 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); 4141 } 4142 update_sits_in_cursum(journal, -i); 4143 up_write(&curseg->journal_rwsem); 4144 } 4145 4146 /* 4147 * CP calls this function, which flushes SIT entries including sit_journal, 4148 * and moves prefree segs to free segs. 4149 */ 4150 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) 4151 { 4152 struct sit_info *sit_i = SIT_I(sbi); 4153 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 4154 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4155 struct f2fs_journal *journal = curseg->journal; 4156 struct sit_entry_set *ses, *tmp; 4157 struct list_head *head = &SM_I(sbi)->sit_entry_set; 4158 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS); 4159 struct seg_entry *se; 4160 4161 down_write(&sit_i->sentry_lock); 4162 4163 if (!sit_i->dirty_sentries) 4164 goto out; 4165 4166 /* 4167 * add and account sit entries of dirty bitmap in sit entry 4168 * set temporarily 4169 */ 4170 add_sits_in_set(sbi); 4171 4172 /* 4173 * if there are no enough space in journal to store dirty sit 4174 * entries, remove all entries from journal and add and account 4175 * them in sit entry set. 4176 */ 4177 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) || 4178 !to_journal) 4179 remove_sits_in_journal(sbi); 4180 4181 /* 4182 * there are two steps to flush sit entries: 4183 * #1, flush sit entries to journal in current cold data summary block. 4184 * #2, flush sit entries to sit page. 4185 */ 4186 list_for_each_entry_safe(ses, tmp, head, set_list) { 4187 struct page *page = NULL; 4188 struct f2fs_sit_block *raw_sit = NULL; 4189 unsigned int start_segno = ses->start_segno; 4190 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, 4191 (unsigned long)MAIN_SEGS(sbi)); 4192 unsigned int segno = start_segno; 4193 4194 if (to_journal && 4195 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) 4196 to_journal = false; 4197 4198 if (to_journal) { 4199 down_write(&curseg->journal_rwsem); 4200 } else { 4201 page = get_next_sit_page(sbi, start_segno); 4202 raw_sit = page_address(page); 4203 } 4204 4205 /* flush dirty sit entries in region of current sit set */ 4206 for_each_set_bit_from(segno, bitmap, end) { 4207 int offset, sit_offset; 4208 4209 se = get_seg_entry(sbi, segno); 4210 #ifdef CONFIG_F2FS_CHECK_FS 4211 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir, 4212 SIT_VBLOCK_MAP_SIZE)) 4213 f2fs_bug_on(sbi, 1); 4214 #endif 4215 4216 /* add discard candidates */ 4217 if (!(cpc->reason & CP_DISCARD)) { 4218 cpc->trim_start = segno; 4219 add_discard_addrs(sbi, cpc, false); 4220 } 4221 4222 if (to_journal) { 4223 offset = f2fs_lookup_journal_in_cursum(journal, 4224 SIT_JOURNAL, segno, 1); 4225 f2fs_bug_on(sbi, offset < 0); 4226 segno_in_journal(journal, offset) = 4227 cpu_to_le32(segno); 4228 seg_info_to_raw_sit(se, 4229 &sit_in_journal(journal, offset)); 4230 check_block_count(sbi, segno, 4231 &sit_in_journal(journal, offset)); 4232 } else { 4233 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 4234 seg_info_to_raw_sit(se, 4235 &raw_sit->entries[sit_offset]); 4236 check_block_count(sbi, segno, 4237 &raw_sit->entries[sit_offset]); 4238 } 4239 4240 __clear_bit(segno, bitmap); 4241 sit_i->dirty_sentries--; 4242 ses->entry_cnt--; 4243 } 4244 4245 if (to_journal) 4246 up_write(&curseg->journal_rwsem); 4247 else 4248 f2fs_put_page(page, 1); 4249 4250 f2fs_bug_on(sbi, ses->entry_cnt); 4251 release_sit_entry_set(ses); 4252 } 4253 4254 f2fs_bug_on(sbi, !list_empty(head)); 4255 f2fs_bug_on(sbi, sit_i->dirty_sentries); 4256 out: 4257 if (cpc->reason & CP_DISCARD) { 4258 __u64 trim_start = cpc->trim_start; 4259 4260 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) 4261 add_discard_addrs(sbi, cpc, false); 4262 4263 cpc->trim_start = trim_start; 4264 } 4265 up_write(&sit_i->sentry_lock); 4266 4267 set_prefree_as_free_segments(sbi); 4268 } 4269 4270 static int build_sit_info(struct f2fs_sb_info *sbi) 4271 { 4272 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 4273 struct sit_info *sit_i; 4274 unsigned int sit_segs, start; 4275 char *src_bitmap, *bitmap; 4276 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size; 4277 4278 /* allocate memory for SIT information */ 4279 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); 4280 if (!sit_i) 4281 return -ENOMEM; 4282 4283 SM_I(sbi)->sit_info = sit_i; 4284 4285 sit_i->sentries = 4286 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), 4287 MAIN_SEGS(sbi)), 4288 GFP_KERNEL); 4289 if (!sit_i->sentries) 4290 return -ENOMEM; 4291 4292 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4293 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size, 4294 GFP_KERNEL); 4295 if (!sit_i->dirty_sentries_bitmap) 4296 return -ENOMEM; 4297 4298 #ifdef CONFIG_F2FS_CHECK_FS 4299 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4; 4300 #else 4301 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3; 4302 #endif 4303 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 4304 if (!sit_i->bitmap) 4305 return -ENOMEM; 4306 4307 bitmap = sit_i->bitmap; 4308 4309 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4310 sit_i->sentries[start].cur_valid_map = bitmap; 4311 bitmap += SIT_VBLOCK_MAP_SIZE; 4312 4313 sit_i->sentries[start].ckpt_valid_map = bitmap; 4314 bitmap += SIT_VBLOCK_MAP_SIZE; 4315 4316 #ifdef CONFIG_F2FS_CHECK_FS 4317 sit_i->sentries[start].cur_valid_map_mir = bitmap; 4318 bitmap += SIT_VBLOCK_MAP_SIZE; 4319 #endif 4320 4321 sit_i->sentries[start].discard_map = bitmap; 4322 bitmap += SIT_VBLOCK_MAP_SIZE; 4323 } 4324 4325 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 4326 if (!sit_i->tmp_map) 4327 return -ENOMEM; 4328 4329 if (__is_large_section(sbi)) { 4330 sit_i->sec_entries = 4331 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), 4332 MAIN_SECS(sbi)), 4333 GFP_KERNEL); 4334 if (!sit_i->sec_entries) 4335 return -ENOMEM; 4336 } 4337 4338 /* get information related with SIT */ 4339 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 4340 4341 /* setup SIT bitmap from ckeckpoint pack */ 4342 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 4343 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 4344 4345 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL); 4346 if (!sit_i->sit_bitmap) 4347 return -ENOMEM; 4348 4349 #ifdef CONFIG_F2FS_CHECK_FS 4350 sit_i->sit_bitmap_mir = kmemdup(src_bitmap, 4351 sit_bitmap_size, GFP_KERNEL); 4352 if (!sit_i->sit_bitmap_mir) 4353 return -ENOMEM; 4354 4355 sit_i->invalid_segmap = f2fs_kvzalloc(sbi, 4356 main_bitmap_size, GFP_KERNEL); 4357 if (!sit_i->invalid_segmap) 4358 return -ENOMEM; 4359 #endif 4360 4361 /* init SIT information */ 4362 sit_i->s_ops = &default_salloc_ops; 4363 4364 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 4365 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; 4366 sit_i->written_valid_blocks = 0; 4367 sit_i->bitmap_size = sit_bitmap_size; 4368 sit_i->dirty_sentries = 0; 4369 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 4370 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 4371 sit_i->mounted_time = ktime_get_boottime_seconds(); 4372 init_rwsem(&sit_i->sentry_lock); 4373 return 0; 4374 } 4375 4376 static int build_free_segmap(struct f2fs_sb_info *sbi) 4377 { 4378 struct free_segmap_info *free_i; 4379 unsigned int bitmap_size, sec_bitmap_size; 4380 4381 /* allocate memory for free segmap information */ 4382 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); 4383 if (!free_i) 4384 return -ENOMEM; 4385 4386 SM_I(sbi)->free_info = free_i; 4387 4388 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4389 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); 4390 if (!free_i->free_segmap) 4391 return -ENOMEM; 4392 4393 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 4394 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); 4395 if (!free_i->free_secmap) 4396 return -ENOMEM; 4397 4398 /* set all segments as dirty temporarily */ 4399 memset(free_i->free_segmap, 0xff, bitmap_size); 4400 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 4401 4402 /* init free segmap information */ 4403 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); 4404 free_i->free_segments = 0; 4405 free_i->free_sections = 0; 4406 spin_lock_init(&free_i->segmap_lock); 4407 return 0; 4408 } 4409 4410 static int build_curseg(struct f2fs_sb_info *sbi) 4411 { 4412 struct curseg_info *array; 4413 int i; 4414 4415 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, 4416 sizeof(*array)), GFP_KERNEL); 4417 if (!array) 4418 return -ENOMEM; 4419 4420 SM_I(sbi)->curseg_array = array; 4421 4422 for (i = 0; i < NO_CHECK_TYPE; i++) { 4423 mutex_init(&array[i].curseg_mutex); 4424 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); 4425 if (!array[i].sum_blk) 4426 return -ENOMEM; 4427 init_rwsem(&array[i].journal_rwsem); 4428 array[i].journal = f2fs_kzalloc(sbi, 4429 sizeof(struct f2fs_journal), GFP_KERNEL); 4430 if (!array[i].journal) 4431 return -ENOMEM; 4432 if (i < NR_PERSISTENT_LOG) 4433 array[i].seg_type = CURSEG_HOT_DATA + i; 4434 else if (i == CURSEG_COLD_DATA_PINNED) 4435 array[i].seg_type = CURSEG_COLD_DATA; 4436 else if (i == CURSEG_ALL_DATA_ATGC) 4437 array[i].seg_type = CURSEG_COLD_DATA; 4438 array[i].segno = NULL_SEGNO; 4439 array[i].next_blkoff = 0; 4440 array[i].inited = false; 4441 } 4442 return restore_curseg_summaries(sbi); 4443 } 4444 4445 static int build_sit_entries(struct f2fs_sb_info *sbi) 4446 { 4447 struct sit_info *sit_i = SIT_I(sbi); 4448 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4449 struct f2fs_journal *journal = curseg->journal; 4450 struct seg_entry *se; 4451 struct f2fs_sit_entry sit; 4452 int sit_blk_cnt = SIT_BLK_CNT(sbi); 4453 unsigned int i, start, end; 4454 unsigned int readed, start_blk = 0; 4455 int err = 0; 4456 block_t total_node_blocks = 0; 4457 4458 do { 4459 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS, 4460 META_SIT, true); 4461 4462 start = start_blk * sit_i->sents_per_block; 4463 end = (start_blk + readed) * sit_i->sents_per_block; 4464 4465 for (; start < end && start < MAIN_SEGS(sbi); start++) { 4466 struct f2fs_sit_block *sit_blk; 4467 struct page *page; 4468 4469 se = &sit_i->sentries[start]; 4470 page = get_current_sit_page(sbi, start); 4471 if (IS_ERR(page)) 4472 return PTR_ERR(page); 4473 sit_blk = (struct f2fs_sit_block *)page_address(page); 4474 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 4475 f2fs_put_page(page, 1); 4476 4477 err = check_block_count(sbi, start, &sit); 4478 if (err) 4479 return err; 4480 seg_info_from_raw_sit(se, &sit); 4481 if (IS_NODESEG(se->type)) 4482 total_node_blocks += se->valid_blocks; 4483 4484 /* build discard map only one time */ 4485 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 4486 memset(se->discard_map, 0xff, 4487 SIT_VBLOCK_MAP_SIZE); 4488 } else { 4489 memcpy(se->discard_map, 4490 se->cur_valid_map, 4491 SIT_VBLOCK_MAP_SIZE); 4492 sbi->discard_blks += 4493 sbi->blocks_per_seg - 4494 se->valid_blocks; 4495 } 4496 4497 if (__is_large_section(sbi)) 4498 get_sec_entry(sbi, start)->valid_blocks += 4499 se->valid_blocks; 4500 } 4501 start_blk += readed; 4502 } while (start_blk < sit_blk_cnt); 4503 4504 down_read(&curseg->journal_rwsem); 4505 for (i = 0; i < sits_in_cursum(journal); i++) { 4506 unsigned int old_valid_blocks; 4507 4508 start = le32_to_cpu(segno_in_journal(journal, i)); 4509 if (start >= MAIN_SEGS(sbi)) { 4510 f2fs_err(sbi, "Wrong journal entry on segno %u", 4511 start); 4512 err = -EFSCORRUPTED; 4513 break; 4514 } 4515 4516 se = &sit_i->sentries[start]; 4517 sit = sit_in_journal(journal, i); 4518 4519 old_valid_blocks = se->valid_blocks; 4520 if (IS_NODESEG(se->type)) 4521 total_node_blocks -= old_valid_blocks; 4522 4523 err = check_block_count(sbi, start, &sit); 4524 if (err) 4525 break; 4526 seg_info_from_raw_sit(se, &sit); 4527 if (IS_NODESEG(se->type)) 4528 total_node_blocks += se->valid_blocks; 4529 4530 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 4531 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE); 4532 } else { 4533 memcpy(se->discard_map, se->cur_valid_map, 4534 SIT_VBLOCK_MAP_SIZE); 4535 sbi->discard_blks += old_valid_blocks; 4536 sbi->discard_blks -= se->valid_blocks; 4537 } 4538 4539 if (__is_large_section(sbi)) { 4540 get_sec_entry(sbi, start)->valid_blocks += 4541 se->valid_blocks; 4542 get_sec_entry(sbi, start)->valid_blocks -= 4543 old_valid_blocks; 4544 } 4545 } 4546 up_read(&curseg->journal_rwsem); 4547 4548 if (!err && total_node_blocks != valid_node_count(sbi)) { 4549 f2fs_err(sbi, "SIT is corrupted node# %u vs %u", 4550 total_node_blocks, valid_node_count(sbi)); 4551 err = -EFSCORRUPTED; 4552 } 4553 4554 return err; 4555 } 4556 4557 static void init_free_segmap(struct f2fs_sb_info *sbi) 4558 { 4559 unsigned int start; 4560 int type; 4561 struct seg_entry *sentry; 4562 4563 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4564 if (f2fs_usable_blks_in_seg(sbi, start) == 0) 4565 continue; 4566 sentry = get_seg_entry(sbi, start); 4567 if (!sentry->valid_blocks) 4568 __set_free(sbi, start); 4569 else 4570 SIT_I(sbi)->written_valid_blocks += 4571 sentry->valid_blocks; 4572 } 4573 4574 /* set use the current segments */ 4575 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 4576 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 4577 4578 __set_test_and_inuse(sbi, curseg_t->segno); 4579 } 4580 } 4581 4582 static void init_dirty_segmap(struct f2fs_sb_info *sbi) 4583 { 4584 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4585 struct free_segmap_info *free_i = FREE_I(sbi); 4586 unsigned int segno = 0, offset = 0, secno; 4587 block_t valid_blocks, usable_blks_in_seg; 4588 block_t blks_per_sec = BLKS_PER_SEC(sbi); 4589 4590 while (1) { 4591 /* find dirty segment based on free segmap */ 4592 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); 4593 if (segno >= MAIN_SEGS(sbi)) 4594 break; 4595 offset = segno + 1; 4596 valid_blocks = get_valid_blocks(sbi, segno, false); 4597 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); 4598 if (valid_blocks == usable_blks_in_seg || !valid_blocks) 4599 continue; 4600 if (valid_blocks > usable_blks_in_seg) { 4601 f2fs_bug_on(sbi, 1); 4602 continue; 4603 } 4604 mutex_lock(&dirty_i->seglist_lock); 4605 __locate_dirty_segment(sbi, segno, DIRTY); 4606 mutex_unlock(&dirty_i->seglist_lock); 4607 } 4608 4609 if (!__is_large_section(sbi)) 4610 return; 4611 4612 mutex_lock(&dirty_i->seglist_lock); 4613 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { 4614 valid_blocks = get_valid_blocks(sbi, segno, true); 4615 secno = GET_SEC_FROM_SEG(sbi, segno); 4616 4617 if (!valid_blocks || valid_blocks == blks_per_sec) 4618 continue; 4619 if (IS_CURSEC(sbi, secno)) 4620 continue; 4621 set_bit(secno, dirty_i->dirty_secmap); 4622 } 4623 mutex_unlock(&dirty_i->seglist_lock); 4624 } 4625 4626 static int init_victim_secmap(struct f2fs_sb_info *sbi) 4627 { 4628 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4629 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 4630 4631 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 4632 if (!dirty_i->victim_secmap) 4633 return -ENOMEM; 4634 return 0; 4635 } 4636 4637 static int build_dirty_segmap(struct f2fs_sb_info *sbi) 4638 { 4639 struct dirty_seglist_info *dirty_i; 4640 unsigned int bitmap_size, i; 4641 4642 /* allocate memory for dirty segments list information */ 4643 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), 4644 GFP_KERNEL); 4645 if (!dirty_i) 4646 return -ENOMEM; 4647 4648 SM_I(sbi)->dirty_info = dirty_i; 4649 mutex_init(&dirty_i->seglist_lock); 4650 4651 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4652 4653 for (i = 0; i < NR_DIRTY_TYPE; i++) { 4654 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, 4655 GFP_KERNEL); 4656 if (!dirty_i->dirty_segmap[i]) 4657 return -ENOMEM; 4658 } 4659 4660 if (__is_large_section(sbi)) { 4661 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 4662 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi, 4663 bitmap_size, GFP_KERNEL); 4664 if (!dirty_i->dirty_secmap) 4665 return -ENOMEM; 4666 } 4667 4668 init_dirty_segmap(sbi); 4669 return init_victim_secmap(sbi); 4670 } 4671 4672 static int sanity_check_curseg(struct f2fs_sb_info *sbi) 4673 { 4674 int i; 4675 4676 /* 4677 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr; 4678 * In LFS curseg, all blkaddr after .next_blkoff should be unused. 4679 */ 4680 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 4681 struct curseg_info *curseg = CURSEG_I(sbi, i); 4682 struct seg_entry *se = get_seg_entry(sbi, curseg->segno); 4683 unsigned int blkofs = curseg->next_blkoff; 4684 4685 sanity_check_seg_type(sbi, curseg->seg_type); 4686 4687 if (f2fs_test_bit(blkofs, se->cur_valid_map)) 4688 goto out; 4689 4690 if (curseg->alloc_type == SSR) 4691 continue; 4692 4693 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) { 4694 if (!f2fs_test_bit(blkofs, se->cur_valid_map)) 4695 continue; 4696 out: 4697 f2fs_err(sbi, 4698 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u", 4699 i, curseg->segno, curseg->alloc_type, 4700 curseg->next_blkoff, blkofs); 4701 return -EFSCORRUPTED; 4702 } 4703 } 4704 return 0; 4705 } 4706 4707 #ifdef CONFIG_BLK_DEV_ZONED 4708 4709 static int check_zone_write_pointer(struct f2fs_sb_info *sbi, 4710 struct f2fs_dev_info *fdev, 4711 struct blk_zone *zone) 4712 { 4713 unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno; 4714 block_t zone_block, wp_block, last_valid_block; 4715 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 4716 int i, s, b, ret; 4717 struct seg_entry *se; 4718 4719 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ) 4720 return 0; 4721 4722 wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block); 4723 wp_segno = GET_SEGNO(sbi, wp_block); 4724 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); 4725 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block); 4726 zone_segno = GET_SEGNO(sbi, zone_block); 4727 zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno); 4728 4729 if (zone_segno >= MAIN_SEGS(sbi)) 4730 return 0; 4731 4732 /* 4733 * Skip check of zones cursegs point to, since 4734 * fix_curseg_write_pointer() checks them. 4735 */ 4736 for (i = 0; i < NO_CHECK_TYPE; i++) 4737 if (zone_secno == GET_SEC_FROM_SEG(sbi, 4738 CURSEG_I(sbi, i)->segno)) 4739 return 0; 4740 4741 /* 4742 * Get last valid block of the zone. 4743 */ 4744 last_valid_block = zone_block - 1; 4745 for (s = sbi->segs_per_sec - 1; s >= 0; s--) { 4746 segno = zone_segno + s; 4747 se = get_seg_entry(sbi, segno); 4748 for (b = sbi->blocks_per_seg - 1; b >= 0; b--) 4749 if (f2fs_test_bit(b, se->cur_valid_map)) { 4750 last_valid_block = START_BLOCK(sbi, segno) + b; 4751 break; 4752 } 4753 if (last_valid_block >= zone_block) 4754 break; 4755 } 4756 4757 /* 4758 * If last valid block is beyond the write pointer, report the 4759 * inconsistency. This inconsistency does not cause write error 4760 * because the zone will not be selected for write operation until 4761 * it get discarded. Just report it. 4762 */ 4763 if (last_valid_block >= wp_block) { 4764 f2fs_notice(sbi, "Valid block beyond write pointer: " 4765 "valid block[0x%x,0x%x] wp[0x%x,0x%x]", 4766 GET_SEGNO(sbi, last_valid_block), 4767 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block), 4768 wp_segno, wp_blkoff); 4769 return 0; 4770 } 4771 4772 /* 4773 * If there is no valid block in the zone and if write pointer is 4774 * not at zone start, reset the write pointer. 4775 */ 4776 if (last_valid_block + 1 == zone_block && zone->wp != zone->start) { 4777 f2fs_notice(sbi, 4778 "Zone without valid block has non-zero write " 4779 "pointer. Reset the write pointer: wp[0x%x,0x%x]", 4780 wp_segno, wp_blkoff); 4781 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block, 4782 zone->len >> log_sectors_per_block); 4783 if (ret) { 4784 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 4785 fdev->path, ret); 4786 return ret; 4787 } 4788 } 4789 4790 return 0; 4791 } 4792 4793 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi, 4794 block_t zone_blkaddr) 4795 { 4796 int i; 4797 4798 for (i = 0; i < sbi->s_ndevs; i++) { 4799 if (!bdev_is_zoned(FDEV(i).bdev)) 4800 continue; 4801 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr && 4802 zone_blkaddr <= FDEV(i).end_blk)) 4803 return &FDEV(i); 4804 } 4805 4806 return NULL; 4807 } 4808 4809 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx, 4810 void *data) 4811 { 4812 memcpy(data, zone, sizeof(struct blk_zone)); 4813 return 0; 4814 } 4815 4816 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) 4817 { 4818 struct curseg_info *cs = CURSEG_I(sbi, type); 4819 struct f2fs_dev_info *zbd; 4820 struct blk_zone zone; 4821 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off; 4822 block_t cs_zone_block, wp_block; 4823 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 4824 sector_t zone_sector; 4825 int err; 4826 4827 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 4828 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 4829 4830 zbd = get_target_zoned_dev(sbi, cs_zone_block); 4831 if (!zbd) 4832 return 0; 4833 4834 /* report zone for the sector the curseg points to */ 4835 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 4836 << log_sectors_per_block; 4837 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 4838 report_one_zone_cb, &zone); 4839 if (err != 1) { 4840 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 4841 zbd->path, err); 4842 return err; 4843 } 4844 4845 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 4846 return 0; 4847 4848 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block); 4849 wp_segno = GET_SEGNO(sbi, wp_block); 4850 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); 4851 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0); 4852 4853 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff && 4854 wp_sector_off == 0) 4855 return 0; 4856 4857 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: " 4858 "curseg[0x%x,0x%x] wp[0x%x,0x%x]", 4859 type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff); 4860 4861 f2fs_notice(sbi, "Assign new section to curseg[%d]: " 4862 "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff); 4863 4864 f2fs_allocate_new_section(sbi, type, true); 4865 4866 /* check consistency of the zone curseg pointed to */ 4867 if (check_zone_write_pointer(sbi, zbd, &zone)) 4868 return -EIO; 4869 4870 /* check newly assigned zone */ 4871 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 4872 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 4873 4874 zbd = get_target_zoned_dev(sbi, cs_zone_block); 4875 if (!zbd) 4876 return 0; 4877 4878 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 4879 << log_sectors_per_block; 4880 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 4881 report_one_zone_cb, &zone); 4882 if (err != 1) { 4883 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 4884 zbd->path, err); 4885 return err; 4886 } 4887 4888 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 4889 return 0; 4890 4891 if (zone.wp != zone.start) { 4892 f2fs_notice(sbi, 4893 "New zone for curseg[%d] is not yet discarded. " 4894 "Reset the zone: curseg[0x%x,0x%x]", 4895 type, cs->segno, cs->next_blkoff); 4896 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, 4897 zone_sector >> log_sectors_per_block, 4898 zone.len >> log_sectors_per_block); 4899 if (err) { 4900 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 4901 zbd->path, err); 4902 return err; 4903 } 4904 } 4905 4906 return 0; 4907 } 4908 4909 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) 4910 { 4911 int i, ret; 4912 4913 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 4914 ret = fix_curseg_write_pointer(sbi, i); 4915 if (ret) 4916 return ret; 4917 } 4918 4919 return 0; 4920 } 4921 4922 struct check_zone_write_pointer_args { 4923 struct f2fs_sb_info *sbi; 4924 struct f2fs_dev_info *fdev; 4925 }; 4926 4927 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx, 4928 void *data) 4929 { 4930 struct check_zone_write_pointer_args *args; 4931 4932 args = (struct check_zone_write_pointer_args *)data; 4933 4934 return check_zone_write_pointer(args->sbi, args->fdev, zone); 4935 } 4936 4937 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) 4938 { 4939 int i, ret; 4940 struct check_zone_write_pointer_args args; 4941 4942 for (i = 0; i < sbi->s_ndevs; i++) { 4943 if (!bdev_is_zoned(FDEV(i).bdev)) 4944 continue; 4945 4946 args.sbi = sbi; 4947 args.fdev = &FDEV(i); 4948 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES, 4949 check_zone_write_pointer_cb, &args); 4950 if (ret < 0) 4951 return ret; 4952 } 4953 4954 return 0; 4955 } 4956 4957 static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx, 4958 unsigned int dev_idx) 4959 { 4960 if (!bdev_is_zoned(FDEV(dev_idx).bdev)) 4961 return true; 4962 return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq); 4963 } 4964 4965 /* Return the zone index in the given device */ 4966 static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno, 4967 int dev_idx) 4968 { 4969 block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); 4970 4971 return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >> 4972 sbi->log_blocks_per_blkz; 4973 } 4974 4975 /* 4976 * Return the usable segments in a section based on the zone's 4977 * corresponding zone capacity. Zone is equal to a section. 4978 */ 4979 static inline unsigned int f2fs_usable_zone_segs_in_sec( 4980 struct f2fs_sb_info *sbi, unsigned int segno) 4981 { 4982 unsigned int dev_idx, zone_idx, unusable_segs_in_sec; 4983 4984 dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 4985 zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx); 4986 4987 /* Conventional zone's capacity is always equal to zone size */ 4988 if (is_conv_zone(sbi, zone_idx, dev_idx)) 4989 return sbi->segs_per_sec; 4990 4991 /* 4992 * If the zone_capacity_blocks array is NULL, then zone capacity 4993 * is equal to the zone size for all zones 4994 */ 4995 if (!FDEV(dev_idx).zone_capacity_blocks) 4996 return sbi->segs_per_sec; 4997 4998 /* Get the segment count beyond zone capacity block */ 4999 unusable_segs_in_sec = (sbi->blocks_per_blkz - 5000 FDEV(dev_idx).zone_capacity_blocks[zone_idx]) >> 5001 sbi->log_blocks_per_seg; 5002 return sbi->segs_per_sec - unusable_segs_in_sec; 5003 } 5004 5005 /* 5006 * Return the number of usable blocks in a segment. The number of blocks 5007 * returned is always equal to the number of blocks in a segment for 5008 * segments fully contained within a sequential zone capacity or a 5009 * conventional zone. For segments partially contained in a sequential 5010 * zone capacity, the number of usable blocks up to the zone capacity 5011 * is returned. 0 is returned in all other cases. 5012 */ 5013 static inline unsigned int f2fs_usable_zone_blks_in_seg( 5014 struct f2fs_sb_info *sbi, unsigned int segno) 5015 { 5016 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr; 5017 unsigned int zone_idx, dev_idx, secno; 5018 5019 secno = GET_SEC_FROM_SEG(sbi, segno); 5020 seg_start = START_BLOCK(sbi, segno); 5021 dev_idx = f2fs_target_device_index(sbi, seg_start); 5022 zone_idx = get_zone_idx(sbi, secno, dev_idx); 5023 5024 /* 5025 * Conventional zone's capacity is always equal to zone size, 5026 * so, blocks per segment is unchanged. 5027 */ 5028 if (is_conv_zone(sbi, zone_idx, dev_idx)) 5029 return sbi->blocks_per_seg; 5030 5031 if (!FDEV(dev_idx).zone_capacity_blocks) 5032 return sbi->blocks_per_seg; 5033 5034 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); 5035 sec_cap_blkaddr = sec_start_blkaddr + 5036 FDEV(dev_idx).zone_capacity_blocks[zone_idx]; 5037 5038 /* 5039 * If segment starts before zone capacity and spans beyond 5040 * zone capacity, then usable blocks are from seg start to 5041 * zone capacity. If the segment starts after the zone capacity, 5042 * then there are no usable blocks. 5043 */ 5044 if (seg_start >= sec_cap_blkaddr) 5045 return 0; 5046 if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr) 5047 return sec_cap_blkaddr - seg_start; 5048 5049 return sbi->blocks_per_seg; 5050 } 5051 #else 5052 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) 5053 { 5054 return 0; 5055 } 5056 5057 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) 5058 { 5059 return 0; 5060 } 5061 5062 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi, 5063 unsigned int segno) 5064 { 5065 return 0; 5066 } 5067 5068 static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi, 5069 unsigned int segno) 5070 { 5071 return 0; 5072 } 5073 #endif 5074 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, 5075 unsigned int segno) 5076 { 5077 if (f2fs_sb_has_blkzoned(sbi)) 5078 return f2fs_usable_zone_blks_in_seg(sbi, segno); 5079 5080 return sbi->blocks_per_seg; 5081 } 5082 5083 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi, 5084 unsigned int segno) 5085 { 5086 if (f2fs_sb_has_blkzoned(sbi)) 5087 return f2fs_usable_zone_segs_in_sec(sbi, segno); 5088 5089 return sbi->segs_per_sec; 5090 } 5091 5092 /* 5093 * Update min, max modified time for cost-benefit GC algorithm 5094 */ 5095 static void init_min_max_mtime(struct f2fs_sb_info *sbi) 5096 { 5097 struct sit_info *sit_i = SIT_I(sbi); 5098 unsigned int segno; 5099 5100 down_write(&sit_i->sentry_lock); 5101 5102 sit_i->min_mtime = ULLONG_MAX; 5103 5104 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { 5105 unsigned int i; 5106 unsigned long long mtime = 0; 5107 5108 for (i = 0; i < sbi->segs_per_sec; i++) 5109 mtime += get_seg_entry(sbi, segno + i)->mtime; 5110 5111 mtime = div_u64(mtime, sbi->segs_per_sec); 5112 5113 if (sit_i->min_mtime > mtime) 5114 sit_i->min_mtime = mtime; 5115 } 5116 sit_i->max_mtime = get_mtime(sbi, false); 5117 sit_i->dirty_max_mtime = 0; 5118 up_write(&sit_i->sentry_lock); 5119 } 5120 5121 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) 5122 { 5123 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 5124 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 5125 struct f2fs_sm_info *sm_info; 5126 int err; 5127 5128 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); 5129 if (!sm_info) 5130 return -ENOMEM; 5131 5132 /* init sm info */ 5133 sbi->sm_info = sm_info; 5134 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 5135 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 5136 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 5137 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 5138 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 5139 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 5140 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 5141 sm_info->rec_prefree_segments = sm_info->main_segments * 5142 DEF_RECLAIM_PREFREE_SEGMENTS / 100; 5143 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS) 5144 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; 5145 5146 if (!f2fs_lfs_mode(sbi)) 5147 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC; 5148 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 5149 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; 5150 sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec; 5151 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS; 5152 sm_info->min_ssr_sections = reserved_sections(sbi); 5153 5154 INIT_LIST_HEAD(&sm_info->sit_entry_set); 5155 5156 init_rwsem(&sm_info->curseg_lock); 5157 5158 if (!f2fs_readonly(sbi->sb)) { 5159 err = f2fs_create_flush_cmd_control(sbi); 5160 if (err) 5161 return err; 5162 } 5163 5164 err = create_discard_cmd_control(sbi); 5165 if (err) 5166 return err; 5167 5168 err = build_sit_info(sbi); 5169 if (err) 5170 return err; 5171 err = build_free_segmap(sbi); 5172 if (err) 5173 return err; 5174 err = build_curseg(sbi); 5175 if (err) 5176 return err; 5177 5178 /* reinit free segmap based on SIT */ 5179 err = build_sit_entries(sbi); 5180 if (err) 5181 return err; 5182 5183 init_free_segmap(sbi); 5184 err = build_dirty_segmap(sbi); 5185 if (err) 5186 return err; 5187 5188 err = sanity_check_curseg(sbi); 5189 if (err) 5190 return err; 5191 5192 init_min_max_mtime(sbi); 5193 return 0; 5194 } 5195 5196 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 5197 enum dirty_type dirty_type) 5198 { 5199 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5200 5201 mutex_lock(&dirty_i->seglist_lock); 5202 kvfree(dirty_i->dirty_segmap[dirty_type]); 5203 dirty_i->nr_dirty[dirty_type] = 0; 5204 mutex_unlock(&dirty_i->seglist_lock); 5205 } 5206 5207 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 5208 { 5209 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5210 5211 kvfree(dirty_i->victim_secmap); 5212 } 5213 5214 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 5215 { 5216 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5217 int i; 5218 5219 if (!dirty_i) 5220 return; 5221 5222 /* discard pre-free/dirty segments list */ 5223 for (i = 0; i < NR_DIRTY_TYPE; i++) 5224 discard_dirty_segmap(sbi, i); 5225 5226 if (__is_large_section(sbi)) { 5227 mutex_lock(&dirty_i->seglist_lock); 5228 kvfree(dirty_i->dirty_secmap); 5229 mutex_unlock(&dirty_i->seglist_lock); 5230 } 5231 5232 destroy_victim_secmap(sbi); 5233 SM_I(sbi)->dirty_info = NULL; 5234 kfree(dirty_i); 5235 } 5236 5237 static void destroy_curseg(struct f2fs_sb_info *sbi) 5238 { 5239 struct curseg_info *array = SM_I(sbi)->curseg_array; 5240 int i; 5241 5242 if (!array) 5243 return; 5244 SM_I(sbi)->curseg_array = NULL; 5245 for (i = 0; i < NR_CURSEG_TYPE; i++) { 5246 kfree(array[i].sum_blk); 5247 kfree(array[i].journal); 5248 } 5249 kfree(array); 5250 } 5251 5252 static void destroy_free_segmap(struct f2fs_sb_info *sbi) 5253 { 5254 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 5255 5256 if (!free_i) 5257 return; 5258 SM_I(sbi)->free_info = NULL; 5259 kvfree(free_i->free_segmap); 5260 kvfree(free_i->free_secmap); 5261 kfree(free_i); 5262 } 5263 5264 static void destroy_sit_info(struct f2fs_sb_info *sbi) 5265 { 5266 struct sit_info *sit_i = SIT_I(sbi); 5267 5268 if (!sit_i) 5269 return; 5270 5271 if (sit_i->sentries) 5272 kvfree(sit_i->bitmap); 5273 kfree(sit_i->tmp_map); 5274 5275 kvfree(sit_i->sentries); 5276 kvfree(sit_i->sec_entries); 5277 kvfree(sit_i->dirty_sentries_bitmap); 5278 5279 SM_I(sbi)->sit_info = NULL; 5280 kvfree(sit_i->sit_bitmap); 5281 #ifdef CONFIG_F2FS_CHECK_FS 5282 kvfree(sit_i->sit_bitmap_mir); 5283 kvfree(sit_i->invalid_segmap); 5284 #endif 5285 kfree(sit_i); 5286 } 5287 5288 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) 5289 { 5290 struct f2fs_sm_info *sm_info = SM_I(sbi); 5291 5292 if (!sm_info) 5293 return; 5294 f2fs_destroy_flush_cmd_control(sbi, true); 5295 destroy_discard_cmd_control(sbi); 5296 destroy_dirty_segmap(sbi); 5297 destroy_curseg(sbi); 5298 destroy_free_segmap(sbi); 5299 destroy_sit_info(sbi); 5300 sbi->sm_info = NULL; 5301 kfree(sm_info); 5302 } 5303 5304 int __init f2fs_create_segment_manager_caches(void) 5305 { 5306 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry", 5307 sizeof(struct discard_entry)); 5308 if (!discard_entry_slab) 5309 goto fail; 5310 5311 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd", 5312 sizeof(struct discard_cmd)); 5313 if (!discard_cmd_slab) 5314 goto destroy_discard_entry; 5315 5316 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set", 5317 sizeof(struct sit_entry_set)); 5318 if (!sit_entry_set_slab) 5319 goto destroy_discard_cmd; 5320 5321 inmem_entry_slab = f2fs_kmem_cache_create("f2fs_inmem_page_entry", 5322 sizeof(struct inmem_pages)); 5323 if (!inmem_entry_slab) 5324 goto destroy_sit_entry_set; 5325 return 0; 5326 5327 destroy_sit_entry_set: 5328 kmem_cache_destroy(sit_entry_set_slab); 5329 destroy_discard_cmd: 5330 kmem_cache_destroy(discard_cmd_slab); 5331 destroy_discard_entry: 5332 kmem_cache_destroy(discard_entry_slab); 5333 fail: 5334 return -ENOMEM; 5335 } 5336 5337 void f2fs_destroy_segment_manager_caches(void) 5338 { 5339 kmem_cache_destroy(sit_entry_set_slab); 5340 kmem_cache_destroy(discard_cmd_slab); 5341 kmem_cache_destroy(discard_entry_slab); 5342 kmem_cache_destroy(inmem_entry_slab); 5343 } 5344