1 /* 2 * fs/f2fs/segment.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/fs.h> 12 #include <linux/f2fs_fs.h> 13 #include <linux/bio.h> 14 #include <linux/blkdev.h> 15 #include <linux/prefetch.h> 16 #include <linux/kthread.h> 17 #include <linux/swap.h> 18 #include <linux/timer.h> 19 #include <linux/freezer.h> 20 #include <linux/sched/signal.h> 21 22 #include "f2fs.h" 23 #include "segment.h" 24 #include "node.h" 25 #include "gc.h" 26 #include "trace.h" 27 #include <trace/events/f2fs.h> 28 29 #define __reverse_ffz(x) __reverse_ffs(~(x)) 30 31 static struct kmem_cache *discard_entry_slab; 32 static struct kmem_cache *discard_cmd_slab; 33 static struct kmem_cache *sit_entry_set_slab; 34 static struct kmem_cache *inmem_entry_slab; 35 36 static unsigned long __reverse_ulong(unsigned char *str) 37 { 38 unsigned long tmp = 0; 39 int shift = 24, idx = 0; 40 41 #if BITS_PER_LONG == 64 42 shift = 56; 43 #endif 44 while (shift >= 0) { 45 tmp |= (unsigned long)str[idx++] << shift; 46 shift -= BITS_PER_BYTE; 47 } 48 return tmp; 49 } 50 51 /* 52 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 53 * MSB and LSB are reversed in a byte by f2fs_set_bit. 54 */ 55 static inline unsigned long __reverse_ffs(unsigned long word) 56 { 57 int num = 0; 58 59 #if BITS_PER_LONG == 64 60 if ((word & 0xffffffff00000000UL) == 0) 61 num += 32; 62 else 63 word >>= 32; 64 #endif 65 if ((word & 0xffff0000) == 0) 66 num += 16; 67 else 68 word >>= 16; 69 70 if ((word & 0xff00) == 0) 71 num += 8; 72 else 73 word >>= 8; 74 75 if ((word & 0xf0) == 0) 76 num += 4; 77 else 78 word >>= 4; 79 80 if ((word & 0xc) == 0) 81 num += 2; 82 else 83 word >>= 2; 84 85 if ((word & 0x2) == 0) 86 num += 1; 87 return num; 88 } 89 90 /* 91 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because 92 * f2fs_set_bit makes MSB and LSB reversed in a byte. 93 * @size must be integral times of unsigned long. 94 * Example: 95 * MSB <--> LSB 96 * f2fs_set_bit(0, bitmap) => 1000 0000 97 * f2fs_set_bit(7, bitmap) => 0000 0001 98 */ 99 static unsigned long __find_rev_next_bit(const unsigned long *addr, 100 unsigned long size, unsigned long offset) 101 { 102 const unsigned long *p = addr + BIT_WORD(offset); 103 unsigned long result = size; 104 unsigned long tmp; 105 106 if (offset >= size) 107 return size; 108 109 size -= (offset & ~(BITS_PER_LONG - 1)); 110 offset %= BITS_PER_LONG; 111 112 while (1) { 113 if (*p == 0) 114 goto pass; 115 116 tmp = __reverse_ulong((unsigned char *)p); 117 118 tmp &= ~0UL >> offset; 119 if (size < BITS_PER_LONG) 120 tmp &= (~0UL << (BITS_PER_LONG - size)); 121 if (tmp) 122 goto found; 123 pass: 124 if (size <= BITS_PER_LONG) 125 break; 126 size -= BITS_PER_LONG; 127 offset = 0; 128 p++; 129 } 130 return result; 131 found: 132 return result - size + __reverse_ffs(tmp); 133 } 134 135 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 136 unsigned long size, unsigned long offset) 137 { 138 const unsigned long *p = addr + BIT_WORD(offset); 139 unsigned long result = size; 140 unsigned long tmp; 141 142 if (offset >= size) 143 return size; 144 145 size -= (offset & ~(BITS_PER_LONG - 1)); 146 offset %= BITS_PER_LONG; 147 148 while (1) { 149 if (*p == ~0UL) 150 goto pass; 151 152 tmp = __reverse_ulong((unsigned char *)p); 153 154 if (offset) 155 tmp |= ~0UL << (BITS_PER_LONG - offset); 156 if (size < BITS_PER_LONG) 157 tmp |= ~0UL >> size; 158 if (tmp != ~0UL) 159 goto found; 160 pass: 161 if (size <= BITS_PER_LONG) 162 break; 163 size -= BITS_PER_LONG; 164 offset = 0; 165 p++; 166 } 167 return result; 168 found: 169 return result - size + __reverse_ffz(tmp); 170 } 171 172 bool need_SSR(struct f2fs_sb_info *sbi) 173 { 174 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); 175 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); 176 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); 177 178 if (test_opt(sbi, LFS)) 179 return false; 180 if (sbi->gc_mode == GC_URGENT) 181 return true; 182 183 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + 184 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); 185 } 186 187 void register_inmem_page(struct inode *inode, struct page *page) 188 { 189 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 190 struct f2fs_inode_info *fi = F2FS_I(inode); 191 struct inmem_pages *new; 192 193 f2fs_trace_pid(page); 194 195 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); 196 SetPagePrivate(page); 197 198 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); 199 200 /* add atomic page indices to the list */ 201 new->page = page; 202 INIT_LIST_HEAD(&new->list); 203 204 /* increase reference count with clean state */ 205 mutex_lock(&fi->inmem_lock); 206 get_page(page); 207 list_add_tail(&new->list, &fi->inmem_pages); 208 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 209 if (list_empty(&fi->inmem_ilist)) 210 list_add_tail(&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE]); 211 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 212 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); 213 mutex_unlock(&fi->inmem_lock); 214 215 trace_f2fs_register_inmem_page(page, INMEM); 216 } 217 218 static int __revoke_inmem_pages(struct inode *inode, 219 struct list_head *head, bool drop, bool recover) 220 { 221 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 222 struct inmem_pages *cur, *tmp; 223 int err = 0; 224 225 list_for_each_entry_safe(cur, tmp, head, list) { 226 struct page *page = cur->page; 227 228 if (drop) 229 trace_f2fs_commit_inmem_page(page, INMEM_DROP); 230 231 lock_page(page); 232 233 f2fs_wait_on_page_writeback(page, DATA, true); 234 235 if (recover) { 236 struct dnode_of_data dn; 237 struct node_info ni; 238 239 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE); 240 retry: 241 set_new_dnode(&dn, inode, NULL, NULL, 0); 242 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 243 if (err) { 244 if (err == -ENOMEM) { 245 congestion_wait(BLK_RW_ASYNC, HZ/50); 246 cond_resched(); 247 goto retry; 248 } 249 err = -EAGAIN; 250 goto next; 251 } 252 get_node_info(sbi, dn.nid, &ni); 253 if (cur->old_addr == NEW_ADDR) { 254 invalidate_blocks(sbi, dn.data_blkaddr); 255 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 256 } else 257 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, 258 cur->old_addr, ni.version, true, true); 259 f2fs_put_dnode(&dn); 260 } 261 next: 262 /* we don't need to invalidate this in the sccessful status */ 263 if (drop || recover) 264 ClearPageUptodate(page); 265 set_page_private(page, 0); 266 ClearPagePrivate(page); 267 f2fs_put_page(page, 1); 268 269 list_del(&cur->list); 270 kmem_cache_free(inmem_entry_slab, cur); 271 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); 272 } 273 return err; 274 } 275 276 void drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) 277 { 278 struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; 279 struct inode *inode; 280 struct f2fs_inode_info *fi; 281 next: 282 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 283 if (list_empty(head)) { 284 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 285 return; 286 } 287 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); 288 inode = igrab(&fi->vfs_inode); 289 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 290 291 if (inode) { 292 if (gc_failure) { 293 if (fi->i_gc_failures[GC_FAILURE_ATOMIC]) 294 goto drop; 295 goto skip; 296 } 297 drop: 298 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST); 299 drop_inmem_pages(inode); 300 iput(inode); 301 } 302 skip: 303 congestion_wait(BLK_RW_ASYNC, HZ/50); 304 cond_resched(); 305 goto next; 306 } 307 308 void drop_inmem_pages(struct inode *inode) 309 { 310 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 311 struct f2fs_inode_info *fi = F2FS_I(inode); 312 313 mutex_lock(&fi->inmem_lock); 314 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false); 315 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 316 if (!list_empty(&fi->inmem_ilist)) 317 list_del_init(&fi->inmem_ilist); 318 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 319 mutex_unlock(&fi->inmem_lock); 320 321 clear_inode_flag(inode, FI_ATOMIC_FILE); 322 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; 323 stat_dec_atomic_write(inode); 324 } 325 326 void drop_inmem_page(struct inode *inode, struct page *page) 327 { 328 struct f2fs_inode_info *fi = F2FS_I(inode); 329 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 330 struct list_head *head = &fi->inmem_pages; 331 struct inmem_pages *cur = NULL; 332 333 f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); 334 335 mutex_lock(&fi->inmem_lock); 336 list_for_each_entry(cur, head, list) { 337 if (cur->page == page) 338 break; 339 } 340 341 f2fs_bug_on(sbi, list_empty(head) || cur->page != page); 342 list_del(&cur->list); 343 mutex_unlock(&fi->inmem_lock); 344 345 dec_page_count(sbi, F2FS_INMEM_PAGES); 346 kmem_cache_free(inmem_entry_slab, cur); 347 348 ClearPageUptodate(page); 349 set_page_private(page, 0); 350 ClearPagePrivate(page); 351 f2fs_put_page(page, 0); 352 353 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE); 354 } 355 356 static int __commit_inmem_pages(struct inode *inode) 357 { 358 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 359 struct f2fs_inode_info *fi = F2FS_I(inode); 360 struct inmem_pages *cur, *tmp; 361 struct f2fs_io_info fio = { 362 .sbi = sbi, 363 .ino = inode->i_ino, 364 .type = DATA, 365 .op = REQ_OP_WRITE, 366 .op_flags = REQ_SYNC | REQ_PRIO, 367 .io_type = FS_DATA_IO, 368 }; 369 struct list_head revoke_list; 370 pgoff_t last_idx = ULONG_MAX; 371 int err = 0; 372 373 INIT_LIST_HEAD(&revoke_list); 374 375 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) { 376 struct page *page = cur->page; 377 378 lock_page(page); 379 if (page->mapping == inode->i_mapping) { 380 trace_f2fs_commit_inmem_page(page, INMEM); 381 382 set_page_dirty(page); 383 f2fs_wait_on_page_writeback(page, DATA, true); 384 if (clear_page_dirty_for_io(page)) { 385 inode_dec_dirty_pages(inode); 386 remove_dirty_inode(inode); 387 } 388 retry: 389 fio.page = page; 390 fio.old_blkaddr = NULL_ADDR; 391 fio.encrypted_page = NULL; 392 fio.need_lock = LOCK_DONE; 393 err = do_write_data_page(&fio); 394 if (err) { 395 if (err == -ENOMEM) { 396 congestion_wait(BLK_RW_ASYNC, HZ/50); 397 cond_resched(); 398 goto retry; 399 } 400 unlock_page(page); 401 break; 402 } 403 /* record old blkaddr for revoking */ 404 cur->old_addr = fio.old_blkaddr; 405 last_idx = page->index; 406 } 407 unlock_page(page); 408 list_move_tail(&cur->list, &revoke_list); 409 } 410 411 if (last_idx != ULONG_MAX) 412 f2fs_submit_merged_write_cond(sbi, inode, 0, last_idx, DATA); 413 414 if (err) { 415 /* 416 * try to revoke all committed pages, but still we could fail 417 * due to no memory or other reason, if that happened, EAGAIN 418 * will be returned, which means in such case, transaction is 419 * already not integrity, caller should use journal to do the 420 * recovery or rewrite & commit last transaction. For other 421 * error number, revoking was done by filesystem itself. 422 */ 423 err = __revoke_inmem_pages(inode, &revoke_list, false, true); 424 425 /* drop all uncommitted pages */ 426 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false); 427 } else { 428 __revoke_inmem_pages(inode, &revoke_list, false, false); 429 } 430 431 return err; 432 } 433 434 int commit_inmem_pages(struct inode *inode) 435 { 436 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 437 struct f2fs_inode_info *fi = F2FS_I(inode); 438 int err; 439 440 f2fs_balance_fs(sbi, true); 441 f2fs_lock_op(sbi); 442 443 set_inode_flag(inode, FI_ATOMIC_COMMIT); 444 445 mutex_lock(&fi->inmem_lock); 446 err = __commit_inmem_pages(inode); 447 448 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 449 if (!list_empty(&fi->inmem_ilist)) 450 list_del_init(&fi->inmem_ilist); 451 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 452 mutex_unlock(&fi->inmem_lock); 453 454 clear_inode_flag(inode, FI_ATOMIC_COMMIT); 455 456 f2fs_unlock_op(sbi); 457 return err; 458 } 459 460 /* 461 * This function balances dirty node and dentry pages. 462 * In addition, it controls garbage collection. 463 */ 464 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) 465 { 466 #ifdef CONFIG_F2FS_FAULT_INJECTION 467 if (time_to_inject(sbi, FAULT_CHECKPOINT)) { 468 f2fs_show_injection_info(FAULT_CHECKPOINT); 469 f2fs_stop_checkpoint(sbi, false); 470 } 471 #endif 472 473 /* balance_fs_bg is able to be pending */ 474 if (need && excess_cached_nats(sbi)) 475 f2fs_balance_fs_bg(sbi); 476 477 /* 478 * We should do GC or end up with checkpoint, if there are so many dirty 479 * dir/node pages without enough free segments. 480 */ 481 if (has_not_enough_free_secs(sbi, 0, 0)) { 482 mutex_lock(&sbi->gc_mutex); 483 f2fs_gc(sbi, false, false, NULL_SEGNO); 484 } 485 } 486 487 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) 488 { 489 /* try to shrink extent cache when there is no enough memory */ 490 if (!available_free_memory(sbi, EXTENT_CACHE)) 491 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); 492 493 /* check the # of cached NAT entries */ 494 if (!available_free_memory(sbi, NAT_ENTRIES)) 495 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); 496 497 if (!available_free_memory(sbi, FREE_NIDS)) 498 try_to_free_nids(sbi, MAX_FREE_NIDS); 499 else 500 build_free_nids(sbi, false, false); 501 502 if (!is_idle(sbi) && !excess_dirty_nats(sbi)) 503 return; 504 505 /* checkpoint is the only way to shrink partial cached entries */ 506 if (!available_free_memory(sbi, NAT_ENTRIES) || 507 !available_free_memory(sbi, INO_ENTRIES) || 508 excess_prefree_segs(sbi) || 509 excess_dirty_nats(sbi) || 510 f2fs_time_over(sbi, CP_TIME)) { 511 if (test_opt(sbi, DATA_FLUSH)) { 512 struct blk_plug plug; 513 514 blk_start_plug(&plug); 515 sync_dirty_inodes(sbi, FILE_INODE); 516 blk_finish_plug(&plug); 517 } 518 f2fs_sync_fs(sbi->sb, true); 519 stat_inc_bg_cp_count(sbi->stat_info); 520 } 521 } 522 523 static int __submit_flush_wait(struct f2fs_sb_info *sbi, 524 struct block_device *bdev) 525 { 526 struct bio *bio = f2fs_bio_alloc(sbi, 0, true); 527 int ret; 528 529 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH; 530 bio_set_dev(bio, bdev); 531 ret = submit_bio_wait(bio); 532 bio_put(bio); 533 534 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), 535 test_opt(sbi, FLUSH_MERGE), ret); 536 return ret; 537 } 538 539 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) 540 { 541 int ret = 0; 542 int i; 543 544 if (!sbi->s_ndevs) 545 return __submit_flush_wait(sbi, sbi->sb->s_bdev); 546 547 for (i = 0; i < sbi->s_ndevs; i++) { 548 if (!is_dirty_device(sbi, ino, i, FLUSH_INO)) 549 continue; 550 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 551 if (ret) 552 break; 553 } 554 return ret; 555 } 556 557 static int issue_flush_thread(void *data) 558 { 559 struct f2fs_sb_info *sbi = data; 560 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 561 wait_queue_head_t *q = &fcc->flush_wait_queue; 562 repeat: 563 if (kthread_should_stop()) 564 return 0; 565 566 sb_start_intwrite(sbi->sb); 567 568 if (!llist_empty(&fcc->issue_list)) { 569 struct flush_cmd *cmd, *next; 570 int ret; 571 572 fcc->dispatch_list = llist_del_all(&fcc->issue_list); 573 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); 574 575 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode); 576 577 ret = submit_flush_wait(sbi, cmd->ino); 578 atomic_inc(&fcc->issued_flush); 579 580 llist_for_each_entry_safe(cmd, next, 581 fcc->dispatch_list, llnode) { 582 cmd->ret = ret; 583 complete(&cmd->wait); 584 } 585 fcc->dispatch_list = NULL; 586 } 587 588 sb_end_intwrite(sbi->sb); 589 590 wait_event_interruptible(*q, 591 kthread_should_stop() || !llist_empty(&fcc->issue_list)); 592 goto repeat; 593 } 594 595 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) 596 { 597 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 598 struct flush_cmd cmd; 599 int ret; 600 601 if (test_opt(sbi, NOBARRIER)) 602 return 0; 603 604 if (!test_opt(sbi, FLUSH_MERGE)) { 605 ret = submit_flush_wait(sbi, ino); 606 atomic_inc(&fcc->issued_flush); 607 return ret; 608 } 609 610 if (atomic_inc_return(&fcc->issing_flush) == 1 || sbi->s_ndevs > 1) { 611 ret = submit_flush_wait(sbi, ino); 612 atomic_dec(&fcc->issing_flush); 613 614 atomic_inc(&fcc->issued_flush); 615 return ret; 616 } 617 618 cmd.ino = ino; 619 init_completion(&cmd.wait); 620 621 llist_add(&cmd.llnode, &fcc->issue_list); 622 623 /* update issue_list before we wake up issue_flush thread */ 624 smp_mb(); 625 626 if (waitqueue_active(&fcc->flush_wait_queue)) 627 wake_up(&fcc->flush_wait_queue); 628 629 if (fcc->f2fs_issue_flush) { 630 wait_for_completion(&cmd.wait); 631 atomic_dec(&fcc->issing_flush); 632 } else { 633 struct llist_node *list; 634 635 list = llist_del_all(&fcc->issue_list); 636 if (!list) { 637 wait_for_completion(&cmd.wait); 638 atomic_dec(&fcc->issing_flush); 639 } else { 640 struct flush_cmd *tmp, *next; 641 642 ret = submit_flush_wait(sbi, ino); 643 644 llist_for_each_entry_safe(tmp, next, list, llnode) { 645 if (tmp == &cmd) { 646 cmd.ret = ret; 647 atomic_dec(&fcc->issing_flush); 648 continue; 649 } 650 tmp->ret = ret; 651 complete(&tmp->wait); 652 } 653 } 654 } 655 656 return cmd.ret; 657 } 658 659 int create_flush_cmd_control(struct f2fs_sb_info *sbi) 660 { 661 dev_t dev = sbi->sb->s_bdev->bd_dev; 662 struct flush_cmd_control *fcc; 663 int err = 0; 664 665 if (SM_I(sbi)->fcc_info) { 666 fcc = SM_I(sbi)->fcc_info; 667 if (fcc->f2fs_issue_flush) 668 return err; 669 goto init_thread; 670 } 671 672 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); 673 if (!fcc) 674 return -ENOMEM; 675 atomic_set(&fcc->issued_flush, 0); 676 atomic_set(&fcc->issing_flush, 0); 677 init_waitqueue_head(&fcc->flush_wait_queue); 678 init_llist_head(&fcc->issue_list); 679 SM_I(sbi)->fcc_info = fcc; 680 if (!test_opt(sbi, FLUSH_MERGE)) 681 return err; 682 683 init_thread: 684 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, 685 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); 686 if (IS_ERR(fcc->f2fs_issue_flush)) { 687 err = PTR_ERR(fcc->f2fs_issue_flush); 688 kfree(fcc); 689 SM_I(sbi)->fcc_info = NULL; 690 return err; 691 } 692 693 return err; 694 } 695 696 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) 697 { 698 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 699 700 if (fcc && fcc->f2fs_issue_flush) { 701 struct task_struct *flush_thread = fcc->f2fs_issue_flush; 702 703 fcc->f2fs_issue_flush = NULL; 704 kthread_stop(flush_thread); 705 } 706 if (free) { 707 kfree(fcc); 708 SM_I(sbi)->fcc_info = NULL; 709 } 710 } 711 712 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) 713 { 714 int ret = 0, i; 715 716 if (!sbi->s_ndevs) 717 return 0; 718 719 for (i = 1; i < sbi->s_ndevs; i++) { 720 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) 721 continue; 722 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 723 if (ret) 724 break; 725 726 spin_lock(&sbi->dev_lock); 727 f2fs_clear_bit(i, (char *)&sbi->dirty_device); 728 spin_unlock(&sbi->dev_lock); 729 } 730 731 return ret; 732 } 733 734 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 735 enum dirty_type dirty_type) 736 { 737 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 738 739 /* need not be added */ 740 if (IS_CURSEG(sbi, segno)) 741 return; 742 743 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 744 dirty_i->nr_dirty[dirty_type]++; 745 746 if (dirty_type == DIRTY) { 747 struct seg_entry *sentry = get_seg_entry(sbi, segno); 748 enum dirty_type t = sentry->type; 749 750 if (unlikely(t >= DIRTY)) { 751 f2fs_bug_on(sbi, 1); 752 return; 753 } 754 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 755 dirty_i->nr_dirty[t]++; 756 } 757 } 758 759 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 760 enum dirty_type dirty_type) 761 { 762 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 763 764 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 765 dirty_i->nr_dirty[dirty_type]--; 766 767 if (dirty_type == DIRTY) { 768 struct seg_entry *sentry = get_seg_entry(sbi, segno); 769 enum dirty_type t = sentry->type; 770 771 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 772 dirty_i->nr_dirty[t]--; 773 774 if (get_valid_blocks(sbi, segno, true) == 0) 775 clear_bit(GET_SEC_FROM_SEG(sbi, segno), 776 dirty_i->victim_secmap); 777 } 778 } 779 780 /* 781 * Should not occur error such as -ENOMEM. 782 * Adding dirty entry into seglist is not critical operation. 783 * If a given segment is one of current working segments, it won't be added. 784 */ 785 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 786 { 787 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 788 unsigned short valid_blocks; 789 790 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) 791 return; 792 793 mutex_lock(&dirty_i->seglist_lock); 794 795 valid_blocks = get_valid_blocks(sbi, segno, false); 796 797 if (valid_blocks == 0) { 798 __locate_dirty_segment(sbi, segno, PRE); 799 __remove_dirty_segment(sbi, segno, DIRTY); 800 } else if (valid_blocks < sbi->blocks_per_seg) { 801 __locate_dirty_segment(sbi, segno, DIRTY); 802 } else { 803 /* Recovery routine with SSR needs this */ 804 __remove_dirty_segment(sbi, segno, DIRTY); 805 } 806 807 mutex_unlock(&dirty_i->seglist_lock); 808 } 809 810 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, 811 struct block_device *bdev, block_t lstart, 812 block_t start, block_t len) 813 { 814 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 815 struct list_head *pend_list; 816 struct discard_cmd *dc; 817 818 f2fs_bug_on(sbi, !len); 819 820 pend_list = &dcc->pend_list[plist_idx(len)]; 821 822 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS); 823 INIT_LIST_HEAD(&dc->list); 824 dc->bdev = bdev; 825 dc->lstart = lstart; 826 dc->start = start; 827 dc->len = len; 828 dc->ref = 0; 829 dc->state = D_PREP; 830 dc->error = 0; 831 init_completion(&dc->wait); 832 list_add_tail(&dc->list, pend_list); 833 atomic_inc(&dcc->discard_cmd_cnt); 834 dcc->undiscard_blks += len; 835 836 return dc; 837 } 838 839 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, 840 struct block_device *bdev, block_t lstart, 841 block_t start, block_t len, 842 struct rb_node *parent, struct rb_node **p) 843 { 844 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 845 struct discard_cmd *dc; 846 847 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); 848 849 rb_link_node(&dc->rb_node, parent, p); 850 rb_insert_color(&dc->rb_node, &dcc->root); 851 852 return dc; 853 } 854 855 static void __detach_discard_cmd(struct discard_cmd_control *dcc, 856 struct discard_cmd *dc) 857 { 858 if (dc->state == D_DONE) 859 atomic_dec(&dcc->issing_discard); 860 861 list_del(&dc->list); 862 rb_erase(&dc->rb_node, &dcc->root); 863 dcc->undiscard_blks -= dc->len; 864 865 kmem_cache_free(discard_cmd_slab, dc); 866 867 atomic_dec(&dcc->discard_cmd_cnt); 868 } 869 870 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, 871 struct discard_cmd *dc) 872 { 873 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 874 875 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len); 876 877 f2fs_bug_on(sbi, dc->ref); 878 879 if (dc->error == -EOPNOTSUPP) 880 dc->error = 0; 881 882 if (dc->error) 883 f2fs_msg(sbi->sb, KERN_INFO, 884 "Issue discard(%u, %u, %u) failed, ret: %d", 885 dc->lstart, dc->start, dc->len, dc->error); 886 __detach_discard_cmd(dcc, dc); 887 } 888 889 static void f2fs_submit_discard_endio(struct bio *bio) 890 { 891 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; 892 893 dc->error = blk_status_to_errno(bio->bi_status); 894 dc->state = D_DONE; 895 complete_all(&dc->wait); 896 bio_put(bio); 897 } 898 899 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, 900 block_t start, block_t end) 901 { 902 #ifdef CONFIG_F2FS_CHECK_FS 903 struct seg_entry *sentry; 904 unsigned int segno; 905 block_t blk = start; 906 unsigned long offset, size, max_blocks = sbi->blocks_per_seg; 907 unsigned long *map; 908 909 while (blk < end) { 910 segno = GET_SEGNO(sbi, blk); 911 sentry = get_seg_entry(sbi, segno); 912 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); 913 914 if (end < START_BLOCK(sbi, segno + 1)) 915 size = GET_BLKOFF_FROM_SEG0(sbi, end); 916 else 917 size = max_blocks; 918 map = (unsigned long *)(sentry->cur_valid_map); 919 offset = __find_rev_next_bit(map, size, offset); 920 f2fs_bug_on(sbi, offset != size); 921 blk = START_BLOCK(sbi, segno + 1); 922 } 923 #endif 924 } 925 926 static void __init_discard_policy(struct f2fs_sb_info *sbi, 927 struct discard_policy *dpolicy, 928 int discard_type, unsigned int granularity) 929 { 930 /* common policy */ 931 dpolicy->type = discard_type; 932 dpolicy->sync = true; 933 dpolicy->granularity = granularity; 934 935 dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST; 936 dpolicy->io_aware_gran = MAX_PLIST_NUM; 937 938 if (discard_type == DPOLICY_BG) { 939 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME; 940 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME; 941 dpolicy->io_aware = true; 942 dpolicy->sync = false; 943 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) { 944 dpolicy->granularity = 1; 945 dpolicy->max_interval = DEF_MIN_DISCARD_ISSUE_TIME; 946 } 947 } else if (discard_type == DPOLICY_FORCE) { 948 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME; 949 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME; 950 dpolicy->io_aware = false; 951 } else if (discard_type == DPOLICY_FSTRIM) { 952 dpolicy->io_aware = false; 953 } else if (discard_type == DPOLICY_UMOUNT) { 954 dpolicy->max_requests = UINT_MAX; 955 dpolicy->io_aware = false; 956 } 957 } 958 959 960 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ 961 static void __submit_discard_cmd(struct f2fs_sb_info *sbi, 962 struct discard_policy *dpolicy, 963 struct discard_cmd *dc) 964 { 965 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 966 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 967 &(dcc->fstrim_list) : &(dcc->wait_list); 968 struct bio *bio = NULL; 969 int flag = dpolicy->sync ? REQ_SYNC : 0; 970 971 if (dc->state != D_PREP) 972 return; 973 974 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 975 return; 976 977 trace_f2fs_issue_discard(dc->bdev, dc->start, dc->len); 978 979 dc->error = __blkdev_issue_discard(dc->bdev, 980 SECTOR_FROM_BLOCK(dc->start), 981 SECTOR_FROM_BLOCK(dc->len), 982 GFP_NOFS, 0, &bio); 983 if (!dc->error) { 984 /* should keep before submission to avoid D_DONE right away */ 985 dc->state = D_SUBMIT; 986 atomic_inc(&dcc->issued_discard); 987 atomic_inc(&dcc->issing_discard); 988 if (bio) { 989 bio->bi_private = dc; 990 bio->bi_end_io = f2fs_submit_discard_endio; 991 bio->bi_opf |= flag; 992 submit_bio(bio); 993 list_move_tail(&dc->list, wait_list); 994 __check_sit_bitmap(sbi, dc->start, dc->start + dc->len); 995 996 f2fs_update_iostat(sbi, FS_DISCARD, 1); 997 } 998 } else { 999 __remove_discard_cmd(sbi, dc); 1000 } 1001 } 1002 1003 static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi, 1004 struct block_device *bdev, block_t lstart, 1005 block_t start, block_t len, 1006 struct rb_node **insert_p, 1007 struct rb_node *insert_parent) 1008 { 1009 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1010 struct rb_node **p; 1011 struct rb_node *parent = NULL; 1012 struct discard_cmd *dc = NULL; 1013 1014 if (insert_p && insert_parent) { 1015 parent = insert_parent; 1016 p = insert_p; 1017 goto do_insert; 1018 } 1019 1020 p = __lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, lstart); 1021 do_insert: 1022 dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, p); 1023 if (!dc) 1024 return NULL; 1025 1026 return dc; 1027 } 1028 1029 static void __relocate_discard_cmd(struct discard_cmd_control *dcc, 1030 struct discard_cmd *dc) 1031 { 1032 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]); 1033 } 1034 1035 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, 1036 struct discard_cmd *dc, block_t blkaddr) 1037 { 1038 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1039 struct discard_info di = dc->di; 1040 bool modified = false; 1041 1042 if (dc->state == D_DONE || dc->len == 1) { 1043 __remove_discard_cmd(sbi, dc); 1044 return; 1045 } 1046 1047 dcc->undiscard_blks -= di.len; 1048 1049 if (blkaddr > di.lstart) { 1050 dc->len = blkaddr - dc->lstart; 1051 dcc->undiscard_blks += dc->len; 1052 __relocate_discard_cmd(dcc, dc); 1053 modified = true; 1054 } 1055 1056 if (blkaddr < di.lstart + di.len - 1) { 1057 if (modified) { 1058 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, 1059 di.start + blkaddr + 1 - di.lstart, 1060 di.lstart + di.len - 1 - blkaddr, 1061 NULL, NULL); 1062 } else { 1063 dc->lstart++; 1064 dc->len--; 1065 dc->start++; 1066 dcc->undiscard_blks += dc->len; 1067 __relocate_discard_cmd(dcc, dc); 1068 } 1069 } 1070 } 1071 1072 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1073 struct block_device *bdev, block_t lstart, 1074 block_t start, block_t len) 1075 { 1076 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1077 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1078 struct discard_cmd *dc; 1079 struct discard_info di = {0}; 1080 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1081 block_t end = lstart + len; 1082 1083 mutex_lock(&dcc->cmd_lock); 1084 1085 dc = (struct discard_cmd *)__lookup_rb_tree_ret(&dcc->root, 1086 NULL, lstart, 1087 (struct rb_entry **)&prev_dc, 1088 (struct rb_entry **)&next_dc, 1089 &insert_p, &insert_parent, true); 1090 if (dc) 1091 prev_dc = dc; 1092 1093 if (!prev_dc) { 1094 di.lstart = lstart; 1095 di.len = next_dc ? next_dc->lstart - lstart : len; 1096 di.len = min(di.len, len); 1097 di.start = start; 1098 } 1099 1100 while (1) { 1101 struct rb_node *node; 1102 bool merged = false; 1103 struct discard_cmd *tdc = NULL; 1104 1105 if (prev_dc) { 1106 di.lstart = prev_dc->lstart + prev_dc->len; 1107 if (di.lstart < lstart) 1108 di.lstart = lstart; 1109 if (di.lstart >= end) 1110 break; 1111 1112 if (!next_dc || next_dc->lstart > end) 1113 di.len = end - di.lstart; 1114 else 1115 di.len = next_dc->lstart - di.lstart; 1116 di.start = start + di.lstart - lstart; 1117 } 1118 1119 if (!di.len) 1120 goto next; 1121 1122 if (prev_dc && prev_dc->state == D_PREP && 1123 prev_dc->bdev == bdev && 1124 __is_discard_back_mergeable(&di, &prev_dc->di)) { 1125 prev_dc->di.len += di.len; 1126 dcc->undiscard_blks += di.len; 1127 __relocate_discard_cmd(dcc, prev_dc); 1128 di = prev_dc->di; 1129 tdc = prev_dc; 1130 merged = true; 1131 } 1132 1133 if (next_dc && next_dc->state == D_PREP && 1134 next_dc->bdev == bdev && 1135 __is_discard_front_mergeable(&di, &next_dc->di)) { 1136 next_dc->di.lstart = di.lstart; 1137 next_dc->di.len += di.len; 1138 next_dc->di.start = di.start; 1139 dcc->undiscard_blks += di.len; 1140 __relocate_discard_cmd(dcc, next_dc); 1141 if (tdc) 1142 __remove_discard_cmd(sbi, tdc); 1143 merged = true; 1144 } 1145 1146 if (!merged) { 1147 __insert_discard_tree(sbi, bdev, di.lstart, di.start, 1148 di.len, NULL, NULL); 1149 } 1150 next: 1151 prev_dc = next_dc; 1152 if (!prev_dc) 1153 break; 1154 1155 node = rb_next(&prev_dc->rb_node); 1156 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1157 } 1158 1159 mutex_unlock(&dcc->cmd_lock); 1160 } 1161 1162 static int __queue_discard_cmd(struct f2fs_sb_info *sbi, 1163 struct block_device *bdev, block_t blkstart, block_t blklen) 1164 { 1165 block_t lblkstart = blkstart; 1166 1167 trace_f2fs_queue_discard(bdev, blkstart, blklen); 1168 1169 if (sbi->s_ndevs) { 1170 int devi = f2fs_target_device_index(sbi, blkstart); 1171 1172 blkstart -= FDEV(devi).start_blk; 1173 } 1174 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); 1175 return 0; 1176 } 1177 1178 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, 1179 struct discard_policy *dpolicy) 1180 { 1181 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1182 struct list_head *pend_list; 1183 struct discard_cmd *dc, *tmp; 1184 struct blk_plug plug; 1185 int i, iter = 0, issued = 0; 1186 bool io_interrupted = false; 1187 1188 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1189 if (i + 1 < dpolicy->granularity) 1190 break; 1191 pend_list = &dcc->pend_list[i]; 1192 1193 mutex_lock(&dcc->cmd_lock); 1194 if (list_empty(pend_list)) 1195 goto next; 1196 f2fs_bug_on(sbi, !__check_rb_tree_consistence(sbi, &dcc->root)); 1197 blk_start_plug(&plug); 1198 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1199 f2fs_bug_on(sbi, dc->state != D_PREP); 1200 1201 if (dpolicy->io_aware && i < dpolicy->io_aware_gran && 1202 !is_idle(sbi)) { 1203 io_interrupted = true; 1204 goto skip; 1205 } 1206 1207 __submit_discard_cmd(sbi, dpolicy, dc); 1208 issued++; 1209 skip: 1210 if (++iter >= dpolicy->max_requests) 1211 break; 1212 } 1213 blk_finish_plug(&plug); 1214 next: 1215 mutex_unlock(&dcc->cmd_lock); 1216 1217 if (iter >= dpolicy->max_requests) 1218 break; 1219 } 1220 1221 if (!issued && io_interrupted) 1222 issued = -1; 1223 1224 return issued; 1225 } 1226 1227 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) 1228 { 1229 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1230 struct list_head *pend_list; 1231 struct discard_cmd *dc, *tmp; 1232 int i; 1233 bool dropped = false; 1234 1235 mutex_lock(&dcc->cmd_lock); 1236 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1237 pend_list = &dcc->pend_list[i]; 1238 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1239 f2fs_bug_on(sbi, dc->state != D_PREP); 1240 __remove_discard_cmd(sbi, dc); 1241 dropped = true; 1242 } 1243 } 1244 mutex_unlock(&dcc->cmd_lock); 1245 1246 return dropped; 1247 } 1248 1249 void drop_discard_cmd(struct f2fs_sb_info *sbi) 1250 { 1251 __drop_discard_cmd(sbi); 1252 } 1253 1254 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, 1255 struct discard_cmd *dc) 1256 { 1257 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1258 unsigned int len = 0; 1259 1260 wait_for_completion_io(&dc->wait); 1261 mutex_lock(&dcc->cmd_lock); 1262 f2fs_bug_on(sbi, dc->state != D_DONE); 1263 dc->ref--; 1264 if (!dc->ref) { 1265 if (!dc->error) 1266 len = dc->len; 1267 __remove_discard_cmd(sbi, dc); 1268 } 1269 mutex_unlock(&dcc->cmd_lock); 1270 1271 return len; 1272 } 1273 1274 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, 1275 struct discard_policy *dpolicy, 1276 block_t start, block_t end) 1277 { 1278 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1279 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1280 &(dcc->fstrim_list) : &(dcc->wait_list); 1281 struct discard_cmd *dc, *tmp; 1282 bool need_wait; 1283 unsigned int trimmed = 0; 1284 1285 next: 1286 need_wait = false; 1287 1288 mutex_lock(&dcc->cmd_lock); 1289 list_for_each_entry_safe(dc, tmp, wait_list, list) { 1290 if (dc->lstart + dc->len <= start || end <= dc->lstart) 1291 continue; 1292 if (dc->len < dpolicy->granularity) 1293 continue; 1294 if (dc->state == D_DONE && !dc->ref) { 1295 wait_for_completion_io(&dc->wait); 1296 if (!dc->error) 1297 trimmed += dc->len; 1298 __remove_discard_cmd(sbi, dc); 1299 } else { 1300 dc->ref++; 1301 need_wait = true; 1302 break; 1303 } 1304 } 1305 mutex_unlock(&dcc->cmd_lock); 1306 1307 if (need_wait) { 1308 trimmed += __wait_one_discard_bio(sbi, dc); 1309 goto next; 1310 } 1311 1312 return trimmed; 1313 } 1314 1315 static void __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1316 struct discard_policy *dpolicy) 1317 { 1318 struct discard_policy dp; 1319 1320 if (dpolicy) { 1321 __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); 1322 return; 1323 } 1324 1325 /* wait all */ 1326 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); 1327 __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1328 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); 1329 __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1330 } 1331 1332 /* This should be covered by global mutex, &sit_i->sentry_lock */ 1333 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) 1334 { 1335 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1336 struct discard_cmd *dc; 1337 bool need_wait = false; 1338 1339 mutex_lock(&dcc->cmd_lock); 1340 dc = (struct discard_cmd *)__lookup_rb_tree(&dcc->root, NULL, blkaddr); 1341 if (dc) { 1342 if (dc->state == D_PREP) { 1343 __punch_discard_cmd(sbi, dc, blkaddr); 1344 } else { 1345 dc->ref++; 1346 need_wait = true; 1347 } 1348 } 1349 mutex_unlock(&dcc->cmd_lock); 1350 1351 if (need_wait) 1352 __wait_one_discard_bio(sbi, dc); 1353 } 1354 1355 void stop_discard_thread(struct f2fs_sb_info *sbi) 1356 { 1357 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1358 1359 if (dcc && dcc->f2fs_issue_discard) { 1360 struct task_struct *discard_thread = dcc->f2fs_issue_discard; 1361 1362 dcc->f2fs_issue_discard = NULL; 1363 kthread_stop(discard_thread); 1364 } 1365 } 1366 1367 /* This comes from f2fs_put_super */ 1368 bool f2fs_wait_discard_bios(struct f2fs_sb_info *sbi) 1369 { 1370 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1371 struct discard_policy dpolicy; 1372 bool dropped; 1373 1374 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, 1375 dcc->discard_granularity); 1376 __issue_discard_cmd(sbi, &dpolicy); 1377 dropped = __drop_discard_cmd(sbi); 1378 1379 /* just to make sure there is no pending discard commands */ 1380 __wait_all_discard_cmd(sbi, NULL); 1381 return dropped; 1382 } 1383 1384 static int issue_discard_thread(void *data) 1385 { 1386 struct f2fs_sb_info *sbi = data; 1387 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1388 wait_queue_head_t *q = &dcc->discard_wait_queue; 1389 struct discard_policy dpolicy; 1390 unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME; 1391 int issued; 1392 1393 set_freezable(); 1394 1395 do { 1396 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, 1397 dcc->discard_granularity); 1398 1399 wait_event_interruptible_timeout(*q, 1400 kthread_should_stop() || freezing(current) || 1401 dcc->discard_wake, 1402 msecs_to_jiffies(wait_ms)); 1403 if (try_to_freeze()) 1404 continue; 1405 if (f2fs_readonly(sbi->sb)) 1406 continue; 1407 if (kthread_should_stop()) 1408 return 0; 1409 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 1410 wait_ms = dpolicy.max_interval; 1411 continue; 1412 } 1413 1414 if (dcc->discard_wake) 1415 dcc->discard_wake = 0; 1416 1417 if (sbi->gc_mode == GC_URGENT) 1418 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); 1419 1420 sb_start_intwrite(sbi->sb); 1421 1422 issued = __issue_discard_cmd(sbi, &dpolicy); 1423 if (issued) { 1424 __wait_all_discard_cmd(sbi, &dpolicy); 1425 wait_ms = dpolicy.min_interval; 1426 } else { 1427 wait_ms = dpolicy.max_interval; 1428 } 1429 1430 sb_end_intwrite(sbi->sb); 1431 1432 } while (!kthread_should_stop()); 1433 return 0; 1434 } 1435 1436 #ifdef CONFIG_BLK_DEV_ZONED 1437 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, 1438 struct block_device *bdev, block_t blkstart, block_t blklen) 1439 { 1440 sector_t sector, nr_sects; 1441 block_t lblkstart = blkstart; 1442 int devi = 0; 1443 1444 if (sbi->s_ndevs) { 1445 devi = f2fs_target_device_index(sbi, blkstart); 1446 blkstart -= FDEV(devi).start_blk; 1447 } 1448 1449 /* 1450 * We need to know the type of the zone: for conventional zones, 1451 * use regular discard if the drive supports it. For sequential 1452 * zones, reset the zone write pointer. 1453 */ 1454 switch (get_blkz_type(sbi, bdev, blkstart)) { 1455 1456 case BLK_ZONE_TYPE_CONVENTIONAL: 1457 if (!blk_queue_discard(bdev_get_queue(bdev))) 1458 return 0; 1459 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen); 1460 case BLK_ZONE_TYPE_SEQWRITE_REQ: 1461 case BLK_ZONE_TYPE_SEQWRITE_PREF: 1462 sector = SECTOR_FROM_BLOCK(blkstart); 1463 nr_sects = SECTOR_FROM_BLOCK(blklen); 1464 1465 if (sector & (bdev_zone_sectors(bdev) - 1) || 1466 nr_sects != bdev_zone_sectors(bdev)) { 1467 f2fs_msg(sbi->sb, KERN_INFO, 1468 "(%d) %s: Unaligned discard attempted (block %x + %x)", 1469 devi, sbi->s_ndevs ? FDEV(devi).path: "", 1470 blkstart, blklen); 1471 return -EIO; 1472 } 1473 trace_f2fs_issue_reset_zone(bdev, blkstart); 1474 return blkdev_reset_zones(bdev, sector, 1475 nr_sects, GFP_NOFS); 1476 default: 1477 /* Unknown zone type: broken device ? */ 1478 return -EIO; 1479 } 1480 } 1481 #endif 1482 1483 static int __issue_discard_async(struct f2fs_sb_info *sbi, 1484 struct block_device *bdev, block_t blkstart, block_t blklen) 1485 { 1486 #ifdef CONFIG_BLK_DEV_ZONED 1487 if (f2fs_sb_has_blkzoned(sbi->sb) && 1488 bdev_zoned_model(bdev) != BLK_ZONED_NONE) 1489 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); 1490 #endif 1491 return __queue_discard_cmd(sbi, bdev, blkstart, blklen); 1492 } 1493 1494 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, 1495 block_t blkstart, block_t blklen) 1496 { 1497 sector_t start = blkstart, len = 0; 1498 struct block_device *bdev; 1499 struct seg_entry *se; 1500 unsigned int offset; 1501 block_t i; 1502 int err = 0; 1503 1504 bdev = f2fs_target_device(sbi, blkstart, NULL); 1505 1506 for (i = blkstart; i < blkstart + blklen; i++, len++) { 1507 if (i != start) { 1508 struct block_device *bdev2 = 1509 f2fs_target_device(sbi, i, NULL); 1510 1511 if (bdev2 != bdev) { 1512 err = __issue_discard_async(sbi, bdev, 1513 start, len); 1514 if (err) 1515 return err; 1516 bdev = bdev2; 1517 start = i; 1518 len = 0; 1519 } 1520 } 1521 1522 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); 1523 offset = GET_BLKOFF_FROM_SEG0(sbi, i); 1524 1525 if (!f2fs_test_and_set_bit(offset, se->discard_map)) 1526 sbi->discard_blks--; 1527 } 1528 1529 if (len) 1530 err = __issue_discard_async(sbi, bdev, start, len); 1531 return err; 1532 } 1533 1534 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, 1535 bool check_only) 1536 { 1537 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 1538 int max_blocks = sbi->blocks_per_seg; 1539 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); 1540 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 1541 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 1542 unsigned long *discard_map = (unsigned long *)se->discard_map; 1543 unsigned long *dmap = SIT_I(sbi)->tmp_map; 1544 unsigned int start = 0, end = -1; 1545 bool force = (cpc->reason & CP_DISCARD); 1546 struct discard_entry *de = NULL; 1547 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; 1548 int i; 1549 1550 if (se->valid_blocks == max_blocks || !f2fs_discard_en(sbi)) 1551 return false; 1552 1553 if (!force) { 1554 if (!test_opt(sbi, DISCARD) || !se->valid_blocks || 1555 SM_I(sbi)->dcc_info->nr_discards >= 1556 SM_I(sbi)->dcc_info->max_discards) 1557 return false; 1558 } 1559 1560 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 1561 for (i = 0; i < entries; i++) 1562 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : 1563 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 1564 1565 while (force || SM_I(sbi)->dcc_info->nr_discards <= 1566 SM_I(sbi)->dcc_info->max_discards) { 1567 start = __find_rev_next_bit(dmap, max_blocks, end + 1); 1568 if (start >= max_blocks) 1569 break; 1570 1571 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); 1572 if (force && start && end != max_blocks 1573 && (end - start) < cpc->trim_minlen) 1574 continue; 1575 1576 if (check_only) 1577 return true; 1578 1579 if (!de) { 1580 de = f2fs_kmem_cache_alloc(discard_entry_slab, 1581 GFP_F2FS_ZERO); 1582 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); 1583 list_add_tail(&de->list, head); 1584 } 1585 1586 for (i = start; i < end; i++) 1587 __set_bit_le(i, (void *)de->discard_map); 1588 1589 SM_I(sbi)->dcc_info->nr_discards += end - start; 1590 } 1591 return false; 1592 } 1593 1594 static void release_discard_addr(struct discard_entry *entry) 1595 { 1596 list_del(&entry->list); 1597 kmem_cache_free(discard_entry_slab, entry); 1598 } 1599 1600 void release_discard_addrs(struct f2fs_sb_info *sbi) 1601 { 1602 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); 1603 struct discard_entry *entry, *this; 1604 1605 /* drop caches */ 1606 list_for_each_entry_safe(entry, this, head, list) 1607 release_discard_addr(entry); 1608 } 1609 1610 /* 1611 * Should call clear_prefree_segments after checkpoint is done. 1612 */ 1613 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 1614 { 1615 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1616 unsigned int segno; 1617 1618 mutex_lock(&dirty_i->seglist_lock); 1619 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) 1620 __set_test_and_free(sbi, segno); 1621 mutex_unlock(&dirty_i->seglist_lock); 1622 } 1623 1624 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc) 1625 { 1626 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1627 struct list_head *head = &dcc->entry_list; 1628 struct discard_entry *entry, *this; 1629 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1630 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 1631 unsigned int start = 0, end = -1; 1632 unsigned int secno, start_segno; 1633 bool force = (cpc->reason & CP_DISCARD); 1634 1635 mutex_lock(&dirty_i->seglist_lock); 1636 1637 while (1) { 1638 int i; 1639 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); 1640 if (start >= MAIN_SEGS(sbi)) 1641 break; 1642 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), 1643 start + 1); 1644 1645 for (i = start; i < end; i++) 1646 clear_bit(i, prefree_map); 1647 1648 dirty_i->nr_dirty[PRE] -= end - start; 1649 1650 if (!test_opt(sbi, DISCARD)) 1651 continue; 1652 1653 if (force && start >= cpc->trim_start && 1654 (end - 1) <= cpc->trim_end) 1655 continue; 1656 1657 if (!test_opt(sbi, LFS) || sbi->segs_per_sec == 1) { 1658 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 1659 (end - start) << sbi->log_blocks_per_seg); 1660 continue; 1661 } 1662 next: 1663 secno = GET_SEC_FROM_SEG(sbi, start); 1664 start_segno = GET_SEG_FROM_SEC(sbi, secno); 1665 if (!IS_CURSEC(sbi, secno) && 1666 !get_valid_blocks(sbi, start, true)) 1667 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), 1668 sbi->segs_per_sec << sbi->log_blocks_per_seg); 1669 1670 start = start_segno + sbi->segs_per_sec; 1671 if (start < end) 1672 goto next; 1673 else 1674 end = start - 1; 1675 } 1676 mutex_unlock(&dirty_i->seglist_lock); 1677 1678 /* send small discards */ 1679 list_for_each_entry_safe(entry, this, head, list) { 1680 unsigned int cur_pos = 0, next_pos, len, total_len = 0; 1681 bool is_valid = test_bit_le(0, entry->discard_map); 1682 1683 find_next: 1684 if (is_valid) { 1685 next_pos = find_next_zero_bit_le(entry->discard_map, 1686 sbi->blocks_per_seg, cur_pos); 1687 len = next_pos - cur_pos; 1688 1689 if (f2fs_sb_has_blkzoned(sbi->sb) || 1690 (force && len < cpc->trim_minlen)) 1691 goto skip; 1692 1693 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, 1694 len); 1695 total_len += len; 1696 } else { 1697 next_pos = find_next_bit_le(entry->discard_map, 1698 sbi->blocks_per_seg, cur_pos); 1699 } 1700 skip: 1701 cur_pos = next_pos; 1702 is_valid = !is_valid; 1703 1704 if (cur_pos < sbi->blocks_per_seg) 1705 goto find_next; 1706 1707 release_discard_addr(entry); 1708 dcc->nr_discards -= total_len; 1709 } 1710 1711 wake_up_discard_thread(sbi, false); 1712 } 1713 1714 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) 1715 { 1716 dev_t dev = sbi->sb->s_bdev->bd_dev; 1717 struct discard_cmd_control *dcc; 1718 int err = 0, i; 1719 1720 if (SM_I(sbi)->dcc_info) { 1721 dcc = SM_I(sbi)->dcc_info; 1722 goto init_thread; 1723 } 1724 1725 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); 1726 if (!dcc) 1727 return -ENOMEM; 1728 1729 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; 1730 INIT_LIST_HEAD(&dcc->entry_list); 1731 for (i = 0; i < MAX_PLIST_NUM; i++) 1732 INIT_LIST_HEAD(&dcc->pend_list[i]); 1733 INIT_LIST_HEAD(&dcc->wait_list); 1734 INIT_LIST_HEAD(&dcc->fstrim_list); 1735 mutex_init(&dcc->cmd_lock); 1736 atomic_set(&dcc->issued_discard, 0); 1737 atomic_set(&dcc->issing_discard, 0); 1738 atomic_set(&dcc->discard_cmd_cnt, 0); 1739 dcc->nr_discards = 0; 1740 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; 1741 dcc->undiscard_blks = 0; 1742 dcc->root = RB_ROOT; 1743 1744 init_waitqueue_head(&dcc->discard_wait_queue); 1745 SM_I(sbi)->dcc_info = dcc; 1746 init_thread: 1747 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, 1748 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); 1749 if (IS_ERR(dcc->f2fs_issue_discard)) { 1750 err = PTR_ERR(dcc->f2fs_issue_discard); 1751 kfree(dcc); 1752 SM_I(sbi)->dcc_info = NULL; 1753 return err; 1754 } 1755 1756 return err; 1757 } 1758 1759 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) 1760 { 1761 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1762 1763 if (!dcc) 1764 return; 1765 1766 stop_discard_thread(sbi); 1767 1768 kfree(dcc); 1769 SM_I(sbi)->dcc_info = NULL; 1770 } 1771 1772 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 1773 { 1774 struct sit_info *sit_i = SIT_I(sbi); 1775 1776 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { 1777 sit_i->dirty_sentries++; 1778 return false; 1779 } 1780 1781 return true; 1782 } 1783 1784 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 1785 unsigned int segno, int modified) 1786 { 1787 struct seg_entry *se = get_seg_entry(sbi, segno); 1788 se->type = type; 1789 if (modified) 1790 __mark_sit_entry_dirty(sbi, segno); 1791 } 1792 1793 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 1794 { 1795 struct seg_entry *se; 1796 unsigned int segno, offset; 1797 long int new_vblocks; 1798 bool exist; 1799 #ifdef CONFIG_F2FS_CHECK_FS 1800 bool mir_exist; 1801 #endif 1802 1803 segno = GET_SEGNO(sbi, blkaddr); 1804 1805 se = get_seg_entry(sbi, segno); 1806 new_vblocks = se->valid_blocks + del; 1807 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 1808 1809 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) || 1810 (new_vblocks > sbi->blocks_per_seg))); 1811 1812 se->valid_blocks = new_vblocks; 1813 se->mtime = get_mtime(sbi); 1814 SIT_I(sbi)->max_mtime = se->mtime; 1815 1816 /* Update valid block bitmap */ 1817 if (del > 0) { 1818 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); 1819 #ifdef CONFIG_F2FS_CHECK_FS 1820 mir_exist = f2fs_test_and_set_bit(offset, 1821 se->cur_valid_map_mir); 1822 if (unlikely(exist != mir_exist)) { 1823 f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " 1824 "when setting bitmap, blk:%u, old bit:%d", 1825 blkaddr, exist); 1826 f2fs_bug_on(sbi, 1); 1827 } 1828 #endif 1829 if (unlikely(exist)) { 1830 f2fs_msg(sbi->sb, KERN_ERR, 1831 "Bitmap was wrongly set, blk:%u", blkaddr); 1832 f2fs_bug_on(sbi, 1); 1833 se->valid_blocks--; 1834 del = 0; 1835 } 1836 1837 if (f2fs_discard_en(sbi) && 1838 !f2fs_test_and_set_bit(offset, se->discard_map)) 1839 sbi->discard_blks--; 1840 1841 /* don't overwrite by SSR to keep node chain */ 1842 if (IS_NODESEG(se->type)) { 1843 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map)) 1844 se->ckpt_valid_blocks++; 1845 } 1846 } else { 1847 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map); 1848 #ifdef CONFIG_F2FS_CHECK_FS 1849 mir_exist = f2fs_test_and_clear_bit(offset, 1850 se->cur_valid_map_mir); 1851 if (unlikely(exist != mir_exist)) { 1852 f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " 1853 "when clearing bitmap, blk:%u, old bit:%d", 1854 blkaddr, exist); 1855 f2fs_bug_on(sbi, 1); 1856 } 1857 #endif 1858 if (unlikely(!exist)) { 1859 f2fs_msg(sbi->sb, KERN_ERR, 1860 "Bitmap was wrongly cleared, blk:%u", blkaddr); 1861 f2fs_bug_on(sbi, 1); 1862 se->valid_blocks++; 1863 del = 0; 1864 } 1865 1866 if (f2fs_discard_en(sbi) && 1867 f2fs_test_and_clear_bit(offset, se->discard_map)) 1868 sbi->discard_blks++; 1869 } 1870 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 1871 se->ckpt_valid_blocks += del; 1872 1873 __mark_sit_entry_dirty(sbi, segno); 1874 1875 /* update total number of valid blocks to be written in ckpt area */ 1876 SIT_I(sbi)->written_valid_blocks += del; 1877 1878 if (sbi->segs_per_sec > 1) 1879 get_sec_entry(sbi, segno)->valid_blocks += del; 1880 } 1881 1882 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) 1883 { 1884 unsigned int segno = GET_SEGNO(sbi, addr); 1885 struct sit_info *sit_i = SIT_I(sbi); 1886 1887 f2fs_bug_on(sbi, addr == NULL_ADDR); 1888 if (addr == NEW_ADDR) 1889 return; 1890 1891 /* add it into sit main buffer */ 1892 down_write(&sit_i->sentry_lock); 1893 1894 update_sit_entry(sbi, addr, -1); 1895 1896 /* add it into dirty seglist */ 1897 locate_dirty_segment(sbi, segno); 1898 1899 up_write(&sit_i->sentry_lock); 1900 } 1901 1902 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) 1903 { 1904 struct sit_info *sit_i = SIT_I(sbi); 1905 unsigned int segno, offset; 1906 struct seg_entry *se; 1907 bool is_cp = false; 1908 1909 if (!is_valid_blkaddr(blkaddr)) 1910 return true; 1911 1912 down_read(&sit_i->sentry_lock); 1913 1914 segno = GET_SEGNO(sbi, blkaddr); 1915 se = get_seg_entry(sbi, segno); 1916 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 1917 1918 if (f2fs_test_bit(offset, se->ckpt_valid_map)) 1919 is_cp = true; 1920 1921 up_read(&sit_i->sentry_lock); 1922 1923 return is_cp; 1924 } 1925 1926 /* 1927 * This function should be resided under the curseg_mutex lock 1928 */ 1929 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, 1930 struct f2fs_summary *sum) 1931 { 1932 struct curseg_info *curseg = CURSEG_I(sbi, type); 1933 void *addr = curseg->sum_blk; 1934 addr += curseg->next_blkoff * sizeof(struct f2fs_summary); 1935 memcpy(addr, sum, sizeof(struct f2fs_summary)); 1936 } 1937 1938 /* 1939 * Calculate the number of current summary pages for writing 1940 */ 1941 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) 1942 { 1943 int valid_sum_count = 0; 1944 int i, sum_in_page; 1945 1946 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 1947 if (sbi->ckpt->alloc_type[i] == SSR) 1948 valid_sum_count += sbi->blocks_per_seg; 1949 else { 1950 if (for_ra) 1951 valid_sum_count += le16_to_cpu( 1952 F2FS_CKPT(sbi)->cur_data_blkoff[i]); 1953 else 1954 valid_sum_count += curseg_blkoff(sbi, i); 1955 } 1956 } 1957 1958 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - 1959 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 1960 if (valid_sum_count <= sum_in_page) 1961 return 1; 1962 else if ((valid_sum_count - sum_in_page) <= 1963 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 1964 return 2; 1965 return 3; 1966 } 1967 1968 /* 1969 * Caller should put this summary page 1970 */ 1971 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) 1972 { 1973 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno)); 1974 } 1975 1976 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr) 1977 { 1978 struct page *page = grab_meta_page(sbi, blk_addr); 1979 1980 memcpy(page_address(page), src, PAGE_SIZE); 1981 set_page_dirty(page); 1982 f2fs_put_page(page, 1); 1983 } 1984 1985 static void write_sum_page(struct f2fs_sb_info *sbi, 1986 struct f2fs_summary_block *sum_blk, block_t blk_addr) 1987 { 1988 update_meta_page(sbi, (void *)sum_blk, blk_addr); 1989 } 1990 1991 static void write_current_sum_page(struct f2fs_sb_info *sbi, 1992 int type, block_t blk_addr) 1993 { 1994 struct curseg_info *curseg = CURSEG_I(sbi, type); 1995 struct page *page = grab_meta_page(sbi, blk_addr); 1996 struct f2fs_summary_block *src = curseg->sum_blk; 1997 struct f2fs_summary_block *dst; 1998 1999 dst = (struct f2fs_summary_block *)page_address(page); 2000 memset(dst, 0, PAGE_SIZE); 2001 2002 mutex_lock(&curseg->curseg_mutex); 2003 2004 down_read(&curseg->journal_rwsem); 2005 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); 2006 up_read(&curseg->journal_rwsem); 2007 2008 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); 2009 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); 2010 2011 mutex_unlock(&curseg->curseg_mutex); 2012 2013 set_page_dirty(page); 2014 f2fs_put_page(page, 1); 2015 } 2016 2017 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) 2018 { 2019 struct curseg_info *curseg = CURSEG_I(sbi, type); 2020 unsigned int segno = curseg->segno + 1; 2021 struct free_segmap_info *free_i = FREE_I(sbi); 2022 2023 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) 2024 return !test_bit(segno, free_i->free_segmap); 2025 return 0; 2026 } 2027 2028 /* 2029 * Find a new segment from the free segments bitmap to right order 2030 * This function should be returned with success, otherwise BUG 2031 */ 2032 static void get_new_segment(struct f2fs_sb_info *sbi, 2033 unsigned int *newseg, bool new_sec, int dir) 2034 { 2035 struct free_segmap_info *free_i = FREE_I(sbi); 2036 unsigned int segno, secno, zoneno; 2037 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; 2038 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); 2039 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); 2040 unsigned int left_start = hint; 2041 bool init = true; 2042 int go_left = 0; 2043 int i; 2044 2045 spin_lock(&free_i->segmap_lock); 2046 2047 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { 2048 segno = find_next_zero_bit(free_i->free_segmap, 2049 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); 2050 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) 2051 goto got_it; 2052 } 2053 find_other_zone: 2054 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); 2055 if (secno >= MAIN_SECS(sbi)) { 2056 if (dir == ALLOC_RIGHT) { 2057 secno = find_next_zero_bit(free_i->free_secmap, 2058 MAIN_SECS(sbi), 0); 2059 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); 2060 } else { 2061 go_left = 1; 2062 left_start = hint - 1; 2063 } 2064 } 2065 if (go_left == 0) 2066 goto skip_left; 2067 2068 while (test_bit(left_start, free_i->free_secmap)) { 2069 if (left_start > 0) { 2070 left_start--; 2071 continue; 2072 } 2073 left_start = find_next_zero_bit(free_i->free_secmap, 2074 MAIN_SECS(sbi), 0); 2075 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); 2076 break; 2077 } 2078 secno = left_start; 2079 skip_left: 2080 segno = GET_SEG_FROM_SEC(sbi, secno); 2081 zoneno = GET_ZONE_FROM_SEC(sbi, secno); 2082 2083 /* give up on finding another zone */ 2084 if (!init) 2085 goto got_it; 2086 if (sbi->secs_per_zone == 1) 2087 goto got_it; 2088 if (zoneno == old_zoneno) 2089 goto got_it; 2090 if (dir == ALLOC_LEFT) { 2091 if (!go_left && zoneno + 1 >= total_zones) 2092 goto got_it; 2093 if (go_left && zoneno == 0) 2094 goto got_it; 2095 } 2096 for (i = 0; i < NR_CURSEG_TYPE; i++) 2097 if (CURSEG_I(sbi, i)->zone == zoneno) 2098 break; 2099 2100 if (i < NR_CURSEG_TYPE) { 2101 /* zone is in user, try another */ 2102 if (go_left) 2103 hint = zoneno * sbi->secs_per_zone - 1; 2104 else if (zoneno + 1 >= total_zones) 2105 hint = 0; 2106 else 2107 hint = (zoneno + 1) * sbi->secs_per_zone; 2108 init = false; 2109 goto find_other_zone; 2110 } 2111 got_it: 2112 /* set it as dirty segment in free segmap */ 2113 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); 2114 __set_inuse(sbi, segno); 2115 *newseg = segno; 2116 spin_unlock(&free_i->segmap_lock); 2117 } 2118 2119 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 2120 { 2121 struct curseg_info *curseg = CURSEG_I(sbi, type); 2122 struct summary_footer *sum_footer; 2123 2124 curseg->segno = curseg->next_segno; 2125 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); 2126 curseg->next_blkoff = 0; 2127 curseg->next_segno = NULL_SEGNO; 2128 2129 sum_footer = &(curseg->sum_blk->footer); 2130 memset(sum_footer, 0, sizeof(struct summary_footer)); 2131 if (IS_DATASEG(type)) 2132 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 2133 if (IS_NODESEG(type)) 2134 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 2135 __set_sit_entry_type(sbi, type, curseg->segno, modified); 2136 } 2137 2138 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) 2139 { 2140 /* if segs_per_sec is large than 1, we need to keep original policy. */ 2141 if (sbi->segs_per_sec != 1) 2142 return CURSEG_I(sbi, type)->segno; 2143 2144 if (test_opt(sbi, NOHEAP) && 2145 (type == CURSEG_HOT_DATA || IS_NODESEG(type))) 2146 return 0; 2147 2148 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) 2149 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; 2150 2151 /* find segments from 0 to reuse freed segments */ 2152 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) 2153 return 0; 2154 2155 return CURSEG_I(sbi, type)->segno; 2156 } 2157 2158 /* 2159 * Allocate a current working segment. 2160 * This function always allocates a free segment in LFS manner. 2161 */ 2162 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 2163 { 2164 struct curseg_info *curseg = CURSEG_I(sbi, type); 2165 unsigned int segno = curseg->segno; 2166 int dir = ALLOC_LEFT; 2167 2168 write_sum_page(sbi, curseg->sum_blk, 2169 GET_SUM_BLOCK(sbi, segno)); 2170 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA) 2171 dir = ALLOC_RIGHT; 2172 2173 if (test_opt(sbi, NOHEAP)) 2174 dir = ALLOC_RIGHT; 2175 2176 segno = __get_next_segno(sbi, type); 2177 get_new_segment(sbi, &segno, new_sec, dir); 2178 curseg->next_segno = segno; 2179 reset_curseg(sbi, type, 1); 2180 curseg->alloc_type = LFS; 2181 } 2182 2183 static void __next_free_blkoff(struct f2fs_sb_info *sbi, 2184 struct curseg_info *seg, block_t start) 2185 { 2186 struct seg_entry *se = get_seg_entry(sbi, seg->segno); 2187 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 2188 unsigned long *target_map = SIT_I(sbi)->tmp_map; 2189 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 2190 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 2191 int i, pos; 2192 2193 for (i = 0; i < entries; i++) 2194 target_map[i] = ckpt_map[i] | cur_map[i]; 2195 2196 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); 2197 2198 seg->next_blkoff = pos; 2199 } 2200 2201 /* 2202 * If a segment is written by LFS manner, next block offset is just obtained 2203 * by increasing the current block offset. However, if a segment is written by 2204 * SSR manner, next block offset obtained by calling __next_free_blkoff 2205 */ 2206 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, 2207 struct curseg_info *seg) 2208 { 2209 if (seg->alloc_type == SSR) 2210 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); 2211 else 2212 seg->next_blkoff++; 2213 } 2214 2215 /* 2216 * This function always allocates a used segment(from dirty seglist) by SSR 2217 * manner, so it should recover the existing segment information of valid blocks 2218 */ 2219 static void change_curseg(struct f2fs_sb_info *sbi, int type) 2220 { 2221 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2222 struct curseg_info *curseg = CURSEG_I(sbi, type); 2223 unsigned int new_segno = curseg->next_segno; 2224 struct f2fs_summary_block *sum_node; 2225 struct page *sum_page; 2226 2227 write_sum_page(sbi, curseg->sum_blk, 2228 GET_SUM_BLOCK(sbi, curseg->segno)); 2229 __set_test_and_inuse(sbi, new_segno); 2230 2231 mutex_lock(&dirty_i->seglist_lock); 2232 __remove_dirty_segment(sbi, new_segno, PRE); 2233 __remove_dirty_segment(sbi, new_segno, DIRTY); 2234 mutex_unlock(&dirty_i->seglist_lock); 2235 2236 reset_curseg(sbi, type, 1); 2237 curseg->alloc_type = SSR; 2238 __next_free_blkoff(sbi, curseg, 0); 2239 2240 sum_page = get_sum_page(sbi, new_segno); 2241 sum_node = (struct f2fs_summary_block *)page_address(sum_page); 2242 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); 2243 f2fs_put_page(sum_page, 1); 2244 } 2245 2246 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) 2247 { 2248 struct curseg_info *curseg = CURSEG_I(sbi, type); 2249 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; 2250 unsigned segno = NULL_SEGNO; 2251 int i, cnt; 2252 bool reversed = false; 2253 2254 /* need_SSR() already forces to do this */ 2255 if (v_ops->get_victim(sbi, &segno, BG_GC, type, SSR)) { 2256 curseg->next_segno = segno; 2257 return 1; 2258 } 2259 2260 /* For node segments, let's do SSR more intensively */ 2261 if (IS_NODESEG(type)) { 2262 if (type >= CURSEG_WARM_NODE) { 2263 reversed = true; 2264 i = CURSEG_COLD_NODE; 2265 } else { 2266 i = CURSEG_HOT_NODE; 2267 } 2268 cnt = NR_CURSEG_NODE_TYPE; 2269 } else { 2270 if (type >= CURSEG_WARM_DATA) { 2271 reversed = true; 2272 i = CURSEG_COLD_DATA; 2273 } else { 2274 i = CURSEG_HOT_DATA; 2275 } 2276 cnt = NR_CURSEG_DATA_TYPE; 2277 } 2278 2279 for (; cnt-- > 0; reversed ? i-- : i++) { 2280 if (i == type) 2281 continue; 2282 if (v_ops->get_victim(sbi, &segno, BG_GC, i, SSR)) { 2283 curseg->next_segno = segno; 2284 return 1; 2285 } 2286 } 2287 return 0; 2288 } 2289 2290 /* 2291 * flush out current segment and replace it with new segment 2292 * This function should be returned with success, otherwise BUG 2293 */ 2294 static void allocate_segment_by_default(struct f2fs_sb_info *sbi, 2295 int type, bool force) 2296 { 2297 struct curseg_info *curseg = CURSEG_I(sbi, type); 2298 2299 if (force) 2300 new_curseg(sbi, type, true); 2301 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && 2302 type == CURSEG_WARM_NODE) 2303 new_curseg(sbi, type, false); 2304 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type)) 2305 new_curseg(sbi, type, false); 2306 else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) 2307 change_curseg(sbi, type); 2308 else 2309 new_curseg(sbi, type, false); 2310 2311 stat_inc_seg_type(sbi, curseg); 2312 } 2313 2314 void allocate_new_segments(struct f2fs_sb_info *sbi) 2315 { 2316 struct curseg_info *curseg; 2317 unsigned int old_segno; 2318 int i; 2319 2320 down_write(&SIT_I(sbi)->sentry_lock); 2321 2322 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2323 curseg = CURSEG_I(sbi, i); 2324 old_segno = curseg->segno; 2325 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); 2326 locate_dirty_segment(sbi, old_segno); 2327 } 2328 2329 up_write(&SIT_I(sbi)->sentry_lock); 2330 } 2331 2332 static const struct segment_allocation default_salloc_ops = { 2333 .allocate_segment = allocate_segment_by_default, 2334 }; 2335 2336 bool exist_trim_candidates(struct f2fs_sb_info *sbi, struct cp_control *cpc) 2337 { 2338 __u64 trim_start = cpc->trim_start; 2339 bool has_candidate = false; 2340 2341 down_write(&SIT_I(sbi)->sentry_lock); 2342 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { 2343 if (add_discard_addrs(sbi, cpc, true)) { 2344 has_candidate = true; 2345 break; 2346 } 2347 } 2348 up_write(&SIT_I(sbi)->sentry_lock); 2349 2350 cpc->trim_start = trim_start; 2351 return has_candidate; 2352 } 2353 2354 static void __issue_discard_cmd_range(struct f2fs_sb_info *sbi, 2355 struct discard_policy *dpolicy, 2356 unsigned int start, unsigned int end) 2357 { 2358 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2359 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 2360 struct rb_node **insert_p = NULL, *insert_parent = NULL; 2361 struct discard_cmd *dc; 2362 struct blk_plug plug; 2363 int issued; 2364 2365 next: 2366 issued = 0; 2367 2368 mutex_lock(&dcc->cmd_lock); 2369 f2fs_bug_on(sbi, !__check_rb_tree_consistence(sbi, &dcc->root)); 2370 2371 dc = (struct discard_cmd *)__lookup_rb_tree_ret(&dcc->root, 2372 NULL, start, 2373 (struct rb_entry **)&prev_dc, 2374 (struct rb_entry **)&next_dc, 2375 &insert_p, &insert_parent, true); 2376 if (!dc) 2377 dc = next_dc; 2378 2379 blk_start_plug(&plug); 2380 2381 while (dc && dc->lstart <= end) { 2382 struct rb_node *node; 2383 2384 if (dc->len < dpolicy->granularity) 2385 goto skip; 2386 2387 if (dc->state != D_PREP) { 2388 list_move_tail(&dc->list, &dcc->fstrim_list); 2389 goto skip; 2390 } 2391 2392 __submit_discard_cmd(sbi, dpolicy, dc); 2393 2394 if (++issued >= dpolicy->max_requests) { 2395 start = dc->lstart + dc->len; 2396 2397 blk_finish_plug(&plug); 2398 mutex_unlock(&dcc->cmd_lock); 2399 __wait_all_discard_cmd(sbi, NULL); 2400 congestion_wait(BLK_RW_ASYNC, HZ/50); 2401 goto next; 2402 } 2403 skip: 2404 node = rb_next(&dc->rb_node); 2405 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 2406 2407 if (fatal_signal_pending(current)) 2408 break; 2409 } 2410 2411 blk_finish_plug(&plug); 2412 mutex_unlock(&dcc->cmd_lock); 2413 } 2414 2415 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) 2416 { 2417 __u64 start = F2FS_BYTES_TO_BLK(range->start); 2418 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; 2419 unsigned int start_segno, end_segno; 2420 block_t start_block, end_block; 2421 struct cp_control cpc; 2422 struct discard_policy dpolicy; 2423 unsigned long long trimmed = 0; 2424 int err = 0; 2425 2426 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) 2427 return -EINVAL; 2428 2429 if (end <= MAIN_BLKADDR(sbi)) 2430 return -EINVAL; 2431 2432 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 2433 f2fs_msg(sbi->sb, KERN_WARNING, 2434 "Found FS corruption, run fsck to fix."); 2435 return -EIO; 2436 } 2437 2438 /* start/end segment number in main_area */ 2439 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); 2440 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : 2441 GET_SEGNO(sbi, end); 2442 2443 cpc.reason = CP_DISCARD; 2444 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); 2445 cpc.trim_start = start_segno; 2446 cpc.trim_end = end_segno; 2447 2448 if (sbi->discard_blks == 0) 2449 goto out; 2450 2451 mutex_lock(&sbi->gc_mutex); 2452 err = write_checkpoint(sbi, &cpc); 2453 mutex_unlock(&sbi->gc_mutex); 2454 if (err) 2455 goto out; 2456 2457 start_block = START_BLOCK(sbi, start_segno); 2458 end_block = START_BLOCK(sbi, end_segno + 1); 2459 2460 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); 2461 __issue_discard_cmd_range(sbi, &dpolicy, start_block, end_block); 2462 2463 /* 2464 * We filed discard candidates, but actually we don't need to wait for 2465 * all of them, since they'll be issued in idle time along with runtime 2466 * discard option. User configuration looks like using runtime discard 2467 * or periodic fstrim instead of it. 2468 */ 2469 if (!test_opt(sbi, DISCARD)) { 2470 trimmed = __wait_discard_cmd_range(sbi, &dpolicy, 2471 start_block, end_block); 2472 range->len = F2FS_BLK_TO_BYTES(trimmed); 2473 } 2474 out: 2475 return err; 2476 } 2477 2478 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) 2479 { 2480 struct curseg_info *curseg = CURSEG_I(sbi, type); 2481 if (curseg->next_blkoff < sbi->blocks_per_seg) 2482 return true; 2483 return false; 2484 } 2485 2486 int rw_hint_to_seg_type(enum rw_hint hint) 2487 { 2488 switch (hint) { 2489 case WRITE_LIFE_SHORT: 2490 return CURSEG_HOT_DATA; 2491 case WRITE_LIFE_EXTREME: 2492 return CURSEG_COLD_DATA; 2493 default: 2494 return CURSEG_WARM_DATA; 2495 } 2496 } 2497 2498 /* This returns write hints for each segment type. This hints will be 2499 * passed down to block layer. There are mapping tables which depend on 2500 * the mount option 'whint_mode'. 2501 * 2502 * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET. 2503 * 2504 * 2) whint_mode=user-based. F2FS tries to pass down hints given by users. 2505 * 2506 * User F2FS Block 2507 * ---- ---- ----- 2508 * META WRITE_LIFE_NOT_SET 2509 * HOT_NODE " 2510 * WARM_NODE " 2511 * COLD_NODE " 2512 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 2513 * extension list " " 2514 * 2515 * -- buffered io 2516 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2517 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2518 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 2519 * WRITE_LIFE_NONE " " 2520 * WRITE_LIFE_MEDIUM " " 2521 * WRITE_LIFE_LONG " " 2522 * 2523 * -- direct io 2524 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2525 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2526 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 2527 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 2528 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 2529 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 2530 * 2531 * 3) whint_mode=fs-based. F2FS passes down hints with its policy. 2532 * 2533 * User F2FS Block 2534 * ---- ---- ----- 2535 * META WRITE_LIFE_MEDIUM; 2536 * HOT_NODE WRITE_LIFE_NOT_SET 2537 * WARM_NODE " 2538 * COLD_NODE WRITE_LIFE_NONE 2539 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 2540 * extension list " " 2541 * 2542 * -- buffered io 2543 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2544 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2545 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_LONG 2546 * WRITE_LIFE_NONE " " 2547 * WRITE_LIFE_MEDIUM " " 2548 * WRITE_LIFE_LONG " " 2549 * 2550 * -- direct io 2551 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2552 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2553 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 2554 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 2555 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 2556 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 2557 */ 2558 2559 enum rw_hint io_type_to_rw_hint(struct f2fs_sb_info *sbi, 2560 enum page_type type, enum temp_type temp) 2561 { 2562 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) { 2563 if (type == DATA) { 2564 if (temp == WARM) 2565 return WRITE_LIFE_NOT_SET; 2566 else if (temp == HOT) 2567 return WRITE_LIFE_SHORT; 2568 else if (temp == COLD) 2569 return WRITE_LIFE_EXTREME; 2570 } else { 2571 return WRITE_LIFE_NOT_SET; 2572 } 2573 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) { 2574 if (type == DATA) { 2575 if (temp == WARM) 2576 return WRITE_LIFE_LONG; 2577 else if (temp == HOT) 2578 return WRITE_LIFE_SHORT; 2579 else if (temp == COLD) 2580 return WRITE_LIFE_EXTREME; 2581 } else if (type == NODE) { 2582 if (temp == WARM || temp == HOT) 2583 return WRITE_LIFE_NOT_SET; 2584 else if (temp == COLD) 2585 return WRITE_LIFE_NONE; 2586 } else if (type == META) { 2587 return WRITE_LIFE_MEDIUM; 2588 } 2589 } 2590 return WRITE_LIFE_NOT_SET; 2591 } 2592 2593 static int __get_segment_type_2(struct f2fs_io_info *fio) 2594 { 2595 if (fio->type == DATA) 2596 return CURSEG_HOT_DATA; 2597 else 2598 return CURSEG_HOT_NODE; 2599 } 2600 2601 static int __get_segment_type_4(struct f2fs_io_info *fio) 2602 { 2603 if (fio->type == DATA) { 2604 struct inode *inode = fio->page->mapping->host; 2605 2606 if (S_ISDIR(inode->i_mode)) 2607 return CURSEG_HOT_DATA; 2608 else 2609 return CURSEG_COLD_DATA; 2610 } else { 2611 if (IS_DNODE(fio->page) && is_cold_node(fio->page)) 2612 return CURSEG_WARM_NODE; 2613 else 2614 return CURSEG_COLD_NODE; 2615 } 2616 } 2617 2618 static int __get_segment_type_6(struct f2fs_io_info *fio) 2619 { 2620 if (fio->type == DATA) { 2621 struct inode *inode = fio->page->mapping->host; 2622 2623 if (is_cold_data(fio->page) || file_is_cold(inode)) 2624 return CURSEG_COLD_DATA; 2625 if (file_is_hot(inode) || 2626 is_inode_flag_set(inode, FI_HOT_DATA) || 2627 is_inode_flag_set(inode, FI_ATOMIC_FILE) || 2628 is_inode_flag_set(inode, FI_VOLATILE_FILE)) 2629 return CURSEG_HOT_DATA; 2630 return rw_hint_to_seg_type(inode->i_write_hint); 2631 } else { 2632 if (IS_DNODE(fio->page)) 2633 return is_cold_node(fio->page) ? CURSEG_WARM_NODE : 2634 CURSEG_HOT_NODE; 2635 return CURSEG_COLD_NODE; 2636 } 2637 } 2638 2639 static int __get_segment_type(struct f2fs_io_info *fio) 2640 { 2641 int type = 0; 2642 2643 switch (F2FS_OPTION(fio->sbi).active_logs) { 2644 case 2: 2645 type = __get_segment_type_2(fio); 2646 break; 2647 case 4: 2648 type = __get_segment_type_4(fio); 2649 break; 2650 case 6: 2651 type = __get_segment_type_6(fio); 2652 break; 2653 default: 2654 f2fs_bug_on(fio->sbi, true); 2655 } 2656 2657 if (IS_HOT(type)) 2658 fio->temp = HOT; 2659 else if (IS_WARM(type)) 2660 fio->temp = WARM; 2661 else 2662 fio->temp = COLD; 2663 return type; 2664 } 2665 2666 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, 2667 block_t old_blkaddr, block_t *new_blkaddr, 2668 struct f2fs_summary *sum, int type, 2669 struct f2fs_io_info *fio, bool add_list) 2670 { 2671 struct sit_info *sit_i = SIT_I(sbi); 2672 struct curseg_info *curseg = CURSEG_I(sbi, type); 2673 2674 down_read(&SM_I(sbi)->curseg_lock); 2675 2676 mutex_lock(&curseg->curseg_mutex); 2677 down_write(&sit_i->sentry_lock); 2678 2679 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 2680 2681 f2fs_wait_discard_bio(sbi, *new_blkaddr); 2682 2683 /* 2684 * __add_sum_entry should be resided under the curseg_mutex 2685 * because, this function updates a summary entry in the 2686 * current summary block. 2687 */ 2688 __add_sum_entry(sbi, type, sum); 2689 2690 __refresh_next_blkoff(sbi, curseg); 2691 2692 stat_inc_block_count(sbi, curseg); 2693 2694 /* 2695 * SIT information should be updated before segment allocation, 2696 * since SSR needs latest valid block information. 2697 */ 2698 update_sit_entry(sbi, *new_blkaddr, 1); 2699 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 2700 update_sit_entry(sbi, old_blkaddr, -1); 2701 2702 if (!__has_curseg_space(sbi, type)) 2703 sit_i->s_ops->allocate_segment(sbi, type, false); 2704 2705 /* 2706 * segment dirty status should be updated after segment allocation, 2707 * so we just need to update status only one time after previous 2708 * segment being closed. 2709 */ 2710 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 2711 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); 2712 2713 up_write(&sit_i->sentry_lock); 2714 2715 if (page && IS_NODESEG(type)) { 2716 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); 2717 2718 f2fs_inode_chksum_set(sbi, page); 2719 } 2720 2721 if (add_list) { 2722 struct f2fs_bio_info *io; 2723 2724 INIT_LIST_HEAD(&fio->list); 2725 fio->in_list = true; 2726 io = sbi->write_io[fio->type] + fio->temp; 2727 spin_lock(&io->io_lock); 2728 list_add_tail(&fio->list, &io->io_list); 2729 spin_unlock(&io->io_lock); 2730 } 2731 2732 mutex_unlock(&curseg->curseg_mutex); 2733 2734 up_read(&SM_I(sbi)->curseg_lock); 2735 } 2736 2737 static void update_device_state(struct f2fs_io_info *fio) 2738 { 2739 struct f2fs_sb_info *sbi = fio->sbi; 2740 unsigned int devidx; 2741 2742 if (!sbi->s_ndevs) 2743 return; 2744 2745 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr); 2746 2747 /* update device state for fsync */ 2748 set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO); 2749 2750 /* update device state for checkpoint */ 2751 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { 2752 spin_lock(&sbi->dev_lock); 2753 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); 2754 spin_unlock(&sbi->dev_lock); 2755 } 2756 } 2757 2758 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) 2759 { 2760 int type = __get_segment_type(fio); 2761 int err; 2762 bool keep_order = (test_opt(fio->sbi, LFS) && type == CURSEG_COLD_DATA); 2763 2764 if (keep_order) 2765 down_read(&fio->sbi->io_order_lock); 2766 reallocate: 2767 allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, 2768 &fio->new_blkaddr, sum, type, fio, true); 2769 2770 /* writeout dirty page into bdev */ 2771 err = f2fs_submit_page_write(fio); 2772 if (err == -EAGAIN) { 2773 fio->old_blkaddr = fio->new_blkaddr; 2774 goto reallocate; 2775 } else if (!err) { 2776 update_device_state(fio); 2777 } 2778 if (keep_order) 2779 up_read(&fio->sbi->io_order_lock); 2780 } 2781 2782 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page, 2783 enum iostat_type io_type) 2784 { 2785 struct f2fs_io_info fio = { 2786 .sbi = sbi, 2787 .type = META, 2788 .temp = HOT, 2789 .op = REQ_OP_WRITE, 2790 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, 2791 .old_blkaddr = page->index, 2792 .new_blkaddr = page->index, 2793 .page = page, 2794 .encrypted_page = NULL, 2795 .in_list = false, 2796 }; 2797 2798 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) 2799 fio.op_flags &= ~REQ_META; 2800 2801 set_page_writeback(page); 2802 ClearPageError(page); 2803 f2fs_submit_page_write(&fio); 2804 2805 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE); 2806 } 2807 2808 void write_node_page(unsigned int nid, struct f2fs_io_info *fio) 2809 { 2810 struct f2fs_summary sum; 2811 2812 set_summary(&sum, nid, 0, 0); 2813 do_write_page(&sum, fio); 2814 2815 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); 2816 } 2817 2818 void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio) 2819 { 2820 struct f2fs_sb_info *sbi = fio->sbi; 2821 struct f2fs_summary sum; 2822 struct node_info ni; 2823 2824 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); 2825 get_node_info(sbi, dn->nid, &ni); 2826 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 2827 do_write_page(&sum, fio); 2828 f2fs_update_data_blkaddr(dn, fio->new_blkaddr); 2829 2830 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE); 2831 } 2832 2833 int rewrite_data_page(struct f2fs_io_info *fio) 2834 { 2835 int err; 2836 struct f2fs_sb_info *sbi = fio->sbi; 2837 2838 fio->new_blkaddr = fio->old_blkaddr; 2839 /* i/o temperature is needed for passing down write hints */ 2840 __get_segment_type(fio); 2841 2842 f2fs_bug_on(sbi, !IS_DATASEG(get_seg_entry(sbi, 2843 GET_SEGNO(sbi, fio->new_blkaddr))->type)); 2844 2845 stat_inc_inplace_blocks(fio->sbi); 2846 2847 err = f2fs_submit_page_bio(fio); 2848 if (!err) 2849 update_device_state(fio); 2850 2851 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); 2852 2853 return err; 2854 } 2855 2856 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, 2857 unsigned int segno) 2858 { 2859 int i; 2860 2861 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { 2862 if (CURSEG_I(sbi, i)->segno == segno) 2863 break; 2864 } 2865 return i; 2866 } 2867 2868 void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, 2869 block_t old_blkaddr, block_t new_blkaddr, 2870 bool recover_curseg, bool recover_newaddr) 2871 { 2872 struct sit_info *sit_i = SIT_I(sbi); 2873 struct curseg_info *curseg; 2874 unsigned int segno, old_cursegno; 2875 struct seg_entry *se; 2876 int type; 2877 unsigned short old_blkoff; 2878 2879 segno = GET_SEGNO(sbi, new_blkaddr); 2880 se = get_seg_entry(sbi, segno); 2881 type = se->type; 2882 2883 down_write(&SM_I(sbi)->curseg_lock); 2884 2885 if (!recover_curseg) { 2886 /* for recovery flow */ 2887 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { 2888 if (old_blkaddr == NULL_ADDR) 2889 type = CURSEG_COLD_DATA; 2890 else 2891 type = CURSEG_WARM_DATA; 2892 } 2893 } else { 2894 if (IS_CURSEG(sbi, segno)) { 2895 /* se->type is volatile as SSR allocation */ 2896 type = __f2fs_get_curseg(sbi, segno); 2897 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); 2898 } else { 2899 type = CURSEG_WARM_DATA; 2900 } 2901 } 2902 2903 f2fs_bug_on(sbi, !IS_DATASEG(type)); 2904 curseg = CURSEG_I(sbi, type); 2905 2906 mutex_lock(&curseg->curseg_mutex); 2907 down_write(&sit_i->sentry_lock); 2908 2909 old_cursegno = curseg->segno; 2910 old_blkoff = curseg->next_blkoff; 2911 2912 /* change the current segment */ 2913 if (segno != curseg->segno) { 2914 curseg->next_segno = segno; 2915 change_curseg(sbi, type); 2916 } 2917 2918 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 2919 __add_sum_entry(sbi, type, sum); 2920 2921 if (!recover_curseg || recover_newaddr) 2922 update_sit_entry(sbi, new_blkaddr, 1); 2923 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 2924 update_sit_entry(sbi, old_blkaddr, -1); 2925 2926 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 2927 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); 2928 2929 locate_dirty_segment(sbi, old_cursegno); 2930 2931 if (recover_curseg) { 2932 if (old_cursegno != curseg->segno) { 2933 curseg->next_segno = old_cursegno; 2934 change_curseg(sbi, type); 2935 } 2936 curseg->next_blkoff = old_blkoff; 2937 } 2938 2939 up_write(&sit_i->sentry_lock); 2940 mutex_unlock(&curseg->curseg_mutex); 2941 up_write(&SM_I(sbi)->curseg_lock); 2942 } 2943 2944 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, 2945 block_t old_addr, block_t new_addr, 2946 unsigned char version, bool recover_curseg, 2947 bool recover_newaddr) 2948 { 2949 struct f2fs_summary sum; 2950 2951 set_summary(&sum, dn->nid, dn->ofs_in_node, version); 2952 2953 __f2fs_replace_block(sbi, &sum, old_addr, new_addr, 2954 recover_curseg, recover_newaddr); 2955 2956 f2fs_update_data_blkaddr(dn, new_addr); 2957 } 2958 2959 void f2fs_wait_on_page_writeback(struct page *page, 2960 enum page_type type, bool ordered) 2961 { 2962 if (PageWriteback(page)) { 2963 struct f2fs_sb_info *sbi = F2FS_P_SB(page); 2964 2965 f2fs_submit_merged_write_cond(sbi, page->mapping->host, 2966 0, page->index, type); 2967 if (ordered) 2968 wait_on_page_writeback(page); 2969 else 2970 wait_for_stable_page(page); 2971 } 2972 } 2973 2974 void f2fs_wait_on_block_writeback(struct f2fs_sb_info *sbi, block_t blkaddr) 2975 { 2976 struct page *cpage; 2977 2978 if (!is_valid_blkaddr(blkaddr)) 2979 return; 2980 2981 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); 2982 if (cpage) { 2983 f2fs_wait_on_page_writeback(cpage, DATA, true); 2984 f2fs_put_page(cpage, 1); 2985 } 2986 } 2987 2988 static void read_compacted_summaries(struct f2fs_sb_info *sbi) 2989 { 2990 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 2991 struct curseg_info *seg_i; 2992 unsigned char *kaddr; 2993 struct page *page; 2994 block_t start; 2995 int i, j, offset; 2996 2997 start = start_sum_block(sbi); 2998 2999 page = get_meta_page(sbi, start++); 3000 kaddr = (unsigned char *)page_address(page); 3001 3002 /* Step 1: restore nat cache */ 3003 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3004 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); 3005 3006 /* Step 2: restore sit cache */ 3007 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3008 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); 3009 offset = 2 * SUM_JOURNAL_SIZE; 3010 3011 /* Step 3: restore summary entries */ 3012 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3013 unsigned short blk_off; 3014 unsigned int segno; 3015 3016 seg_i = CURSEG_I(sbi, i); 3017 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 3018 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 3019 seg_i->next_segno = segno; 3020 reset_curseg(sbi, i, 0); 3021 seg_i->alloc_type = ckpt->alloc_type[i]; 3022 seg_i->next_blkoff = blk_off; 3023 3024 if (seg_i->alloc_type == SSR) 3025 blk_off = sbi->blocks_per_seg; 3026 3027 for (j = 0; j < blk_off; j++) { 3028 struct f2fs_summary *s; 3029 s = (struct f2fs_summary *)(kaddr + offset); 3030 seg_i->sum_blk->entries[j] = *s; 3031 offset += SUMMARY_SIZE; 3032 if (offset + SUMMARY_SIZE <= PAGE_SIZE - 3033 SUM_FOOTER_SIZE) 3034 continue; 3035 3036 f2fs_put_page(page, 1); 3037 page = NULL; 3038 3039 page = get_meta_page(sbi, start++); 3040 kaddr = (unsigned char *)page_address(page); 3041 offset = 0; 3042 } 3043 } 3044 f2fs_put_page(page, 1); 3045 } 3046 3047 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 3048 { 3049 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3050 struct f2fs_summary_block *sum; 3051 struct curseg_info *curseg; 3052 struct page *new; 3053 unsigned short blk_off; 3054 unsigned int segno = 0; 3055 block_t blk_addr = 0; 3056 3057 /* get segment number and block addr */ 3058 if (IS_DATASEG(type)) { 3059 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 3060 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 3061 CURSEG_HOT_DATA]); 3062 if (__exist_node_summaries(sbi)) 3063 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); 3064 else 3065 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 3066 } else { 3067 segno = le32_to_cpu(ckpt->cur_node_segno[type - 3068 CURSEG_HOT_NODE]); 3069 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 3070 CURSEG_HOT_NODE]); 3071 if (__exist_node_summaries(sbi)) 3072 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 3073 type - CURSEG_HOT_NODE); 3074 else 3075 blk_addr = GET_SUM_BLOCK(sbi, segno); 3076 } 3077 3078 new = get_meta_page(sbi, blk_addr); 3079 sum = (struct f2fs_summary_block *)page_address(new); 3080 3081 if (IS_NODESEG(type)) { 3082 if (__exist_node_summaries(sbi)) { 3083 struct f2fs_summary *ns = &sum->entries[0]; 3084 int i; 3085 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { 3086 ns->version = 0; 3087 ns->ofs_in_node = 0; 3088 } 3089 } else { 3090 restore_node_summary(sbi, segno, sum); 3091 } 3092 } 3093 3094 /* set uncompleted segment to curseg */ 3095 curseg = CURSEG_I(sbi, type); 3096 mutex_lock(&curseg->curseg_mutex); 3097 3098 /* update journal info */ 3099 down_write(&curseg->journal_rwsem); 3100 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); 3101 up_write(&curseg->journal_rwsem); 3102 3103 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); 3104 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); 3105 curseg->next_segno = segno; 3106 reset_curseg(sbi, type, 0); 3107 curseg->alloc_type = ckpt->alloc_type[type]; 3108 curseg->next_blkoff = blk_off; 3109 mutex_unlock(&curseg->curseg_mutex); 3110 f2fs_put_page(new, 1); 3111 return 0; 3112 } 3113 3114 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 3115 { 3116 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; 3117 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; 3118 int type = CURSEG_HOT_DATA; 3119 int err; 3120 3121 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { 3122 int npages = npages_for_summary_flush(sbi, true); 3123 3124 if (npages >= 2) 3125 ra_meta_pages(sbi, start_sum_block(sbi), npages, 3126 META_CP, true); 3127 3128 /* restore for compacted data summary */ 3129 read_compacted_summaries(sbi); 3130 type = CURSEG_HOT_NODE; 3131 } 3132 3133 if (__exist_node_summaries(sbi)) 3134 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type), 3135 NR_CURSEG_TYPE - type, META_CP, true); 3136 3137 for (; type <= CURSEG_COLD_NODE; type++) { 3138 err = read_normal_summaries(sbi, type); 3139 if (err) 3140 return err; 3141 } 3142 3143 /* sanity check for summary blocks */ 3144 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || 3145 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) 3146 return -EINVAL; 3147 3148 return 0; 3149 } 3150 3151 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 3152 { 3153 struct page *page; 3154 unsigned char *kaddr; 3155 struct f2fs_summary *summary; 3156 struct curseg_info *seg_i; 3157 int written_size = 0; 3158 int i, j; 3159 3160 page = grab_meta_page(sbi, blkaddr++); 3161 kaddr = (unsigned char *)page_address(page); 3162 memset(kaddr, 0, PAGE_SIZE); 3163 3164 /* Step 1: write nat cache */ 3165 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3166 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); 3167 written_size += SUM_JOURNAL_SIZE; 3168 3169 /* Step 2: write sit cache */ 3170 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3171 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); 3172 written_size += SUM_JOURNAL_SIZE; 3173 3174 /* Step 3: write summary entries */ 3175 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3176 unsigned short blkoff; 3177 seg_i = CURSEG_I(sbi, i); 3178 if (sbi->ckpt->alloc_type[i] == SSR) 3179 blkoff = sbi->blocks_per_seg; 3180 else 3181 blkoff = curseg_blkoff(sbi, i); 3182 3183 for (j = 0; j < blkoff; j++) { 3184 if (!page) { 3185 page = grab_meta_page(sbi, blkaddr++); 3186 kaddr = (unsigned char *)page_address(page); 3187 memset(kaddr, 0, PAGE_SIZE); 3188 written_size = 0; 3189 } 3190 summary = (struct f2fs_summary *)(kaddr + written_size); 3191 *summary = seg_i->sum_blk->entries[j]; 3192 written_size += SUMMARY_SIZE; 3193 3194 if (written_size + SUMMARY_SIZE <= PAGE_SIZE - 3195 SUM_FOOTER_SIZE) 3196 continue; 3197 3198 set_page_dirty(page); 3199 f2fs_put_page(page, 1); 3200 page = NULL; 3201 } 3202 } 3203 if (page) { 3204 set_page_dirty(page); 3205 f2fs_put_page(page, 1); 3206 } 3207 } 3208 3209 static void write_normal_summaries(struct f2fs_sb_info *sbi, 3210 block_t blkaddr, int type) 3211 { 3212 int i, end; 3213 if (IS_DATASEG(type)) 3214 end = type + NR_CURSEG_DATA_TYPE; 3215 else 3216 end = type + NR_CURSEG_NODE_TYPE; 3217 3218 for (i = type; i < end; i++) 3219 write_current_sum_page(sbi, i, blkaddr + (i - type)); 3220 } 3221 3222 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 3223 { 3224 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) 3225 write_compacted_summaries(sbi, start_blk); 3226 else 3227 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 3228 } 3229 3230 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 3231 { 3232 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 3233 } 3234 3235 int lookup_journal_in_cursum(struct f2fs_journal *journal, int type, 3236 unsigned int val, int alloc) 3237 { 3238 int i; 3239 3240 if (type == NAT_JOURNAL) { 3241 for (i = 0; i < nats_in_cursum(journal); i++) { 3242 if (le32_to_cpu(nid_in_journal(journal, i)) == val) 3243 return i; 3244 } 3245 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) 3246 return update_nats_in_cursum(journal, 1); 3247 } else if (type == SIT_JOURNAL) { 3248 for (i = 0; i < sits_in_cursum(journal); i++) 3249 if (le32_to_cpu(segno_in_journal(journal, i)) == val) 3250 return i; 3251 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) 3252 return update_sits_in_cursum(journal, 1); 3253 } 3254 return -1; 3255 } 3256 3257 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 3258 unsigned int segno) 3259 { 3260 return get_meta_page(sbi, current_sit_addr(sbi, segno)); 3261 } 3262 3263 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 3264 unsigned int start) 3265 { 3266 struct sit_info *sit_i = SIT_I(sbi); 3267 struct page *page; 3268 pgoff_t src_off, dst_off; 3269 3270 src_off = current_sit_addr(sbi, start); 3271 dst_off = next_sit_addr(sbi, src_off); 3272 3273 page = grab_meta_page(sbi, dst_off); 3274 seg_info_to_sit_page(sbi, page, start); 3275 3276 set_page_dirty(page); 3277 set_to_next_sit(sit_i, start); 3278 3279 return page; 3280 } 3281 3282 static struct sit_entry_set *grab_sit_entry_set(void) 3283 { 3284 struct sit_entry_set *ses = 3285 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS); 3286 3287 ses->entry_cnt = 0; 3288 INIT_LIST_HEAD(&ses->set_list); 3289 return ses; 3290 } 3291 3292 static void release_sit_entry_set(struct sit_entry_set *ses) 3293 { 3294 list_del(&ses->set_list); 3295 kmem_cache_free(sit_entry_set_slab, ses); 3296 } 3297 3298 static void adjust_sit_entry_set(struct sit_entry_set *ses, 3299 struct list_head *head) 3300 { 3301 struct sit_entry_set *next = ses; 3302 3303 if (list_is_last(&ses->set_list, head)) 3304 return; 3305 3306 list_for_each_entry_continue(next, head, set_list) 3307 if (ses->entry_cnt <= next->entry_cnt) 3308 break; 3309 3310 list_move_tail(&ses->set_list, &next->set_list); 3311 } 3312 3313 static void add_sit_entry(unsigned int segno, struct list_head *head) 3314 { 3315 struct sit_entry_set *ses; 3316 unsigned int start_segno = START_SEGNO(segno); 3317 3318 list_for_each_entry(ses, head, set_list) { 3319 if (ses->start_segno == start_segno) { 3320 ses->entry_cnt++; 3321 adjust_sit_entry_set(ses, head); 3322 return; 3323 } 3324 } 3325 3326 ses = grab_sit_entry_set(); 3327 3328 ses->start_segno = start_segno; 3329 ses->entry_cnt++; 3330 list_add(&ses->set_list, head); 3331 } 3332 3333 static void add_sits_in_set(struct f2fs_sb_info *sbi) 3334 { 3335 struct f2fs_sm_info *sm_info = SM_I(sbi); 3336 struct list_head *set_list = &sm_info->sit_entry_set; 3337 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; 3338 unsigned int segno; 3339 3340 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) 3341 add_sit_entry(segno, set_list); 3342 } 3343 3344 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) 3345 { 3346 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 3347 struct f2fs_journal *journal = curseg->journal; 3348 int i; 3349 3350 down_write(&curseg->journal_rwsem); 3351 for (i = 0; i < sits_in_cursum(journal); i++) { 3352 unsigned int segno; 3353 bool dirtied; 3354 3355 segno = le32_to_cpu(segno_in_journal(journal, i)); 3356 dirtied = __mark_sit_entry_dirty(sbi, segno); 3357 3358 if (!dirtied) 3359 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); 3360 } 3361 update_sits_in_cursum(journal, -i); 3362 up_write(&curseg->journal_rwsem); 3363 } 3364 3365 /* 3366 * CP calls this function, which flushes SIT entries including sit_journal, 3367 * and moves prefree segs to free segs. 3368 */ 3369 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) 3370 { 3371 struct sit_info *sit_i = SIT_I(sbi); 3372 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 3373 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 3374 struct f2fs_journal *journal = curseg->journal; 3375 struct sit_entry_set *ses, *tmp; 3376 struct list_head *head = &SM_I(sbi)->sit_entry_set; 3377 bool to_journal = true; 3378 struct seg_entry *se; 3379 3380 down_write(&sit_i->sentry_lock); 3381 3382 if (!sit_i->dirty_sentries) 3383 goto out; 3384 3385 /* 3386 * add and account sit entries of dirty bitmap in sit entry 3387 * set temporarily 3388 */ 3389 add_sits_in_set(sbi); 3390 3391 /* 3392 * if there are no enough space in journal to store dirty sit 3393 * entries, remove all entries from journal and add and account 3394 * them in sit entry set. 3395 */ 3396 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL)) 3397 remove_sits_in_journal(sbi); 3398 3399 /* 3400 * there are two steps to flush sit entries: 3401 * #1, flush sit entries to journal in current cold data summary block. 3402 * #2, flush sit entries to sit page. 3403 */ 3404 list_for_each_entry_safe(ses, tmp, head, set_list) { 3405 struct page *page = NULL; 3406 struct f2fs_sit_block *raw_sit = NULL; 3407 unsigned int start_segno = ses->start_segno; 3408 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, 3409 (unsigned long)MAIN_SEGS(sbi)); 3410 unsigned int segno = start_segno; 3411 3412 if (to_journal && 3413 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) 3414 to_journal = false; 3415 3416 if (to_journal) { 3417 down_write(&curseg->journal_rwsem); 3418 } else { 3419 page = get_next_sit_page(sbi, start_segno); 3420 raw_sit = page_address(page); 3421 } 3422 3423 /* flush dirty sit entries in region of current sit set */ 3424 for_each_set_bit_from(segno, bitmap, end) { 3425 int offset, sit_offset; 3426 3427 se = get_seg_entry(sbi, segno); 3428 #ifdef CONFIG_F2FS_CHECK_FS 3429 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir, 3430 SIT_VBLOCK_MAP_SIZE)) 3431 f2fs_bug_on(sbi, 1); 3432 #endif 3433 3434 /* add discard candidates */ 3435 if (!(cpc->reason & CP_DISCARD)) { 3436 cpc->trim_start = segno; 3437 add_discard_addrs(sbi, cpc, false); 3438 } 3439 3440 if (to_journal) { 3441 offset = lookup_journal_in_cursum(journal, 3442 SIT_JOURNAL, segno, 1); 3443 f2fs_bug_on(sbi, offset < 0); 3444 segno_in_journal(journal, offset) = 3445 cpu_to_le32(segno); 3446 seg_info_to_raw_sit(se, 3447 &sit_in_journal(journal, offset)); 3448 check_block_count(sbi, segno, 3449 &sit_in_journal(journal, offset)); 3450 } else { 3451 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 3452 seg_info_to_raw_sit(se, 3453 &raw_sit->entries[sit_offset]); 3454 check_block_count(sbi, segno, 3455 &raw_sit->entries[sit_offset]); 3456 } 3457 3458 __clear_bit(segno, bitmap); 3459 sit_i->dirty_sentries--; 3460 ses->entry_cnt--; 3461 } 3462 3463 if (to_journal) 3464 up_write(&curseg->journal_rwsem); 3465 else 3466 f2fs_put_page(page, 1); 3467 3468 f2fs_bug_on(sbi, ses->entry_cnt); 3469 release_sit_entry_set(ses); 3470 } 3471 3472 f2fs_bug_on(sbi, !list_empty(head)); 3473 f2fs_bug_on(sbi, sit_i->dirty_sentries); 3474 out: 3475 if (cpc->reason & CP_DISCARD) { 3476 __u64 trim_start = cpc->trim_start; 3477 3478 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) 3479 add_discard_addrs(sbi, cpc, false); 3480 3481 cpc->trim_start = trim_start; 3482 } 3483 up_write(&sit_i->sentry_lock); 3484 3485 set_prefree_as_free_segments(sbi); 3486 } 3487 3488 static int build_sit_info(struct f2fs_sb_info *sbi) 3489 { 3490 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 3491 struct sit_info *sit_i; 3492 unsigned int sit_segs, start; 3493 char *src_bitmap; 3494 unsigned int bitmap_size; 3495 3496 /* allocate memory for SIT information */ 3497 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); 3498 if (!sit_i) 3499 return -ENOMEM; 3500 3501 SM_I(sbi)->sit_info = sit_i; 3502 3503 sit_i->sentries = f2fs_kvzalloc(sbi, MAIN_SEGS(sbi) * 3504 sizeof(struct seg_entry), GFP_KERNEL); 3505 if (!sit_i->sentries) 3506 return -ENOMEM; 3507 3508 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 3509 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, bitmap_size, 3510 GFP_KERNEL); 3511 if (!sit_i->dirty_sentries_bitmap) 3512 return -ENOMEM; 3513 3514 for (start = 0; start < MAIN_SEGS(sbi); start++) { 3515 sit_i->sentries[start].cur_valid_map 3516 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3517 sit_i->sentries[start].ckpt_valid_map 3518 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3519 if (!sit_i->sentries[start].cur_valid_map || 3520 !sit_i->sentries[start].ckpt_valid_map) 3521 return -ENOMEM; 3522 3523 #ifdef CONFIG_F2FS_CHECK_FS 3524 sit_i->sentries[start].cur_valid_map_mir 3525 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3526 if (!sit_i->sentries[start].cur_valid_map_mir) 3527 return -ENOMEM; 3528 #endif 3529 3530 if (f2fs_discard_en(sbi)) { 3531 sit_i->sentries[start].discard_map 3532 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, 3533 GFP_KERNEL); 3534 if (!sit_i->sentries[start].discard_map) 3535 return -ENOMEM; 3536 } 3537 } 3538 3539 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3540 if (!sit_i->tmp_map) 3541 return -ENOMEM; 3542 3543 if (sbi->segs_per_sec > 1) { 3544 sit_i->sec_entries = f2fs_kvzalloc(sbi, MAIN_SECS(sbi) * 3545 sizeof(struct sec_entry), GFP_KERNEL); 3546 if (!sit_i->sec_entries) 3547 return -ENOMEM; 3548 } 3549 3550 /* get information related with SIT */ 3551 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 3552 3553 /* setup SIT bitmap from ckeckpoint pack */ 3554 bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 3555 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 3556 3557 sit_i->sit_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); 3558 if (!sit_i->sit_bitmap) 3559 return -ENOMEM; 3560 3561 #ifdef CONFIG_F2FS_CHECK_FS 3562 sit_i->sit_bitmap_mir = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); 3563 if (!sit_i->sit_bitmap_mir) 3564 return -ENOMEM; 3565 #endif 3566 3567 /* init SIT information */ 3568 sit_i->s_ops = &default_salloc_ops; 3569 3570 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 3571 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; 3572 sit_i->written_valid_blocks = 0; 3573 sit_i->bitmap_size = bitmap_size; 3574 sit_i->dirty_sentries = 0; 3575 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 3576 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 3577 sit_i->mounted_time = ktime_get_real_seconds(); 3578 init_rwsem(&sit_i->sentry_lock); 3579 return 0; 3580 } 3581 3582 static int build_free_segmap(struct f2fs_sb_info *sbi) 3583 { 3584 struct free_segmap_info *free_i; 3585 unsigned int bitmap_size, sec_bitmap_size; 3586 3587 /* allocate memory for free segmap information */ 3588 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); 3589 if (!free_i) 3590 return -ENOMEM; 3591 3592 SM_I(sbi)->free_info = free_i; 3593 3594 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 3595 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); 3596 if (!free_i->free_segmap) 3597 return -ENOMEM; 3598 3599 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 3600 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); 3601 if (!free_i->free_secmap) 3602 return -ENOMEM; 3603 3604 /* set all segments as dirty temporarily */ 3605 memset(free_i->free_segmap, 0xff, bitmap_size); 3606 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 3607 3608 /* init free segmap information */ 3609 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); 3610 free_i->free_segments = 0; 3611 free_i->free_sections = 0; 3612 spin_lock_init(&free_i->segmap_lock); 3613 return 0; 3614 } 3615 3616 static int build_curseg(struct f2fs_sb_info *sbi) 3617 { 3618 struct curseg_info *array; 3619 int i; 3620 3621 array = f2fs_kzalloc(sbi, sizeof(*array) * NR_CURSEG_TYPE, GFP_KERNEL); 3622 if (!array) 3623 return -ENOMEM; 3624 3625 SM_I(sbi)->curseg_array = array; 3626 3627 for (i = 0; i < NR_CURSEG_TYPE; i++) { 3628 mutex_init(&array[i].curseg_mutex); 3629 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); 3630 if (!array[i].sum_blk) 3631 return -ENOMEM; 3632 init_rwsem(&array[i].journal_rwsem); 3633 array[i].journal = f2fs_kzalloc(sbi, 3634 sizeof(struct f2fs_journal), GFP_KERNEL); 3635 if (!array[i].journal) 3636 return -ENOMEM; 3637 array[i].segno = NULL_SEGNO; 3638 array[i].next_blkoff = 0; 3639 } 3640 return restore_curseg_summaries(sbi); 3641 } 3642 3643 static int build_sit_entries(struct f2fs_sb_info *sbi) 3644 { 3645 struct sit_info *sit_i = SIT_I(sbi); 3646 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 3647 struct f2fs_journal *journal = curseg->journal; 3648 struct seg_entry *se; 3649 struct f2fs_sit_entry sit; 3650 int sit_blk_cnt = SIT_BLK_CNT(sbi); 3651 unsigned int i, start, end; 3652 unsigned int readed, start_blk = 0; 3653 int err = 0; 3654 block_t total_node_blocks = 0; 3655 3656 do { 3657 readed = ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES, 3658 META_SIT, true); 3659 3660 start = start_blk * sit_i->sents_per_block; 3661 end = (start_blk + readed) * sit_i->sents_per_block; 3662 3663 for (; start < end && start < MAIN_SEGS(sbi); start++) { 3664 struct f2fs_sit_block *sit_blk; 3665 struct page *page; 3666 3667 se = &sit_i->sentries[start]; 3668 page = get_current_sit_page(sbi, start); 3669 sit_blk = (struct f2fs_sit_block *)page_address(page); 3670 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 3671 f2fs_put_page(page, 1); 3672 3673 err = check_block_count(sbi, start, &sit); 3674 if (err) 3675 return err; 3676 seg_info_from_raw_sit(se, &sit); 3677 if (IS_NODESEG(se->type)) 3678 total_node_blocks += se->valid_blocks; 3679 3680 /* build discard map only one time */ 3681 if (f2fs_discard_en(sbi)) { 3682 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 3683 memset(se->discard_map, 0xff, 3684 SIT_VBLOCK_MAP_SIZE); 3685 } else { 3686 memcpy(se->discard_map, 3687 se->cur_valid_map, 3688 SIT_VBLOCK_MAP_SIZE); 3689 sbi->discard_blks += 3690 sbi->blocks_per_seg - 3691 se->valid_blocks; 3692 } 3693 } 3694 3695 if (sbi->segs_per_sec > 1) 3696 get_sec_entry(sbi, start)->valid_blocks += 3697 se->valid_blocks; 3698 } 3699 start_blk += readed; 3700 } while (start_blk < sit_blk_cnt); 3701 3702 down_read(&curseg->journal_rwsem); 3703 for (i = 0; i < sits_in_cursum(journal); i++) { 3704 unsigned int old_valid_blocks; 3705 3706 start = le32_to_cpu(segno_in_journal(journal, i)); 3707 if (start >= MAIN_SEGS(sbi)) { 3708 f2fs_msg(sbi->sb, KERN_ERR, 3709 "Wrong journal entry on segno %u", 3710 start); 3711 set_sbi_flag(sbi, SBI_NEED_FSCK); 3712 err = -EINVAL; 3713 break; 3714 } 3715 3716 se = &sit_i->sentries[start]; 3717 sit = sit_in_journal(journal, i); 3718 3719 old_valid_blocks = se->valid_blocks; 3720 if (IS_NODESEG(se->type)) 3721 total_node_blocks -= old_valid_blocks; 3722 3723 err = check_block_count(sbi, start, &sit); 3724 if (err) 3725 break; 3726 seg_info_from_raw_sit(se, &sit); 3727 if (IS_NODESEG(se->type)) 3728 total_node_blocks += se->valid_blocks; 3729 3730 if (f2fs_discard_en(sbi)) { 3731 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 3732 memset(se->discard_map, 0xff, 3733 SIT_VBLOCK_MAP_SIZE); 3734 } else { 3735 memcpy(se->discard_map, se->cur_valid_map, 3736 SIT_VBLOCK_MAP_SIZE); 3737 sbi->discard_blks += old_valid_blocks; 3738 sbi->discard_blks -= se->valid_blocks; 3739 } 3740 } 3741 3742 if (sbi->segs_per_sec > 1) { 3743 get_sec_entry(sbi, start)->valid_blocks += 3744 se->valid_blocks; 3745 get_sec_entry(sbi, start)->valid_blocks -= 3746 old_valid_blocks; 3747 } 3748 } 3749 up_read(&curseg->journal_rwsem); 3750 3751 if (!err && total_node_blocks != valid_node_count(sbi)) { 3752 f2fs_msg(sbi->sb, KERN_ERR, 3753 "SIT is corrupted node# %u vs %u", 3754 total_node_blocks, valid_node_count(sbi)); 3755 set_sbi_flag(sbi, SBI_NEED_FSCK); 3756 err = -EINVAL; 3757 } 3758 3759 return err; 3760 } 3761 3762 static void init_free_segmap(struct f2fs_sb_info *sbi) 3763 { 3764 unsigned int start; 3765 int type; 3766 3767 for (start = 0; start < MAIN_SEGS(sbi); start++) { 3768 struct seg_entry *sentry = get_seg_entry(sbi, start); 3769 if (!sentry->valid_blocks) 3770 __set_free(sbi, start); 3771 else 3772 SIT_I(sbi)->written_valid_blocks += 3773 sentry->valid_blocks; 3774 } 3775 3776 /* set use the current segments */ 3777 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 3778 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 3779 __set_test_and_inuse(sbi, curseg_t->segno); 3780 } 3781 } 3782 3783 static void init_dirty_segmap(struct f2fs_sb_info *sbi) 3784 { 3785 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 3786 struct free_segmap_info *free_i = FREE_I(sbi); 3787 unsigned int segno = 0, offset = 0; 3788 unsigned short valid_blocks; 3789 3790 while (1) { 3791 /* find dirty segment based on free segmap */ 3792 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); 3793 if (segno >= MAIN_SEGS(sbi)) 3794 break; 3795 offset = segno + 1; 3796 valid_blocks = get_valid_blocks(sbi, segno, false); 3797 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks) 3798 continue; 3799 if (valid_blocks > sbi->blocks_per_seg) { 3800 f2fs_bug_on(sbi, 1); 3801 continue; 3802 } 3803 mutex_lock(&dirty_i->seglist_lock); 3804 __locate_dirty_segment(sbi, segno, DIRTY); 3805 mutex_unlock(&dirty_i->seglist_lock); 3806 } 3807 } 3808 3809 static int init_victim_secmap(struct f2fs_sb_info *sbi) 3810 { 3811 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 3812 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 3813 3814 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 3815 if (!dirty_i->victim_secmap) 3816 return -ENOMEM; 3817 return 0; 3818 } 3819 3820 static int build_dirty_segmap(struct f2fs_sb_info *sbi) 3821 { 3822 struct dirty_seglist_info *dirty_i; 3823 unsigned int bitmap_size, i; 3824 3825 /* allocate memory for dirty segments list information */ 3826 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), 3827 GFP_KERNEL); 3828 if (!dirty_i) 3829 return -ENOMEM; 3830 3831 SM_I(sbi)->dirty_info = dirty_i; 3832 mutex_init(&dirty_i->seglist_lock); 3833 3834 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 3835 3836 for (i = 0; i < NR_DIRTY_TYPE; i++) { 3837 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, 3838 GFP_KERNEL); 3839 if (!dirty_i->dirty_segmap[i]) 3840 return -ENOMEM; 3841 } 3842 3843 init_dirty_segmap(sbi); 3844 return init_victim_secmap(sbi); 3845 } 3846 3847 /* 3848 * Update min, max modified time for cost-benefit GC algorithm 3849 */ 3850 static void init_min_max_mtime(struct f2fs_sb_info *sbi) 3851 { 3852 struct sit_info *sit_i = SIT_I(sbi); 3853 unsigned int segno; 3854 3855 down_write(&sit_i->sentry_lock); 3856 3857 sit_i->min_mtime = ULLONG_MAX; 3858 3859 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { 3860 unsigned int i; 3861 unsigned long long mtime = 0; 3862 3863 for (i = 0; i < sbi->segs_per_sec; i++) 3864 mtime += get_seg_entry(sbi, segno + i)->mtime; 3865 3866 mtime = div_u64(mtime, sbi->segs_per_sec); 3867 3868 if (sit_i->min_mtime > mtime) 3869 sit_i->min_mtime = mtime; 3870 } 3871 sit_i->max_mtime = get_mtime(sbi); 3872 up_write(&sit_i->sentry_lock); 3873 } 3874 3875 int build_segment_manager(struct f2fs_sb_info *sbi) 3876 { 3877 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 3878 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3879 struct f2fs_sm_info *sm_info; 3880 int err; 3881 3882 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); 3883 if (!sm_info) 3884 return -ENOMEM; 3885 3886 /* init sm info */ 3887 sbi->sm_info = sm_info; 3888 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 3889 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 3890 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 3891 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 3892 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 3893 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 3894 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 3895 sm_info->rec_prefree_segments = sm_info->main_segments * 3896 DEF_RECLAIM_PREFREE_SEGMENTS / 100; 3897 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS) 3898 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; 3899 3900 if (!test_opt(sbi, LFS)) 3901 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC; 3902 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 3903 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; 3904 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS; 3905 sm_info->min_ssr_sections = reserved_sections(sbi); 3906 3907 INIT_LIST_HEAD(&sm_info->sit_entry_set); 3908 3909 init_rwsem(&sm_info->curseg_lock); 3910 3911 if (!f2fs_readonly(sbi->sb)) { 3912 err = create_flush_cmd_control(sbi); 3913 if (err) 3914 return err; 3915 } 3916 3917 err = create_discard_cmd_control(sbi); 3918 if (err) 3919 return err; 3920 3921 err = build_sit_info(sbi); 3922 if (err) 3923 return err; 3924 err = build_free_segmap(sbi); 3925 if (err) 3926 return err; 3927 err = build_curseg(sbi); 3928 if (err) 3929 return err; 3930 3931 /* reinit free segmap based on SIT */ 3932 err = build_sit_entries(sbi); 3933 if (err) 3934 return err; 3935 3936 init_free_segmap(sbi); 3937 err = build_dirty_segmap(sbi); 3938 if (err) 3939 return err; 3940 3941 init_min_max_mtime(sbi); 3942 return 0; 3943 } 3944 3945 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 3946 enum dirty_type dirty_type) 3947 { 3948 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 3949 3950 mutex_lock(&dirty_i->seglist_lock); 3951 kvfree(dirty_i->dirty_segmap[dirty_type]); 3952 dirty_i->nr_dirty[dirty_type] = 0; 3953 mutex_unlock(&dirty_i->seglist_lock); 3954 } 3955 3956 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 3957 { 3958 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 3959 kvfree(dirty_i->victim_secmap); 3960 } 3961 3962 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 3963 { 3964 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 3965 int i; 3966 3967 if (!dirty_i) 3968 return; 3969 3970 /* discard pre-free/dirty segments list */ 3971 for (i = 0; i < NR_DIRTY_TYPE; i++) 3972 discard_dirty_segmap(sbi, i); 3973 3974 destroy_victim_secmap(sbi); 3975 SM_I(sbi)->dirty_info = NULL; 3976 kfree(dirty_i); 3977 } 3978 3979 static void destroy_curseg(struct f2fs_sb_info *sbi) 3980 { 3981 struct curseg_info *array = SM_I(sbi)->curseg_array; 3982 int i; 3983 3984 if (!array) 3985 return; 3986 SM_I(sbi)->curseg_array = NULL; 3987 for (i = 0; i < NR_CURSEG_TYPE; i++) { 3988 kfree(array[i].sum_blk); 3989 kfree(array[i].journal); 3990 } 3991 kfree(array); 3992 } 3993 3994 static void destroy_free_segmap(struct f2fs_sb_info *sbi) 3995 { 3996 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 3997 if (!free_i) 3998 return; 3999 SM_I(sbi)->free_info = NULL; 4000 kvfree(free_i->free_segmap); 4001 kvfree(free_i->free_secmap); 4002 kfree(free_i); 4003 } 4004 4005 static void destroy_sit_info(struct f2fs_sb_info *sbi) 4006 { 4007 struct sit_info *sit_i = SIT_I(sbi); 4008 unsigned int start; 4009 4010 if (!sit_i) 4011 return; 4012 4013 if (sit_i->sentries) { 4014 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4015 kfree(sit_i->sentries[start].cur_valid_map); 4016 #ifdef CONFIG_F2FS_CHECK_FS 4017 kfree(sit_i->sentries[start].cur_valid_map_mir); 4018 #endif 4019 kfree(sit_i->sentries[start].ckpt_valid_map); 4020 kfree(sit_i->sentries[start].discard_map); 4021 } 4022 } 4023 kfree(sit_i->tmp_map); 4024 4025 kvfree(sit_i->sentries); 4026 kvfree(sit_i->sec_entries); 4027 kvfree(sit_i->dirty_sentries_bitmap); 4028 4029 SM_I(sbi)->sit_info = NULL; 4030 kfree(sit_i->sit_bitmap); 4031 #ifdef CONFIG_F2FS_CHECK_FS 4032 kfree(sit_i->sit_bitmap_mir); 4033 #endif 4034 kfree(sit_i); 4035 } 4036 4037 void destroy_segment_manager(struct f2fs_sb_info *sbi) 4038 { 4039 struct f2fs_sm_info *sm_info = SM_I(sbi); 4040 4041 if (!sm_info) 4042 return; 4043 destroy_flush_cmd_control(sbi, true); 4044 destroy_discard_cmd_control(sbi); 4045 destroy_dirty_segmap(sbi); 4046 destroy_curseg(sbi); 4047 destroy_free_segmap(sbi); 4048 destroy_sit_info(sbi); 4049 sbi->sm_info = NULL; 4050 kfree(sm_info); 4051 } 4052 4053 int __init create_segment_manager_caches(void) 4054 { 4055 discard_entry_slab = f2fs_kmem_cache_create("discard_entry", 4056 sizeof(struct discard_entry)); 4057 if (!discard_entry_slab) 4058 goto fail; 4059 4060 discard_cmd_slab = f2fs_kmem_cache_create("discard_cmd", 4061 sizeof(struct discard_cmd)); 4062 if (!discard_cmd_slab) 4063 goto destroy_discard_entry; 4064 4065 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set", 4066 sizeof(struct sit_entry_set)); 4067 if (!sit_entry_set_slab) 4068 goto destroy_discard_cmd; 4069 4070 inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry", 4071 sizeof(struct inmem_pages)); 4072 if (!inmem_entry_slab) 4073 goto destroy_sit_entry_set; 4074 return 0; 4075 4076 destroy_sit_entry_set: 4077 kmem_cache_destroy(sit_entry_set_slab); 4078 destroy_discard_cmd: 4079 kmem_cache_destroy(discard_cmd_slab); 4080 destroy_discard_entry: 4081 kmem_cache_destroy(discard_entry_slab); 4082 fail: 4083 return -ENOMEM; 4084 } 4085 4086 void destroy_segment_manager_caches(void) 4087 { 4088 kmem_cache_destroy(sit_entry_set_slab); 4089 kmem_cache_destroy(discard_cmd_slab); 4090 kmem_cache_destroy(discard_entry_slab); 4091 kmem_cache_destroy(inmem_entry_slab); 4092 } 4093