1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/segment.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/prefetch.h> 13 #include <linux/kthread.h> 14 #include <linux/swap.h> 15 #include <linux/timer.h> 16 #include <linux/freezer.h> 17 #include <linux/sched/signal.h> 18 19 #include "f2fs.h" 20 #include "segment.h" 21 #include "node.h" 22 #include "gc.h" 23 #include "trace.h" 24 #include <trace/events/f2fs.h> 25 26 #define __reverse_ffz(x) __reverse_ffs(~(x)) 27 28 static struct kmem_cache *discard_entry_slab; 29 static struct kmem_cache *discard_cmd_slab; 30 static struct kmem_cache *sit_entry_set_slab; 31 static struct kmem_cache *inmem_entry_slab; 32 33 static unsigned long __reverse_ulong(unsigned char *str) 34 { 35 unsigned long tmp = 0; 36 int shift = 24, idx = 0; 37 38 #if BITS_PER_LONG == 64 39 shift = 56; 40 #endif 41 while (shift >= 0) { 42 tmp |= (unsigned long)str[idx++] << shift; 43 shift -= BITS_PER_BYTE; 44 } 45 return tmp; 46 } 47 48 /* 49 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 50 * MSB and LSB are reversed in a byte by f2fs_set_bit. 51 */ 52 static inline unsigned long __reverse_ffs(unsigned long word) 53 { 54 int num = 0; 55 56 #if BITS_PER_LONG == 64 57 if ((word & 0xffffffff00000000UL) == 0) 58 num += 32; 59 else 60 word >>= 32; 61 #endif 62 if ((word & 0xffff0000) == 0) 63 num += 16; 64 else 65 word >>= 16; 66 67 if ((word & 0xff00) == 0) 68 num += 8; 69 else 70 word >>= 8; 71 72 if ((word & 0xf0) == 0) 73 num += 4; 74 else 75 word >>= 4; 76 77 if ((word & 0xc) == 0) 78 num += 2; 79 else 80 word >>= 2; 81 82 if ((word & 0x2) == 0) 83 num += 1; 84 return num; 85 } 86 87 /* 88 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because 89 * f2fs_set_bit makes MSB and LSB reversed in a byte. 90 * @size must be integral times of unsigned long. 91 * Example: 92 * MSB <--> LSB 93 * f2fs_set_bit(0, bitmap) => 1000 0000 94 * f2fs_set_bit(7, bitmap) => 0000 0001 95 */ 96 static unsigned long __find_rev_next_bit(const unsigned long *addr, 97 unsigned long size, unsigned long offset) 98 { 99 const unsigned long *p = addr + BIT_WORD(offset); 100 unsigned long result = size; 101 unsigned long tmp; 102 103 if (offset >= size) 104 return size; 105 106 size -= (offset & ~(BITS_PER_LONG - 1)); 107 offset %= BITS_PER_LONG; 108 109 while (1) { 110 if (*p == 0) 111 goto pass; 112 113 tmp = __reverse_ulong((unsigned char *)p); 114 115 tmp &= ~0UL >> offset; 116 if (size < BITS_PER_LONG) 117 tmp &= (~0UL << (BITS_PER_LONG - size)); 118 if (tmp) 119 goto found; 120 pass: 121 if (size <= BITS_PER_LONG) 122 break; 123 size -= BITS_PER_LONG; 124 offset = 0; 125 p++; 126 } 127 return result; 128 found: 129 return result - size + __reverse_ffs(tmp); 130 } 131 132 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 133 unsigned long size, unsigned long offset) 134 { 135 const unsigned long *p = addr + BIT_WORD(offset); 136 unsigned long result = size; 137 unsigned long tmp; 138 139 if (offset >= size) 140 return size; 141 142 size -= (offset & ~(BITS_PER_LONG - 1)); 143 offset %= BITS_PER_LONG; 144 145 while (1) { 146 if (*p == ~0UL) 147 goto pass; 148 149 tmp = __reverse_ulong((unsigned char *)p); 150 151 if (offset) 152 tmp |= ~0UL << (BITS_PER_LONG - offset); 153 if (size < BITS_PER_LONG) 154 tmp |= ~0UL >> size; 155 if (tmp != ~0UL) 156 goto found; 157 pass: 158 if (size <= BITS_PER_LONG) 159 break; 160 size -= BITS_PER_LONG; 161 offset = 0; 162 p++; 163 } 164 return result; 165 found: 166 return result - size + __reverse_ffz(tmp); 167 } 168 169 bool f2fs_need_SSR(struct f2fs_sb_info *sbi) 170 { 171 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); 172 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); 173 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); 174 175 if (test_opt(sbi, LFS)) 176 return false; 177 if (sbi->gc_mode == GC_URGENT) 178 return true; 179 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 180 return true; 181 182 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + 183 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); 184 } 185 186 void f2fs_register_inmem_page(struct inode *inode, struct page *page) 187 { 188 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 189 struct f2fs_inode_info *fi = F2FS_I(inode); 190 struct inmem_pages *new; 191 192 f2fs_trace_pid(page); 193 194 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); 195 SetPagePrivate(page); 196 197 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); 198 199 /* add atomic page indices to the list */ 200 new->page = page; 201 INIT_LIST_HEAD(&new->list); 202 203 /* increase reference count with clean state */ 204 mutex_lock(&fi->inmem_lock); 205 get_page(page); 206 list_add_tail(&new->list, &fi->inmem_pages); 207 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 208 if (list_empty(&fi->inmem_ilist)) 209 list_add_tail(&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE]); 210 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 211 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); 212 mutex_unlock(&fi->inmem_lock); 213 214 trace_f2fs_register_inmem_page(page, INMEM); 215 } 216 217 static int __revoke_inmem_pages(struct inode *inode, 218 struct list_head *head, bool drop, bool recover) 219 { 220 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 221 struct inmem_pages *cur, *tmp; 222 int err = 0; 223 224 list_for_each_entry_safe(cur, tmp, head, list) { 225 struct page *page = cur->page; 226 227 if (drop) 228 trace_f2fs_commit_inmem_page(page, INMEM_DROP); 229 230 lock_page(page); 231 232 f2fs_wait_on_page_writeback(page, DATA, true); 233 234 if (recover) { 235 struct dnode_of_data dn; 236 struct node_info ni; 237 238 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE); 239 retry: 240 set_new_dnode(&dn, inode, NULL, NULL, 0); 241 err = f2fs_get_dnode_of_data(&dn, page->index, 242 LOOKUP_NODE); 243 if (err) { 244 if (err == -ENOMEM) { 245 congestion_wait(BLK_RW_ASYNC, HZ/50); 246 cond_resched(); 247 goto retry; 248 } 249 err = -EAGAIN; 250 goto next; 251 } 252 253 err = f2fs_get_node_info(sbi, dn.nid, &ni); 254 if (err) { 255 f2fs_put_dnode(&dn); 256 return err; 257 } 258 259 if (cur->old_addr == NEW_ADDR) { 260 f2fs_invalidate_blocks(sbi, dn.data_blkaddr); 261 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 262 } else 263 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, 264 cur->old_addr, ni.version, true, true); 265 f2fs_put_dnode(&dn); 266 } 267 next: 268 /* we don't need to invalidate this in the sccessful status */ 269 if (drop || recover) { 270 ClearPageUptodate(page); 271 clear_cold_data(page); 272 } 273 set_page_private(page, 0); 274 ClearPagePrivate(page); 275 f2fs_put_page(page, 1); 276 277 list_del(&cur->list); 278 kmem_cache_free(inmem_entry_slab, cur); 279 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); 280 } 281 return err; 282 } 283 284 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) 285 { 286 struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; 287 struct inode *inode; 288 struct f2fs_inode_info *fi; 289 next: 290 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 291 if (list_empty(head)) { 292 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 293 return; 294 } 295 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); 296 inode = igrab(&fi->vfs_inode); 297 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 298 299 if (inode) { 300 if (gc_failure) { 301 if (fi->i_gc_failures[GC_FAILURE_ATOMIC]) 302 goto drop; 303 goto skip; 304 } 305 drop: 306 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST); 307 f2fs_drop_inmem_pages(inode); 308 iput(inode); 309 } 310 skip: 311 congestion_wait(BLK_RW_ASYNC, HZ/50); 312 cond_resched(); 313 goto next; 314 } 315 316 void f2fs_drop_inmem_pages(struct inode *inode) 317 { 318 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 319 struct f2fs_inode_info *fi = F2FS_I(inode); 320 321 mutex_lock(&fi->inmem_lock); 322 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false); 323 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 324 if (!list_empty(&fi->inmem_ilist)) 325 list_del_init(&fi->inmem_ilist); 326 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 327 mutex_unlock(&fi->inmem_lock); 328 329 clear_inode_flag(inode, FI_ATOMIC_FILE); 330 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0; 331 stat_dec_atomic_write(inode); 332 } 333 334 void f2fs_drop_inmem_page(struct inode *inode, struct page *page) 335 { 336 struct f2fs_inode_info *fi = F2FS_I(inode); 337 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 338 struct list_head *head = &fi->inmem_pages; 339 struct inmem_pages *cur = NULL; 340 341 f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); 342 343 mutex_lock(&fi->inmem_lock); 344 list_for_each_entry(cur, head, list) { 345 if (cur->page == page) 346 break; 347 } 348 349 f2fs_bug_on(sbi, list_empty(head) || cur->page != page); 350 list_del(&cur->list); 351 mutex_unlock(&fi->inmem_lock); 352 353 dec_page_count(sbi, F2FS_INMEM_PAGES); 354 kmem_cache_free(inmem_entry_slab, cur); 355 356 ClearPageUptodate(page); 357 set_page_private(page, 0); 358 ClearPagePrivate(page); 359 f2fs_put_page(page, 0); 360 361 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE); 362 } 363 364 static int __f2fs_commit_inmem_pages(struct inode *inode) 365 { 366 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 367 struct f2fs_inode_info *fi = F2FS_I(inode); 368 struct inmem_pages *cur, *tmp; 369 struct f2fs_io_info fio = { 370 .sbi = sbi, 371 .ino = inode->i_ino, 372 .type = DATA, 373 .op = REQ_OP_WRITE, 374 .op_flags = REQ_SYNC | REQ_PRIO, 375 .io_type = FS_DATA_IO, 376 }; 377 struct list_head revoke_list; 378 bool submit_bio = false; 379 int err = 0; 380 381 INIT_LIST_HEAD(&revoke_list); 382 383 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) { 384 struct page *page = cur->page; 385 386 lock_page(page); 387 if (page->mapping == inode->i_mapping) { 388 trace_f2fs_commit_inmem_page(page, INMEM); 389 390 f2fs_wait_on_page_writeback(page, DATA, true); 391 392 set_page_dirty(page); 393 if (clear_page_dirty_for_io(page)) { 394 inode_dec_dirty_pages(inode); 395 f2fs_remove_dirty_inode(inode); 396 } 397 retry: 398 fio.page = page; 399 fio.old_blkaddr = NULL_ADDR; 400 fio.encrypted_page = NULL; 401 fio.need_lock = LOCK_DONE; 402 err = f2fs_do_write_data_page(&fio); 403 if (err) { 404 if (err == -ENOMEM) { 405 congestion_wait(BLK_RW_ASYNC, HZ/50); 406 cond_resched(); 407 goto retry; 408 } 409 unlock_page(page); 410 break; 411 } 412 /* record old blkaddr for revoking */ 413 cur->old_addr = fio.old_blkaddr; 414 submit_bio = true; 415 } 416 unlock_page(page); 417 list_move_tail(&cur->list, &revoke_list); 418 } 419 420 if (submit_bio) 421 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA); 422 423 if (err) { 424 /* 425 * try to revoke all committed pages, but still we could fail 426 * due to no memory or other reason, if that happened, EAGAIN 427 * will be returned, which means in such case, transaction is 428 * already not integrity, caller should use journal to do the 429 * recovery or rewrite & commit last transaction. For other 430 * error number, revoking was done by filesystem itself. 431 */ 432 err = __revoke_inmem_pages(inode, &revoke_list, false, true); 433 434 /* drop all uncommitted pages */ 435 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false); 436 } else { 437 __revoke_inmem_pages(inode, &revoke_list, false, false); 438 } 439 440 return err; 441 } 442 443 int f2fs_commit_inmem_pages(struct inode *inode) 444 { 445 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 446 struct f2fs_inode_info *fi = F2FS_I(inode); 447 int err; 448 449 f2fs_balance_fs(sbi, true); 450 451 down_write(&fi->i_gc_rwsem[WRITE]); 452 453 f2fs_lock_op(sbi); 454 set_inode_flag(inode, FI_ATOMIC_COMMIT); 455 456 mutex_lock(&fi->inmem_lock); 457 err = __f2fs_commit_inmem_pages(inode); 458 459 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); 460 if (!list_empty(&fi->inmem_ilist)) 461 list_del_init(&fi->inmem_ilist); 462 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); 463 mutex_unlock(&fi->inmem_lock); 464 465 clear_inode_flag(inode, FI_ATOMIC_COMMIT); 466 467 f2fs_unlock_op(sbi); 468 up_write(&fi->i_gc_rwsem[WRITE]); 469 470 return err; 471 } 472 473 /* 474 * This function balances dirty node and dentry pages. 475 * In addition, it controls garbage collection. 476 */ 477 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) 478 { 479 if (time_to_inject(sbi, FAULT_CHECKPOINT)) { 480 f2fs_show_injection_info(FAULT_CHECKPOINT); 481 f2fs_stop_checkpoint(sbi, false); 482 } 483 484 /* balance_fs_bg is able to be pending */ 485 if (need && excess_cached_nats(sbi)) 486 f2fs_balance_fs_bg(sbi); 487 488 if (f2fs_is_checkpoint_ready(sbi)) 489 return; 490 491 /* 492 * We should do GC or end up with checkpoint, if there are so many dirty 493 * dir/node pages without enough free segments. 494 */ 495 if (has_not_enough_free_secs(sbi, 0, 0)) { 496 mutex_lock(&sbi->gc_mutex); 497 f2fs_gc(sbi, false, false, NULL_SEGNO); 498 } 499 } 500 501 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) 502 { 503 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 504 return; 505 506 /* try to shrink extent cache when there is no enough memory */ 507 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE)) 508 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); 509 510 /* check the # of cached NAT entries */ 511 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) 512 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); 513 514 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) 515 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); 516 else 517 f2fs_build_free_nids(sbi, false, false); 518 519 if (!is_idle(sbi, REQ_TIME) && 520 (!excess_dirty_nats(sbi) && !excess_dirty_nodes(sbi))) 521 return; 522 523 /* checkpoint is the only way to shrink partial cached entries */ 524 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES) || 525 !f2fs_available_free_memory(sbi, INO_ENTRIES) || 526 excess_prefree_segs(sbi) || 527 excess_dirty_nats(sbi) || 528 excess_dirty_nodes(sbi) || 529 f2fs_time_over(sbi, CP_TIME)) { 530 if (test_opt(sbi, DATA_FLUSH)) { 531 struct blk_plug plug; 532 533 blk_start_plug(&plug); 534 f2fs_sync_dirty_inodes(sbi, FILE_INODE); 535 blk_finish_plug(&plug); 536 } 537 f2fs_sync_fs(sbi->sb, true); 538 stat_inc_bg_cp_count(sbi->stat_info); 539 } 540 } 541 542 static int __submit_flush_wait(struct f2fs_sb_info *sbi, 543 struct block_device *bdev) 544 { 545 struct bio *bio = f2fs_bio_alloc(sbi, 0, true); 546 int ret; 547 548 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH; 549 bio_set_dev(bio, bdev); 550 ret = submit_bio_wait(bio); 551 bio_put(bio); 552 553 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), 554 test_opt(sbi, FLUSH_MERGE), ret); 555 return ret; 556 } 557 558 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) 559 { 560 int ret = 0; 561 int i; 562 563 if (!sbi->s_ndevs) 564 return __submit_flush_wait(sbi, sbi->sb->s_bdev); 565 566 for (i = 0; i < sbi->s_ndevs; i++) { 567 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) 568 continue; 569 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 570 if (ret) 571 break; 572 } 573 return ret; 574 } 575 576 static int issue_flush_thread(void *data) 577 { 578 struct f2fs_sb_info *sbi = data; 579 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 580 wait_queue_head_t *q = &fcc->flush_wait_queue; 581 repeat: 582 if (kthread_should_stop()) 583 return 0; 584 585 sb_start_intwrite(sbi->sb); 586 587 if (!llist_empty(&fcc->issue_list)) { 588 struct flush_cmd *cmd, *next; 589 int ret; 590 591 fcc->dispatch_list = llist_del_all(&fcc->issue_list); 592 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); 593 594 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode); 595 596 ret = submit_flush_wait(sbi, cmd->ino); 597 atomic_inc(&fcc->issued_flush); 598 599 llist_for_each_entry_safe(cmd, next, 600 fcc->dispatch_list, llnode) { 601 cmd->ret = ret; 602 complete(&cmd->wait); 603 } 604 fcc->dispatch_list = NULL; 605 } 606 607 sb_end_intwrite(sbi->sb); 608 609 wait_event_interruptible(*q, 610 kthread_should_stop() || !llist_empty(&fcc->issue_list)); 611 goto repeat; 612 } 613 614 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) 615 { 616 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 617 struct flush_cmd cmd; 618 int ret; 619 620 if (test_opt(sbi, NOBARRIER)) 621 return 0; 622 623 if (!test_opt(sbi, FLUSH_MERGE)) { 624 atomic_inc(&fcc->issing_flush); 625 ret = submit_flush_wait(sbi, ino); 626 atomic_dec(&fcc->issing_flush); 627 atomic_inc(&fcc->issued_flush); 628 return ret; 629 } 630 631 if (atomic_inc_return(&fcc->issing_flush) == 1 || sbi->s_ndevs > 1) { 632 ret = submit_flush_wait(sbi, ino); 633 atomic_dec(&fcc->issing_flush); 634 635 atomic_inc(&fcc->issued_flush); 636 return ret; 637 } 638 639 cmd.ino = ino; 640 init_completion(&cmd.wait); 641 642 llist_add(&cmd.llnode, &fcc->issue_list); 643 644 /* update issue_list before we wake up issue_flush thread */ 645 smp_mb(); 646 647 if (waitqueue_active(&fcc->flush_wait_queue)) 648 wake_up(&fcc->flush_wait_queue); 649 650 if (fcc->f2fs_issue_flush) { 651 wait_for_completion(&cmd.wait); 652 atomic_dec(&fcc->issing_flush); 653 } else { 654 struct llist_node *list; 655 656 list = llist_del_all(&fcc->issue_list); 657 if (!list) { 658 wait_for_completion(&cmd.wait); 659 atomic_dec(&fcc->issing_flush); 660 } else { 661 struct flush_cmd *tmp, *next; 662 663 ret = submit_flush_wait(sbi, ino); 664 665 llist_for_each_entry_safe(tmp, next, list, llnode) { 666 if (tmp == &cmd) { 667 cmd.ret = ret; 668 atomic_dec(&fcc->issing_flush); 669 continue; 670 } 671 tmp->ret = ret; 672 complete(&tmp->wait); 673 } 674 } 675 } 676 677 return cmd.ret; 678 } 679 680 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) 681 { 682 dev_t dev = sbi->sb->s_bdev->bd_dev; 683 struct flush_cmd_control *fcc; 684 int err = 0; 685 686 if (SM_I(sbi)->fcc_info) { 687 fcc = SM_I(sbi)->fcc_info; 688 if (fcc->f2fs_issue_flush) 689 return err; 690 goto init_thread; 691 } 692 693 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); 694 if (!fcc) 695 return -ENOMEM; 696 atomic_set(&fcc->issued_flush, 0); 697 atomic_set(&fcc->issing_flush, 0); 698 init_waitqueue_head(&fcc->flush_wait_queue); 699 init_llist_head(&fcc->issue_list); 700 SM_I(sbi)->fcc_info = fcc; 701 if (!test_opt(sbi, FLUSH_MERGE)) 702 return err; 703 704 init_thread: 705 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, 706 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); 707 if (IS_ERR(fcc->f2fs_issue_flush)) { 708 err = PTR_ERR(fcc->f2fs_issue_flush); 709 kvfree(fcc); 710 SM_I(sbi)->fcc_info = NULL; 711 return err; 712 } 713 714 return err; 715 } 716 717 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) 718 { 719 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 720 721 if (fcc && fcc->f2fs_issue_flush) { 722 struct task_struct *flush_thread = fcc->f2fs_issue_flush; 723 724 fcc->f2fs_issue_flush = NULL; 725 kthread_stop(flush_thread); 726 } 727 if (free) { 728 kvfree(fcc); 729 SM_I(sbi)->fcc_info = NULL; 730 } 731 } 732 733 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) 734 { 735 int ret = 0, i; 736 737 if (!sbi->s_ndevs) 738 return 0; 739 740 for (i = 1; i < sbi->s_ndevs; i++) { 741 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) 742 continue; 743 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 744 if (ret) 745 break; 746 747 spin_lock(&sbi->dev_lock); 748 f2fs_clear_bit(i, (char *)&sbi->dirty_device); 749 spin_unlock(&sbi->dev_lock); 750 } 751 752 return ret; 753 } 754 755 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 756 enum dirty_type dirty_type) 757 { 758 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 759 760 /* need not be added */ 761 if (IS_CURSEG(sbi, segno)) 762 return; 763 764 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 765 dirty_i->nr_dirty[dirty_type]++; 766 767 if (dirty_type == DIRTY) { 768 struct seg_entry *sentry = get_seg_entry(sbi, segno); 769 enum dirty_type t = sentry->type; 770 771 if (unlikely(t >= DIRTY)) { 772 f2fs_bug_on(sbi, 1); 773 return; 774 } 775 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 776 dirty_i->nr_dirty[t]++; 777 } 778 } 779 780 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 781 enum dirty_type dirty_type) 782 { 783 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 784 785 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 786 dirty_i->nr_dirty[dirty_type]--; 787 788 if (dirty_type == DIRTY) { 789 struct seg_entry *sentry = get_seg_entry(sbi, segno); 790 enum dirty_type t = sentry->type; 791 792 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 793 dirty_i->nr_dirty[t]--; 794 795 if (get_valid_blocks(sbi, segno, true) == 0) 796 clear_bit(GET_SEC_FROM_SEG(sbi, segno), 797 dirty_i->victim_secmap); 798 } 799 } 800 801 /* 802 * Should not occur error such as -ENOMEM. 803 * Adding dirty entry into seglist is not critical operation. 804 * If a given segment is one of current working segments, it won't be added. 805 */ 806 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 807 { 808 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 809 unsigned short valid_blocks, ckpt_valid_blocks; 810 811 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) 812 return; 813 814 mutex_lock(&dirty_i->seglist_lock); 815 816 valid_blocks = get_valid_blocks(sbi, segno, false); 817 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno); 818 819 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || 820 ckpt_valid_blocks == sbi->blocks_per_seg)) { 821 __locate_dirty_segment(sbi, segno, PRE); 822 __remove_dirty_segment(sbi, segno, DIRTY); 823 } else if (valid_blocks < sbi->blocks_per_seg) { 824 __locate_dirty_segment(sbi, segno, DIRTY); 825 } else { 826 /* Recovery routine with SSR needs this */ 827 __remove_dirty_segment(sbi, segno, DIRTY); 828 } 829 830 mutex_unlock(&dirty_i->seglist_lock); 831 } 832 833 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */ 834 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) 835 { 836 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 837 unsigned int segno; 838 839 mutex_lock(&dirty_i->seglist_lock); 840 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 841 if (get_valid_blocks(sbi, segno, false)) 842 continue; 843 if (IS_CURSEG(sbi, segno)) 844 continue; 845 __locate_dirty_segment(sbi, segno, PRE); 846 __remove_dirty_segment(sbi, segno, DIRTY); 847 } 848 mutex_unlock(&dirty_i->seglist_lock); 849 } 850 851 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi) 852 { 853 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 854 block_t ovp = overprovision_segments(sbi) << sbi->log_blocks_per_seg; 855 block_t holes[2] = {0, 0}; /* DATA and NODE */ 856 struct seg_entry *se; 857 unsigned int segno; 858 859 mutex_lock(&dirty_i->seglist_lock); 860 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 861 se = get_seg_entry(sbi, segno); 862 if (IS_NODESEG(se->type)) 863 holes[NODE] += sbi->blocks_per_seg - se->valid_blocks; 864 else 865 holes[DATA] += sbi->blocks_per_seg - se->valid_blocks; 866 } 867 mutex_unlock(&dirty_i->seglist_lock); 868 869 if (holes[DATA] > ovp || holes[NODE] > ovp) 870 return -EAGAIN; 871 return 0; 872 } 873 874 /* This is only used by SBI_CP_DISABLED */ 875 static unsigned int get_free_segment(struct f2fs_sb_info *sbi) 876 { 877 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 878 unsigned int segno = 0; 879 880 mutex_lock(&dirty_i->seglist_lock); 881 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 882 if (get_valid_blocks(sbi, segno, false)) 883 continue; 884 if (get_ckpt_valid_blocks(sbi, segno)) 885 continue; 886 mutex_unlock(&dirty_i->seglist_lock); 887 return segno; 888 } 889 mutex_unlock(&dirty_i->seglist_lock); 890 return NULL_SEGNO; 891 } 892 893 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, 894 struct block_device *bdev, block_t lstart, 895 block_t start, block_t len) 896 { 897 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 898 struct list_head *pend_list; 899 struct discard_cmd *dc; 900 901 f2fs_bug_on(sbi, !len); 902 903 pend_list = &dcc->pend_list[plist_idx(len)]; 904 905 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS); 906 INIT_LIST_HEAD(&dc->list); 907 dc->bdev = bdev; 908 dc->lstart = lstart; 909 dc->start = start; 910 dc->len = len; 911 dc->ref = 0; 912 dc->state = D_PREP; 913 dc->issuing = 0; 914 dc->error = 0; 915 init_completion(&dc->wait); 916 list_add_tail(&dc->list, pend_list); 917 spin_lock_init(&dc->lock); 918 dc->bio_ref = 0; 919 atomic_inc(&dcc->discard_cmd_cnt); 920 dcc->undiscard_blks += len; 921 922 return dc; 923 } 924 925 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, 926 struct block_device *bdev, block_t lstart, 927 block_t start, block_t len, 928 struct rb_node *parent, struct rb_node **p, 929 bool leftmost) 930 { 931 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 932 struct discard_cmd *dc; 933 934 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); 935 936 rb_link_node(&dc->rb_node, parent, p); 937 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); 938 939 return dc; 940 } 941 942 static void __detach_discard_cmd(struct discard_cmd_control *dcc, 943 struct discard_cmd *dc) 944 { 945 if (dc->state == D_DONE) 946 atomic_sub(dc->issuing, &dcc->issing_discard); 947 948 list_del(&dc->list); 949 rb_erase_cached(&dc->rb_node, &dcc->root); 950 dcc->undiscard_blks -= dc->len; 951 952 kmem_cache_free(discard_cmd_slab, dc); 953 954 atomic_dec(&dcc->discard_cmd_cnt); 955 } 956 957 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, 958 struct discard_cmd *dc) 959 { 960 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 961 unsigned long flags; 962 963 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len); 964 965 spin_lock_irqsave(&dc->lock, flags); 966 if (dc->bio_ref) { 967 spin_unlock_irqrestore(&dc->lock, flags); 968 return; 969 } 970 spin_unlock_irqrestore(&dc->lock, flags); 971 972 f2fs_bug_on(sbi, dc->ref); 973 974 if (dc->error == -EOPNOTSUPP) 975 dc->error = 0; 976 977 if (dc->error) 978 printk_ratelimited( 979 "%sF2FS-fs: Issue discard(%u, %u, %u) failed, ret: %d", 980 KERN_INFO, dc->lstart, dc->start, dc->len, dc->error); 981 __detach_discard_cmd(dcc, dc); 982 } 983 984 static void f2fs_submit_discard_endio(struct bio *bio) 985 { 986 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; 987 unsigned long flags; 988 989 dc->error = blk_status_to_errno(bio->bi_status); 990 991 spin_lock_irqsave(&dc->lock, flags); 992 dc->bio_ref--; 993 if (!dc->bio_ref && dc->state == D_SUBMIT) { 994 dc->state = D_DONE; 995 complete_all(&dc->wait); 996 } 997 spin_unlock_irqrestore(&dc->lock, flags); 998 bio_put(bio); 999 } 1000 1001 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, 1002 block_t start, block_t end) 1003 { 1004 #ifdef CONFIG_F2FS_CHECK_FS 1005 struct seg_entry *sentry; 1006 unsigned int segno; 1007 block_t blk = start; 1008 unsigned long offset, size, max_blocks = sbi->blocks_per_seg; 1009 unsigned long *map; 1010 1011 while (blk < end) { 1012 segno = GET_SEGNO(sbi, blk); 1013 sentry = get_seg_entry(sbi, segno); 1014 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); 1015 1016 if (end < START_BLOCK(sbi, segno + 1)) 1017 size = GET_BLKOFF_FROM_SEG0(sbi, end); 1018 else 1019 size = max_blocks; 1020 map = (unsigned long *)(sentry->cur_valid_map); 1021 offset = __find_rev_next_bit(map, size, offset); 1022 f2fs_bug_on(sbi, offset != size); 1023 blk = START_BLOCK(sbi, segno + 1); 1024 } 1025 #endif 1026 } 1027 1028 static void __init_discard_policy(struct f2fs_sb_info *sbi, 1029 struct discard_policy *dpolicy, 1030 int discard_type, unsigned int granularity) 1031 { 1032 /* common policy */ 1033 dpolicy->type = discard_type; 1034 dpolicy->sync = true; 1035 dpolicy->ordered = false; 1036 dpolicy->granularity = granularity; 1037 1038 dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST; 1039 dpolicy->io_aware_gran = MAX_PLIST_NUM; 1040 1041 if (discard_type == DPOLICY_BG) { 1042 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME; 1043 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME; 1044 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME; 1045 dpolicy->io_aware = true; 1046 dpolicy->sync = false; 1047 dpolicy->ordered = true; 1048 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) { 1049 dpolicy->granularity = 1; 1050 dpolicy->max_interval = DEF_MIN_DISCARD_ISSUE_TIME; 1051 } 1052 } else if (discard_type == DPOLICY_FORCE) { 1053 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME; 1054 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME; 1055 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME; 1056 dpolicy->io_aware = false; 1057 } else if (discard_type == DPOLICY_FSTRIM) { 1058 dpolicy->io_aware = false; 1059 } else if (discard_type == DPOLICY_UMOUNT) { 1060 dpolicy->max_requests = UINT_MAX; 1061 dpolicy->io_aware = false; 1062 } 1063 } 1064 1065 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1066 struct block_device *bdev, block_t lstart, 1067 block_t start, block_t len); 1068 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ 1069 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, 1070 struct discard_policy *dpolicy, 1071 struct discard_cmd *dc, 1072 unsigned int *issued) 1073 { 1074 struct block_device *bdev = dc->bdev; 1075 struct request_queue *q = bdev_get_queue(bdev); 1076 unsigned int max_discard_blocks = 1077 SECTOR_TO_BLOCK(q->limits.max_discard_sectors); 1078 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1079 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1080 &(dcc->fstrim_list) : &(dcc->wait_list); 1081 int flag = dpolicy->sync ? REQ_SYNC : 0; 1082 block_t lstart, start, len, total_len; 1083 int err = 0; 1084 1085 if (dc->state != D_PREP) 1086 return 0; 1087 1088 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 1089 return 0; 1090 1091 trace_f2fs_issue_discard(bdev, dc->start, dc->len); 1092 1093 lstart = dc->lstart; 1094 start = dc->start; 1095 len = dc->len; 1096 total_len = len; 1097 1098 dc->len = 0; 1099 1100 while (total_len && *issued < dpolicy->max_requests && !err) { 1101 struct bio *bio = NULL; 1102 unsigned long flags; 1103 bool last = true; 1104 1105 if (len > max_discard_blocks) { 1106 len = max_discard_blocks; 1107 last = false; 1108 } 1109 1110 (*issued)++; 1111 if (*issued == dpolicy->max_requests) 1112 last = true; 1113 1114 dc->len += len; 1115 1116 if (time_to_inject(sbi, FAULT_DISCARD)) { 1117 f2fs_show_injection_info(FAULT_DISCARD); 1118 err = -EIO; 1119 goto submit; 1120 } 1121 err = __blkdev_issue_discard(bdev, 1122 SECTOR_FROM_BLOCK(start), 1123 SECTOR_FROM_BLOCK(len), 1124 GFP_NOFS, 0, &bio); 1125 submit: 1126 if (err) { 1127 spin_lock_irqsave(&dc->lock, flags); 1128 if (dc->state == D_PARTIAL) 1129 dc->state = D_SUBMIT; 1130 spin_unlock_irqrestore(&dc->lock, flags); 1131 1132 break; 1133 } 1134 1135 f2fs_bug_on(sbi, !bio); 1136 1137 /* 1138 * should keep before submission to avoid D_DONE 1139 * right away 1140 */ 1141 spin_lock_irqsave(&dc->lock, flags); 1142 if (last) 1143 dc->state = D_SUBMIT; 1144 else 1145 dc->state = D_PARTIAL; 1146 dc->bio_ref++; 1147 spin_unlock_irqrestore(&dc->lock, flags); 1148 1149 atomic_inc(&dcc->issing_discard); 1150 dc->issuing++; 1151 list_move_tail(&dc->list, wait_list); 1152 1153 /* sanity check on discard range */ 1154 __check_sit_bitmap(sbi, start, start + len); 1155 1156 bio->bi_private = dc; 1157 bio->bi_end_io = f2fs_submit_discard_endio; 1158 bio->bi_opf |= flag; 1159 submit_bio(bio); 1160 1161 atomic_inc(&dcc->issued_discard); 1162 1163 f2fs_update_iostat(sbi, FS_DISCARD, 1); 1164 1165 lstart += len; 1166 start += len; 1167 total_len -= len; 1168 len = total_len; 1169 } 1170 1171 if (!err && len) 1172 __update_discard_tree_range(sbi, bdev, lstart, start, len); 1173 return err; 1174 } 1175 1176 static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi, 1177 struct block_device *bdev, block_t lstart, 1178 block_t start, block_t len, 1179 struct rb_node **insert_p, 1180 struct rb_node *insert_parent) 1181 { 1182 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1183 struct rb_node **p; 1184 struct rb_node *parent = NULL; 1185 struct discard_cmd *dc = NULL; 1186 bool leftmost = true; 1187 1188 if (insert_p && insert_parent) { 1189 parent = insert_parent; 1190 p = insert_p; 1191 goto do_insert; 1192 } 1193 1194 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, 1195 lstart, &leftmost); 1196 do_insert: 1197 dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, 1198 p, leftmost); 1199 if (!dc) 1200 return NULL; 1201 1202 return dc; 1203 } 1204 1205 static void __relocate_discard_cmd(struct discard_cmd_control *dcc, 1206 struct discard_cmd *dc) 1207 { 1208 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]); 1209 } 1210 1211 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, 1212 struct discard_cmd *dc, block_t blkaddr) 1213 { 1214 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1215 struct discard_info di = dc->di; 1216 bool modified = false; 1217 1218 if (dc->state == D_DONE || dc->len == 1) { 1219 __remove_discard_cmd(sbi, dc); 1220 return; 1221 } 1222 1223 dcc->undiscard_blks -= di.len; 1224 1225 if (blkaddr > di.lstart) { 1226 dc->len = blkaddr - dc->lstart; 1227 dcc->undiscard_blks += dc->len; 1228 __relocate_discard_cmd(dcc, dc); 1229 modified = true; 1230 } 1231 1232 if (blkaddr < di.lstart + di.len - 1) { 1233 if (modified) { 1234 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, 1235 di.start + blkaddr + 1 - di.lstart, 1236 di.lstart + di.len - 1 - blkaddr, 1237 NULL, NULL); 1238 } else { 1239 dc->lstart++; 1240 dc->len--; 1241 dc->start++; 1242 dcc->undiscard_blks += dc->len; 1243 __relocate_discard_cmd(dcc, dc); 1244 } 1245 } 1246 } 1247 1248 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1249 struct block_device *bdev, block_t lstart, 1250 block_t start, block_t len) 1251 { 1252 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1253 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1254 struct discard_cmd *dc; 1255 struct discard_info di = {0}; 1256 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1257 struct request_queue *q = bdev_get_queue(bdev); 1258 unsigned int max_discard_blocks = 1259 SECTOR_TO_BLOCK(q->limits.max_discard_sectors); 1260 block_t end = lstart + len; 1261 1262 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 1263 NULL, lstart, 1264 (struct rb_entry **)&prev_dc, 1265 (struct rb_entry **)&next_dc, 1266 &insert_p, &insert_parent, true, NULL); 1267 if (dc) 1268 prev_dc = dc; 1269 1270 if (!prev_dc) { 1271 di.lstart = lstart; 1272 di.len = next_dc ? next_dc->lstart - lstart : len; 1273 di.len = min(di.len, len); 1274 di.start = start; 1275 } 1276 1277 while (1) { 1278 struct rb_node *node; 1279 bool merged = false; 1280 struct discard_cmd *tdc = NULL; 1281 1282 if (prev_dc) { 1283 di.lstart = prev_dc->lstart + prev_dc->len; 1284 if (di.lstart < lstart) 1285 di.lstart = lstart; 1286 if (di.lstart >= end) 1287 break; 1288 1289 if (!next_dc || next_dc->lstart > end) 1290 di.len = end - di.lstart; 1291 else 1292 di.len = next_dc->lstart - di.lstart; 1293 di.start = start + di.lstart - lstart; 1294 } 1295 1296 if (!di.len) 1297 goto next; 1298 1299 if (prev_dc && prev_dc->state == D_PREP && 1300 prev_dc->bdev == bdev && 1301 __is_discard_back_mergeable(&di, &prev_dc->di, 1302 max_discard_blocks)) { 1303 prev_dc->di.len += di.len; 1304 dcc->undiscard_blks += di.len; 1305 __relocate_discard_cmd(dcc, prev_dc); 1306 di = prev_dc->di; 1307 tdc = prev_dc; 1308 merged = true; 1309 } 1310 1311 if (next_dc && next_dc->state == D_PREP && 1312 next_dc->bdev == bdev && 1313 __is_discard_front_mergeable(&di, &next_dc->di, 1314 max_discard_blocks)) { 1315 next_dc->di.lstart = di.lstart; 1316 next_dc->di.len += di.len; 1317 next_dc->di.start = di.start; 1318 dcc->undiscard_blks += di.len; 1319 __relocate_discard_cmd(dcc, next_dc); 1320 if (tdc) 1321 __remove_discard_cmd(sbi, tdc); 1322 merged = true; 1323 } 1324 1325 if (!merged) { 1326 __insert_discard_tree(sbi, bdev, di.lstart, di.start, 1327 di.len, NULL, NULL); 1328 } 1329 next: 1330 prev_dc = next_dc; 1331 if (!prev_dc) 1332 break; 1333 1334 node = rb_next(&prev_dc->rb_node); 1335 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1336 } 1337 } 1338 1339 static int __queue_discard_cmd(struct f2fs_sb_info *sbi, 1340 struct block_device *bdev, block_t blkstart, block_t blklen) 1341 { 1342 block_t lblkstart = blkstart; 1343 1344 trace_f2fs_queue_discard(bdev, blkstart, blklen); 1345 1346 if (sbi->s_ndevs) { 1347 int devi = f2fs_target_device_index(sbi, blkstart); 1348 1349 blkstart -= FDEV(devi).start_blk; 1350 } 1351 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); 1352 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); 1353 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1354 return 0; 1355 } 1356 1357 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, 1358 struct discard_policy *dpolicy) 1359 { 1360 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1361 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1362 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1363 struct discard_cmd *dc; 1364 struct blk_plug plug; 1365 unsigned int pos = dcc->next_pos; 1366 unsigned int issued = 0; 1367 bool io_interrupted = false; 1368 1369 mutex_lock(&dcc->cmd_lock); 1370 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 1371 NULL, pos, 1372 (struct rb_entry **)&prev_dc, 1373 (struct rb_entry **)&next_dc, 1374 &insert_p, &insert_parent, true, NULL); 1375 if (!dc) 1376 dc = next_dc; 1377 1378 blk_start_plug(&plug); 1379 1380 while (dc) { 1381 struct rb_node *node; 1382 int err = 0; 1383 1384 if (dc->state != D_PREP) 1385 goto next; 1386 1387 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { 1388 io_interrupted = true; 1389 break; 1390 } 1391 1392 dcc->next_pos = dc->lstart + dc->len; 1393 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1394 1395 if (issued >= dpolicy->max_requests) 1396 break; 1397 next: 1398 node = rb_next(&dc->rb_node); 1399 if (err) 1400 __remove_discard_cmd(sbi, dc); 1401 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1402 } 1403 1404 blk_finish_plug(&plug); 1405 1406 if (!dc) 1407 dcc->next_pos = 0; 1408 1409 mutex_unlock(&dcc->cmd_lock); 1410 1411 if (!issued && io_interrupted) 1412 issued = -1; 1413 1414 return issued; 1415 } 1416 1417 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, 1418 struct discard_policy *dpolicy) 1419 { 1420 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1421 struct list_head *pend_list; 1422 struct discard_cmd *dc, *tmp; 1423 struct blk_plug plug; 1424 int i, issued = 0; 1425 bool io_interrupted = false; 1426 1427 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1428 if (i + 1 < dpolicy->granularity) 1429 break; 1430 1431 if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered) 1432 return __issue_discard_cmd_orderly(sbi, dpolicy); 1433 1434 pend_list = &dcc->pend_list[i]; 1435 1436 mutex_lock(&dcc->cmd_lock); 1437 if (list_empty(pend_list)) 1438 goto next; 1439 if (unlikely(dcc->rbtree_check)) 1440 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, 1441 &dcc->root)); 1442 blk_start_plug(&plug); 1443 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1444 f2fs_bug_on(sbi, dc->state != D_PREP); 1445 1446 if (dpolicy->io_aware && i < dpolicy->io_aware_gran && 1447 !is_idle(sbi, DISCARD_TIME)) { 1448 io_interrupted = true; 1449 break; 1450 } 1451 1452 __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1453 1454 if (issued >= dpolicy->max_requests) 1455 break; 1456 } 1457 blk_finish_plug(&plug); 1458 next: 1459 mutex_unlock(&dcc->cmd_lock); 1460 1461 if (issued >= dpolicy->max_requests || io_interrupted) 1462 break; 1463 } 1464 1465 if (!issued && io_interrupted) 1466 issued = -1; 1467 1468 return issued; 1469 } 1470 1471 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) 1472 { 1473 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1474 struct list_head *pend_list; 1475 struct discard_cmd *dc, *tmp; 1476 int i; 1477 bool dropped = false; 1478 1479 mutex_lock(&dcc->cmd_lock); 1480 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1481 pend_list = &dcc->pend_list[i]; 1482 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1483 f2fs_bug_on(sbi, dc->state != D_PREP); 1484 __remove_discard_cmd(sbi, dc); 1485 dropped = true; 1486 } 1487 } 1488 mutex_unlock(&dcc->cmd_lock); 1489 1490 return dropped; 1491 } 1492 1493 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) 1494 { 1495 __drop_discard_cmd(sbi); 1496 } 1497 1498 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, 1499 struct discard_cmd *dc) 1500 { 1501 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1502 unsigned int len = 0; 1503 1504 wait_for_completion_io(&dc->wait); 1505 mutex_lock(&dcc->cmd_lock); 1506 f2fs_bug_on(sbi, dc->state != D_DONE); 1507 dc->ref--; 1508 if (!dc->ref) { 1509 if (!dc->error) 1510 len = dc->len; 1511 __remove_discard_cmd(sbi, dc); 1512 } 1513 mutex_unlock(&dcc->cmd_lock); 1514 1515 return len; 1516 } 1517 1518 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, 1519 struct discard_policy *dpolicy, 1520 block_t start, block_t end) 1521 { 1522 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1523 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1524 &(dcc->fstrim_list) : &(dcc->wait_list); 1525 struct discard_cmd *dc, *tmp; 1526 bool need_wait; 1527 unsigned int trimmed = 0; 1528 1529 next: 1530 need_wait = false; 1531 1532 mutex_lock(&dcc->cmd_lock); 1533 list_for_each_entry_safe(dc, tmp, wait_list, list) { 1534 if (dc->lstart + dc->len <= start || end <= dc->lstart) 1535 continue; 1536 if (dc->len < dpolicy->granularity) 1537 continue; 1538 if (dc->state == D_DONE && !dc->ref) { 1539 wait_for_completion_io(&dc->wait); 1540 if (!dc->error) 1541 trimmed += dc->len; 1542 __remove_discard_cmd(sbi, dc); 1543 } else { 1544 dc->ref++; 1545 need_wait = true; 1546 break; 1547 } 1548 } 1549 mutex_unlock(&dcc->cmd_lock); 1550 1551 if (need_wait) { 1552 trimmed += __wait_one_discard_bio(sbi, dc); 1553 goto next; 1554 } 1555 1556 return trimmed; 1557 } 1558 1559 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1560 struct discard_policy *dpolicy) 1561 { 1562 struct discard_policy dp; 1563 unsigned int discard_blks; 1564 1565 if (dpolicy) 1566 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); 1567 1568 /* wait all */ 1569 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); 1570 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1571 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); 1572 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1573 1574 return discard_blks; 1575 } 1576 1577 /* This should be covered by global mutex, &sit_i->sentry_lock */ 1578 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) 1579 { 1580 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1581 struct discard_cmd *dc; 1582 bool need_wait = false; 1583 1584 mutex_lock(&dcc->cmd_lock); 1585 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root, 1586 NULL, blkaddr); 1587 if (dc) { 1588 if (dc->state == D_PREP) { 1589 __punch_discard_cmd(sbi, dc, blkaddr); 1590 } else { 1591 dc->ref++; 1592 need_wait = true; 1593 } 1594 } 1595 mutex_unlock(&dcc->cmd_lock); 1596 1597 if (need_wait) 1598 __wait_one_discard_bio(sbi, dc); 1599 } 1600 1601 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) 1602 { 1603 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1604 1605 if (dcc && dcc->f2fs_issue_discard) { 1606 struct task_struct *discard_thread = dcc->f2fs_issue_discard; 1607 1608 dcc->f2fs_issue_discard = NULL; 1609 kthread_stop(discard_thread); 1610 } 1611 } 1612 1613 /* This comes from f2fs_put_super */ 1614 bool f2fs_wait_discard_bios(struct f2fs_sb_info *sbi) 1615 { 1616 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1617 struct discard_policy dpolicy; 1618 bool dropped; 1619 1620 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, 1621 dcc->discard_granularity); 1622 __issue_discard_cmd(sbi, &dpolicy); 1623 dropped = __drop_discard_cmd(sbi); 1624 1625 /* just to make sure there is no pending discard commands */ 1626 __wait_all_discard_cmd(sbi, NULL); 1627 1628 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); 1629 return dropped; 1630 } 1631 1632 static int issue_discard_thread(void *data) 1633 { 1634 struct f2fs_sb_info *sbi = data; 1635 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1636 wait_queue_head_t *q = &dcc->discard_wait_queue; 1637 struct discard_policy dpolicy; 1638 unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME; 1639 int issued; 1640 1641 set_freezable(); 1642 1643 do { 1644 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, 1645 dcc->discard_granularity); 1646 1647 wait_event_interruptible_timeout(*q, 1648 kthread_should_stop() || freezing(current) || 1649 dcc->discard_wake, 1650 msecs_to_jiffies(wait_ms)); 1651 1652 if (dcc->discard_wake) 1653 dcc->discard_wake = 0; 1654 1655 if (try_to_freeze()) 1656 continue; 1657 if (f2fs_readonly(sbi->sb)) 1658 continue; 1659 if (kthread_should_stop()) 1660 return 0; 1661 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 1662 wait_ms = dpolicy.max_interval; 1663 continue; 1664 } 1665 1666 if (sbi->gc_mode == GC_URGENT) 1667 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); 1668 1669 sb_start_intwrite(sbi->sb); 1670 1671 issued = __issue_discard_cmd(sbi, &dpolicy); 1672 if (issued > 0) { 1673 __wait_all_discard_cmd(sbi, &dpolicy); 1674 wait_ms = dpolicy.min_interval; 1675 } else if (issued == -1){ 1676 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); 1677 if (!wait_ms) 1678 wait_ms = dpolicy.mid_interval; 1679 } else { 1680 wait_ms = dpolicy.max_interval; 1681 } 1682 1683 sb_end_intwrite(sbi->sb); 1684 1685 } while (!kthread_should_stop()); 1686 return 0; 1687 } 1688 1689 #ifdef CONFIG_BLK_DEV_ZONED 1690 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, 1691 struct block_device *bdev, block_t blkstart, block_t blklen) 1692 { 1693 sector_t sector, nr_sects; 1694 block_t lblkstart = blkstart; 1695 int devi = 0; 1696 1697 if (sbi->s_ndevs) { 1698 devi = f2fs_target_device_index(sbi, blkstart); 1699 blkstart -= FDEV(devi).start_blk; 1700 } 1701 1702 /* 1703 * We need to know the type of the zone: for conventional zones, 1704 * use regular discard if the drive supports it. For sequential 1705 * zones, reset the zone write pointer. 1706 */ 1707 switch (get_blkz_type(sbi, bdev, blkstart)) { 1708 1709 case BLK_ZONE_TYPE_CONVENTIONAL: 1710 if (!blk_queue_discard(bdev_get_queue(bdev))) 1711 return 0; 1712 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen); 1713 case BLK_ZONE_TYPE_SEQWRITE_REQ: 1714 case BLK_ZONE_TYPE_SEQWRITE_PREF: 1715 sector = SECTOR_FROM_BLOCK(blkstart); 1716 nr_sects = SECTOR_FROM_BLOCK(blklen); 1717 1718 if (sector & (bdev_zone_sectors(bdev) - 1) || 1719 nr_sects != bdev_zone_sectors(bdev)) { 1720 f2fs_msg(sbi->sb, KERN_INFO, 1721 "(%d) %s: Unaligned discard attempted (block %x + %x)", 1722 devi, sbi->s_ndevs ? FDEV(devi).path: "", 1723 blkstart, blklen); 1724 return -EIO; 1725 } 1726 trace_f2fs_issue_reset_zone(bdev, blkstart); 1727 return blkdev_reset_zones(bdev, sector, 1728 nr_sects, GFP_NOFS); 1729 default: 1730 /* Unknown zone type: broken device ? */ 1731 return -EIO; 1732 } 1733 } 1734 #endif 1735 1736 static int __issue_discard_async(struct f2fs_sb_info *sbi, 1737 struct block_device *bdev, block_t blkstart, block_t blklen) 1738 { 1739 #ifdef CONFIG_BLK_DEV_ZONED 1740 if (f2fs_sb_has_blkzoned(sbi) && 1741 bdev_zoned_model(bdev) != BLK_ZONED_NONE) 1742 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); 1743 #endif 1744 return __queue_discard_cmd(sbi, bdev, blkstart, blklen); 1745 } 1746 1747 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, 1748 block_t blkstart, block_t blklen) 1749 { 1750 sector_t start = blkstart, len = 0; 1751 struct block_device *bdev; 1752 struct seg_entry *se; 1753 unsigned int offset; 1754 block_t i; 1755 int err = 0; 1756 1757 bdev = f2fs_target_device(sbi, blkstart, NULL); 1758 1759 for (i = blkstart; i < blkstart + blklen; i++, len++) { 1760 if (i != start) { 1761 struct block_device *bdev2 = 1762 f2fs_target_device(sbi, i, NULL); 1763 1764 if (bdev2 != bdev) { 1765 err = __issue_discard_async(sbi, bdev, 1766 start, len); 1767 if (err) 1768 return err; 1769 bdev = bdev2; 1770 start = i; 1771 len = 0; 1772 } 1773 } 1774 1775 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); 1776 offset = GET_BLKOFF_FROM_SEG0(sbi, i); 1777 1778 if (!f2fs_test_and_set_bit(offset, se->discard_map)) 1779 sbi->discard_blks--; 1780 } 1781 1782 if (len) 1783 err = __issue_discard_async(sbi, bdev, start, len); 1784 return err; 1785 } 1786 1787 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, 1788 bool check_only) 1789 { 1790 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 1791 int max_blocks = sbi->blocks_per_seg; 1792 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); 1793 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 1794 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 1795 unsigned long *discard_map = (unsigned long *)se->discard_map; 1796 unsigned long *dmap = SIT_I(sbi)->tmp_map; 1797 unsigned int start = 0, end = -1; 1798 bool force = (cpc->reason & CP_DISCARD); 1799 struct discard_entry *de = NULL; 1800 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; 1801 int i; 1802 1803 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi)) 1804 return false; 1805 1806 if (!force) { 1807 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks || 1808 SM_I(sbi)->dcc_info->nr_discards >= 1809 SM_I(sbi)->dcc_info->max_discards) 1810 return false; 1811 } 1812 1813 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 1814 for (i = 0; i < entries; i++) 1815 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : 1816 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 1817 1818 while (force || SM_I(sbi)->dcc_info->nr_discards <= 1819 SM_I(sbi)->dcc_info->max_discards) { 1820 start = __find_rev_next_bit(dmap, max_blocks, end + 1); 1821 if (start >= max_blocks) 1822 break; 1823 1824 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); 1825 if (force && start && end != max_blocks 1826 && (end - start) < cpc->trim_minlen) 1827 continue; 1828 1829 if (check_only) 1830 return true; 1831 1832 if (!de) { 1833 de = f2fs_kmem_cache_alloc(discard_entry_slab, 1834 GFP_F2FS_ZERO); 1835 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); 1836 list_add_tail(&de->list, head); 1837 } 1838 1839 for (i = start; i < end; i++) 1840 __set_bit_le(i, (void *)de->discard_map); 1841 1842 SM_I(sbi)->dcc_info->nr_discards += end - start; 1843 } 1844 return false; 1845 } 1846 1847 static void release_discard_addr(struct discard_entry *entry) 1848 { 1849 list_del(&entry->list); 1850 kmem_cache_free(discard_entry_slab, entry); 1851 } 1852 1853 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) 1854 { 1855 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); 1856 struct discard_entry *entry, *this; 1857 1858 /* drop caches */ 1859 list_for_each_entry_safe(entry, this, head, list) 1860 release_discard_addr(entry); 1861 } 1862 1863 /* 1864 * Should call f2fs_clear_prefree_segments after checkpoint is done. 1865 */ 1866 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 1867 { 1868 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1869 unsigned int segno; 1870 1871 mutex_lock(&dirty_i->seglist_lock); 1872 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) 1873 __set_test_and_free(sbi, segno); 1874 mutex_unlock(&dirty_i->seglist_lock); 1875 } 1876 1877 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, 1878 struct cp_control *cpc) 1879 { 1880 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1881 struct list_head *head = &dcc->entry_list; 1882 struct discard_entry *entry, *this; 1883 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1884 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 1885 unsigned int start = 0, end = -1; 1886 unsigned int secno, start_segno; 1887 bool force = (cpc->reason & CP_DISCARD); 1888 bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi); 1889 1890 mutex_lock(&dirty_i->seglist_lock); 1891 1892 while (1) { 1893 int i; 1894 1895 if (need_align && end != -1) 1896 end--; 1897 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); 1898 if (start >= MAIN_SEGS(sbi)) 1899 break; 1900 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), 1901 start + 1); 1902 1903 if (need_align) { 1904 start = rounddown(start, sbi->segs_per_sec); 1905 end = roundup(end, sbi->segs_per_sec); 1906 } 1907 1908 for (i = start; i < end; i++) { 1909 if (test_and_clear_bit(i, prefree_map)) 1910 dirty_i->nr_dirty[PRE]--; 1911 } 1912 1913 if (!f2fs_realtime_discard_enable(sbi)) 1914 continue; 1915 1916 if (force && start >= cpc->trim_start && 1917 (end - 1) <= cpc->trim_end) 1918 continue; 1919 1920 if (!test_opt(sbi, LFS) || !__is_large_section(sbi)) { 1921 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 1922 (end - start) << sbi->log_blocks_per_seg); 1923 continue; 1924 } 1925 next: 1926 secno = GET_SEC_FROM_SEG(sbi, start); 1927 start_segno = GET_SEG_FROM_SEC(sbi, secno); 1928 if (!IS_CURSEC(sbi, secno) && 1929 !get_valid_blocks(sbi, start, true)) 1930 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), 1931 sbi->segs_per_sec << sbi->log_blocks_per_seg); 1932 1933 start = start_segno + sbi->segs_per_sec; 1934 if (start < end) 1935 goto next; 1936 else 1937 end = start - 1; 1938 } 1939 mutex_unlock(&dirty_i->seglist_lock); 1940 1941 /* send small discards */ 1942 list_for_each_entry_safe(entry, this, head, list) { 1943 unsigned int cur_pos = 0, next_pos, len, total_len = 0; 1944 bool is_valid = test_bit_le(0, entry->discard_map); 1945 1946 find_next: 1947 if (is_valid) { 1948 next_pos = find_next_zero_bit_le(entry->discard_map, 1949 sbi->blocks_per_seg, cur_pos); 1950 len = next_pos - cur_pos; 1951 1952 if (f2fs_sb_has_blkzoned(sbi) || 1953 (force && len < cpc->trim_minlen)) 1954 goto skip; 1955 1956 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, 1957 len); 1958 total_len += len; 1959 } else { 1960 next_pos = find_next_bit_le(entry->discard_map, 1961 sbi->blocks_per_seg, cur_pos); 1962 } 1963 skip: 1964 cur_pos = next_pos; 1965 is_valid = !is_valid; 1966 1967 if (cur_pos < sbi->blocks_per_seg) 1968 goto find_next; 1969 1970 release_discard_addr(entry); 1971 dcc->nr_discards -= total_len; 1972 } 1973 1974 wake_up_discard_thread(sbi, false); 1975 } 1976 1977 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) 1978 { 1979 dev_t dev = sbi->sb->s_bdev->bd_dev; 1980 struct discard_cmd_control *dcc; 1981 int err = 0, i; 1982 1983 if (SM_I(sbi)->dcc_info) { 1984 dcc = SM_I(sbi)->dcc_info; 1985 goto init_thread; 1986 } 1987 1988 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); 1989 if (!dcc) 1990 return -ENOMEM; 1991 1992 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; 1993 INIT_LIST_HEAD(&dcc->entry_list); 1994 for (i = 0; i < MAX_PLIST_NUM; i++) 1995 INIT_LIST_HEAD(&dcc->pend_list[i]); 1996 INIT_LIST_HEAD(&dcc->wait_list); 1997 INIT_LIST_HEAD(&dcc->fstrim_list); 1998 mutex_init(&dcc->cmd_lock); 1999 atomic_set(&dcc->issued_discard, 0); 2000 atomic_set(&dcc->issing_discard, 0); 2001 atomic_set(&dcc->discard_cmd_cnt, 0); 2002 dcc->nr_discards = 0; 2003 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; 2004 dcc->undiscard_blks = 0; 2005 dcc->next_pos = 0; 2006 dcc->root = RB_ROOT_CACHED; 2007 dcc->rbtree_check = false; 2008 2009 init_waitqueue_head(&dcc->discard_wait_queue); 2010 SM_I(sbi)->dcc_info = dcc; 2011 init_thread: 2012 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, 2013 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); 2014 if (IS_ERR(dcc->f2fs_issue_discard)) { 2015 err = PTR_ERR(dcc->f2fs_issue_discard); 2016 kvfree(dcc); 2017 SM_I(sbi)->dcc_info = NULL; 2018 return err; 2019 } 2020 2021 return err; 2022 } 2023 2024 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) 2025 { 2026 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2027 2028 if (!dcc) 2029 return; 2030 2031 f2fs_stop_discard_thread(sbi); 2032 2033 kvfree(dcc); 2034 SM_I(sbi)->dcc_info = NULL; 2035 } 2036 2037 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 2038 { 2039 struct sit_info *sit_i = SIT_I(sbi); 2040 2041 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { 2042 sit_i->dirty_sentries++; 2043 return false; 2044 } 2045 2046 return true; 2047 } 2048 2049 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 2050 unsigned int segno, int modified) 2051 { 2052 struct seg_entry *se = get_seg_entry(sbi, segno); 2053 se->type = type; 2054 if (modified) 2055 __mark_sit_entry_dirty(sbi, segno); 2056 } 2057 2058 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 2059 { 2060 struct seg_entry *se; 2061 unsigned int segno, offset; 2062 long int new_vblocks; 2063 bool exist; 2064 #ifdef CONFIG_F2FS_CHECK_FS 2065 bool mir_exist; 2066 #endif 2067 2068 segno = GET_SEGNO(sbi, blkaddr); 2069 2070 se = get_seg_entry(sbi, segno); 2071 new_vblocks = se->valid_blocks + del; 2072 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2073 2074 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) || 2075 (new_vblocks > sbi->blocks_per_seg))); 2076 2077 se->valid_blocks = new_vblocks; 2078 se->mtime = get_mtime(sbi, false); 2079 if (se->mtime > SIT_I(sbi)->max_mtime) 2080 SIT_I(sbi)->max_mtime = se->mtime; 2081 2082 /* Update valid block bitmap */ 2083 if (del > 0) { 2084 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); 2085 #ifdef CONFIG_F2FS_CHECK_FS 2086 mir_exist = f2fs_test_and_set_bit(offset, 2087 se->cur_valid_map_mir); 2088 if (unlikely(exist != mir_exist)) { 2089 f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " 2090 "when setting bitmap, blk:%u, old bit:%d", 2091 blkaddr, exist); 2092 f2fs_bug_on(sbi, 1); 2093 } 2094 #endif 2095 if (unlikely(exist)) { 2096 f2fs_msg(sbi->sb, KERN_ERR, 2097 "Bitmap was wrongly set, blk:%u", blkaddr); 2098 f2fs_bug_on(sbi, 1); 2099 se->valid_blocks--; 2100 del = 0; 2101 } 2102 2103 if (!f2fs_test_and_set_bit(offset, se->discard_map)) 2104 sbi->discard_blks--; 2105 2106 /* don't overwrite by SSR to keep node chain */ 2107 if (IS_NODESEG(se->type) && 2108 !is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { 2109 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map)) 2110 se->ckpt_valid_blocks++; 2111 } 2112 } else { 2113 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map); 2114 #ifdef CONFIG_F2FS_CHECK_FS 2115 mir_exist = f2fs_test_and_clear_bit(offset, 2116 se->cur_valid_map_mir); 2117 if (unlikely(exist != mir_exist)) { 2118 f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " 2119 "when clearing bitmap, blk:%u, old bit:%d", 2120 blkaddr, exist); 2121 f2fs_bug_on(sbi, 1); 2122 } 2123 #endif 2124 if (unlikely(!exist)) { 2125 f2fs_msg(sbi->sb, KERN_ERR, 2126 "Bitmap was wrongly cleared, blk:%u", blkaddr); 2127 f2fs_bug_on(sbi, 1); 2128 se->valid_blocks++; 2129 del = 0; 2130 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2131 /* 2132 * If checkpoints are off, we must not reuse data that 2133 * was used in the previous checkpoint. If it was used 2134 * before, we must track that to know how much space we 2135 * really have. 2136 */ 2137 if (f2fs_test_bit(offset, se->ckpt_valid_map)) 2138 sbi->unusable_block_count++; 2139 } 2140 2141 if (f2fs_test_and_clear_bit(offset, se->discard_map)) 2142 sbi->discard_blks++; 2143 } 2144 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 2145 se->ckpt_valid_blocks += del; 2146 2147 __mark_sit_entry_dirty(sbi, segno); 2148 2149 /* update total number of valid blocks to be written in ckpt area */ 2150 SIT_I(sbi)->written_valid_blocks += del; 2151 2152 if (__is_large_section(sbi)) 2153 get_sec_entry(sbi, segno)->valid_blocks += del; 2154 } 2155 2156 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) 2157 { 2158 unsigned int segno = GET_SEGNO(sbi, addr); 2159 struct sit_info *sit_i = SIT_I(sbi); 2160 2161 f2fs_bug_on(sbi, addr == NULL_ADDR); 2162 if (addr == NEW_ADDR) 2163 return; 2164 2165 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr); 2166 2167 /* add it into sit main buffer */ 2168 down_write(&sit_i->sentry_lock); 2169 2170 update_sit_entry(sbi, addr, -1); 2171 2172 /* add it into dirty seglist */ 2173 locate_dirty_segment(sbi, segno); 2174 2175 up_write(&sit_i->sentry_lock); 2176 } 2177 2178 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) 2179 { 2180 struct sit_info *sit_i = SIT_I(sbi); 2181 unsigned int segno, offset; 2182 struct seg_entry *se; 2183 bool is_cp = false; 2184 2185 if (!is_valid_data_blkaddr(sbi, blkaddr)) 2186 return true; 2187 2188 down_read(&sit_i->sentry_lock); 2189 2190 segno = GET_SEGNO(sbi, blkaddr); 2191 se = get_seg_entry(sbi, segno); 2192 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2193 2194 if (f2fs_test_bit(offset, se->ckpt_valid_map)) 2195 is_cp = true; 2196 2197 up_read(&sit_i->sentry_lock); 2198 2199 return is_cp; 2200 } 2201 2202 /* 2203 * This function should be resided under the curseg_mutex lock 2204 */ 2205 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, 2206 struct f2fs_summary *sum) 2207 { 2208 struct curseg_info *curseg = CURSEG_I(sbi, type); 2209 void *addr = curseg->sum_blk; 2210 addr += curseg->next_blkoff * sizeof(struct f2fs_summary); 2211 memcpy(addr, sum, sizeof(struct f2fs_summary)); 2212 } 2213 2214 /* 2215 * Calculate the number of current summary pages for writing 2216 */ 2217 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) 2218 { 2219 int valid_sum_count = 0; 2220 int i, sum_in_page; 2221 2222 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2223 if (sbi->ckpt->alloc_type[i] == SSR) 2224 valid_sum_count += sbi->blocks_per_seg; 2225 else { 2226 if (for_ra) 2227 valid_sum_count += le16_to_cpu( 2228 F2FS_CKPT(sbi)->cur_data_blkoff[i]); 2229 else 2230 valid_sum_count += curseg_blkoff(sbi, i); 2231 } 2232 } 2233 2234 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - 2235 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 2236 if (valid_sum_count <= sum_in_page) 2237 return 1; 2238 else if ((valid_sum_count - sum_in_page) <= 2239 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 2240 return 2; 2241 return 3; 2242 } 2243 2244 /* 2245 * Caller should put this summary page 2246 */ 2247 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) 2248 { 2249 return f2fs_get_meta_page_nofail(sbi, GET_SUM_BLOCK(sbi, segno)); 2250 } 2251 2252 void f2fs_update_meta_page(struct f2fs_sb_info *sbi, 2253 void *src, block_t blk_addr) 2254 { 2255 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); 2256 2257 memcpy(page_address(page), src, PAGE_SIZE); 2258 set_page_dirty(page); 2259 f2fs_put_page(page, 1); 2260 } 2261 2262 static void write_sum_page(struct f2fs_sb_info *sbi, 2263 struct f2fs_summary_block *sum_blk, block_t blk_addr) 2264 { 2265 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr); 2266 } 2267 2268 static void write_current_sum_page(struct f2fs_sb_info *sbi, 2269 int type, block_t blk_addr) 2270 { 2271 struct curseg_info *curseg = CURSEG_I(sbi, type); 2272 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); 2273 struct f2fs_summary_block *src = curseg->sum_blk; 2274 struct f2fs_summary_block *dst; 2275 2276 dst = (struct f2fs_summary_block *)page_address(page); 2277 memset(dst, 0, PAGE_SIZE); 2278 2279 mutex_lock(&curseg->curseg_mutex); 2280 2281 down_read(&curseg->journal_rwsem); 2282 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); 2283 up_read(&curseg->journal_rwsem); 2284 2285 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); 2286 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); 2287 2288 mutex_unlock(&curseg->curseg_mutex); 2289 2290 set_page_dirty(page); 2291 f2fs_put_page(page, 1); 2292 } 2293 2294 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) 2295 { 2296 struct curseg_info *curseg = CURSEG_I(sbi, type); 2297 unsigned int segno = curseg->segno + 1; 2298 struct free_segmap_info *free_i = FREE_I(sbi); 2299 2300 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) 2301 return !test_bit(segno, free_i->free_segmap); 2302 return 0; 2303 } 2304 2305 /* 2306 * Find a new segment from the free segments bitmap to right order 2307 * This function should be returned with success, otherwise BUG 2308 */ 2309 static void get_new_segment(struct f2fs_sb_info *sbi, 2310 unsigned int *newseg, bool new_sec, int dir) 2311 { 2312 struct free_segmap_info *free_i = FREE_I(sbi); 2313 unsigned int segno, secno, zoneno; 2314 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; 2315 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); 2316 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); 2317 unsigned int left_start = hint; 2318 bool init = true; 2319 int go_left = 0; 2320 int i; 2321 2322 spin_lock(&free_i->segmap_lock); 2323 2324 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { 2325 segno = find_next_zero_bit(free_i->free_segmap, 2326 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); 2327 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) 2328 goto got_it; 2329 } 2330 find_other_zone: 2331 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); 2332 if (secno >= MAIN_SECS(sbi)) { 2333 if (dir == ALLOC_RIGHT) { 2334 secno = find_next_zero_bit(free_i->free_secmap, 2335 MAIN_SECS(sbi), 0); 2336 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); 2337 } else { 2338 go_left = 1; 2339 left_start = hint - 1; 2340 } 2341 } 2342 if (go_left == 0) 2343 goto skip_left; 2344 2345 while (test_bit(left_start, free_i->free_secmap)) { 2346 if (left_start > 0) { 2347 left_start--; 2348 continue; 2349 } 2350 left_start = find_next_zero_bit(free_i->free_secmap, 2351 MAIN_SECS(sbi), 0); 2352 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); 2353 break; 2354 } 2355 secno = left_start; 2356 skip_left: 2357 segno = GET_SEG_FROM_SEC(sbi, secno); 2358 zoneno = GET_ZONE_FROM_SEC(sbi, secno); 2359 2360 /* give up on finding another zone */ 2361 if (!init) 2362 goto got_it; 2363 if (sbi->secs_per_zone == 1) 2364 goto got_it; 2365 if (zoneno == old_zoneno) 2366 goto got_it; 2367 if (dir == ALLOC_LEFT) { 2368 if (!go_left && zoneno + 1 >= total_zones) 2369 goto got_it; 2370 if (go_left && zoneno == 0) 2371 goto got_it; 2372 } 2373 for (i = 0; i < NR_CURSEG_TYPE; i++) 2374 if (CURSEG_I(sbi, i)->zone == zoneno) 2375 break; 2376 2377 if (i < NR_CURSEG_TYPE) { 2378 /* zone is in user, try another */ 2379 if (go_left) 2380 hint = zoneno * sbi->secs_per_zone - 1; 2381 else if (zoneno + 1 >= total_zones) 2382 hint = 0; 2383 else 2384 hint = (zoneno + 1) * sbi->secs_per_zone; 2385 init = false; 2386 goto find_other_zone; 2387 } 2388 got_it: 2389 /* set it as dirty segment in free segmap */ 2390 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); 2391 __set_inuse(sbi, segno); 2392 *newseg = segno; 2393 spin_unlock(&free_i->segmap_lock); 2394 } 2395 2396 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 2397 { 2398 struct curseg_info *curseg = CURSEG_I(sbi, type); 2399 struct summary_footer *sum_footer; 2400 2401 curseg->segno = curseg->next_segno; 2402 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); 2403 curseg->next_blkoff = 0; 2404 curseg->next_segno = NULL_SEGNO; 2405 2406 sum_footer = &(curseg->sum_blk->footer); 2407 memset(sum_footer, 0, sizeof(struct summary_footer)); 2408 if (IS_DATASEG(type)) 2409 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 2410 if (IS_NODESEG(type)) 2411 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 2412 __set_sit_entry_type(sbi, type, curseg->segno, modified); 2413 } 2414 2415 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) 2416 { 2417 /* if segs_per_sec is large than 1, we need to keep original policy. */ 2418 if (__is_large_section(sbi)) 2419 return CURSEG_I(sbi, type)->segno; 2420 2421 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 2422 return 0; 2423 2424 if (test_opt(sbi, NOHEAP) && 2425 (type == CURSEG_HOT_DATA || IS_NODESEG(type))) 2426 return 0; 2427 2428 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) 2429 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; 2430 2431 /* find segments from 0 to reuse freed segments */ 2432 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) 2433 return 0; 2434 2435 return CURSEG_I(sbi, type)->segno; 2436 } 2437 2438 /* 2439 * Allocate a current working segment. 2440 * This function always allocates a free segment in LFS manner. 2441 */ 2442 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 2443 { 2444 struct curseg_info *curseg = CURSEG_I(sbi, type); 2445 unsigned int segno = curseg->segno; 2446 int dir = ALLOC_LEFT; 2447 2448 write_sum_page(sbi, curseg->sum_blk, 2449 GET_SUM_BLOCK(sbi, segno)); 2450 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA) 2451 dir = ALLOC_RIGHT; 2452 2453 if (test_opt(sbi, NOHEAP)) 2454 dir = ALLOC_RIGHT; 2455 2456 segno = __get_next_segno(sbi, type); 2457 get_new_segment(sbi, &segno, new_sec, dir); 2458 curseg->next_segno = segno; 2459 reset_curseg(sbi, type, 1); 2460 curseg->alloc_type = LFS; 2461 } 2462 2463 static void __next_free_blkoff(struct f2fs_sb_info *sbi, 2464 struct curseg_info *seg, block_t start) 2465 { 2466 struct seg_entry *se = get_seg_entry(sbi, seg->segno); 2467 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 2468 unsigned long *target_map = SIT_I(sbi)->tmp_map; 2469 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 2470 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 2471 int i, pos; 2472 2473 for (i = 0; i < entries; i++) 2474 target_map[i] = ckpt_map[i] | cur_map[i]; 2475 2476 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); 2477 2478 seg->next_blkoff = pos; 2479 } 2480 2481 /* 2482 * If a segment is written by LFS manner, next block offset is just obtained 2483 * by increasing the current block offset. However, if a segment is written by 2484 * SSR manner, next block offset obtained by calling __next_free_blkoff 2485 */ 2486 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, 2487 struct curseg_info *seg) 2488 { 2489 if (seg->alloc_type == SSR) 2490 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); 2491 else 2492 seg->next_blkoff++; 2493 } 2494 2495 /* 2496 * This function always allocates a used segment(from dirty seglist) by SSR 2497 * manner, so it should recover the existing segment information of valid blocks 2498 */ 2499 static void change_curseg(struct f2fs_sb_info *sbi, int type) 2500 { 2501 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2502 struct curseg_info *curseg = CURSEG_I(sbi, type); 2503 unsigned int new_segno = curseg->next_segno; 2504 struct f2fs_summary_block *sum_node; 2505 struct page *sum_page; 2506 2507 write_sum_page(sbi, curseg->sum_blk, 2508 GET_SUM_BLOCK(sbi, curseg->segno)); 2509 __set_test_and_inuse(sbi, new_segno); 2510 2511 mutex_lock(&dirty_i->seglist_lock); 2512 __remove_dirty_segment(sbi, new_segno, PRE); 2513 __remove_dirty_segment(sbi, new_segno, DIRTY); 2514 mutex_unlock(&dirty_i->seglist_lock); 2515 2516 reset_curseg(sbi, type, 1); 2517 curseg->alloc_type = SSR; 2518 __next_free_blkoff(sbi, curseg, 0); 2519 2520 sum_page = f2fs_get_sum_page(sbi, new_segno); 2521 f2fs_bug_on(sbi, IS_ERR(sum_page)); 2522 sum_node = (struct f2fs_summary_block *)page_address(sum_page); 2523 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); 2524 f2fs_put_page(sum_page, 1); 2525 } 2526 2527 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) 2528 { 2529 struct curseg_info *curseg = CURSEG_I(sbi, type); 2530 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; 2531 unsigned segno = NULL_SEGNO; 2532 int i, cnt; 2533 bool reversed = false; 2534 2535 /* f2fs_need_SSR() already forces to do this */ 2536 if (v_ops->get_victim(sbi, &segno, BG_GC, type, SSR)) { 2537 curseg->next_segno = segno; 2538 return 1; 2539 } 2540 2541 /* For node segments, let's do SSR more intensively */ 2542 if (IS_NODESEG(type)) { 2543 if (type >= CURSEG_WARM_NODE) { 2544 reversed = true; 2545 i = CURSEG_COLD_NODE; 2546 } else { 2547 i = CURSEG_HOT_NODE; 2548 } 2549 cnt = NR_CURSEG_NODE_TYPE; 2550 } else { 2551 if (type >= CURSEG_WARM_DATA) { 2552 reversed = true; 2553 i = CURSEG_COLD_DATA; 2554 } else { 2555 i = CURSEG_HOT_DATA; 2556 } 2557 cnt = NR_CURSEG_DATA_TYPE; 2558 } 2559 2560 for (; cnt-- > 0; reversed ? i-- : i++) { 2561 if (i == type) 2562 continue; 2563 if (v_ops->get_victim(sbi, &segno, BG_GC, i, SSR)) { 2564 curseg->next_segno = segno; 2565 return 1; 2566 } 2567 } 2568 2569 /* find valid_blocks=0 in dirty list */ 2570 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2571 segno = get_free_segment(sbi); 2572 if (segno != NULL_SEGNO) { 2573 curseg->next_segno = segno; 2574 return 1; 2575 } 2576 } 2577 return 0; 2578 } 2579 2580 /* 2581 * flush out current segment and replace it with new segment 2582 * This function should be returned with success, otherwise BUG 2583 */ 2584 static void allocate_segment_by_default(struct f2fs_sb_info *sbi, 2585 int type, bool force) 2586 { 2587 struct curseg_info *curseg = CURSEG_I(sbi, type); 2588 2589 if (force) 2590 new_curseg(sbi, type, true); 2591 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && 2592 type == CURSEG_WARM_NODE) 2593 new_curseg(sbi, type, false); 2594 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type) && 2595 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 2596 new_curseg(sbi, type, false); 2597 else if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type)) 2598 change_curseg(sbi, type); 2599 else 2600 new_curseg(sbi, type, false); 2601 2602 stat_inc_seg_type(sbi, curseg); 2603 } 2604 2605 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) 2606 { 2607 struct curseg_info *curseg; 2608 unsigned int old_segno; 2609 int i; 2610 2611 down_write(&SIT_I(sbi)->sentry_lock); 2612 2613 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2614 curseg = CURSEG_I(sbi, i); 2615 old_segno = curseg->segno; 2616 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); 2617 locate_dirty_segment(sbi, old_segno); 2618 } 2619 2620 up_write(&SIT_I(sbi)->sentry_lock); 2621 } 2622 2623 static const struct segment_allocation default_salloc_ops = { 2624 .allocate_segment = allocate_segment_by_default, 2625 }; 2626 2627 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, 2628 struct cp_control *cpc) 2629 { 2630 __u64 trim_start = cpc->trim_start; 2631 bool has_candidate = false; 2632 2633 down_write(&SIT_I(sbi)->sentry_lock); 2634 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { 2635 if (add_discard_addrs(sbi, cpc, true)) { 2636 has_candidate = true; 2637 break; 2638 } 2639 } 2640 up_write(&SIT_I(sbi)->sentry_lock); 2641 2642 cpc->trim_start = trim_start; 2643 return has_candidate; 2644 } 2645 2646 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, 2647 struct discard_policy *dpolicy, 2648 unsigned int start, unsigned int end) 2649 { 2650 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2651 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 2652 struct rb_node **insert_p = NULL, *insert_parent = NULL; 2653 struct discard_cmd *dc; 2654 struct blk_plug plug; 2655 int issued; 2656 unsigned int trimmed = 0; 2657 2658 next: 2659 issued = 0; 2660 2661 mutex_lock(&dcc->cmd_lock); 2662 if (unlikely(dcc->rbtree_check)) 2663 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, 2664 &dcc->root)); 2665 2666 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root, 2667 NULL, start, 2668 (struct rb_entry **)&prev_dc, 2669 (struct rb_entry **)&next_dc, 2670 &insert_p, &insert_parent, true, NULL); 2671 if (!dc) 2672 dc = next_dc; 2673 2674 blk_start_plug(&plug); 2675 2676 while (dc && dc->lstart <= end) { 2677 struct rb_node *node; 2678 int err = 0; 2679 2680 if (dc->len < dpolicy->granularity) 2681 goto skip; 2682 2683 if (dc->state != D_PREP) { 2684 list_move_tail(&dc->list, &dcc->fstrim_list); 2685 goto skip; 2686 } 2687 2688 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 2689 2690 if (issued >= dpolicy->max_requests) { 2691 start = dc->lstart + dc->len; 2692 2693 if (err) 2694 __remove_discard_cmd(sbi, dc); 2695 2696 blk_finish_plug(&plug); 2697 mutex_unlock(&dcc->cmd_lock); 2698 trimmed += __wait_all_discard_cmd(sbi, NULL); 2699 congestion_wait(BLK_RW_ASYNC, HZ/50); 2700 goto next; 2701 } 2702 skip: 2703 node = rb_next(&dc->rb_node); 2704 if (err) 2705 __remove_discard_cmd(sbi, dc); 2706 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 2707 2708 if (fatal_signal_pending(current)) 2709 break; 2710 } 2711 2712 blk_finish_plug(&plug); 2713 mutex_unlock(&dcc->cmd_lock); 2714 2715 return trimmed; 2716 } 2717 2718 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) 2719 { 2720 __u64 start = F2FS_BYTES_TO_BLK(range->start); 2721 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; 2722 unsigned int start_segno, end_segno; 2723 block_t start_block, end_block; 2724 struct cp_control cpc; 2725 struct discard_policy dpolicy; 2726 unsigned long long trimmed = 0; 2727 int err = 0; 2728 bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi); 2729 2730 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) 2731 return -EINVAL; 2732 2733 if (end < MAIN_BLKADDR(sbi)) 2734 goto out; 2735 2736 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 2737 f2fs_msg(sbi->sb, KERN_WARNING, 2738 "Found FS corruption, run fsck to fix."); 2739 return -EIO; 2740 } 2741 2742 /* start/end segment number in main_area */ 2743 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); 2744 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : 2745 GET_SEGNO(sbi, end); 2746 if (need_align) { 2747 start_segno = rounddown(start_segno, sbi->segs_per_sec); 2748 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1; 2749 } 2750 2751 cpc.reason = CP_DISCARD; 2752 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); 2753 cpc.trim_start = start_segno; 2754 cpc.trim_end = end_segno; 2755 2756 if (sbi->discard_blks == 0) 2757 goto out; 2758 2759 mutex_lock(&sbi->gc_mutex); 2760 err = f2fs_write_checkpoint(sbi, &cpc); 2761 mutex_unlock(&sbi->gc_mutex); 2762 if (err) 2763 goto out; 2764 2765 /* 2766 * We filed discard candidates, but actually we don't need to wait for 2767 * all of them, since they'll be issued in idle time along with runtime 2768 * discard option. User configuration looks like using runtime discard 2769 * or periodic fstrim instead of it. 2770 */ 2771 if (f2fs_realtime_discard_enable(sbi)) 2772 goto out; 2773 2774 start_block = START_BLOCK(sbi, start_segno); 2775 end_block = START_BLOCK(sbi, end_segno + 1); 2776 2777 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); 2778 trimmed = __issue_discard_cmd_range(sbi, &dpolicy, 2779 start_block, end_block); 2780 2781 trimmed += __wait_discard_cmd_range(sbi, &dpolicy, 2782 start_block, end_block); 2783 out: 2784 if (!err) 2785 range->len = F2FS_BLK_TO_BYTES(trimmed); 2786 return err; 2787 } 2788 2789 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) 2790 { 2791 struct curseg_info *curseg = CURSEG_I(sbi, type); 2792 if (curseg->next_blkoff < sbi->blocks_per_seg) 2793 return true; 2794 return false; 2795 } 2796 2797 int f2fs_rw_hint_to_seg_type(enum rw_hint hint) 2798 { 2799 switch (hint) { 2800 case WRITE_LIFE_SHORT: 2801 return CURSEG_HOT_DATA; 2802 case WRITE_LIFE_EXTREME: 2803 return CURSEG_COLD_DATA; 2804 default: 2805 return CURSEG_WARM_DATA; 2806 } 2807 } 2808 2809 /* This returns write hints for each segment type. This hints will be 2810 * passed down to block layer. There are mapping tables which depend on 2811 * the mount option 'whint_mode'. 2812 * 2813 * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET. 2814 * 2815 * 2) whint_mode=user-based. F2FS tries to pass down hints given by users. 2816 * 2817 * User F2FS Block 2818 * ---- ---- ----- 2819 * META WRITE_LIFE_NOT_SET 2820 * HOT_NODE " 2821 * WARM_NODE " 2822 * COLD_NODE " 2823 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 2824 * extension list " " 2825 * 2826 * -- buffered io 2827 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2828 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2829 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 2830 * WRITE_LIFE_NONE " " 2831 * WRITE_LIFE_MEDIUM " " 2832 * WRITE_LIFE_LONG " " 2833 * 2834 * -- direct io 2835 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2836 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2837 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 2838 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 2839 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 2840 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 2841 * 2842 * 3) whint_mode=fs-based. F2FS passes down hints with its policy. 2843 * 2844 * User F2FS Block 2845 * ---- ---- ----- 2846 * META WRITE_LIFE_MEDIUM; 2847 * HOT_NODE WRITE_LIFE_NOT_SET 2848 * WARM_NODE " 2849 * COLD_NODE WRITE_LIFE_NONE 2850 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 2851 * extension list " " 2852 * 2853 * -- buffered io 2854 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2855 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2856 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_LONG 2857 * WRITE_LIFE_NONE " " 2858 * WRITE_LIFE_MEDIUM " " 2859 * WRITE_LIFE_LONG " " 2860 * 2861 * -- direct io 2862 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 2863 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 2864 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 2865 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 2866 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 2867 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 2868 */ 2869 2870 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, 2871 enum page_type type, enum temp_type temp) 2872 { 2873 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) { 2874 if (type == DATA) { 2875 if (temp == WARM) 2876 return WRITE_LIFE_NOT_SET; 2877 else if (temp == HOT) 2878 return WRITE_LIFE_SHORT; 2879 else if (temp == COLD) 2880 return WRITE_LIFE_EXTREME; 2881 } else { 2882 return WRITE_LIFE_NOT_SET; 2883 } 2884 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) { 2885 if (type == DATA) { 2886 if (temp == WARM) 2887 return WRITE_LIFE_LONG; 2888 else if (temp == HOT) 2889 return WRITE_LIFE_SHORT; 2890 else if (temp == COLD) 2891 return WRITE_LIFE_EXTREME; 2892 } else if (type == NODE) { 2893 if (temp == WARM || temp == HOT) 2894 return WRITE_LIFE_NOT_SET; 2895 else if (temp == COLD) 2896 return WRITE_LIFE_NONE; 2897 } else if (type == META) { 2898 return WRITE_LIFE_MEDIUM; 2899 } 2900 } 2901 return WRITE_LIFE_NOT_SET; 2902 } 2903 2904 static int __get_segment_type_2(struct f2fs_io_info *fio) 2905 { 2906 if (fio->type == DATA) 2907 return CURSEG_HOT_DATA; 2908 else 2909 return CURSEG_HOT_NODE; 2910 } 2911 2912 static int __get_segment_type_4(struct f2fs_io_info *fio) 2913 { 2914 if (fio->type == DATA) { 2915 struct inode *inode = fio->page->mapping->host; 2916 2917 if (S_ISDIR(inode->i_mode)) 2918 return CURSEG_HOT_DATA; 2919 else 2920 return CURSEG_COLD_DATA; 2921 } else { 2922 if (IS_DNODE(fio->page) && is_cold_node(fio->page)) 2923 return CURSEG_WARM_NODE; 2924 else 2925 return CURSEG_COLD_NODE; 2926 } 2927 } 2928 2929 static int __get_segment_type_6(struct f2fs_io_info *fio) 2930 { 2931 if (fio->type == DATA) { 2932 struct inode *inode = fio->page->mapping->host; 2933 2934 if (is_cold_data(fio->page) || file_is_cold(inode)) 2935 return CURSEG_COLD_DATA; 2936 if (file_is_hot(inode) || 2937 is_inode_flag_set(inode, FI_HOT_DATA) || 2938 f2fs_is_atomic_file(inode) || 2939 f2fs_is_volatile_file(inode)) 2940 return CURSEG_HOT_DATA; 2941 return f2fs_rw_hint_to_seg_type(inode->i_write_hint); 2942 } else { 2943 if (IS_DNODE(fio->page)) 2944 return is_cold_node(fio->page) ? CURSEG_WARM_NODE : 2945 CURSEG_HOT_NODE; 2946 return CURSEG_COLD_NODE; 2947 } 2948 } 2949 2950 static int __get_segment_type(struct f2fs_io_info *fio) 2951 { 2952 int type = 0; 2953 2954 switch (F2FS_OPTION(fio->sbi).active_logs) { 2955 case 2: 2956 type = __get_segment_type_2(fio); 2957 break; 2958 case 4: 2959 type = __get_segment_type_4(fio); 2960 break; 2961 case 6: 2962 type = __get_segment_type_6(fio); 2963 break; 2964 default: 2965 f2fs_bug_on(fio->sbi, true); 2966 } 2967 2968 if (IS_HOT(type)) 2969 fio->temp = HOT; 2970 else if (IS_WARM(type)) 2971 fio->temp = WARM; 2972 else 2973 fio->temp = COLD; 2974 return type; 2975 } 2976 2977 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, 2978 block_t old_blkaddr, block_t *new_blkaddr, 2979 struct f2fs_summary *sum, int type, 2980 struct f2fs_io_info *fio, bool add_list) 2981 { 2982 struct sit_info *sit_i = SIT_I(sbi); 2983 struct curseg_info *curseg = CURSEG_I(sbi, type); 2984 2985 down_read(&SM_I(sbi)->curseg_lock); 2986 2987 mutex_lock(&curseg->curseg_mutex); 2988 down_write(&sit_i->sentry_lock); 2989 2990 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 2991 2992 f2fs_wait_discard_bio(sbi, *new_blkaddr); 2993 2994 /* 2995 * __add_sum_entry should be resided under the curseg_mutex 2996 * because, this function updates a summary entry in the 2997 * current summary block. 2998 */ 2999 __add_sum_entry(sbi, type, sum); 3000 3001 __refresh_next_blkoff(sbi, curseg); 3002 3003 stat_inc_block_count(sbi, curseg); 3004 3005 /* 3006 * SIT information should be updated before segment allocation, 3007 * since SSR needs latest valid block information. 3008 */ 3009 update_sit_entry(sbi, *new_blkaddr, 1); 3010 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 3011 update_sit_entry(sbi, old_blkaddr, -1); 3012 3013 if (!__has_curseg_space(sbi, type)) 3014 sit_i->s_ops->allocate_segment(sbi, type, false); 3015 3016 /* 3017 * segment dirty status should be updated after segment allocation, 3018 * so we just need to update status only one time after previous 3019 * segment being closed. 3020 */ 3021 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 3022 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); 3023 3024 up_write(&sit_i->sentry_lock); 3025 3026 if (page && IS_NODESEG(type)) { 3027 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); 3028 3029 f2fs_inode_chksum_set(sbi, page); 3030 } 3031 3032 if (add_list) { 3033 struct f2fs_bio_info *io; 3034 3035 INIT_LIST_HEAD(&fio->list); 3036 fio->in_list = true; 3037 fio->retry = false; 3038 io = sbi->write_io[fio->type] + fio->temp; 3039 spin_lock(&io->io_lock); 3040 list_add_tail(&fio->list, &io->io_list); 3041 spin_unlock(&io->io_lock); 3042 } 3043 3044 mutex_unlock(&curseg->curseg_mutex); 3045 3046 up_read(&SM_I(sbi)->curseg_lock); 3047 } 3048 3049 static void update_device_state(struct f2fs_io_info *fio) 3050 { 3051 struct f2fs_sb_info *sbi = fio->sbi; 3052 unsigned int devidx; 3053 3054 if (!sbi->s_ndevs) 3055 return; 3056 3057 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr); 3058 3059 /* update device state for fsync */ 3060 f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO); 3061 3062 /* update device state for checkpoint */ 3063 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { 3064 spin_lock(&sbi->dev_lock); 3065 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); 3066 spin_unlock(&sbi->dev_lock); 3067 } 3068 } 3069 3070 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) 3071 { 3072 int type = __get_segment_type(fio); 3073 bool keep_order = (test_opt(fio->sbi, LFS) && type == CURSEG_COLD_DATA); 3074 3075 if (keep_order) 3076 down_read(&fio->sbi->io_order_lock); 3077 reallocate: 3078 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, 3079 &fio->new_blkaddr, sum, type, fio, true); 3080 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) 3081 invalidate_mapping_pages(META_MAPPING(fio->sbi), 3082 fio->old_blkaddr, fio->old_blkaddr); 3083 3084 /* writeout dirty page into bdev */ 3085 f2fs_submit_page_write(fio); 3086 if (fio->retry) { 3087 fio->old_blkaddr = fio->new_blkaddr; 3088 goto reallocate; 3089 } 3090 3091 update_device_state(fio); 3092 3093 if (keep_order) 3094 up_read(&fio->sbi->io_order_lock); 3095 } 3096 3097 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page, 3098 enum iostat_type io_type) 3099 { 3100 struct f2fs_io_info fio = { 3101 .sbi = sbi, 3102 .type = META, 3103 .temp = HOT, 3104 .op = REQ_OP_WRITE, 3105 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, 3106 .old_blkaddr = page->index, 3107 .new_blkaddr = page->index, 3108 .page = page, 3109 .encrypted_page = NULL, 3110 .in_list = false, 3111 }; 3112 3113 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) 3114 fio.op_flags &= ~REQ_META; 3115 3116 set_page_writeback(page); 3117 ClearPageError(page); 3118 f2fs_submit_page_write(&fio); 3119 3120 stat_inc_meta_count(sbi, page->index); 3121 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE); 3122 } 3123 3124 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) 3125 { 3126 struct f2fs_summary sum; 3127 3128 set_summary(&sum, nid, 0, 0); 3129 do_write_page(&sum, fio); 3130 3131 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); 3132 } 3133 3134 void f2fs_outplace_write_data(struct dnode_of_data *dn, 3135 struct f2fs_io_info *fio) 3136 { 3137 struct f2fs_sb_info *sbi = fio->sbi; 3138 struct f2fs_summary sum; 3139 3140 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); 3141 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); 3142 do_write_page(&sum, fio); 3143 f2fs_update_data_blkaddr(dn, fio->new_blkaddr); 3144 3145 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE); 3146 } 3147 3148 int f2fs_inplace_write_data(struct f2fs_io_info *fio) 3149 { 3150 int err; 3151 struct f2fs_sb_info *sbi = fio->sbi; 3152 3153 fio->new_blkaddr = fio->old_blkaddr; 3154 /* i/o temperature is needed for passing down write hints */ 3155 __get_segment_type(fio); 3156 3157 f2fs_bug_on(sbi, !IS_DATASEG(get_seg_entry(sbi, 3158 GET_SEGNO(sbi, fio->new_blkaddr))->type)); 3159 3160 stat_inc_inplace_blocks(fio->sbi); 3161 3162 err = f2fs_submit_page_bio(fio); 3163 if (!err) 3164 update_device_state(fio); 3165 3166 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); 3167 3168 return err; 3169 } 3170 3171 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, 3172 unsigned int segno) 3173 { 3174 int i; 3175 3176 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { 3177 if (CURSEG_I(sbi, i)->segno == segno) 3178 break; 3179 } 3180 return i; 3181 } 3182 3183 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, 3184 block_t old_blkaddr, block_t new_blkaddr, 3185 bool recover_curseg, bool recover_newaddr) 3186 { 3187 struct sit_info *sit_i = SIT_I(sbi); 3188 struct curseg_info *curseg; 3189 unsigned int segno, old_cursegno; 3190 struct seg_entry *se; 3191 int type; 3192 unsigned short old_blkoff; 3193 3194 segno = GET_SEGNO(sbi, new_blkaddr); 3195 se = get_seg_entry(sbi, segno); 3196 type = se->type; 3197 3198 down_write(&SM_I(sbi)->curseg_lock); 3199 3200 if (!recover_curseg) { 3201 /* for recovery flow */ 3202 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { 3203 if (old_blkaddr == NULL_ADDR) 3204 type = CURSEG_COLD_DATA; 3205 else 3206 type = CURSEG_WARM_DATA; 3207 } 3208 } else { 3209 if (IS_CURSEG(sbi, segno)) { 3210 /* se->type is volatile as SSR allocation */ 3211 type = __f2fs_get_curseg(sbi, segno); 3212 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); 3213 } else { 3214 type = CURSEG_WARM_DATA; 3215 } 3216 } 3217 3218 f2fs_bug_on(sbi, !IS_DATASEG(type)); 3219 curseg = CURSEG_I(sbi, type); 3220 3221 mutex_lock(&curseg->curseg_mutex); 3222 down_write(&sit_i->sentry_lock); 3223 3224 old_cursegno = curseg->segno; 3225 old_blkoff = curseg->next_blkoff; 3226 3227 /* change the current segment */ 3228 if (segno != curseg->segno) { 3229 curseg->next_segno = segno; 3230 change_curseg(sbi, type); 3231 } 3232 3233 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 3234 __add_sum_entry(sbi, type, sum); 3235 3236 if (!recover_curseg || recover_newaddr) 3237 update_sit_entry(sbi, new_blkaddr, 1); 3238 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { 3239 invalidate_mapping_pages(META_MAPPING(sbi), 3240 old_blkaddr, old_blkaddr); 3241 update_sit_entry(sbi, old_blkaddr, -1); 3242 } 3243 3244 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 3245 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); 3246 3247 locate_dirty_segment(sbi, old_cursegno); 3248 3249 if (recover_curseg) { 3250 if (old_cursegno != curseg->segno) { 3251 curseg->next_segno = old_cursegno; 3252 change_curseg(sbi, type); 3253 } 3254 curseg->next_blkoff = old_blkoff; 3255 } 3256 3257 up_write(&sit_i->sentry_lock); 3258 mutex_unlock(&curseg->curseg_mutex); 3259 up_write(&SM_I(sbi)->curseg_lock); 3260 } 3261 3262 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, 3263 block_t old_addr, block_t new_addr, 3264 unsigned char version, bool recover_curseg, 3265 bool recover_newaddr) 3266 { 3267 struct f2fs_summary sum; 3268 3269 set_summary(&sum, dn->nid, dn->ofs_in_node, version); 3270 3271 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, 3272 recover_curseg, recover_newaddr); 3273 3274 f2fs_update_data_blkaddr(dn, new_addr); 3275 } 3276 3277 void f2fs_wait_on_page_writeback(struct page *page, 3278 enum page_type type, bool ordered) 3279 { 3280 if (PageWriteback(page)) { 3281 struct f2fs_sb_info *sbi = F2FS_P_SB(page); 3282 3283 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type); 3284 if (ordered) 3285 wait_on_page_writeback(page); 3286 else 3287 wait_for_stable_page(page); 3288 } 3289 } 3290 3291 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) 3292 { 3293 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3294 struct page *cpage; 3295 3296 if (!f2fs_post_read_required(inode)) 3297 return; 3298 3299 if (!is_valid_data_blkaddr(sbi, blkaddr)) 3300 return; 3301 3302 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); 3303 if (cpage) { 3304 f2fs_wait_on_page_writeback(cpage, DATA, true); 3305 f2fs_put_page(cpage, 1); 3306 } 3307 } 3308 3309 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, 3310 block_t len) 3311 { 3312 block_t i; 3313 3314 for (i = 0; i < len; i++) 3315 f2fs_wait_on_block_writeback(inode, blkaddr + i); 3316 } 3317 3318 static int read_compacted_summaries(struct f2fs_sb_info *sbi) 3319 { 3320 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3321 struct curseg_info *seg_i; 3322 unsigned char *kaddr; 3323 struct page *page; 3324 block_t start; 3325 int i, j, offset; 3326 3327 start = start_sum_block(sbi); 3328 3329 page = f2fs_get_meta_page(sbi, start++); 3330 if (IS_ERR(page)) 3331 return PTR_ERR(page); 3332 kaddr = (unsigned char *)page_address(page); 3333 3334 /* Step 1: restore nat cache */ 3335 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3336 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); 3337 3338 /* Step 2: restore sit cache */ 3339 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3340 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); 3341 offset = 2 * SUM_JOURNAL_SIZE; 3342 3343 /* Step 3: restore summary entries */ 3344 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3345 unsigned short blk_off; 3346 unsigned int segno; 3347 3348 seg_i = CURSEG_I(sbi, i); 3349 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 3350 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 3351 seg_i->next_segno = segno; 3352 reset_curseg(sbi, i, 0); 3353 seg_i->alloc_type = ckpt->alloc_type[i]; 3354 seg_i->next_blkoff = blk_off; 3355 3356 if (seg_i->alloc_type == SSR) 3357 blk_off = sbi->blocks_per_seg; 3358 3359 for (j = 0; j < blk_off; j++) { 3360 struct f2fs_summary *s; 3361 s = (struct f2fs_summary *)(kaddr + offset); 3362 seg_i->sum_blk->entries[j] = *s; 3363 offset += SUMMARY_SIZE; 3364 if (offset + SUMMARY_SIZE <= PAGE_SIZE - 3365 SUM_FOOTER_SIZE) 3366 continue; 3367 3368 f2fs_put_page(page, 1); 3369 page = NULL; 3370 3371 page = f2fs_get_meta_page(sbi, start++); 3372 if (IS_ERR(page)) 3373 return PTR_ERR(page); 3374 kaddr = (unsigned char *)page_address(page); 3375 offset = 0; 3376 } 3377 } 3378 f2fs_put_page(page, 1); 3379 return 0; 3380 } 3381 3382 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 3383 { 3384 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 3385 struct f2fs_summary_block *sum; 3386 struct curseg_info *curseg; 3387 struct page *new; 3388 unsigned short blk_off; 3389 unsigned int segno = 0; 3390 block_t blk_addr = 0; 3391 int err = 0; 3392 3393 /* get segment number and block addr */ 3394 if (IS_DATASEG(type)) { 3395 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 3396 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 3397 CURSEG_HOT_DATA]); 3398 if (__exist_node_summaries(sbi)) 3399 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); 3400 else 3401 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 3402 } else { 3403 segno = le32_to_cpu(ckpt->cur_node_segno[type - 3404 CURSEG_HOT_NODE]); 3405 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 3406 CURSEG_HOT_NODE]); 3407 if (__exist_node_summaries(sbi)) 3408 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 3409 type - CURSEG_HOT_NODE); 3410 else 3411 blk_addr = GET_SUM_BLOCK(sbi, segno); 3412 } 3413 3414 new = f2fs_get_meta_page(sbi, blk_addr); 3415 if (IS_ERR(new)) 3416 return PTR_ERR(new); 3417 sum = (struct f2fs_summary_block *)page_address(new); 3418 3419 if (IS_NODESEG(type)) { 3420 if (__exist_node_summaries(sbi)) { 3421 struct f2fs_summary *ns = &sum->entries[0]; 3422 int i; 3423 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { 3424 ns->version = 0; 3425 ns->ofs_in_node = 0; 3426 } 3427 } else { 3428 err = f2fs_restore_node_summary(sbi, segno, sum); 3429 if (err) 3430 goto out; 3431 } 3432 } 3433 3434 /* set uncompleted segment to curseg */ 3435 curseg = CURSEG_I(sbi, type); 3436 mutex_lock(&curseg->curseg_mutex); 3437 3438 /* update journal info */ 3439 down_write(&curseg->journal_rwsem); 3440 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); 3441 up_write(&curseg->journal_rwsem); 3442 3443 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); 3444 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); 3445 curseg->next_segno = segno; 3446 reset_curseg(sbi, type, 0); 3447 curseg->alloc_type = ckpt->alloc_type[type]; 3448 curseg->next_blkoff = blk_off; 3449 mutex_unlock(&curseg->curseg_mutex); 3450 out: 3451 f2fs_put_page(new, 1); 3452 return err; 3453 } 3454 3455 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 3456 { 3457 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; 3458 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; 3459 int type = CURSEG_HOT_DATA; 3460 int err; 3461 3462 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { 3463 int npages = f2fs_npages_for_summary_flush(sbi, true); 3464 3465 if (npages >= 2) 3466 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, 3467 META_CP, true); 3468 3469 /* restore for compacted data summary */ 3470 err = read_compacted_summaries(sbi); 3471 if (err) 3472 return err; 3473 type = CURSEG_HOT_NODE; 3474 } 3475 3476 if (__exist_node_summaries(sbi)) 3477 f2fs_ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type), 3478 NR_CURSEG_TYPE - type, META_CP, true); 3479 3480 for (; type <= CURSEG_COLD_NODE; type++) { 3481 err = read_normal_summaries(sbi, type); 3482 if (err) 3483 return err; 3484 } 3485 3486 /* sanity check for summary blocks */ 3487 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || 3488 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) 3489 return -EINVAL; 3490 3491 return 0; 3492 } 3493 3494 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 3495 { 3496 struct page *page; 3497 unsigned char *kaddr; 3498 struct f2fs_summary *summary; 3499 struct curseg_info *seg_i; 3500 int written_size = 0; 3501 int i, j; 3502 3503 page = f2fs_grab_meta_page(sbi, blkaddr++); 3504 kaddr = (unsigned char *)page_address(page); 3505 memset(kaddr, 0, PAGE_SIZE); 3506 3507 /* Step 1: write nat cache */ 3508 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 3509 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); 3510 written_size += SUM_JOURNAL_SIZE; 3511 3512 /* Step 2: write sit cache */ 3513 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 3514 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); 3515 written_size += SUM_JOURNAL_SIZE; 3516 3517 /* Step 3: write summary entries */ 3518 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 3519 unsigned short blkoff; 3520 seg_i = CURSEG_I(sbi, i); 3521 if (sbi->ckpt->alloc_type[i] == SSR) 3522 blkoff = sbi->blocks_per_seg; 3523 else 3524 blkoff = curseg_blkoff(sbi, i); 3525 3526 for (j = 0; j < blkoff; j++) { 3527 if (!page) { 3528 page = f2fs_grab_meta_page(sbi, blkaddr++); 3529 kaddr = (unsigned char *)page_address(page); 3530 memset(kaddr, 0, PAGE_SIZE); 3531 written_size = 0; 3532 } 3533 summary = (struct f2fs_summary *)(kaddr + written_size); 3534 *summary = seg_i->sum_blk->entries[j]; 3535 written_size += SUMMARY_SIZE; 3536 3537 if (written_size + SUMMARY_SIZE <= PAGE_SIZE - 3538 SUM_FOOTER_SIZE) 3539 continue; 3540 3541 set_page_dirty(page); 3542 f2fs_put_page(page, 1); 3543 page = NULL; 3544 } 3545 } 3546 if (page) { 3547 set_page_dirty(page); 3548 f2fs_put_page(page, 1); 3549 } 3550 } 3551 3552 static void write_normal_summaries(struct f2fs_sb_info *sbi, 3553 block_t blkaddr, int type) 3554 { 3555 int i, end; 3556 if (IS_DATASEG(type)) 3557 end = type + NR_CURSEG_DATA_TYPE; 3558 else 3559 end = type + NR_CURSEG_NODE_TYPE; 3560 3561 for (i = type; i < end; i++) 3562 write_current_sum_page(sbi, i, blkaddr + (i - type)); 3563 } 3564 3565 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 3566 { 3567 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) 3568 write_compacted_summaries(sbi, start_blk); 3569 else 3570 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 3571 } 3572 3573 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 3574 { 3575 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 3576 } 3577 3578 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, 3579 unsigned int val, int alloc) 3580 { 3581 int i; 3582 3583 if (type == NAT_JOURNAL) { 3584 for (i = 0; i < nats_in_cursum(journal); i++) { 3585 if (le32_to_cpu(nid_in_journal(journal, i)) == val) 3586 return i; 3587 } 3588 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) 3589 return update_nats_in_cursum(journal, 1); 3590 } else if (type == SIT_JOURNAL) { 3591 for (i = 0; i < sits_in_cursum(journal); i++) 3592 if (le32_to_cpu(segno_in_journal(journal, i)) == val) 3593 return i; 3594 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) 3595 return update_sits_in_cursum(journal, 1); 3596 } 3597 return -1; 3598 } 3599 3600 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 3601 unsigned int segno) 3602 { 3603 return f2fs_get_meta_page_nofail(sbi, current_sit_addr(sbi, segno)); 3604 } 3605 3606 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 3607 unsigned int start) 3608 { 3609 struct sit_info *sit_i = SIT_I(sbi); 3610 struct page *page; 3611 pgoff_t src_off, dst_off; 3612 3613 src_off = current_sit_addr(sbi, start); 3614 dst_off = next_sit_addr(sbi, src_off); 3615 3616 page = f2fs_grab_meta_page(sbi, dst_off); 3617 seg_info_to_sit_page(sbi, page, start); 3618 3619 set_page_dirty(page); 3620 set_to_next_sit(sit_i, start); 3621 3622 return page; 3623 } 3624 3625 static struct sit_entry_set *grab_sit_entry_set(void) 3626 { 3627 struct sit_entry_set *ses = 3628 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS); 3629 3630 ses->entry_cnt = 0; 3631 INIT_LIST_HEAD(&ses->set_list); 3632 return ses; 3633 } 3634 3635 static void release_sit_entry_set(struct sit_entry_set *ses) 3636 { 3637 list_del(&ses->set_list); 3638 kmem_cache_free(sit_entry_set_slab, ses); 3639 } 3640 3641 static void adjust_sit_entry_set(struct sit_entry_set *ses, 3642 struct list_head *head) 3643 { 3644 struct sit_entry_set *next = ses; 3645 3646 if (list_is_last(&ses->set_list, head)) 3647 return; 3648 3649 list_for_each_entry_continue(next, head, set_list) 3650 if (ses->entry_cnt <= next->entry_cnt) 3651 break; 3652 3653 list_move_tail(&ses->set_list, &next->set_list); 3654 } 3655 3656 static void add_sit_entry(unsigned int segno, struct list_head *head) 3657 { 3658 struct sit_entry_set *ses; 3659 unsigned int start_segno = START_SEGNO(segno); 3660 3661 list_for_each_entry(ses, head, set_list) { 3662 if (ses->start_segno == start_segno) { 3663 ses->entry_cnt++; 3664 adjust_sit_entry_set(ses, head); 3665 return; 3666 } 3667 } 3668 3669 ses = grab_sit_entry_set(); 3670 3671 ses->start_segno = start_segno; 3672 ses->entry_cnt++; 3673 list_add(&ses->set_list, head); 3674 } 3675 3676 static void add_sits_in_set(struct f2fs_sb_info *sbi) 3677 { 3678 struct f2fs_sm_info *sm_info = SM_I(sbi); 3679 struct list_head *set_list = &sm_info->sit_entry_set; 3680 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; 3681 unsigned int segno; 3682 3683 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) 3684 add_sit_entry(segno, set_list); 3685 } 3686 3687 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) 3688 { 3689 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 3690 struct f2fs_journal *journal = curseg->journal; 3691 int i; 3692 3693 down_write(&curseg->journal_rwsem); 3694 for (i = 0; i < sits_in_cursum(journal); i++) { 3695 unsigned int segno; 3696 bool dirtied; 3697 3698 segno = le32_to_cpu(segno_in_journal(journal, i)); 3699 dirtied = __mark_sit_entry_dirty(sbi, segno); 3700 3701 if (!dirtied) 3702 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); 3703 } 3704 update_sits_in_cursum(journal, -i); 3705 up_write(&curseg->journal_rwsem); 3706 } 3707 3708 /* 3709 * CP calls this function, which flushes SIT entries including sit_journal, 3710 * and moves prefree segs to free segs. 3711 */ 3712 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) 3713 { 3714 struct sit_info *sit_i = SIT_I(sbi); 3715 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 3716 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 3717 struct f2fs_journal *journal = curseg->journal; 3718 struct sit_entry_set *ses, *tmp; 3719 struct list_head *head = &SM_I(sbi)->sit_entry_set; 3720 bool to_journal = true; 3721 struct seg_entry *se; 3722 3723 down_write(&sit_i->sentry_lock); 3724 3725 if (!sit_i->dirty_sentries) 3726 goto out; 3727 3728 /* 3729 * add and account sit entries of dirty bitmap in sit entry 3730 * set temporarily 3731 */ 3732 add_sits_in_set(sbi); 3733 3734 /* 3735 * if there are no enough space in journal to store dirty sit 3736 * entries, remove all entries from journal and add and account 3737 * them in sit entry set. 3738 */ 3739 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL)) 3740 remove_sits_in_journal(sbi); 3741 3742 /* 3743 * there are two steps to flush sit entries: 3744 * #1, flush sit entries to journal in current cold data summary block. 3745 * #2, flush sit entries to sit page. 3746 */ 3747 list_for_each_entry_safe(ses, tmp, head, set_list) { 3748 struct page *page = NULL; 3749 struct f2fs_sit_block *raw_sit = NULL; 3750 unsigned int start_segno = ses->start_segno; 3751 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, 3752 (unsigned long)MAIN_SEGS(sbi)); 3753 unsigned int segno = start_segno; 3754 3755 if (to_journal && 3756 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) 3757 to_journal = false; 3758 3759 if (to_journal) { 3760 down_write(&curseg->journal_rwsem); 3761 } else { 3762 page = get_next_sit_page(sbi, start_segno); 3763 raw_sit = page_address(page); 3764 } 3765 3766 /* flush dirty sit entries in region of current sit set */ 3767 for_each_set_bit_from(segno, bitmap, end) { 3768 int offset, sit_offset; 3769 3770 se = get_seg_entry(sbi, segno); 3771 #ifdef CONFIG_F2FS_CHECK_FS 3772 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir, 3773 SIT_VBLOCK_MAP_SIZE)) 3774 f2fs_bug_on(sbi, 1); 3775 #endif 3776 3777 /* add discard candidates */ 3778 if (!(cpc->reason & CP_DISCARD)) { 3779 cpc->trim_start = segno; 3780 add_discard_addrs(sbi, cpc, false); 3781 } 3782 3783 if (to_journal) { 3784 offset = f2fs_lookup_journal_in_cursum(journal, 3785 SIT_JOURNAL, segno, 1); 3786 f2fs_bug_on(sbi, offset < 0); 3787 segno_in_journal(journal, offset) = 3788 cpu_to_le32(segno); 3789 seg_info_to_raw_sit(se, 3790 &sit_in_journal(journal, offset)); 3791 check_block_count(sbi, segno, 3792 &sit_in_journal(journal, offset)); 3793 } else { 3794 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 3795 seg_info_to_raw_sit(se, 3796 &raw_sit->entries[sit_offset]); 3797 check_block_count(sbi, segno, 3798 &raw_sit->entries[sit_offset]); 3799 } 3800 3801 __clear_bit(segno, bitmap); 3802 sit_i->dirty_sentries--; 3803 ses->entry_cnt--; 3804 } 3805 3806 if (to_journal) 3807 up_write(&curseg->journal_rwsem); 3808 else 3809 f2fs_put_page(page, 1); 3810 3811 f2fs_bug_on(sbi, ses->entry_cnt); 3812 release_sit_entry_set(ses); 3813 } 3814 3815 f2fs_bug_on(sbi, !list_empty(head)); 3816 f2fs_bug_on(sbi, sit_i->dirty_sentries); 3817 out: 3818 if (cpc->reason & CP_DISCARD) { 3819 __u64 trim_start = cpc->trim_start; 3820 3821 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) 3822 add_discard_addrs(sbi, cpc, false); 3823 3824 cpc->trim_start = trim_start; 3825 } 3826 up_write(&sit_i->sentry_lock); 3827 3828 set_prefree_as_free_segments(sbi); 3829 } 3830 3831 static int build_sit_info(struct f2fs_sb_info *sbi) 3832 { 3833 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 3834 struct sit_info *sit_i; 3835 unsigned int sit_segs, start; 3836 char *src_bitmap; 3837 unsigned int bitmap_size; 3838 3839 /* allocate memory for SIT information */ 3840 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); 3841 if (!sit_i) 3842 return -ENOMEM; 3843 3844 SM_I(sbi)->sit_info = sit_i; 3845 3846 sit_i->sentries = 3847 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), 3848 MAIN_SEGS(sbi)), 3849 GFP_KERNEL); 3850 if (!sit_i->sentries) 3851 return -ENOMEM; 3852 3853 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 3854 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, bitmap_size, 3855 GFP_KERNEL); 3856 if (!sit_i->dirty_sentries_bitmap) 3857 return -ENOMEM; 3858 3859 for (start = 0; start < MAIN_SEGS(sbi); start++) { 3860 sit_i->sentries[start].cur_valid_map 3861 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3862 sit_i->sentries[start].ckpt_valid_map 3863 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3864 if (!sit_i->sentries[start].cur_valid_map || 3865 !sit_i->sentries[start].ckpt_valid_map) 3866 return -ENOMEM; 3867 3868 #ifdef CONFIG_F2FS_CHECK_FS 3869 sit_i->sentries[start].cur_valid_map_mir 3870 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3871 if (!sit_i->sentries[start].cur_valid_map_mir) 3872 return -ENOMEM; 3873 #endif 3874 3875 sit_i->sentries[start].discard_map 3876 = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, 3877 GFP_KERNEL); 3878 if (!sit_i->sentries[start].discard_map) 3879 return -ENOMEM; 3880 } 3881 3882 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 3883 if (!sit_i->tmp_map) 3884 return -ENOMEM; 3885 3886 if (__is_large_section(sbi)) { 3887 sit_i->sec_entries = 3888 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), 3889 MAIN_SECS(sbi)), 3890 GFP_KERNEL); 3891 if (!sit_i->sec_entries) 3892 return -ENOMEM; 3893 } 3894 3895 /* get information related with SIT */ 3896 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 3897 3898 /* setup SIT bitmap from ckeckpoint pack */ 3899 bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 3900 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 3901 3902 sit_i->sit_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); 3903 if (!sit_i->sit_bitmap) 3904 return -ENOMEM; 3905 3906 #ifdef CONFIG_F2FS_CHECK_FS 3907 sit_i->sit_bitmap_mir = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); 3908 if (!sit_i->sit_bitmap_mir) 3909 return -ENOMEM; 3910 #endif 3911 3912 /* init SIT information */ 3913 sit_i->s_ops = &default_salloc_ops; 3914 3915 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 3916 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; 3917 sit_i->written_valid_blocks = 0; 3918 sit_i->bitmap_size = bitmap_size; 3919 sit_i->dirty_sentries = 0; 3920 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 3921 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 3922 sit_i->mounted_time = ktime_get_real_seconds(); 3923 init_rwsem(&sit_i->sentry_lock); 3924 return 0; 3925 } 3926 3927 static int build_free_segmap(struct f2fs_sb_info *sbi) 3928 { 3929 struct free_segmap_info *free_i; 3930 unsigned int bitmap_size, sec_bitmap_size; 3931 3932 /* allocate memory for free segmap information */ 3933 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); 3934 if (!free_i) 3935 return -ENOMEM; 3936 3937 SM_I(sbi)->free_info = free_i; 3938 3939 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 3940 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); 3941 if (!free_i->free_segmap) 3942 return -ENOMEM; 3943 3944 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 3945 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); 3946 if (!free_i->free_secmap) 3947 return -ENOMEM; 3948 3949 /* set all segments as dirty temporarily */ 3950 memset(free_i->free_segmap, 0xff, bitmap_size); 3951 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 3952 3953 /* init free segmap information */ 3954 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); 3955 free_i->free_segments = 0; 3956 free_i->free_sections = 0; 3957 spin_lock_init(&free_i->segmap_lock); 3958 return 0; 3959 } 3960 3961 static int build_curseg(struct f2fs_sb_info *sbi) 3962 { 3963 struct curseg_info *array; 3964 int i; 3965 3966 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, sizeof(*array)), 3967 GFP_KERNEL); 3968 if (!array) 3969 return -ENOMEM; 3970 3971 SM_I(sbi)->curseg_array = array; 3972 3973 for (i = 0; i < NR_CURSEG_TYPE; i++) { 3974 mutex_init(&array[i].curseg_mutex); 3975 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); 3976 if (!array[i].sum_blk) 3977 return -ENOMEM; 3978 init_rwsem(&array[i].journal_rwsem); 3979 array[i].journal = f2fs_kzalloc(sbi, 3980 sizeof(struct f2fs_journal), GFP_KERNEL); 3981 if (!array[i].journal) 3982 return -ENOMEM; 3983 array[i].segno = NULL_SEGNO; 3984 array[i].next_blkoff = 0; 3985 } 3986 return restore_curseg_summaries(sbi); 3987 } 3988 3989 static int build_sit_entries(struct f2fs_sb_info *sbi) 3990 { 3991 struct sit_info *sit_i = SIT_I(sbi); 3992 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 3993 struct f2fs_journal *journal = curseg->journal; 3994 struct seg_entry *se; 3995 struct f2fs_sit_entry sit; 3996 int sit_blk_cnt = SIT_BLK_CNT(sbi); 3997 unsigned int i, start, end; 3998 unsigned int readed, start_blk = 0; 3999 int err = 0; 4000 block_t total_node_blocks = 0; 4001 4002 do { 4003 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES, 4004 META_SIT, true); 4005 4006 start = start_blk * sit_i->sents_per_block; 4007 end = (start_blk + readed) * sit_i->sents_per_block; 4008 4009 for (; start < end && start < MAIN_SEGS(sbi); start++) { 4010 struct f2fs_sit_block *sit_blk; 4011 struct page *page; 4012 4013 se = &sit_i->sentries[start]; 4014 page = get_current_sit_page(sbi, start); 4015 if (IS_ERR(page)) 4016 return PTR_ERR(page); 4017 sit_blk = (struct f2fs_sit_block *)page_address(page); 4018 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 4019 f2fs_put_page(page, 1); 4020 4021 err = check_block_count(sbi, start, &sit); 4022 if (err) 4023 return err; 4024 seg_info_from_raw_sit(se, &sit); 4025 if (IS_NODESEG(se->type)) 4026 total_node_blocks += se->valid_blocks; 4027 4028 /* build discard map only one time */ 4029 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 4030 memset(se->discard_map, 0xff, 4031 SIT_VBLOCK_MAP_SIZE); 4032 } else { 4033 memcpy(se->discard_map, 4034 se->cur_valid_map, 4035 SIT_VBLOCK_MAP_SIZE); 4036 sbi->discard_blks += 4037 sbi->blocks_per_seg - 4038 se->valid_blocks; 4039 } 4040 4041 if (__is_large_section(sbi)) 4042 get_sec_entry(sbi, start)->valid_blocks += 4043 se->valid_blocks; 4044 } 4045 start_blk += readed; 4046 } while (start_blk < sit_blk_cnt); 4047 4048 down_read(&curseg->journal_rwsem); 4049 for (i = 0; i < sits_in_cursum(journal); i++) { 4050 unsigned int old_valid_blocks; 4051 4052 start = le32_to_cpu(segno_in_journal(journal, i)); 4053 if (start >= MAIN_SEGS(sbi)) { 4054 f2fs_msg(sbi->sb, KERN_ERR, 4055 "Wrong journal entry on segno %u", 4056 start); 4057 set_sbi_flag(sbi, SBI_NEED_FSCK); 4058 err = -EINVAL; 4059 break; 4060 } 4061 4062 se = &sit_i->sentries[start]; 4063 sit = sit_in_journal(journal, i); 4064 4065 old_valid_blocks = se->valid_blocks; 4066 if (IS_NODESEG(se->type)) 4067 total_node_blocks -= old_valid_blocks; 4068 4069 err = check_block_count(sbi, start, &sit); 4070 if (err) 4071 break; 4072 seg_info_from_raw_sit(se, &sit); 4073 if (IS_NODESEG(se->type)) 4074 total_node_blocks += se->valid_blocks; 4075 4076 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 4077 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE); 4078 } else { 4079 memcpy(se->discard_map, se->cur_valid_map, 4080 SIT_VBLOCK_MAP_SIZE); 4081 sbi->discard_blks += old_valid_blocks; 4082 sbi->discard_blks -= se->valid_blocks; 4083 } 4084 4085 if (__is_large_section(sbi)) { 4086 get_sec_entry(sbi, start)->valid_blocks += 4087 se->valid_blocks; 4088 get_sec_entry(sbi, start)->valid_blocks -= 4089 old_valid_blocks; 4090 } 4091 } 4092 up_read(&curseg->journal_rwsem); 4093 4094 if (!err && total_node_blocks != valid_node_count(sbi)) { 4095 f2fs_msg(sbi->sb, KERN_ERR, 4096 "SIT is corrupted node# %u vs %u", 4097 total_node_blocks, valid_node_count(sbi)); 4098 set_sbi_flag(sbi, SBI_NEED_FSCK); 4099 err = -EINVAL; 4100 } 4101 4102 return err; 4103 } 4104 4105 static void init_free_segmap(struct f2fs_sb_info *sbi) 4106 { 4107 unsigned int start; 4108 int type; 4109 4110 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4111 struct seg_entry *sentry = get_seg_entry(sbi, start); 4112 if (!sentry->valid_blocks) 4113 __set_free(sbi, start); 4114 else 4115 SIT_I(sbi)->written_valid_blocks += 4116 sentry->valid_blocks; 4117 } 4118 4119 /* set use the current segments */ 4120 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 4121 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 4122 __set_test_and_inuse(sbi, curseg_t->segno); 4123 } 4124 } 4125 4126 static void init_dirty_segmap(struct f2fs_sb_info *sbi) 4127 { 4128 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4129 struct free_segmap_info *free_i = FREE_I(sbi); 4130 unsigned int segno = 0, offset = 0; 4131 unsigned short valid_blocks; 4132 4133 while (1) { 4134 /* find dirty segment based on free segmap */ 4135 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); 4136 if (segno >= MAIN_SEGS(sbi)) 4137 break; 4138 offset = segno + 1; 4139 valid_blocks = get_valid_blocks(sbi, segno, false); 4140 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks) 4141 continue; 4142 if (valid_blocks > sbi->blocks_per_seg) { 4143 f2fs_bug_on(sbi, 1); 4144 continue; 4145 } 4146 mutex_lock(&dirty_i->seglist_lock); 4147 __locate_dirty_segment(sbi, segno, DIRTY); 4148 mutex_unlock(&dirty_i->seglist_lock); 4149 } 4150 } 4151 4152 static int init_victim_secmap(struct f2fs_sb_info *sbi) 4153 { 4154 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4155 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 4156 4157 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 4158 if (!dirty_i->victim_secmap) 4159 return -ENOMEM; 4160 return 0; 4161 } 4162 4163 static int build_dirty_segmap(struct f2fs_sb_info *sbi) 4164 { 4165 struct dirty_seglist_info *dirty_i; 4166 unsigned int bitmap_size, i; 4167 4168 /* allocate memory for dirty segments list information */ 4169 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), 4170 GFP_KERNEL); 4171 if (!dirty_i) 4172 return -ENOMEM; 4173 4174 SM_I(sbi)->dirty_info = dirty_i; 4175 mutex_init(&dirty_i->seglist_lock); 4176 4177 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4178 4179 for (i = 0; i < NR_DIRTY_TYPE; i++) { 4180 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, 4181 GFP_KERNEL); 4182 if (!dirty_i->dirty_segmap[i]) 4183 return -ENOMEM; 4184 } 4185 4186 init_dirty_segmap(sbi); 4187 return init_victim_secmap(sbi); 4188 } 4189 4190 /* 4191 * Update min, max modified time for cost-benefit GC algorithm 4192 */ 4193 static void init_min_max_mtime(struct f2fs_sb_info *sbi) 4194 { 4195 struct sit_info *sit_i = SIT_I(sbi); 4196 unsigned int segno; 4197 4198 down_write(&sit_i->sentry_lock); 4199 4200 sit_i->min_mtime = ULLONG_MAX; 4201 4202 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { 4203 unsigned int i; 4204 unsigned long long mtime = 0; 4205 4206 for (i = 0; i < sbi->segs_per_sec; i++) 4207 mtime += get_seg_entry(sbi, segno + i)->mtime; 4208 4209 mtime = div_u64(mtime, sbi->segs_per_sec); 4210 4211 if (sit_i->min_mtime > mtime) 4212 sit_i->min_mtime = mtime; 4213 } 4214 sit_i->max_mtime = get_mtime(sbi, false); 4215 up_write(&sit_i->sentry_lock); 4216 } 4217 4218 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) 4219 { 4220 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 4221 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 4222 struct f2fs_sm_info *sm_info; 4223 int err; 4224 4225 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); 4226 if (!sm_info) 4227 return -ENOMEM; 4228 4229 /* init sm info */ 4230 sbi->sm_info = sm_info; 4231 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 4232 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 4233 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 4234 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 4235 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 4236 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 4237 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 4238 sm_info->rec_prefree_segments = sm_info->main_segments * 4239 DEF_RECLAIM_PREFREE_SEGMENTS / 100; 4240 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS) 4241 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; 4242 4243 if (!test_opt(sbi, LFS)) 4244 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC; 4245 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 4246 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; 4247 sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec; 4248 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS; 4249 sm_info->min_ssr_sections = reserved_sections(sbi); 4250 4251 INIT_LIST_HEAD(&sm_info->sit_entry_set); 4252 4253 init_rwsem(&sm_info->curseg_lock); 4254 4255 if (!f2fs_readonly(sbi->sb)) { 4256 err = f2fs_create_flush_cmd_control(sbi); 4257 if (err) 4258 return err; 4259 } 4260 4261 err = create_discard_cmd_control(sbi); 4262 if (err) 4263 return err; 4264 4265 err = build_sit_info(sbi); 4266 if (err) 4267 return err; 4268 err = build_free_segmap(sbi); 4269 if (err) 4270 return err; 4271 err = build_curseg(sbi); 4272 if (err) 4273 return err; 4274 4275 /* reinit free segmap based on SIT */ 4276 err = build_sit_entries(sbi); 4277 if (err) 4278 return err; 4279 4280 init_free_segmap(sbi); 4281 err = build_dirty_segmap(sbi); 4282 if (err) 4283 return err; 4284 4285 init_min_max_mtime(sbi); 4286 return 0; 4287 } 4288 4289 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 4290 enum dirty_type dirty_type) 4291 { 4292 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4293 4294 mutex_lock(&dirty_i->seglist_lock); 4295 kvfree(dirty_i->dirty_segmap[dirty_type]); 4296 dirty_i->nr_dirty[dirty_type] = 0; 4297 mutex_unlock(&dirty_i->seglist_lock); 4298 } 4299 4300 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 4301 { 4302 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4303 kvfree(dirty_i->victim_secmap); 4304 } 4305 4306 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 4307 { 4308 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 4309 int i; 4310 4311 if (!dirty_i) 4312 return; 4313 4314 /* discard pre-free/dirty segments list */ 4315 for (i = 0; i < NR_DIRTY_TYPE; i++) 4316 discard_dirty_segmap(sbi, i); 4317 4318 destroy_victim_secmap(sbi); 4319 SM_I(sbi)->dirty_info = NULL; 4320 kvfree(dirty_i); 4321 } 4322 4323 static void destroy_curseg(struct f2fs_sb_info *sbi) 4324 { 4325 struct curseg_info *array = SM_I(sbi)->curseg_array; 4326 int i; 4327 4328 if (!array) 4329 return; 4330 SM_I(sbi)->curseg_array = NULL; 4331 for (i = 0; i < NR_CURSEG_TYPE; i++) { 4332 kvfree(array[i].sum_blk); 4333 kvfree(array[i].journal); 4334 } 4335 kvfree(array); 4336 } 4337 4338 static void destroy_free_segmap(struct f2fs_sb_info *sbi) 4339 { 4340 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 4341 if (!free_i) 4342 return; 4343 SM_I(sbi)->free_info = NULL; 4344 kvfree(free_i->free_segmap); 4345 kvfree(free_i->free_secmap); 4346 kvfree(free_i); 4347 } 4348 4349 static void destroy_sit_info(struct f2fs_sb_info *sbi) 4350 { 4351 struct sit_info *sit_i = SIT_I(sbi); 4352 unsigned int start; 4353 4354 if (!sit_i) 4355 return; 4356 4357 if (sit_i->sentries) { 4358 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4359 kvfree(sit_i->sentries[start].cur_valid_map); 4360 #ifdef CONFIG_F2FS_CHECK_FS 4361 kvfree(sit_i->sentries[start].cur_valid_map_mir); 4362 #endif 4363 kvfree(sit_i->sentries[start].ckpt_valid_map); 4364 kvfree(sit_i->sentries[start].discard_map); 4365 } 4366 } 4367 kvfree(sit_i->tmp_map); 4368 4369 kvfree(sit_i->sentries); 4370 kvfree(sit_i->sec_entries); 4371 kvfree(sit_i->dirty_sentries_bitmap); 4372 4373 SM_I(sbi)->sit_info = NULL; 4374 kvfree(sit_i->sit_bitmap); 4375 #ifdef CONFIG_F2FS_CHECK_FS 4376 kvfree(sit_i->sit_bitmap_mir); 4377 #endif 4378 kvfree(sit_i); 4379 } 4380 4381 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) 4382 { 4383 struct f2fs_sm_info *sm_info = SM_I(sbi); 4384 4385 if (!sm_info) 4386 return; 4387 f2fs_destroy_flush_cmd_control(sbi, true); 4388 destroy_discard_cmd_control(sbi); 4389 destroy_dirty_segmap(sbi); 4390 destroy_curseg(sbi); 4391 destroy_free_segmap(sbi); 4392 destroy_sit_info(sbi); 4393 sbi->sm_info = NULL; 4394 kvfree(sm_info); 4395 } 4396 4397 int __init f2fs_create_segment_manager_caches(void) 4398 { 4399 discard_entry_slab = f2fs_kmem_cache_create("discard_entry", 4400 sizeof(struct discard_entry)); 4401 if (!discard_entry_slab) 4402 goto fail; 4403 4404 discard_cmd_slab = f2fs_kmem_cache_create("discard_cmd", 4405 sizeof(struct discard_cmd)); 4406 if (!discard_cmd_slab) 4407 goto destroy_discard_entry; 4408 4409 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set", 4410 sizeof(struct sit_entry_set)); 4411 if (!sit_entry_set_slab) 4412 goto destroy_discard_cmd; 4413 4414 inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry", 4415 sizeof(struct inmem_pages)); 4416 if (!inmem_entry_slab) 4417 goto destroy_sit_entry_set; 4418 return 0; 4419 4420 destroy_sit_entry_set: 4421 kmem_cache_destroy(sit_entry_set_slab); 4422 destroy_discard_cmd: 4423 kmem_cache_destroy(discard_cmd_slab); 4424 destroy_discard_entry: 4425 kmem_cache_destroy(discard_entry_slab); 4426 fail: 4427 return -ENOMEM; 4428 } 4429 4430 void f2fs_destroy_segment_manager_caches(void) 4431 { 4432 kmem_cache_destroy(sit_entry_set_slab); 4433 kmem_cache_destroy(discard_cmd_slab); 4434 kmem_cache_destroy(discard_entry_slab); 4435 kmem_cache_destroy(inmem_entry_slab); 4436 } 4437