node.c (1d1df41c5a33359a00e919d54eaebfb789711fdc) node.c (e4544b63a7ee49e7fbebf35ece0a6acd3b9617ae)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/node.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 368 unchanged lines hidden (view full) ---

377}
378
379int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid)
380{
381 struct f2fs_nm_info *nm_i = NM_I(sbi);
382 struct nat_entry *e;
383 bool need = false;
384
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/node.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 368 unchanged lines hidden (view full) ---

377}
378
379int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid)
380{
381 struct f2fs_nm_info *nm_i = NM_I(sbi);
382 struct nat_entry *e;
383 bool need = false;
384
385 down_read(&nm_i->nat_tree_lock);
385 f2fs_down_read(&nm_i->nat_tree_lock);
386 e = __lookup_nat_cache(nm_i, nid);
387 if (e) {
388 if (!get_nat_flag(e, IS_CHECKPOINTED) &&
389 !get_nat_flag(e, HAS_FSYNCED_INODE))
390 need = true;
391 }
386 e = __lookup_nat_cache(nm_i, nid);
387 if (e) {
388 if (!get_nat_flag(e, IS_CHECKPOINTED) &&
389 !get_nat_flag(e, HAS_FSYNCED_INODE))
390 need = true;
391 }
392 up_read(&nm_i->nat_tree_lock);
392 f2fs_up_read(&nm_i->nat_tree_lock);
393 return need;
394}
395
396bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
397{
398 struct f2fs_nm_info *nm_i = NM_I(sbi);
399 struct nat_entry *e;
400 bool is_cp = true;
401
393 return need;
394}
395
396bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
397{
398 struct f2fs_nm_info *nm_i = NM_I(sbi);
399 struct nat_entry *e;
400 bool is_cp = true;
401
402 down_read(&nm_i->nat_tree_lock);
402 f2fs_down_read(&nm_i->nat_tree_lock);
403 e = __lookup_nat_cache(nm_i, nid);
404 if (e && !get_nat_flag(e, IS_CHECKPOINTED))
405 is_cp = false;
403 e = __lookup_nat_cache(nm_i, nid);
404 if (e && !get_nat_flag(e, IS_CHECKPOINTED))
405 is_cp = false;
406 up_read(&nm_i->nat_tree_lock);
406 f2fs_up_read(&nm_i->nat_tree_lock);
407 return is_cp;
408}
409
410bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino)
411{
412 struct f2fs_nm_info *nm_i = NM_I(sbi);
413 struct nat_entry *e;
414 bool need_update = true;
415
407 return is_cp;
408}
409
410bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino)
411{
412 struct f2fs_nm_info *nm_i = NM_I(sbi);
413 struct nat_entry *e;
414 bool need_update = true;
415
416 down_read(&nm_i->nat_tree_lock);
416 f2fs_down_read(&nm_i->nat_tree_lock);
417 e = __lookup_nat_cache(nm_i, ino);
418 if (e && get_nat_flag(e, HAS_LAST_FSYNC) &&
419 (get_nat_flag(e, IS_CHECKPOINTED) ||
420 get_nat_flag(e, HAS_FSYNCED_INODE)))
421 need_update = false;
417 e = __lookup_nat_cache(nm_i, ino);
418 if (e && get_nat_flag(e, HAS_LAST_FSYNC) &&
419 (get_nat_flag(e, IS_CHECKPOINTED) ||
420 get_nat_flag(e, HAS_FSYNCED_INODE)))
421 need_update = false;
422 up_read(&nm_i->nat_tree_lock);
422 f2fs_up_read(&nm_i->nat_tree_lock);
423 return need_update;
424}
425
426/* must be locked by nat_tree_lock */
427static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid,
428 struct f2fs_nat_entry *ne)
429{
430 struct f2fs_nm_info *nm_i = NM_I(sbi);
431 struct nat_entry *new, *e;
432
433 /* Let's mitigate lock contention of nat_tree_lock during checkpoint */
423 return need_update;
424}
425
426/* must be locked by nat_tree_lock */
427static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid,
428 struct f2fs_nat_entry *ne)
429{
430 struct f2fs_nm_info *nm_i = NM_I(sbi);
431 struct nat_entry *new, *e;
432
433 /* Let's mitigate lock contention of nat_tree_lock during checkpoint */
434 if (rwsem_is_locked(&sbi->cp_global_sem))
434 if (f2fs_rwsem_is_locked(&sbi->cp_global_sem))
435 return;
436
437 new = __alloc_nat_entry(sbi, nid, false);
438 if (!new)
439 return;
440
435 return;
436
437 new = __alloc_nat_entry(sbi, nid, false);
438 if (!new)
439 return;
440
441 down_write(&nm_i->nat_tree_lock);
441 f2fs_down_write(&nm_i->nat_tree_lock);
442 e = __lookup_nat_cache(nm_i, nid);
443 if (!e)
444 e = __init_nat_entry(nm_i, new, ne, false);
445 else
446 f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) ||
447 nat_get_blkaddr(e) !=
448 le32_to_cpu(ne->block_addr) ||
449 nat_get_version(e) != ne->version);
442 e = __lookup_nat_cache(nm_i, nid);
443 if (!e)
444 e = __init_nat_entry(nm_i, new, ne, false);
445 else
446 f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) ||
447 nat_get_blkaddr(e) !=
448 le32_to_cpu(ne->block_addr) ||
449 nat_get_version(e) != ne->version);
450 up_write(&nm_i->nat_tree_lock);
450 f2fs_up_write(&nm_i->nat_tree_lock);
451 if (e != new)
452 __free_nat_entry(new);
453}
454
455static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
456 block_t new_blkaddr, bool fsync_done)
457{
458 struct f2fs_nm_info *nm_i = NM_I(sbi);
459 struct nat_entry *e;
460 struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true);
461
451 if (e != new)
452 __free_nat_entry(new);
453}
454
455static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
456 block_t new_blkaddr, bool fsync_done)
457{
458 struct f2fs_nm_info *nm_i = NM_I(sbi);
459 struct nat_entry *e;
460 struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true);
461
462 down_write(&nm_i->nat_tree_lock);
462 f2fs_down_write(&nm_i->nat_tree_lock);
463 e = __lookup_nat_cache(nm_i, ni->nid);
464 if (!e) {
465 e = __init_nat_entry(nm_i, new, NULL, true);
466 copy_node_info(&e->ni, ni);
467 f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR);
468 } else if (new_blkaddr == NEW_ADDR) {
469 /*
470 * when nid is reallocated,

--- 32 unchanged lines hidden (view full) ---

503 /* update fsync_mark if its inode nat entry is still alive */
504 if (ni->nid != ni->ino)
505 e = __lookup_nat_cache(nm_i, ni->ino);
506 if (e) {
507 if (fsync_done && ni->nid == ni->ino)
508 set_nat_flag(e, HAS_FSYNCED_INODE, true);
509 set_nat_flag(e, HAS_LAST_FSYNC, fsync_done);
510 }
463 e = __lookup_nat_cache(nm_i, ni->nid);
464 if (!e) {
465 e = __init_nat_entry(nm_i, new, NULL, true);
466 copy_node_info(&e->ni, ni);
467 f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR);
468 } else if (new_blkaddr == NEW_ADDR) {
469 /*
470 * when nid is reallocated,

--- 32 unchanged lines hidden (view full) ---

503 /* update fsync_mark if its inode nat entry is still alive */
504 if (ni->nid != ni->ino)
505 e = __lookup_nat_cache(nm_i, ni->ino);
506 if (e) {
507 if (fsync_done && ni->nid == ni->ino)
508 set_nat_flag(e, HAS_FSYNCED_INODE, true);
509 set_nat_flag(e, HAS_LAST_FSYNC, fsync_done);
510 }
511 up_write(&nm_i->nat_tree_lock);
511 f2fs_up_write(&nm_i->nat_tree_lock);
512}
513
514int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
515{
516 struct f2fs_nm_info *nm_i = NM_I(sbi);
517 int nr = nr_shrink;
518
512}
513
514int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
515{
516 struct f2fs_nm_info *nm_i = NM_I(sbi);
517 int nr = nr_shrink;
518
519 if (!down_write_trylock(&nm_i->nat_tree_lock))
519 if (!f2fs_down_write_trylock(&nm_i->nat_tree_lock))
520 return 0;
521
522 spin_lock(&nm_i->nat_list_lock);
523 while (nr_shrink) {
524 struct nat_entry *ne;
525
526 if (list_empty(&nm_i->nat_entries))
527 break;

--- 5 unchanged lines hidden (view full) ---

533
534 __del_from_nat_cache(nm_i, ne);
535 nr_shrink--;
536
537 spin_lock(&nm_i->nat_list_lock);
538 }
539 spin_unlock(&nm_i->nat_list_lock);
540
520 return 0;
521
522 spin_lock(&nm_i->nat_list_lock);
523 while (nr_shrink) {
524 struct nat_entry *ne;
525
526 if (list_empty(&nm_i->nat_entries))
527 break;

--- 5 unchanged lines hidden (view full) ---

533
534 __del_from_nat_cache(nm_i, ne);
535 nr_shrink--;
536
537 spin_lock(&nm_i->nat_list_lock);
538 }
539 spin_unlock(&nm_i->nat_list_lock);
540
541 up_write(&nm_i->nat_tree_lock);
541 f2fs_up_write(&nm_i->nat_tree_lock);
542 return nr - nr_shrink;
543}
544
545int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
546 struct node_info *ni, bool checkpoint_context)
547{
548 struct f2fs_nm_info *nm_i = NM_I(sbi);
549 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);

--- 5 unchanged lines hidden (view full) ---

555 struct nat_entry *e;
556 pgoff_t index;
557 block_t blkaddr;
558 int i;
559
560 ni->nid = nid;
561retry:
562 /* Check nat cache */
542 return nr - nr_shrink;
543}
544
545int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
546 struct node_info *ni, bool checkpoint_context)
547{
548 struct f2fs_nm_info *nm_i = NM_I(sbi);
549 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);

--- 5 unchanged lines hidden (view full) ---

555 struct nat_entry *e;
556 pgoff_t index;
557 block_t blkaddr;
558 int i;
559
560 ni->nid = nid;
561retry:
562 /* Check nat cache */
563 down_read(&nm_i->nat_tree_lock);
563 f2fs_down_read(&nm_i->nat_tree_lock);
564 e = __lookup_nat_cache(nm_i, nid);
565 if (e) {
566 ni->ino = nat_get_ino(e);
567 ni->blk_addr = nat_get_blkaddr(e);
568 ni->version = nat_get_version(e);
564 e = __lookup_nat_cache(nm_i, nid);
565 if (e) {
566 ni->ino = nat_get_ino(e);
567 ni->blk_addr = nat_get_blkaddr(e);
568 ni->version = nat_get_version(e);
569 up_read(&nm_i->nat_tree_lock);
569 f2fs_up_read(&nm_i->nat_tree_lock);
570 return 0;
571 }
572
573 /*
574 * Check current segment summary by trying to grab journal_rwsem first.
575 * This sem is on the critical path on the checkpoint requiring the above
576 * nat_tree_lock. Therefore, we should retry, if we failed to grab here
577 * while not bothering checkpoint.
578 */
570 return 0;
571 }
572
573 /*
574 * Check current segment summary by trying to grab journal_rwsem first.
575 * This sem is on the critical path on the checkpoint requiring the above
576 * nat_tree_lock. Therefore, we should retry, if we failed to grab here
577 * while not bothering checkpoint.
578 */
579 if (!rwsem_is_locked(&sbi->cp_global_sem) || checkpoint_context) {
579 if (!f2fs_rwsem_is_locked(&sbi->cp_global_sem) || checkpoint_context) {
580 down_read(&curseg->journal_rwsem);
580 down_read(&curseg->journal_rwsem);
581 } else if (rwsem_is_contended(&nm_i->nat_tree_lock) ||
581 } else if (f2fs_rwsem_is_contended(&nm_i->nat_tree_lock) ||
582 !down_read_trylock(&curseg->journal_rwsem)) {
582 !down_read_trylock(&curseg->journal_rwsem)) {
583 up_read(&nm_i->nat_tree_lock);
583 f2fs_up_read(&nm_i->nat_tree_lock);
584 goto retry;
585 }
586
587 i = f2fs_lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0);
588 if (i >= 0) {
589 ne = nat_in_journal(journal, i);
590 node_info_from_raw_nat(ni, &ne);
591 }
584 goto retry;
585 }
586
587 i = f2fs_lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0);
588 if (i >= 0) {
589 ne = nat_in_journal(journal, i);
590 node_info_from_raw_nat(ni, &ne);
591 }
592 up_read(&curseg->journal_rwsem);
592 up_read(&curseg->journal_rwsem);
593 if (i >= 0) {
593 if (i >= 0) {
594 up_read(&nm_i->nat_tree_lock);
594 f2fs_up_read(&nm_i->nat_tree_lock);
595 goto cache;
596 }
597
598 /* Fill node_info from nat page */
599 index = current_nat_addr(sbi, nid);
595 goto cache;
596 }
597
598 /* Fill node_info from nat page */
599 index = current_nat_addr(sbi, nid);
600 up_read(&nm_i->nat_tree_lock);
600 f2fs_up_read(&nm_i->nat_tree_lock);
601
602 page = f2fs_get_meta_page(sbi, index);
603 if (IS_ERR(page))
604 return PTR_ERR(page);
605
606 nat_blk = (struct f2fs_nat_block *)page_address(page);
607 ne = nat_blk->entries[nid - start_nid];
608 node_info_from_raw_nat(ni, &ne);

--- 995 unchanged lines hidden (view full) ---

1604 /* get old block addr of this node page */
1605 nid = nid_of_node(page);
1606 f2fs_bug_on(sbi, page->index != nid);
1607
1608 if (f2fs_get_node_info(sbi, nid, &ni, !do_balance))
1609 goto redirty_out;
1610
1611 if (wbc->for_reclaim) {
601
602 page = f2fs_get_meta_page(sbi, index);
603 if (IS_ERR(page))
604 return PTR_ERR(page);
605
606 nat_blk = (struct f2fs_nat_block *)page_address(page);
607 ne = nat_blk->entries[nid - start_nid];
608 node_info_from_raw_nat(ni, &ne);

--- 995 unchanged lines hidden (view full) ---

1604 /* get old block addr of this node page */
1605 nid = nid_of_node(page);
1606 f2fs_bug_on(sbi, page->index != nid);
1607
1608 if (f2fs_get_node_info(sbi, nid, &ni, !do_balance))
1609 goto redirty_out;
1610
1611 if (wbc->for_reclaim) {
1612 if (!down_read_trylock(&sbi->node_write))
1612 if (!f2fs_down_read_trylock(&sbi->node_write))
1613 goto redirty_out;
1614 } else {
1613 goto redirty_out;
1614 } else {
1615 down_read(&sbi->node_write);
1615 f2fs_down_read(&sbi->node_write);
1616 }
1617
1618 /* This page is already truncated */
1619 if (unlikely(ni.blk_addr == NULL_ADDR)) {
1620 ClearPageUptodate(page);
1621 dec_page_count(sbi, F2FS_DIRTY_NODES);
1616 }
1617
1618 /* This page is already truncated */
1619 if (unlikely(ni.blk_addr == NULL_ADDR)) {
1620 ClearPageUptodate(page);
1621 dec_page_count(sbi, F2FS_DIRTY_NODES);
1622 up_read(&sbi->node_write);
1622 f2fs_up_read(&sbi->node_write);
1623 unlock_page(page);
1624 return 0;
1625 }
1626
1627 if (__is_valid_data_blkaddr(ni.blk_addr) &&
1628 !f2fs_is_valid_blkaddr(sbi, ni.blk_addr,
1629 DATA_GENERIC_ENHANCE)) {
1623 unlock_page(page);
1624 return 0;
1625 }
1626
1627 if (__is_valid_data_blkaddr(ni.blk_addr) &&
1628 !f2fs_is_valid_blkaddr(sbi, ni.blk_addr,
1629 DATA_GENERIC_ENHANCE)) {
1630 up_read(&sbi->node_write);
1630 f2fs_up_read(&sbi->node_write);
1631 goto redirty_out;
1632 }
1633
1634 if (atomic && !test_opt(sbi, NOBARRIER))
1635 fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
1636
1637 /* should add to global list before clearing PAGECACHE status */
1638 if (f2fs_in_warm_node_list(sbi, page)) {

--- 4 unchanged lines hidden (view full) ---

1643
1644 set_page_writeback(page);
1645 ClearPageError(page);
1646
1647 fio.old_blkaddr = ni.blk_addr;
1648 f2fs_do_write_node_page(nid, &fio);
1649 set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(page));
1650 dec_page_count(sbi, F2FS_DIRTY_NODES);
1631 goto redirty_out;
1632 }
1633
1634 if (atomic && !test_opt(sbi, NOBARRIER))
1635 fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
1636
1637 /* should add to global list before clearing PAGECACHE status */
1638 if (f2fs_in_warm_node_list(sbi, page)) {

--- 4 unchanged lines hidden (view full) ---

1643
1644 set_page_writeback(page);
1645 ClearPageError(page);
1646
1647 fio.old_blkaddr = ni.blk_addr;
1648 f2fs_do_write_node_page(nid, &fio);
1649 set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(page));
1650 dec_page_count(sbi, F2FS_DIRTY_NODES);
1651 up_read(&sbi->node_write);
1651 f2fs_up_read(&sbi->node_write);
1652
1653 if (wbc->for_reclaim) {
1654 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, NODE);
1655 submitted = NULL;
1656 }
1657
1658 unlock_page(page);
1659

--- 560 unchanged lines hidden (view full) ---

2220}
2221
2222bool f2fs_nat_bitmap_enabled(struct f2fs_sb_info *sbi)
2223{
2224 struct f2fs_nm_info *nm_i = NM_I(sbi);
2225 unsigned int i;
2226 bool ret = true;
2227
1652
1653 if (wbc->for_reclaim) {
1654 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, NODE);
1655 submitted = NULL;
1656 }
1657
1658 unlock_page(page);
1659

--- 560 unchanged lines hidden (view full) ---

2220}
2221
2222bool f2fs_nat_bitmap_enabled(struct f2fs_sb_info *sbi)
2223{
2224 struct f2fs_nm_info *nm_i = NM_I(sbi);
2225 unsigned int i;
2226 bool ret = true;
2227
2228 down_read(&nm_i->nat_tree_lock);
2228 f2fs_down_read(&nm_i->nat_tree_lock);
2229 for (i = 0; i < nm_i->nat_blocks; i++) {
2230 if (!test_bit_le(i, nm_i->nat_block_bitmap)) {
2231 ret = false;
2232 break;
2233 }
2234 }
2229 for (i = 0; i < nm_i->nat_blocks; i++) {
2230 if (!test_bit_le(i, nm_i->nat_block_bitmap)) {
2231 ret = false;
2232 break;
2233 }
2234 }
2235 up_read(&nm_i->nat_tree_lock);
2235 f2fs_up_read(&nm_i->nat_tree_lock);
2236
2237 return ret;
2238}
2239
2240static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid,
2241 bool set, bool build)
2242{
2243 struct f2fs_nm_info *nm_i = NM_I(sbi);

--- 166 unchanged lines hidden (view full) ---

2410}
2411
2412static void scan_free_nid_bits(struct f2fs_sb_info *sbi)
2413{
2414 struct f2fs_nm_info *nm_i = NM_I(sbi);
2415 unsigned int i, idx;
2416 nid_t nid;
2417
2236
2237 return ret;
2238}
2239
2240static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid,
2241 bool set, bool build)
2242{
2243 struct f2fs_nm_info *nm_i = NM_I(sbi);

--- 166 unchanged lines hidden (view full) ---

2410}
2411
2412static void scan_free_nid_bits(struct f2fs_sb_info *sbi)
2413{
2414 struct f2fs_nm_info *nm_i = NM_I(sbi);
2415 unsigned int i, idx;
2416 nid_t nid;
2417
2418 down_read(&nm_i->nat_tree_lock);
2418 f2fs_down_read(&nm_i->nat_tree_lock);
2419
2420 for (i = 0; i < nm_i->nat_blocks; i++) {
2421 if (!test_bit_le(i, nm_i->nat_block_bitmap))
2422 continue;
2423 if (!nm_i->free_nid_count[i])
2424 continue;
2425 for (idx = 0; idx < NAT_ENTRY_PER_BLOCK; idx++) {
2426 idx = find_next_bit_le(nm_i->free_nid_bitmap[i],

--- 6 unchanged lines hidden (view full) ---

2433
2434 if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS)
2435 goto out;
2436 }
2437 }
2438out:
2439 scan_curseg_cache(sbi);
2440
2419
2420 for (i = 0; i < nm_i->nat_blocks; i++) {
2421 if (!test_bit_le(i, nm_i->nat_block_bitmap))
2422 continue;
2423 if (!nm_i->free_nid_count[i])
2424 continue;
2425 for (idx = 0; idx < NAT_ENTRY_PER_BLOCK; idx++) {
2426 idx = find_next_bit_le(nm_i->free_nid_bitmap[i],

--- 6 unchanged lines hidden (view full) ---

2433
2434 if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS)
2435 goto out;
2436 }
2437 }
2438out:
2439 scan_curseg_cache(sbi);
2440
2441 up_read(&nm_i->nat_tree_lock);
2441 f2fs_up_read(&nm_i->nat_tree_lock);
2442}
2443
2444static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi,
2445 bool sync, bool mount)
2446{
2447 struct f2fs_nm_info *nm_i = NM_I(sbi);
2448 int i = 0, ret;
2449 nid_t nid = nm_i->next_scan_nid;

--- 18 unchanged lines hidden (view full) ---

2468 if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK)
2469 return 0;
2470 }
2471
2472 /* readahead nat pages to be scanned */
2473 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES,
2474 META_NAT, true);
2475
2442}
2443
2444static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi,
2445 bool sync, bool mount)
2446{
2447 struct f2fs_nm_info *nm_i = NM_I(sbi);
2448 int i = 0, ret;
2449 nid_t nid = nm_i->next_scan_nid;

--- 18 unchanged lines hidden (view full) ---

2468 if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK)
2469 return 0;
2470 }
2471
2472 /* readahead nat pages to be scanned */
2473 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES,
2474 META_NAT, true);
2475
2476 down_read(&nm_i->nat_tree_lock);
2476 f2fs_down_read(&nm_i->nat_tree_lock);
2477
2478 while (1) {
2479 if (!test_bit_le(NAT_BLOCK_OFFSET(nid),
2480 nm_i->nat_block_bitmap)) {
2481 struct page *page = get_current_nat_page(sbi, nid);
2482
2483 if (IS_ERR(page)) {
2484 ret = PTR_ERR(page);
2485 } else {
2486 ret = scan_nat_page(sbi, page, nid);
2487 f2fs_put_page(page, 1);
2488 }
2489
2490 if (ret) {
2477
2478 while (1) {
2479 if (!test_bit_le(NAT_BLOCK_OFFSET(nid),
2480 nm_i->nat_block_bitmap)) {
2481 struct page *page = get_current_nat_page(sbi, nid);
2482
2483 if (IS_ERR(page)) {
2484 ret = PTR_ERR(page);
2485 } else {
2486 ret = scan_nat_page(sbi, page, nid);
2487 f2fs_put_page(page, 1);
2488 }
2489
2490 if (ret) {
2491 up_read(&nm_i->nat_tree_lock);
2491 f2fs_up_read(&nm_i->nat_tree_lock);
2492 f2fs_err(sbi, "NAT is corrupt, run fsck to fix it");
2493 return ret;
2494 }
2495 }
2496
2497 nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
2498 if (unlikely(nid >= nm_i->max_nid))
2499 nid = 0;
2500
2501 if (++i >= FREE_NID_PAGES)
2502 break;
2503 }
2504
2505 /* go to the next free nat pages to find free nids abundantly */
2506 nm_i->next_scan_nid = nid;
2507
2508 /* find free nids from current sum_pages */
2509 scan_curseg_cache(sbi);
2510
2492 f2fs_err(sbi, "NAT is corrupt, run fsck to fix it");
2493 return ret;
2494 }
2495 }
2496
2497 nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
2498 if (unlikely(nid >= nm_i->max_nid))
2499 nid = 0;
2500
2501 if (++i >= FREE_NID_PAGES)
2502 break;
2503 }
2504
2505 /* go to the next free nat pages to find free nids abundantly */
2506 nm_i->next_scan_nid = nid;
2507
2508 /* find free nids from current sum_pages */
2509 scan_curseg_cache(sbi);
2510
2511 up_read(&nm_i->nat_tree_lock);
2511 f2fs_up_read(&nm_i->nat_tree_lock);
2512
2513 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid),
2514 nm_i->ra_nid_pages, META_NAT, false);
2515
2516 return 0;
2517}
2518
2519int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount)

--- 428 unchanged lines hidden (view full) ---

2948 __update_nat_bits(nm_i, nat_index, valid);
2949}
2950
2951void f2fs_enable_nat_bits(struct f2fs_sb_info *sbi)
2952{
2953 struct f2fs_nm_info *nm_i = NM_I(sbi);
2954 unsigned int nat_ofs;
2955
2512
2513 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid),
2514 nm_i->ra_nid_pages, META_NAT, false);
2515
2516 return 0;
2517}
2518
2519int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount)

--- 428 unchanged lines hidden (view full) ---

2948 __update_nat_bits(nm_i, nat_index, valid);
2949}
2950
2951void f2fs_enable_nat_bits(struct f2fs_sb_info *sbi)
2952{
2953 struct f2fs_nm_info *nm_i = NM_I(sbi);
2954 unsigned int nat_ofs;
2955
2956 down_read(&nm_i->nat_tree_lock);
2956 f2fs_down_read(&nm_i->nat_tree_lock);
2957
2958 for (nat_ofs = 0; nat_ofs < nm_i->nat_blocks; nat_ofs++) {
2959 unsigned int valid = 0, nid_ofs = 0;
2960
2961 /* handle nid zero due to it should never be used */
2962 if (unlikely(nat_ofs == 0)) {
2963 valid = 1;
2964 nid_ofs = 1;
2965 }
2966
2967 for (; nid_ofs < NAT_ENTRY_PER_BLOCK; nid_ofs++) {
2968 if (!test_bit_le(nid_ofs,
2969 nm_i->free_nid_bitmap[nat_ofs]))
2970 valid++;
2971 }
2972
2973 __update_nat_bits(nm_i, nat_ofs, valid);
2974 }
2975
2957
2958 for (nat_ofs = 0; nat_ofs < nm_i->nat_blocks; nat_ofs++) {
2959 unsigned int valid = 0, nid_ofs = 0;
2960
2961 /* handle nid zero due to it should never be used */
2962 if (unlikely(nat_ofs == 0)) {
2963 valid = 1;
2964 nid_ofs = 1;
2965 }
2966
2967 for (; nid_ofs < NAT_ENTRY_PER_BLOCK; nid_ofs++) {
2968 if (!test_bit_le(nid_ofs,
2969 nm_i->free_nid_bitmap[nat_ofs]))
2970 valid++;
2971 }
2972
2973 __update_nat_bits(nm_i, nat_ofs, valid);
2974 }
2975
2976 up_read(&nm_i->nat_tree_lock);
2976 f2fs_up_read(&nm_i->nat_tree_lock);
2977}
2978
2979static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
2980 struct nat_entry_set *set, struct cp_control *cpc)
2981{
2982 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2983 struct f2fs_journal *journal = curseg->journal;
2984 nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;

--- 81 unchanged lines hidden (view full) ---

3066 LIST_HEAD(sets);
3067 int err = 0;
3068
3069 /*
3070 * during unmount, let's flush nat_bits before checking
3071 * nat_cnt[DIRTY_NAT].
3072 */
3073 if (cpc->reason & CP_UMOUNT) {
2977}
2978
2979static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
2980 struct nat_entry_set *set, struct cp_control *cpc)
2981{
2982 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2983 struct f2fs_journal *journal = curseg->journal;
2984 nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;

--- 81 unchanged lines hidden (view full) ---

3066 LIST_HEAD(sets);
3067 int err = 0;
3068
3069 /*
3070 * during unmount, let's flush nat_bits before checking
3071 * nat_cnt[DIRTY_NAT].
3072 */
3073 if (cpc->reason & CP_UMOUNT) {
3074 down_write(&nm_i->nat_tree_lock);
3074 f2fs_down_write(&nm_i->nat_tree_lock);
3075 remove_nats_in_journal(sbi);
3075 remove_nats_in_journal(sbi);
3076 up_write(&nm_i->nat_tree_lock);
3076 f2fs_up_write(&nm_i->nat_tree_lock);
3077 }
3078
3079 if (!nm_i->nat_cnt[DIRTY_NAT])
3080 return 0;
3081
3077 }
3078
3079 if (!nm_i->nat_cnt[DIRTY_NAT])
3080 return 0;
3081
3082 down_write(&nm_i->nat_tree_lock);
3082 f2fs_down_write(&nm_i->nat_tree_lock);
3083
3084 /*
3085 * if there are no enough space in journal to store dirty nat
3086 * entries, remove all entries from journal and merge them
3087 * into nat entry set.
3088 */
3089 if (cpc->reason & CP_UMOUNT ||
3090 !__has_cursum_space(journal,

--- 12 unchanged lines hidden (view full) ---

3103
3104 /* flush dirty nats in nat entry set */
3105 list_for_each_entry_safe(set, tmp, &sets, set_list) {
3106 err = __flush_nat_entry_set(sbi, set, cpc);
3107 if (err)
3108 break;
3109 }
3110
3083
3084 /*
3085 * if there are no enough space in journal to store dirty nat
3086 * entries, remove all entries from journal and merge them
3087 * into nat entry set.
3088 */
3089 if (cpc->reason & CP_UMOUNT ||
3090 !__has_cursum_space(journal,

--- 12 unchanged lines hidden (view full) ---

3103
3104 /* flush dirty nats in nat entry set */
3105 list_for_each_entry_safe(set, tmp, &sets, set_list) {
3106 err = __flush_nat_entry_set(sbi, set, cpc);
3107 if (err)
3108 break;
3109 }
3110
3111 up_write(&nm_i->nat_tree_lock);
3111 f2fs_up_write(&nm_i->nat_tree_lock);
3112 /* Allow dirty nats by node block allocation in write_begin */
3113
3114 return err;
3115}
3116
3117static int __get_nat_bitmaps(struct f2fs_sb_info *sbi)
3118{
3119 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);

--- 103 unchanged lines hidden (view full) ---

3223 INIT_LIST_HEAD(&nm_i->free_nid_list);
3224 INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
3225 INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
3226 INIT_LIST_HEAD(&nm_i->nat_entries);
3227 spin_lock_init(&nm_i->nat_list_lock);
3228
3229 mutex_init(&nm_i->build_lock);
3230 spin_lock_init(&nm_i->nid_list_lock);
3112 /* Allow dirty nats by node block allocation in write_begin */
3113
3114 return err;
3115}
3116
3117static int __get_nat_bitmaps(struct f2fs_sb_info *sbi)
3118{
3119 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);

--- 103 unchanged lines hidden (view full) ---

3223 INIT_LIST_HEAD(&nm_i->free_nid_list);
3224 INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
3225 INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
3226 INIT_LIST_HEAD(&nm_i->nat_entries);
3227 spin_lock_init(&nm_i->nat_list_lock);
3228
3229 mutex_init(&nm_i->build_lock);
3230 spin_lock_init(&nm_i->nid_list_lock);
3231 init_rwsem(&nm_i->nat_tree_lock);
3231 init_f2fs_rwsem(&nm_i->nat_tree_lock);
3232
3233 nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
3234 nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
3235 version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
3236 nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
3237 GFP_KERNEL);
3238 if (!nm_i->nat_bitmap)
3239 return -ENOMEM;

--- 89 unchanged lines hidden (view full) ---

3329 spin_lock(&nm_i->nid_list_lock);
3330 }
3331 f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID]);
3332 f2fs_bug_on(sbi, nm_i->nid_cnt[PREALLOC_NID]);
3333 f2fs_bug_on(sbi, !list_empty(&nm_i->free_nid_list));
3334 spin_unlock(&nm_i->nid_list_lock);
3335
3336 /* destroy nat cache */
3232
3233 nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
3234 nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
3235 version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
3236 nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
3237 GFP_KERNEL);
3238 if (!nm_i->nat_bitmap)
3239 return -ENOMEM;

--- 89 unchanged lines hidden (view full) ---

3329 spin_lock(&nm_i->nid_list_lock);
3330 }
3331 f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID]);
3332 f2fs_bug_on(sbi, nm_i->nid_cnt[PREALLOC_NID]);
3333 f2fs_bug_on(sbi, !list_empty(&nm_i->free_nid_list));
3334 spin_unlock(&nm_i->nid_list_lock);
3335
3336 /* destroy nat cache */
3337 down_write(&nm_i->nat_tree_lock);
3337 f2fs_down_write(&nm_i->nat_tree_lock);
3338 while ((found = __gang_lookup_nat_cache(nm_i,
3339 nid, NATVEC_SIZE, natvec))) {
3340 unsigned idx;
3341
3342 nid = nat_get_nid(natvec[found - 1]) + 1;
3343 for (idx = 0; idx < found; idx++) {
3344 spin_lock(&nm_i->nat_list_lock);
3345 list_del(&natvec[idx]->list);

--- 13 unchanged lines hidden (view full) ---

3359 nid = setvec[found - 1]->set + 1;
3360 for (idx = 0; idx < found; idx++) {
3361 /* entry_cnt is not zero, when cp_error was occurred */
3362 f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list));
3363 radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set);
3364 kmem_cache_free(nat_entry_set_slab, setvec[idx]);
3365 }
3366 }
3338 while ((found = __gang_lookup_nat_cache(nm_i,
3339 nid, NATVEC_SIZE, natvec))) {
3340 unsigned idx;
3341
3342 nid = nat_get_nid(natvec[found - 1]) + 1;
3343 for (idx = 0; idx < found; idx++) {
3344 spin_lock(&nm_i->nat_list_lock);
3345 list_del(&natvec[idx]->list);

--- 13 unchanged lines hidden (view full) ---

3359 nid = setvec[found - 1]->set + 1;
3360 for (idx = 0; idx < found; idx++) {
3361 /* entry_cnt is not zero, when cp_error was occurred */
3362 f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list));
3363 radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set);
3364 kmem_cache_free(nat_entry_set_slab, setvec[idx]);
3365 }
3366 }
3367 up_write(&nm_i->nat_tree_lock);
3367 f2fs_up_write(&nm_i->nat_tree_lock);
3368
3369 kvfree(nm_i->nat_block_bitmap);
3370 if (nm_i->free_nid_bitmap) {
3371 int i;
3372
3373 for (i = 0; i < nm_i->nat_blocks; i++)
3374 kvfree(nm_i->free_nid_bitmap[i]);
3375 kvfree(nm_i->free_nid_bitmap);

--- 52 unchanged lines hidden --- 		3368
3369 kvfree(nm_i->nat_block_bitmap);
3370 if (nm_i->free_nid_bitmap) {
3371 int i;
3372
3373 for (i = 0; i < nm_i->nat_blocks; i++)
3374 kvfree(nm_i->free_nid_bitmap[i]);
3375 kvfree(nm_i->free_nid_bitmap);

--- 52 unchanged lines hidden ---