data.c (1c52283265a462a100ae63ddf58b4e5884acde86) data.c (e4544b63a7ee49e7fbebf35ece0a6acd3b9617ae)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 576 unchanged lines hidden (view full) ---

585}
586
587static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
588 enum page_type type, enum temp_type temp)
589{
590 enum page_type btype = PAGE_TYPE_OF_BIO(type);
591 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
592
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 576 unchanged lines hidden (view full) ---

585}
586
587static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
588 enum page_type type, enum temp_type temp)
589{
590 enum page_type btype = PAGE_TYPE_OF_BIO(type);
591 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
592
593 down_write(&io->io_rwsem);
593 f2fs_down_write(&io->io_rwsem);
594
595 /* change META to META_FLUSH in the checkpoint procedure */
596 if (type >= META_FLUSH) {
597 io->fio.type = META_FLUSH;
598 io->fio.op = REQ_OP_WRITE;
599 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
600 if (!test_opt(sbi, NOBARRIER))
601 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
602 }
603 __submit_merged_bio(io);
594
595 /* change META to META_FLUSH in the checkpoint procedure */
596 if (type >= META_FLUSH) {
597 io->fio.type = META_FLUSH;
598 io->fio.op = REQ_OP_WRITE;
599 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
600 if (!test_opt(sbi, NOBARRIER))
601 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
602 }
603 __submit_merged_bio(io);
604 up_write(&io->io_rwsem);
604 f2fs_up_write(&io->io_rwsem);
605}
606
607static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
608 struct inode *inode, struct page *page,
609 nid_t ino, enum page_type type, bool force)
610{
611 enum temp_type temp;
612 bool ret = true;
613
614 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
615 if (!force) {
616 enum page_type btype = PAGE_TYPE_OF_BIO(type);
617 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
618
605}
606
607static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
608 struct inode *inode, struct page *page,
609 nid_t ino, enum page_type type, bool force)
610{
611 enum temp_type temp;
612 bool ret = true;
613
614 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
615 if (!force) {
616 enum page_type btype = PAGE_TYPE_OF_BIO(type);
617 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
618
619 down_read(&io->io_rwsem);
619 f2fs_down_read(&io->io_rwsem);
620 ret = __has_merged_page(io->bio, inode, page, ino);
620 ret = __has_merged_page(io->bio, inode, page, ino);
621 up_read(&io->io_rwsem);
621 f2fs_up_read(&io->io_rwsem);
622 }
623 if (ret)
624 __f2fs_submit_merged_write(sbi, type, temp);
625
626 /* TODO: use HOT temp only for meta pages now. */
627 if (type >= META)
628 break;
629 }

--- 107 unchanged lines hidden (view full) ---

737
738 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL);
739 be->bio = bio;
740 bio_get(bio);
741
742 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
743 f2fs_bug_on(sbi, 1);
744
622 }
623 if (ret)
624 __f2fs_submit_merged_write(sbi, type, temp);
625
626 /* TODO: use HOT temp only for meta pages now. */
627 if (type >= META)
628 break;
629 }

--- 107 unchanged lines hidden (view full) ---

737
738 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL);
739 be->bio = bio;
740 bio_get(bio);
741
742 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
743 f2fs_bug_on(sbi, 1);
744
745 down_write(&io->bio_list_lock);
745 f2fs_down_write(&io->bio_list_lock);
746 list_add_tail(&be->list, &io->bio_list);
746 list_add_tail(&be->list, &io->bio_list);
747 up_write(&io->bio_list_lock);
747 f2fs_up_write(&io->bio_list_lock);
748}
749
750static void del_bio_entry(struct bio_entry *be)
751{
752 list_del(&be->list);
753 kmem_cache_free(bio_entry_slab, be);
754}
755

--- 5 unchanged lines hidden (view full) ---

761 bool found = false;
762 int ret = -EAGAIN;
763
764 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
765 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
766 struct list_head *head = &io->bio_list;
767 struct bio_entry *be;
768
748}
749
750static void del_bio_entry(struct bio_entry *be)
751{
752 list_del(&be->list);
753 kmem_cache_free(bio_entry_slab, be);
754}
755

--- 5 unchanged lines hidden (view full) ---

761 bool found = false;
762 int ret = -EAGAIN;
763
764 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
765 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
766 struct list_head *head = &io->bio_list;
767 struct bio_entry *be;
768
769 down_write(&io->bio_list_lock);
769 f2fs_down_write(&io->bio_list_lock);
770 list_for_each_entry(be, head, list) {
771 if (be->bio != *bio)
772 continue;
773
774 found = true;
775
776 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio,
777 *fio->last_block,

--- 7 unchanged lines hidden (view full) ---

785 break;
786 }
787
788 /* page can't be merged into bio; submit the bio */
789 del_bio_entry(be);
790 __submit_bio(sbi, *bio, DATA);
791 break;
792 }
770 list_for_each_entry(be, head, list) {
771 if (be->bio != *bio)
772 continue;
773
774 found = true;
775
776 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio,
777 *fio->last_block,

--- 7 unchanged lines hidden (view full) ---

785 break;
786 }
787
788 /* page can't be merged into bio; submit the bio */
789 del_bio_entry(be);
790 __submit_bio(sbi, *bio, DATA);
791 break;
792 }
793 up_write(&io->bio_list_lock);
793 f2fs_up_write(&io->bio_list_lock);
794 }
795
796 if (ret) {
797 bio_put(*bio);
798 *bio = NULL;
799 }
800
801 return ret;

--- 9 unchanged lines hidden (view full) ---

811 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
812 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
813 struct list_head *head = &io->bio_list;
814 struct bio_entry *be;
815
816 if (list_empty(head))
817 continue;
818
794 }
795
796 if (ret) {
797 bio_put(*bio);
798 *bio = NULL;
799 }
800
801 return ret;

--- 9 unchanged lines hidden (view full) ---

811 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
812 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
813 struct list_head *head = &io->bio_list;
814 struct bio_entry *be;
815
816 if (list_empty(head))
817 continue;
818
819 down_read(&io->bio_list_lock);
819 f2fs_down_read(&io->bio_list_lock);
820 list_for_each_entry(be, head, list) {
821 if (target)
822 found = (target == be->bio);
823 else
824 found = __has_merged_page(be->bio, NULL,
825 page, 0);
826 if (found)
827 break;
828 }
820 list_for_each_entry(be, head, list) {
821 if (target)
822 found = (target == be->bio);
823 else
824 found = __has_merged_page(be->bio, NULL,
825 page, 0);
826 if (found)
827 break;
828 }
829 up_read(&io->bio_list_lock);
829 f2fs_up_read(&io->bio_list_lock);
830
831 if (!found)
832 continue;
833
834 found = false;
835
830
831 if (!found)
832 continue;
833
834 found = false;
835
836 down_write(&io->bio_list_lock);
836 f2fs_down_write(&io->bio_list_lock);
837 list_for_each_entry(be, head, list) {
838 if (target)
839 found = (target == be->bio);
840 else
841 found = __has_merged_page(be->bio, NULL,
842 page, 0);
843 if (found) {
844 target = be->bio;
845 del_bio_entry(be);
846 break;
847 }
848 }
837 list_for_each_entry(be, head, list) {
838 if (target)
839 found = (target == be->bio);
840 else
841 found = __has_merged_page(be->bio, NULL,
842 page, 0);
843 if (found) {
844 target = be->bio;
845 del_bio_entry(be);
846 break;
847 }
848 }
849 up_write(&io->bio_list_lock);
849 f2fs_up_write(&io->bio_list_lock);
850 }
851
852 if (found)
853 __submit_bio(sbi, target, DATA);
854 if (bio && *bio) {
855 bio_put(*bio);
856 *bio = NULL;
857 }

--- 43 unchanged lines hidden (view full) ---

901{
902 struct f2fs_sb_info *sbi = fio->sbi;
903 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
904 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
905 struct page *bio_page;
906
907 f2fs_bug_on(sbi, is_read_io(fio->op));
908
850 }
851
852 if (found)
853 __submit_bio(sbi, target, DATA);
854 if (bio && *bio) {
855 bio_put(*bio);
856 *bio = NULL;
857 }

--- 43 unchanged lines hidden (view full) ---

901{
902 struct f2fs_sb_info *sbi = fio->sbi;
903 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
904 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
905 struct page *bio_page;
906
907 f2fs_bug_on(sbi, is_read_io(fio->op));
908
909 down_write(&io->io_rwsem);
909 f2fs_down_write(&io->io_rwsem);
910next:
911 if (fio->in_list) {
912 spin_lock(&io->io_lock);
913 if (list_empty(&io->io_list)) {
914 spin_unlock(&io->io_lock);
915 goto out;
916 }
917 fio = list_first_entry(&io->io_list,

--- 50 unchanged lines hidden (view full) ---

968 trace_f2fs_submit_page_write(fio->page, fio);
969skip:
970 if (fio->in_list)
971 goto next;
972out:
973 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
974 !f2fs_is_checkpoint_ready(sbi))
975 __submit_merged_bio(io);
910next:
911 if (fio->in_list) {
912 spin_lock(&io->io_lock);
913 if (list_empty(&io->io_list)) {
914 spin_unlock(&io->io_lock);
915 goto out;
916 }
917 fio = list_first_entry(&io->io_list,

--- 50 unchanged lines hidden (view full) ---

968 trace_f2fs_submit_page_write(fio->page, fio);
969skip:
970 if (fio->in_list)
971 goto next;
972out:
973 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
974 !f2fs_is_checkpoint_ready(sbi))
975 __submit_merged_bio(io);
976 up_write(&io->io_rwsem);
976 f2fs_up_write(&io->io_rwsem);
977}
978
979static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
980 unsigned nr_pages, unsigned op_flag,
981 pgoff_t first_idx, bool for_write)
982{
983 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
984 struct bio *bio;

--- 393 unchanged lines hidden (view full) ---

1378 f2fs_update_data_blkaddr(dn, dn->data_blkaddr);
1379 return 0;
1380}
1381
1382void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1383{
1384 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1385 if (lock)
977}
978
979static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
980 unsigned nr_pages, unsigned op_flag,
981 pgoff_t first_idx, bool for_write)
982{
983 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
984 struct bio *bio;

--- 393 unchanged lines hidden (view full) ---

1378 f2fs_update_data_blkaddr(dn, dn->data_blkaddr);
1379 return 0;
1380}
1381
1382void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1383{
1384 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1385 if (lock)
1386 down_read(&sbi->node_change);
1386 f2fs_down_read(&sbi->node_change);
1387 else
1387 else
1388 up_read(&sbi->node_change);
1388 f2fs_up_read(&sbi->node_change);
1389 } else {
1390 if (lock)
1391 f2fs_lock_op(sbi);
1392 else
1393 f2fs_unlock_op(sbi);
1394 }
1395}
1396

--- 1347 unchanged lines hidden (view full) ---

2744 /* Dentry/quota blocks are controlled by checkpoint */
2745 if (S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) {
2746 /*
2747 * We need to wait for node_write to avoid block allocation during
2748 * checkpoint. This can only happen to quota writes which can cause
2749 * the below discard race condition.
2750 */
2751 if (IS_NOQUOTA(inode))
1389 } else {
1390 if (lock)
1391 f2fs_lock_op(sbi);
1392 else
1393 f2fs_unlock_op(sbi);
1394 }
1395}
1396

--- 1347 unchanged lines hidden (view full) ---

2744 /* Dentry/quota blocks are controlled by checkpoint */
2745 if (S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) {
2746 /*
2747 * We need to wait for node_write to avoid block allocation during
2748 * checkpoint. This can only happen to quota writes which can cause
2749 * the below discard race condition.
2750 */
2751 if (IS_NOQUOTA(inode))
2752 down_read(&sbi->node_write);
2752 f2fs_down_read(&sbi->node_write);
2753
2754 fio.need_lock = LOCK_DONE;
2755 err = f2fs_do_write_data_page(&fio);
2756
2757 if (IS_NOQUOTA(inode))
2753
2754 fio.need_lock = LOCK_DONE;
2755 err = f2fs_do_write_data_page(&fio);
2756
2757 if (IS_NOQUOTA(inode))
2758 up_read(&sbi->node_write);
2758 f2fs_up_read(&sbi->node_write);
2759
2760 goto done;
2761 }
2762
2763 if (!wbc->for_reclaim)
2764 need_balance_fs = true;
2765 else if (has_not_enough_free_secs(sbi, 0, 0))
2766 goto redirty_out;

--- 441 unchanged lines hidden (view full) ---

3208{
3209 loff_t i_size = i_size_read(inode);
3210
3211 if (IS_NOQUOTA(inode))
3212 return;
3213
3214 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3215 if (to > i_size && !f2fs_verity_in_progress(inode)) {
2759
2760 goto done;
2761 }
2762
2763 if (!wbc->for_reclaim)
2764 need_balance_fs = true;
2765 else if (has_not_enough_free_secs(sbi, 0, 0))
2766 goto redirty_out;

--- 441 unchanged lines hidden (view full) ---

3208{
3209 loff_t i_size = i_size_read(inode);
3210
3211 if (IS_NOQUOTA(inode))
3212 return;
3213
3214 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3215 if (to > i_size && !f2fs_verity_in_progress(inode)) {
3216 down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3216 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3217 filemap_invalidate_lock(inode->i_mapping);
3218
3219 truncate_pagecache(inode, i_size);
3220 f2fs_truncate_blocks(inode, i_size, true);
3221
3222 filemap_invalidate_unlock(inode->i_mapping);
3217 filemap_invalidate_lock(inode->i_mapping);
3218
3219 truncate_pagecache(inode, i_size);
3220 f2fs_truncate_blocks(inode, i_size, true);
3221
3222 filemap_invalidate_unlock(inode->i_mapping);
3223 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3223 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3224 }
3225}
3226
3227static int prepare_write_begin(struct f2fs_sb_info *sbi,
3228 struct page *page, loff_t pos, unsigned len,
3229 block_t *blk_addr, bool *node_changed)
3230{
3231 struct inode *inode = page->mapping->host;

--- 484 unchanged lines hidden (view full) ---

3716{
3717 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3718 unsigned int blkofs;
3719 unsigned int blk_per_sec = BLKS_PER_SEC(sbi);
3720 unsigned int secidx = start_blk / blk_per_sec;
3721 unsigned int end_sec = secidx + blkcnt / blk_per_sec;
3722 int ret = 0;
3723
3224 }
3225}
3226
3227static int prepare_write_begin(struct f2fs_sb_info *sbi,
3228 struct page *page, loff_t pos, unsigned len,
3229 block_t *blk_addr, bool *node_changed)
3230{
3231 struct inode *inode = page->mapping->host;

--- 484 unchanged lines hidden (view full) ---

3716{
3717 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3718 unsigned int blkofs;
3719 unsigned int blk_per_sec = BLKS_PER_SEC(sbi);
3720 unsigned int secidx = start_blk / blk_per_sec;
3721 unsigned int end_sec = secidx + blkcnt / blk_per_sec;
3722 int ret = 0;
3723
3724 down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3724 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3725 filemap_invalidate_lock(inode->i_mapping);
3726
3727 set_inode_flag(inode, FI_ALIGNED_WRITE);
3728
3729 for (; secidx < end_sec; secidx++) {
3725 filemap_invalidate_lock(inode->i_mapping);
3726
3727 set_inode_flag(inode, FI_ALIGNED_WRITE);
3728
3729 for (; secidx < end_sec; secidx++) {
3730 down_write(&sbi->pin_sem);
3730 f2fs_down_write(&sbi->pin_sem);
3731
3732 f2fs_lock_op(sbi);
3733 f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false);
3734 f2fs_unlock_op(sbi);
3735
3736 set_inode_flag(inode, FI_DO_DEFRAG);
3737
3738 for (blkofs = 0; blkofs < blk_per_sec; blkofs++) {
3739 struct page *page;
3740 unsigned int blkidx = secidx * blk_per_sec + blkofs;
3741
3742 page = f2fs_get_lock_data_page(inode, blkidx, true);
3743 if (IS_ERR(page)) {
3731
3732 f2fs_lock_op(sbi);
3733 f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false);
3734 f2fs_unlock_op(sbi);
3735
3736 set_inode_flag(inode, FI_DO_DEFRAG);
3737
3738 for (blkofs = 0; blkofs < blk_per_sec; blkofs++) {
3739 struct page *page;
3740 unsigned int blkidx = secidx * blk_per_sec + blkofs;
3741
3742 page = f2fs_get_lock_data_page(inode, blkidx, true);
3743 if (IS_ERR(page)) {
3744 up_write(&sbi->pin_sem);
3744 f2fs_up_write(&sbi->pin_sem);
3745 ret = PTR_ERR(page);
3746 goto done;
3747 }
3748
3749 set_page_dirty(page);
3750 f2fs_put_page(page, 1);
3751 }
3752
3753 clear_inode_flag(inode, FI_DO_DEFRAG);
3754
3755 ret = filemap_fdatawrite(inode->i_mapping);
3756
3745 ret = PTR_ERR(page);
3746 goto done;
3747 }
3748
3749 set_page_dirty(page);
3750 f2fs_put_page(page, 1);
3751 }
3752
3753 clear_inode_flag(inode, FI_DO_DEFRAG);
3754
3755 ret = filemap_fdatawrite(inode->i_mapping);
3756
3757 up_write(&sbi->pin_sem);
3757 f2fs_up_write(&sbi->pin_sem);
3758
3759 if (ret)
3760 break;
3761 }
3762
3763done:
3764 clear_inode_flag(inode, FI_DO_DEFRAG);
3765 clear_inode_flag(inode, FI_ALIGNED_WRITE);
3766
3767 filemap_invalidate_unlock(inode->i_mapping);
3758
3759 if (ret)
3760 break;
3761 }
3762
3763done:
3764 clear_inode_flag(inode, FI_DO_DEFRAG);
3765 clear_inode_flag(inode, FI_ALIGNED_WRITE);
3766
3767 filemap_invalidate_unlock(inode->i_mapping);
3768 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3768 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3769
3770 return ret;
3771}
3772
3773static int check_swap_activate(struct swap_info_struct *sis,
3774 struct file *swap_file, sector_t *span)
3775{
3776 struct address_space *mapping = swap_file->f_mapping;

--- 302 unchanged lines hidden --- 		3769
3770 return ret;
3771}
3772
3773static int check_swap_activate(struct swap_info_struct *sis,
3774 struct file *swap_file, sector_t *span)
3775{
3776 struct address_space *mapping = swap_file->f_mapping;

--- 302 unchanged lines hidden ---