data.c (9a87ffc99ec8eb8d35eed7c4f816d75f5cc9662e) data.c (b62e71be2110d8b52bf5faf3c3ed7ca1a0c113a5)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 79 unchanged lines hidden (view full) ---

88 return F2FS_RD_NODE;
89 }
90 return F2FS_RD_DATA;
91}
92
93/* postprocessing steps for read bios */
94enum bio_post_read_step {
95#ifdef CONFIG_FS_ENCRYPTION
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 79 unchanged lines hidden (view full) ---

88 return F2FS_RD_NODE;
89 }
90 return F2FS_RD_DATA;
91}
92
93/* postprocessing steps for read bios */
94enum bio_post_read_step {
95#ifdef CONFIG_FS_ENCRYPTION
96 STEP_DECRYPT = 1 << 0,
96 STEP_DECRYPT = BIT(0),
97#else
98 STEP_DECRYPT = 0, /* compile out the decryption-related code */
99#endif
100#ifdef CONFIG_F2FS_FS_COMPRESSION
97#else
98 STEP_DECRYPT = 0, /* compile out the decryption-related code */
99#endif
100#ifdef CONFIG_F2FS_FS_COMPRESSION
101 STEP_DECOMPRESS = 1 << 1,
101 STEP_DECOMPRESS = BIT(1),
102#else
103 STEP_DECOMPRESS = 0, /* compile out the decompression-related code */
104#endif
105#ifdef CONFIG_FS_VERITY
102#else
103 STEP_DECOMPRESS = 0, /* compile out the decompression-related code */
104#endif
105#ifdef CONFIG_FS_VERITY
106 STEP_VERITY = 1 << 2,
106 STEP_VERITY = BIT(2),
107#else
108 STEP_VERITY = 0, /* compile out the verity-related code */
109#endif
110};
111
112struct bio_post_read_ctx {
113 struct bio *bio;
114 struct f2fs_sb_info *sbi;

--- 300 unchanged lines hidden (view full) ---

415 for (i = 0; i < sbi->s_ndevs; i++)
416 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
417 return i;
418 return 0;
419}
420
421static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio)
422{
107#else
108 STEP_VERITY = 0, /* compile out the verity-related code */
109#endif
110};
111
112struct bio_post_read_ctx {
113 struct bio *bio;
114 struct f2fs_sb_info *sbi;

--- 300 unchanged lines hidden (view full) ---

415 for (i = 0; i < sbi->s_ndevs; i++)
416 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
417 return i;
418 return 0;
419}
420
421static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio)
422{
423 unsigned int temp_mask = (1 << NR_TEMP_TYPE) - 1;
423 unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0);
424 unsigned int fua_flag, meta_flag, io_flag;
425 blk_opf_t op_flags = 0;
426
427 if (fio->op != REQ_OP_WRITE)
428 return 0;
429 if (fio->type == DATA)
430 io_flag = fio->sbi->data_io_flag;
431 else if (fio->type == NODE)

--- 5 unchanged lines hidden (view full) ---

437 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask;
438
439 /*
440 * data/node io flag bits per temp:
441 * REQ_META | REQ_FUA |
442 * 5 | 4 | 3 | 2 | 1 | 0 |
443 * Cold | Warm | Hot | Cold | Warm | Hot |
444 */
424 unsigned int fua_flag, meta_flag, io_flag;
425 blk_opf_t op_flags = 0;
426
427 if (fio->op != REQ_OP_WRITE)
428 return 0;
429 if (fio->type == DATA)
430 io_flag = fio->sbi->data_io_flag;
431 else if (fio->type == NODE)

--- 5 unchanged lines hidden (view full) ---

437 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask;
438
439 /*
440 * data/node io flag bits per temp:
441 * REQ_META | REQ_FUA |
442 * 5 | 4 | 3 | 2 | 1 | 0 |
443 * Cold | Warm | Hot | Cold | Warm | Hot |
444 */
445 if ((1 << fio->temp) & meta_flag)
445 if (BIT(fio->temp) & meta_flag)
446 op_flags |= REQ_META;
446 op_flags |= REQ_META;
447 if ((1 << fio->temp) & fua_flag)
447 if (BIT(fio->temp) & fua_flag)
448 op_flags |= REQ_FUA;
449 return op_flags;
450}
451
452static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
453{
454 struct f2fs_sb_info *sbi = fio->sbi;
455 struct block_device *bdev;

--- 413 unchanged lines hidden (view full) ---

869
870void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
871 struct bio **bio, struct page *page)
872{
873 enum temp_type temp;
874 bool found = false;
875 struct bio *target = bio ? *bio : NULL;
876
448 op_flags |= REQ_FUA;
449 return op_flags;
450}
451
452static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
453{
454 struct f2fs_sb_info *sbi = fio->sbi;
455 struct block_device *bdev;

--- 413 unchanged lines hidden (view full) ---

869
870void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
871 struct bio **bio, struct page *page)
872{
873 enum temp_type temp;
874 bool found = false;
875 struct bio *target = bio ? *bio : NULL;
876
877 f2fs_bug_on(sbi, !target && !page);
878
877 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
878 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
879 struct list_head *head = &io->bio_list;
880 struct bio_entry *be;
881
882 if (list_empty(head))
883 continue;
884

--- 1345 unchanged lines hidden (view full) ---

2230 if (!from_dnode)
2231 goto skip_reading_dnode;
2232
2233 set_new_dnode(&dn, inode, NULL, NULL, 0);
2234 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
2235 if (ret)
2236 goto out;
2237
879 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
880 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
881 struct list_head *head = &io->bio_list;
882 struct bio_entry *be;
883
884 if (list_empty(head))
885 continue;
886

--- 1345 unchanged lines hidden (view full) ---

2232 if (!from_dnode)
2233 goto skip_reading_dnode;
2234
2235 set_new_dnode(&dn, inode, NULL, NULL, 0);
2236 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
2237 if (ret)
2238 goto out;
2239
2240 if (unlikely(f2fs_cp_error(sbi))) {
2241 ret = -EIO;
2242 goto out_put_dnode;
2243 }
2238 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR);
2239
2240skip_reading_dnode:
2241 for (i = 1; i < cc->cluster_size; i++) {
2242 block_t blkaddr;
2243
2244 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page,
2245 dn.ofs_in_node + i) :

--- 547 unchanged lines hidden (view full) ---

2793
2794 /* we should bypass data pages to proceed the kworker jobs */
2795 if (unlikely(f2fs_cp_error(sbi))) {
2796 mapping_set_error(page->mapping, -EIO);
2797 /*
2798 * don't drop any dirty dentry pages for keeping lastest
2799 * directory structure.
2800 */
2244 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR);
2245
2246skip_reading_dnode:
2247 for (i = 1; i < cc->cluster_size; i++) {
2248 block_t blkaddr;
2249
2250 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page,
2251 dn.ofs_in_node + i) :

--- 547 unchanged lines hidden (view full) ---

2799
2800 /* we should bypass data pages to proceed the kworker jobs */
2801 if (unlikely(f2fs_cp_error(sbi))) {
2802 mapping_set_error(page->mapping, -EIO);
2803 /*
2804 * don't drop any dirty dentry pages for keeping lastest
2805 * directory structure.
2806 */
2801 if (S_ISDIR(inode->i_mode))
2807 if (S_ISDIR(inode->i_mode) &&
2808 !is_sbi_flag_set(sbi, SBI_IS_CLOSE))
2802 goto redirty_out;
2809 goto redirty_out;
2810
2811 /* keep data pages in remount-ro mode */
2812 if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY)
2813 goto redirty_out;
2803 goto out;
2804 }
2805
2806 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2807 goto redirty_out;
2808
2809 if (page->index < end_index ||
2810 f2fs_verity_in_progress(inode) ||

--- 82 unchanged lines hidden (view full) ---

2893 }
2894 unlock_page(page);
2895 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2896 !F2FS_I(inode)->wb_task && allow_balance)
2897 f2fs_balance_fs(sbi, need_balance_fs);
2898
2899 if (unlikely(f2fs_cp_error(sbi))) {
2900 f2fs_submit_merged_write(sbi, DATA);
2814 goto out;
2815 }
2816
2817 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2818 goto redirty_out;
2819
2820 if (page->index < end_index ||
2821 f2fs_verity_in_progress(inode) ||

--- 82 unchanged lines hidden (view full) ---

2904 }
2905 unlock_page(page);
2906 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2907 !F2FS_I(inode)->wb_task && allow_balance)
2908 f2fs_balance_fs(sbi, need_balance_fs);
2909
2910 if (unlikely(f2fs_cp_error(sbi))) {
2911 f2fs_submit_merged_write(sbi, DATA);
2901 f2fs_submit_merged_ipu_write(sbi, bio, NULL);
2912 if (bio && *bio)
2913 f2fs_submit_merged_ipu_write(sbi, bio, NULL);
2902 submitted = NULL;
2903 }
2904
2905 if (submitted)
2906 *submitted = fio.submitted;
2907
2908 return 0;
2909

--- 208 unchanged lines hidden (view full) ---

3118 }
3119
3120 if (!folio_test_dirty(folio)) {
3121 /* someone wrote it for us */
3122 goto continue_unlock;
3123 }
3124
3125 if (folio_test_writeback(folio)) {
2914 submitted = NULL;
2915 }
2916
2917 if (submitted)
2918 *submitted = fio.submitted;
2919
2920 return 0;
2921

--- 208 unchanged lines hidden (view full) ---

3130 }
3131
3132 if (!folio_test_dirty(folio)) {
3133 /* someone wrote it for us */
3134 goto continue_unlock;
3135 }
3136
3137 if (folio_test_writeback(folio)) {
3126 if (wbc->sync_mode != WB_SYNC_NONE)
3127 f2fs_wait_on_page_writeback(
3128 &folio->page,
3129 DATA, true, true);
3130 else
3138 if (wbc->sync_mode == WB_SYNC_NONE)
3131 goto continue_unlock;
3139 goto continue_unlock;
3140 f2fs_wait_on_page_writeback(&folio->page, DATA, true, true);
3132 }
3133
3134 if (!folio_clear_dirty_for_io(folio))
3135 goto continue_unlock;
3136
3137#ifdef CONFIG_F2FS_FS_COMPRESSION
3138 if (f2fs_compressed_file(inode)) {
3139 folio_get(folio);

--- 341 unchanged lines hidden (view full) ---

3481
3482unlock_out:
3483 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO);
3484 return err;
3485}
3486
3487static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi,
3488 struct page *page, loff_t pos, unsigned int len,
3141 }
3142
3143 if (!folio_clear_dirty_for_io(folio))
3144 goto continue_unlock;
3145
3146#ifdef CONFIG_F2FS_FS_COMPRESSION
3147 if (f2fs_compressed_file(inode)) {
3148 folio_get(folio);

--- 341 unchanged lines hidden (view full) ---

3490
3491unlock_out:
3492 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO);
3493 return err;
3494}
3495
3496static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi,
3497 struct page *page, loff_t pos, unsigned int len,
3489 block_t *blk_addr, bool *node_changed)
3498 block_t *blk_addr, bool *node_changed, bool *use_cow)
3490{
3491 struct inode *inode = page->mapping->host;
3492 struct inode *cow_inode = F2FS_I(inode)->cow_inode;
3493 pgoff_t index = page->index;
3494 int err = 0;
3495 block_t ori_blk_addr = NULL_ADDR;
3496
3497 /* If pos is beyond the end of file, reserve a new block in COW inode */
3498 if ((pos & PAGE_MASK) >= i_size_read(inode))
3499 goto reserve_block;
3500
3501 /* Look for the block in COW inode first */
3502 err = __find_data_block(cow_inode, index, blk_addr);
3499{
3500 struct inode *inode = page->mapping->host;
3501 struct inode *cow_inode = F2FS_I(inode)->cow_inode;
3502 pgoff_t index = page->index;
3503 int err = 0;
3504 block_t ori_blk_addr = NULL_ADDR;
3505
3506 /* If pos is beyond the end of file, reserve a new block in COW inode */
3507 if ((pos & PAGE_MASK) >= i_size_read(inode))
3508 goto reserve_block;
3509
3510 /* Look for the block in COW inode first */
3511 err = __find_data_block(cow_inode, index, blk_addr);
3503 if (err)
3512 if (err) {
3504 return err;
3513 return err;
3505 else if (*blk_addr != NULL_ADDR)
3514 } else if (*blk_addr != NULL_ADDR) {
3515 *use_cow = true;
3506 return 0;
3516 return 0;
3517 }
3507
3508 if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE))
3509 goto reserve_block;
3510
3511 /* Look for the block in the original inode */
3512 err = __find_data_block(inode, index, &ori_blk_addr);
3513 if (err)
3514 return err;

--- 13 unchanged lines hidden (view full) ---

3528static int f2fs_write_begin(struct file *file, struct address_space *mapping,
3529 loff_t pos, unsigned len, struct page **pagep, void **fsdata)
3530{
3531 struct inode *inode = mapping->host;
3532 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3533 struct page *page = NULL;
3534 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
3535 bool need_balance = false;
3518
3519 if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE))
3520 goto reserve_block;
3521
3522 /* Look for the block in the original inode */
3523 err = __find_data_block(inode, index, &ori_blk_addr);
3524 if (err)
3525 return err;

--- 13 unchanged lines hidden (view full) ---

3539static int f2fs_write_begin(struct file *file, struct address_space *mapping,
3540 loff_t pos, unsigned len, struct page **pagep, void **fsdata)
3541{
3542 struct inode *inode = mapping->host;
3543 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3544 struct page *page = NULL;
3545 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
3546 bool need_balance = false;
3547 bool use_cow = false;
3536 block_t blkaddr = NULL_ADDR;
3537 int err = 0;
3538
3539 trace_f2fs_write_begin(inode, pos, len);
3540
3541 if (!f2fs_is_checkpoint_ready(sbi)) {
3542 err = -ENOSPC;
3543 goto fail;

--- 43 unchanged lines hidden (view full) ---

3587 }
3588
3589 /* TODO: cluster can be compressed due to race with .writepage */
3590
3591 *pagep = page;
3592
3593 if (f2fs_is_atomic_file(inode))
3594 err = prepare_atomic_write_begin(sbi, page, pos, len,
3548 block_t blkaddr = NULL_ADDR;
3549 int err = 0;
3550
3551 trace_f2fs_write_begin(inode, pos, len);
3552
3553 if (!f2fs_is_checkpoint_ready(sbi)) {
3554 err = -ENOSPC;
3555 goto fail;

--- 43 unchanged lines hidden (view full) ---

3599 }
3600
3601 /* TODO: cluster can be compressed due to race with .writepage */
3602
3603 *pagep = page;
3604
3605 if (f2fs_is_atomic_file(inode))
3606 err = prepare_atomic_write_begin(sbi, page, pos, len,
3595 &blkaddr, &need_balance);
3607 &blkaddr, &need_balance, &use_cow);
3596 else
3597 err = prepare_write_begin(sbi, page, pos, len,
3598 &blkaddr, &need_balance);
3599 if (err)
3600 goto fail;
3601
3602 if (need_balance && !IS_NOQUOTA(inode) &&
3603 has_not_enough_free_secs(sbi, 0, 0)) {

--- 23 unchanged lines hidden (view full) ---

3627 SetPageUptodate(page);
3628 } else {
3629 if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
3630 DATA_GENERIC_ENHANCE_READ)) {
3631 err = -EFSCORRUPTED;
3632 f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
3633 goto fail;
3634 }
3608 else
3609 err = prepare_write_begin(sbi, page, pos, len,
3610 &blkaddr, &need_balance);
3611 if (err)
3612 goto fail;
3613
3614 if (need_balance && !IS_NOQUOTA(inode) &&
3615 has_not_enough_free_secs(sbi, 0, 0)) {

--- 23 unchanged lines hidden (view full) ---

3639 SetPageUptodate(page);
3640 } else {
3641 if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
3642 DATA_GENERIC_ENHANCE_READ)) {
3643 err = -EFSCORRUPTED;
3644 f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
3645 goto fail;
3646 }
3635 err = f2fs_submit_page_read(inode, page, blkaddr, 0, true);
3647 err = f2fs_submit_page_read(use_cow ?
3648 F2FS_I(inode)->cow_inode : inode, page,
3649 blkaddr, 0, true);
3636 if (err)
3637 goto fail;
3638
3639 lock_page(page);
3640 if (unlikely(page->mapping != mapping)) {
3641 f2fs_put_page(page, 1);
3642 goto repeat;
3643 }

--- 76 unchanged lines hidden (view full) ---

3720 dec_page_count(sbi, F2FS_DIRTY_META);
3721 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
3722 dec_page_count(sbi, F2FS_DIRTY_NODES);
3723 } else {
3724 inode_dec_dirty_pages(inode);
3725 f2fs_remove_dirty_inode(inode);
3726 }
3727 }
3650 if (err)
3651 goto fail;
3652
3653 lock_page(page);
3654 if (unlikely(page->mapping != mapping)) {
3655 f2fs_put_page(page, 1);
3656 goto repeat;
3657 }

--- 76 unchanged lines hidden (view full) ---

3734 dec_page_count(sbi, F2FS_DIRTY_META);
3735 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
3736 dec_page_count(sbi, F2FS_DIRTY_NODES);
3737 } else {
3738 inode_dec_dirty_pages(inode);
3739 f2fs_remove_dirty_inode(inode);
3740 }
3741 }
3728
3729 clear_page_private_reference(&folio->page);
3730 clear_page_private_gcing(&folio->page);
3731
3732 if (test_opt(sbi, COMPRESS_CACHE) &&
3733 inode->i_ino == F2FS_COMPRESS_INO(sbi))
3734 clear_page_private_data(&folio->page);
3735
3736 folio_detach_private(folio);
3742 clear_page_private_all(&folio->page);
3737}
3738
3739bool f2fs_release_folio(struct folio *folio, gfp_t wait)
3740{
3743}
3744
3745bool f2fs_release_folio(struct folio *folio, gfp_t wait)
3746{
3741 struct f2fs_sb_info *sbi;
3742
3743 /* If this is dirty folio, keep private data */
3744 if (folio_test_dirty(folio))
3745 return false;
3746
3747 /* If this is dirty folio, keep private data */
3748 if (folio_test_dirty(folio))
3749 return false;
3750
3747 sbi = F2FS_M_SB(folio->mapping);
3748 if (test_opt(sbi, COMPRESS_CACHE)) {
3749 struct inode *inode = folio->mapping->host;
3750
3751 if (inode->i_ino == F2FS_COMPRESS_INO(sbi))
3752 clear_page_private_data(&folio->page);
3753 }
3754
3755 clear_page_private_reference(&folio->page);
3756 clear_page_private_gcing(&folio->page);
3757
3758 folio_detach_private(folio);
3751 clear_page_private_all(&folio->page);
3759 return true;
3760}
3761
3762static bool f2fs_dirty_data_folio(struct address_space *mapping,
3763 struct folio *folio)
3764{
3765 struct inode *inode = mapping->host;
3766

--- 452 unchanged lines hidden --- 		3752 return true;
3753}
3754
3755static bool f2fs_dirty_data_folio(struct address_space *mapping,
3756 struct folio *folio)
3757{
3758 struct inode *inode = mapping->host;
3759

--- 452 unchanged lines hidden ---