data.c (aaccf3c97418f169afdbb5855e9cbcbda34e90fd) data.c (0a4daae5ffea39f5015334e4d18a6a80b447cae4)
1// SPDX-License-Identifier: GPL-2.0
1/*
2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
2/*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/buffer_head.h>
14#include <linux/mpage.h>
15#include <linux/writeback.h>
16#include <linux/backing-dev.h>
17#include <linux/pagevec.h>

--- 103 unchanged lines hidden (view full) ---

121
122static bool f2fs_bio_post_read_required(struct bio *bio)
123{
124 return bio->bi_private && !bio->bi_status;
125}
126
127static void f2fs_read_end_io(struct bio *bio)
128{
7 */
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/buffer_head.h>
11#include <linux/mpage.h>
12#include <linux/writeback.h>
13#include <linux/backing-dev.h>
14#include <linux/pagevec.h>

--- 103 unchanged lines hidden (view full) ---

118
119static bool f2fs_bio_post_read_required(struct bio *bio)
120{
121 return bio->bi_private && !bio->bi_status;
122}
123
124static void f2fs_read_end_io(struct bio *bio)
125{
129 if (time_to_inject(F2FS_P_SB(bio_first_page_all(bio)), FAULT_IO)) {
130 f2fs_show_injection_info(FAULT_IO);
126 if (time_to_inject(F2FS_P_SB(bio_first_page_all(bio)),
127 FAULT_READ_IO)) {
128 f2fs_show_injection_info(FAULT_READ_IO);
131 bio->bi_status = BLK_STS_IOERR;
132 }
133
134 if (f2fs_bio_post_read_required(bio)) {
135 struct bio_post_read_ctx *ctx = bio->bi_private;
136
137 ctx->cur_step = STEP_INITIAL;
138 bio_post_read_processing(ctx);

--- 4 unchanged lines hidden (view full) ---

143}
144
145static void f2fs_write_end_io(struct bio *bio)
146{
147 struct f2fs_sb_info *sbi = bio->bi_private;
148 struct bio_vec *bvec;
149 int i;
150
129 bio->bi_status = BLK_STS_IOERR;
130 }
131
132 if (f2fs_bio_post_read_required(bio)) {
133 struct bio_post_read_ctx *ctx = bio->bi_private;
134
135 ctx->cur_step = STEP_INITIAL;
136 bio_post_read_processing(ctx);

--- 4 unchanged lines hidden (view full) ---

141}
142
143static void f2fs_write_end_io(struct bio *bio)
144{
145 struct f2fs_sb_info *sbi = bio->bi_private;
146 struct bio_vec *bvec;
147 int i;
148
149 if (time_to_inject(sbi, FAULT_WRITE_IO)) {
150 f2fs_show_injection_info(FAULT_WRITE_IO);
151 bio->bi_status = BLK_STS_IOERR;
152 }
153
151 bio_for_each_segment_all(bvec, bio, i) {
152 struct page *page = bvec->bv_page;
153 enum count_type type = WB_DATA_TYPE(page);
154
155 if (IS_DUMMY_WRITTEN_PAGE(page)) {
156 set_page_private(page, (unsigned long)NULL);
157 ClearPagePrivate(page);
158 unlock_page(page);

--- 369 unchanged lines hidden (view full) ---

528 io->last_block_in_bio = fio->new_blkaddr;
529 f2fs_trace_ios(fio, 0);
530
531 trace_f2fs_submit_page_write(fio->page, fio);
532skip:
533 if (fio->in_list)
534 goto next;
535out:
154 bio_for_each_segment_all(bvec, bio, i) {
155 struct page *page = bvec->bv_page;
156 enum count_type type = WB_DATA_TYPE(page);
157
158 if (IS_DUMMY_WRITTEN_PAGE(page)) {
159 set_page_private(page, (unsigned long)NULL);
160 ClearPagePrivate(page);
161 unlock_page(page);

--- 369 unchanged lines hidden (view full) ---

531 io->last_block_in_bio = fio->new_blkaddr;
532 f2fs_trace_ios(fio, 0);
533
534 trace_f2fs_submit_page_write(fio->page, fio);
535skip:
536 if (fio->in_list)
537 goto next;
538out:
539 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))
540 __submit_merged_bio(io);
536 up_write(&io->io_rwsem);
537}
538
539static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
540 unsigned nr_pages, unsigned op_flag)
541{
542 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
543 struct bio *bio;

--- 16 unchanged lines hidden (view full) ---

560 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
561 if (!ctx) {
562 bio_put(bio);
563 return ERR_PTR(-ENOMEM);
564 }
565 ctx->bio = bio;
566 ctx->enabled_steps = post_read_steps;
567 bio->bi_private = ctx;
541 up_write(&io->io_rwsem);
542}
543
544static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
545 unsigned nr_pages, unsigned op_flag)
546{
547 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
548 struct bio *bio;

--- 16 unchanged lines hidden (view full) ---

565 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
566 if (!ctx) {
567 bio_put(bio);
568 return ERR_PTR(-ENOMEM);
569 }
570 ctx->bio = bio;
571 ctx->enabled_steps = post_read_steps;
572 bio->bi_private = ctx;
568
569 /* wait the page to be moved by cleaning */
570 f2fs_wait_on_block_writeback(sbi, blkaddr);
571 }
572
573 return bio;
574}
575
576/* This can handle encryption stuffs */
577static int f2fs_submit_page_read(struct inode *inode, struct page *page,
578 block_t blkaddr)
579{
580 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0);
581
582 if (IS_ERR(bio))
583 return PTR_ERR(bio);
584
573 }
574
575 return bio;
576}
577
578/* This can handle encryption stuffs */
579static int f2fs_submit_page_read(struct inode *inode, struct page *page,
580 block_t blkaddr)
581{
582 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0);
583
584 if (IS_ERR(bio))
585 return PTR_ERR(bio);
586
587 /* wait for GCed page writeback via META_MAPPING */
588 f2fs_wait_on_block_writeback(inode, blkaddr);
589
585 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
586 bio_put(bio);
587 return -EFAULT;
588 }
589 __submit_bio(F2FS_I_SB(inode), bio, DATA);
590 return 0;
591}
592

--- 278 unchanged lines hidden (view full) ---

871}
872
873static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
874{
875 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
876 struct f2fs_summary sum;
877 struct node_info ni;
878 block_t old_blkaddr;
590 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
591 bio_put(bio);
592 return -EFAULT;
593 }
594 __submit_bio(F2FS_I_SB(inode), bio, DATA);
595 return 0;
596}
597

--- 278 unchanged lines hidden (view full) ---

876}
877
878static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
879{
880 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
881 struct f2fs_summary sum;
882 struct node_info ni;
883 block_t old_blkaddr;
879 pgoff_t fofs;
880 blkcnt_t count = 1;
881 int err;
882
883 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
884 return -EPERM;
885
886 err = f2fs_get_node_info(sbi, dn->nid, &ni);
887 if (err)

--- 12 unchanged lines hidden (view full) ---

900 old_blkaddr = dn->data_blkaddr;
901 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
902 &sum, seg_type, NULL, false);
903 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
904 invalidate_mapping_pages(META_MAPPING(sbi),
905 old_blkaddr, old_blkaddr);
906 f2fs_set_data_blkaddr(dn);
907
884 blkcnt_t count = 1;
885 int err;
886
887 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
888 return -EPERM;
889
890 err = f2fs_get_node_info(sbi, dn->nid, &ni);
891 if (err)

--- 12 unchanged lines hidden (view full) ---

904 old_blkaddr = dn->data_blkaddr;
905 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
906 &sum, seg_type, NULL, false);
907 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
908 invalidate_mapping_pages(META_MAPPING(sbi),
909 old_blkaddr, old_blkaddr);
910 f2fs_set_data_blkaddr(dn);
911
908 /* update i_size */
909 fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
910 dn->ofs_in_node;
911 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
912 f2fs_i_size_write(dn->inode,
913 ((loff_t)(fofs + 1) << PAGE_SHIFT));
912 /*
913 * i_size will be updated by direct_IO. Otherwise, we'll get stale
914 * data from unwritten block via dio_read.
915 */
914 return 0;
915}
916
917int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
918{
919 struct inode *inode = file_inode(iocb->ki_filp);
920 struct f2fs_map_blocks map;
921 int flag;

--- 149 unchanged lines hidden (view full) ---

1071 goto sync_out;
1072 }
1073 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1074 if (blkaddr == NULL_ADDR) {
1075 prealloc++;
1076 last_ofs_in_node = dn.ofs_in_node;
1077 }
1078 } else {
916 return 0;
917}
918
919int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
920{
921 struct inode *inode = file_inode(iocb->ki_filp);
922 struct f2fs_map_blocks map;
923 int flag;

--- 149 unchanged lines hidden (view full) ---

1073 goto sync_out;
1074 }
1075 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1076 if (blkaddr == NULL_ADDR) {
1077 prealloc++;
1078 last_ofs_in_node = dn.ofs_in_node;
1079 }
1080 } else {
1081 WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO &&
1082 flag != F2FS_GET_BLOCK_DIO);
1079 err = __allocate_data_block(&dn,
1080 map->m_seg_type);
1081 if (!err)
1082 set_inode_flag(inode, FI_APPEND_WRITE);
1083 }
1084 if (err)
1085 goto sync_out;
1086 map->m_flags |= F2FS_MAP_NEW;

--- 163 unchanged lines hidden (view full) ---

1250 flag, next_pgofs,
1251 NO_CHECK_TYPE);
1252}
1253
1254static int get_data_block_dio(struct inode *inode, sector_t iblock,
1255 struct buffer_head *bh_result, int create)
1256{
1257 return __get_data_block(inode, iblock, bh_result, create,
1083 err = __allocate_data_block(&dn,
1084 map->m_seg_type);
1085 if (!err)
1086 set_inode_flag(inode, FI_APPEND_WRITE);
1087 }
1088 if (err)
1089 goto sync_out;
1090 map->m_flags |= F2FS_MAP_NEW;

--- 163 unchanged lines hidden (view full) ---

1254 flag, next_pgofs,
1255 NO_CHECK_TYPE);
1256}
1257
1258static int get_data_block_dio(struct inode *inode, sector_t iblock,
1259 struct buffer_head *bh_result, int create)
1260{
1261 return __get_data_block(inode, iblock, bh_result, create,
1258 F2FS_GET_BLOCK_DEFAULT, NULL,
1262 F2FS_GET_BLOCK_DIO, NULL,
1259 f2fs_rw_hint_to_seg_type(
1260 inode->i_write_hint));
1261}
1262
1263static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1264 struct buffer_head *bh_result, int create)
1265{
1266 /* Block number less than F2FS MAX BLOCKS */

--- 286 unchanged lines hidden (view full) ---

1553 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
1554 is_readahead ? REQ_RAHEAD : 0);
1555 if (IS_ERR(bio)) {
1556 bio = NULL;
1557 goto set_error_page;
1558 }
1559 }
1560
1263 f2fs_rw_hint_to_seg_type(
1264 inode->i_write_hint));
1265}
1266
1267static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1268 struct buffer_head *bh_result, int create)
1269{
1270 /* Block number less than F2FS MAX BLOCKS */

--- 286 unchanged lines hidden (view full) ---

1557 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
1558 is_readahead ? REQ_RAHEAD : 0);
1559 if (IS_ERR(bio)) {
1560 bio = NULL;
1561 goto set_error_page;
1562 }
1563 }
1564
1565 /*
1566 * If the page is under writeback, we need to wait for
1567 * its completion to see the correct decrypted data.
1568 */
1569 f2fs_wait_on_block_writeback(inode, block_nr);
1570
1561 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1562 goto submit_and_realloc;
1563
1564 last_block_in_bio = block_nr;
1565 goto next_page;
1566set_error_page:
1567 SetPageError(page);
1568 zero_user_segment(page, 0, PAGE_SIZE);

--- 51 unchanged lines hidden (view full) ---

1620 struct inode *inode = fio->page->mapping->host;
1621 struct page *mpage;
1622 gfp_t gfp_flags = GFP_NOFS;
1623
1624 if (!f2fs_encrypted_file(inode))
1625 return 0;
1626
1627 /* wait for GCed page writeback via META_MAPPING */
1571 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1572 goto submit_and_realloc;
1573
1574 last_block_in_bio = block_nr;
1575 goto next_page;
1576set_error_page:
1577 SetPageError(page);
1578 zero_user_segment(page, 0, PAGE_SIZE);

--- 51 unchanged lines hidden (view full) ---

1630 struct inode *inode = fio->page->mapping->host;
1631 struct page *mpage;
1632 gfp_t gfp_flags = GFP_NOFS;
1633
1634 if (!f2fs_encrypted_file(inode))
1635 return 0;
1636
1637 /* wait for GCed page writeback via META_MAPPING */
1628 f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr);
1638 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
1629
1630retry_encrypt:
1631 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1632 PAGE_SIZE, 0, fio->page->index, gfp_flags);
1633 if (IS_ERR(fio->encrypted_page)) {
1634 /* flush pending IOs and wait for a while in the ENOMEM case */
1635 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1636 f2fs_flush_merged_writes(fio->sbi);

--- 740 unchanged lines hidden (view full) ---

2377 /* The page got truncated from under us */
2378 f2fs_put_page(page, 1);
2379 goto repeat;
2380 }
2381 }
2382
2383 f2fs_wait_on_page_writeback(page, DATA, false);
2384
1639
1640retry_encrypt:
1641 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1642 PAGE_SIZE, 0, fio->page->index, gfp_flags);
1643 if (IS_ERR(fio->encrypted_page)) {
1644 /* flush pending IOs and wait for a while in the ENOMEM case */
1645 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1646 f2fs_flush_merged_writes(fio->sbi);

--- 740 unchanged lines hidden (view full) ---

2387 /* The page got truncated from under us */
2388 f2fs_put_page(page, 1);
2389 goto repeat;
2390 }
2391 }
2392
2393 f2fs_wait_on_page_writeback(page, DATA, false);
2394
2385 /* wait for GCed page writeback via META_MAPPING */
2386 if (f2fs_post_read_required(inode))
2387 f2fs_wait_on_block_writeback(sbi, blkaddr);
2388
2389 if (len == PAGE_SIZE || PageUptodate(page))
2390 return 0;
2391
2392 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2393 zero_user_segment(page, len, PAGE_SIZE);
2394 return 0;
2395 }
2396

--- 337 unchanged lines hidden --- 		2395 if (len == PAGE_SIZE || PageUptodate(page))
2396 return 0;
2397
2398 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2399 zero_user_segment(page, len, PAGE_SIZE);
2400 return 0;
2401 }
2402

--- 337 unchanged lines hidden ---