data.c (da8c7fecc9c7ba91b6d5ff5726189f269686a40c) data.c (bc29835a9d4860df93a663d659e07dfdd8b4f629)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 493 unchanged lines hidden (view full) ---

502 * read/write raw data without encryption.
503 */
504 if (fio && fio->encrypted_page)
505 return !bio_has_crypt_ctx(bio);
506
507 return fscrypt_mergeable_bio(bio, inode, next_idx);
508}
509
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 493 unchanged lines hidden (view full) ---

502 * read/write raw data without encryption.
503 */
504 if (fio && fio->encrypted_page)
505 return !bio_has_crypt_ctx(bio);
506
507 return fscrypt_mergeable_bio(bio, inode, next_idx);
508}
509
510static inline void __submit_bio(struct f2fs_sb_info *sbi,
511 struct bio *bio, enum page_type type)
510void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
511 enum page_type type)
512{
512{
513 if (!is_read_io(bio_op(bio))) {
514 unsigned int start;
513 WARN_ON_ONCE(!is_read_io(bio_op(bio)));
514 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
515
515
516 if (type != DATA && type != NODE)
517 goto submit_io;
516 iostat_update_submit_ctx(bio, type);
517 submit_bio(bio);
518}
518
519
519 if (f2fs_lfs_mode(sbi) && current->plug)
520 blk_finish_plug(current->plug);
520static void f2fs_align_write_bio(struct f2fs_sb_info *sbi, struct bio *bio)
521{
522 unsigned int start =
523 (bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS) % F2FS_IO_SIZE(sbi);
521
524
522 if (!F2FS_IO_ALIGNED(sbi))
523 goto submit_io;
525 if (start == 0)
526 return;
524
527
525 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
526 start %= F2FS_IO_SIZE(sbi);
528 /* fill dummy pages */
529 for (; start < F2FS_IO_SIZE(sbi); start++) {
530 struct page *page =
531 mempool_alloc(sbi->write_io_dummy,
532 GFP_NOIO | __GFP_NOFAIL);
533 f2fs_bug_on(sbi, !page);
527
534
528 if (start == 0)
529 goto submit_io;
535 lock_page(page);
530
536
531 /* fill dummy pages */
532 for (; start < F2FS_IO_SIZE(sbi); start++) {
533 struct page *page =
534 mempool_alloc(sbi->write_io_dummy,
535 GFP_NOIO | __GFP_NOFAIL);
536 f2fs_bug_on(sbi, !page);
537 zero_user_segment(page, 0, PAGE_SIZE);
538 set_page_private_dummy(page);
537
539
538 lock_page(page);
540 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
541 f2fs_bug_on(sbi, 1);
542 }
543}
539
544
540 zero_user_segment(page, 0, PAGE_SIZE);
541 set_page_private_dummy(page);
545static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio,
546 enum page_type type)
547{
548 WARN_ON_ONCE(is_read_io(bio_op(bio)));
542
549
543 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
544 f2fs_bug_on(sbi, 1);
550 if (type == DATA || type == NODE) {
551 if (f2fs_lfs_mode(sbi) && current->plug)
552 blk_finish_plug(current->plug);
553
554 if (F2FS_IO_ALIGNED(sbi)) {
555 f2fs_align_write_bio(sbi, bio);
556 /*
557 * In the NODE case, we lose next block address chain.
558 * So, we need to do checkpoint in f2fs_sync_file.
559 */
560 if (type == NODE)
561 set_sbi_flag(sbi, SBI_NEED_CP);
545 }
562 }
546 /*
547 * In the NODE case, we lose next block address chain. So, we
548 * need to do checkpoint in f2fs_sync_file.
549 */
550 if (type == NODE)
551 set_sbi_flag(sbi, SBI_NEED_CP);
552 }
563 }
553submit_io:
554 if (is_read_io(bio_op(bio)))
555 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
556 else
557 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
558
564
565 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
559 iostat_update_submit_ctx(bio, type);
560 submit_bio(bio);
561}
562
566 iostat_update_submit_ctx(bio, type);
567 submit_bio(bio);
568}
569
563void f2fs_submit_bio(struct f2fs_sb_info *sbi,
564 struct bio *bio, enum page_type type)
565{
566 __submit_bio(sbi, bio, type);
567}
568
569static void __submit_merged_bio(struct f2fs_bio_info *io)
570{
571 struct f2fs_io_info *fio = &io->fio;
572
573 if (!io->bio)
574 return;
575
570static void __submit_merged_bio(struct f2fs_bio_info *io)
571{
572 struct f2fs_io_info *fio = &io->fio;
573
574 if (!io->bio)
575 return;
576
576 if (is_read_io(fio->op))
577 if (is_read_io(fio->op)) {
577 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
578 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
578 else
579 f2fs_submit_read_bio(io->sbi, io->bio, fio->type);
580 } else {
579 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
581 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
580
581 __submit_bio(io->sbi, io->bio, fio->type);
582 f2fs_submit_write_bio(io->sbi, io->bio, fio->type);
583 }
582 io->bio = NULL;
583}
584
585static bool __has_merged_page(struct bio *bio, struct inode *inode,
586 struct page *page, nid_t ino)
587{
588 struct bio_vec *bvec;
589 struct bvec_iter_all iter_all;

--- 151 unchanged lines hidden (view full) ---

741 }
742
743 if (fio->io_wbc && !is_read_io(fio->op))
744 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
745
746 inc_page_count(fio->sbi, is_read_io(fio->op) ?
747 __read_io_type(page) : WB_DATA_TYPE(fio->page));
748
584 io->bio = NULL;
585}
586
587static bool __has_merged_page(struct bio *bio, struct inode *inode,
588 struct page *page, nid_t ino)
589{
590 struct bio_vec *bvec;
591 struct bvec_iter_all iter_all;

--- 151 unchanged lines hidden (view full) ---

743 }
744
745 if (fio->io_wbc && !is_read_io(fio->op))
746 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
747
748 inc_page_count(fio->sbi, is_read_io(fio->op) ?
749 __read_io_type(page) : WB_DATA_TYPE(fio->page));
750
749 __submit_bio(fio->sbi, bio, fio->type);
751 if (is_read_io(bio_op(bio)))
752 f2fs_submit_read_bio(fio->sbi, bio, fio->type);
753 else
754 f2fs_submit_write_bio(fio->sbi, bio, fio->type);
750 return 0;
751}
752
753static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
754 block_t last_blkaddr, block_t cur_blkaddr)
755{
756 if (unlikely(sbi->max_io_bytes &&
757 bio->bi_iter.bi_size >= sbi->max_io_bytes))

--- 85 unchanged lines hidden (view full) ---

843 bio_add_page(*bio, page, PAGE_SIZE, 0) ==
844 PAGE_SIZE) {
845 ret = 0;
846 break;
847 }
848
849 /* page can't be merged into bio; submit the bio */
850 del_bio_entry(be);
755 return 0;
756}
757
758static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
759 block_t last_blkaddr, block_t cur_blkaddr)
760{
761 if (unlikely(sbi->max_io_bytes &&
762 bio->bi_iter.bi_size >= sbi->max_io_bytes))

--- 85 unchanged lines hidden (view full) ---

848 bio_add_page(*bio, page, PAGE_SIZE, 0) ==
849 PAGE_SIZE) {
850 ret = 0;
851 break;
852 }
853
854 /* page can't be merged into bio; submit the bio */
855 del_bio_entry(be);
851 __submit_bio(sbi, *bio, DATA);
856 f2fs_submit_write_bio(sbi, *bio, DATA);
852 break;
853 }
854 f2fs_up_write(&io->bio_list_lock);
855 }
856
857 if (ret) {
858 bio_put(*bio);
859 *bio = NULL;

--- 46 unchanged lines hidden (view full) ---

906 del_bio_entry(be);
907 break;
908 }
909 }
910 f2fs_up_write(&io->bio_list_lock);
911 }
912
913 if (found)
857 break;
858 }
859 f2fs_up_write(&io->bio_list_lock);
860 }
861
862 if (ret) {
863 bio_put(*bio);
864 *bio = NULL;

--- 46 unchanged lines hidden (view full) ---

911 del_bio_entry(be);
912 break;
913 }
914 }
915 f2fs_up_write(&io->bio_list_lock);
916 }
917
918 if (found)
914 __submit_bio(sbi, target, DATA);
919 f2fs_submit_write_bio(sbi, target, DATA);
915 if (bio && *bio) {
916 bio_put(*bio);
917 *bio = NULL;
918 }
919}
920
921int f2fs_merge_page_bio(struct f2fs_io_info *fio)
922{

--- 179 unchanged lines hidden (view full) ---

1102 f2fs_wait_on_block_writeback(inode, blkaddr);
1103
1104 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
1105 bio_put(bio);
1106 return -EFAULT;
1107 }
1108 inc_page_count(sbi, F2FS_RD_DATA);
1109 f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE);
920 if (bio && *bio) {
921 bio_put(*bio);
922 *bio = NULL;
923 }
924}
925
926int f2fs_merge_page_bio(struct f2fs_io_info *fio)
927{

--- 179 unchanged lines hidden (view full) ---

1107 f2fs_wait_on_block_writeback(inode, blkaddr);
1108
1109 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
1110 bio_put(bio);
1111 return -EFAULT;
1112 }
1113 inc_page_count(sbi, F2FS_RD_DATA);
1114 f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE);
1110 __submit_bio(sbi, bio, DATA);
1115 f2fs_submit_read_bio(sbi, bio, DATA);
1111 return 0;
1112}
1113
1114static void __set_data_blkaddr(struct dnode_of_data *dn)
1115{
1116 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
1117 __le32 *addr_array;
1118 int base = 0;

--- 1013 unchanged lines hidden (view full) ---

2132 /*
2133 * This page will go to BIO. Do we need to send this
2134 * BIO off first?
2135 */
2136 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio,
2137 *last_block_in_bio, block_nr) ||
2138 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2139submit_and_realloc:
1116 return 0;
1117}
1118
1119static void __set_data_blkaddr(struct dnode_of_data *dn)
1120{
1121 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
1122 __le32 *addr_array;
1123 int base = 0;

--- 1013 unchanged lines hidden (view full) ---

2137 /*
2138 * This page will go to BIO. Do we need to send this
2139 * BIO off first?
2140 */
2141 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio,
2142 *last_block_in_bio, block_nr) ||
2143 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2144submit_and_realloc:
2140 __submit_bio(F2FS_I_SB(inode), bio, DATA);
2145 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
2141 bio = NULL;
2142 }
2143 if (bio == NULL) {
2144 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
2145 is_readahead ? REQ_RAHEAD : 0, page->index,
2146 false);
2147 if (IS_ERR(bio)) {
2148 ret = PTR_ERR(bio);

--- 130 unchanged lines hidden (view full) ---

2279 f2fs_decompress_cluster(dic, true);
2280 continue;
2281 }
2282
2283 if (bio && (!page_is_mergeable(sbi, bio,
2284 *last_block_in_bio, blkaddr) ||
2285 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2286submit_and_realloc:
2146 bio = NULL;
2147 }
2148 if (bio == NULL) {
2149 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
2150 is_readahead ? REQ_RAHEAD : 0, page->index,
2151 false);
2152 if (IS_ERR(bio)) {
2153 ret = PTR_ERR(bio);

--- 130 unchanged lines hidden (view full) ---

2284 f2fs_decompress_cluster(dic, true);
2285 continue;
2286 }
2287
2288 if (bio && (!page_is_mergeable(sbi, bio,
2289 *last_block_in_bio, blkaddr) ||
2290 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2291submit_and_realloc:
2287 __submit_bio(sbi, bio, DATA);
2292 f2fs_submit_read_bio(sbi, bio, DATA);
2288 bio = NULL;
2289 }
2290
2291 if (!bio) {
2292 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages,
2293 is_readahead ? REQ_RAHEAD : 0,
2294 page->index, for_write);
2295 if (IS_ERR(bio)) {

--- 144 unchanged lines hidden (view full) ---

2440 &last_block_in_bio,
2441 rac != NULL, false);
2442 f2fs_destroy_compress_ctx(&cc, false);
2443 }
2444 }
2445#endif
2446 }
2447 if (bio)
2293 bio = NULL;
2294 }
2295
2296 if (!bio) {
2297 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages,
2298 is_readahead ? REQ_RAHEAD : 0,
2299 page->index, for_write);
2300 if (IS_ERR(bio)) {

--- 144 unchanged lines hidden (view full) ---

2445 &last_block_in_bio,
2446 rac != NULL, false);
2447 f2fs_destroy_compress_ctx(&cc, false);
2448 }
2449 }
2450#endif
2451 }
2452 if (bio)
2448 __submit_bio(F2FS_I_SB(inode), bio, DATA);
2453 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
2449 return ret;
2450}
2451
2452static int f2fs_read_data_folio(struct file *file, struct folio *folio)
2453{
2454 struct page *page = &folio->page;
2455 struct inode *inode = page_file_mapping(page)->host;
2456 int ret = -EAGAIN;

--- 1735 unchanged lines hidden --- 		2454 return ret;
2455}
2456
2457static int f2fs_read_data_folio(struct file *file, struct folio *folio)
2458{
2459 struct page *page = &folio->page;
2460 struct inode *inode = page_file_mapping(page)->host;
2461 int ret = -EAGAIN;

--- 1735 unchanged lines hidden ---