1 /* 2 * fs/f2fs/data.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/fs.h> 12 #include <linux/f2fs_fs.h> 13 #include <linux/buffer_head.h> 14 #include <linux/mpage.h> 15 #include <linux/writeback.h> 16 #include <linux/backing-dev.h> 17 #include <linux/pagevec.h> 18 #include <linux/blkdev.h> 19 #include <linux/bio.h> 20 #include <linux/prefetch.h> 21 #include <linux/uio.h> 22 #include <linux/mm.h> 23 #include <linux/memcontrol.h> 24 #include <linux/cleancache.h> 25 #include <linux/sched/signal.h> 26 27 #include "f2fs.h" 28 #include "node.h" 29 #include "segment.h" 30 #include "trace.h" 31 #include <trace/events/f2fs.h> 32 33 static bool __is_cp_guaranteed(struct page *page) 34 { 35 struct address_space *mapping = page->mapping; 36 struct inode *inode; 37 struct f2fs_sb_info *sbi; 38 39 if (!mapping) 40 return false; 41 42 inode = mapping->host; 43 sbi = F2FS_I_SB(inode); 44 45 if (inode->i_ino == F2FS_META_INO(sbi) || 46 inode->i_ino == F2FS_NODE_INO(sbi) || 47 S_ISDIR(inode->i_mode) || 48 is_cold_data(page)) 49 return true; 50 return false; 51 } 52 53 static void f2fs_read_end_io(struct bio *bio) 54 { 55 struct bio_vec *bvec; 56 int i; 57 58 #ifdef CONFIG_F2FS_FAULT_INJECTION 59 if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) { 60 f2fs_show_injection_info(FAULT_IO); 61 bio->bi_status = BLK_STS_IOERR; 62 } 63 #endif 64 65 if (f2fs_bio_encrypted(bio)) { 66 if (bio->bi_status) { 67 fscrypt_release_ctx(bio->bi_private); 68 } else { 69 fscrypt_decrypt_bio_pages(bio->bi_private, bio); 70 return; 71 } 72 } 73 74 bio_for_each_segment_all(bvec, bio, i) { 75 struct page *page = bvec->bv_page; 76 77 if (!bio->bi_status) { 78 if (!PageUptodate(page)) 79 SetPageUptodate(page); 80 } else { 81 ClearPageUptodate(page); 82 SetPageError(page); 83 } 84 unlock_page(page); 85 } 86 bio_put(bio); 87 } 88 89 static void f2fs_write_end_io(struct bio *bio) 90 { 91 struct f2fs_sb_info *sbi = bio->bi_private; 92 struct bio_vec *bvec; 93 int i; 94 95 bio_for_each_segment_all(bvec, bio, i) { 96 struct page *page = bvec->bv_page; 97 enum count_type type = WB_DATA_TYPE(page); 98 99 if (IS_DUMMY_WRITTEN_PAGE(page)) { 100 set_page_private(page, (unsigned long)NULL); 101 ClearPagePrivate(page); 102 unlock_page(page); 103 mempool_free(page, sbi->write_io_dummy); 104 105 if (unlikely(bio->bi_status)) 106 f2fs_stop_checkpoint(sbi, true); 107 continue; 108 } 109 110 fscrypt_pullback_bio_page(&page, true); 111 112 if (unlikely(bio->bi_status)) { 113 mapping_set_error(page->mapping, -EIO); 114 f2fs_stop_checkpoint(sbi, true); 115 } 116 dec_page_count(sbi, type); 117 clear_cold_data(page); 118 end_page_writeback(page); 119 } 120 if (!get_pages(sbi, F2FS_WB_CP_DATA) && 121 wq_has_sleeper(&sbi->cp_wait)) 122 wake_up(&sbi->cp_wait); 123 124 bio_put(bio); 125 } 126 127 /* 128 * Return true, if pre_bio's bdev is same as its target device. 129 */ 130 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, 131 block_t blk_addr, struct bio *bio) 132 { 133 struct block_device *bdev = sbi->sb->s_bdev; 134 int i; 135 136 for (i = 0; i < sbi->s_ndevs; i++) { 137 if (FDEV(i).start_blk <= blk_addr && 138 FDEV(i).end_blk >= blk_addr) { 139 blk_addr -= FDEV(i).start_blk; 140 bdev = FDEV(i).bdev; 141 break; 142 } 143 } 144 if (bio) { 145 bio_set_dev(bio, bdev); 146 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); 147 } 148 return bdev; 149 } 150 151 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) 152 { 153 int i; 154 155 for (i = 0; i < sbi->s_ndevs; i++) 156 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) 157 return i; 158 return 0; 159 } 160 161 static bool __same_bdev(struct f2fs_sb_info *sbi, 162 block_t blk_addr, struct bio *bio) 163 { 164 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL); 165 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno; 166 } 167 168 /* 169 * Low-level block read/write IO operations. 170 */ 171 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, 172 int npages, bool is_read) 173 { 174 struct bio *bio; 175 176 bio = f2fs_bio_alloc(sbi, npages, true); 177 178 f2fs_target_device(sbi, blk_addr, bio); 179 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; 180 bio->bi_private = is_read ? NULL : sbi; 181 182 return bio; 183 } 184 185 static inline void __submit_bio(struct f2fs_sb_info *sbi, 186 struct bio *bio, enum page_type type) 187 { 188 if (!is_read_io(bio_op(bio))) { 189 unsigned int start; 190 191 if (f2fs_sb_mounted_blkzoned(sbi->sb) && 192 current->plug && (type == DATA || type == NODE)) 193 blk_finish_plug(current->plug); 194 195 if (type != DATA && type != NODE) 196 goto submit_io; 197 198 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS; 199 start %= F2FS_IO_SIZE(sbi); 200 201 if (start == 0) 202 goto submit_io; 203 204 /* fill dummy pages */ 205 for (; start < F2FS_IO_SIZE(sbi); start++) { 206 struct page *page = 207 mempool_alloc(sbi->write_io_dummy, 208 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL); 209 f2fs_bug_on(sbi, !page); 210 211 SetPagePrivate(page); 212 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE); 213 lock_page(page); 214 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 215 f2fs_bug_on(sbi, 1); 216 } 217 /* 218 * In the NODE case, we lose next block address chain. So, we 219 * need to do checkpoint in f2fs_sync_file. 220 */ 221 if (type == NODE) 222 set_sbi_flag(sbi, SBI_NEED_CP); 223 } 224 submit_io: 225 if (is_read_io(bio_op(bio))) 226 trace_f2fs_submit_read_bio(sbi->sb, type, bio); 227 else 228 trace_f2fs_submit_write_bio(sbi->sb, type, bio); 229 submit_bio(bio); 230 } 231 232 static void __submit_merged_bio(struct f2fs_bio_info *io) 233 { 234 struct f2fs_io_info *fio = &io->fio; 235 236 if (!io->bio) 237 return; 238 239 bio_set_op_attrs(io->bio, fio->op, fio->op_flags); 240 241 if (is_read_io(fio->op)) 242 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); 243 else 244 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); 245 246 __submit_bio(io->sbi, io->bio, fio->type); 247 io->bio = NULL; 248 } 249 250 static bool __has_merged_page(struct f2fs_bio_info *io, 251 struct inode *inode, nid_t ino, pgoff_t idx) 252 { 253 struct bio_vec *bvec; 254 struct page *target; 255 int i; 256 257 if (!io->bio) 258 return false; 259 260 if (!inode && !ino) 261 return true; 262 263 bio_for_each_segment_all(bvec, io->bio, i) { 264 265 if (bvec->bv_page->mapping) 266 target = bvec->bv_page; 267 else 268 target = fscrypt_control_page(bvec->bv_page); 269 270 if (idx != target->index) 271 continue; 272 273 if (inode && inode == target->mapping->host) 274 return true; 275 if (ino && ino == ino_of_node(target)) 276 return true; 277 } 278 279 return false; 280 } 281 282 static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode, 283 nid_t ino, pgoff_t idx, enum page_type type) 284 { 285 enum page_type btype = PAGE_TYPE_OF_BIO(type); 286 enum temp_type temp; 287 struct f2fs_bio_info *io; 288 bool ret = false; 289 290 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 291 io = sbi->write_io[btype] + temp; 292 293 down_read(&io->io_rwsem); 294 ret = __has_merged_page(io, inode, ino, idx); 295 up_read(&io->io_rwsem); 296 297 /* TODO: use HOT temp only for meta pages now. */ 298 if (ret || btype == META) 299 break; 300 } 301 return ret; 302 } 303 304 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, 305 enum page_type type, enum temp_type temp) 306 { 307 enum page_type btype = PAGE_TYPE_OF_BIO(type); 308 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 309 310 down_write(&io->io_rwsem); 311 312 /* change META to META_FLUSH in the checkpoint procedure */ 313 if (type >= META_FLUSH) { 314 io->fio.type = META_FLUSH; 315 io->fio.op = REQ_OP_WRITE; 316 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC; 317 if (!test_opt(sbi, NOBARRIER)) 318 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA; 319 } 320 __submit_merged_bio(io); 321 up_write(&io->io_rwsem); 322 } 323 324 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, 325 struct inode *inode, nid_t ino, pgoff_t idx, 326 enum page_type type, bool force) 327 { 328 enum temp_type temp; 329 330 if (!force && !has_merged_page(sbi, inode, ino, idx, type)) 331 return; 332 333 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 334 335 __f2fs_submit_merged_write(sbi, type, temp); 336 337 /* TODO: use HOT temp only for meta pages now. */ 338 if (type >= META) 339 break; 340 } 341 } 342 343 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) 344 { 345 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true); 346 } 347 348 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, 349 struct inode *inode, nid_t ino, pgoff_t idx, 350 enum page_type type) 351 { 352 __submit_merged_write_cond(sbi, inode, ino, idx, type, false); 353 } 354 355 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) 356 { 357 f2fs_submit_merged_write(sbi, DATA); 358 f2fs_submit_merged_write(sbi, NODE); 359 f2fs_submit_merged_write(sbi, META); 360 } 361 362 /* 363 * Fill the locked page with data located in the block address. 364 * A caller needs to unlock the page on failure. 365 */ 366 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 367 { 368 struct bio *bio; 369 struct page *page = fio->encrypted_page ? 370 fio->encrypted_page : fio->page; 371 372 trace_f2fs_submit_page_bio(page, fio); 373 f2fs_trace_ios(fio, 0); 374 375 /* Allocate a new bio */ 376 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, 1, is_read_io(fio->op)); 377 378 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 379 bio_put(bio); 380 return -EFAULT; 381 } 382 bio_set_op_attrs(bio, fio->op, fio->op_flags); 383 384 __submit_bio(fio->sbi, bio, fio->type); 385 386 if (!is_read_io(fio->op)) 387 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page)); 388 return 0; 389 } 390 391 int f2fs_submit_page_write(struct f2fs_io_info *fio) 392 { 393 struct f2fs_sb_info *sbi = fio->sbi; 394 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 395 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; 396 struct page *bio_page; 397 int err = 0; 398 399 f2fs_bug_on(sbi, is_read_io(fio->op)); 400 401 down_write(&io->io_rwsem); 402 next: 403 if (fio->in_list) { 404 spin_lock(&io->io_lock); 405 if (list_empty(&io->io_list)) { 406 spin_unlock(&io->io_lock); 407 goto out_fail; 408 } 409 fio = list_first_entry(&io->io_list, 410 struct f2fs_io_info, list); 411 list_del(&fio->list); 412 spin_unlock(&io->io_lock); 413 } 414 415 if (fio->old_blkaddr != NEW_ADDR) 416 verify_block_addr(sbi, fio->old_blkaddr); 417 verify_block_addr(sbi, fio->new_blkaddr); 418 419 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; 420 421 /* set submitted = true as a return value */ 422 fio->submitted = true; 423 424 inc_page_count(sbi, WB_DATA_TYPE(bio_page)); 425 426 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 || 427 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) || 428 !__same_bdev(sbi, fio->new_blkaddr, io->bio))) 429 __submit_merged_bio(io); 430 alloc_new: 431 if (io->bio == NULL) { 432 if ((fio->type == DATA || fio->type == NODE) && 433 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { 434 err = -EAGAIN; 435 dec_page_count(sbi, WB_DATA_TYPE(bio_page)); 436 goto out_fail; 437 } 438 io->bio = __bio_alloc(sbi, fio->new_blkaddr, 439 BIO_MAX_PAGES, false); 440 io->fio = *fio; 441 } 442 443 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { 444 __submit_merged_bio(io); 445 goto alloc_new; 446 } 447 448 io->last_block_in_bio = fio->new_blkaddr; 449 f2fs_trace_ios(fio, 0); 450 451 trace_f2fs_submit_page_write(fio->page, fio); 452 453 if (fio->in_list) 454 goto next; 455 out_fail: 456 up_write(&io->io_rwsem); 457 return err; 458 } 459 460 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, 461 unsigned nr_pages) 462 { 463 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 464 struct fscrypt_ctx *ctx = NULL; 465 struct bio *bio; 466 467 if (f2fs_encrypted_file(inode)) { 468 ctx = fscrypt_get_ctx(inode, GFP_NOFS); 469 if (IS_ERR(ctx)) 470 return ERR_CAST(ctx); 471 472 /* wait the page to be moved by cleaning */ 473 f2fs_wait_on_block_writeback(sbi, blkaddr); 474 } 475 476 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false); 477 if (!bio) { 478 if (ctx) 479 fscrypt_release_ctx(ctx); 480 return ERR_PTR(-ENOMEM); 481 } 482 f2fs_target_device(sbi, blkaddr, bio); 483 bio->bi_end_io = f2fs_read_end_io; 484 bio->bi_private = ctx; 485 bio_set_op_attrs(bio, REQ_OP_READ, 0); 486 487 return bio; 488 } 489 490 /* This can handle encryption stuffs */ 491 static int f2fs_submit_page_read(struct inode *inode, struct page *page, 492 block_t blkaddr) 493 { 494 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1); 495 496 if (IS_ERR(bio)) 497 return PTR_ERR(bio); 498 499 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 500 bio_put(bio); 501 return -EFAULT; 502 } 503 __submit_bio(F2FS_I_SB(inode), bio, DATA); 504 return 0; 505 } 506 507 static void __set_data_blkaddr(struct dnode_of_data *dn) 508 { 509 struct f2fs_node *rn = F2FS_NODE(dn->node_page); 510 __le32 *addr_array; 511 int base = 0; 512 513 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode)) 514 base = get_extra_isize(dn->inode); 515 516 /* Get physical address of data block */ 517 addr_array = blkaddr_in_node(rn); 518 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 519 } 520 521 /* 522 * Lock ordering for the change of data block address: 523 * ->data_page 524 * ->node_page 525 * update block addresses in the node page 526 */ 527 void set_data_blkaddr(struct dnode_of_data *dn) 528 { 529 f2fs_wait_on_page_writeback(dn->node_page, NODE, true); 530 __set_data_blkaddr(dn); 531 if (set_page_dirty(dn->node_page)) 532 dn->node_changed = true; 533 } 534 535 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 536 { 537 dn->data_blkaddr = blkaddr; 538 set_data_blkaddr(dn); 539 f2fs_update_extent_cache(dn); 540 } 541 542 /* dn->ofs_in_node will be returned with up-to-date last block pointer */ 543 int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) 544 { 545 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 546 int err; 547 548 if (!count) 549 return 0; 550 551 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 552 return -EPERM; 553 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) 554 return err; 555 556 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, 557 dn->ofs_in_node, count); 558 559 f2fs_wait_on_page_writeback(dn->node_page, NODE, true); 560 561 for (; count > 0; dn->ofs_in_node++) { 562 block_t blkaddr = datablock_addr(dn->inode, 563 dn->node_page, dn->ofs_in_node); 564 if (blkaddr == NULL_ADDR) { 565 dn->data_blkaddr = NEW_ADDR; 566 __set_data_blkaddr(dn); 567 count--; 568 } 569 } 570 571 if (set_page_dirty(dn->node_page)) 572 dn->node_changed = true; 573 return 0; 574 } 575 576 /* Should keep dn->ofs_in_node unchanged */ 577 int reserve_new_block(struct dnode_of_data *dn) 578 { 579 unsigned int ofs_in_node = dn->ofs_in_node; 580 int ret; 581 582 ret = reserve_new_blocks(dn, 1); 583 dn->ofs_in_node = ofs_in_node; 584 return ret; 585 } 586 587 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 588 { 589 bool need_put = dn->inode_page ? false : true; 590 int err; 591 592 err = get_dnode_of_data(dn, index, ALLOC_NODE); 593 if (err) 594 return err; 595 596 if (dn->data_blkaddr == NULL_ADDR) 597 err = reserve_new_block(dn); 598 if (err || need_put) 599 f2fs_put_dnode(dn); 600 return err; 601 } 602 603 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) 604 { 605 struct extent_info ei = {0,0,0}; 606 struct inode *inode = dn->inode; 607 608 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 609 dn->data_blkaddr = ei.blk + index - ei.fofs; 610 return 0; 611 } 612 613 return f2fs_reserve_block(dn, index); 614 } 615 616 struct page *get_read_data_page(struct inode *inode, pgoff_t index, 617 int op_flags, bool for_write) 618 { 619 struct address_space *mapping = inode->i_mapping; 620 struct dnode_of_data dn; 621 struct page *page; 622 struct extent_info ei = {0,0,0}; 623 int err; 624 625 page = f2fs_grab_cache_page(mapping, index, for_write); 626 if (!page) 627 return ERR_PTR(-ENOMEM); 628 629 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 630 dn.data_blkaddr = ei.blk + index - ei.fofs; 631 goto got_it; 632 } 633 634 set_new_dnode(&dn, inode, NULL, NULL, 0); 635 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 636 if (err) 637 goto put_err; 638 f2fs_put_dnode(&dn); 639 640 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 641 err = -ENOENT; 642 goto put_err; 643 } 644 got_it: 645 if (PageUptodate(page)) { 646 unlock_page(page); 647 return page; 648 } 649 650 /* 651 * A new dentry page is allocated but not able to be written, since its 652 * new inode page couldn't be allocated due to -ENOSPC. 653 * In such the case, its blkaddr can be remained as NEW_ADDR. 654 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata. 655 */ 656 if (dn.data_blkaddr == NEW_ADDR) { 657 zero_user_segment(page, 0, PAGE_SIZE); 658 if (!PageUptodate(page)) 659 SetPageUptodate(page); 660 unlock_page(page); 661 return page; 662 } 663 664 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr); 665 if (err) 666 goto put_err; 667 return page; 668 669 put_err: 670 f2fs_put_page(page, 1); 671 return ERR_PTR(err); 672 } 673 674 struct page *find_data_page(struct inode *inode, pgoff_t index) 675 { 676 struct address_space *mapping = inode->i_mapping; 677 struct page *page; 678 679 page = find_get_page(mapping, index); 680 if (page && PageUptodate(page)) 681 return page; 682 f2fs_put_page(page, 0); 683 684 page = get_read_data_page(inode, index, 0, false); 685 if (IS_ERR(page)) 686 return page; 687 688 if (PageUptodate(page)) 689 return page; 690 691 wait_on_page_locked(page); 692 if (unlikely(!PageUptodate(page))) { 693 f2fs_put_page(page, 0); 694 return ERR_PTR(-EIO); 695 } 696 return page; 697 } 698 699 /* 700 * If it tries to access a hole, return an error. 701 * Because, the callers, functions in dir.c and GC, should be able to know 702 * whether this page exists or not. 703 */ 704 struct page *get_lock_data_page(struct inode *inode, pgoff_t index, 705 bool for_write) 706 { 707 struct address_space *mapping = inode->i_mapping; 708 struct page *page; 709 repeat: 710 page = get_read_data_page(inode, index, 0, for_write); 711 if (IS_ERR(page)) 712 return page; 713 714 /* wait for read completion */ 715 lock_page(page); 716 if (unlikely(page->mapping != mapping)) { 717 f2fs_put_page(page, 1); 718 goto repeat; 719 } 720 if (unlikely(!PageUptodate(page))) { 721 f2fs_put_page(page, 1); 722 return ERR_PTR(-EIO); 723 } 724 return page; 725 } 726 727 /* 728 * Caller ensures that this data page is never allocated. 729 * A new zero-filled data page is allocated in the page cache. 730 * 731 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 732 * f2fs_unlock_op(). 733 * Note that, ipage is set only by make_empty_dir, and if any error occur, 734 * ipage should be released by this function. 735 */ 736 struct page *get_new_data_page(struct inode *inode, 737 struct page *ipage, pgoff_t index, bool new_i_size) 738 { 739 struct address_space *mapping = inode->i_mapping; 740 struct page *page; 741 struct dnode_of_data dn; 742 int err; 743 744 page = f2fs_grab_cache_page(mapping, index, true); 745 if (!page) { 746 /* 747 * before exiting, we should make sure ipage will be released 748 * if any error occur. 749 */ 750 f2fs_put_page(ipage, 1); 751 return ERR_PTR(-ENOMEM); 752 } 753 754 set_new_dnode(&dn, inode, ipage, NULL, 0); 755 err = f2fs_reserve_block(&dn, index); 756 if (err) { 757 f2fs_put_page(page, 1); 758 return ERR_PTR(err); 759 } 760 if (!ipage) 761 f2fs_put_dnode(&dn); 762 763 if (PageUptodate(page)) 764 goto got_it; 765 766 if (dn.data_blkaddr == NEW_ADDR) { 767 zero_user_segment(page, 0, PAGE_SIZE); 768 if (!PageUptodate(page)) 769 SetPageUptodate(page); 770 } else { 771 f2fs_put_page(page, 1); 772 773 /* if ipage exists, blkaddr should be NEW_ADDR */ 774 f2fs_bug_on(F2FS_I_SB(inode), ipage); 775 page = get_lock_data_page(inode, index, true); 776 if (IS_ERR(page)) 777 return page; 778 } 779 got_it: 780 if (new_i_size && i_size_read(inode) < 781 ((loff_t)(index + 1) << PAGE_SHIFT)) 782 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT)); 783 return page; 784 } 785 786 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) 787 { 788 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 789 struct f2fs_summary sum; 790 struct node_info ni; 791 pgoff_t fofs; 792 blkcnt_t count = 1; 793 int err; 794 795 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 796 return -EPERM; 797 798 dn->data_blkaddr = datablock_addr(dn->inode, 799 dn->node_page, dn->ofs_in_node); 800 if (dn->data_blkaddr == NEW_ADDR) 801 goto alloc; 802 803 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) 804 return err; 805 806 alloc: 807 get_node_info(sbi, dn->nid, &ni); 808 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 809 810 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr, 811 &sum, seg_type, NULL, false); 812 set_data_blkaddr(dn); 813 814 /* update i_size */ 815 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) + 816 dn->ofs_in_node; 817 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT)) 818 f2fs_i_size_write(dn->inode, 819 ((loff_t)(fofs + 1) << PAGE_SHIFT)); 820 return 0; 821 } 822 823 static inline bool __force_buffered_io(struct inode *inode, int rw) 824 { 825 return (f2fs_encrypted_file(inode) || 826 (rw == WRITE && test_opt(F2FS_I_SB(inode), LFS)) || 827 F2FS_I_SB(inode)->s_ndevs); 828 } 829 830 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from) 831 { 832 struct inode *inode = file_inode(iocb->ki_filp); 833 struct f2fs_map_blocks map; 834 int err = 0; 835 836 /* convert inline data for Direct I/O*/ 837 if (iocb->ki_flags & IOCB_DIRECT) { 838 err = f2fs_convert_inline_inode(inode); 839 if (err) 840 return err; 841 } 842 843 if (is_inode_flag_set(inode, FI_NO_PREALLOC)) 844 return 0; 845 846 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos); 847 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from)); 848 if (map.m_len > map.m_lblk) 849 map.m_len -= map.m_lblk; 850 else 851 map.m_len = 0; 852 853 map.m_next_pgofs = NULL; 854 map.m_seg_type = NO_CHECK_TYPE; 855 856 if (iocb->ki_flags & IOCB_DIRECT) { 857 map.m_seg_type = rw_hint_to_seg_type(iocb->ki_hint); 858 return f2fs_map_blocks(inode, &map, 1, 859 __force_buffered_io(inode, WRITE) ? 860 F2FS_GET_BLOCK_PRE_AIO : 861 F2FS_GET_BLOCK_PRE_DIO); 862 } 863 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) { 864 err = f2fs_convert_inline_inode(inode); 865 if (err) 866 return err; 867 } 868 if (!f2fs_has_inline_data(inode)) { 869 err = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO); 870 if (map.m_len > 0 && err == -ENOSPC) { 871 set_inode_flag(inode, FI_NO_PREALLOC); 872 err = 0; 873 } 874 return err; 875 } 876 return err; 877 } 878 879 static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock) 880 { 881 if (flag == F2FS_GET_BLOCK_PRE_AIO) { 882 if (lock) 883 down_read(&sbi->node_change); 884 else 885 up_read(&sbi->node_change); 886 } else { 887 if (lock) 888 f2fs_lock_op(sbi); 889 else 890 f2fs_unlock_op(sbi); 891 } 892 } 893 894 /* 895 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with 896 * f2fs_map_blocks structure. 897 * If original data blocks are allocated, then give them to blockdev. 898 * Otherwise, 899 * a. preallocate requested block addresses 900 * b. do not use extent cache for better performance 901 * c. give the block addresses to blockdev 902 */ 903 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 904 int create, int flag) 905 { 906 unsigned int maxblocks = map->m_len; 907 struct dnode_of_data dn; 908 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 909 int mode = create ? ALLOC_NODE : LOOKUP_NODE; 910 pgoff_t pgofs, end_offset, end; 911 int err = 0, ofs = 1; 912 unsigned int ofs_in_node, last_ofs_in_node; 913 blkcnt_t prealloc; 914 struct extent_info ei = {0,0,0}; 915 block_t blkaddr; 916 917 if (!maxblocks) 918 return 0; 919 920 map->m_len = 0; 921 map->m_flags = 0; 922 923 /* it only supports block size == page size */ 924 pgofs = (pgoff_t)map->m_lblk; 925 end = pgofs + maxblocks; 926 927 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) { 928 map->m_pblk = ei.blk + pgofs - ei.fofs; 929 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); 930 map->m_flags = F2FS_MAP_MAPPED; 931 goto out; 932 } 933 934 next_dnode: 935 if (create) 936 __do_map_lock(sbi, flag, true); 937 938 /* When reading holes, we need its node page */ 939 set_new_dnode(&dn, inode, NULL, NULL, 0); 940 err = get_dnode_of_data(&dn, pgofs, mode); 941 if (err) { 942 if (flag == F2FS_GET_BLOCK_BMAP) 943 map->m_pblk = 0; 944 if (err == -ENOENT) { 945 err = 0; 946 if (map->m_next_pgofs) 947 *map->m_next_pgofs = 948 get_next_page_offset(&dn, pgofs); 949 } 950 goto unlock_out; 951 } 952 953 prealloc = 0; 954 last_ofs_in_node = ofs_in_node = dn.ofs_in_node; 955 end_offset = ADDRS_PER_PAGE(dn.node_page, inode); 956 957 next_block: 958 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node); 959 960 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) { 961 if (create) { 962 if (unlikely(f2fs_cp_error(sbi))) { 963 err = -EIO; 964 goto sync_out; 965 } 966 if (flag == F2FS_GET_BLOCK_PRE_AIO) { 967 if (blkaddr == NULL_ADDR) { 968 prealloc++; 969 last_ofs_in_node = dn.ofs_in_node; 970 } 971 } else { 972 err = __allocate_data_block(&dn, 973 map->m_seg_type); 974 if (!err) 975 set_inode_flag(inode, FI_APPEND_WRITE); 976 } 977 if (err) 978 goto sync_out; 979 map->m_flags |= F2FS_MAP_NEW; 980 blkaddr = dn.data_blkaddr; 981 } else { 982 if (flag == F2FS_GET_BLOCK_BMAP) { 983 map->m_pblk = 0; 984 goto sync_out; 985 } 986 if (flag == F2FS_GET_BLOCK_FIEMAP && 987 blkaddr == NULL_ADDR) { 988 if (map->m_next_pgofs) 989 *map->m_next_pgofs = pgofs + 1; 990 } 991 if (flag != F2FS_GET_BLOCK_FIEMAP || 992 blkaddr != NEW_ADDR) 993 goto sync_out; 994 } 995 } 996 997 if (flag == F2FS_GET_BLOCK_PRE_AIO) 998 goto skip; 999 1000 if (map->m_len == 0) { 1001 /* preallocated unwritten block should be mapped for fiemap. */ 1002 if (blkaddr == NEW_ADDR) 1003 map->m_flags |= F2FS_MAP_UNWRITTEN; 1004 map->m_flags |= F2FS_MAP_MAPPED; 1005 1006 map->m_pblk = blkaddr; 1007 map->m_len = 1; 1008 } else if ((map->m_pblk != NEW_ADDR && 1009 blkaddr == (map->m_pblk + ofs)) || 1010 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) || 1011 flag == F2FS_GET_BLOCK_PRE_DIO) { 1012 ofs++; 1013 map->m_len++; 1014 } else { 1015 goto sync_out; 1016 } 1017 1018 skip: 1019 dn.ofs_in_node++; 1020 pgofs++; 1021 1022 /* preallocate blocks in batch for one dnode page */ 1023 if (flag == F2FS_GET_BLOCK_PRE_AIO && 1024 (pgofs == end || dn.ofs_in_node == end_offset)) { 1025 1026 dn.ofs_in_node = ofs_in_node; 1027 err = reserve_new_blocks(&dn, prealloc); 1028 if (err) 1029 goto sync_out; 1030 1031 map->m_len += dn.ofs_in_node - ofs_in_node; 1032 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { 1033 err = -ENOSPC; 1034 goto sync_out; 1035 } 1036 dn.ofs_in_node = end_offset; 1037 } 1038 1039 if (pgofs >= end) 1040 goto sync_out; 1041 else if (dn.ofs_in_node < end_offset) 1042 goto next_block; 1043 1044 f2fs_put_dnode(&dn); 1045 1046 if (create) { 1047 __do_map_lock(sbi, flag, false); 1048 f2fs_balance_fs(sbi, dn.node_changed); 1049 } 1050 goto next_dnode; 1051 1052 sync_out: 1053 f2fs_put_dnode(&dn); 1054 unlock_out: 1055 if (create) { 1056 __do_map_lock(sbi, flag, false); 1057 f2fs_balance_fs(sbi, dn.node_changed); 1058 } 1059 out: 1060 trace_f2fs_map_blocks(inode, map, err); 1061 return err; 1062 } 1063 1064 static int __get_data_block(struct inode *inode, sector_t iblock, 1065 struct buffer_head *bh, int create, int flag, 1066 pgoff_t *next_pgofs, int seg_type) 1067 { 1068 struct f2fs_map_blocks map; 1069 int err; 1070 1071 map.m_lblk = iblock; 1072 map.m_len = bh->b_size >> inode->i_blkbits; 1073 map.m_next_pgofs = next_pgofs; 1074 map.m_seg_type = seg_type; 1075 1076 err = f2fs_map_blocks(inode, &map, create, flag); 1077 if (!err) { 1078 map_bh(bh, inode->i_sb, map.m_pblk); 1079 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; 1080 bh->b_size = (u64)map.m_len << inode->i_blkbits; 1081 } 1082 return err; 1083 } 1084 1085 static int get_data_block(struct inode *inode, sector_t iblock, 1086 struct buffer_head *bh_result, int create, int flag, 1087 pgoff_t *next_pgofs) 1088 { 1089 return __get_data_block(inode, iblock, bh_result, create, 1090 flag, next_pgofs, 1091 NO_CHECK_TYPE); 1092 } 1093 1094 static int get_data_block_dio(struct inode *inode, sector_t iblock, 1095 struct buffer_head *bh_result, int create) 1096 { 1097 return __get_data_block(inode, iblock, bh_result, create, 1098 F2FS_GET_BLOCK_DEFAULT, NULL, 1099 rw_hint_to_seg_type( 1100 inode->i_write_hint)); 1101 } 1102 1103 static int get_data_block_bmap(struct inode *inode, sector_t iblock, 1104 struct buffer_head *bh_result, int create) 1105 { 1106 /* Block number less than F2FS MAX BLOCKS */ 1107 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks)) 1108 return -EFBIG; 1109 1110 return __get_data_block(inode, iblock, bh_result, create, 1111 F2FS_GET_BLOCK_BMAP, NULL, 1112 NO_CHECK_TYPE); 1113 } 1114 1115 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset) 1116 { 1117 return (offset >> inode->i_blkbits); 1118 } 1119 1120 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk) 1121 { 1122 return (blk << inode->i_blkbits); 1123 } 1124 1125 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 1126 u64 start, u64 len) 1127 { 1128 struct buffer_head map_bh; 1129 sector_t start_blk, last_blk; 1130 pgoff_t next_pgofs; 1131 u64 logical = 0, phys = 0, size = 0; 1132 u32 flags = 0; 1133 int ret = 0; 1134 1135 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); 1136 if (ret) 1137 return ret; 1138 1139 if (f2fs_has_inline_data(inode)) { 1140 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 1141 if (ret != -EAGAIN) 1142 return ret; 1143 } 1144 1145 inode_lock(inode); 1146 1147 if (logical_to_blk(inode, len) == 0) 1148 len = blk_to_logical(inode, 1); 1149 1150 start_blk = logical_to_blk(inode, start); 1151 last_blk = logical_to_blk(inode, start + len - 1); 1152 1153 next: 1154 memset(&map_bh, 0, sizeof(struct buffer_head)); 1155 map_bh.b_size = len; 1156 1157 ret = get_data_block(inode, start_blk, &map_bh, 0, 1158 F2FS_GET_BLOCK_FIEMAP, &next_pgofs); 1159 if (ret) 1160 goto out; 1161 1162 /* HOLE */ 1163 if (!buffer_mapped(&map_bh)) { 1164 start_blk = next_pgofs; 1165 1166 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode, 1167 F2FS_I_SB(inode)->max_file_blocks)) 1168 goto prep_next; 1169 1170 flags |= FIEMAP_EXTENT_LAST; 1171 } 1172 1173 if (size) { 1174 if (f2fs_encrypted_inode(inode)) 1175 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; 1176 1177 ret = fiemap_fill_next_extent(fieinfo, logical, 1178 phys, size, flags); 1179 } 1180 1181 if (start_blk > last_blk || ret) 1182 goto out; 1183 1184 logical = blk_to_logical(inode, start_blk); 1185 phys = blk_to_logical(inode, map_bh.b_blocknr); 1186 size = map_bh.b_size; 1187 flags = 0; 1188 if (buffer_unwritten(&map_bh)) 1189 flags = FIEMAP_EXTENT_UNWRITTEN; 1190 1191 start_blk += logical_to_blk(inode, size); 1192 1193 prep_next: 1194 cond_resched(); 1195 if (fatal_signal_pending(current)) 1196 ret = -EINTR; 1197 else 1198 goto next; 1199 out: 1200 if (ret == 1) 1201 ret = 0; 1202 1203 inode_unlock(inode); 1204 return ret; 1205 } 1206 1207 /* 1208 * This function was originally taken from fs/mpage.c, and customized for f2fs. 1209 * Major change was from block_size == page_size in f2fs by default. 1210 */ 1211 static int f2fs_mpage_readpages(struct address_space *mapping, 1212 struct list_head *pages, struct page *page, 1213 unsigned nr_pages) 1214 { 1215 struct bio *bio = NULL; 1216 sector_t last_block_in_bio = 0; 1217 struct inode *inode = mapping->host; 1218 const unsigned blkbits = inode->i_blkbits; 1219 const unsigned blocksize = 1 << blkbits; 1220 sector_t block_in_file; 1221 sector_t last_block; 1222 sector_t last_block_in_file; 1223 sector_t block_nr; 1224 struct f2fs_map_blocks map; 1225 1226 map.m_pblk = 0; 1227 map.m_lblk = 0; 1228 map.m_len = 0; 1229 map.m_flags = 0; 1230 map.m_next_pgofs = NULL; 1231 map.m_seg_type = NO_CHECK_TYPE; 1232 1233 for (; nr_pages; nr_pages--) { 1234 if (pages) { 1235 page = list_last_entry(pages, struct page, lru); 1236 1237 prefetchw(&page->flags); 1238 list_del(&page->lru); 1239 if (add_to_page_cache_lru(page, mapping, 1240 page->index, 1241 readahead_gfp_mask(mapping))) 1242 goto next_page; 1243 } 1244 1245 block_in_file = (sector_t)page->index; 1246 last_block = block_in_file + nr_pages; 1247 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> 1248 blkbits; 1249 if (last_block > last_block_in_file) 1250 last_block = last_block_in_file; 1251 1252 /* 1253 * Map blocks using the previous result first. 1254 */ 1255 if ((map.m_flags & F2FS_MAP_MAPPED) && 1256 block_in_file > map.m_lblk && 1257 block_in_file < (map.m_lblk + map.m_len)) 1258 goto got_it; 1259 1260 /* 1261 * Then do more f2fs_map_blocks() calls until we are 1262 * done with this page. 1263 */ 1264 map.m_flags = 0; 1265 1266 if (block_in_file < last_block) { 1267 map.m_lblk = block_in_file; 1268 map.m_len = last_block - block_in_file; 1269 1270 if (f2fs_map_blocks(inode, &map, 0, 1271 F2FS_GET_BLOCK_DEFAULT)) 1272 goto set_error_page; 1273 } 1274 got_it: 1275 if ((map.m_flags & F2FS_MAP_MAPPED)) { 1276 block_nr = map.m_pblk + block_in_file - map.m_lblk; 1277 SetPageMappedToDisk(page); 1278 1279 if (!PageUptodate(page) && !cleancache_get_page(page)) { 1280 SetPageUptodate(page); 1281 goto confused; 1282 } 1283 } else { 1284 zero_user_segment(page, 0, PAGE_SIZE); 1285 if (!PageUptodate(page)) 1286 SetPageUptodate(page); 1287 unlock_page(page); 1288 goto next_page; 1289 } 1290 1291 /* 1292 * This page will go to BIO. Do we need to send this 1293 * BIO off first? 1294 */ 1295 if (bio && (last_block_in_bio != block_nr - 1 || 1296 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) { 1297 submit_and_realloc: 1298 __submit_bio(F2FS_I_SB(inode), bio, DATA); 1299 bio = NULL; 1300 } 1301 if (bio == NULL) { 1302 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages); 1303 if (IS_ERR(bio)) { 1304 bio = NULL; 1305 goto set_error_page; 1306 } 1307 } 1308 1309 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 1310 goto submit_and_realloc; 1311 1312 last_block_in_bio = block_nr; 1313 goto next_page; 1314 set_error_page: 1315 SetPageError(page); 1316 zero_user_segment(page, 0, PAGE_SIZE); 1317 unlock_page(page); 1318 goto next_page; 1319 confused: 1320 if (bio) { 1321 __submit_bio(F2FS_I_SB(inode), bio, DATA); 1322 bio = NULL; 1323 } 1324 unlock_page(page); 1325 next_page: 1326 if (pages) 1327 put_page(page); 1328 } 1329 BUG_ON(pages && !list_empty(pages)); 1330 if (bio) 1331 __submit_bio(F2FS_I_SB(inode), bio, DATA); 1332 return 0; 1333 } 1334 1335 static int f2fs_read_data_page(struct file *file, struct page *page) 1336 { 1337 struct inode *inode = page->mapping->host; 1338 int ret = -EAGAIN; 1339 1340 trace_f2fs_readpage(page, DATA); 1341 1342 /* If the file has inline data, try to read it directly */ 1343 if (f2fs_has_inline_data(inode)) 1344 ret = f2fs_read_inline_data(inode, page); 1345 if (ret == -EAGAIN) 1346 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1); 1347 return ret; 1348 } 1349 1350 static int f2fs_read_data_pages(struct file *file, 1351 struct address_space *mapping, 1352 struct list_head *pages, unsigned nr_pages) 1353 { 1354 struct inode *inode = mapping->host; 1355 struct page *page = list_last_entry(pages, struct page, lru); 1356 1357 trace_f2fs_readpages(inode, page, nr_pages); 1358 1359 /* If the file has inline data, skip readpages */ 1360 if (f2fs_has_inline_data(inode)) 1361 return 0; 1362 1363 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages); 1364 } 1365 1366 static int encrypt_one_page(struct f2fs_io_info *fio) 1367 { 1368 struct inode *inode = fio->page->mapping->host; 1369 gfp_t gfp_flags = GFP_NOFS; 1370 1371 if (!f2fs_encrypted_file(inode)) 1372 return 0; 1373 1374 /* wait for GCed encrypted page writeback */ 1375 f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr); 1376 1377 retry_encrypt: 1378 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page, 1379 PAGE_SIZE, 0, fio->page->index, gfp_flags); 1380 if (!IS_ERR(fio->encrypted_page)) 1381 return 0; 1382 1383 /* flush pending IOs and wait for a while in the ENOMEM case */ 1384 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { 1385 f2fs_flush_merged_writes(fio->sbi); 1386 congestion_wait(BLK_RW_ASYNC, HZ/50); 1387 gfp_flags |= __GFP_NOFAIL; 1388 goto retry_encrypt; 1389 } 1390 return PTR_ERR(fio->encrypted_page); 1391 } 1392 1393 static inline bool need_inplace_update(struct f2fs_io_info *fio) 1394 { 1395 struct inode *inode = fio->page->mapping->host; 1396 1397 if (S_ISDIR(inode->i_mode) || f2fs_is_atomic_file(inode)) 1398 return false; 1399 if (is_cold_data(fio->page)) 1400 return false; 1401 if (IS_ATOMIC_WRITTEN_PAGE(fio->page)) 1402 return false; 1403 1404 return need_inplace_update_policy(inode, fio); 1405 } 1406 1407 static inline bool valid_ipu_blkaddr(struct f2fs_io_info *fio) 1408 { 1409 if (fio->old_blkaddr == NEW_ADDR) 1410 return false; 1411 if (fio->old_blkaddr == NULL_ADDR) 1412 return false; 1413 return true; 1414 } 1415 1416 int do_write_data_page(struct f2fs_io_info *fio) 1417 { 1418 struct page *page = fio->page; 1419 struct inode *inode = page->mapping->host; 1420 struct dnode_of_data dn; 1421 struct extent_info ei = {0,0,0}; 1422 bool ipu_force = false; 1423 int err = 0; 1424 1425 set_new_dnode(&dn, inode, NULL, NULL, 0); 1426 if (need_inplace_update(fio) && 1427 f2fs_lookup_extent_cache(inode, page->index, &ei)) { 1428 fio->old_blkaddr = ei.blk + page->index - ei.fofs; 1429 1430 if (valid_ipu_blkaddr(fio)) { 1431 ipu_force = true; 1432 fio->need_lock = LOCK_DONE; 1433 goto got_it; 1434 } 1435 } 1436 1437 /* Deadlock due to between page->lock and f2fs_lock_op */ 1438 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) 1439 return -EAGAIN; 1440 1441 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 1442 if (err) 1443 goto out; 1444 1445 fio->old_blkaddr = dn.data_blkaddr; 1446 1447 /* This page is already truncated */ 1448 if (fio->old_blkaddr == NULL_ADDR) { 1449 ClearPageUptodate(page); 1450 goto out_writepage; 1451 } 1452 got_it: 1453 /* 1454 * If current allocation needs SSR, 1455 * it had better in-place writes for updated data. 1456 */ 1457 if (ipu_force || (valid_ipu_blkaddr(fio) && need_inplace_update(fio))) { 1458 err = encrypt_one_page(fio); 1459 if (err) 1460 goto out_writepage; 1461 1462 set_page_writeback(page); 1463 f2fs_put_dnode(&dn); 1464 if (fio->need_lock == LOCK_REQ) 1465 f2fs_unlock_op(fio->sbi); 1466 err = rewrite_data_page(fio); 1467 trace_f2fs_do_write_data_page(fio->page, IPU); 1468 set_inode_flag(inode, FI_UPDATE_WRITE); 1469 return err; 1470 } 1471 1472 if (fio->need_lock == LOCK_RETRY) { 1473 if (!f2fs_trylock_op(fio->sbi)) { 1474 err = -EAGAIN; 1475 goto out_writepage; 1476 } 1477 fio->need_lock = LOCK_REQ; 1478 } 1479 1480 err = encrypt_one_page(fio); 1481 if (err) 1482 goto out_writepage; 1483 1484 set_page_writeback(page); 1485 1486 /* LFS mode write path */ 1487 write_data_page(&dn, fio); 1488 trace_f2fs_do_write_data_page(page, OPU); 1489 set_inode_flag(inode, FI_APPEND_WRITE); 1490 if (page->index == 0) 1491 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN); 1492 out_writepage: 1493 f2fs_put_dnode(&dn); 1494 out: 1495 if (fio->need_lock == LOCK_REQ) 1496 f2fs_unlock_op(fio->sbi); 1497 return err; 1498 } 1499 1500 static int __write_data_page(struct page *page, bool *submitted, 1501 struct writeback_control *wbc, 1502 enum iostat_type io_type) 1503 { 1504 struct inode *inode = page->mapping->host; 1505 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1506 loff_t i_size = i_size_read(inode); 1507 const pgoff_t end_index = ((unsigned long long) i_size) 1508 >> PAGE_SHIFT; 1509 loff_t psize = (page->index + 1) << PAGE_SHIFT; 1510 unsigned offset = 0; 1511 bool need_balance_fs = false; 1512 int err = 0; 1513 struct f2fs_io_info fio = { 1514 .sbi = sbi, 1515 .ino = inode->i_ino, 1516 .type = DATA, 1517 .op = REQ_OP_WRITE, 1518 .op_flags = wbc_to_write_flags(wbc), 1519 .old_blkaddr = NULL_ADDR, 1520 .page = page, 1521 .encrypted_page = NULL, 1522 .submitted = false, 1523 .need_lock = LOCK_RETRY, 1524 .io_type = io_type, 1525 }; 1526 1527 trace_f2fs_writepage(page, DATA); 1528 1529 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1530 goto redirty_out; 1531 1532 if (page->index < end_index) 1533 goto write; 1534 1535 /* 1536 * If the offset is out-of-range of file size, 1537 * this page does not have to be written to disk. 1538 */ 1539 offset = i_size & (PAGE_SIZE - 1); 1540 if ((page->index >= end_index + 1) || !offset) 1541 goto out; 1542 1543 zero_user_segment(page, offset, PAGE_SIZE); 1544 write: 1545 if (f2fs_is_drop_cache(inode)) 1546 goto out; 1547 /* we should not write 0'th page having journal header */ 1548 if (f2fs_is_volatile_file(inode) && (!page->index || 1549 (!wbc->for_reclaim && 1550 available_free_memory(sbi, BASE_CHECK)))) 1551 goto redirty_out; 1552 1553 /* we should bypass data pages to proceed the kworkder jobs */ 1554 if (unlikely(f2fs_cp_error(sbi))) { 1555 mapping_set_error(page->mapping, -EIO); 1556 goto out; 1557 } 1558 1559 /* Dentry blocks are controlled by checkpoint */ 1560 if (S_ISDIR(inode->i_mode)) { 1561 fio.need_lock = LOCK_DONE; 1562 err = do_write_data_page(&fio); 1563 goto done; 1564 } 1565 1566 if (!wbc->for_reclaim) 1567 need_balance_fs = true; 1568 else if (has_not_enough_free_secs(sbi, 0, 0)) 1569 goto redirty_out; 1570 else 1571 set_inode_flag(inode, FI_HOT_DATA); 1572 1573 err = -EAGAIN; 1574 if (f2fs_has_inline_data(inode)) { 1575 err = f2fs_write_inline_data(inode, page); 1576 if (!err) 1577 goto out; 1578 } 1579 1580 if (err == -EAGAIN) { 1581 err = do_write_data_page(&fio); 1582 if (err == -EAGAIN) { 1583 fio.need_lock = LOCK_REQ; 1584 err = do_write_data_page(&fio); 1585 } 1586 } 1587 1588 down_write(&F2FS_I(inode)->i_sem); 1589 if (F2FS_I(inode)->last_disk_size < psize) 1590 F2FS_I(inode)->last_disk_size = psize; 1591 up_write(&F2FS_I(inode)->i_sem); 1592 1593 done: 1594 if (err && err != -ENOENT) 1595 goto redirty_out; 1596 1597 out: 1598 inode_dec_dirty_pages(inode); 1599 if (err) 1600 ClearPageUptodate(page); 1601 1602 if (wbc->for_reclaim) { 1603 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA); 1604 clear_inode_flag(inode, FI_HOT_DATA); 1605 remove_dirty_inode(inode); 1606 submitted = NULL; 1607 } 1608 1609 unlock_page(page); 1610 if (!S_ISDIR(inode->i_mode)) 1611 f2fs_balance_fs(sbi, need_balance_fs); 1612 1613 if (unlikely(f2fs_cp_error(sbi))) { 1614 f2fs_submit_merged_write(sbi, DATA); 1615 submitted = NULL; 1616 } 1617 1618 if (submitted) 1619 *submitted = fio.submitted; 1620 1621 return 0; 1622 1623 redirty_out: 1624 redirty_page_for_writepage(wbc, page); 1625 if (!err) 1626 return AOP_WRITEPAGE_ACTIVATE; 1627 unlock_page(page); 1628 return err; 1629 } 1630 1631 static int f2fs_write_data_page(struct page *page, 1632 struct writeback_control *wbc) 1633 { 1634 return __write_data_page(page, NULL, wbc, FS_DATA_IO); 1635 } 1636 1637 /* 1638 * This function was copied from write_cche_pages from mm/page-writeback.c. 1639 * The major change is making write step of cold data page separately from 1640 * warm/hot data page. 1641 */ 1642 static int f2fs_write_cache_pages(struct address_space *mapping, 1643 struct writeback_control *wbc, 1644 enum iostat_type io_type) 1645 { 1646 int ret = 0; 1647 int done = 0; 1648 struct pagevec pvec; 1649 int nr_pages; 1650 pgoff_t uninitialized_var(writeback_index); 1651 pgoff_t index; 1652 pgoff_t end; /* Inclusive */ 1653 pgoff_t done_index; 1654 pgoff_t last_idx = ULONG_MAX; 1655 int cycled; 1656 int range_whole = 0; 1657 int tag; 1658 1659 pagevec_init(&pvec); 1660 1661 if (get_dirty_pages(mapping->host) <= 1662 SM_I(F2FS_M_SB(mapping))->min_hot_blocks) 1663 set_inode_flag(mapping->host, FI_HOT_DATA); 1664 else 1665 clear_inode_flag(mapping->host, FI_HOT_DATA); 1666 1667 if (wbc->range_cyclic) { 1668 writeback_index = mapping->writeback_index; /* prev offset */ 1669 index = writeback_index; 1670 if (index == 0) 1671 cycled = 1; 1672 else 1673 cycled = 0; 1674 end = -1; 1675 } else { 1676 index = wbc->range_start >> PAGE_SHIFT; 1677 end = wbc->range_end >> PAGE_SHIFT; 1678 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 1679 range_whole = 1; 1680 cycled = 1; /* ignore range_cyclic tests */ 1681 } 1682 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1683 tag = PAGECACHE_TAG_TOWRITE; 1684 else 1685 tag = PAGECACHE_TAG_DIRTY; 1686 retry: 1687 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1688 tag_pages_for_writeback(mapping, index, end); 1689 done_index = index; 1690 while (!done && (index <= end)) { 1691 int i; 1692 1693 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, 1694 tag); 1695 if (nr_pages == 0) 1696 break; 1697 1698 for (i = 0; i < nr_pages; i++) { 1699 struct page *page = pvec.pages[i]; 1700 bool submitted = false; 1701 1702 done_index = page->index; 1703 retry_write: 1704 lock_page(page); 1705 1706 if (unlikely(page->mapping != mapping)) { 1707 continue_unlock: 1708 unlock_page(page); 1709 continue; 1710 } 1711 1712 if (!PageDirty(page)) { 1713 /* someone wrote it for us */ 1714 goto continue_unlock; 1715 } 1716 1717 if (PageWriteback(page)) { 1718 if (wbc->sync_mode != WB_SYNC_NONE) 1719 f2fs_wait_on_page_writeback(page, 1720 DATA, true); 1721 else 1722 goto continue_unlock; 1723 } 1724 1725 BUG_ON(PageWriteback(page)); 1726 if (!clear_page_dirty_for_io(page)) 1727 goto continue_unlock; 1728 1729 ret = __write_data_page(page, &submitted, wbc, io_type); 1730 if (unlikely(ret)) { 1731 /* 1732 * keep nr_to_write, since vfs uses this to 1733 * get # of written pages. 1734 */ 1735 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1736 unlock_page(page); 1737 ret = 0; 1738 continue; 1739 } else if (ret == -EAGAIN) { 1740 ret = 0; 1741 if (wbc->sync_mode == WB_SYNC_ALL) { 1742 cond_resched(); 1743 congestion_wait(BLK_RW_ASYNC, 1744 HZ/50); 1745 goto retry_write; 1746 } 1747 continue; 1748 } 1749 done_index = page->index + 1; 1750 done = 1; 1751 break; 1752 } else if (submitted) { 1753 last_idx = page->index; 1754 } 1755 1756 /* give a priority to WB_SYNC threads */ 1757 if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) || 1758 --wbc->nr_to_write <= 0) && 1759 wbc->sync_mode == WB_SYNC_NONE) { 1760 done = 1; 1761 break; 1762 } 1763 } 1764 pagevec_release(&pvec); 1765 cond_resched(); 1766 } 1767 1768 if (!cycled && !done) { 1769 cycled = 1; 1770 index = 0; 1771 end = writeback_index - 1; 1772 goto retry; 1773 } 1774 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 1775 mapping->writeback_index = done_index; 1776 1777 if (last_idx != ULONG_MAX) 1778 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, 1779 0, last_idx, DATA); 1780 1781 return ret; 1782 } 1783 1784 int __f2fs_write_data_pages(struct address_space *mapping, 1785 struct writeback_control *wbc, 1786 enum iostat_type io_type) 1787 { 1788 struct inode *inode = mapping->host; 1789 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1790 struct blk_plug plug; 1791 int ret; 1792 1793 /* deal with chardevs and other special file */ 1794 if (!mapping->a_ops->writepage) 1795 return 0; 1796 1797 /* skip writing if there is no dirty page in this inode */ 1798 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 1799 return 0; 1800 1801 /* during POR, we don't need to trigger writepage at all. */ 1802 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1803 goto skip_write; 1804 1805 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE && 1806 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 1807 available_free_memory(sbi, DIRTY_DENTS)) 1808 goto skip_write; 1809 1810 /* skip writing during file defragment */ 1811 if (is_inode_flag_set(inode, FI_DO_DEFRAG)) 1812 goto skip_write; 1813 1814 trace_f2fs_writepages(mapping->host, wbc, DATA); 1815 1816 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */ 1817 if (wbc->sync_mode == WB_SYNC_ALL) 1818 atomic_inc(&sbi->wb_sync_req); 1819 else if (atomic_read(&sbi->wb_sync_req)) 1820 goto skip_write; 1821 1822 blk_start_plug(&plug); 1823 ret = f2fs_write_cache_pages(mapping, wbc, io_type); 1824 blk_finish_plug(&plug); 1825 1826 if (wbc->sync_mode == WB_SYNC_ALL) 1827 atomic_dec(&sbi->wb_sync_req); 1828 /* 1829 * if some pages were truncated, we cannot guarantee its mapping->host 1830 * to detect pending bios. 1831 */ 1832 1833 remove_dirty_inode(inode); 1834 return ret; 1835 1836 skip_write: 1837 wbc->pages_skipped += get_dirty_pages(inode); 1838 trace_f2fs_writepages(mapping->host, wbc, DATA); 1839 return 0; 1840 } 1841 1842 static int f2fs_write_data_pages(struct address_space *mapping, 1843 struct writeback_control *wbc) 1844 { 1845 struct inode *inode = mapping->host; 1846 1847 return __f2fs_write_data_pages(mapping, wbc, 1848 F2FS_I(inode)->cp_task == current ? 1849 FS_CP_DATA_IO : FS_DATA_IO); 1850 } 1851 1852 static void f2fs_write_failed(struct address_space *mapping, loff_t to) 1853 { 1854 struct inode *inode = mapping->host; 1855 loff_t i_size = i_size_read(inode); 1856 1857 if (to > i_size) { 1858 down_write(&F2FS_I(inode)->i_mmap_sem); 1859 truncate_pagecache(inode, i_size); 1860 truncate_blocks(inode, i_size, true); 1861 up_write(&F2FS_I(inode)->i_mmap_sem); 1862 } 1863 } 1864 1865 static int prepare_write_begin(struct f2fs_sb_info *sbi, 1866 struct page *page, loff_t pos, unsigned len, 1867 block_t *blk_addr, bool *node_changed) 1868 { 1869 struct inode *inode = page->mapping->host; 1870 pgoff_t index = page->index; 1871 struct dnode_of_data dn; 1872 struct page *ipage; 1873 bool locked = false; 1874 struct extent_info ei = {0,0,0}; 1875 int err = 0; 1876 1877 /* 1878 * we already allocated all the blocks, so we don't need to get 1879 * the block addresses when there is no need to fill the page. 1880 */ 1881 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE && 1882 !is_inode_flag_set(inode, FI_NO_PREALLOC)) 1883 return 0; 1884 1885 if (f2fs_has_inline_data(inode) || 1886 (pos & PAGE_MASK) >= i_size_read(inode)) { 1887 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true); 1888 locked = true; 1889 } 1890 restart: 1891 /* check inline_data */ 1892 ipage = get_node_page(sbi, inode->i_ino); 1893 if (IS_ERR(ipage)) { 1894 err = PTR_ERR(ipage); 1895 goto unlock_out; 1896 } 1897 1898 set_new_dnode(&dn, inode, ipage, ipage, 0); 1899 1900 if (f2fs_has_inline_data(inode)) { 1901 if (pos + len <= MAX_INLINE_DATA(inode)) { 1902 read_inline_data(page, ipage); 1903 set_inode_flag(inode, FI_DATA_EXIST); 1904 if (inode->i_nlink) 1905 set_inline_node(ipage); 1906 } else { 1907 err = f2fs_convert_inline_page(&dn, page); 1908 if (err) 1909 goto out; 1910 if (dn.data_blkaddr == NULL_ADDR) 1911 err = f2fs_get_block(&dn, index); 1912 } 1913 } else if (locked) { 1914 err = f2fs_get_block(&dn, index); 1915 } else { 1916 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 1917 dn.data_blkaddr = ei.blk + index - ei.fofs; 1918 } else { 1919 /* hole case */ 1920 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 1921 if (err || dn.data_blkaddr == NULL_ADDR) { 1922 f2fs_put_dnode(&dn); 1923 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, 1924 true); 1925 locked = true; 1926 goto restart; 1927 } 1928 } 1929 } 1930 1931 /* convert_inline_page can make node_changed */ 1932 *blk_addr = dn.data_blkaddr; 1933 *node_changed = dn.node_changed; 1934 out: 1935 f2fs_put_dnode(&dn); 1936 unlock_out: 1937 if (locked) 1938 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false); 1939 return err; 1940 } 1941 1942 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 1943 loff_t pos, unsigned len, unsigned flags, 1944 struct page **pagep, void **fsdata) 1945 { 1946 struct inode *inode = mapping->host; 1947 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1948 struct page *page = NULL; 1949 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT; 1950 bool need_balance = false; 1951 block_t blkaddr = NULL_ADDR; 1952 int err = 0; 1953 1954 trace_f2fs_write_begin(inode, pos, len, flags); 1955 1956 if (f2fs_is_atomic_file(inode) && 1957 !available_free_memory(sbi, INMEM_PAGES)) { 1958 err = -ENOMEM; 1959 goto fail; 1960 } 1961 1962 /* 1963 * We should check this at this moment to avoid deadlock on inode page 1964 * and #0 page. The locking rule for inline_data conversion should be: 1965 * lock_page(page #0) -> lock_page(inode_page) 1966 */ 1967 if (index != 0) { 1968 err = f2fs_convert_inline_inode(inode); 1969 if (err) 1970 goto fail; 1971 } 1972 repeat: 1973 /* 1974 * Do not use grab_cache_page_write_begin() to avoid deadlock due to 1975 * wait_for_stable_page. Will wait that below with our IO control. 1976 */ 1977 page = f2fs_pagecache_get_page(mapping, index, 1978 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS); 1979 if (!page) { 1980 err = -ENOMEM; 1981 goto fail; 1982 } 1983 1984 *pagep = page; 1985 1986 err = prepare_write_begin(sbi, page, pos, len, 1987 &blkaddr, &need_balance); 1988 if (err) 1989 goto fail; 1990 1991 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) { 1992 unlock_page(page); 1993 f2fs_balance_fs(sbi, true); 1994 lock_page(page); 1995 if (page->mapping != mapping) { 1996 /* The page got truncated from under us */ 1997 f2fs_put_page(page, 1); 1998 goto repeat; 1999 } 2000 } 2001 2002 f2fs_wait_on_page_writeback(page, DATA, false); 2003 2004 /* wait for GCed encrypted page writeback */ 2005 if (f2fs_encrypted_file(inode)) 2006 f2fs_wait_on_block_writeback(sbi, blkaddr); 2007 2008 if (len == PAGE_SIZE || PageUptodate(page)) 2009 return 0; 2010 2011 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) { 2012 zero_user_segment(page, len, PAGE_SIZE); 2013 return 0; 2014 } 2015 2016 if (blkaddr == NEW_ADDR) { 2017 zero_user_segment(page, 0, PAGE_SIZE); 2018 SetPageUptodate(page); 2019 } else { 2020 err = f2fs_submit_page_read(inode, page, blkaddr); 2021 if (err) 2022 goto fail; 2023 2024 lock_page(page); 2025 if (unlikely(page->mapping != mapping)) { 2026 f2fs_put_page(page, 1); 2027 goto repeat; 2028 } 2029 if (unlikely(!PageUptodate(page))) { 2030 err = -EIO; 2031 goto fail; 2032 } 2033 } 2034 return 0; 2035 2036 fail: 2037 f2fs_put_page(page, 1); 2038 f2fs_write_failed(mapping, pos + len); 2039 if (f2fs_is_atomic_file(inode)) 2040 drop_inmem_pages_all(sbi); 2041 return err; 2042 } 2043 2044 static int f2fs_write_end(struct file *file, 2045 struct address_space *mapping, 2046 loff_t pos, unsigned len, unsigned copied, 2047 struct page *page, void *fsdata) 2048 { 2049 struct inode *inode = page->mapping->host; 2050 2051 trace_f2fs_write_end(inode, pos, len, copied); 2052 2053 /* 2054 * This should be come from len == PAGE_SIZE, and we expect copied 2055 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and 2056 * let generic_perform_write() try to copy data again through copied=0. 2057 */ 2058 if (!PageUptodate(page)) { 2059 if (unlikely(copied != len)) 2060 copied = 0; 2061 else 2062 SetPageUptodate(page); 2063 } 2064 if (!copied) 2065 goto unlock_out; 2066 2067 set_page_dirty(page); 2068 2069 if (pos + copied > i_size_read(inode)) 2070 f2fs_i_size_write(inode, pos + copied); 2071 unlock_out: 2072 f2fs_put_page(page, 1); 2073 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 2074 return copied; 2075 } 2076 2077 static int check_direct_IO(struct inode *inode, struct iov_iter *iter, 2078 loff_t offset) 2079 { 2080 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1; 2081 2082 if (offset & blocksize_mask) 2083 return -EINVAL; 2084 2085 if (iov_iter_alignment(iter) & blocksize_mask) 2086 return -EINVAL; 2087 2088 return 0; 2089 } 2090 2091 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 2092 { 2093 struct address_space *mapping = iocb->ki_filp->f_mapping; 2094 struct inode *inode = mapping->host; 2095 size_t count = iov_iter_count(iter); 2096 loff_t offset = iocb->ki_pos; 2097 int rw = iov_iter_rw(iter); 2098 int err; 2099 2100 err = check_direct_IO(inode, iter, offset); 2101 if (err) 2102 return err; 2103 2104 if (__force_buffered_io(inode, rw)) 2105 return 0; 2106 2107 trace_f2fs_direct_IO_enter(inode, offset, count, rw); 2108 2109 down_read(&F2FS_I(inode)->dio_rwsem[rw]); 2110 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio); 2111 up_read(&F2FS_I(inode)->dio_rwsem[rw]); 2112 2113 if (rw == WRITE) { 2114 if (err > 0) { 2115 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO, 2116 err); 2117 set_inode_flag(inode, FI_UPDATE_WRITE); 2118 } else if (err < 0) { 2119 f2fs_write_failed(mapping, offset + count); 2120 } 2121 } 2122 2123 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err); 2124 2125 return err; 2126 } 2127 2128 void f2fs_invalidate_page(struct page *page, unsigned int offset, 2129 unsigned int length) 2130 { 2131 struct inode *inode = page->mapping->host; 2132 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2133 2134 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 2135 (offset % PAGE_SIZE || length != PAGE_SIZE)) 2136 return; 2137 2138 if (PageDirty(page)) { 2139 if (inode->i_ino == F2FS_META_INO(sbi)) { 2140 dec_page_count(sbi, F2FS_DIRTY_META); 2141 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { 2142 dec_page_count(sbi, F2FS_DIRTY_NODES); 2143 } else { 2144 inode_dec_dirty_pages(inode); 2145 remove_dirty_inode(inode); 2146 } 2147 } 2148 2149 /* This is atomic written page, keep Private */ 2150 if (IS_ATOMIC_WRITTEN_PAGE(page)) 2151 return drop_inmem_page(inode, page); 2152 2153 set_page_private(page, 0); 2154 ClearPagePrivate(page); 2155 } 2156 2157 int f2fs_release_page(struct page *page, gfp_t wait) 2158 { 2159 /* If this is dirty page, keep PagePrivate */ 2160 if (PageDirty(page)) 2161 return 0; 2162 2163 /* This is atomic written page, keep Private */ 2164 if (IS_ATOMIC_WRITTEN_PAGE(page)) 2165 return 0; 2166 2167 set_page_private(page, 0); 2168 ClearPagePrivate(page); 2169 return 1; 2170 } 2171 2172 /* 2173 * This was copied from __set_page_dirty_buffers which gives higher performance 2174 * in very high speed storages. (e.g., pmem) 2175 */ 2176 void f2fs_set_page_dirty_nobuffers(struct page *page) 2177 { 2178 struct address_space *mapping = page->mapping; 2179 unsigned long flags; 2180 2181 if (unlikely(!mapping)) 2182 return; 2183 2184 spin_lock(&mapping->private_lock); 2185 lock_page_memcg(page); 2186 SetPageDirty(page); 2187 spin_unlock(&mapping->private_lock); 2188 2189 spin_lock_irqsave(&mapping->tree_lock, flags); 2190 WARN_ON_ONCE(!PageUptodate(page)); 2191 account_page_dirtied(page, mapping); 2192 radix_tree_tag_set(&mapping->page_tree, 2193 page_index(page), PAGECACHE_TAG_DIRTY); 2194 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2195 unlock_page_memcg(page); 2196 2197 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 2198 return; 2199 } 2200 2201 static int f2fs_set_data_page_dirty(struct page *page) 2202 { 2203 struct address_space *mapping = page->mapping; 2204 struct inode *inode = mapping->host; 2205 2206 trace_f2fs_set_page_dirty(page, DATA); 2207 2208 if (!PageUptodate(page)) 2209 SetPageUptodate(page); 2210 2211 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) { 2212 if (!IS_ATOMIC_WRITTEN_PAGE(page)) { 2213 register_inmem_page(inode, page); 2214 return 1; 2215 } 2216 /* 2217 * Previously, this page has been registered, we just 2218 * return here. 2219 */ 2220 return 0; 2221 } 2222 2223 if (!PageDirty(page)) { 2224 f2fs_set_page_dirty_nobuffers(page); 2225 update_dirty_page(inode, page); 2226 return 1; 2227 } 2228 return 0; 2229 } 2230 2231 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 2232 { 2233 struct inode *inode = mapping->host; 2234 2235 if (f2fs_has_inline_data(inode)) 2236 return 0; 2237 2238 /* make sure allocating whole blocks */ 2239 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 2240 filemap_write_and_wait(mapping); 2241 2242 return generic_block_bmap(mapping, block, get_data_block_bmap); 2243 } 2244 2245 #ifdef CONFIG_MIGRATION 2246 #include <linux/migrate.h> 2247 2248 int f2fs_migrate_page(struct address_space *mapping, 2249 struct page *newpage, struct page *page, enum migrate_mode mode) 2250 { 2251 int rc, extra_count; 2252 struct f2fs_inode_info *fi = F2FS_I(mapping->host); 2253 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page); 2254 2255 BUG_ON(PageWriteback(page)); 2256 2257 /* migrating an atomic written page is safe with the inmem_lock hold */ 2258 if (atomic_written) { 2259 if (mode != MIGRATE_SYNC) 2260 return -EBUSY; 2261 if (!mutex_trylock(&fi->inmem_lock)) 2262 return -EAGAIN; 2263 } 2264 2265 /* 2266 * A reference is expected if PagePrivate set when move mapping, 2267 * however F2FS breaks this for maintaining dirty page counts when 2268 * truncating pages. So here adjusting the 'extra_count' make it work. 2269 */ 2270 extra_count = (atomic_written ? 1 : 0) - page_has_private(page); 2271 rc = migrate_page_move_mapping(mapping, newpage, 2272 page, NULL, mode, extra_count); 2273 if (rc != MIGRATEPAGE_SUCCESS) { 2274 if (atomic_written) 2275 mutex_unlock(&fi->inmem_lock); 2276 return rc; 2277 } 2278 2279 if (atomic_written) { 2280 struct inmem_pages *cur; 2281 list_for_each_entry(cur, &fi->inmem_pages, list) 2282 if (cur->page == page) { 2283 cur->page = newpage; 2284 break; 2285 } 2286 mutex_unlock(&fi->inmem_lock); 2287 put_page(page); 2288 get_page(newpage); 2289 } 2290 2291 if (PagePrivate(page)) 2292 SetPagePrivate(newpage); 2293 set_page_private(newpage, page_private(page)); 2294 2295 if (mode != MIGRATE_SYNC_NO_COPY) 2296 migrate_page_copy(newpage, page); 2297 else 2298 migrate_page_states(newpage, page); 2299 2300 return MIGRATEPAGE_SUCCESS; 2301 } 2302 #endif 2303 2304 const struct address_space_operations f2fs_dblock_aops = { 2305 .readpage = f2fs_read_data_page, 2306 .readpages = f2fs_read_data_pages, 2307 .writepage = f2fs_write_data_page, 2308 .writepages = f2fs_write_data_pages, 2309 .write_begin = f2fs_write_begin, 2310 .write_end = f2fs_write_end, 2311 .set_page_dirty = f2fs_set_data_page_dirty, 2312 .invalidatepage = f2fs_invalidate_page, 2313 .releasepage = f2fs_release_page, 2314 .direct_IO = f2fs_direct_IO, 2315 .bmap = f2fs_bmap, 2316 #ifdef CONFIG_MIGRATION 2317 .migratepage = f2fs_migrate_page, 2318 #endif 2319 }; 2320