1 /* 2 * fs/f2fs/data.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/fs.h> 12 #include <linux/f2fs_fs.h> 13 #include <linux/buffer_head.h> 14 #include <linux/mpage.h> 15 #include <linux/writeback.h> 16 #include <linux/backing-dev.h> 17 #include <linux/pagevec.h> 18 #include <linux/blkdev.h> 19 #include <linux/bio.h> 20 #include <linux/prefetch.h> 21 #include <linux/uio.h> 22 #include <linux/cleancache.h> 23 24 #include "f2fs.h" 25 #include "node.h" 26 #include "segment.h" 27 #include "trace.h" 28 #include <trace/events/f2fs.h> 29 30 static void f2fs_read_end_io(struct bio *bio) 31 { 32 struct bio_vec *bvec; 33 int i; 34 35 if (f2fs_bio_encrypted(bio)) { 36 if (bio->bi_error) { 37 f2fs_release_crypto_ctx(bio->bi_private); 38 } else { 39 f2fs_end_io_crypto_work(bio->bi_private, bio); 40 return; 41 } 42 } 43 44 bio_for_each_segment_all(bvec, bio, i) { 45 struct page *page = bvec->bv_page; 46 47 if (!bio->bi_error) { 48 SetPageUptodate(page); 49 } else { 50 ClearPageUptodate(page); 51 SetPageError(page); 52 } 53 unlock_page(page); 54 } 55 bio_put(bio); 56 } 57 58 static void f2fs_write_end_io(struct bio *bio) 59 { 60 struct f2fs_sb_info *sbi = bio->bi_private; 61 struct bio_vec *bvec; 62 int i; 63 64 bio_for_each_segment_all(bvec, bio, i) { 65 struct page *page = bvec->bv_page; 66 67 f2fs_restore_and_release_control_page(&page); 68 69 if (unlikely(bio->bi_error)) { 70 set_page_dirty(page); 71 set_bit(AS_EIO, &page->mapping->flags); 72 f2fs_stop_checkpoint(sbi); 73 } 74 end_page_writeback(page); 75 dec_page_count(sbi, F2FS_WRITEBACK); 76 } 77 78 if (!get_pages(sbi, F2FS_WRITEBACK) && 79 !list_empty(&sbi->cp_wait.task_list)) 80 wake_up(&sbi->cp_wait); 81 82 bio_put(bio); 83 } 84 85 /* 86 * Low-level block read/write IO operations. 87 */ 88 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, 89 int npages, bool is_read) 90 { 91 struct bio *bio; 92 93 bio = f2fs_bio_alloc(npages); 94 95 bio->bi_bdev = sbi->sb->s_bdev; 96 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); 97 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; 98 bio->bi_private = is_read ? NULL : sbi; 99 100 return bio; 101 } 102 103 static void __submit_merged_bio(struct f2fs_bio_info *io) 104 { 105 struct f2fs_io_info *fio = &io->fio; 106 107 if (!io->bio) 108 return; 109 110 if (is_read_io(fio->rw)) 111 trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio); 112 else 113 trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio); 114 115 submit_bio(fio->rw, io->bio); 116 io->bio = NULL; 117 } 118 119 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi, 120 enum page_type type, int rw) 121 { 122 enum page_type btype = PAGE_TYPE_OF_BIO(type); 123 struct f2fs_bio_info *io; 124 125 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype]; 126 127 down_write(&io->io_rwsem); 128 129 /* change META to META_FLUSH in the checkpoint procedure */ 130 if (type >= META_FLUSH) { 131 io->fio.type = META_FLUSH; 132 if (test_opt(sbi, NOBARRIER)) 133 io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO; 134 else 135 io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO; 136 } 137 __submit_merged_bio(io); 138 up_write(&io->io_rwsem); 139 } 140 141 /* 142 * Fill the locked page with data located in the block address. 143 * Return unlocked page. 144 */ 145 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 146 { 147 struct bio *bio; 148 struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; 149 150 trace_f2fs_submit_page_bio(page, fio); 151 f2fs_trace_ios(fio, 0); 152 153 /* Allocate a new bio */ 154 bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw)); 155 156 if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) { 157 bio_put(bio); 158 return -EFAULT; 159 } 160 161 submit_bio(fio->rw, bio); 162 return 0; 163 } 164 165 void f2fs_submit_page_mbio(struct f2fs_io_info *fio) 166 { 167 struct f2fs_sb_info *sbi = fio->sbi; 168 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 169 struct f2fs_bio_info *io; 170 bool is_read = is_read_io(fio->rw); 171 struct page *bio_page; 172 173 io = is_read ? &sbi->read_io : &sbi->write_io[btype]; 174 175 verify_block_addr(sbi, fio->blk_addr); 176 177 down_write(&io->io_rwsem); 178 179 if (!is_read) 180 inc_page_count(sbi, F2FS_WRITEBACK); 181 182 if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 || 183 io->fio.rw != fio->rw)) 184 __submit_merged_bio(io); 185 alloc_new: 186 if (io->bio == NULL) { 187 int bio_blocks = MAX_BIO_BLOCKS(sbi); 188 189 io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read); 190 io->fio = *fio; 191 } 192 193 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; 194 195 if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) < 196 PAGE_CACHE_SIZE) { 197 __submit_merged_bio(io); 198 goto alloc_new; 199 } 200 201 io->last_block_in_bio = fio->blk_addr; 202 f2fs_trace_ios(fio, 0); 203 204 up_write(&io->io_rwsem); 205 trace_f2fs_submit_page_mbio(fio->page, fio); 206 } 207 208 /* 209 * Lock ordering for the change of data block address: 210 * ->data_page 211 * ->node_page 212 * update block addresses in the node page 213 */ 214 void set_data_blkaddr(struct dnode_of_data *dn) 215 { 216 struct f2fs_node *rn; 217 __le32 *addr_array; 218 struct page *node_page = dn->node_page; 219 unsigned int ofs_in_node = dn->ofs_in_node; 220 221 f2fs_wait_on_page_writeback(node_page, NODE); 222 223 rn = F2FS_NODE(node_page); 224 225 /* Get physical address of data block */ 226 addr_array = blkaddr_in_node(rn); 227 addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 228 set_page_dirty(node_page); 229 } 230 231 int reserve_new_block(struct dnode_of_data *dn) 232 { 233 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 234 235 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 236 return -EPERM; 237 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 238 return -ENOSPC; 239 240 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node); 241 242 dn->data_blkaddr = NEW_ADDR; 243 set_data_blkaddr(dn); 244 mark_inode_dirty(dn->inode); 245 sync_inode_page(dn); 246 return 0; 247 } 248 249 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 250 { 251 bool need_put = dn->inode_page ? false : true; 252 int err; 253 254 err = get_dnode_of_data(dn, index, ALLOC_NODE); 255 if (err) 256 return err; 257 258 if (dn->data_blkaddr == NULL_ADDR) 259 err = reserve_new_block(dn); 260 if (err || need_put) 261 f2fs_put_dnode(dn); 262 return err; 263 } 264 265 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) 266 { 267 struct extent_info ei; 268 struct inode *inode = dn->inode; 269 270 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 271 dn->data_blkaddr = ei.blk + index - ei.fofs; 272 return 0; 273 } 274 275 return f2fs_reserve_block(dn, index); 276 } 277 278 struct page *get_read_data_page(struct inode *inode, pgoff_t index, 279 int rw, bool for_write) 280 { 281 struct address_space *mapping = inode->i_mapping; 282 struct dnode_of_data dn; 283 struct page *page; 284 struct extent_info ei; 285 int err; 286 struct f2fs_io_info fio = { 287 .sbi = F2FS_I_SB(inode), 288 .type = DATA, 289 .rw = rw, 290 .encrypted_page = NULL, 291 }; 292 293 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 294 return read_mapping_page(mapping, index, NULL); 295 296 page = f2fs_grab_cache_page(mapping, index, for_write); 297 if (!page) 298 return ERR_PTR(-ENOMEM); 299 300 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 301 dn.data_blkaddr = ei.blk + index - ei.fofs; 302 goto got_it; 303 } 304 305 set_new_dnode(&dn, inode, NULL, NULL, 0); 306 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 307 if (err) 308 goto put_err; 309 f2fs_put_dnode(&dn); 310 311 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 312 err = -ENOENT; 313 goto put_err; 314 } 315 got_it: 316 if (PageUptodate(page)) { 317 unlock_page(page); 318 return page; 319 } 320 321 /* 322 * A new dentry page is allocated but not able to be written, since its 323 * new inode page couldn't be allocated due to -ENOSPC. 324 * In such the case, its blkaddr can be remained as NEW_ADDR. 325 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata. 326 */ 327 if (dn.data_blkaddr == NEW_ADDR) { 328 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 329 SetPageUptodate(page); 330 unlock_page(page); 331 return page; 332 } 333 334 fio.blk_addr = dn.data_blkaddr; 335 fio.page = page; 336 err = f2fs_submit_page_bio(&fio); 337 if (err) 338 goto put_err; 339 return page; 340 341 put_err: 342 f2fs_put_page(page, 1); 343 return ERR_PTR(err); 344 } 345 346 struct page *find_data_page(struct inode *inode, pgoff_t index) 347 { 348 struct address_space *mapping = inode->i_mapping; 349 struct page *page; 350 351 page = find_get_page(mapping, index); 352 if (page && PageUptodate(page)) 353 return page; 354 f2fs_put_page(page, 0); 355 356 page = get_read_data_page(inode, index, READ_SYNC, false); 357 if (IS_ERR(page)) 358 return page; 359 360 if (PageUptodate(page)) 361 return page; 362 363 wait_on_page_locked(page); 364 if (unlikely(!PageUptodate(page))) { 365 f2fs_put_page(page, 0); 366 return ERR_PTR(-EIO); 367 } 368 return page; 369 } 370 371 /* 372 * If it tries to access a hole, return an error. 373 * Because, the callers, functions in dir.c and GC, should be able to know 374 * whether this page exists or not. 375 */ 376 struct page *get_lock_data_page(struct inode *inode, pgoff_t index, 377 bool for_write) 378 { 379 struct address_space *mapping = inode->i_mapping; 380 struct page *page; 381 repeat: 382 page = get_read_data_page(inode, index, READ_SYNC, for_write); 383 if (IS_ERR(page)) 384 return page; 385 386 /* wait for read completion */ 387 lock_page(page); 388 if (unlikely(!PageUptodate(page))) { 389 f2fs_put_page(page, 1); 390 return ERR_PTR(-EIO); 391 } 392 if (unlikely(page->mapping != mapping)) { 393 f2fs_put_page(page, 1); 394 goto repeat; 395 } 396 return page; 397 } 398 399 /* 400 * Caller ensures that this data page is never allocated. 401 * A new zero-filled data page is allocated in the page cache. 402 * 403 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 404 * f2fs_unlock_op(). 405 * Note that, ipage is set only by make_empty_dir, and if any error occur, 406 * ipage should be released by this function. 407 */ 408 struct page *get_new_data_page(struct inode *inode, 409 struct page *ipage, pgoff_t index, bool new_i_size) 410 { 411 struct address_space *mapping = inode->i_mapping; 412 struct page *page; 413 struct dnode_of_data dn; 414 int err; 415 repeat: 416 page = f2fs_grab_cache_page(mapping, index, true); 417 if (!page) { 418 /* 419 * before exiting, we should make sure ipage will be released 420 * if any error occur. 421 */ 422 f2fs_put_page(ipage, 1); 423 return ERR_PTR(-ENOMEM); 424 } 425 426 set_new_dnode(&dn, inode, ipage, NULL, 0); 427 err = f2fs_reserve_block(&dn, index); 428 if (err) { 429 f2fs_put_page(page, 1); 430 return ERR_PTR(err); 431 } 432 if (!ipage) 433 f2fs_put_dnode(&dn); 434 435 if (PageUptodate(page)) 436 goto got_it; 437 438 if (dn.data_blkaddr == NEW_ADDR) { 439 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 440 SetPageUptodate(page); 441 } else { 442 f2fs_put_page(page, 1); 443 444 page = get_read_data_page(inode, index, READ_SYNC, true); 445 if (IS_ERR(page)) 446 goto repeat; 447 448 /* wait for read completion */ 449 lock_page(page); 450 } 451 got_it: 452 if (new_i_size && i_size_read(inode) < 453 ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)) { 454 i_size_write(inode, ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)); 455 /* Only the directory inode sets new_i_size */ 456 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR); 457 } 458 return page; 459 } 460 461 static int __allocate_data_block(struct dnode_of_data *dn) 462 { 463 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 464 struct f2fs_inode_info *fi = F2FS_I(dn->inode); 465 struct f2fs_summary sum; 466 struct node_info ni; 467 int seg = CURSEG_WARM_DATA; 468 pgoff_t fofs; 469 470 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 471 return -EPERM; 472 473 dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node); 474 if (dn->data_blkaddr == NEW_ADDR) 475 goto alloc; 476 477 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 478 return -ENOSPC; 479 480 alloc: 481 get_node_info(sbi, dn->nid, &ni); 482 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 483 484 if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page) 485 seg = CURSEG_DIRECT_IO; 486 487 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr, 488 &sum, seg); 489 set_data_blkaddr(dn); 490 491 /* update i_size */ 492 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) + 493 dn->ofs_in_node; 494 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)) 495 i_size_write(dn->inode, 496 ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)); 497 498 /* direct IO doesn't use extent cache to maximize the performance */ 499 f2fs_drop_largest_extent(dn->inode, fofs); 500 501 return 0; 502 } 503 504 static void __allocate_data_blocks(struct inode *inode, loff_t offset, 505 size_t count) 506 { 507 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 508 struct dnode_of_data dn; 509 u64 start = F2FS_BYTES_TO_BLK(offset); 510 u64 len = F2FS_BYTES_TO_BLK(count); 511 bool allocated; 512 u64 end_offset; 513 514 while (len) { 515 f2fs_balance_fs(sbi); 516 f2fs_lock_op(sbi); 517 518 /* When reading holes, we need its node page */ 519 set_new_dnode(&dn, inode, NULL, NULL, 0); 520 if (get_dnode_of_data(&dn, start, ALLOC_NODE)) 521 goto out; 522 523 allocated = false; 524 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 525 526 while (dn.ofs_in_node < end_offset && len) { 527 block_t blkaddr; 528 529 if (unlikely(f2fs_cp_error(sbi))) 530 goto sync_out; 531 532 blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); 533 if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) { 534 if (__allocate_data_block(&dn)) 535 goto sync_out; 536 allocated = true; 537 } 538 len--; 539 start++; 540 dn.ofs_in_node++; 541 } 542 543 if (allocated) 544 sync_inode_page(&dn); 545 546 f2fs_put_dnode(&dn); 547 f2fs_unlock_op(sbi); 548 } 549 return; 550 551 sync_out: 552 if (allocated) 553 sync_inode_page(&dn); 554 f2fs_put_dnode(&dn); 555 out: 556 f2fs_unlock_op(sbi); 557 return; 558 } 559 560 /* 561 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with 562 * f2fs_map_blocks structure. 563 * If original data blocks are allocated, then give them to blockdev. 564 * Otherwise, 565 * a. preallocate requested block addresses 566 * b. do not use extent cache for better performance 567 * c. give the block addresses to blockdev 568 */ 569 static int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 570 int create, int flag) 571 { 572 unsigned int maxblocks = map->m_len; 573 struct dnode_of_data dn; 574 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 575 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA; 576 pgoff_t pgofs, end_offset; 577 int err = 0, ofs = 1; 578 struct extent_info ei; 579 bool allocated = false; 580 581 map->m_len = 0; 582 map->m_flags = 0; 583 584 /* it only supports block size == page size */ 585 pgofs = (pgoff_t)map->m_lblk; 586 587 if (f2fs_lookup_extent_cache(inode, pgofs, &ei)) { 588 map->m_pblk = ei.blk + pgofs - ei.fofs; 589 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); 590 map->m_flags = F2FS_MAP_MAPPED; 591 goto out; 592 } 593 594 if (create) 595 f2fs_lock_op(F2FS_I_SB(inode)); 596 597 /* When reading holes, we need its node page */ 598 set_new_dnode(&dn, inode, NULL, NULL, 0); 599 err = get_dnode_of_data(&dn, pgofs, mode); 600 if (err) { 601 if (err == -ENOENT) 602 err = 0; 603 goto unlock_out; 604 } 605 606 if (dn.data_blkaddr == NEW_ADDR || dn.data_blkaddr == NULL_ADDR) { 607 if (create) { 608 if (unlikely(f2fs_cp_error(sbi))) { 609 err = -EIO; 610 goto put_out; 611 } 612 err = __allocate_data_block(&dn); 613 if (err) 614 goto put_out; 615 allocated = true; 616 map->m_flags = F2FS_MAP_NEW; 617 } else { 618 if (flag != F2FS_GET_BLOCK_FIEMAP || 619 dn.data_blkaddr != NEW_ADDR) { 620 if (flag == F2FS_GET_BLOCK_BMAP) 621 err = -ENOENT; 622 goto put_out; 623 } 624 625 /* 626 * preallocated unwritten block should be mapped 627 * for fiemap. 628 */ 629 if (dn.data_blkaddr == NEW_ADDR) 630 map->m_flags = F2FS_MAP_UNWRITTEN; 631 } 632 } 633 634 map->m_flags |= F2FS_MAP_MAPPED; 635 map->m_pblk = dn.data_blkaddr; 636 map->m_len = 1; 637 638 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 639 dn.ofs_in_node++; 640 pgofs++; 641 642 get_next: 643 if (dn.ofs_in_node >= end_offset) { 644 if (allocated) 645 sync_inode_page(&dn); 646 allocated = false; 647 f2fs_put_dnode(&dn); 648 649 set_new_dnode(&dn, inode, NULL, NULL, 0); 650 err = get_dnode_of_data(&dn, pgofs, mode); 651 if (err) { 652 if (err == -ENOENT) 653 err = 0; 654 goto unlock_out; 655 } 656 657 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 658 } 659 660 if (maxblocks > map->m_len) { 661 block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); 662 663 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) { 664 if (create) { 665 if (unlikely(f2fs_cp_error(sbi))) { 666 err = -EIO; 667 goto sync_out; 668 } 669 err = __allocate_data_block(&dn); 670 if (err) 671 goto sync_out; 672 allocated = true; 673 map->m_flags |= F2FS_MAP_NEW; 674 blkaddr = dn.data_blkaddr; 675 } else { 676 /* 677 * we only merge preallocated unwritten blocks 678 * for fiemap. 679 */ 680 if (flag != F2FS_GET_BLOCK_FIEMAP || 681 blkaddr != NEW_ADDR) 682 goto sync_out; 683 } 684 } 685 686 /* Give more consecutive addresses for the readahead */ 687 if ((map->m_pblk != NEW_ADDR && 688 blkaddr == (map->m_pblk + ofs)) || 689 (map->m_pblk == NEW_ADDR && 690 blkaddr == NEW_ADDR)) { 691 ofs++; 692 dn.ofs_in_node++; 693 pgofs++; 694 map->m_len++; 695 goto get_next; 696 } 697 } 698 sync_out: 699 if (allocated) 700 sync_inode_page(&dn); 701 put_out: 702 f2fs_put_dnode(&dn); 703 unlock_out: 704 if (create) 705 f2fs_unlock_op(F2FS_I_SB(inode)); 706 out: 707 trace_f2fs_map_blocks(inode, map, err); 708 return err; 709 } 710 711 static int __get_data_block(struct inode *inode, sector_t iblock, 712 struct buffer_head *bh, int create, int flag) 713 { 714 struct f2fs_map_blocks map; 715 int ret; 716 717 map.m_lblk = iblock; 718 map.m_len = bh->b_size >> inode->i_blkbits; 719 720 ret = f2fs_map_blocks(inode, &map, create, flag); 721 if (!ret) { 722 map_bh(bh, inode->i_sb, map.m_pblk); 723 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; 724 bh->b_size = map.m_len << inode->i_blkbits; 725 } 726 return ret; 727 } 728 729 static int get_data_block(struct inode *inode, sector_t iblock, 730 struct buffer_head *bh_result, int create, int flag) 731 { 732 return __get_data_block(inode, iblock, bh_result, create, flag); 733 } 734 735 static int get_data_block_dio(struct inode *inode, sector_t iblock, 736 struct buffer_head *bh_result, int create) 737 { 738 return __get_data_block(inode, iblock, bh_result, create, 739 F2FS_GET_BLOCK_DIO); 740 } 741 742 static int get_data_block_bmap(struct inode *inode, sector_t iblock, 743 struct buffer_head *bh_result, int create) 744 { 745 return __get_data_block(inode, iblock, bh_result, create, 746 F2FS_GET_BLOCK_BMAP); 747 } 748 749 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset) 750 { 751 return (offset >> inode->i_blkbits); 752 } 753 754 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk) 755 { 756 return (blk << inode->i_blkbits); 757 } 758 759 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 760 u64 start, u64 len) 761 { 762 struct buffer_head map_bh; 763 sector_t start_blk, last_blk; 764 loff_t isize = i_size_read(inode); 765 u64 logical = 0, phys = 0, size = 0; 766 u32 flags = 0; 767 bool past_eof = false, whole_file = false; 768 int ret = 0; 769 770 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); 771 if (ret) 772 return ret; 773 774 mutex_lock(&inode->i_mutex); 775 776 if (len >= isize) { 777 whole_file = true; 778 len = isize; 779 } 780 781 if (logical_to_blk(inode, len) == 0) 782 len = blk_to_logical(inode, 1); 783 784 start_blk = logical_to_blk(inode, start); 785 last_blk = logical_to_blk(inode, start + len - 1); 786 next: 787 memset(&map_bh, 0, sizeof(struct buffer_head)); 788 map_bh.b_size = len; 789 790 ret = get_data_block(inode, start_blk, &map_bh, 0, 791 F2FS_GET_BLOCK_FIEMAP); 792 if (ret) 793 goto out; 794 795 /* HOLE */ 796 if (!buffer_mapped(&map_bh)) { 797 start_blk++; 798 799 if (!past_eof && blk_to_logical(inode, start_blk) >= isize) 800 past_eof = 1; 801 802 if (past_eof && size) { 803 flags |= FIEMAP_EXTENT_LAST; 804 ret = fiemap_fill_next_extent(fieinfo, logical, 805 phys, size, flags); 806 } else if (size) { 807 ret = fiemap_fill_next_extent(fieinfo, logical, 808 phys, size, flags); 809 size = 0; 810 } 811 812 /* if we have holes up to/past EOF then we're done */ 813 if (start_blk > last_blk || past_eof || ret) 814 goto out; 815 } else { 816 if (start_blk > last_blk && !whole_file) { 817 ret = fiemap_fill_next_extent(fieinfo, logical, 818 phys, size, flags); 819 goto out; 820 } 821 822 /* 823 * if size != 0 then we know we already have an extent 824 * to add, so add it. 825 */ 826 if (size) { 827 ret = fiemap_fill_next_extent(fieinfo, logical, 828 phys, size, flags); 829 if (ret) 830 goto out; 831 } 832 833 logical = blk_to_logical(inode, start_blk); 834 phys = blk_to_logical(inode, map_bh.b_blocknr); 835 size = map_bh.b_size; 836 flags = 0; 837 if (buffer_unwritten(&map_bh)) 838 flags = FIEMAP_EXTENT_UNWRITTEN; 839 840 start_blk += logical_to_blk(inode, size); 841 842 /* 843 * If we are past the EOF, then we need to make sure as 844 * soon as we find a hole that the last extent we found 845 * is marked with FIEMAP_EXTENT_LAST 846 */ 847 if (!past_eof && logical + size >= isize) 848 past_eof = true; 849 } 850 cond_resched(); 851 if (fatal_signal_pending(current)) 852 ret = -EINTR; 853 else 854 goto next; 855 out: 856 if (ret == 1) 857 ret = 0; 858 859 mutex_unlock(&inode->i_mutex); 860 return ret; 861 } 862 863 /* 864 * This function was originally taken from fs/mpage.c, and customized for f2fs. 865 * Major change was from block_size == page_size in f2fs by default. 866 */ 867 static int f2fs_mpage_readpages(struct address_space *mapping, 868 struct list_head *pages, struct page *page, 869 unsigned nr_pages) 870 { 871 struct bio *bio = NULL; 872 unsigned page_idx; 873 sector_t last_block_in_bio = 0; 874 struct inode *inode = mapping->host; 875 const unsigned blkbits = inode->i_blkbits; 876 const unsigned blocksize = 1 << blkbits; 877 sector_t block_in_file; 878 sector_t last_block; 879 sector_t last_block_in_file; 880 sector_t block_nr; 881 struct block_device *bdev = inode->i_sb->s_bdev; 882 struct f2fs_map_blocks map; 883 884 map.m_pblk = 0; 885 map.m_lblk = 0; 886 map.m_len = 0; 887 map.m_flags = 0; 888 889 for (page_idx = 0; nr_pages; page_idx++, nr_pages--) { 890 891 prefetchw(&page->flags); 892 if (pages) { 893 page = list_entry(pages->prev, struct page, lru); 894 list_del(&page->lru); 895 if (add_to_page_cache_lru(page, mapping, 896 page->index, GFP_KERNEL)) 897 goto next_page; 898 } 899 900 block_in_file = (sector_t)page->index; 901 last_block = block_in_file + nr_pages; 902 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> 903 blkbits; 904 if (last_block > last_block_in_file) 905 last_block = last_block_in_file; 906 907 /* 908 * Map blocks using the previous result first. 909 */ 910 if ((map.m_flags & F2FS_MAP_MAPPED) && 911 block_in_file > map.m_lblk && 912 block_in_file < (map.m_lblk + map.m_len)) 913 goto got_it; 914 915 /* 916 * Then do more f2fs_map_blocks() calls until we are 917 * done with this page. 918 */ 919 map.m_flags = 0; 920 921 if (block_in_file < last_block) { 922 map.m_lblk = block_in_file; 923 map.m_len = last_block - block_in_file; 924 925 if (f2fs_map_blocks(inode, &map, 0, 926 F2FS_GET_BLOCK_READ)) 927 goto set_error_page; 928 } 929 got_it: 930 if ((map.m_flags & F2FS_MAP_MAPPED)) { 931 block_nr = map.m_pblk + block_in_file - map.m_lblk; 932 SetPageMappedToDisk(page); 933 934 if (!PageUptodate(page) && !cleancache_get_page(page)) { 935 SetPageUptodate(page); 936 goto confused; 937 } 938 } else { 939 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 940 SetPageUptodate(page); 941 unlock_page(page); 942 goto next_page; 943 } 944 945 /* 946 * This page will go to BIO. Do we need to send this 947 * BIO off first? 948 */ 949 if (bio && (last_block_in_bio != block_nr - 1)) { 950 submit_and_realloc: 951 submit_bio(READ, bio); 952 bio = NULL; 953 } 954 if (bio == NULL) { 955 struct f2fs_crypto_ctx *ctx = NULL; 956 957 if (f2fs_encrypted_inode(inode) && 958 S_ISREG(inode->i_mode)) { 959 960 ctx = f2fs_get_crypto_ctx(inode); 961 if (IS_ERR(ctx)) 962 goto set_error_page; 963 964 /* wait the page to be moved by cleaning */ 965 f2fs_wait_on_encrypted_page_writeback( 966 F2FS_I_SB(inode), block_nr); 967 } 968 969 bio = bio_alloc(GFP_KERNEL, 970 min_t(int, nr_pages, BIO_MAX_PAGES)); 971 if (!bio) { 972 if (ctx) 973 f2fs_release_crypto_ctx(ctx); 974 goto set_error_page; 975 } 976 bio->bi_bdev = bdev; 977 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr); 978 bio->bi_end_io = f2fs_read_end_io; 979 bio->bi_private = ctx; 980 } 981 982 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 983 goto submit_and_realloc; 984 985 last_block_in_bio = block_nr; 986 goto next_page; 987 set_error_page: 988 SetPageError(page); 989 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 990 unlock_page(page); 991 goto next_page; 992 confused: 993 if (bio) { 994 submit_bio(READ, bio); 995 bio = NULL; 996 } 997 unlock_page(page); 998 next_page: 999 if (pages) 1000 page_cache_release(page); 1001 } 1002 BUG_ON(pages && !list_empty(pages)); 1003 if (bio) 1004 submit_bio(READ, bio); 1005 return 0; 1006 } 1007 1008 static int f2fs_read_data_page(struct file *file, struct page *page) 1009 { 1010 struct inode *inode = page->mapping->host; 1011 int ret = -EAGAIN; 1012 1013 trace_f2fs_readpage(page, DATA); 1014 1015 /* If the file has inline data, try to read it directly */ 1016 if (f2fs_has_inline_data(inode)) 1017 ret = f2fs_read_inline_data(inode, page); 1018 if (ret == -EAGAIN) 1019 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1); 1020 return ret; 1021 } 1022 1023 static int f2fs_read_data_pages(struct file *file, 1024 struct address_space *mapping, 1025 struct list_head *pages, unsigned nr_pages) 1026 { 1027 struct inode *inode = file->f_mapping->host; 1028 struct page *page = list_entry(pages->prev, struct page, lru); 1029 1030 trace_f2fs_readpages(inode, page, nr_pages); 1031 1032 /* If the file has inline data, skip readpages */ 1033 if (f2fs_has_inline_data(inode)) 1034 return 0; 1035 1036 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages); 1037 } 1038 1039 int do_write_data_page(struct f2fs_io_info *fio) 1040 { 1041 struct page *page = fio->page; 1042 struct inode *inode = page->mapping->host; 1043 struct dnode_of_data dn; 1044 int err = 0; 1045 1046 set_new_dnode(&dn, inode, NULL, NULL, 0); 1047 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 1048 if (err) 1049 return err; 1050 1051 fio->blk_addr = dn.data_blkaddr; 1052 1053 /* This page is already truncated */ 1054 if (fio->blk_addr == NULL_ADDR) { 1055 ClearPageUptodate(page); 1056 goto out_writepage; 1057 } 1058 1059 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { 1060 1061 /* wait for GCed encrypted page writeback */ 1062 f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode), 1063 fio->blk_addr); 1064 1065 fio->encrypted_page = f2fs_encrypt(inode, fio->page); 1066 if (IS_ERR(fio->encrypted_page)) { 1067 err = PTR_ERR(fio->encrypted_page); 1068 goto out_writepage; 1069 } 1070 } 1071 1072 set_page_writeback(page); 1073 1074 /* 1075 * If current allocation needs SSR, 1076 * it had better in-place writes for updated data. 1077 */ 1078 if (unlikely(fio->blk_addr != NEW_ADDR && 1079 !is_cold_data(page) && 1080 need_inplace_update(inode))) { 1081 rewrite_data_page(fio); 1082 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE); 1083 trace_f2fs_do_write_data_page(page, IPU); 1084 } else { 1085 write_data_page(&dn, fio); 1086 set_data_blkaddr(&dn); 1087 f2fs_update_extent_cache(&dn); 1088 trace_f2fs_do_write_data_page(page, OPU); 1089 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); 1090 if (page->index == 0) 1091 set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN); 1092 } 1093 out_writepage: 1094 f2fs_put_dnode(&dn); 1095 return err; 1096 } 1097 1098 static int f2fs_write_data_page(struct page *page, 1099 struct writeback_control *wbc) 1100 { 1101 struct inode *inode = page->mapping->host; 1102 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1103 loff_t i_size = i_size_read(inode); 1104 const pgoff_t end_index = ((unsigned long long) i_size) 1105 >> PAGE_CACHE_SHIFT; 1106 unsigned offset = 0; 1107 bool need_balance_fs = false; 1108 int err = 0; 1109 struct f2fs_io_info fio = { 1110 .sbi = sbi, 1111 .type = DATA, 1112 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE, 1113 .page = page, 1114 .encrypted_page = NULL, 1115 }; 1116 1117 trace_f2fs_writepage(page, DATA); 1118 1119 if (page->index < end_index) 1120 goto write; 1121 1122 /* 1123 * If the offset is out-of-range of file size, 1124 * this page does not have to be written to disk. 1125 */ 1126 offset = i_size & (PAGE_CACHE_SIZE - 1); 1127 if ((page->index >= end_index + 1) || !offset) 1128 goto out; 1129 1130 zero_user_segment(page, offset, PAGE_CACHE_SIZE); 1131 write: 1132 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1133 goto redirty_out; 1134 if (f2fs_is_drop_cache(inode)) 1135 goto out; 1136 if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim && 1137 available_free_memory(sbi, BASE_CHECK)) 1138 goto redirty_out; 1139 1140 /* Dentry blocks are controlled by checkpoint */ 1141 if (S_ISDIR(inode->i_mode)) { 1142 if (unlikely(f2fs_cp_error(sbi))) 1143 goto redirty_out; 1144 err = do_write_data_page(&fio); 1145 goto done; 1146 } 1147 1148 /* we should bypass data pages to proceed the kworkder jobs */ 1149 if (unlikely(f2fs_cp_error(sbi))) { 1150 SetPageError(page); 1151 goto out; 1152 } 1153 1154 if (!wbc->for_reclaim) 1155 need_balance_fs = true; 1156 else if (has_not_enough_free_secs(sbi, 0)) 1157 goto redirty_out; 1158 1159 err = -EAGAIN; 1160 f2fs_lock_op(sbi); 1161 if (f2fs_has_inline_data(inode)) 1162 err = f2fs_write_inline_data(inode, page); 1163 if (err == -EAGAIN) 1164 err = do_write_data_page(&fio); 1165 f2fs_unlock_op(sbi); 1166 done: 1167 if (err && err != -ENOENT) 1168 goto redirty_out; 1169 1170 clear_cold_data(page); 1171 out: 1172 inode_dec_dirty_pages(inode); 1173 if (err) 1174 ClearPageUptodate(page); 1175 unlock_page(page); 1176 if (need_balance_fs) 1177 f2fs_balance_fs(sbi); 1178 if (wbc->for_reclaim) 1179 f2fs_submit_merged_bio(sbi, DATA, WRITE); 1180 return 0; 1181 1182 redirty_out: 1183 redirty_page_for_writepage(wbc, page); 1184 return AOP_WRITEPAGE_ACTIVATE; 1185 } 1186 1187 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc, 1188 void *data) 1189 { 1190 struct address_space *mapping = data; 1191 int ret = mapping->a_ops->writepage(page, wbc); 1192 mapping_set_error(mapping, ret); 1193 return ret; 1194 } 1195 1196 /* 1197 * This function was copied from write_cche_pages from mm/page-writeback.c. 1198 * The major change is making write step of cold data page separately from 1199 * warm/hot data page. 1200 */ 1201 static int f2fs_write_cache_pages(struct address_space *mapping, 1202 struct writeback_control *wbc, writepage_t writepage, 1203 void *data) 1204 { 1205 int ret = 0; 1206 int done = 0; 1207 struct pagevec pvec; 1208 int nr_pages; 1209 pgoff_t uninitialized_var(writeback_index); 1210 pgoff_t index; 1211 pgoff_t end; /* Inclusive */ 1212 pgoff_t done_index; 1213 int cycled; 1214 int range_whole = 0; 1215 int tag; 1216 int step = 0; 1217 1218 pagevec_init(&pvec, 0); 1219 next: 1220 if (wbc->range_cyclic) { 1221 writeback_index = mapping->writeback_index; /* prev offset */ 1222 index = writeback_index; 1223 if (index == 0) 1224 cycled = 1; 1225 else 1226 cycled = 0; 1227 end = -1; 1228 } else { 1229 index = wbc->range_start >> PAGE_CACHE_SHIFT; 1230 end = wbc->range_end >> PAGE_CACHE_SHIFT; 1231 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 1232 range_whole = 1; 1233 cycled = 1; /* ignore range_cyclic tests */ 1234 } 1235 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1236 tag = PAGECACHE_TAG_TOWRITE; 1237 else 1238 tag = PAGECACHE_TAG_DIRTY; 1239 retry: 1240 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1241 tag_pages_for_writeback(mapping, index, end); 1242 done_index = index; 1243 while (!done && (index <= end)) { 1244 int i; 1245 1246 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 1247 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1); 1248 if (nr_pages == 0) 1249 break; 1250 1251 for (i = 0; i < nr_pages; i++) { 1252 struct page *page = pvec.pages[i]; 1253 1254 if (page->index > end) { 1255 done = 1; 1256 break; 1257 } 1258 1259 done_index = page->index; 1260 1261 lock_page(page); 1262 1263 if (unlikely(page->mapping != mapping)) { 1264 continue_unlock: 1265 unlock_page(page); 1266 continue; 1267 } 1268 1269 if (!PageDirty(page)) { 1270 /* someone wrote it for us */ 1271 goto continue_unlock; 1272 } 1273 1274 if (step == is_cold_data(page)) 1275 goto continue_unlock; 1276 1277 if (PageWriteback(page)) { 1278 if (wbc->sync_mode != WB_SYNC_NONE) 1279 f2fs_wait_on_page_writeback(page, DATA); 1280 else 1281 goto continue_unlock; 1282 } 1283 1284 BUG_ON(PageWriteback(page)); 1285 if (!clear_page_dirty_for_io(page)) 1286 goto continue_unlock; 1287 1288 ret = (*writepage)(page, wbc, data); 1289 if (unlikely(ret)) { 1290 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1291 unlock_page(page); 1292 ret = 0; 1293 } else { 1294 done_index = page->index + 1; 1295 done = 1; 1296 break; 1297 } 1298 } 1299 1300 if (--wbc->nr_to_write <= 0 && 1301 wbc->sync_mode == WB_SYNC_NONE) { 1302 done = 1; 1303 break; 1304 } 1305 } 1306 pagevec_release(&pvec); 1307 cond_resched(); 1308 } 1309 1310 if (step < 1) { 1311 step++; 1312 goto next; 1313 } 1314 1315 if (!cycled && !done) { 1316 cycled = 1; 1317 index = 0; 1318 end = writeback_index - 1; 1319 goto retry; 1320 } 1321 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 1322 mapping->writeback_index = done_index; 1323 1324 return ret; 1325 } 1326 1327 static int f2fs_write_data_pages(struct address_space *mapping, 1328 struct writeback_control *wbc) 1329 { 1330 struct inode *inode = mapping->host; 1331 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1332 bool locked = false; 1333 int ret; 1334 long diff; 1335 1336 trace_f2fs_writepages(mapping->host, wbc, DATA); 1337 1338 /* deal with chardevs and other special file */ 1339 if (!mapping->a_ops->writepage) 1340 return 0; 1341 1342 /* skip writing if there is no dirty page in this inode */ 1343 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 1344 return 0; 1345 1346 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE && 1347 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 1348 available_free_memory(sbi, DIRTY_DENTS)) 1349 goto skip_write; 1350 1351 /* during POR, we don't need to trigger writepage at all. */ 1352 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1353 goto skip_write; 1354 1355 diff = nr_pages_to_write(sbi, DATA, wbc); 1356 1357 if (!S_ISDIR(inode->i_mode)) { 1358 mutex_lock(&sbi->writepages); 1359 locked = true; 1360 } 1361 ret = f2fs_write_cache_pages(mapping, wbc, __f2fs_writepage, mapping); 1362 f2fs_submit_merged_bio(sbi, DATA, WRITE); 1363 if (locked) 1364 mutex_unlock(&sbi->writepages); 1365 1366 remove_dirty_dir_inode(inode); 1367 1368 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); 1369 return ret; 1370 1371 skip_write: 1372 wbc->pages_skipped += get_dirty_pages(inode); 1373 return 0; 1374 } 1375 1376 static void f2fs_write_failed(struct address_space *mapping, loff_t to) 1377 { 1378 struct inode *inode = mapping->host; 1379 1380 if (to > inode->i_size) { 1381 truncate_pagecache(inode, inode->i_size); 1382 truncate_blocks(inode, inode->i_size, true); 1383 } 1384 } 1385 1386 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 1387 loff_t pos, unsigned len, unsigned flags, 1388 struct page **pagep, void **fsdata) 1389 { 1390 struct inode *inode = mapping->host; 1391 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1392 struct page *page = NULL; 1393 struct page *ipage; 1394 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT; 1395 struct dnode_of_data dn; 1396 int err = 0; 1397 1398 trace_f2fs_write_begin(inode, pos, len, flags); 1399 1400 f2fs_balance_fs(sbi); 1401 1402 /* 1403 * We should check this at this moment to avoid deadlock on inode page 1404 * and #0 page. The locking rule for inline_data conversion should be: 1405 * lock_page(page #0) -> lock_page(inode_page) 1406 */ 1407 if (index != 0) { 1408 err = f2fs_convert_inline_inode(inode); 1409 if (err) 1410 goto fail; 1411 } 1412 repeat: 1413 page = grab_cache_page_write_begin(mapping, index, flags); 1414 if (!page) { 1415 err = -ENOMEM; 1416 goto fail; 1417 } 1418 1419 *pagep = page; 1420 1421 f2fs_lock_op(sbi); 1422 1423 /* check inline_data */ 1424 ipage = get_node_page(sbi, inode->i_ino); 1425 if (IS_ERR(ipage)) { 1426 err = PTR_ERR(ipage); 1427 goto unlock_fail; 1428 } 1429 1430 set_new_dnode(&dn, inode, ipage, ipage, 0); 1431 1432 if (f2fs_has_inline_data(inode)) { 1433 if (pos + len <= MAX_INLINE_DATA) { 1434 read_inline_data(page, ipage); 1435 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); 1436 sync_inode_page(&dn); 1437 goto put_next; 1438 } 1439 err = f2fs_convert_inline_page(&dn, page); 1440 if (err) 1441 goto put_fail; 1442 } 1443 1444 err = f2fs_get_block(&dn, index); 1445 if (err) 1446 goto put_fail; 1447 put_next: 1448 f2fs_put_dnode(&dn); 1449 f2fs_unlock_op(sbi); 1450 1451 f2fs_wait_on_page_writeback(page, DATA); 1452 1453 /* wait for GCed encrypted page writeback */ 1454 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 1455 f2fs_wait_on_encrypted_page_writeback(sbi, dn.data_blkaddr); 1456 1457 if (len == PAGE_CACHE_SIZE) 1458 goto out_update; 1459 if (PageUptodate(page)) 1460 goto out_clear; 1461 1462 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) { 1463 unsigned start = pos & (PAGE_CACHE_SIZE - 1); 1464 unsigned end = start + len; 1465 1466 /* Reading beyond i_size is simple: memset to zero */ 1467 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE); 1468 goto out_update; 1469 } 1470 1471 if (dn.data_blkaddr == NEW_ADDR) { 1472 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 1473 } else { 1474 struct f2fs_io_info fio = { 1475 .sbi = sbi, 1476 .type = DATA, 1477 .rw = READ_SYNC, 1478 .blk_addr = dn.data_blkaddr, 1479 .page = page, 1480 .encrypted_page = NULL, 1481 }; 1482 err = f2fs_submit_page_bio(&fio); 1483 if (err) 1484 goto fail; 1485 1486 lock_page(page); 1487 if (unlikely(!PageUptodate(page))) { 1488 err = -EIO; 1489 goto fail; 1490 } 1491 if (unlikely(page->mapping != mapping)) { 1492 f2fs_put_page(page, 1); 1493 goto repeat; 1494 } 1495 1496 /* avoid symlink page */ 1497 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { 1498 err = f2fs_decrypt_one(inode, page); 1499 if (err) 1500 goto fail; 1501 } 1502 } 1503 out_update: 1504 SetPageUptodate(page); 1505 out_clear: 1506 clear_cold_data(page); 1507 return 0; 1508 1509 put_fail: 1510 f2fs_put_dnode(&dn); 1511 unlock_fail: 1512 f2fs_unlock_op(sbi); 1513 fail: 1514 f2fs_put_page(page, 1); 1515 f2fs_write_failed(mapping, pos + len); 1516 return err; 1517 } 1518 1519 static int f2fs_write_end(struct file *file, 1520 struct address_space *mapping, 1521 loff_t pos, unsigned len, unsigned copied, 1522 struct page *page, void *fsdata) 1523 { 1524 struct inode *inode = page->mapping->host; 1525 1526 trace_f2fs_write_end(inode, pos, len, copied); 1527 1528 set_page_dirty(page); 1529 1530 if (pos + copied > i_size_read(inode)) { 1531 i_size_write(inode, pos + copied); 1532 mark_inode_dirty(inode); 1533 update_inode_page(inode); 1534 } 1535 1536 f2fs_put_page(page, 1); 1537 return copied; 1538 } 1539 1540 static int check_direct_IO(struct inode *inode, struct iov_iter *iter, 1541 loff_t offset) 1542 { 1543 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1; 1544 1545 if (offset & blocksize_mask) 1546 return -EINVAL; 1547 1548 if (iov_iter_alignment(iter) & blocksize_mask) 1549 return -EINVAL; 1550 1551 return 0; 1552 } 1553 1554 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter, 1555 loff_t offset) 1556 { 1557 struct file *file = iocb->ki_filp; 1558 struct address_space *mapping = file->f_mapping; 1559 struct inode *inode = mapping->host; 1560 size_t count = iov_iter_count(iter); 1561 int err; 1562 1563 /* we don't need to use inline_data strictly */ 1564 if (f2fs_has_inline_data(inode)) { 1565 err = f2fs_convert_inline_inode(inode); 1566 if (err) 1567 return err; 1568 } 1569 1570 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 1571 return 0; 1572 1573 err = check_direct_IO(inode, iter, offset); 1574 if (err) 1575 return err; 1576 1577 trace_f2fs_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); 1578 1579 if (iov_iter_rw(iter) == WRITE) { 1580 __allocate_data_blocks(inode, offset, count); 1581 if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { 1582 err = -EIO; 1583 goto out; 1584 } 1585 } 1586 1587 err = blockdev_direct_IO(iocb, inode, iter, offset, get_data_block_dio); 1588 out: 1589 if (err < 0 && iov_iter_rw(iter) == WRITE) 1590 f2fs_write_failed(mapping, offset + count); 1591 1592 trace_f2fs_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), err); 1593 1594 return err; 1595 } 1596 1597 void f2fs_invalidate_page(struct page *page, unsigned int offset, 1598 unsigned int length) 1599 { 1600 struct inode *inode = page->mapping->host; 1601 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1602 1603 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 1604 (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE)) 1605 return; 1606 1607 if (PageDirty(page)) { 1608 if (inode->i_ino == F2FS_META_INO(sbi)) 1609 dec_page_count(sbi, F2FS_DIRTY_META); 1610 else if (inode->i_ino == F2FS_NODE_INO(sbi)) 1611 dec_page_count(sbi, F2FS_DIRTY_NODES); 1612 else 1613 inode_dec_dirty_pages(inode); 1614 } 1615 1616 /* This is atomic written page, keep Private */ 1617 if (IS_ATOMIC_WRITTEN_PAGE(page)) 1618 return; 1619 1620 ClearPagePrivate(page); 1621 } 1622 1623 int f2fs_release_page(struct page *page, gfp_t wait) 1624 { 1625 /* If this is dirty page, keep PagePrivate */ 1626 if (PageDirty(page)) 1627 return 0; 1628 1629 /* This is atomic written page, keep Private */ 1630 if (IS_ATOMIC_WRITTEN_PAGE(page)) 1631 return 0; 1632 1633 ClearPagePrivate(page); 1634 return 1; 1635 } 1636 1637 static int f2fs_set_data_page_dirty(struct page *page) 1638 { 1639 struct address_space *mapping = page->mapping; 1640 struct inode *inode = mapping->host; 1641 1642 trace_f2fs_set_page_dirty(page, DATA); 1643 1644 SetPageUptodate(page); 1645 1646 if (f2fs_is_atomic_file(inode)) { 1647 if (!IS_ATOMIC_WRITTEN_PAGE(page)) { 1648 register_inmem_page(inode, page); 1649 return 1; 1650 } 1651 /* 1652 * Previously, this page has been registered, we just 1653 * return here. 1654 */ 1655 return 0; 1656 } 1657 1658 if (!PageDirty(page)) { 1659 __set_page_dirty_nobuffers(page); 1660 update_dirty_page(inode, page); 1661 return 1; 1662 } 1663 return 0; 1664 } 1665 1666 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 1667 { 1668 struct inode *inode = mapping->host; 1669 1670 if (f2fs_has_inline_data(inode)) 1671 return 0; 1672 1673 /* make sure allocating whole blocks */ 1674 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 1675 filemap_write_and_wait(mapping); 1676 1677 return generic_block_bmap(mapping, block, get_data_block_bmap); 1678 } 1679 1680 const struct address_space_operations f2fs_dblock_aops = { 1681 .readpage = f2fs_read_data_page, 1682 .readpages = f2fs_read_data_pages, 1683 .writepage = f2fs_write_data_page, 1684 .writepages = f2fs_write_data_pages, 1685 .write_begin = f2fs_write_begin, 1686 .write_end = f2fs_write_end, 1687 .set_page_dirty = f2fs_set_data_page_dirty, 1688 .invalidatepage = f2fs_invalidate_page, 1689 .releasepage = f2fs_release_page, 1690 .direct_IO = f2fs_direct_IO, 1691 .bmap = f2fs_bmap, 1692 }; 1693