1 /* 2 * fs/f2fs/data.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/fs.h> 12 #include <linux/f2fs_fs.h> 13 #include <linux/buffer_head.h> 14 #include <linux/mpage.h> 15 #include <linux/writeback.h> 16 #include <linux/backing-dev.h> 17 #include <linux/pagevec.h> 18 #include <linux/blkdev.h> 19 #include <linux/bio.h> 20 #include <linux/prefetch.h> 21 #include <linux/uio.h> 22 #include <linux/cleancache.h> 23 24 #include "f2fs.h" 25 #include "node.h" 26 #include "segment.h" 27 #include "trace.h" 28 #include <trace/events/f2fs.h> 29 30 static void f2fs_read_end_io(struct bio *bio) 31 { 32 struct bio_vec *bvec; 33 int i; 34 35 if (f2fs_bio_encrypted(bio)) { 36 if (bio->bi_error) { 37 f2fs_release_crypto_ctx(bio->bi_private); 38 } else { 39 f2fs_end_io_crypto_work(bio->bi_private, bio); 40 return; 41 } 42 } 43 44 bio_for_each_segment_all(bvec, bio, i) { 45 struct page *page = bvec->bv_page; 46 47 if (!bio->bi_error) { 48 SetPageUptodate(page); 49 } else { 50 ClearPageUptodate(page); 51 SetPageError(page); 52 } 53 unlock_page(page); 54 } 55 bio_put(bio); 56 } 57 58 static void f2fs_write_end_io(struct bio *bio) 59 { 60 struct f2fs_sb_info *sbi = bio->bi_private; 61 struct bio_vec *bvec; 62 int i; 63 64 bio_for_each_segment_all(bvec, bio, i) { 65 struct page *page = bvec->bv_page; 66 67 f2fs_restore_and_release_control_page(&page); 68 69 if (unlikely(bio->bi_error)) { 70 set_page_dirty(page); 71 set_bit(AS_EIO, &page->mapping->flags); 72 f2fs_stop_checkpoint(sbi); 73 } 74 end_page_writeback(page); 75 dec_page_count(sbi, F2FS_WRITEBACK); 76 } 77 78 if (!get_pages(sbi, F2FS_WRITEBACK) && 79 !list_empty(&sbi->cp_wait.task_list)) 80 wake_up(&sbi->cp_wait); 81 82 bio_put(bio); 83 } 84 85 /* 86 * Low-level block read/write IO operations. 87 */ 88 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, 89 int npages, bool is_read) 90 { 91 struct bio *bio; 92 93 bio = f2fs_bio_alloc(npages); 94 95 bio->bi_bdev = sbi->sb->s_bdev; 96 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); 97 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; 98 bio->bi_private = is_read ? NULL : sbi; 99 100 return bio; 101 } 102 103 static void __submit_merged_bio(struct f2fs_bio_info *io) 104 { 105 struct f2fs_io_info *fio = &io->fio; 106 107 if (!io->bio) 108 return; 109 110 if (is_read_io(fio->rw)) 111 trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio); 112 else 113 trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio); 114 115 submit_bio(fio->rw, io->bio); 116 io->bio = NULL; 117 } 118 119 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi, 120 enum page_type type, int rw) 121 { 122 enum page_type btype = PAGE_TYPE_OF_BIO(type); 123 struct f2fs_bio_info *io; 124 125 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype]; 126 127 down_write(&io->io_rwsem); 128 129 /* change META to META_FLUSH in the checkpoint procedure */ 130 if (type >= META_FLUSH) { 131 io->fio.type = META_FLUSH; 132 if (test_opt(sbi, NOBARRIER)) 133 io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO; 134 else 135 io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO; 136 } 137 __submit_merged_bio(io); 138 up_write(&io->io_rwsem); 139 } 140 141 /* 142 * Fill the locked page with data located in the block address. 143 * Return unlocked page. 144 */ 145 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 146 { 147 struct bio *bio; 148 struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; 149 150 trace_f2fs_submit_page_bio(page, fio); 151 f2fs_trace_ios(fio, 0); 152 153 /* Allocate a new bio */ 154 bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw)); 155 156 if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) { 157 bio_put(bio); 158 return -EFAULT; 159 } 160 161 submit_bio(fio->rw, bio); 162 return 0; 163 } 164 165 void f2fs_submit_page_mbio(struct f2fs_io_info *fio) 166 { 167 struct f2fs_sb_info *sbi = fio->sbi; 168 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 169 struct f2fs_bio_info *io; 170 bool is_read = is_read_io(fio->rw); 171 struct page *bio_page; 172 173 io = is_read ? &sbi->read_io : &sbi->write_io[btype]; 174 175 verify_block_addr(sbi, fio->blk_addr); 176 177 down_write(&io->io_rwsem); 178 179 if (!is_read) 180 inc_page_count(sbi, F2FS_WRITEBACK); 181 182 if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 || 183 io->fio.rw != fio->rw)) 184 __submit_merged_bio(io); 185 alloc_new: 186 if (io->bio == NULL) { 187 int bio_blocks = MAX_BIO_BLOCKS(sbi); 188 189 io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read); 190 io->fio = *fio; 191 } 192 193 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; 194 195 if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) < 196 PAGE_CACHE_SIZE) { 197 __submit_merged_bio(io); 198 goto alloc_new; 199 } 200 201 io->last_block_in_bio = fio->blk_addr; 202 f2fs_trace_ios(fio, 0); 203 204 up_write(&io->io_rwsem); 205 trace_f2fs_submit_page_mbio(fio->page, fio); 206 } 207 208 /* 209 * Lock ordering for the change of data block address: 210 * ->data_page 211 * ->node_page 212 * update block addresses in the node page 213 */ 214 void set_data_blkaddr(struct dnode_of_data *dn) 215 { 216 struct f2fs_node *rn; 217 __le32 *addr_array; 218 struct page *node_page = dn->node_page; 219 unsigned int ofs_in_node = dn->ofs_in_node; 220 221 f2fs_wait_on_page_writeback(node_page, NODE); 222 223 rn = F2FS_NODE(node_page); 224 225 /* Get physical address of data block */ 226 addr_array = blkaddr_in_node(rn); 227 addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 228 if (set_page_dirty(node_page)) 229 dn->node_changed = true; 230 } 231 232 int reserve_new_block(struct dnode_of_data *dn) 233 { 234 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 235 236 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 237 return -EPERM; 238 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 239 return -ENOSPC; 240 241 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node); 242 243 dn->data_blkaddr = NEW_ADDR; 244 set_data_blkaddr(dn); 245 mark_inode_dirty(dn->inode); 246 sync_inode_page(dn); 247 return 0; 248 } 249 250 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 251 { 252 bool need_put = dn->inode_page ? false : true; 253 int err; 254 255 err = get_dnode_of_data(dn, index, ALLOC_NODE); 256 if (err) 257 return err; 258 259 if (dn->data_blkaddr == NULL_ADDR) 260 err = reserve_new_block(dn); 261 if (err || need_put) 262 f2fs_put_dnode(dn); 263 return err; 264 } 265 266 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) 267 { 268 struct extent_info ei; 269 struct inode *inode = dn->inode; 270 271 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 272 dn->data_blkaddr = ei.blk + index - ei.fofs; 273 return 0; 274 } 275 276 return f2fs_reserve_block(dn, index); 277 } 278 279 struct page *get_read_data_page(struct inode *inode, pgoff_t index, 280 int rw, bool for_write) 281 { 282 struct address_space *mapping = inode->i_mapping; 283 struct dnode_of_data dn; 284 struct page *page; 285 struct extent_info ei; 286 int err; 287 struct f2fs_io_info fio = { 288 .sbi = F2FS_I_SB(inode), 289 .type = DATA, 290 .rw = rw, 291 .encrypted_page = NULL, 292 }; 293 294 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 295 return read_mapping_page(mapping, index, NULL); 296 297 page = f2fs_grab_cache_page(mapping, index, for_write); 298 if (!page) 299 return ERR_PTR(-ENOMEM); 300 301 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 302 dn.data_blkaddr = ei.blk + index - ei.fofs; 303 goto got_it; 304 } 305 306 set_new_dnode(&dn, inode, NULL, NULL, 0); 307 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 308 if (err) 309 goto put_err; 310 f2fs_put_dnode(&dn); 311 312 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 313 err = -ENOENT; 314 goto put_err; 315 } 316 got_it: 317 if (PageUptodate(page)) { 318 unlock_page(page); 319 return page; 320 } 321 322 /* 323 * A new dentry page is allocated but not able to be written, since its 324 * new inode page couldn't be allocated due to -ENOSPC. 325 * In such the case, its blkaddr can be remained as NEW_ADDR. 326 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata. 327 */ 328 if (dn.data_blkaddr == NEW_ADDR) { 329 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 330 SetPageUptodate(page); 331 unlock_page(page); 332 return page; 333 } 334 335 fio.blk_addr = dn.data_blkaddr; 336 fio.page = page; 337 err = f2fs_submit_page_bio(&fio); 338 if (err) 339 goto put_err; 340 return page; 341 342 put_err: 343 f2fs_put_page(page, 1); 344 return ERR_PTR(err); 345 } 346 347 struct page *find_data_page(struct inode *inode, pgoff_t index) 348 { 349 struct address_space *mapping = inode->i_mapping; 350 struct page *page; 351 352 page = find_get_page(mapping, index); 353 if (page && PageUptodate(page)) 354 return page; 355 f2fs_put_page(page, 0); 356 357 page = get_read_data_page(inode, index, READ_SYNC, false); 358 if (IS_ERR(page)) 359 return page; 360 361 if (PageUptodate(page)) 362 return page; 363 364 wait_on_page_locked(page); 365 if (unlikely(!PageUptodate(page))) { 366 f2fs_put_page(page, 0); 367 return ERR_PTR(-EIO); 368 } 369 return page; 370 } 371 372 /* 373 * If it tries to access a hole, return an error. 374 * Because, the callers, functions in dir.c and GC, should be able to know 375 * whether this page exists or not. 376 */ 377 struct page *get_lock_data_page(struct inode *inode, pgoff_t index, 378 bool for_write) 379 { 380 struct address_space *mapping = inode->i_mapping; 381 struct page *page; 382 repeat: 383 page = get_read_data_page(inode, index, READ_SYNC, for_write); 384 if (IS_ERR(page)) 385 return page; 386 387 /* wait for read completion */ 388 lock_page(page); 389 if (unlikely(!PageUptodate(page))) { 390 f2fs_put_page(page, 1); 391 return ERR_PTR(-EIO); 392 } 393 if (unlikely(page->mapping != mapping)) { 394 f2fs_put_page(page, 1); 395 goto repeat; 396 } 397 return page; 398 } 399 400 /* 401 * Caller ensures that this data page is never allocated. 402 * A new zero-filled data page is allocated in the page cache. 403 * 404 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 405 * f2fs_unlock_op(). 406 * Note that, ipage is set only by make_empty_dir, and if any error occur, 407 * ipage should be released by this function. 408 */ 409 struct page *get_new_data_page(struct inode *inode, 410 struct page *ipage, pgoff_t index, bool new_i_size) 411 { 412 struct address_space *mapping = inode->i_mapping; 413 struct page *page; 414 struct dnode_of_data dn; 415 int err; 416 417 page = f2fs_grab_cache_page(mapping, index, true); 418 if (!page) { 419 /* 420 * before exiting, we should make sure ipage will be released 421 * if any error occur. 422 */ 423 f2fs_put_page(ipage, 1); 424 return ERR_PTR(-ENOMEM); 425 } 426 427 set_new_dnode(&dn, inode, ipage, NULL, 0); 428 err = f2fs_reserve_block(&dn, index); 429 if (err) { 430 f2fs_put_page(page, 1); 431 return ERR_PTR(err); 432 } 433 if (!ipage) 434 f2fs_put_dnode(&dn); 435 436 if (PageUptodate(page)) 437 goto got_it; 438 439 if (dn.data_blkaddr == NEW_ADDR) { 440 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 441 SetPageUptodate(page); 442 } else { 443 f2fs_put_page(page, 1); 444 445 /* if ipage exists, blkaddr should be NEW_ADDR */ 446 f2fs_bug_on(F2FS_I_SB(inode), ipage); 447 page = get_lock_data_page(inode, index, true); 448 if (IS_ERR(page)) 449 return page; 450 } 451 got_it: 452 if (new_i_size && i_size_read(inode) < 453 ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)) { 454 i_size_write(inode, ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)); 455 /* Only the directory inode sets new_i_size */ 456 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR); 457 } 458 return page; 459 } 460 461 static int __allocate_data_block(struct dnode_of_data *dn) 462 { 463 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 464 struct f2fs_inode_info *fi = F2FS_I(dn->inode); 465 struct f2fs_summary sum; 466 struct node_info ni; 467 int seg = CURSEG_WARM_DATA; 468 pgoff_t fofs; 469 470 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 471 return -EPERM; 472 473 dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node); 474 if (dn->data_blkaddr == NEW_ADDR) 475 goto alloc; 476 477 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 478 return -ENOSPC; 479 480 alloc: 481 get_node_info(sbi, dn->nid, &ni); 482 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 483 484 if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page) 485 seg = CURSEG_DIRECT_IO; 486 487 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr, 488 &sum, seg); 489 set_data_blkaddr(dn); 490 491 /* update i_size */ 492 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) + 493 dn->ofs_in_node; 494 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)) 495 i_size_write(dn->inode, 496 ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)); 497 return 0; 498 } 499 500 static int __allocate_data_blocks(struct inode *inode, loff_t offset, 501 size_t count) 502 { 503 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 504 struct dnode_of_data dn; 505 u64 start = F2FS_BYTES_TO_BLK(offset); 506 u64 len = F2FS_BYTES_TO_BLK(count); 507 bool allocated; 508 u64 end_offset; 509 int err = 0; 510 511 while (len) { 512 f2fs_lock_op(sbi); 513 514 /* When reading holes, we need its node page */ 515 set_new_dnode(&dn, inode, NULL, NULL, 0); 516 err = get_dnode_of_data(&dn, start, ALLOC_NODE); 517 if (err) 518 goto out; 519 520 allocated = false; 521 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 522 523 while (dn.ofs_in_node < end_offset && len) { 524 block_t blkaddr; 525 526 if (unlikely(f2fs_cp_error(sbi))) { 527 err = -EIO; 528 goto sync_out; 529 } 530 531 blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); 532 if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) { 533 err = __allocate_data_block(&dn); 534 if (err) 535 goto sync_out; 536 allocated = true; 537 } 538 len--; 539 start++; 540 dn.ofs_in_node++; 541 } 542 543 if (allocated) 544 sync_inode_page(&dn); 545 546 f2fs_put_dnode(&dn); 547 f2fs_unlock_op(sbi); 548 549 f2fs_balance_fs(sbi, dn.node_changed); 550 } 551 return err; 552 553 sync_out: 554 if (allocated) 555 sync_inode_page(&dn); 556 f2fs_put_dnode(&dn); 557 out: 558 f2fs_unlock_op(sbi); 559 f2fs_balance_fs(sbi, dn.node_changed); 560 return err; 561 } 562 563 /* 564 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with 565 * f2fs_map_blocks structure. 566 * If original data blocks are allocated, then give them to blockdev. 567 * Otherwise, 568 * a. preallocate requested block addresses 569 * b. do not use extent cache for better performance 570 * c. give the block addresses to blockdev 571 */ 572 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 573 int create, int flag) 574 { 575 unsigned int maxblocks = map->m_len; 576 struct dnode_of_data dn; 577 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 578 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA; 579 pgoff_t pgofs, end_offset; 580 int err = 0, ofs = 1; 581 struct extent_info ei; 582 bool allocated = false; 583 block_t blkaddr; 584 585 map->m_len = 0; 586 map->m_flags = 0; 587 588 /* it only supports block size == page size */ 589 pgofs = (pgoff_t)map->m_lblk; 590 591 if (f2fs_lookup_extent_cache(inode, pgofs, &ei)) { 592 map->m_pblk = ei.blk + pgofs - ei.fofs; 593 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); 594 map->m_flags = F2FS_MAP_MAPPED; 595 goto out; 596 } 597 598 if (create) 599 f2fs_lock_op(sbi); 600 601 /* When reading holes, we need its node page */ 602 set_new_dnode(&dn, inode, NULL, NULL, 0); 603 err = get_dnode_of_data(&dn, pgofs, mode); 604 if (err) { 605 if (err == -ENOENT) 606 err = 0; 607 goto unlock_out; 608 } 609 610 if (dn.data_blkaddr == NEW_ADDR || dn.data_blkaddr == NULL_ADDR) { 611 if (create) { 612 if (unlikely(f2fs_cp_error(sbi))) { 613 err = -EIO; 614 goto put_out; 615 } 616 err = __allocate_data_block(&dn); 617 if (err) 618 goto put_out; 619 allocated = true; 620 map->m_flags = F2FS_MAP_NEW; 621 } else { 622 if (flag != F2FS_GET_BLOCK_FIEMAP || 623 dn.data_blkaddr != NEW_ADDR) { 624 if (flag == F2FS_GET_BLOCK_BMAP) 625 err = -ENOENT; 626 goto put_out; 627 } 628 629 /* 630 * preallocated unwritten block should be mapped 631 * for fiemap. 632 */ 633 if (dn.data_blkaddr == NEW_ADDR) 634 map->m_flags = F2FS_MAP_UNWRITTEN; 635 } 636 } 637 638 map->m_flags |= F2FS_MAP_MAPPED; 639 map->m_pblk = dn.data_blkaddr; 640 map->m_len = 1; 641 642 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 643 dn.ofs_in_node++; 644 pgofs++; 645 646 get_next: 647 if (map->m_len >= maxblocks) 648 goto sync_out; 649 650 if (dn.ofs_in_node >= end_offset) { 651 if (allocated) 652 sync_inode_page(&dn); 653 allocated = false; 654 f2fs_put_dnode(&dn); 655 656 if (create) { 657 f2fs_unlock_op(sbi); 658 f2fs_balance_fs(sbi, dn.node_changed); 659 f2fs_lock_op(sbi); 660 } 661 662 set_new_dnode(&dn, inode, NULL, NULL, 0); 663 err = get_dnode_of_data(&dn, pgofs, mode); 664 if (err) { 665 if (err == -ENOENT) 666 err = 0; 667 goto unlock_out; 668 } 669 670 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 671 } 672 673 blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); 674 675 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) { 676 if (create) { 677 if (unlikely(f2fs_cp_error(sbi))) { 678 err = -EIO; 679 goto sync_out; 680 } 681 err = __allocate_data_block(&dn); 682 if (err) 683 goto sync_out; 684 allocated = true; 685 map->m_flags |= F2FS_MAP_NEW; 686 blkaddr = dn.data_blkaddr; 687 } else { 688 /* 689 * we only merge preallocated unwritten blocks 690 * for fiemap. 691 */ 692 if (flag != F2FS_GET_BLOCK_FIEMAP || 693 blkaddr != NEW_ADDR) 694 goto sync_out; 695 } 696 } 697 698 /* Give more consecutive addresses for the readahead */ 699 if ((map->m_pblk != NEW_ADDR && 700 blkaddr == (map->m_pblk + ofs)) || 701 (map->m_pblk == NEW_ADDR && 702 blkaddr == NEW_ADDR)) { 703 ofs++; 704 dn.ofs_in_node++; 705 pgofs++; 706 map->m_len++; 707 goto get_next; 708 } 709 710 sync_out: 711 if (allocated) 712 sync_inode_page(&dn); 713 put_out: 714 f2fs_put_dnode(&dn); 715 unlock_out: 716 if (create) { 717 f2fs_unlock_op(sbi); 718 f2fs_balance_fs(sbi, dn.node_changed); 719 } 720 out: 721 trace_f2fs_map_blocks(inode, map, err); 722 return err; 723 } 724 725 static int __get_data_block(struct inode *inode, sector_t iblock, 726 struct buffer_head *bh, int create, int flag) 727 { 728 struct f2fs_map_blocks map; 729 int ret; 730 731 map.m_lblk = iblock; 732 map.m_len = bh->b_size >> inode->i_blkbits; 733 734 ret = f2fs_map_blocks(inode, &map, create, flag); 735 if (!ret) { 736 map_bh(bh, inode->i_sb, map.m_pblk); 737 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; 738 bh->b_size = map.m_len << inode->i_blkbits; 739 } 740 return ret; 741 } 742 743 static int get_data_block(struct inode *inode, sector_t iblock, 744 struct buffer_head *bh_result, int create, int flag) 745 { 746 return __get_data_block(inode, iblock, bh_result, create, flag); 747 } 748 749 static int get_data_block_dio(struct inode *inode, sector_t iblock, 750 struct buffer_head *bh_result, int create) 751 { 752 return __get_data_block(inode, iblock, bh_result, create, 753 F2FS_GET_BLOCK_DIO); 754 } 755 756 static int get_data_block_bmap(struct inode *inode, sector_t iblock, 757 struct buffer_head *bh_result, int create) 758 { 759 /* Block number less than F2FS MAX BLOCKS */ 760 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks)) 761 return -EFBIG; 762 763 return __get_data_block(inode, iblock, bh_result, create, 764 F2FS_GET_BLOCK_BMAP); 765 } 766 767 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset) 768 { 769 return (offset >> inode->i_blkbits); 770 } 771 772 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk) 773 { 774 return (blk << inode->i_blkbits); 775 } 776 777 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 778 u64 start, u64 len) 779 { 780 struct buffer_head map_bh; 781 sector_t start_blk, last_blk; 782 loff_t isize; 783 u64 logical = 0, phys = 0, size = 0; 784 u32 flags = 0; 785 int ret = 0; 786 787 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); 788 if (ret) 789 return ret; 790 791 if (f2fs_has_inline_data(inode)) { 792 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 793 if (ret != -EAGAIN) 794 return ret; 795 } 796 797 inode_lock(inode); 798 799 isize = i_size_read(inode); 800 if (start >= isize) 801 goto out; 802 803 if (start + len > isize) 804 len = isize - start; 805 806 if (logical_to_blk(inode, len) == 0) 807 len = blk_to_logical(inode, 1); 808 809 start_blk = logical_to_blk(inode, start); 810 last_blk = logical_to_blk(inode, start + len - 1); 811 812 next: 813 memset(&map_bh, 0, sizeof(struct buffer_head)); 814 map_bh.b_size = len; 815 816 ret = get_data_block(inode, start_blk, &map_bh, 0, 817 F2FS_GET_BLOCK_FIEMAP); 818 if (ret) 819 goto out; 820 821 /* HOLE */ 822 if (!buffer_mapped(&map_bh)) { 823 /* Go through holes util pass the EOF */ 824 if (blk_to_logical(inode, start_blk++) < isize) 825 goto prep_next; 826 /* Found a hole beyond isize means no more extents. 827 * Note that the premise is that filesystems don't 828 * punch holes beyond isize and keep size unchanged. 829 */ 830 flags |= FIEMAP_EXTENT_LAST; 831 } 832 833 if (size) { 834 if (f2fs_encrypted_inode(inode)) 835 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; 836 837 ret = fiemap_fill_next_extent(fieinfo, logical, 838 phys, size, flags); 839 } 840 841 if (start_blk > last_blk || ret) 842 goto out; 843 844 logical = blk_to_logical(inode, start_blk); 845 phys = blk_to_logical(inode, map_bh.b_blocknr); 846 size = map_bh.b_size; 847 flags = 0; 848 if (buffer_unwritten(&map_bh)) 849 flags = FIEMAP_EXTENT_UNWRITTEN; 850 851 start_blk += logical_to_blk(inode, size); 852 853 prep_next: 854 cond_resched(); 855 if (fatal_signal_pending(current)) 856 ret = -EINTR; 857 else 858 goto next; 859 out: 860 if (ret == 1) 861 ret = 0; 862 863 inode_unlock(inode); 864 return ret; 865 } 866 867 /* 868 * This function was originally taken from fs/mpage.c, and customized for f2fs. 869 * Major change was from block_size == page_size in f2fs by default. 870 */ 871 static int f2fs_mpage_readpages(struct address_space *mapping, 872 struct list_head *pages, struct page *page, 873 unsigned nr_pages) 874 { 875 struct bio *bio = NULL; 876 unsigned page_idx; 877 sector_t last_block_in_bio = 0; 878 struct inode *inode = mapping->host; 879 const unsigned blkbits = inode->i_blkbits; 880 const unsigned blocksize = 1 << blkbits; 881 sector_t block_in_file; 882 sector_t last_block; 883 sector_t last_block_in_file; 884 sector_t block_nr; 885 struct block_device *bdev = inode->i_sb->s_bdev; 886 struct f2fs_map_blocks map; 887 888 map.m_pblk = 0; 889 map.m_lblk = 0; 890 map.m_len = 0; 891 map.m_flags = 0; 892 893 for (page_idx = 0; nr_pages; page_idx++, nr_pages--) { 894 895 prefetchw(&page->flags); 896 if (pages) { 897 page = list_entry(pages->prev, struct page, lru); 898 list_del(&page->lru); 899 if (add_to_page_cache_lru(page, mapping, 900 page->index, GFP_KERNEL)) 901 goto next_page; 902 } 903 904 block_in_file = (sector_t)page->index; 905 last_block = block_in_file + nr_pages; 906 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> 907 blkbits; 908 if (last_block > last_block_in_file) 909 last_block = last_block_in_file; 910 911 /* 912 * Map blocks using the previous result first. 913 */ 914 if ((map.m_flags & F2FS_MAP_MAPPED) && 915 block_in_file > map.m_lblk && 916 block_in_file < (map.m_lblk + map.m_len)) 917 goto got_it; 918 919 /* 920 * Then do more f2fs_map_blocks() calls until we are 921 * done with this page. 922 */ 923 map.m_flags = 0; 924 925 if (block_in_file < last_block) { 926 map.m_lblk = block_in_file; 927 map.m_len = last_block - block_in_file; 928 929 if (f2fs_map_blocks(inode, &map, 0, 930 F2FS_GET_BLOCK_READ)) 931 goto set_error_page; 932 } 933 got_it: 934 if ((map.m_flags & F2FS_MAP_MAPPED)) { 935 block_nr = map.m_pblk + block_in_file - map.m_lblk; 936 SetPageMappedToDisk(page); 937 938 if (!PageUptodate(page) && !cleancache_get_page(page)) { 939 SetPageUptodate(page); 940 goto confused; 941 } 942 } else { 943 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 944 SetPageUptodate(page); 945 unlock_page(page); 946 goto next_page; 947 } 948 949 /* 950 * This page will go to BIO. Do we need to send this 951 * BIO off first? 952 */ 953 if (bio && (last_block_in_bio != block_nr - 1)) { 954 submit_and_realloc: 955 submit_bio(READ, bio); 956 bio = NULL; 957 } 958 if (bio == NULL) { 959 struct f2fs_crypto_ctx *ctx = NULL; 960 961 if (f2fs_encrypted_inode(inode) && 962 S_ISREG(inode->i_mode)) { 963 964 ctx = f2fs_get_crypto_ctx(inode); 965 if (IS_ERR(ctx)) 966 goto set_error_page; 967 968 /* wait the page to be moved by cleaning */ 969 f2fs_wait_on_encrypted_page_writeback( 970 F2FS_I_SB(inode), block_nr); 971 } 972 973 bio = bio_alloc(GFP_KERNEL, 974 min_t(int, nr_pages, BIO_MAX_PAGES)); 975 if (!bio) { 976 if (ctx) 977 f2fs_release_crypto_ctx(ctx); 978 goto set_error_page; 979 } 980 bio->bi_bdev = bdev; 981 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr); 982 bio->bi_end_io = f2fs_read_end_io; 983 bio->bi_private = ctx; 984 } 985 986 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 987 goto submit_and_realloc; 988 989 last_block_in_bio = block_nr; 990 goto next_page; 991 set_error_page: 992 SetPageError(page); 993 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 994 unlock_page(page); 995 goto next_page; 996 confused: 997 if (bio) { 998 submit_bio(READ, bio); 999 bio = NULL; 1000 } 1001 unlock_page(page); 1002 next_page: 1003 if (pages) 1004 page_cache_release(page); 1005 } 1006 BUG_ON(pages && !list_empty(pages)); 1007 if (bio) 1008 submit_bio(READ, bio); 1009 return 0; 1010 } 1011 1012 static int f2fs_read_data_page(struct file *file, struct page *page) 1013 { 1014 struct inode *inode = page->mapping->host; 1015 int ret = -EAGAIN; 1016 1017 trace_f2fs_readpage(page, DATA); 1018 1019 /* If the file has inline data, try to read it directly */ 1020 if (f2fs_has_inline_data(inode)) 1021 ret = f2fs_read_inline_data(inode, page); 1022 if (ret == -EAGAIN) 1023 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1); 1024 return ret; 1025 } 1026 1027 static int f2fs_read_data_pages(struct file *file, 1028 struct address_space *mapping, 1029 struct list_head *pages, unsigned nr_pages) 1030 { 1031 struct inode *inode = file->f_mapping->host; 1032 struct page *page = list_entry(pages->prev, struct page, lru); 1033 1034 trace_f2fs_readpages(inode, page, nr_pages); 1035 1036 /* If the file has inline data, skip readpages */ 1037 if (f2fs_has_inline_data(inode)) 1038 return 0; 1039 1040 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages); 1041 } 1042 1043 int do_write_data_page(struct f2fs_io_info *fio) 1044 { 1045 struct page *page = fio->page; 1046 struct inode *inode = page->mapping->host; 1047 struct dnode_of_data dn; 1048 int err = 0; 1049 1050 set_new_dnode(&dn, inode, NULL, NULL, 0); 1051 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 1052 if (err) 1053 return err; 1054 1055 fio->blk_addr = dn.data_blkaddr; 1056 1057 /* This page is already truncated */ 1058 if (fio->blk_addr == NULL_ADDR) { 1059 ClearPageUptodate(page); 1060 goto out_writepage; 1061 } 1062 1063 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { 1064 1065 /* wait for GCed encrypted page writeback */ 1066 f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode), 1067 fio->blk_addr); 1068 1069 fio->encrypted_page = f2fs_encrypt(inode, fio->page); 1070 if (IS_ERR(fio->encrypted_page)) { 1071 err = PTR_ERR(fio->encrypted_page); 1072 goto out_writepage; 1073 } 1074 } 1075 1076 set_page_writeback(page); 1077 1078 /* 1079 * If current allocation needs SSR, 1080 * it had better in-place writes for updated data. 1081 */ 1082 if (unlikely(fio->blk_addr != NEW_ADDR && 1083 !is_cold_data(page) && 1084 !IS_ATOMIC_WRITTEN_PAGE(page) && 1085 need_inplace_update(inode))) { 1086 rewrite_data_page(fio); 1087 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE); 1088 trace_f2fs_do_write_data_page(page, IPU); 1089 } else { 1090 write_data_page(&dn, fio); 1091 set_data_blkaddr(&dn); 1092 f2fs_update_extent_cache(&dn); 1093 trace_f2fs_do_write_data_page(page, OPU); 1094 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); 1095 if (page->index == 0) 1096 set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN); 1097 } 1098 out_writepage: 1099 f2fs_put_dnode(&dn); 1100 return err; 1101 } 1102 1103 static int f2fs_write_data_page(struct page *page, 1104 struct writeback_control *wbc) 1105 { 1106 struct inode *inode = page->mapping->host; 1107 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1108 loff_t i_size = i_size_read(inode); 1109 const pgoff_t end_index = ((unsigned long long) i_size) 1110 >> PAGE_CACHE_SHIFT; 1111 unsigned offset = 0; 1112 bool need_balance_fs = false; 1113 int err = 0; 1114 struct f2fs_io_info fio = { 1115 .sbi = sbi, 1116 .type = DATA, 1117 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE, 1118 .page = page, 1119 .encrypted_page = NULL, 1120 }; 1121 1122 trace_f2fs_writepage(page, DATA); 1123 1124 if (page->index < end_index) 1125 goto write; 1126 1127 /* 1128 * If the offset is out-of-range of file size, 1129 * this page does not have to be written to disk. 1130 */ 1131 offset = i_size & (PAGE_CACHE_SIZE - 1); 1132 if ((page->index >= end_index + 1) || !offset) 1133 goto out; 1134 1135 zero_user_segment(page, offset, PAGE_CACHE_SIZE); 1136 write: 1137 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1138 goto redirty_out; 1139 if (f2fs_is_drop_cache(inode)) 1140 goto out; 1141 if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim && 1142 available_free_memory(sbi, BASE_CHECK)) 1143 goto redirty_out; 1144 1145 /* Dentry blocks are controlled by checkpoint */ 1146 if (S_ISDIR(inode->i_mode)) { 1147 if (unlikely(f2fs_cp_error(sbi))) 1148 goto redirty_out; 1149 err = do_write_data_page(&fio); 1150 goto done; 1151 } 1152 1153 /* we should bypass data pages to proceed the kworkder jobs */ 1154 if (unlikely(f2fs_cp_error(sbi))) { 1155 SetPageError(page); 1156 goto out; 1157 } 1158 1159 if (!wbc->for_reclaim) 1160 need_balance_fs = true; 1161 else if (has_not_enough_free_secs(sbi, 0)) 1162 goto redirty_out; 1163 1164 err = -EAGAIN; 1165 f2fs_lock_op(sbi); 1166 if (f2fs_has_inline_data(inode)) 1167 err = f2fs_write_inline_data(inode, page); 1168 if (err == -EAGAIN) 1169 err = do_write_data_page(&fio); 1170 f2fs_unlock_op(sbi); 1171 done: 1172 if (err && err != -ENOENT) 1173 goto redirty_out; 1174 1175 clear_cold_data(page); 1176 out: 1177 inode_dec_dirty_pages(inode); 1178 if (err) 1179 ClearPageUptodate(page); 1180 unlock_page(page); 1181 f2fs_balance_fs(sbi, need_balance_fs); 1182 if (wbc->for_reclaim || unlikely(f2fs_cp_error(sbi))) { 1183 f2fs_submit_merged_bio(sbi, DATA, WRITE); 1184 remove_dirty_inode(inode); 1185 } 1186 return 0; 1187 1188 redirty_out: 1189 redirty_page_for_writepage(wbc, page); 1190 return AOP_WRITEPAGE_ACTIVATE; 1191 } 1192 1193 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc, 1194 void *data) 1195 { 1196 struct address_space *mapping = data; 1197 int ret = mapping->a_ops->writepage(page, wbc); 1198 mapping_set_error(mapping, ret); 1199 return ret; 1200 } 1201 1202 /* 1203 * This function was copied from write_cche_pages from mm/page-writeback.c. 1204 * The major change is making write step of cold data page separately from 1205 * warm/hot data page. 1206 */ 1207 static int f2fs_write_cache_pages(struct address_space *mapping, 1208 struct writeback_control *wbc, writepage_t writepage, 1209 void *data) 1210 { 1211 int ret = 0; 1212 int done = 0; 1213 struct pagevec pvec; 1214 int nr_pages; 1215 pgoff_t uninitialized_var(writeback_index); 1216 pgoff_t index; 1217 pgoff_t end; /* Inclusive */ 1218 pgoff_t done_index; 1219 int cycled; 1220 int range_whole = 0; 1221 int tag; 1222 int step = 0; 1223 1224 pagevec_init(&pvec, 0); 1225 next: 1226 if (wbc->range_cyclic) { 1227 writeback_index = mapping->writeback_index; /* prev offset */ 1228 index = writeback_index; 1229 if (index == 0) 1230 cycled = 1; 1231 else 1232 cycled = 0; 1233 end = -1; 1234 } else { 1235 index = wbc->range_start >> PAGE_CACHE_SHIFT; 1236 end = wbc->range_end >> PAGE_CACHE_SHIFT; 1237 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 1238 range_whole = 1; 1239 cycled = 1; /* ignore range_cyclic tests */ 1240 } 1241 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1242 tag = PAGECACHE_TAG_TOWRITE; 1243 else 1244 tag = PAGECACHE_TAG_DIRTY; 1245 retry: 1246 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1247 tag_pages_for_writeback(mapping, index, end); 1248 done_index = index; 1249 while (!done && (index <= end)) { 1250 int i; 1251 1252 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 1253 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1); 1254 if (nr_pages == 0) 1255 break; 1256 1257 for (i = 0; i < nr_pages; i++) { 1258 struct page *page = pvec.pages[i]; 1259 1260 if (page->index > end) { 1261 done = 1; 1262 break; 1263 } 1264 1265 done_index = page->index; 1266 1267 lock_page(page); 1268 1269 if (unlikely(page->mapping != mapping)) { 1270 continue_unlock: 1271 unlock_page(page); 1272 continue; 1273 } 1274 1275 if (!PageDirty(page)) { 1276 /* someone wrote it for us */ 1277 goto continue_unlock; 1278 } 1279 1280 if (step == is_cold_data(page)) 1281 goto continue_unlock; 1282 1283 if (PageWriteback(page)) { 1284 if (wbc->sync_mode != WB_SYNC_NONE) 1285 f2fs_wait_on_page_writeback(page, DATA); 1286 else 1287 goto continue_unlock; 1288 } 1289 1290 BUG_ON(PageWriteback(page)); 1291 if (!clear_page_dirty_for_io(page)) 1292 goto continue_unlock; 1293 1294 ret = (*writepage)(page, wbc, data); 1295 if (unlikely(ret)) { 1296 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1297 unlock_page(page); 1298 ret = 0; 1299 } else { 1300 done_index = page->index + 1; 1301 done = 1; 1302 break; 1303 } 1304 } 1305 1306 if (--wbc->nr_to_write <= 0 && 1307 wbc->sync_mode == WB_SYNC_NONE) { 1308 done = 1; 1309 break; 1310 } 1311 } 1312 pagevec_release(&pvec); 1313 cond_resched(); 1314 } 1315 1316 if (step < 1) { 1317 step++; 1318 goto next; 1319 } 1320 1321 if (!cycled && !done) { 1322 cycled = 1; 1323 index = 0; 1324 end = writeback_index - 1; 1325 goto retry; 1326 } 1327 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 1328 mapping->writeback_index = done_index; 1329 1330 return ret; 1331 } 1332 1333 static int f2fs_write_data_pages(struct address_space *mapping, 1334 struct writeback_control *wbc) 1335 { 1336 struct inode *inode = mapping->host; 1337 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1338 bool locked = false; 1339 int ret; 1340 long diff; 1341 1342 trace_f2fs_writepages(mapping->host, wbc, DATA); 1343 1344 /* deal with chardevs and other special file */ 1345 if (!mapping->a_ops->writepage) 1346 return 0; 1347 1348 /* skip writing if there is no dirty page in this inode */ 1349 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 1350 return 0; 1351 1352 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE && 1353 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 1354 available_free_memory(sbi, DIRTY_DENTS)) 1355 goto skip_write; 1356 1357 /* skip writing during file defragment */ 1358 if (is_inode_flag_set(F2FS_I(inode), FI_DO_DEFRAG)) 1359 goto skip_write; 1360 1361 /* during POR, we don't need to trigger writepage at all. */ 1362 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1363 goto skip_write; 1364 1365 diff = nr_pages_to_write(sbi, DATA, wbc); 1366 1367 if (!S_ISDIR(inode->i_mode)) { 1368 mutex_lock(&sbi->writepages); 1369 locked = true; 1370 } 1371 ret = f2fs_write_cache_pages(mapping, wbc, __f2fs_writepage, mapping); 1372 f2fs_submit_merged_bio(sbi, DATA, WRITE); 1373 if (locked) 1374 mutex_unlock(&sbi->writepages); 1375 1376 remove_dirty_inode(inode); 1377 1378 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); 1379 return ret; 1380 1381 skip_write: 1382 wbc->pages_skipped += get_dirty_pages(inode); 1383 return 0; 1384 } 1385 1386 static void f2fs_write_failed(struct address_space *mapping, loff_t to) 1387 { 1388 struct inode *inode = mapping->host; 1389 loff_t i_size = i_size_read(inode); 1390 1391 if (to > i_size) { 1392 truncate_pagecache(inode, i_size); 1393 truncate_blocks(inode, i_size, true); 1394 } 1395 } 1396 1397 static int prepare_write_begin(struct f2fs_sb_info *sbi, 1398 struct page *page, loff_t pos, unsigned len, 1399 block_t *blk_addr, bool *node_changed) 1400 { 1401 struct inode *inode = page->mapping->host; 1402 pgoff_t index = page->index; 1403 struct dnode_of_data dn; 1404 struct page *ipage; 1405 bool locked = false; 1406 struct extent_info ei; 1407 int err = 0; 1408 1409 if (f2fs_has_inline_data(inode) || 1410 (pos & PAGE_CACHE_MASK) >= i_size_read(inode)) { 1411 f2fs_lock_op(sbi); 1412 locked = true; 1413 } 1414 restart: 1415 /* check inline_data */ 1416 ipage = get_node_page(sbi, inode->i_ino); 1417 if (IS_ERR(ipage)) { 1418 err = PTR_ERR(ipage); 1419 goto unlock_out; 1420 } 1421 1422 set_new_dnode(&dn, inode, ipage, ipage, 0); 1423 1424 if (f2fs_has_inline_data(inode)) { 1425 if (pos + len <= MAX_INLINE_DATA) { 1426 read_inline_data(page, ipage); 1427 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); 1428 sync_inode_page(&dn); 1429 } else { 1430 err = f2fs_convert_inline_page(&dn, page); 1431 if (err) 1432 goto out; 1433 if (dn.data_blkaddr == NULL_ADDR) 1434 err = f2fs_get_block(&dn, index); 1435 } 1436 } else if (locked) { 1437 err = f2fs_get_block(&dn, index); 1438 } else { 1439 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 1440 dn.data_blkaddr = ei.blk + index - ei.fofs; 1441 } else { 1442 bool restart = false; 1443 1444 /* hole case */ 1445 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 1446 if (err || (!err && dn.data_blkaddr == NULL_ADDR)) 1447 restart = true; 1448 if (restart) { 1449 f2fs_put_dnode(&dn); 1450 f2fs_lock_op(sbi); 1451 locked = true; 1452 goto restart; 1453 } 1454 } 1455 } 1456 1457 /* convert_inline_page can make node_changed */ 1458 *blk_addr = dn.data_blkaddr; 1459 *node_changed = dn.node_changed; 1460 out: 1461 f2fs_put_dnode(&dn); 1462 unlock_out: 1463 if (locked) 1464 f2fs_unlock_op(sbi); 1465 return err; 1466 } 1467 1468 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 1469 loff_t pos, unsigned len, unsigned flags, 1470 struct page **pagep, void **fsdata) 1471 { 1472 struct inode *inode = mapping->host; 1473 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1474 struct page *page = NULL; 1475 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT; 1476 bool need_balance = false; 1477 block_t blkaddr = NULL_ADDR; 1478 int err = 0; 1479 1480 trace_f2fs_write_begin(inode, pos, len, flags); 1481 1482 /* 1483 * We should check this at this moment to avoid deadlock on inode page 1484 * and #0 page. The locking rule for inline_data conversion should be: 1485 * lock_page(page #0) -> lock_page(inode_page) 1486 */ 1487 if (index != 0) { 1488 err = f2fs_convert_inline_inode(inode); 1489 if (err) 1490 goto fail; 1491 } 1492 repeat: 1493 page = grab_cache_page_write_begin(mapping, index, flags); 1494 if (!page) { 1495 err = -ENOMEM; 1496 goto fail; 1497 } 1498 1499 *pagep = page; 1500 1501 err = prepare_write_begin(sbi, page, pos, len, 1502 &blkaddr, &need_balance); 1503 if (err) 1504 goto fail; 1505 1506 if (need_balance && has_not_enough_free_secs(sbi, 0)) { 1507 unlock_page(page); 1508 f2fs_balance_fs(sbi, true); 1509 lock_page(page); 1510 if (page->mapping != mapping) { 1511 /* The page got truncated from under us */ 1512 f2fs_put_page(page, 1); 1513 goto repeat; 1514 } 1515 } 1516 1517 f2fs_wait_on_page_writeback(page, DATA); 1518 1519 /* wait for GCed encrypted page writeback */ 1520 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 1521 f2fs_wait_on_encrypted_page_writeback(sbi, blkaddr); 1522 1523 if (len == PAGE_CACHE_SIZE) 1524 goto out_update; 1525 if (PageUptodate(page)) 1526 goto out_clear; 1527 1528 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) { 1529 unsigned start = pos & (PAGE_CACHE_SIZE - 1); 1530 unsigned end = start + len; 1531 1532 /* Reading beyond i_size is simple: memset to zero */ 1533 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE); 1534 goto out_update; 1535 } 1536 1537 if (blkaddr == NEW_ADDR) { 1538 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 1539 } else { 1540 struct f2fs_io_info fio = { 1541 .sbi = sbi, 1542 .type = DATA, 1543 .rw = READ_SYNC, 1544 .blk_addr = blkaddr, 1545 .page = page, 1546 .encrypted_page = NULL, 1547 }; 1548 err = f2fs_submit_page_bio(&fio); 1549 if (err) 1550 goto fail; 1551 1552 lock_page(page); 1553 if (unlikely(!PageUptodate(page))) { 1554 err = -EIO; 1555 goto fail; 1556 } 1557 if (unlikely(page->mapping != mapping)) { 1558 f2fs_put_page(page, 1); 1559 goto repeat; 1560 } 1561 1562 /* avoid symlink page */ 1563 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { 1564 err = f2fs_decrypt_one(inode, page); 1565 if (err) 1566 goto fail; 1567 } 1568 } 1569 out_update: 1570 SetPageUptodate(page); 1571 out_clear: 1572 clear_cold_data(page); 1573 return 0; 1574 1575 fail: 1576 f2fs_put_page(page, 1); 1577 f2fs_write_failed(mapping, pos + len); 1578 return err; 1579 } 1580 1581 static int f2fs_write_end(struct file *file, 1582 struct address_space *mapping, 1583 loff_t pos, unsigned len, unsigned copied, 1584 struct page *page, void *fsdata) 1585 { 1586 struct inode *inode = page->mapping->host; 1587 1588 trace_f2fs_write_end(inode, pos, len, copied); 1589 1590 set_page_dirty(page); 1591 1592 if (pos + copied > i_size_read(inode)) { 1593 i_size_write(inode, pos + copied); 1594 mark_inode_dirty(inode); 1595 update_inode_page(inode); 1596 } 1597 1598 f2fs_put_page(page, 1); 1599 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 1600 return copied; 1601 } 1602 1603 static int check_direct_IO(struct inode *inode, struct iov_iter *iter, 1604 loff_t offset) 1605 { 1606 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1; 1607 1608 if (offset & blocksize_mask) 1609 return -EINVAL; 1610 1611 if (iov_iter_alignment(iter) & blocksize_mask) 1612 return -EINVAL; 1613 1614 return 0; 1615 } 1616 1617 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter, 1618 loff_t offset) 1619 { 1620 struct file *file = iocb->ki_filp; 1621 struct address_space *mapping = file->f_mapping; 1622 struct inode *inode = mapping->host; 1623 size_t count = iov_iter_count(iter); 1624 int err; 1625 1626 /* we don't need to use inline_data strictly */ 1627 err = f2fs_convert_inline_inode(inode); 1628 if (err) 1629 return err; 1630 1631 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 1632 return 0; 1633 1634 err = check_direct_IO(inode, iter, offset); 1635 if (err) 1636 return err; 1637 1638 trace_f2fs_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); 1639 1640 if (iov_iter_rw(iter) == WRITE) { 1641 err = __allocate_data_blocks(inode, offset, count); 1642 if (err) 1643 goto out; 1644 } 1645 1646 err = blockdev_direct_IO(iocb, inode, iter, offset, get_data_block_dio); 1647 out: 1648 if (err < 0 && iov_iter_rw(iter) == WRITE) 1649 f2fs_write_failed(mapping, offset + count); 1650 1651 trace_f2fs_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), err); 1652 1653 return err; 1654 } 1655 1656 void f2fs_invalidate_page(struct page *page, unsigned int offset, 1657 unsigned int length) 1658 { 1659 struct inode *inode = page->mapping->host; 1660 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1661 1662 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 1663 (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE)) 1664 return; 1665 1666 if (PageDirty(page)) { 1667 if (inode->i_ino == F2FS_META_INO(sbi)) 1668 dec_page_count(sbi, F2FS_DIRTY_META); 1669 else if (inode->i_ino == F2FS_NODE_INO(sbi)) 1670 dec_page_count(sbi, F2FS_DIRTY_NODES); 1671 else 1672 inode_dec_dirty_pages(inode); 1673 } 1674 1675 /* This is atomic written page, keep Private */ 1676 if (IS_ATOMIC_WRITTEN_PAGE(page)) 1677 return; 1678 1679 ClearPagePrivate(page); 1680 } 1681 1682 int f2fs_release_page(struct page *page, gfp_t wait) 1683 { 1684 /* If this is dirty page, keep PagePrivate */ 1685 if (PageDirty(page)) 1686 return 0; 1687 1688 /* This is atomic written page, keep Private */ 1689 if (IS_ATOMIC_WRITTEN_PAGE(page)) 1690 return 0; 1691 1692 ClearPagePrivate(page); 1693 return 1; 1694 } 1695 1696 static int f2fs_set_data_page_dirty(struct page *page) 1697 { 1698 struct address_space *mapping = page->mapping; 1699 struct inode *inode = mapping->host; 1700 1701 trace_f2fs_set_page_dirty(page, DATA); 1702 1703 SetPageUptodate(page); 1704 1705 if (f2fs_is_atomic_file(inode)) { 1706 if (!IS_ATOMIC_WRITTEN_PAGE(page)) { 1707 register_inmem_page(inode, page); 1708 return 1; 1709 } 1710 /* 1711 * Previously, this page has been registered, we just 1712 * return here. 1713 */ 1714 return 0; 1715 } 1716 1717 if (!PageDirty(page)) { 1718 __set_page_dirty_nobuffers(page); 1719 update_dirty_page(inode, page); 1720 return 1; 1721 } 1722 return 0; 1723 } 1724 1725 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 1726 { 1727 struct inode *inode = mapping->host; 1728 1729 if (f2fs_has_inline_data(inode)) 1730 return 0; 1731 1732 /* make sure allocating whole blocks */ 1733 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 1734 filemap_write_and_wait(mapping); 1735 1736 return generic_block_bmap(mapping, block, get_data_block_bmap); 1737 } 1738 1739 const struct address_space_operations f2fs_dblock_aops = { 1740 .readpage = f2fs_read_data_page, 1741 .readpages = f2fs_read_data_pages, 1742 .writepage = f2fs_write_data_page, 1743 .writepages = f2fs_write_data_pages, 1744 .write_begin = f2fs_write_begin, 1745 .write_end = f2fs_write_end, 1746 .set_page_dirty = f2fs_set_data_page_dirty, 1747 .invalidatepage = f2fs_invalidate_page, 1748 .releasepage = f2fs_release_page, 1749 .direct_IO = f2fs_direct_IO, 1750 .bmap = f2fs_bmap, 1751 }; 1752