1 /* 2 * fs/f2fs/data.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/fs.h> 12 #include <linux/f2fs_fs.h> 13 #include <linux/buffer_head.h> 14 #include <linux/mpage.h> 15 #include <linux/aio.h> 16 #include <linux/writeback.h> 17 #include <linux/backing-dev.h> 18 #include <linux/blkdev.h> 19 #include <linux/bio.h> 20 #include <linux/prefetch.h> 21 22 #include "f2fs.h" 23 #include "node.h" 24 #include "segment.h" 25 #include <trace/events/f2fs.h> 26 27 static void f2fs_read_end_io(struct bio *bio, int err) 28 { 29 struct bio_vec *bvec; 30 int i; 31 32 bio_for_each_segment_all(bvec, bio, i) { 33 struct page *page = bvec->bv_page; 34 35 if (!err) { 36 SetPageUptodate(page); 37 } else { 38 ClearPageUptodate(page); 39 SetPageError(page); 40 } 41 unlock_page(page); 42 } 43 bio_put(bio); 44 } 45 46 static void f2fs_write_end_io(struct bio *bio, int err) 47 { 48 struct f2fs_sb_info *sbi = bio->bi_private; 49 struct bio_vec *bvec; 50 int i; 51 52 bio_for_each_segment_all(bvec, bio, i) { 53 struct page *page = bvec->bv_page; 54 55 if (unlikely(err)) { 56 set_page_dirty(page); 57 set_bit(AS_EIO, &page->mapping->flags); 58 f2fs_stop_checkpoint(sbi); 59 } 60 end_page_writeback(page); 61 dec_page_count(sbi, F2FS_WRITEBACK); 62 } 63 64 if (!get_pages(sbi, F2FS_WRITEBACK) && 65 !list_empty(&sbi->cp_wait.task_list)) 66 wake_up(&sbi->cp_wait); 67 68 bio_put(bio); 69 } 70 71 /* 72 * Low-level block read/write IO operations. 73 */ 74 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, 75 int npages, bool is_read) 76 { 77 struct bio *bio; 78 79 /* No failure on bio allocation */ 80 bio = bio_alloc(GFP_NOIO, npages); 81 82 bio->bi_bdev = sbi->sb->s_bdev; 83 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); 84 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; 85 bio->bi_private = sbi; 86 87 return bio; 88 } 89 90 static void __submit_merged_bio(struct f2fs_bio_info *io) 91 { 92 struct f2fs_io_info *fio = &io->fio; 93 94 if (!io->bio) 95 return; 96 97 if (is_read_io(fio->rw)) 98 trace_f2fs_submit_read_bio(io->sbi->sb, fio->rw, 99 fio->type, io->bio); 100 else 101 trace_f2fs_submit_write_bio(io->sbi->sb, fio->rw, 102 fio->type, io->bio); 103 104 submit_bio(fio->rw, io->bio); 105 io->bio = NULL; 106 } 107 108 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi, 109 enum page_type type, int rw) 110 { 111 enum page_type btype = PAGE_TYPE_OF_BIO(type); 112 struct f2fs_bio_info *io; 113 114 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype]; 115 116 down_write(&io->io_rwsem); 117 118 /* change META to META_FLUSH in the checkpoint procedure */ 119 if (type >= META_FLUSH) { 120 io->fio.type = META_FLUSH; 121 if (test_opt(sbi, NOBARRIER)) 122 io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO; 123 else 124 io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO; 125 } 126 __submit_merged_bio(io); 127 up_write(&io->io_rwsem); 128 } 129 130 /* 131 * Fill the locked page with data located in the block address. 132 * Return unlocked page. 133 */ 134 int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page, 135 block_t blk_addr, int rw) 136 { 137 struct bio *bio; 138 139 trace_f2fs_submit_page_bio(page, blk_addr, rw); 140 141 /* Allocate a new bio */ 142 bio = __bio_alloc(sbi, blk_addr, 1, is_read_io(rw)); 143 144 if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) { 145 bio_put(bio); 146 f2fs_put_page(page, 1); 147 return -EFAULT; 148 } 149 150 submit_bio(rw, bio); 151 return 0; 152 } 153 154 void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page, 155 block_t blk_addr, struct f2fs_io_info *fio) 156 { 157 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 158 struct f2fs_bio_info *io; 159 bool is_read = is_read_io(fio->rw); 160 161 io = is_read ? &sbi->read_io : &sbi->write_io[btype]; 162 163 verify_block_addr(sbi, blk_addr); 164 165 down_write(&io->io_rwsem); 166 167 if (!is_read) 168 inc_page_count(sbi, F2FS_WRITEBACK); 169 170 if (io->bio && (io->last_block_in_bio != blk_addr - 1 || 171 io->fio.rw != fio->rw)) 172 __submit_merged_bio(io); 173 alloc_new: 174 if (io->bio == NULL) { 175 int bio_blocks = MAX_BIO_BLOCKS(sbi); 176 177 io->bio = __bio_alloc(sbi, blk_addr, bio_blocks, is_read); 178 io->fio = *fio; 179 } 180 181 if (bio_add_page(io->bio, page, PAGE_CACHE_SIZE, 0) < 182 PAGE_CACHE_SIZE) { 183 __submit_merged_bio(io); 184 goto alloc_new; 185 } 186 187 io->last_block_in_bio = blk_addr; 188 189 up_write(&io->io_rwsem); 190 trace_f2fs_submit_page_mbio(page, fio->rw, fio->type, blk_addr); 191 } 192 193 /* 194 * Lock ordering for the change of data block address: 195 * ->data_page 196 * ->node_page 197 * update block addresses in the node page 198 */ 199 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr) 200 { 201 struct f2fs_node *rn; 202 __le32 *addr_array; 203 struct page *node_page = dn->node_page; 204 unsigned int ofs_in_node = dn->ofs_in_node; 205 206 f2fs_wait_on_page_writeback(node_page, NODE); 207 208 rn = F2FS_NODE(node_page); 209 210 /* Get physical address of data block */ 211 addr_array = blkaddr_in_node(rn); 212 addr_array[ofs_in_node] = cpu_to_le32(new_addr); 213 set_page_dirty(node_page); 214 } 215 216 int reserve_new_block(struct dnode_of_data *dn) 217 { 218 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 219 220 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 221 return -EPERM; 222 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 223 return -ENOSPC; 224 225 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node); 226 227 __set_data_blkaddr(dn, NEW_ADDR); 228 dn->data_blkaddr = NEW_ADDR; 229 mark_inode_dirty(dn->inode); 230 sync_inode_page(dn); 231 return 0; 232 } 233 234 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 235 { 236 bool need_put = dn->inode_page ? false : true; 237 int err; 238 239 err = get_dnode_of_data(dn, index, ALLOC_NODE); 240 if (err) 241 return err; 242 243 if (dn->data_blkaddr == NULL_ADDR) 244 err = reserve_new_block(dn); 245 if (err || need_put) 246 f2fs_put_dnode(dn); 247 return err; 248 } 249 250 static int check_extent_cache(struct inode *inode, pgoff_t pgofs, 251 struct buffer_head *bh_result) 252 { 253 struct f2fs_inode_info *fi = F2FS_I(inode); 254 pgoff_t start_fofs, end_fofs; 255 block_t start_blkaddr; 256 257 if (is_inode_flag_set(fi, FI_NO_EXTENT)) 258 return 0; 259 260 read_lock(&fi->ext.ext_lock); 261 if (fi->ext.len == 0) { 262 read_unlock(&fi->ext.ext_lock); 263 return 0; 264 } 265 266 stat_inc_total_hit(inode->i_sb); 267 268 start_fofs = fi->ext.fofs; 269 end_fofs = fi->ext.fofs + fi->ext.len - 1; 270 start_blkaddr = fi->ext.blk_addr; 271 272 if (pgofs >= start_fofs && pgofs <= end_fofs) { 273 unsigned int blkbits = inode->i_sb->s_blocksize_bits; 274 size_t count; 275 276 clear_buffer_new(bh_result); 277 map_bh(bh_result, inode->i_sb, 278 start_blkaddr + pgofs - start_fofs); 279 count = end_fofs - pgofs + 1; 280 if (count < (UINT_MAX >> blkbits)) 281 bh_result->b_size = (count << blkbits); 282 else 283 bh_result->b_size = UINT_MAX; 284 285 stat_inc_read_hit(inode->i_sb); 286 read_unlock(&fi->ext.ext_lock); 287 return 1; 288 } 289 read_unlock(&fi->ext.ext_lock); 290 return 0; 291 } 292 293 void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn) 294 { 295 struct f2fs_inode_info *fi = F2FS_I(dn->inode); 296 pgoff_t fofs, start_fofs, end_fofs; 297 block_t start_blkaddr, end_blkaddr; 298 int need_update = true; 299 300 f2fs_bug_on(F2FS_I_SB(dn->inode), blk_addr == NEW_ADDR); 301 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) + 302 dn->ofs_in_node; 303 304 /* Update the page address in the parent node */ 305 __set_data_blkaddr(dn, blk_addr); 306 307 if (is_inode_flag_set(fi, FI_NO_EXTENT)) 308 return; 309 310 write_lock(&fi->ext.ext_lock); 311 312 start_fofs = fi->ext.fofs; 313 end_fofs = fi->ext.fofs + fi->ext.len - 1; 314 start_blkaddr = fi->ext.blk_addr; 315 end_blkaddr = fi->ext.blk_addr + fi->ext.len - 1; 316 317 /* Drop and initialize the matched extent */ 318 if (fi->ext.len == 1 && fofs == start_fofs) 319 fi->ext.len = 0; 320 321 /* Initial extent */ 322 if (fi->ext.len == 0) { 323 if (blk_addr != NULL_ADDR) { 324 fi->ext.fofs = fofs; 325 fi->ext.blk_addr = blk_addr; 326 fi->ext.len = 1; 327 } 328 goto end_update; 329 } 330 331 /* Front merge */ 332 if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) { 333 fi->ext.fofs--; 334 fi->ext.blk_addr--; 335 fi->ext.len++; 336 goto end_update; 337 } 338 339 /* Back merge */ 340 if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) { 341 fi->ext.len++; 342 goto end_update; 343 } 344 345 /* Split the existing extent */ 346 if (fi->ext.len > 1 && 347 fofs >= start_fofs && fofs <= end_fofs) { 348 if ((end_fofs - fofs) < (fi->ext.len >> 1)) { 349 fi->ext.len = fofs - start_fofs; 350 } else { 351 fi->ext.fofs = fofs + 1; 352 fi->ext.blk_addr = start_blkaddr + 353 fofs - start_fofs + 1; 354 fi->ext.len -= fofs - start_fofs + 1; 355 } 356 } else { 357 need_update = false; 358 } 359 360 /* Finally, if the extent is very fragmented, let's drop the cache. */ 361 if (fi->ext.len < F2FS_MIN_EXTENT_LEN) { 362 fi->ext.len = 0; 363 set_inode_flag(fi, FI_NO_EXTENT); 364 need_update = true; 365 } 366 end_update: 367 write_unlock(&fi->ext.ext_lock); 368 if (need_update) 369 sync_inode_page(dn); 370 return; 371 } 372 373 struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync) 374 { 375 struct address_space *mapping = inode->i_mapping; 376 struct dnode_of_data dn; 377 struct page *page; 378 int err; 379 380 page = find_get_page(mapping, index); 381 if (page && PageUptodate(page)) 382 return page; 383 f2fs_put_page(page, 0); 384 385 set_new_dnode(&dn, inode, NULL, NULL, 0); 386 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 387 if (err) 388 return ERR_PTR(err); 389 f2fs_put_dnode(&dn); 390 391 if (dn.data_blkaddr == NULL_ADDR) 392 return ERR_PTR(-ENOENT); 393 394 /* By fallocate(), there is no cached page, but with NEW_ADDR */ 395 if (unlikely(dn.data_blkaddr == NEW_ADDR)) 396 return ERR_PTR(-EINVAL); 397 398 page = grab_cache_page(mapping, index); 399 if (!page) 400 return ERR_PTR(-ENOMEM); 401 402 if (PageUptodate(page)) { 403 unlock_page(page); 404 return page; 405 } 406 407 err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, dn.data_blkaddr, 408 sync ? READ_SYNC : READA); 409 if (err) 410 return ERR_PTR(err); 411 412 if (sync) { 413 wait_on_page_locked(page); 414 if (unlikely(!PageUptodate(page))) { 415 f2fs_put_page(page, 0); 416 return ERR_PTR(-EIO); 417 } 418 } 419 return page; 420 } 421 422 /* 423 * If it tries to access a hole, return an error. 424 * Because, the callers, functions in dir.c and GC, should be able to know 425 * whether this page exists or not. 426 */ 427 struct page *get_lock_data_page(struct inode *inode, pgoff_t index) 428 { 429 struct address_space *mapping = inode->i_mapping; 430 struct dnode_of_data dn; 431 struct page *page; 432 int err; 433 434 repeat: 435 page = grab_cache_page(mapping, index); 436 if (!page) 437 return ERR_PTR(-ENOMEM); 438 439 set_new_dnode(&dn, inode, NULL, NULL, 0); 440 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 441 if (err) { 442 f2fs_put_page(page, 1); 443 return ERR_PTR(err); 444 } 445 f2fs_put_dnode(&dn); 446 447 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 448 f2fs_put_page(page, 1); 449 return ERR_PTR(-ENOENT); 450 } 451 452 if (PageUptodate(page)) 453 return page; 454 455 /* 456 * A new dentry page is allocated but not able to be written, since its 457 * new inode page couldn't be allocated due to -ENOSPC. 458 * In such the case, its blkaddr can be remained as NEW_ADDR. 459 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata. 460 */ 461 if (dn.data_blkaddr == NEW_ADDR) { 462 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 463 SetPageUptodate(page); 464 return page; 465 } 466 467 err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, 468 dn.data_blkaddr, READ_SYNC); 469 if (err) 470 return ERR_PTR(err); 471 472 lock_page(page); 473 if (unlikely(!PageUptodate(page))) { 474 f2fs_put_page(page, 1); 475 return ERR_PTR(-EIO); 476 } 477 if (unlikely(page->mapping != mapping)) { 478 f2fs_put_page(page, 1); 479 goto repeat; 480 } 481 return page; 482 } 483 484 /* 485 * Caller ensures that this data page is never allocated. 486 * A new zero-filled data page is allocated in the page cache. 487 * 488 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 489 * f2fs_unlock_op(). 490 * Note that, ipage is set only by make_empty_dir. 491 */ 492 struct page *get_new_data_page(struct inode *inode, 493 struct page *ipage, pgoff_t index, bool new_i_size) 494 { 495 struct address_space *mapping = inode->i_mapping; 496 struct page *page; 497 struct dnode_of_data dn; 498 int err; 499 500 set_new_dnode(&dn, inode, ipage, NULL, 0); 501 err = f2fs_reserve_block(&dn, index); 502 if (err) 503 return ERR_PTR(err); 504 repeat: 505 page = grab_cache_page(mapping, index); 506 if (!page) { 507 err = -ENOMEM; 508 goto put_err; 509 } 510 511 if (PageUptodate(page)) 512 return page; 513 514 if (dn.data_blkaddr == NEW_ADDR) { 515 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 516 SetPageUptodate(page); 517 } else { 518 err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, 519 dn.data_blkaddr, READ_SYNC); 520 if (err) 521 goto put_err; 522 523 lock_page(page); 524 if (unlikely(!PageUptodate(page))) { 525 f2fs_put_page(page, 1); 526 err = -EIO; 527 goto put_err; 528 } 529 if (unlikely(page->mapping != mapping)) { 530 f2fs_put_page(page, 1); 531 goto repeat; 532 } 533 } 534 535 if (new_i_size && 536 i_size_read(inode) < ((index + 1) << PAGE_CACHE_SHIFT)) { 537 i_size_write(inode, ((index + 1) << PAGE_CACHE_SHIFT)); 538 /* Only the directory inode sets new_i_size */ 539 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR); 540 } 541 return page; 542 543 put_err: 544 f2fs_put_dnode(&dn); 545 return ERR_PTR(err); 546 } 547 548 static int __allocate_data_block(struct dnode_of_data *dn) 549 { 550 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 551 struct f2fs_inode_info *fi = F2FS_I(dn->inode); 552 struct f2fs_summary sum; 553 block_t new_blkaddr; 554 struct node_info ni; 555 pgoff_t fofs; 556 int type; 557 558 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 559 return -EPERM; 560 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 561 return -ENOSPC; 562 563 __set_data_blkaddr(dn, NEW_ADDR); 564 dn->data_blkaddr = NEW_ADDR; 565 566 get_node_info(sbi, dn->nid, &ni); 567 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 568 569 type = CURSEG_WARM_DATA; 570 571 allocate_data_block(sbi, NULL, NULL_ADDR, &new_blkaddr, &sum, type); 572 573 /* direct IO doesn't use extent cache to maximize the performance */ 574 set_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT); 575 update_extent_cache(new_blkaddr, dn); 576 clear_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT); 577 578 /* update i_size */ 579 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) + 580 dn->ofs_in_node; 581 if (i_size_read(dn->inode) < ((fofs + 1) << PAGE_CACHE_SHIFT)) 582 i_size_write(dn->inode, ((fofs + 1) << PAGE_CACHE_SHIFT)); 583 584 dn->data_blkaddr = new_blkaddr; 585 return 0; 586 } 587 588 /* 589 * get_data_block() now supported readahead/bmap/rw direct_IO with mapped bh. 590 * If original data blocks are allocated, then give them to blockdev. 591 * Otherwise, 592 * a. preallocate requested block addresses 593 * b. do not use extent cache for better performance 594 * c. give the block addresses to blockdev 595 */ 596 static int __get_data_block(struct inode *inode, sector_t iblock, 597 struct buffer_head *bh_result, int create, bool fiemap) 598 { 599 unsigned int blkbits = inode->i_sb->s_blocksize_bits; 600 unsigned maxblocks = bh_result->b_size >> blkbits; 601 struct dnode_of_data dn; 602 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA; 603 pgoff_t pgofs, end_offset; 604 int err = 0, ofs = 1; 605 bool allocated = false; 606 607 /* Get the page offset from the block offset(iblock) */ 608 pgofs = (pgoff_t)(iblock >> (PAGE_CACHE_SHIFT - blkbits)); 609 610 if (check_extent_cache(inode, pgofs, bh_result)) 611 goto out; 612 613 if (create) { 614 f2fs_balance_fs(F2FS_I_SB(inode)); 615 f2fs_lock_op(F2FS_I_SB(inode)); 616 } 617 618 /* When reading holes, we need its node page */ 619 set_new_dnode(&dn, inode, NULL, NULL, 0); 620 err = get_dnode_of_data(&dn, pgofs, mode); 621 if (err) { 622 if (err == -ENOENT) 623 err = 0; 624 goto unlock_out; 625 } 626 if (dn.data_blkaddr == NEW_ADDR && !fiemap) 627 goto put_out; 628 629 if (dn.data_blkaddr != NULL_ADDR) { 630 map_bh(bh_result, inode->i_sb, dn.data_blkaddr); 631 } else if (create) { 632 err = __allocate_data_block(&dn); 633 if (err) 634 goto put_out; 635 allocated = true; 636 map_bh(bh_result, inode->i_sb, dn.data_blkaddr); 637 } else { 638 goto put_out; 639 } 640 641 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 642 bh_result->b_size = (((size_t)1) << blkbits); 643 dn.ofs_in_node++; 644 pgofs++; 645 646 get_next: 647 if (dn.ofs_in_node >= end_offset) { 648 if (allocated) 649 sync_inode_page(&dn); 650 allocated = false; 651 f2fs_put_dnode(&dn); 652 653 set_new_dnode(&dn, inode, NULL, NULL, 0); 654 err = get_dnode_of_data(&dn, pgofs, mode); 655 if (err) { 656 if (err == -ENOENT) 657 err = 0; 658 goto unlock_out; 659 } 660 if (dn.data_blkaddr == NEW_ADDR && !fiemap) 661 goto put_out; 662 663 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 664 } 665 666 if (maxblocks > (bh_result->b_size >> blkbits)) { 667 block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); 668 if (blkaddr == NULL_ADDR && create) { 669 err = __allocate_data_block(&dn); 670 if (err) 671 goto sync_out; 672 allocated = true; 673 blkaddr = dn.data_blkaddr; 674 } 675 /* Give more consecutive addresses for the readahead */ 676 if (blkaddr == (bh_result->b_blocknr + ofs)) { 677 ofs++; 678 dn.ofs_in_node++; 679 pgofs++; 680 bh_result->b_size += (((size_t)1) << blkbits); 681 goto get_next; 682 } 683 } 684 sync_out: 685 if (allocated) 686 sync_inode_page(&dn); 687 put_out: 688 f2fs_put_dnode(&dn); 689 unlock_out: 690 if (create) 691 f2fs_unlock_op(F2FS_I_SB(inode)); 692 out: 693 trace_f2fs_get_data_block(inode, iblock, bh_result, err); 694 return err; 695 } 696 697 static int get_data_block(struct inode *inode, sector_t iblock, 698 struct buffer_head *bh_result, int create) 699 { 700 return __get_data_block(inode, iblock, bh_result, create, false); 701 } 702 703 static int get_data_block_fiemap(struct inode *inode, sector_t iblock, 704 struct buffer_head *bh_result, int create) 705 { 706 return __get_data_block(inode, iblock, bh_result, create, true); 707 } 708 709 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 710 u64 start, u64 len) 711 { 712 return generic_block_fiemap(inode, fieinfo, 713 start, len, get_data_block_fiemap); 714 } 715 716 static int f2fs_read_data_page(struct file *file, struct page *page) 717 { 718 struct inode *inode = page->mapping->host; 719 int ret = -EAGAIN; 720 721 trace_f2fs_readpage(page, DATA); 722 723 /* If the file has inline data, try to read it directly */ 724 if (f2fs_has_inline_data(inode)) 725 ret = f2fs_read_inline_data(inode, page); 726 if (ret == -EAGAIN) 727 ret = mpage_readpage(page, get_data_block); 728 729 return ret; 730 } 731 732 static int f2fs_read_data_pages(struct file *file, 733 struct address_space *mapping, 734 struct list_head *pages, unsigned nr_pages) 735 { 736 struct inode *inode = file->f_mapping->host; 737 738 /* If the file has inline data, skip readpages */ 739 if (f2fs_has_inline_data(inode)) 740 return 0; 741 742 return mpage_readpages(mapping, pages, nr_pages, get_data_block); 743 } 744 745 int do_write_data_page(struct page *page, struct f2fs_io_info *fio) 746 { 747 struct inode *inode = page->mapping->host; 748 block_t old_blkaddr, new_blkaddr; 749 struct dnode_of_data dn; 750 int err = 0; 751 752 set_new_dnode(&dn, inode, NULL, NULL, 0); 753 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 754 if (err) 755 return err; 756 757 old_blkaddr = dn.data_blkaddr; 758 759 /* This page is already truncated */ 760 if (old_blkaddr == NULL_ADDR) 761 goto out_writepage; 762 763 set_page_writeback(page); 764 765 /* 766 * If current allocation needs SSR, 767 * it had better in-place writes for updated data. 768 */ 769 if (unlikely(old_blkaddr != NEW_ADDR && 770 !is_cold_data(page) && 771 need_inplace_update(inode))) { 772 rewrite_data_page(page, old_blkaddr, fio); 773 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE); 774 } else { 775 write_data_page(page, &dn, &new_blkaddr, fio); 776 update_extent_cache(new_blkaddr, &dn); 777 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); 778 } 779 out_writepage: 780 f2fs_put_dnode(&dn); 781 return err; 782 } 783 784 static int f2fs_write_data_page(struct page *page, 785 struct writeback_control *wbc) 786 { 787 struct inode *inode = page->mapping->host; 788 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 789 loff_t i_size = i_size_read(inode); 790 const pgoff_t end_index = ((unsigned long long) i_size) 791 >> PAGE_CACHE_SHIFT; 792 unsigned offset = 0; 793 bool need_balance_fs = false; 794 int err = 0; 795 struct f2fs_io_info fio = { 796 .type = DATA, 797 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE, 798 }; 799 800 trace_f2fs_writepage(page, DATA); 801 802 if (page->index < end_index) 803 goto write; 804 805 /* 806 * If the offset is out-of-range of file size, 807 * this page does not have to be written to disk. 808 */ 809 offset = i_size & (PAGE_CACHE_SIZE - 1); 810 if ((page->index >= end_index + 1) || !offset) 811 goto out; 812 813 zero_user_segment(page, offset, PAGE_CACHE_SIZE); 814 write: 815 if (unlikely(sbi->por_doing)) 816 goto redirty_out; 817 818 /* Dentry blocks are controlled by checkpoint */ 819 if (S_ISDIR(inode->i_mode)) { 820 if (unlikely(f2fs_cp_error(sbi))) 821 goto redirty_out; 822 err = do_write_data_page(page, &fio); 823 goto done; 824 } 825 826 /* we should bypass data pages to proceed the kworkder jobs */ 827 if (unlikely(f2fs_cp_error(sbi))) { 828 SetPageError(page); 829 unlock_page(page); 830 goto out; 831 } 832 833 if (!wbc->for_reclaim) 834 need_balance_fs = true; 835 else if (has_not_enough_free_secs(sbi, 0)) 836 goto redirty_out; 837 838 err = -EAGAIN; 839 f2fs_lock_op(sbi); 840 if (f2fs_has_inline_data(inode)) 841 err = f2fs_write_inline_data(inode, page); 842 if (err == -EAGAIN) 843 err = do_write_data_page(page, &fio); 844 f2fs_unlock_op(sbi); 845 done: 846 if (err && err != -ENOENT) 847 goto redirty_out; 848 849 clear_cold_data(page); 850 out: 851 inode_dec_dirty_pages(inode); 852 unlock_page(page); 853 if (need_balance_fs) 854 f2fs_balance_fs(sbi); 855 if (wbc->for_reclaim) 856 f2fs_submit_merged_bio(sbi, DATA, WRITE); 857 return 0; 858 859 redirty_out: 860 redirty_page_for_writepage(wbc, page); 861 return AOP_WRITEPAGE_ACTIVATE; 862 } 863 864 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc, 865 void *data) 866 { 867 struct address_space *mapping = data; 868 int ret = mapping->a_ops->writepage(page, wbc); 869 mapping_set_error(mapping, ret); 870 return ret; 871 } 872 873 static int f2fs_write_data_pages(struct address_space *mapping, 874 struct writeback_control *wbc) 875 { 876 struct inode *inode = mapping->host; 877 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 878 bool locked = false; 879 int ret; 880 long diff; 881 882 trace_f2fs_writepages(mapping->host, wbc, DATA); 883 884 /* deal with chardevs and other special file */ 885 if (!mapping->a_ops->writepage) 886 return 0; 887 888 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE && 889 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 890 available_free_memory(sbi, DIRTY_DENTS)) 891 goto skip_write; 892 893 diff = nr_pages_to_write(sbi, DATA, wbc); 894 895 if (!S_ISDIR(inode->i_mode)) { 896 mutex_lock(&sbi->writepages); 897 locked = true; 898 } 899 ret = write_cache_pages(mapping, wbc, __f2fs_writepage, mapping); 900 if (locked) 901 mutex_unlock(&sbi->writepages); 902 903 f2fs_submit_merged_bio(sbi, DATA, WRITE); 904 905 remove_dirty_dir_inode(inode); 906 907 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); 908 return ret; 909 910 skip_write: 911 wbc->pages_skipped += get_dirty_pages(inode); 912 return 0; 913 } 914 915 static void f2fs_write_failed(struct address_space *mapping, loff_t to) 916 { 917 struct inode *inode = mapping->host; 918 919 if (to > inode->i_size) { 920 truncate_pagecache(inode, inode->i_size); 921 truncate_blocks(inode, inode->i_size, true); 922 } 923 } 924 925 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 926 loff_t pos, unsigned len, unsigned flags, 927 struct page **pagep, void **fsdata) 928 { 929 struct inode *inode = mapping->host; 930 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 931 struct page *page, *ipage; 932 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT; 933 struct dnode_of_data dn; 934 int err = 0; 935 936 trace_f2fs_write_begin(inode, pos, len, flags); 937 938 f2fs_balance_fs(sbi); 939 940 /* 941 * We should check this at this moment to avoid deadlock on inode page 942 * and #0 page. The locking rule for inline_data conversion should be: 943 * lock_page(page #0) -> lock_page(inode_page) 944 */ 945 if (index != 0) { 946 err = f2fs_convert_inline_inode(inode); 947 if (err) 948 goto fail; 949 } 950 repeat: 951 page = grab_cache_page_write_begin(mapping, index, flags); 952 if (!page) { 953 err = -ENOMEM; 954 goto fail; 955 } 956 957 *pagep = page; 958 959 f2fs_lock_op(sbi); 960 961 /* check inline_data */ 962 ipage = get_node_page(sbi, inode->i_ino); 963 if (IS_ERR(ipage)) { 964 err = PTR_ERR(ipage); 965 goto unlock_fail; 966 } 967 968 set_new_dnode(&dn, inode, ipage, ipage, 0); 969 970 if (f2fs_has_inline_data(inode)) { 971 if (pos + len <= MAX_INLINE_DATA) { 972 read_inline_data(page, ipage); 973 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); 974 sync_inode_page(&dn); 975 goto put_next; 976 } 977 err = f2fs_convert_inline_page(&dn, page); 978 if (err) 979 goto put_fail; 980 } 981 err = f2fs_reserve_block(&dn, index); 982 if (err) 983 goto put_fail; 984 put_next: 985 f2fs_put_dnode(&dn); 986 f2fs_unlock_op(sbi); 987 988 if ((len == PAGE_CACHE_SIZE) || PageUptodate(page)) 989 return 0; 990 991 f2fs_wait_on_page_writeback(page, DATA); 992 993 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) { 994 unsigned start = pos & (PAGE_CACHE_SIZE - 1); 995 unsigned end = start + len; 996 997 /* Reading beyond i_size is simple: memset to zero */ 998 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE); 999 goto out; 1000 } 1001 1002 if (dn.data_blkaddr == NEW_ADDR) { 1003 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 1004 } else { 1005 err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr, 1006 READ_SYNC); 1007 if (err) 1008 goto fail; 1009 1010 lock_page(page); 1011 if (unlikely(!PageUptodate(page))) { 1012 f2fs_put_page(page, 1); 1013 err = -EIO; 1014 goto fail; 1015 } 1016 if (unlikely(page->mapping != mapping)) { 1017 f2fs_put_page(page, 1); 1018 goto repeat; 1019 } 1020 } 1021 out: 1022 SetPageUptodate(page); 1023 clear_cold_data(page); 1024 return 0; 1025 1026 put_fail: 1027 f2fs_put_dnode(&dn); 1028 unlock_fail: 1029 f2fs_unlock_op(sbi); 1030 f2fs_put_page(page, 1); 1031 fail: 1032 f2fs_write_failed(mapping, pos + len); 1033 return err; 1034 } 1035 1036 static int f2fs_write_end(struct file *file, 1037 struct address_space *mapping, 1038 loff_t pos, unsigned len, unsigned copied, 1039 struct page *page, void *fsdata) 1040 { 1041 struct inode *inode = page->mapping->host; 1042 1043 trace_f2fs_write_end(inode, pos, len, copied); 1044 1045 set_page_dirty(page); 1046 1047 if (pos + copied > i_size_read(inode)) { 1048 i_size_write(inode, pos + copied); 1049 mark_inode_dirty(inode); 1050 update_inode_page(inode); 1051 } 1052 1053 f2fs_put_page(page, 1); 1054 return copied; 1055 } 1056 1057 static int check_direct_IO(struct inode *inode, int rw, 1058 struct iov_iter *iter, loff_t offset) 1059 { 1060 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1; 1061 1062 if (rw == READ) 1063 return 0; 1064 1065 if (offset & blocksize_mask) 1066 return -EINVAL; 1067 1068 if (iov_iter_alignment(iter) & blocksize_mask) 1069 return -EINVAL; 1070 1071 return 0; 1072 } 1073 1074 static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb, 1075 struct iov_iter *iter, loff_t offset) 1076 { 1077 struct file *file = iocb->ki_filp; 1078 struct address_space *mapping = file->f_mapping; 1079 struct inode *inode = mapping->host; 1080 size_t count = iov_iter_count(iter); 1081 int err; 1082 1083 /* we don't need to use inline_data strictly */ 1084 if (f2fs_has_inline_data(inode)) { 1085 err = f2fs_convert_inline_inode(inode); 1086 if (err) 1087 return err; 1088 } 1089 1090 if (check_direct_IO(inode, rw, iter, offset)) 1091 return 0; 1092 1093 trace_f2fs_direct_IO_enter(inode, offset, count, rw); 1094 1095 err = blockdev_direct_IO(rw, iocb, inode, iter, offset, get_data_block); 1096 if (err < 0 && (rw & WRITE)) 1097 f2fs_write_failed(mapping, offset + count); 1098 1099 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err); 1100 1101 return err; 1102 } 1103 1104 static void f2fs_invalidate_data_page(struct page *page, unsigned int offset, 1105 unsigned int length) 1106 { 1107 struct inode *inode = page->mapping->host; 1108 1109 if (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE) 1110 return; 1111 1112 if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode)) 1113 invalidate_inmem_page(inode, page); 1114 1115 if (PageDirty(page)) 1116 inode_dec_dirty_pages(inode); 1117 ClearPagePrivate(page); 1118 } 1119 1120 static int f2fs_release_data_page(struct page *page, gfp_t wait) 1121 { 1122 ClearPagePrivate(page); 1123 return 1; 1124 } 1125 1126 static int f2fs_set_data_page_dirty(struct page *page) 1127 { 1128 struct address_space *mapping = page->mapping; 1129 struct inode *inode = mapping->host; 1130 1131 trace_f2fs_set_page_dirty(page, DATA); 1132 1133 SetPageUptodate(page); 1134 1135 if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode)) { 1136 register_inmem_page(inode, page); 1137 return 1; 1138 } 1139 1140 mark_inode_dirty(inode); 1141 1142 if (!PageDirty(page)) { 1143 __set_page_dirty_nobuffers(page); 1144 update_dirty_page(inode, page); 1145 return 1; 1146 } 1147 return 0; 1148 } 1149 1150 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 1151 { 1152 struct inode *inode = mapping->host; 1153 1154 /* we don't need to use inline_data strictly */ 1155 if (f2fs_has_inline_data(inode)) { 1156 int err = f2fs_convert_inline_inode(inode); 1157 if (err) 1158 return err; 1159 } 1160 return generic_block_bmap(mapping, block, get_data_block); 1161 } 1162 1163 const struct address_space_operations f2fs_dblock_aops = { 1164 .readpage = f2fs_read_data_page, 1165 .readpages = f2fs_read_data_pages, 1166 .writepage = f2fs_write_data_page, 1167 .writepages = f2fs_write_data_pages, 1168 .write_begin = f2fs_write_begin, 1169 .write_end = f2fs_write_end, 1170 .set_page_dirty = f2fs_set_data_page_dirty, 1171 .invalidatepage = f2fs_invalidate_data_page, 1172 .releasepage = f2fs_release_data_page, 1173 .direct_IO = f2fs_direct_IO, 1174 .bmap = f2fs_bmap, 1175 }; 1176