1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/data.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/buffer_head.h> 11 #include <linux/mpage.h> 12 #include <linux/writeback.h> 13 #include <linux/backing-dev.h> 14 #include <linux/pagevec.h> 15 #include <linux/blkdev.h> 16 #include <linux/bio.h> 17 #include <linux/prefetch.h> 18 #include <linux/uio.h> 19 #include <linux/cleancache.h> 20 #include <linux/sched/signal.h> 21 22 #include "f2fs.h" 23 #include "node.h" 24 #include "segment.h" 25 #include "trace.h" 26 #include <trace/events/f2fs.h> 27 28 #define NUM_PREALLOC_POST_READ_CTXS 128 29 30 static struct kmem_cache *bio_post_read_ctx_cache; 31 static mempool_t *bio_post_read_ctx_pool; 32 33 static bool __is_cp_guaranteed(struct page *page) 34 { 35 struct address_space *mapping = page->mapping; 36 struct inode *inode; 37 struct f2fs_sb_info *sbi; 38 39 if (!mapping) 40 return false; 41 42 inode = mapping->host; 43 sbi = F2FS_I_SB(inode); 44 45 if (inode->i_ino == F2FS_META_INO(sbi) || 46 inode->i_ino == F2FS_NODE_INO(sbi) || 47 S_ISDIR(inode->i_mode) || 48 (S_ISREG(inode->i_mode) && 49 is_inode_flag_set(inode, FI_ATOMIC_FILE)) || 50 is_cold_data(page)) 51 return true; 52 return false; 53 } 54 55 /* postprocessing steps for read bios */ 56 enum bio_post_read_step { 57 STEP_INITIAL = 0, 58 STEP_DECRYPT, 59 }; 60 61 struct bio_post_read_ctx { 62 struct bio *bio; 63 struct work_struct work; 64 unsigned int cur_step; 65 unsigned int enabled_steps; 66 }; 67 68 static void __read_end_io(struct bio *bio) 69 { 70 struct page *page; 71 struct bio_vec *bv; 72 int i; 73 74 bio_for_each_segment_all(bv, bio, i) { 75 page = bv->bv_page; 76 77 /* PG_error was set if any post_read step failed */ 78 if (bio->bi_status || PageError(page)) { 79 ClearPageUptodate(page); 80 /* will re-read again later */ 81 ClearPageError(page); 82 } else { 83 SetPageUptodate(page); 84 } 85 unlock_page(page); 86 } 87 if (bio->bi_private) 88 mempool_free(bio->bi_private, bio_post_read_ctx_pool); 89 bio_put(bio); 90 } 91 92 static void bio_post_read_processing(struct bio_post_read_ctx *ctx); 93 94 static void decrypt_work(struct work_struct *work) 95 { 96 struct bio_post_read_ctx *ctx = 97 container_of(work, struct bio_post_read_ctx, work); 98 99 fscrypt_decrypt_bio(ctx->bio); 100 101 bio_post_read_processing(ctx); 102 } 103 104 static void bio_post_read_processing(struct bio_post_read_ctx *ctx) 105 { 106 switch (++ctx->cur_step) { 107 case STEP_DECRYPT: 108 if (ctx->enabled_steps & (1 << STEP_DECRYPT)) { 109 INIT_WORK(&ctx->work, decrypt_work); 110 fscrypt_enqueue_decrypt_work(&ctx->work); 111 return; 112 } 113 ctx->cur_step++; 114 /* fall-through */ 115 default: 116 __read_end_io(ctx->bio); 117 } 118 } 119 120 static bool f2fs_bio_post_read_required(struct bio *bio) 121 { 122 return bio->bi_private && !bio->bi_status; 123 } 124 125 static void f2fs_read_end_io(struct bio *bio) 126 { 127 if (time_to_inject(F2FS_P_SB(bio_first_page_all(bio)), 128 FAULT_READ_IO)) { 129 f2fs_show_injection_info(FAULT_READ_IO); 130 bio->bi_status = BLK_STS_IOERR; 131 } 132 133 if (f2fs_bio_post_read_required(bio)) { 134 struct bio_post_read_ctx *ctx = bio->bi_private; 135 136 ctx->cur_step = STEP_INITIAL; 137 bio_post_read_processing(ctx); 138 return; 139 } 140 141 __read_end_io(bio); 142 } 143 144 static void f2fs_write_end_io(struct bio *bio) 145 { 146 struct f2fs_sb_info *sbi = bio->bi_private; 147 struct bio_vec *bvec; 148 int i; 149 150 if (time_to_inject(sbi, FAULT_WRITE_IO)) { 151 f2fs_show_injection_info(FAULT_WRITE_IO); 152 bio->bi_status = BLK_STS_IOERR; 153 } 154 155 bio_for_each_segment_all(bvec, bio, i) { 156 struct page *page = bvec->bv_page; 157 enum count_type type = WB_DATA_TYPE(page); 158 159 if (IS_DUMMY_WRITTEN_PAGE(page)) { 160 set_page_private(page, (unsigned long)NULL); 161 ClearPagePrivate(page); 162 unlock_page(page); 163 mempool_free(page, sbi->write_io_dummy); 164 165 if (unlikely(bio->bi_status)) 166 f2fs_stop_checkpoint(sbi, true); 167 continue; 168 } 169 170 fscrypt_pullback_bio_page(&page, true); 171 172 if (unlikely(bio->bi_status)) { 173 mapping_set_error(page->mapping, -EIO); 174 if (type == F2FS_WB_CP_DATA) 175 f2fs_stop_checkpoint(sbi, true); 176 } 177 178 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && 179 page->index != nid_of_node(page)); 180 181 dec_page_count(sbi, type); 182 if (f2fs_in_warm_node_list(sbi, page)) 183 f2fs_del_fsync_node_entry(sbi, page); 184 clear_cold_data(page); 185 end_page_writeback(page); 186 } 187 if (!get_pages(sbi, F2FS_WB_CP_DATA) && 188 wq_has_sleeper(&sbi->cp_wait)) 189 wake_up(&sbi->cp_wait); 190 191 bio_put(bio); 192 } 193 194 /* 195 * Return true, if pre_bio's bdev is same as its target device. 196 */ 197 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, 198 block_t blk_addr, struct bio *bio) 199 { 200 struct block_device *bdev = sbi->sb->s_bdev; 201 int i; 202 203 for (i = 0; i < sbi->s_ndevs; i++) { 204 if (FDEV(i).start_blk <= blk_addr && 205 FDEV(i).end_blk >= blk_addr) { 206 blk_addr -= FDEV(i).start_blk; 207 bdev = FDEV(i).bdev; 208 break; 209 } 210 } 211 if (bio) { 212 bio_set_dev(bio, bdev); 213 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); 214 } 215 return bdev; 216 } 217 218 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) 219 { 220 int i; 221 222 for (i = 0; i < sbi->s_ndevs; i++) 223 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) 224 return i; 225 return 0; 226 } 227 228 static bool __same_bdev(struct f2fs_sb_info *sbi, 229 block_t blk_addr, struct bio *bio) 230 { 231 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL); 232 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno; 233 } 234 235 /* 236 * Low-level block read/write IO operations. 237 */ 238 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, 239 struct writeback_control *wbc, 240 int npages, bool is_read, 241 enum page_type type, enum temp_type temp) 242 { 243 struct bio *bio; 244 245 bio = f2fs_bio_alloc(sbi, npages, true); 246 247 f2fs_target_device(sbi, blk_addr, bio); 248 if (is_read) { 249 bio->bi_end_io = f2fs_read_end_io; 250 bio->bi_private = NULL; 251 } else { 252 bio->bi_end_io = f2fs_write_end_io; 253 bio->bi_private = sbi; 254 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, type, temp); 255 } 256 if (wbc) 257 wbc_init_bio(wbc, bio); 258 259 return bio; 260 } 261 262 static inline void __submit_bio(struct f2fs_sb_info *sbi, 263 struct bio *bio, enum page_type type) 264 { 265 if (!is_read_io(bio_op(bio))) { 266 unsigned int start; 267 268 if (type != DATA && type != NODE) 269 goto submit_io; 270 271 if (test_opt(sbi, LFS) && current->plug) 272 blk_finish_plug(current->plug); 273 274 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS; 275 start %= F2FS_IO_SIZE(sbi); 276 277 if (start == 0) 278 goto submit_io; 279 280 /* fill dummy pages */ 281 for (; start < F2FS_IO_SIZE(sbi); start++) { 282 struct page *page = 283 mempool_alloc(sbi->write_io_dummy, 284 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL); 285 f2fs_bug_on(sbi, !page); 286 287 SetPagePrivate(page); 288 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE); 289 lock_page(page); 290 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 291 f2fs_bug_on(sbi, 1); 292 } 293 /* 294 * In the NODE case, we lose next block address chain. So, we 295 * need to do checkpoint in f2fs_sync_file. 296 */ 297 if (type == NODE) 298 set_sbi_flag(sbi, SBI_NEED_CP); 299 } 300 submit_io: 301 if (is_read_io(bio_op(bio))) 302 trace_f2fs_submit_read_bio(sbi->sb, type, bio); 303 else 304 trace_f2fs_submit_write_bio(sbi->sb, type, bio); 305 submit_bio(bio); 306 } 307 308 static void __submit_merged_bio(struct f2fs_bio_info *io) 309 { 310 struct f2fs_io_info *fio = &io->fio; 311 312 if (!io->bio) 313 return; 314 315 bio_set_op_attrs(io->bio, fio->op, fio->op_flags); 316 317 if (is_read_io(fio->op)) 318 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); 319 else 320 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); 321 322 __submit_bio(io->sbi, io->bio, fio->type); 323 io->bio = NULL; 324 } 325 326 static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode, 327 struct page *page, nid_t ino) 328 { 329 struct bio_vec *bvec; 330 struct page *target; 331 int i; 332 333 if (!io->bio) 334 return false; 335 336 if (!inode && !page && !ino) 337 return true; 338 339 bio_for_each_segment_all(bvec, io->bio, i) { 340 341 if (bvec->bv_page->mapping) 342 target = bvec->bv_page; 343 else 344 target = fscrypt_control_page(bvec->bv_page); 345 346 if (inode && inode == target->mapping->host) 347 return true; 348 if (page && page == target) 349 return true; 350 if (ino && ino == ino_of_node(target)) 351 return true; 352 } 353 354 return false; 355 } 356 357 static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode, 358 struct page *page, nid_t ino, 359 enum page_type type) 360 { 361 enum page_type btype = PAGE_TYPE_OF_BIO(type); 362 enum temp_type temp; 363 struct f2fs_bio_info *io; 364 bool ret = false; 365 366 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 367 io = sbi->write_io[btype] + temp; 368 369 down_read(&io->io_rwsem); 370 ret = __has_merged_page(io, inode, page, ino); 371 up_read(&io->io_rwsem); 372 373 /* TODO: use HOT temp only for meta pages now. */ 374 if (ret || btype == META) 375 break; 376 } 377 return ret; 378 } 379 380 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, 381 enum page_type type, enum temp_type temp) 382 { 383 enum page_type btype = PAGE_TYPE_OF_BIO(type); 384 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 385 386 down_write(&io->io_rwsem); 387 388 /* change META to META_FLUSH in the checkpoint procedure */ 389 if (type >= META_FLUSH) { 390 io->fio.type = META_FLUSH; 391 io->fio.op = REQ_OP_WRITE; 392 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC; 393 if (!test_opt(sbi, NOBARRIER)) 394 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA; 395 } 396 __submit_merged_bio(io); 397 up_write(&io->io_rwsem); 398 } 399 400 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, 401 struct inode *inode, struct page *page, 402 nid_t ino, enum page_type type, bool force) 403 { 404 enum temp_type temp; 405 406 if (!force && !has_merged_page(sbi, inode, page, ino, type)) 407 return; 408 409 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 410 411 __f2fs_submit_merged_write(sbi, type, temp); 412 413 /* TODO: use HOT temp only for meta pages now. */ 414 if (type >= META) 415 break; 416 } 417 } 418 419 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) 420 { 421 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true); 422 } 423 424 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, 425 struct inode *inode, struct page *page, 426 nid_t ino, enum page_type type) 427 { 428 __submit_merged_write_cond(sbi, inode, page, ino, type, false); 429 } 430 431 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) 432 { 433 f2fs_submit_merged_write(sbi, DATA); 434 f2fs_submit_merged_write(sbi, NODE); 435 f2fs_submit_merged_write(sbi, META); 436 } 437 438 /* 439 * Fill the locked page with data located in the block address. 440 * A caller needs to unlock the page on failure. 441 */ 442 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 443 { 444 struct bio *bio; 445 struct page *page = fio->encrypted_page ? 446 fio->encrypted_page : fio->page; 447 448 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 449 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) 450 return -EFAULT; 451 452 trace_f2fs_submit_page_bio(page, fio); 453 f2fs_trace_ios(fio, 0); 454 455 /* Allocate a new bio */ 456 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc, 457 1, is_read_io(fio->op), fio->type, fio->temp); 458 459 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 460 bio_put(bio); 461 return -EFAULT; 462 } 463 bio_set_op_attrs(bio, fio->op, fio->op_flags); 464 465 __submit_bio(fio->sbi, bio, fio->type); 466 467 if (!is_read_io(fio->op)) 468 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page)); 469 return 0; 470 } 471 472 void f2fs_submit_page_write(struct f2fs_io_info *fio) 473 { 474 struct f2fs_sb_info *sbi = fio->sbi; 475 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 476 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; 477 struct page *bio_page; 478 479 f2fs_bug_on(sbi, is_read_io(fio->op)); 480 481 down_write(&io->io_rwsem); 482 next: 483 if (fio->in_list) { 484 spin_lock(&io->io_lock); 485 if (list_empty(&io->io_list)) { 486 spin_unlock(&io->io_lock); 487 goto out; 488 } 489 fio = list_first_entry(&io->io_list, 490 struct f2fs_io_info, list); 491 list_del(&fio->list); 492 spin_unlock(&io->io_lock); 493 } 494 495 if (__is_valid_data_blkaddr(fio->old_blkaddr)) 496 verify_block_addr(fio, fio->old_blkaddr); 497 verify_block_addr(fio, fio->new_blkaddr); 498 499 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; 500 501 /* set submitted = true as a return value */ 502 fio->submitted = true; 503 504 inc_page_count(sbi, WB_DATA_TYPE(bio_page)); 505 506 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 || 507 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) || 508 !__same_bdev(sbi, fio->new_blkaddr, io->bio))) 509 __submit_merged_bio(io); 510 alloc_new: 511 if (io->bio == NULL) { 512 if ((fio->type == DATA || fio->type == NODE) && 513 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { 514 dec_page_count(sbi, WB_DATA_TYPE(bio_page)); 515 fio->retry = true; 516 goto skip; 517 } 518 io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc, 519 BIO_MAX_PAGES, false, 520 fio->type, fio->temp); 521 io->fio = *fio; 522 } 523 524 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { 525 __submit_merged_bio(io); 526 goto alloc_new; 527 } 528 529 if (fio->io_wbc) 530 wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE); 531 532 io->last_block_in_bio = fio->new_blkaddr; 533 f2fs_trace_ios(fio, 0); 534 535 trace_f2fs_submit_page_write(fio->page, fio); 536 skip: 537 if (fio->in_list) 538 goto next; 539 out: 540 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || 541 f2fs_is_checkpoint_ready(sbi)) 542 __submit_merged_bio(io); 543 up_write(&io->io_rwsem); 544 } 545 546 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, 547 unsigned nr_pages, unsigned op_flag) 548 { 549 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 550 struct bio *bio; 551 struct bio_post_read_ctx *ctx; 552 unsigned int post_read_steps = 0; 553 554 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) 555 return ERR_PTR(-EFAULT); 556 557 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false); 558 if (!bio) 559 return ERR_PTR(-ENOMEM); 560 f2fs_target_device(sbi, blkaddr, bio); 561 bio->bi_end_io = f2fs_read_end_io; 562 bio_set_op_attrs(bio, REQ_OP_READ, op_flag); 563 564 if (f2fs_encrypted_file(inode)) 565 post_read_steps |= 1 << STEP_DECRYPT; 566 if (post_read_steps) { 567 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); 568 if (!ctx) { 569 bio_put(bio); 570 return ERR_PTR(-ENOMEM); 571 } 572 ctx->bio = bio; 573 ctx->enabled_steps = post_read_steps; 574 bio->bi_private = ctx; 575 } 576 577 return bio; 578 } 579 580 /* This can handle encryption stuffs */ 581 static int f2fs_submit_page_read(struct inode *inode, struct page *page, 582 block_t blkaddr) 583 { 584 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0); 585 586 if (IS_ERR(bio)) 587 return PTR_ERR(bio); 588 589 /* wait for GCed page writeback via META_MAPPING */ 590 f2fs_wait_on_block_writeback(inode, blkaddr); 591 592 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 593 bio_put(bio); 594 return -EFAULT; 595 } 596 ClearPageError(page); 597 __submit_bio(F2FS_I_SB(inode), bio, DATA); 598 return 0; 599 } 600 601 static void __set_data_blkaddr(struct dnode_of_data *dn) 602 { 603 struct f2fs_node *rn = F2FS_NODE(dn->node_page); 604 __le32 *addr_array; 605 int base = 0; 606 607 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode)) 608 base = get_extra_isize(dn->inode); 609 610 /* Get physical address of data block */ 611 addr_array = blkaddr_in_node(rn); 612 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 613 } 614 615 /* 616 * Lock ordering for the change of data block address: 617 * ->data_page 618 * ->node_page 619 * update block addresses in the node page 620 */ 621 void f2fs_set_data_blkaddr(struct dnode_of_data *dn) 622 { 623 f2fs_wait_on_page_writeback(dn->node_page, NODE, true); 624 __set_data_blkaddr(dn); 625 if (set_page_dirty(dn->node_page)) 626 dn->node_changed = true; 627 } 628 629 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 630 { 631 dn->data_blkaddr = blkaddr; 632 f2fs_set_data_blkaddr(dn); 633 f2fs_update_extent_cache(dn); 634 } 635 636 /* dn->ofs_in_node will be returned with up-to-date last block pointer */ 637 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) 638 { 639 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 640 int err; 641 642 if (!count) 643 return 0; 644 645 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 646 return -EPERM; 647 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) 648 return err; 649 650 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, 651 dn->ofs_in_node, count); 652 653 f2fs_wait_on_page_writeback(dn->node_page, NODE, true); 654 655 for (; count > 0; dn->ofs_in_node++) { 656 block_t blkaddr = datablock_addr(dn->inode, 657 dn->node_page, dn->ofs_in_node); 658 if (blkaddr == NULL_ADDR) { 659 dn->data_blkaddr = NEW_ADDR; 660 __set_data_blkaddr(dn); 661 count--; 662 } 663 } 664 665 if (set_page_dirty(dn->node_page)) 666 dn->node_changed = true; 667 return 0; 668 } 669 670 /* Should keep dn->ofs_in_node unchanged */ 671 int f2fs_reserve_new_block(struct dnode_of_data *dn) 672 { 673 unsigned int ofs_in_node = dn->ofs_in_node; 674 int ret; 675 676 ret = f2fs_reserve_new_blocks(dn, 1); 677 dn->ofs_in_node = ofs_in_node; 678 return ret; 679 } 680 681 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 682 { 683 bool need_put = dn->inode_page ? false : true; 684 int err; 685 686 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE); 687 if (err) 688 return err; 689 690 if (dn->data_blkaddr == NULL_ADDR) 691 err = f2fs_reserve_new_block(dn); 692 if (err || need_put) 693 f2fs_put_dnode(dn); 694 return err; 695 } 696 697 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) 698 { 699 struct extent_info ei = {0,0,0}; 700 struct inode *inode = dn->inode; 701 702 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 703 dn->data_blkaddr = ei.blk + index - ei.fofs; 704 return 0; 705 } 706 707 return f2fs_reserve_block(dn, index); 708 } 709 710 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, 711 int op_flags, bool for_write) 712 { 713 struct address_space *mapping = inode->i_mapping; 714 struct dnode_of_data dn; 715 struct page *page; 716 struct extent_info ei = {0,0,0}; 717 int err; 718 719 page = f2fs_grab_cache_page(mapping, index, for_write); 720 if (!page) 721 return ERR_PTR(-ENOMEM); 722 723 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 724 dn.data_blkaddr = ei.blk + index - ei.fofs; 725 goto got_it; 726 } 727 728 set_new_dnode(&dn, inode, NULL, NULL, 0); 729 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 730 if (err) 731 goto put_err; 732 f2fs_put_dnode(&dn); 733 734 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 735 err = -ENOENT; 736 goto put_err; 737 } 738 got_it: 739 if (PageUptodate(page)) { 740 unlock_page(page); 741 return page; 742 } 743 744 /* 745 * A new dentry page is allocated but not able to be written, since its 746 * new inode page couldn't be allocated due to -ENOSPC. 747 * In such the case, its blkaddr can be remained as NEW_ADDR. 748 * see, f2fs_add_link -> f2fs_get_new_data_page -> 749 * f2fs_init_inode_metadata. 750 */ 751 if (dn.data_blkaddr == NEW_ADDR) { 752 zero_user_segment(page, 0, PAGE_SIZE); 753 if (!PageUptodate(page)) 754 SetPageUptodate(page); 755 unlock_page(page); 756 return page; 757 } 758 759 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr); 760 if (err) 761 goto put_err; 762 return page; 763 764 put_err: 765 f2fs_put_page(page, 1); 766 return ERR_PTR(err); 767 } 768 769 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index) 770 { 771 struct address_space *mapping = inode->i_mapping; 772 struct page *page; 773 774 page = find_get_page(mapping, index); 775 if (page && PageUptodate(page)) 776 return page; 777 f2fs_put_page(page, 0); 778 779 page = f2fs_get_read_data_page(inode, index, 0, false); 780 if (IS_ERR(page)) 781 return page; 782 783 if (PageUptodate(page)) 784 return page; 785 786 wait_on_page_locked(page); 787 if (unlikely(!PageUptodate(page))) { 788 f2fs_put_page(page, 0); 789 return ERR_PTR(-EIO); 790 } 791 return page; 792 } 793 794 /* 795 * If it tries to access a hole, return an error. 796 * Because, the callers, functions in dir.c and GC, should be able to know 797 * whether this page exists or not. 798 */ 799 struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index, 800 bool for_write) 801 { 802 struct address_space *mapping = inode->i_mapping; 803 struct page *page; 804 repeat: 805 page = f2fs_get_read_data_page(inode, index, 0, for_write); 806 if (IS_ERR(page)) 807 return page; 808 809 /* wait for read completion */ 810 lock_page(page); 811 if (unlikely(page->mapping != mapping)) { 812 f2fs_put_page(page, 1); 813 goto repeat; 814 } 815 if (unlikely(!PageUptodate(page))) { 816 f2fs_put_page(page, 1); 817 return ERR_PTR(-EIO); 818 } 819 return page; 820 } 821 822 /* 823 * Caller ensures that this data page is never allocated. 824 * A new zero-filled data page is allocated in the page cache. 825 * 826 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 827 * f2fs_unlock_op(). 828 * Note that, ipage is set only by make_empty_dir, and if any error occur, 829 * ipage should be released by this function. 830 */ 831 struct page *f2fs_get_new_data_page(struct inode *inode, 832 struct page *ipage, pgoff_t index, bool new_i_size) 833 { 834 struct address_space *mapping = inode->i_mapping; 835 struct page *page; 836 struct dnode_of_data dn; 837 int err; 838 839 page = f2fs_grab_cache_page(mapping, index, true); 840 if (!page) { 841 /* 842 * before exiting, we should make sure ipage will be released 843 * if any error occur. 844 */ 845 f2fs_put_page(ipage, 1); 846 return ERR_PTR(-ENOMEM); 847 } 848 849 set_new_dnode(&dn, inode, ipage, NULL, 0); 850 err = f2fs_reserve_block(&dn, index); 851 if (err) { 852 f2fs_put_page(page, 1); 853 return ERR_PTR(err); 854 } 855 if (!ipage) 856 f2fs_put_dnode(&dn); 857 858 if (PageUptodate(page)) 859 goto got_it; 860 861 if (dn.data_blkaddr == NEW_ADDR) { 862 zero_user_segment(page, 0, PAGE_SIZE); 863 if (!PageUptodate(page)) 864 SetPageUptodate(page); 865 } else { 866 f2fs_put_page(page, 1); 867 868 /* if ipage exists, blkaddr should be NEW_ADDR */ 869 f2fs_bug_on(F2FS_I_SB(inode), ipage); 870 page = f2fs_get_lock_data_page(inode, index, true); 871 if (IS_ERR(page)) 872 return page; 873 } 874 got_it: 875 if (new_i_size && i_size_read(inode) < 876 ((loff_t)(index + 1) << PAGE_SHIFT)) 877 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT)); 878 return page; 879 } 880 881 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) 882 { 883 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 884 struct f2fs_summary sum; 885 struct node_info ni; 886 block_t old_blkaddr; 887 blkcnt_t count = 1; 888 int err; 889 890 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 891 return -EPERM; 892 893 err = f2fs_get_node_info(sbi, dn->nid, &ni); 894 if (err) 895 return err; 896 897 dn->data_blkaddr = datablock_addr(dn->inode, 898 dn->node_page, dn->ofs_in_node); 899 if (dn->data_blkaddr != NULL_ADDR) 900 goto alloc; 901 902 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) 903 return err; 904 905 alloc: 906 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 907 old_blkaddr = dn->data_blkaddr; 908 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr, 909 &sum, seg_type, NULL, false); 910 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 911 invalidate_mapping_pages(META_MAPPING(sbi), 912 old_blkaddr, old_blkaddr); 913 f2fs_set_data_blkaddr(dn); 914 915 /* 916 * i_size will be updated by direct_IO. Otherwise, we'll get stale 917 * data from unwritten block via dio_read. 918 */ 919 return 0; 920 } 921 922 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from) 923 { 924 struct inode *inode = file_inode(iocb->ki_filp); 925 struct f2fs_map_blocks map; 926 int flag; 927 int err = 0; 928 bool direct_io = iocb->ki_flags & IOCB_DIRECT; 929 930 /* convert inline data for Direct I/O*/ 931 if (direct_io) { 932 err = f2fs_convert_inline_inode(inode); 933 if (err) 934 return err; 935 } 936 937 if (is_inode_flag_set(inode, FI_NO_PREALLOC)) 938 return 0; 939 940 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos); 941 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from)); 942 if (map.m_len > map.m_lblk) 943 map.m_len -= map.m_lblk; 944 else 945 map.m_len = 0; 946 947 map.m_next_pgofs = NULL; 948 map.m_next_extent = NULL; 949 map.m_seg_type = NO_CHECK_TYPE; 950 951 if (direct_io) { 952 map.m_seg_type = f2fs_rw_hint_to_seg_type(iocb->ki_hint); 953 flag = f2fs_force_buffered_io(inode, iocb, from) ? 954 F2FS_GET_BLOCK_PRE_AIO : 955 F2FS_GET_BLOCK_PRE_DIO; 956 goto map_blocks; 957 } 958 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) { 959 err = f2fs_convert_inline_inode(inode); 960 if (err) 961 return err; 962 } 963 if (f2fs_has_inline_data(inode)) 964 return err; 965 966 flag = F2FS_GET_BLOCK_PRE_AIO; 967 968 map_blocks: 969 err = f2fs_map_blocks(inode, &map, 1, flag); 970 if (map.m_len > 0 && err == -ENOSPC) { 971 if (!direct_io) 972 set_inode_flag(inode, FI_NO_PREALLOC); 973 err = 0; 974 } 975 return err; 976 } 977 978 void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock) 979 { 980 if (flag == F2FS_GET_BLOCK_PRE_AIO) { 981 if (lock) 982 down_read(&sbi->node_change); 983 else 984 up_read(&sbi->node_change); 985 } else { 986 if (lock) 987 f2fs_lock_op(sbi); 988 else 989 f2fs_unlock_op(sbi); 990 } 991 } 992 993 /* 994 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with 995 * f2fs_map_blocks structure. 996 * If original data blocks are allocated, then give them to blockdev. 997 * Otherwise, 998 * a. preallocate requested block addresses 999 * b. do not use extent cache for better performance 1000 * c. give the block addresses to blockdev 1001 */ 1002 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 1003 int create, int flag) 1004 { 1005 unsigned int maxblocks = map->m_len; 1006 struct dnode_of_data dn; 1007 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1008 int mode = create ? ALLOC_NODE : LOOKUP_NODE; 1009 pgoff_t pgofs, end_offset, end; 1010 int err = 0, ofs = 1; 1011 unsigned int ofs_in_node, last_ofs_in_node; 1012 blkcnt_t prealloc; 1013 struct extent_info ei = {0,0,0}; 1014 block_t blkaddr; 1015 unsigned int start_pgofs; 1016 1017 if (!maxblocks) 1018 return 0; 1019 1020 map->m_len = 0; 1021 map->m_flags = 0; 1022 1023 /* it only supports block size == page size */ 1024 pgofs = (pgoff_t)map->m_lblk; 1025 end = pgofs + maxblocks; 1026 1027 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) { 1028 map->m_pblk = ei.blk + pgofs - ei.fofs; 1029 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); 1030 map->m_flags = F2FS_MAP_MAPPED; 1031 if (map->m_next_extent) 1032 *map->m_next_extent = pgofs + map->m_len; 1033 goto out; 1034 } 1035 1036 next_dnode: 1037 if (create) 1038 __do_map_lock(sbi, flag, true); 1039 1040 /* When reading holes, we need its node page */ 1041 set_new_dnode(&dn, inode, NULL, NULL, 0); 1042 err = f2fs_get_dnode_of_data(&dn, pgofs, mode); 1043 if (err) { 1044 if (flag == F2FS_GET_BLOCK_BMAP) 1045 map->m_pblk = 0; 1046 if (err == -ENOENT) { 1047 err = 0; 1048 if (map->m_next_pgofs) 1049 *map->m_next_pgofs = 1050 f2fs_get_next_page_offset(&dn, pgofs); 1051 if (map->m_next_extent) 1052 *map->m_next_extent = 1053 f2fs_get_next_page_offset(&dn, pgofs); 1054 } 1055 goto unlock_out; 1056 } 1057 1058 start_pgofs = pgofs; 1059 prealloc = 0; 1060 last_ofs_in_node = ofs_in_node = dn.ofs_in_node; 1061 end_offset = ADDRS_PER_PAGE(dn.node_page, inode); 1062 1063 next_block: 1064 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node); 1065 1066 if (__is_valid_data_blkaddr(blkaddr) && 1067 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) { 1068 err = -EFAULT; 1069 goto sync_out; 1070 } 1071 1072 if (is_valid_data_blkaddr(sbi, blkaddr)) { 1073 /* use out-place-update for driect IO under LFS mode */ 1074 if (test_opt(sbi, LFS) && create && 1075 flag == F2FS_GET_BLOCK_DIO) { 1076 err = __allocate_data_block(&dn, map->m_seg_type); 1077 if (!err) 1078 set_inode_flag(inode, FI_APPEND_WRITE); 1079 } 1080 } else { 1081 if (create) { 1082 if (unlikely(f2fs_cp_error(sbi))) { 1083 err = -EIO; 1084 goto sync_out; 1085 } 1086 if (flag == F2FS_GET_BLOCK_PRE_AIO) { 1087 if (blkaddr == NULL_ADDR) { 1088 prealloc++; 1089 last_ofs_in_node = dn.ofs_in_node; 1090 } 1091 } else { 1092 WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO && 1093 flag != F2FS_GET_BLOCK_DIO); 1094 err = __allocate_data_block(&dn, 1095 map->m_seg_type); 1096 if (!err) 1097 set_inode_flag(inode, FI_APPEND_WRITE); 1098 } 1099 if (err) 1100 goto sync_out; 1101 map->m_flags |= F2FS_MAP_NEW; 1102 blkaddr = dn.data_blkaddr; 1103 } else { 1104 if (flag == F2FS_GET_BLOCK_BMAP) { 1105 map->m_pblk = 0; 1106 goto sync_out; 1107 } 1108 if (flag == F2FS_GET_BLOCK_PRECACHE) 1109 goto sync_out; 1110 if (flag == F2FS_GET_BLOCK_FIEMAP && 1111 blkaddr == NULL_ADDR) { 1112 if (map->m_next_pgofs) 1113 *map->m_next_pgofs = pgofs + 1; 1114 goto sync_out; 1115 } 1116 if (flag != F2FS_GET_BLOCK_FIEMAP) { 1117 /* for defragment case */ 1118 if (map->m_next_pgofs) 1119 *map->m_next_pgofs = pgofs + 1; 1120 goto sync_out; 1121 } 1122 } 1123 } 1124 1125 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1126 goto skip; 1127 1128 if (map->m_len == 0) { 1129 /* preallocated unwritten block should be mapped for fiemap. */ 1130 if (blkaddr == NEW_ADDR) 1131 map->m_flags |= F2FS_MAP_UNWRITTEN; 1132 map->m_flags |= F2FS_MAP_MAPPED; 1133 1134 map->m_pblk = blkaddr; 1135 map->m_len = 1; 1136 } else if ((map->m_pblk != NEW_ADDR && 1137 blkaddr == (map->m_pblk + ofs)) || 1138 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) || 1139 flag == F2FS_GET_BLOCK_PRE_DIO) { 1140 ofs++; 1141 map->m_len++; 1142 } else { 1143 goto sync_out; 1144 } 1145 1146 skip: 1147 dn.ofs_in_node++; 1148 pgofs++; 1149 1150 /* preallocate blocks in batch for one dnode page */ 1151 if (flag == F2FS_GET_BLOCK_PRE_AIO && 1152 (pgofs == end || dn.ofs_in_node == end_offset)) { 1153 1154 dn.ofs_in_node = ofs_in_node; 1155 err = f2fs_reserve_new_blocks(&dn, prealloc); 1156 if (err) 1157 goto sync_out; 1158 1159 map->m_len += dn.ofs_in_node - ofs_in_node; 1160 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { 1161 err = -ENOSPC; 1162 goto sync_out; 1163 } 1164 dn.ofs_in_node = end_offset; 1165 } 1166 1167 if (pgofs >= end) 1168 goto sync_out; 1169 else if (dn.ofs_in_node < end_offset) 1170 goto next_block; 1171 1172 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1173 if (map->m_flags & F2FS_MAP_MAPPED) { 1174 unsigned int ofs = start_pgofs - map->m_lblk; 1175 1176 f2fs_update_extent_cache_range(&dn, 1177 start_pgofs, map->m_pblk + ofs, 1178 map->m_len - ofs); 1179 } 1180 } 1181 1182 f2fs_put_dnode(&dn); 1183 1184 if (create) { 1185 __do_map_lock(sbi, flag, false); 1186 f2fs_balance_fs(sbi, dn.node_changed); 1187 } 1188 goto next_dnode; 1189 1190 sync_out: 1191 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1192 if (map->m_flags & F2FS_MAP_MAPPED) { 1193 unsigned int ofs = start_pgofs - map->m_lblk; 1194 1195 f2fs_update_extent_cache_range(&dn, 1196 start_pgofs, map->m_pblk + ofs, 1197 map->m_len - ofs); 1198 } 1199 if (map->m_next_extent) 1200 *map->m_next_extent = pgofs + 1; 1201 } 1202 f2fs_put_dnode(&dn); 1203 unlock_out: 1204 if (create) { 1205 __do_map_lock(sbi, flag, false); 1206 f2fs_balance_fs(sbi, dn.node_changed); 1207 } 1208 out: 1209 trace_f2fs_map_blocks(inode, map, err); 1210 return err; 1211 } 1212 1213 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) 1214 { 1215 struct f2fs_map_blocks map; 1216 block_t last_lblk; 1217 int err; 1218 1219 if (pos + len > i_size_read(inode)) 1220 return false; 1221 1222 map.m_lblk = F2FS_BYTES_TO_BLK(pos); 1223 map.m_next_pgofs = NULL; 1224 map.m_next_extent = NULL; 1225 map.m_seg_type = NO_CHECK_TYPE; 1226 last_lblk = F2FS_BLK_ALIGN(pos + len); 1227 1228 while (map.m_lblk < last_lblk) { 1229 map.m_len = last_lblk - map.m_lblk; 1230 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT); 1231 if (err || map.m_len == 0) 1232 return false; 1233 map.m_lblk += map.m_len; 1234 } 1235 return true; 1236 } 1237 1238 static int __get_data_block(struct inode *inode, sector_t iblock, 1239 struct buffer_head *bh, int create, int flag, 1240 pgoff_t *next_pgofs, int seg_type) 1241 { 1242 struct f2fs_map_blocks map; 1243 int err; 1244 1245 map.m_lblk = iblock; 1246 map.m_len = bh->b_size >> inode->i_blkbits; 1247 map.m_next_pgofs = next_pgofs; 1248 map.m_next_extent = NULL; 1249 map.m_seg_type = seg_type; 1250 1251 err = f2fs_map_blocks(inode, &map, create, flag); 1252 if (!err) { 1253 map_bh(bh, inode->i_sb, map.m_pblk); 1254 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; 1255 bh->b_size = (u64)map.m_len << inode->i_blkbits; 1256 } 1257 return err; 1258 } 1259 1260 static int get_data_block(struct inode *inode, sector_t iblock, 1261 struct buffer_head *bh_result, int create, int flag, 1262 pgoff_t *next_pgofs) 1263 { 1264 return __get_data_block(inode, iblock, bh_result, create, 1265 flag, next_pgofs, 1266 NO_CHECK_TYPE); 1267 } 1268 1269 static int get_data_block_dio(struct inode *inode, sector_t iblock, 1270 struct buffer_head *bh_result, int create) 1271 { 1272 return __get_data_block(inode, iblock, bh_result, create, 1273 F2FS_GET_BLOCK_DIO, NULL, 1274 f2fs_rw_hint_to_seg_type( 1275 inode->i_write_hint)); 1276 } 1277 1278 static int get_data_block_bmap(struct inode *inode, sector_t iblock, 1279 struct buffer_head *bh_result, int create) 1280 { 1281 /* Block number less than F2FS MAX BLOCKS */ 1282 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks)) 1283 return -EFBIG; 1284 1285 return __get_data_block(inode, iblock, bh_result, create, 1286 F2FS_GET_BLOCK_BMAP, NULL, 1287 NO_CHECK_TYPE); 1288 } 1289 1290 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset) 1291 { 1292 return (offset >> inode->i_blkbits); 1293 } 1294 1295 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk) 1296 { 1297 return (blk << inode->i_blkbits); 1298 } 1299 1300 static int f2fs_xattr_fiemap(struct inode *inode, 1301 struct fiemap_extent_info *fieinfo) 1302 { 1303 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1304 struct page *page; 1305 struct node_info ni; 1306 __u64 phys = 0, len; 1307 __u32 flags; 1308 nid_t xnid = F2FS_I(inode)->i_xattr_nid; 1309 int err = 0; 1310 1311 if (f2fs_has_inline_xattr(inode)) { 1312 int offset; 1313 1314 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), 1315 inode->i_ino, false); 1316 if (!page) 1317 return -ENOMEM; 1318 1319 err = f2fs_get_node_info(sbi, inode->i_ino, &ni); 1320 if (err) { 1321 f2fs_put_page(page, 1); 1322 return err; 1323 } 1324 1325 phys = (__u64)blk_to_logical(inode, ni.blk_addr); 1326 offset = offsetof(struct f2fs_inode, i_addr) + 1327 sizeof(__le32) * (DEF_ADDRS_PER_INODE - 1328 get_inline_xattr_addrs(inode)); 1329 1330 phys += offset; 1331 len = inline_xattr_size(inode); 1332 1333 f2fs_put_page(page, 1); 1334 1335 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; 1336 1337 if (!xnid) 1338 flags |= FIEMAP_EXTENT_LAST; 1339 1340 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1341 if (err || err == 1) 1342 return err; 1343 } 1344 1345 if (xnid) { 1346 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false); 1347 if (!page) 1348 return -ENOMEM; 1349 1350 err = f2fs_get_node_info(sbi, xnid, &ni); 1351 if (err) { 1352 f2fs_put_page(page, 1); 1353 return err; 1354 } 1355 1356 phys = (__u64)blk_to_logical(inode, ni.blk_addr); 1357 len = inode->i_sb->s_blocksize; 1358 1359 f2fs_put_page(page, 1); 1360 1361 flags = FIEMAP_EXTENT_LAST; 1362 } 1363 1364 if (phys) 1365 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1366 1367 return (err < 0 ? err : 0); 1368 } 1369 1370 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 1371 u64 start, u64 len) 1372 { 1373 struct buffer_head map_bh; 1374 sector_t start_blk, last_blk; 1375 pgoff_t next_pgofs; 1376 u64 logical = 0, phys = 0, size = 0; 1377 u32 flags = 0; 1378 int ret = 0; 1379 1380 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { 1381 ret = f2fs_precache_extents(inode); 1382 if (ret) 1383 return ret; 1384 } 1385 1386 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR); 1387 if (ret) 1388 return ret; 1389 1390 inode_lock(inode); 1391 1392 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { 1393 ret = f2fs_xattr_fiemap(inode, fieinfo); 1394 goto out; 1395 } 1396 1397 if (f2fs_has_inline_data(inode)) { 1398 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 1399 if (ret != -EAGAIN) 1400 goto out; 1401 } 1402 1403 if (logical_to_blk(inode, len) == 0) 1404 len = blk_to_logical(inode, 1); 1405 1406 start_blk = logical_to_blk(inode, start); 1407 last_blk = logical_to_blk(inode, start + len - 1); 1408 1409 next: 1410 memset(&map_bh, 0, sizeof(struct buffer_head)); 1411 map_bh.b_size = len; 1412 1413 ret = get_data_block(inode, start_blk, &map_bh, 0, 1414 F2FS_GET_BLOCK_FIEMAP, &next_pgofs); 1415 if (ret) 1416 goto out; 1417 1418 /* HOLE */ 1419 if (!buffer_mapped(&map_bh)) { 1420 start_blk = next_pgofs; 1421 1422 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode, 1423 F2FS_I_SB(inode)->max_file_blocks)) 1424 goto prep_next; 1425 1426 flags |= FIEMAP_EXTENT_LAST; 1427 } 1428 1429 if (size) { 1430 if (f2fs_encrypted_inode(inode)) 1431 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; 1432 1433 ret = fiemap_fill_next_extent(fieinfo, logical, 1434 phys, size, flags); 1435 } 1436 1437 if (start_blk > last_blk || ret) 1438 goto out; 1439 1440 logical = blk_to_logical(inode, start_blk); 1441 phys = blk_to_logical(inode, map_bh.b_blocknr); 1442 size = map_bh.b_size; 1443 flags = 0; 1444 if (buffer_unwritten(&map_bh)) 1445 flags = FIEMAP_EXTENT_UNWRITTEN; 1446 1447 start_blk += logical_to_blk(inode, size); 1448 1449 prep_next: 1450 cond_resched(); 1451 if (fatal_signal_pending(current)) 1452 ret = -EINTR; 1453 else 1454 goto next; 1455 out: 1456 if (ret == 1) 1457 ret = 0; 1458 1459 inode_unlock(inode); 1460 return ret; 1461 } 1462 1463 /* 1464 * This function was originally taken from fs/mpage.c, and customized for f2fs. 1465 * Major change was from block_size == page_size in f2fs by default. 1466 * 1467 * Note that the aops->readpages() function is ONLY used for read-ahead. If 1468 * this function ever deviates from doing just read-ahead, it should either 1469 * use ->readpage() or do the necessary surgery to decouple ->readpages() 1470 * from read-ahead. 1471 */ 1472 static int f2fs_mpage_readpages(struct address_space *mapping, 1473 struct list_head *pages, struct page *page, 1474 unsigned nr_pages, bool is_readahead) 1475 { 1476 struct bio *bio = NULL; 1477 sector_t last_block_in_bio = 0; 1478 struct inode *inode = mapping->host; 1479 const unsigned blkbits = inode->i_blkbits; 1480 const unsigned blocksize = 1 << blkbits; 1481 sector_t block_in_file; 1482 sector_t last_block; 1483 sector_t last_block_in_file; 1484 sector_t block_nr; 1485 struct f2fs_map_blocks map; 1486 1487 map.m_pblk = 0; 1488 map.m_lblk = 0; 1489 map.m_len = 0; 1490 map.m_flags = 0; 1491 map.m_next_pgofs = NULL; 1492 map.m_next_extent = NULL; 1493 map.m_seg_type = NO_CHECK_TYPE; 1494 1495 for (; nr_pages; nr_pages--) { 1496 if (pages) { 1497 page = list_last_entry(pages, struct page, lru); 1498 1499 prefetchw(&page->flags); 1500 list_del(&page->lru); 1501 if (add_to_page_cache_lru(page, mapping, 1502 page->index, 1503 readahead_gfp_mask(mapping))) 1504 goto next_page; 1505 } 1506 1507 block_in_file = (sector_t)page->index; 1508 last_block = block_in_file + nr_pages; 1509 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> 1510 blkbits; 1511 if (last_block > last_block_in_file) 1512 last_block = last_block_in_file; 1513 1514 /* 1515 * Map blocks using the previous result first. 1516 */ 1517 if ((map.m_flags & F2FS_MAP_MAPPED) && 1518 block_in_file > map.m_lblk && 1519 block_in_file < (map.m_lblk + map.m_len)) 1520 goto got_it; 1521 1522 /* 1523 * Then do more f2fs_map_blocks() calls until we are 1524 * done with this page. 1525 */ 1526 map.m_flags = 0; 1527 1528 if (block_in_file < last_block) { 1529 map.m_lblk = block_in_file; 1530 map.m_len = last_block - block_in_file; 1531 1532 if (f2fs_map_blocks(inode, &map, 0, 1533 F2FS_GET_BLOCK_DEFAULT)) 1534 goto set_error_page; 1535 } 1536 got_it: 1537 if ((map.m_flags & F2FS_MAP_MAPPED)) { 1538 block_nr = map.m_pblk + block_in_file - map.m_lblk; 1539 SetPageMappedToDisk(page); 1540 1541 if (!PageUptodate(page) && !cleancache_get_page(page)) { 1542 SetPageUptodate(page); 1543 goto confused; 1544 } 1545 1546 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, 1547 DATA_GENERIC)) 1548 goto set_error_page; 1549 } else { 1550 zero_user_segment(page, 0, PAGE_SIZE); 1551 if (!PageUptodate(page)) 1552 SetPageUptodate(page); 1553 unlock_page(page); 1554 goto next_page; 1555 } 1556 1557 /* 1558 * This page will go to BIO. Do we need to send this 1559 * BIO off first? 1560 */ 1561 if (bio && (last_block_in_bio != block_nr - 1 || 1562 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) { 1563 submit_and_realloc: 1564 __submit_bio(F2FS_I_SB(inode), bio, DATA); 1565 bio = NULL; 1566 } 1567 if (bio == NULL) { 1568 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, 1569 is_readahead ? REQ_RAHEAD : 0); 1570 if (IS_ERR(bio)) { 1571 bio = NULL; 1572 goto set_error_page; 1573 } 1574 } 1575 1576 /* 1577 * If the page is under writeback, we need to wait for 1578 * its completion to see the correct decrypted data. 1579 */ 1580 f2fs_wait_on_block_writeback(inode, block_nr); 1581 1582 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 1583 goto submit_and_realloc; 1584 1585 ClearPageError(page); 1586 last_block_in_bio = block_nr; 1587 goto next_page; 1588 set_error_page: 1589 SetPageError(page); 1590 zero_user_segment(page, 0, PAGE_SIZE); 1591 unlock_page(page); 1592 goto next_page; 1593 confused: 1594 if (bio) { 1595 __submit_bio(F2FS_I_SB(inode), bio, DATA); 1596 bio = NULL; 1597 } 1598 unlock_page(page); 1599 next_page: 1600 if (pages) 1601 put_page(page); 1602 } 1603 BUG_ON(pages && !list_empty(pages)); 1604 if (bio) 1605 __submit_bio(F2FS_I_SB(inode), bio, DATA); 1606 return 0; 1607 } 1608 1609 static int f2fs_read_data_page(struct file *file, struct page *page) 1610 { 1611 struct inode *inode = page->mapping->host; 1612 int ret = -EAGAIN; 1613 1614 trace_f2fs_readpage(page, DATA); 1615 1616 /* If the file has inline data, try to read it directly */ 1617 if (f2fs_has_inline_data(inode)) 1618 ret = f2fs_read_inline_data(inode, page); 1619 if (ret == -EAGAIN) 1620 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1, false); 1621 return ret; 1622 } 1623 1624 static int f2fs_read_data_pages(struct file *file, 1625 struct address_space *mapping, 1626 struct list_head *pages, unsigned nr_pages) 1627 { 1628 struct inode *inode = mapping->host; 1629 struct page *page = list_last_entry(pages, struct page, lru); 1630 1631 trace_f2fs_readpages(inode, page, nr_pages); 1632 1633 /* If the file has inline data, skip readpages */ 1634 if (f2fs_has_inline_data(inode)) 1635 return 0; 1636 1637 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages, true); 1638 } 1639 1640 static int encrypt_one_page(struct f2fs_io_info *fio) 1641 { 1642 struct inode *inode = fio->page->mapping->host; 1643 struct page *mpage; 1644 gfp_t gfp_flags = GFP_NOFS; 1645 1646 if (!f2fs_encrypted_file(inode)) 1647 return 0; 1648 1649 /* wait for GCed page writeback via META_MAPPING */ 1650 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); 1651 1652 retry_encrypt: 1653 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page, 1654 PAGE_SIZE, 0, fio->page->index, gfp_flags); 1655 if (IS_ERR(fio->encrypted_page)) { 1656 /* flush pending IOs and wait for a while in the ENOMEM case */ 1657 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { 1658 f2fs_flush_merged_writes(fio->sbi); 1659 congestion_wait(BLK_RW_ASYNC, HZ/50); 1660 gfp_flags |= __GFP_NOFAIL; 1661 goto retry_encrypt; 1662 } 1663 return PTR_ERR(fio->encrypted_page); 1664 } 1665 1666 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr); 1667 if (mpage) { 1668 if (PageUptodate(mpage)) 1669 memcpy(page_address(mpage), 1670 page_address(fio->encrypted_page), PAGE_SIZE); 1671 f2fs_put_page(mpage, 1); 1672 } 1673 return 0; 1674 } 1675 1676 static inline bool check_inplace_update_policy(struct inode *inode, 1677 struct f2fs_io_info *fio) 1678 { 1679 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1680 unsigned int policy = SM_I(sbi)->ipu_policy; 1681 1682 if (policy & (0x1 << F2FS_IPU_FORCE)) 1683 return true; 1684 if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi)) 1685 return true; 1686 if (policy & (0x1 << F2FS_IPU_UTIL) && 1687 utilization(sbi) > SM_I(sbi)->min_ipu_util) 1688 return true; 1689 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) && 1690 utilization(sbi) > SM_I(sbi)->min_ipu_util) 1691 return true; 1692 1693 /* 1694 * IPU for rewrite async pages 1695 */ 1696 if (policy & (0x1 << F2FS_IPU_ASYNC) && 1697 fio && fio->op == REQ_OP_WRITE && 1698 !(fio->op_flags & REQ_SYNC) && 1699 !f2fs_encrypted_inode(inode)) 1700 return true; 1701 1702 /* this is only set during fdatasync */ 1703 if (policy & (0x1 << F2FS_IPU_FSYNC) && 1704 is_inode_flag_set(inode, FI_NEED_IPU)) 1705 return true; 1706 1707 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 1708 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 1709 return true; 1710 1711 return false; 1712 } 1713 1714 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) 1715 { 1716 if (f2fs_is_pinned_file(inode)) 1717 return true; 1718 1719 /* if this is cold file, we should overwrite to avoid fragmentation */ 1720 if (file_is_cold(inode)) 1721 return true; 1722 1723 return check_inplace_update_policy(inode, fio); 1724 } 1725 1726 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) 1727 { 1728 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1729 1730 if (test_opt(sbi, LFS)) 1731 return true; 1732 if (S_ISDIR(inode->i_mode)) 1733 return true; 1734 if (f2fs_is_atomic_file(inode)) 1735 return true; 1736 if (fio) { 1737 if (is_cold_data(fio->page)) 1738 return true; 1739 if (IS_ATOMIC_WRITTEN_PAGE(fio->page)) 1740 return true; 1741 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 1742 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 1743 return true; 1744 } 1745 return false; 1746 } 1747 1748 static inline bool need_inplace_update(struct f2fs_io_info *fio) 1749 { 1750 struct inode *inode = fio->page->mapping->host; 1751 1752 if (f2fs_should_update_outplace(inode, fio)) 1753 return false; 1754 1755 return f2fs_should_update_inplace(inode, fio); 1756 } 1757 1758 int f2fs_do_write_data_page(struct f2fs_io_info *fio) 1759 { 1760 struct page *page = fio->page; 1761 struct inode *inode = page->mapping->host; 1762 struct dnode_of_data dn; 1763 struct extent_info ei = {0,0,0}; 1764 struct node_info ni; 1765 bool ipu_force = false; 1766 int err = 0; 1767 1768 set_new_dnode(&dn, inode, NULL, NULL, 0); 1769 if (need_inplace_update(fio) && 1770 f2fs_lookup_extent_cache(inode, page->index, &ei)) { 1771 fio->old_blkaddr = ei.blk + page->index - ei.fofs; 1772 1773 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 1774 DATA_GENERIC)) 1775 return -EFAULT; 1776 1777 ipu_force = true; 1778 fio->need_lock = LOCK_DONE; 1779 goto got_it; 1780 } 1781 1782 /* Deadlock due to between page->lock and f2fs_lock_op */ 1783 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) 1784 return -EAGAIN; 1785 1786 err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 1787 if (err) 1788 goto out; 1789 1790 fio->old_blkaddr = dn.data_blkaddr; 1791 1792 /* This page is already truncated */ 1793 if (fio->old_blkaddr == NULL_ADDR) { 1794 ClearPageUptodate(page); 1795 goto out_writepage; 1796 } 1797 got_it: 1798 if (__is_valid_data_blkaddr(fio->old_blkaddr) && 1799 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 1800 DATA_GENERIC)) { 1801 err = -EFAULT; 1802 goto out_writepage; 1803 } 1804 /* 1805 * If current allocation needs SSR, 1806 * it had better in-place writes for updated data. 1807 */ 1808 if (ipu_force || (is_valid_data_blkaddr(fio->sbi, fio->old_blkaddr) && 1809 need_inplace_update(fio))) { 1810 err = encrypt_one_page(fio); 1811 if (err) 1812 goto out_writepage; 1813 1814 set_page_writeback(page); 1815 ClearPageError(page); 1816 f2fs_put_dnode(&dn); 1817 if (fio->need_lock == LOCK_REQ) 1818 f2fs_unlock_op(fio->sbi); 1819 err = f2fs_inplace_write_data(fio); 1820 trace_f2fs_do_write_data_page(fio->page, IPU); 1821 set_inode_flag(inode, FI_UPDATE_WRITE); 1822 return err; 1823 } 1824 1825 if (fio->need_lock == LOCK_RETRY) { 1826 if (!f2fs_trylock_op(fio->sbi)) { 1827 err = -EAGAIN; 1828 goto out_writepage; 1829 } 1830 fio->need_lock = LOCK_REQ; 1831 } 1832 1833 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni); 1834 if (err) 1835 goto out_writepage; 1836 1837 fio->version = ni.version; 1838 1839 err = encrypt_one_page(fio); 1840 if (err) 1841 goto out_writepage; 1842 1843 set_page_writeback(page); 1844 ClearPageError(page); 1845 1846 /* LFS mode write path */ 1847 f2fs_outplace_write_data(&dn, fio); 1848 trace_f2fs_do_write_data_page(page, OPU); 1849 set_inode_flag(inode, FI_APPEND_WRITE); 1850 if (page->index == 0) 1851 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN); 1852 out_writepage: 1853 f2fs_put_dnode(&dn); 1854 out: 1855 if (fio->need_lock == LOCK_REQ) 1856 f2fs_unlock_op(fio->sbi); 1857 return err; 1858 } 1859 1860 static int __write_data_page(struct page *page, bool *submitted, 1861 struct writeback_control *wbc, 1862 enum iostat_type io_type) 1863 { 1864 struct inode *inode = page->mapping->host; 1865 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1866 loff_t i_size = i_size_read(inode); 1867 const pgoff_t end_index = ((unsigned long long) i_size) 1868 >> PAGE_SHIFT; 1869 loff_t psize = (page->index + 1) << PAGE_SHIFT; 1870 unsigned offset = 0; 1871 bool need_balance_fs = false; 1872 int err = 0; 1873 struct f2fs_io_info fio = { 1874 .sbi = sbi, 1875 .ino = inode->i_ino, 1876 .type = DATA, 1877 .op = REQ_OP_WRITE, 1878 .op_flags = wbc_to_write_flags(wbc), 1879 .old_blkaddr = NULL_ADDR, 1880 .page = page, 1881 .encrypted_page = NULL, 1882 .submitted = false, 1883 .need_lock = LOCK_RETRY, 1884 .io_type = io_type, 1885 .io_wbc = wbc, 1886 }; 1887 1888 trace_f2fs_writepage(page, DATA); 1889 1890 /* we should bypass data pages to proceed the kworkder jobs */ 1891 if (unlikely(f2fs_cp_error(sbi))) { 1892 mapping_set_error(page->mapping, -EIO); 1893 /* 1894 * don't drop any dirty dentry pages for keeping lastest 1895 * directory structure. 1896 */ 1897 if (S_ISDIR(inode->i_mode)) 1898 goto redirty_out; 1899 goto out; 1900 } 1901 1902 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1903 goto redirty_out; 1904 1905 if (page->index < end_index) 1906 goto write; 1907 1908 /* 1909 * If the offset is out-of-range of file size, 1910 * this page does not have to be written to disk. 1911 */ 1912 offset = i_size & (PAGE_SIZE - 1); 1913 if ((page->index >= end_index + 1) || !offset) 1914 goto out; 1915 1916 zero_user_segment(page, offset, PAGE_SIZE); 1917 write: 1918 if (f2fs_is_drop_cache(inode)) 1919 goto out; 1920 /* we should not write 0'th page having journal header */ 1921 if (f2fs_is_volatile_file(inode) && (!page->index || 1922 (!wbc->for_reclaim && 1923 f2fs_available_free_memory(sbi, BASE_CHECK)))) 1924 goto redirty_out; 1925 1926 /* Dentry blocks are controlled by checkpoint */ 1927 if (S_ISDIR(inode->i_mode)) { 1928 fio.need_lock = LOCK_DONE; 1929 err = f2fs_do_write_data_page(&fio); 1930 goto done; 1931 } 1932 1933 if (!wbc->for_reclaim) 1934 need_balance_fs = true; 1935 else if (has_not_enough_free_secs(sbi, 0, 0)) 1936 goto redirty_out; 1937 else 1938 set_inode_flag(inode, FI_HOT_DATA); 1939 1940 err = -EAGAIN; 1941 if (f2fs_has_inline_data(inode)) { 1942 err = f2fs_write_inline_data(inode, page); 1943 if (!err) 1944 goto out; 1945 } 1946 1947 if (err == -EAGAIN) { 1948 err = f2fs_do_write_data_page(&fio); 1949 if (err == -EAGAIN) { 1950 fio.need_lock = LOCK_REQ; 1951 err = f2fs_do_write_data_page(&fio); 1952 } 1953 } 1954 1955 if (err) { 1956 file_set_keep_isize(inode); 1957 } else { 1958 down_write(&F2FS_I(inode)->i_sem); 1959 if (F2FS_I(inode)->last_disk_size < psize) 1960 F2FS_I(inode)->last_disk_size = psize; 1961 up_write(&F2FS_I(inode)->i_sem); 1962 } 1963 1964 done: 1965 if (err && err != -ENOENT) 1966 goto redirty_out; 1967 1968 out: 1969 inode_dec_dirty_pages(inode); 1970 if (err) 1971 ClearPageUptodate(page); 1972 1973 if (wbc->for_reclaim) { 1974 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA); 1975 clear_inode_flag(inode, FI_HOT_DATA); 1976 f2fs_remove_dirty_inode(inode); 1977 submitted = NULL; 1978 } 1979 1980 unlock_page(page); 1981 if (!S_ISDIR(inode->i_mode)) 1982 f2fs_balance_fs(sbi, need_balance_fs); 1983 1984 if (unlikely(f2fs_cp_error(sbi))) { 1985 f2fs_submit_merged_write(sbi, DATA); 1986 submitted = NULL; 1987 } 1988 1989 if (submitted) 1990 *submitted = fio.submitted; 1991 1992 return 0; 1993 1994 redirty_out: 1995 redirty_page_for_writepage(wbc, page); 1996 /* 1997 * pageout() in MM traslates EAGAIN, so calls handle_write_error() 1998 * -> mapping_set_error() -> set_bit(AS_EIO, ...). 1999 * file_write_and_wait_range() will see EIO error, which is critical 2000 * to return value of fsync() followed by atomic_write failure to user. 2001 */ 2002 if (!err || wbc->for_reclaim) 2003 return AOP_WRITEPAGE_ACTIVATE; 2004 unlock_page(page); 2005 return err; 2006 } 2007 2008 static int f2fs_write_data_page(struct page *page, 2009 struct writeback_control *wbc) 2010 { 2011 return __write_data_page(page, NULL, wbc, FS_DATA_IO); 2012 } 2013 2014 /* 2015 * This function was copied from write_cche_pages from mm/page-writeback.c. 2016 * The major change is making write step of cold data page separately from 2017 * warm/hot data page. 2018 */ 2019 static int f2fs_write_cache_pages(struct address_space *mapping, 2020 struct writeback_control *wbc, 2021 enum iostat_type io_type) 2022 { 2023 int ret = 0; 2024 int done = 0; 2025 struct pagevec pvec; 2026 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 2027 int nr_pages; 2028 pgoff_t uninitialized_var(writeback_index); 2029 pgoff_t index; 2030 pgoff_t end; /* Inclusive */ 2031 pgoff_t done_index; 2032 int cycled; 2033 int range_whole = 0; 2034 int tag; 2035 int nwritten = 0; 2036 2037 pagevec_init(&pvec); 2038 2039 if (get_dirty_pages(mapping->host) <= 2040 SM_I(F2FS_M_SB(mapping))->min_hot_blocks) 2041 set_inode_flag(mapping->host, FI_HOT_DATA); 2042 else 2043 clear_inode_flag(mapping->host, FI_HOT_DATA); 2044 2045 if (wbc->range_cyclic) { 2046 writeback_index = mapping->writeback_index; /* prev offset */ 2047 index = writeback_index; 2048 if (index == 0) 2049 cycled = 1; 2050 else 2051 cycled = 0; 2052 end = -1; 2053 } else { 2054 index = wbc->range_start >> PAGE_SHIFT; 2055 end = wbc->range_end >> PAGE_SHIFT; 2056 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2057 range_whole = 1; 2058 cycled = 1; /* ignore range_cyclic tests */ 2059 } 2060 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2061 tag = PAGECACHE_TAG_TOWRITE; 2062 else 2063 tag = PAGECACHE_TAG_DIRTY; 2064 retry: 2065 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2066 tag_pages_for_writeback(mapping, index, end); 2067 done_index = index; 2068 while (!done && (index <= end)) { 2069 int i; 2070 2071 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, 2072 tag); 2073 if (nr_pages == 0) 2074 break; 2075 2076 for (i = 0; i < nr_pages; i++) { 2077 struct page *page = pvec.pages[i]; 2078 bool submitted = false; 2079 2080 /* give a priority to WB_SYNC threads */ 2081 if (atomic_read(&sbi->wb_sync_req[DATA]) && 2082 wbc->sync_mode == WB_SYNC_NONE) { 2083 done = 1; 2084 break; 2085 } 2086 2087 done_index = page->index; 2088 retry_write: 2089 lock_page(page); 2090 2091 if (unlikely(page->mapping != mapping)) { 2092 continue_unlock: 2093 unlock_page(page); 2094 continue; 2095 } 2096 2097 if (!PageDirty(page)) { 2098 /* someone wrote it for us */ 2099 goto continue_unlock; 2100 } 2101 2102 if (PageWriteback(page)) { 2103 if (wbc->sync_mode != WB_SYNC_NONE) 2104 f2fs_wait_on_page_writeback(page, 2105 DATA, true); 2106 else 2107 goto continue_unlock; 2108 } 2109 2110 BUG_ON(PageWriteback(page)); 2111 if (!clear_page_dirty_for_io(page)) 2112 goto continue_unlock; 2113 2114 ret = __write_data_page(page, &submitted, wbc, io_type); 2115 if (unlikely(ret)) { 2116 /* 2117 * keep nr_to_write, since vfs uses this to 2118 * get # of written pages. 2119 */ 2120 if (ret == AOP_WRITEPAGE_ACTIVATE) { 2121 unlock_page(page); 2122 ret = 0; 2123 continue; 2124 } else if (ret == -EAGAIN) { 2125 ret = 0; 2126 if (wbc->sync_mode == WB_SYNC_ALL) { 2127 cond_resched(); 2128 congestion_wait(BLK_RW_ASYNC, 2129 HZ/50); 2130 goto retry_write; 2131 } 2132 continue; 2133 } 2134 done_index = page->index + 1; 2135 done = 1; 2136 break; 2137 } else if (submitted) { 2138 nwritten++; 2139 } 2140 2141 if (--wbc->nr_to_write <= 0 && 2142 wbc->sync_mode == WB_SYNC_NONE) { 2143 done = 1; 2144 break; 2145 } 2146 } 2147 pagevec_release(&pvec); 2148 cond_resched(); 2149 } 2150 2151 if (!cycled && !done) { 2152 cycled = 1; 2153 index = 0; 2154 end = writeback_index - 1; 2155 goto retry; 2156 } 2157 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 2158 mapping->writeback_index = done_index; 2159 2160 if (nwritten) 2161 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, 2162 NULL, 0, DATA); 2163 2164 return ret; 2165 } 2166 2167 static inline bool __should_serialize_io(struct inode *inode, 2168 struct writeback_control *wbc) 2169 { 2170 if (!S_ISREG(inode->i_mode)) 2171 return false; 2172 if (wbc->sync_mode != WB_SYNC_ALL) 2173 return true; 2174 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks) 2175 return true; 2176 return false; 2177 } 2178 2179 static int __f2fs_write_data_pages(struct address_space *mapping, 2180 struct writeback_control *wbc, 2181 enum iostat_type io_type) 2182 { 2183 struct inode *inode = mapping->host; 2184 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2185 struct blk_plug plug; 2186 int ret; 2187 bool locked = false; 2188 2189 /* deal with chardevs and other special file */ 2190 if (!mapping->a_ops->writepage) 2191 return 0; 2192 2193 /* skip writing if there is no dirty page in this inode */ 2194 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 2195 return 0; 2196 2197 /* during POR, we don't need to trigger writepage at all. */ 2198 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 2199 goto skip_write; 2200 2201 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE && 2202 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 2203 f2fs_available_free_memory(sbi, DIRTY_DENTS)) 2204 goto skip_write; 2205 2206 /* skip writing during file defragment */ 2207 if (is_inode_flag_set(inode, FI_DO_DEFRAG)) 2208 goto skip_write; 2209 2210 trace_f2fs_writepages(mapping->host, wbc, DATA); 2211 2212 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */ 2213 if (wbc->sync_mode == WB_SYNC_ALL) 2214 atomic_inc(&sbi->wb_sync_req[DATA]); 2215 else if (atomic_read(&sbi->wb_sync_req[DATA])) 2216 goto skip_write; 2217 2218 if (__should_serialize_io(inode, wbc)) { 2219 mutex_lock(&sbi->writepages); 2220 locked = true; 2221 } 2222 2223 blk_start_plug(&plug); 2224 ret = f2fs_write_cache_pages(mapping, wbc, io_type); 2225 blk_finish_plug(&plug); 2226 2227 if (locked) 2228 mutex_unlock(&sbi->writepages); 2229 2230 if (wbc->sync_mode == WB_SYNC_ALL) 2231 atomic_dec(&sbi->wb_sync_req[DATA]); 2232 /* 2233 * if some pages were truncated, we cannot guarantee its mapping->host 2234 * to detect pending bios. 2235 */ 2236 2237 f2fs_remove_dirty_inode(inode); 2238 return ret; 2239 2240 skip_write: 2241 wbc->pages_skipped += get_dirty_pages(inode); 2242 trace_f2fs_writepages(mapping->host, wbc, DATA); 2243 return 0; 2244 } 2245 2246 static int f2fs_write_data_pages(struct address_space *mapping, 2247 struct writeback_control *wbc) 2248 { 2249 struct inode *inode = mapping->host; 2250 2251 return __f2fs_write_data_pages(mapping, wbc, 2252 F2FS_I(inode)->cp_task == current ? 2253 FS_CP_DATA_IO : FS_DATA_IO); 2254 } 2255 2256 static void f2fs_write_failed(struct address_space *mapping, loff_t to) 2257 { 2258 struct inode *inode = mapping->host; 2259 loff_t i_size = i_size_read(inode); 2260 2261 if (to > i_size) { 2262 down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 2263 down_write(&F2FS_I(inode)->i_mmap_sem); 2264 2265 truncate_pagecache(inode, i_size); 2266 f2fs_truncate_blocks(inode, i_size, true); 2267 2268 up_write(&F2FS_I(inode)->i_mmap_sem); 2269 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 2270 } 2271 } 2272 2273 static int prepare_write_begin(struct f2fs_sb_info *sbi, 2274 struct page *page, loff_t pos, unsigned len, 2275 block_t *blk_addr, bool *node_changed) 2276 { 2277 struct inode *inode = page->mapping->host; 2278 pgoff_t index = page->index; 2279 struct dnode_of_data dn; 2280 struct page *ipage; 2281 bool locked = false; 2282 struct extent_info ei = {0,0,0}; 2283 int err = 0; 2284 2285 /* 2286 * we already allocated all the blocks, so we don't need to get 2287 * the block addresses when there is no need to fill the page. 2288 */ 2289 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE && 2290 !is_inode_flag_set(inode, FI_NO_PREALLOC)) 2291 return 0; 2292 2293 if (f2fs_has_inline_data(inode) || 2294 (pos & PAGE_MASK) >= i_size_read(inode)) { 2295 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true); 2296 locked = true; 2297 } 2298 restart: 2299 /* check inline_data */ 2300 ipage = f2fs_get_node_page(sbi, inode->i_ino); 2301 if (IS_ERR(ipage)) { 2302 err = PTR_ERR(ipage); 2303 goto unlock_out; 2304 } 2305 2306 set_new_dnode(&dn, inode, ipage, ipage, 0); 2307 2308 if (f2fs_has_inline_data(inode)) { 2309 if (pos + len <= MAX_INLINE_DATA(inode)) { 2310 f2fs_do_read_inline_data(page, ipage); 2311 set_inode_flag(inode, FI_DATA_EXIST); 2312 if (inode->i_nlink) 2313 set_inline_node(ipage); 2314 } else { 2315 err = f2fs_convert_inline_page(&dn, page); 2316 if (err) 2317 goto out; 2318 if (dn.data_blkaddr == NULL_ADDR) 2319 err = f2fs_get_block(&dn, index); 2320 } 2321 } else if (locked) { 2322 err = f2fs_get_block(&dn, index); 2323 } else { 2324 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 2325 dn.data_blkaddr = ei.blk + index - ei.fofs; 2326 } else { 2327 /* hole case */ 2328 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 2329 if (err || dn.data_blkaddr == NULL_ADDR) { 2330 f2fs_put_dnode(&dn); 2331 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, 2332 true); 2333 locked = true; 2334 goto restart; 2335 } 2336 } 2337 } 2338 2339 /* convert_inline_page can make node_changed */ 2340 *blk_addr = dn.data_blkaddr; 2341 *node_changed = dn.node_changed; 2342 out: 2343 f2fs_put_dnode(&dn); 2344 unlock_out: 2345 if (locked) 2346 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false); 2347 return err; 2348 } 2349 2350 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 2351 loff_t pos, unsigned len, unsigned flags, 2352 struct page **pagep, void **fsdata) 2353 { 2354 struct inode *inode = mapping->host; 2355 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2356 struct page *page = NULL; 2357 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT; 2358 bool need_balance = false, drop_atomic = false; 2359 block_t blkaddr = NULL_ADDR; 2360 int err = 0; 2361 2362 trace_f2fs_write_begin(inode, pos, len, flags); 2363 2364 err = f2fs_is_checkpoint_ready(sbi); 2365 if (err) 2366 goto fail; 2367 2368 if ((f2fs_is_atomic_file(inode) && 2369 !f2fs_available_free_memory(sbi, INMEM_PAGES)) || 2370 is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST)) { 2371 err = -ENOMEM; 2372 drop_atomic = true; 2373 goto fail; 2374 } 2375 2376 /* 2377 * We should check this at this moment to avoid deadlock on inode page 2378 * and #0 page. The locking rule for inline_data conversion should be: 2379 * lock_page(page #0) -> lock_page(inode_page) 2380 */ 2381 if (index != 0) { 2382 err = f2fs_convert_inline_inode(inode); 2383 if (err) 2384 goto fail; 2385 } 2386 repeat: 2387 /* 2388 * Do not use grab_cache_page_write_begin() to avoid deadlock due to 2389 * wait_for_stable_page. Will wait that below with our IO control. 2390 */ 2391 page = f2fs_pagecache_get_page(mapping, index, 2392 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS); 2393 if (!page) { 2394 err = -ENOMEM; 2395 goto fail; 2396 } 2397 2398 *pagep = page; 2399 2400 err = prepare_write_begin(sbi, page, pos, len, 2401 &blkaddr, &need_balance); 2402 if (err) 2403 goto fail; 2404 2405 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) { 2406 unlock_page(page); 2407 f2fs_balance_fs(sbi, true); 2408 lock_page(page); 2409 if (page->mapping != mapping) { 2410 /* The page got truncated from under us */ 2411 f2fs_put_page(page, 1); 2412 goto repeat; 2413 } 2414 } 2415 2416 f2fs_wait_on_page_writeback(page, DATA, false); 2417 2418 if (len == PAGE_SIZE || PageUptodate(page)) 2419 return 0; 2420 2421 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) { 2422 zero_user_segment(page, len, PAGE_SIZE); 2423 return 0; 2424 } 2425 2426 if (blkaddr == NEW_ADDR) { 2427 zero_user_segment(page, 0, PAGE_SIZE); 2428 SetPageUptodate(page); 2429 } else { 2430 err = f2fs_submit_page_read(inode, page, blkaddr); 2431 if (err) 2432 goto fail; 2433 2434 lock_page(page); 2435 if (unlikely(page->mapping != mapping)) { 2436 f2fs_put_page(page, 1); 2437 goto repeat; 2438 } 2439 if (unlikely(!PageUptodate(page))) { 2440 err = -EIO; 2441 goto fail; 2442 } 2443 } 2444 return 0; 2445 2446 fail: 2447 f2fs_put_page(page, 1); 2448 f2fs_write_failed(mapping, pos + len); 2449 if (drop_atomic) 2450 f2fs_drop_inmem_pages_all(sbi, false); 2451 return err; 2452 } 2453 2454 static int f2fs_write_end(struct file *file, 2455 struct address_space *mapping, 2456 loff_t pos, unsigned len, unsigned copied, 2457 struct page *page, void *fsdata) 2458 { 2459 struct inode *inode = page->mapping->host; 2460 2461 trace_f2fs_write_end(inode, pos, len, copied); 2462 2463 /* 2464 * This should be come from len == PAGE_SIZE, and we expect copied 2465 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and 2466 * let generic_perform_write() try to copy data again through copied=0. 2467 */ 2468 if (!PageUptodate(page)) { 2469 if (unlikely(copied != len)) 2470 copied = 0; 2471 else 2472 SetPageUptodate(page); 2473 } 2474 if (!copied) 2475 goto unlock_out; 2476 2477 set_page_dirty(page); 2478 2479 if (pos + copied > i_size_read(inode)) 2480 f2fs_i_size_write(inode, pos + copied); 2481 unlock_out: 2482 f2fs_put_page(page, 1); 2483 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 2484 return copied; 2485 } 2486 2487 static int check_direct_IO(struct inode *inode, struct iov_iter *iter, 2488 loff_t offset) 2489 { 2490 unsigned i_blkbits = READ_ONCE(inode->i_blkbits); 2491 unsigned blkbits = i_blkbits; 2492 unsigned blocksize_mask = (1 << blkbits) - 1; 2493 unsigned long align = offset | iov_iter_alignment(iter); 2494 struct block_device *bdev = inode->i_sb->s_bdev; 2495 2496 if (align & blocksize_mask) { 2497 if (bdev) 2498 blkbits = blksize_bits(bdev_logical_block_size(bdev)); 2499 blocksize_mask = (1 << blkbits) - 1; 2500 if (align & blocksize_mask) 2501 return -EINVAL; 2502 return 1; 2503 } 2504 return 0; 2505 } 2506 2507 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 2508 { 2509 struct address_space *mapping = iocb->ki_filp->f_mapping; 2510 struct inode *inode = mapping->host; 2511 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2512 struct f2fs_inode_info *fi = F2FS_I(inode); 2513 size_t count = iov_iter_count(iter); 2514 loff_t offset = iocb->ki_pos; 2515 int rw = iov_iter_rw(iter); 2516 int err; 2517 enum rw_hint hint = iocb->ki_hint; 2518 int whint_mode = F2FS_OPTION(sbi).whint_mode; 2519 bool do_opu; 2520 2521 err = check_direct_IO(inode, iter, offset); 2522 if (err) 2523 return err < 0 ? err : 0; 2524 2525 if (f2fs_force_buffered_io(inode, iocb, iter)) 2526 return 0; 2527 2528 do_opu = allow_outplace_dio(inode, iocb, iter); 2529 2530 trace_f2fs_direct_IO_enter(inode, offset, count, rw); 2531 2532 if (rw == WRITE && whint_mode == WHINT_MODE_OFF) 2533 iocb->ki_hint = WRITE_LIFE_NOT_SET; 2534 2535 if (iocb->ki_flags & IOCB_NOWAIT) { 2536 if (!down_read_trylock(&fi->i_gc_rwsem[rw])) { 2537 iocb->ki_hint = hint; 2538 err = -EAGAIN; 2539 goto out; 2540 } 2541 if (do_opu && !down_read_trylock(&fi->i_gc_rwsem[READ])) { 2542 up_read(&fi->i_gc_rwsem[rw]); 2543 iocb->ki_hint = hint; 2544 err = -EAGAIN; 2545 goto out; 2546 } 2547 } else { 2548 down_read(&fi->i_gc_rwsem[rw]); 2549 if (do_opu) 2550 down_read(&fi->i_gc_rwsem[READ]); 2551 } 2552 2553 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio); 2554 2555 if (do_opu) 2556 up_read(&fi->i_gc_rwsem[READ]); 2557 2558 up_read(&fi->i_gc_rwsem[rw]); 2559 2560 if (rw == WRITE) { 2561 if (whint_mode == WHINT_MODE_OFF) 2562 iocb->ki_hint = hint; 2563 if (err > 0) { 2564 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO, 2565 err); 2566 if (!do_opu) 2567 set_inode_flag(inode, FI_UPDATE_WRITE); 2568 } else if (err < 0) { 2569 f2fs_write_failed(mapping, offset + count); 2570 } 2571 } 2572 2573 out: 2574 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err); 2575 2576 return err; 2577 } 2578 2579 void f2fs_invalidate_page(struct page *page, unsigned int offset, 2580 unsigned int length) 2581 { 2582 struct inode *inode = page->mapping->host; 2583 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2584 2585 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 2586 (offset % PAGE_SIZE || length != PAGE_SIZE)) 2587 return; 2588 2589 if (PageDirty(page)) { 2590 if (inode->i_ino == F2FS_META_INO(sbi)) { 2591 dec_page_count(sbi, F2FS_DIRTY_META); 2592 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { 2593 dec_page_count(sbi, F2FS_DIRTY_NODES); 2594 } else { 2595 inode_dec_dirty_pages(inode); 2596 f2fs_remove_dirty_inode(inode); 2597 } 2598 } 2599 2600 /* This is atomic written page, keep Private */ 2601 if (IS_ATOMIC_WRITTEN_PAGE(page)) 2602 return f2fs_drop_inmem_page(inode, page); 2603 2604 set_page_private(page, 0); 2605 ClearPagePrivate(page); 2606 } 2607 2608 int f2fs_release_page(struct page *page, gfp_t wait) 2609 { 2610 /* If this is dirty page, keep PagePrivate */ 2611 if (PageDirty(page)) 2612 return 0; 2613 2614 /* This is atomic written page, keep Private */ 2615 if (IS_ATOMIC_WRITTEN_PAGE(page)) 2616 return 0; 2617 2618 set_page_private(page, 0); 2619 ClearPagePrivate(page); 2620 return 1; 2621 } 2622 2623 static int f2fs_set_data_page_dirty(struct page *page) 2624 { 2625 struct address_space *mapping = page->mapping; 2626 struct inode *inode = mapping->host; 2627 2628 trace_f2fs_set_page_dirty(page, DATA); 2629 2630 if (!PageUptodate(page)) 2631 SetPageUptodate(page); 2632 2633 /* don't remain PG_checked flag which was set during GC */ 2634 if (is_cold_data(page)) 2635 clear_cold_data(page); 2636 2637 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) { 2638 if (!IS_ATOMIC_WRITTEN_PAGE(page)) { 2639 f2fs_register_inmem_page(inode, page); 2640 return 1; 2641 } 2642 /* 2643 * Previously, this page has been registered, we just 2644 * return here. 2645 */ 2646 return 0; 2647 } 2648 2649 if (!PageDirty(page)) { 2650 __set_page_dirty_nobuffers(page); 2651 f2fs_update_dirty_page(inode, page); 2652 return 1; 2653 } 2654 return 0; 2655 } 2656 2657 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 2658 { 2659 struct inode *inode = mapping->host; 2660 2661 if (f2fs_has_inline_data(inode)) 2662 return 0; 2663 2664 /* make sure allocating whole blocks */ 2665 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 2666 filemap_write_and_wait(mapping); 2667 2668 return generic_block_bmap(mapping, block, get_data_block_bmap); 2669 } 2670 2671 #ifdef CONFIG_MIGRATION 2672 #include <linux/migrate.h> 2673 2674 int f2fs_migrate_page(struct address_space *mapping, 2675 struct page *newpage, struct page *page, enum migrate_mode mode) 2676 { 2677 int rc, extra_count; 2678 struct f2fs_inode_info *fi = F2FS_I(mapping->host); 2679 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page); 2680 2681 BUG_ON(PageWriteback(page)); 2682 2683 /* migrating an atomic written page is safe with the inmem_lock hold */ 2684 if (atomic_written) { 2685 if (mode != MIGRATE_SYNC) 2686 return -EBUSY; 2687 if (!mutex_trylock(&fi->inmem_lock)) 2688 return -EAGAIN; 2689 } 2690 2691 /* 2692 * A reference is expected if PagePrivate set when move mapping, 2693 * however F2FS breaks this for maintaining dirty page counts when 2694 * truncating pages. So here adjusting the 'extra_count' make it work. 2695 */ 2696 extra_count = (atomic_written ? 1 : 0) - page_has_private(page); 2697 rc = migrate_page_move_mapping(mapping, newpage, 2698 page, NULL, mode, extra_count); 2699 if (rc != MIGRATEPAGE_SUCCESS) { 2700 if (atomic_written) 2701 mutex_unlock(&fi->inmem_lock); 2702 return rc; 2703 } 2704 2705 if (atomic_written) { 2706 struct inmem_pages *cur; 2707 list_for_each_entry(cur, &fi->inmem_pages, list) 2708 if (cur->page == page) { 2709 cur->page = newpage; 2710 break; 2711 } 2712 mutex_unlock(&fi->inmem_lock); 2713 put_page(page); 2714 get_page(newpage); 2715 } 2716 2717 if (PagePrivate(page)) 2718 SetPagePrivate(newpage); 2719 set_page_private(newpage, page_private(page)); 2720 2721 if (mode != MIGRATE_SYNC_NO_COPY) 2722 migrate_page_copy(newpage, page); 2723 else 2724 migrate_page_states(newpage, page); 2725 2726 return MIGRATEPAGE_SUCCESS; 2727 } 2728 #endif 2729 2730 const struct address_space_operations f2fs_dblock_aops = { 2731 .readpage = f2fs_read_data_page, 2732 .readpages = f2fs_read_data_pages, 2733 .writepage = f2fs_write_data_page, 2734 .writepages = f2fs_write_data_pages, 2735 .write_begin = f2fs_write_begin, 2736 .write_end = f2fs_write_end, 2737 .set_page_dirty = f2fs_set_data_page_dirty, 2738 .invalidatepage = f2fs_invalidate_page, 2739 .releasepage = f2fs_release_page, 2740 .direct_IO = f2fs_direct_IO, 2741 .bmap = f2fs_bmap, 2742 #ifdef CONFIG_MIGRATION 2743 .migratepage = f2fs_migrate_page, 2744 #endif 2745 }; 2746 2747 void f2fs_clear_radix_tree_dirty_tag(struct page *page) 2748 { 2749 struct address_space *mapping = page_mapping(page); 2750 unsigned long flags; 2751 2752 xa_lock_irqsave(&mapping->i_pages, flags); 2753 radix_tree_tag_clear(&mapping->i_pages, page_index(page), 2754 PAGECACHE_TAG_DIRTY); 2755 xa_unlock_irqrestore(&mapping->i_pages, flags); 2756 } 2757 2758 int __init f2fs_init_post_read_processing(void) 2759 { 2760 bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, 0); 2761 if (!bio_post_read_ctx_cache) 2762 goto fail; 2763 bio_post_read_ctx_pool = 2764 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS, 2765 bio_post_read_ctx_cache); 2766 if (!bio_post_read_ctx_pool) 2767 goto fail_free_cache; 2768 return 0; 2769 2770 fail_free_cache: 2771 kmem_cache_destroy(bio_post_read_ctx_cache); 2772 fail: 2773 return -ENOMEM; 2774 } 2775 2776 void __exit f2fs_destroy_post_read_processing(void) 2777 { 2778 mempool_destroy(bio_post_read_ctx_pool); 2779 kmem_cache_destroy(bio_post_read_ctx_cache); 2780 } 2781