1 /* 2 * linux/fs/ext4/inode.c 3 * 4 * Copyright (C) 1992, 1993, 1994, 1995 5 * Remy Card (card@masi.ibp.fr) 6 * Laboratoire MASI - Institut Blaise Pascal 7 * Universite Pierre et Marie Curie (Paris VI) 8 * 9 * from 10 * 11 * linux/fs/minix/inode.c 12 * 13 * Copyright (C) 1991, 1992 Linus Torvalds 14 * 15 * 64-bit file support on 64-bit platforms by Jakub Jelinek 16 * (jj@sunsite.ms.mff.cuni.cz) 17 * 18 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000 19 */ 20 21 #include <linux/fs.h> 22 #include <linux/time.h> 23 #include <linux/jbd2.h> 24 #include <linux/highuid.h> 25 #include <linux/pagemap.h> 26 #include <linux/quotaops.h> 27 #include <linux/string.h> 28 #include <linux/buffer_head.h> 29 #include <linux/writeback.h> 30 #include <linux/pagevec.h> 31 #include <linux/mpage.h> 32 #include <linux/namei.h> 33 #include <linux/uio.h> 34 #include <linux/bio.h> 35 #include <linux/workqueue.h> 36 #include <linux/kernel.h> 37 #include <linux/printk.h> 38 #include <linux/slab.h> 39 #include <linux/ratelimit.h> 40 41 #include "ext4_jbd2.h" 42 #include "xattr.h" 43 #include "acl.h" 44 #include "truncate.h" 45 46 #include <trace/events/ext4.h> 47 48 #define MPAGE_DA_EXTENT_TAIL 0x01 49 50 static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw, 51 struct ext4_inode_info *ei) 52 { 53 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 54 __u16 csum_lo; 55 __u16 csum_hi = 0; 56 __u32 csum; 57 58 csum_lo = raw->i_checksum_lo; 59 raw->i_checksum_lo = 0; 60 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 61 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) { 62 csum_hi = raw->i_checksum_hi; 63 raw->i_checksum_hi = 0; 64 } 65 66 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, 67 EXT4_INODE_SIZE(inode->i_sb)); 68 69 raw->i_checksum_lo = csum_lo; 70 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 71 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 72 raw->i_checksum_hi = csum_hi; 73 74 return csum; 75 } 76 77 static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw, 78 struct ext4_inode_info *ei) 79 { 80 __u32 provided, calculated; 81 82 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 83 cpu_to_le32(EXT4_OS_LINUX) || 84 !EXT4_HAS_RO_COMPAT_FEATURE(inode->i_sb, 85 EXT4_FEATURE_RO_COMPAT_METADATA_CSUM)) 86 return 1; 87 88 provided = le16_to_cpu(raw->i_checksum_lo); 89 calculated = ext4_inode_csum(inode, raw, ei); 90 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 91 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 92 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16; 93 else 94 calculated &= 0xFFFF; 95 96 return provided == calculated; 97 } 98 99 static void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, 100 struct ext4_inode_info *ei) 101 { 102 __u32 csum; 103 104 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 105 cpu_to_le32(EXT4_OS_LINUX) || 106 !EXT4_HAS_RO_COMPAT_FEATURE(inode->i_sb, 107 EXT4_FEATURE_RO_COMPAT_METADATA_CSUM)) 108 return; 109 110 csum = ext4_inode_csum(inode, raw, ei); 111 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF); 112 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 113 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 114 raw->i_checksum_hi = cpu_to_le16(csum >> 16); 115 } 116 117 static inline int ext4_begin_ordered_truncate(struct inode *inode, 118 loff_t new_size) 119 { 120 trace_ext4_begin_ordered_truncate(inode, new_size); 121 /* 122 * If jinode is zero, then we never opened the file for 123 * writing, so there's no need to call 124 * jbd2_journal_begin_ordered_truncate() since there's no 125 * outstanding writes we need to flush. 126 */ 127 if (!EXT4_I(inode)->jinode) 128 return 0; 129 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), 130 EXT4_I(inode)->jinode, 131 new_size); 132 } 133 134 static void ext4_invalidatepage(struct page *page, unsigned long offset); 135 static int noalloc_get_block_write(struct inode *inode, sector_t iblock, 136 struct buffer_head *bh_result, int create); 137 static int ext4_set_bh_endio(struct buffer_head *bh, struct inode *inode); 138 static void ext4_end_io_buffer_write(struct buffer_head *bh, int uptodate); 139 static int __ext4_journalled_writepage(struct page *page, unsigned int len); 140 static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh); 141 static int ext4_discard_partial_page_buffers_no_lock(handle_t *handle, 142 struct inode *inode, struct page *page, loff_t from, 143 loff_t length, int flags); 144 145 /* 146 * Test whether an inode is a fast symlink. 147 */ 148 static int ext4_inode_is_fast_symlink(struct inode *inode) 149 { 150 int ea_blocks = EXT4_I(inode)->i_file_acl ? 151 (inode->i_sb->s_blocksize >> 9) : 0; 152 153 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0); 154 } 155 156 /* 157 * Restart the transaction associated with *handle. This does a commit, 158 * so before we call here everything must be consistently dirtied against 159 * this transaction. 160 */ 161 int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode, 162 int nblocks) 163 { 164 int ret; 165 166 /* 167 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this 168 * moment, get_block can be called only for blocks inside i_size since 169 * page cache has been already dropped and writes are blocked by 170 * i_mutex. So we can safely drop the i_data_sem here. 171 */ 172 BUG_ON(EXT4_JOURNAL(inode) == NULL); 173 jbd_debug(2, "restarting handle %p\n", handle); 174 up_write(&EXT4_I(inode)->i_data_sem); 175 ret = ext4_journal_restart(handle, nblocks); 176 down_write(&EXT4_I(inode)->i_data_sem); 177 ext4_discard_preallocations(inode); 178 179 return ret; 180 } 181 182 /* 183 * Called at the last iput() if i_nlink is zero. 184 */ 185 void ext4_evict_inode(struct inode *inode) 186 { 187 handle_t *handle; 188 int err; 189 190 trace_ext4_evict_inode(inode); 191 192 ext4_ioend_wait(inode); 193 194 if (inode->i_nlink) { 195 /* 196 * When journalling data dirty buffers are tracked only in the 197 * journal. So although mm thinks everything is clean and 198 * ready for reaping the inode might still have some pages to 199 * write in the running transaction or waiting to be 200 * checkpointed. Thus calling jbd2_journal_invalidatepage() 201 * (via truncate_inode_pages()) to discard these buffers can 202 * cause data loss. Also even if we did not discard these 203 * buffers, we would have no way to find them after the inode 204 * is reaped and thus user could see stale data if he tries to 205 * read them before the transaction is checkpointed. So be 206 * careful and force everything to disk here... We use 207 * ei->i_datasync_tid to store the newest transaction 208 * containing inode's data. 209 * 210 * Note that directories do not have this problem because they 211 * don't use page cache. 212 */ 213 if (ext4_should_journal_data(inode) && 214 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode))) { 215 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 216 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid; 217 218 jbd2_log_start_commit(journal, commit_tid); 219 jbd2_log_wait_commit(journal, commit_tid); 220 filemap_write_and_wait(&inode->i_data); 221 } 222 truncate_inode_pages(&inode->i_data, 0); 223 goto no_delete; 224 } 225 226 if (!is_bad_inode(inode)) 227 dquot_initialize(inode); 228 229 if (ext4_should_order_data(inode)) 230 ext4_begin_ordered_truncate(inode, 0); 231 truncate_inode_pages(&inode->i_data, 0); 232 233 if (is_bad_inode(inode)) 234 goto no_delete; 235 236 /* 237 * Protect us against freezing - iput() caller didn't have to have any 238 * protection against it 239 */ 240 sb_start_intwrite(inode->i_sb); 241 handle = ext4_journal_start(inode, ext4_blocks_for_truncate(inode)+3); 242 if (IS_ERR(handle)) { 243 ext4_std_error(inode->i_sb, PTR_ERR(handle)); 244 /* 245 * If we're going to skip the normal cleanup, we still need to 246 * make sure that the in-core orphan linked list is properly 247 * cleaned up. 248 */ 249 ext4_orphan_del(NULL, inode); 250 sb_end_intwrite(inode->i_sb); 251 goto no_delete; 252 } 253 254 if (IS_SYNC(inode)) 255 ext4_handle_sync(handle); 256 inode->i_size = 0; 257 err = ext4_mark_inode_dirty(handle, inode); 258 if (err) { 259 ext4_warning(inode->i_sb, 260 "couldn't mark inode dirty (err %d)", err); 261 goto stop_handle; 262 } 263 if (inode->i_blocks) 264 ext4_truncate(inode); 265 266 /* 267 * ext4_ext_truncate() doesn't reserve any slop when it 268 * restarts journal transactions; therefore there may not be 269 * enough credits left in the handle to remove the inode from 270 * the orphan list and set the dtime field. 271 */ 272 if (!ext4_handle_has_enough_credits(handle, 3)) { 273 err = ext4_journal_extend(handle, 3); 274 if (err > 0) 275 err = ext4_journal_restart(handle, 3); 276 if (err != 0) { 277 ext4_warning(inode->i_sb, 278 "couldn't extend journal (err %d)", err); 279 stop_handle: 280 ext4_journal_stop(handle); 281 ext4_orphan_del(NULL, inode); 282 sb_end_intwrite(inode->i_sb); 283 goto no_delete; 284 } 285 } 286 287 /* 288 * Kill off the orphan record which ext4_truncate created. 289 * AKPM: I think this can be inside the above `if'. 290 * Note that ext4_orphan_del() has to be able to cope with the 291 * deletion of a non-existent orphan - this is because we don't 292 * know if ext4_truncate() actually created an orphan record. 293 * (Well, we could do this if we need to, but heck - it works) 294 */ 295 ext4_orphan_del(handle, inode); 296 EXT4_I(inode)->i_dtime = get_seconds(); 297 298 /* 299 * One subtle ordering requirement: if anything has gone wrong 300 * (transaction abort, IO errors, whatever), then we can still 301 * do these next steps (the fs will already have been marked as 302 * having errors), but we can't free the inode if the mark_dirty 303 * fails. 304 */ 305 if (ext4_mark_inode_dirty(handle, inode)) 306 /* If that failed, just do the required in-core inode clear. */ 307 ext4_clear_inode(inode); 308 else 309 ext4_free_inode(handle, inode); 310 ext4_journal_stop(handle); 311 sb_end_intwrite(inode->i_sb); 312 return; 313 no_delete: 314 ext4_clear_inode(inode); /* We must guarantee clearing of inode... */ 315 } 316 317 #ifdef CONFIG_QUOTA 318 qsize_t *ext4_get_reserved_space(struct inode *inode) 319 { 320 return &EXT4_I(inode)->i_reserved_quota; 321 } 322 #endif 323 324 /* 325 * Calculate the number of metadata blocks need to reserve 326 * to allocate a block located at @lblock 327 */ 328 static int ext4_calc_metadata_amount(struct inode *inode, ext4_lblk_t lblock) 329 { 330 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 331 return ext4_ext_calc_metadata_amount(inode, lblock); 332 333 return ext4_ind_calc_metadata_amount(inode, lblock); 334 } 335 336 /* 337 * Called with i_data_sem down, which is important since we can call 338 * ext4_discard_preallocations() from here. 339 */ 340 void ext4_da_update_reserve_space(struct inode *inode, 341 int used, int quota_claim) 342 { 343 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 344 struct ext4_inode_info *ei = EXT4_I(inode); 345 346 spin_lock(&ei->i_block_reservation_lock); 347 trace_ext4_da_update_reserve_space(inode, used, quota_claim); 348 if (unlikely(used > ei->i_reserved_data_blocks)) { 349 ext4_msg(inode->i_sb, KERN_NOTICE, "%s: ino %lu, used %d " 350 "with only %d reserved data blocks", 351 __func__, inode->i_ino, used, 352 ei->i_reserved_data_blocks); 353 WARN_ON(1); 354 used = ei->i_reserved_data_blocks; 355 } 356 357 if (unlikely(ei->i_allocated_meta_blocks > ei->i_reserved_meta_blocks)) { 358 ext4_msg(inode->i_sb, KERN_NOTICE, "%s: ino %lu, allocated %d " 359 "with only %d reserved metadata blocks\n", __func__, 360 inode->i_ino, ei->i_allocated_meta_blocks, 361 ei->i_reserved_meta_blocks); 362 WARN_ON(1); 363 ei->i_allocated_meta_blocks = ei->i_reserved_meta_blocks; 364 } 365 366 /* Update per-inode reservations */ 367 ei->i_reserved_data_blocks -= used; 368 ei->i_reserved_meta_blocks -= ei->i_allocated_meta_blocks; 369 percpu_counter_sub(&sbi->s_dirtyclusters_counter, 370 used + ei->i_allocated_meta_blocks); 371 ei->i_allocated_meta_blocks = 0; 372 373 if (ei->i_reserved_data_blocks == 0) { 374 /* 375 * We can release all of the reserved metadata blocks 376 * only when we have written all of the delayed 377 * allocation blocks. 378 */ 379 percpu_counter_sub(&sbi->s_dirtyclusters_counter, 380 ei->i_reserved_meta_blocks); 381 ei->i_reserved_meta_blocks = 0; 382 ei->i_da_metadata_calc_len = 0; 383 } 384 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 385 386 /* Update quota subsystem for data blocks */ 387 if (quota_claim) 388 dquot_claim_block(inode, EXT4_C2B(sbi, used)); 389 else { 390 /* 391 * We did fallocate with an offset that is already delayed 392 * allocated. So on delayed allocated writeback we should 393 * not re-claim the quota for fallocated blocks. 394 */ 395 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used)); 396 } 397 398 /* 399 * If we have done all the pending block allocations and if 400 * there aren't any writers on the inode, we can discard the 401 * inode's preallocations. 402 */ 403 if ((ei->i_reserved_data_blocks == 0) && 404 (atomic_read(&inode->i_writecount) == 0)) 405 ext4_discard_preallocations(inode); 406 } 407 408 static int __check_block_validity(struct inode *inode, const char *func, 409 unsigned int line, 410 struct ext4_map_blocks *map) 411 { 412 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), map->m_pblk, 413 map->m_len)) { 414 ext4_error_inode(inode, func, line, map->m_pblk, 415 "lblock %lu mapped to illegal pblock " 416 "(length %d)", (unsigned long) map->m_lblk, 417 map->m_len); 418 return -EIO; 419 } 420 return 0; 421 } 422 423 #define check_block_validity(inode, map) \ 424 __check_block_validity((inode), __func__, __LINE__, (map)) 425 426 /* 427 * Return the number of contiguous dirty pages in a given inode 428 * starting at page frame idx. 429 */ 430 static pgoff_t ext4_num_dirty_pages(struct inode *inode, pgoff_t idx, 431 unsigned int max_pages) 432 { 433 struct address_space *mapping = inode->i_mapping; 434 pgoff_t index; 435 struct pagevec pvec; 436 pgoff_t num = 0; 437 int i, nr_pages, done = 0; 438 439 if (max_pages == 0) 440 return 0; 441 pagevec_init(&pvec, 0); 442 while (!done) { 443 index = idx; 444 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 445 PAGECACHE_TAG_DIRTY, 446 (pgoff_t)PAGEVEC_SIZE); 447 if (nr_pages == 0) 448 break; 449 for (i = 0; i < nr_pages; i++) { 450 struct page *page = pvec.pages[i]; 451 struct buffer_head *bh, *head; 452 453 lock_page(page); 454 if (unlikely(page->mapping != mapping) || 455 !PageDirty(page) || 456 PageWriteback(page) || 457 page->index != idx) { 458 done = 1; 459 unlock_page(page); 460 break; 461 } 462 if (page_has_buffers(page)) { 463 bh = head = page_buffers(page); 464 do { 465 if (!buffer_delay(bh) && 466 !buffer_unwritten(bh)) 467 done = 1; 468 bh = bh->b_this_page; 469 } while (!done && (bh != head)); 470 } 471 unlock_page(page); 472 if (done) 473 break; 474 idx++; 475 num++; 476 if (num >= max_pages) { 477 done = 1; 478 break; 479 } 480 } 481 pagevec_release(&pvec); 482 } 483 return num; 484 } 485 486 /* 487 * The ext4_map_blocks() function tries to look up the requested blocks, 488 * and returns if the blocks are already mapped. 489 * 490 * Otherwise it takes the write lock of the i_data_sem and allocate blocks 491 * and store the allocated blocks in the result buffer head and mark it 492 * mapped. 493 * 494 * If file type is extents based, it will call ext4_ext_map_blocks(), 495 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping 496 * based files 497 * 498 * On success, it returns the number of blocks being mapped or allocate. 499 * if create==0 and the blocks are pre-allocated and uninitialized block, 500 * the result buffer head is unmapped. If the create ==1, it will make sure 501 * the buffer head is mapped. 502 * 503 * It returns 0 if plain look up failed (blocks have not been allocated), in 504 * that case, buffer head is unmapped 505 * 506 * It returns the error in case of allocation failure. 507 */ 508 int ext4_map_blocks(handle_t *handle, struct inode *inode, 509 struct ext4_map_blocks *map, int flags) 510 { 511 int retval; 512 513 map->m_flags = 0; 514 ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u," 515 "logical block %lu\n", inode->i_ino, flags, map->m_len, 516 (unsigned long) map->m_lblk); 517 /* 518 * Try to see if we can get the block without requesting a new 519 * file system block. 520 */ 521 if (!(flags & EXT4_GET_BLOCKS_NO_LOCK)) 522 down_read((&EXT4_I(inode)->i_data_sem)); 523 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 524 retval = ext4_ext_map_blocks(handle, inode, map, flags & 525 EXT4_GET_BLOCKS_KEEP_SIZE); 526 } else { 527 retval = ext4_ind_map_blocks(handle, inode, map, flags & 528 EXT4_GET_BLOCKS_KEEP_SIZE); 529 } 530 if (!(flags & EXT4_GET_BLOCKS_NO_LOCK)) 531 up_read((&EXT4_I(inode)->i_data_sem)); 532 533 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 534 int ret; 535 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) { 536 /* delayed alloc may be allocated by fallocate and 537 * coverted to initialized by directIO. 538 * we need to handle delayed extent here. 539 */ 540 down_write((&EXT4_I(inode)->i_data_sem)); 541 goto delayed_mapped; 542 } 543 ret = check_block_validity(inode, map); 544 if (ret != 0) 545 return ret; 546 } 547 548 /* If it is only a block(s) look up */ 549 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) 550 return retval; 551 552 /* 553 * Returns if the blocks have already allocated 554 * 555 * Note that if blocks have been preallocated 556 * ext4_ext_get_block() returns the create = 0 557 * with buffer head unmapped. 558 */ 559 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) 560 return retval; 561 562 /* 563 * When we call get_blocks without the create flag, the 564 * BH_Unwritten flag could have gotten set if the blocks 565 * requested were part of a uninitialized extent. We need to 566 * clear this flag now that we are committed to convert all or 567 * part of the uninitialized extent to be an initialized 568 * extent. This is because we need to avoid the combination 569 * of BH_Unwritten and BH_Mapped flags being simultaneously 570 * set on the buffer_head. 571 */ 572 map->m_flags &= ~EXT4_MAP_UNWRITTEN; 573 574 /* 575 * New blocks allocate and/or writing to uninitialized extent 576 * will possibly result in updating i_data, so we take 577 * the write lock of i_data_sem, and call get_blocks() 578 * with create == 1 flag. 579 */ 580 down_write((&EXT4_I(inode)->i_data_sem)); 581 582 /* 583 * if the caller is from delayed allocation writeout path 584 * we have already reserved fs blocks for allocation 585 * let the underlying get_block() function know to 586 * avoid double accounting 587 */ 588 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) 589 ext4_set_inode_state(inode, EXT4_STATE_DELALLOC_RESERVED); 590 /* 591 * We need to check for EXT4 here because migrate 592 * could have changed the inode type in between 593 */ 594 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 595 retval = ext4_ext_map_blocks(handle, inode, map, flags); 596 } else { 597 retval = ext4_ind_map_blocks(handle, inode, map, flags); 598 599 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) { 600 /* 601 * We allocated new blocks which will result in 602 * i_data's format changing. Force the migrate 603 * to fail by clearing migrate flags 604 */ 605 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE); 606 } 607 608 /* 609 * Update reserved blocks/metadata blocks after successful 610 * block allocation which had been deferred till now. We don't 611 * support fallocate for non extent files. So we can update 612 * reserve space here. 613 */ 614 if ((retval > 0) && 615 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)) 616 ext4_da_update_reserve_space(inode, retval, 1); 617 } 618 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) { 619 ext4_clear_inode_state(inode, EXT4_STATE_DELALLOC_RESERVED); 620 621 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 622 int ret; 623 delayed_mapped: 624 /* delayed allocation blocks has been allocated */ 625 ret = ext4_es_remove_extent(inode, map->m_lblk, 626 map->m_len); 627 if (ret < 0) 628 retval = ret; 629 } 630 } 631 632 up_write((&EXT4_I(inode)->i_data_sem)); 633 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 634 int ret = check_block_validity(inode, map); 635 if (ret != 0) 636 return ret; 637 } 638 return retval; 639 } 640 641 /* Maximum number of blocks we map for direct IO at once. */ 642 #define DIO_MAX_BLOCKS 4096 643 644 static int _ext4_get_block(struct inode *inode, sector_t iblock, 645 struct buffer_head *bh, int flags) 646 { 647 handle_t *handle = ext4_journal_current_handle(); 648 struct ext4_map_blocks map; 649 int ret = 0, started = 0; 650 int dio_credits; 651 652 if (ext4_has_inline_data(inode)) 653 return -ERANGE; 654 655 map.m_lblk = iblock; 656 map.m_len = bh->b_size >> inode->i_blkbits; 657 658 if (flags && !(flags & EXT4_GET_BLOCKS_NO_LOCK) && !handle) { 659 /* Direct IO write... */ 660 if (map.m_len > DIO_MAX_BLOCKS) 661 map.m_len = DIO_MAX_BLOCKS; 662 dio_credits = ext4_chunk_trans_blocks(inode, map.m_len); 663 handle = ext4_journal_start(inode, dio_credits); 664 if (IS_ERR(handle)) { 665 ret = PTR_ERR(handle); 666 return ret; 667 } 668 started = 1; 669 } 670 671 ret = ext4_map_blocks(handle, inode, &map, flags); 672 if (ret > 0) { 673 map_bh(bh, inode->i_sb, map.m_pblk); 674 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | map.m_flags; 675 bh->b_size = inode->i_sb->s_blocksize * map.m_len; 676 ret = 0; 677 } 678 if (started) 679 ext4_journal_stop(handle); 680 return ret; 681 } 682 683 int ext4_get_block(struct inode *inode, sector_t iblock, 684 struct buffer_head *bh, int create) 685 { 686 return _ext4_get_block(inode, iblock, bh, 687 create ? EXT4_GET_BLOCKS_CREATE : 0); 688 } 689 690 /* 691 * `handle' can be NULL if create is zero 692 */ 693 struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode, 694 ext4_lblk_t block, int create, int *errp) 695 { 696 struct ext4_map_blocks map; 697 struct buffer_head *bh; 698 int fatal = 0, err; 699 700 J_ASSERT(handle != NULL || create == 0); 701 702 map.m_lblk = block; 703 map.m_len = 1; 704 err = ext4_map_blocks(handle, inode, &map, 705 create ? EXT4_GET_BLOCKS_CREATE : 0); 706 707 /* ensure we send some value back into *errp */ 708 *errp = 0; 709 710 if (err < 0) 711 *errp = err; 712 if (err <= 0) 713 return NULL; 714 715 bh = sb_getblk(inode->i_sb, map.m_pblk); 716 if (!bh) { 717 *errp = -EIO; 718 return NULL; 719 } 720 if (map.m_flags & EXT4_MAP_NEW) { 721 J_ASSERT(create != 0); 722 J_ASSERT(handle != NULL); 723 724 /* 725 * Now that we do not always journal data, we should 726 * keep in mind whether this should always journal the 727 * new buffer as metadata. For now, regular file 728 * writes use ext4_get_block instead, so it's not a 729 * problem. 730 */ 731 lock_buffer(bh); 732 BUFFER_TRACE(bh, "call get_create_access"); 733 fatal = ext4_journal_get_create_access(handle, bh); 734 if (!fatal && !buffer_uptodate(bh)) { 735 memset(bh->b_data, 0, inode->i_sb->s_blocksize); 736 set_buffer_uptodate(bh); 737 } 738 unlock_buffer(bh); 739 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 740 err = ext4_handle_dirty_metadata(handle, inode, bh); 741 if (!fatal) 742 fatal = err; 743 } else { 744 BUFFER_TRACE(bh, "not a new buffer"); 745 } 746 if (fatal) { 747 *errp = fatal; 748 brelse(bh); 749 bh = NULL; 750 } 751 return bh; 752 } 753 754 struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode, 755 ext4_lblk_t block, int create, int *err) 756 { 757 struct buffer_head *bh; 758 759 bh = ext4_getblk(handle, inode, block, create, err); 760 if (!bh) 761 return bh; 762 if (buffer_uptodate(bh)) 763 return bh; 764 ll_rw_block(READ | REQ_META | REQ_PRIO, 1, &bh); 765 wait_on_buffer(bh); 766 if (buffer_uptodate(bh)) 767 return bh; 768 put_bh(bh); 769 *err = -EIO; 770 return NULL; 771 } 772 773 int ext4_walk_page_buffers(handle_t *handle, 774 struct buffer_head *head, 775 unsigned from, 776 unsigned to, 777 int *partial, 778 int (*fn)(handle_t *handle, 779 struct buffer_head *bh)) 780 { 781 struct buffer_head *bh; 782 unsigned block_start, block_end; 783 unsigned blocksize = head->b_size; 784 int err, ret = 0; 785 struct buffer_head *next; 786 787 for (bh = head, block_start = 0; 788 ret == 0 && (bh != head || !block_start); 789 block_start = block_end, bh = next) { 790 next = bh->b_this_page; 791 block_end = block_start + blocksize; 792 if (block_end <= from || block_start >= to) { 793 if (partial && !buffer_uptodate(bh)) 794 *partial = 1; 795 continue; 796 } 797 err = (*fn)(handle, bh); 798 if (!ret) 799 ret = err; 800 } 801 return ret; 802 } 803 804 /* 805 * To preserve ordering, it is essential that the hole instantiation and 806 * the data write be encapsulated in a single transaction. We cannot 807 * close off a transaction and start a new one between the ext4_get_block() 808 * and the commit_write(). So doing the jbd2_journal_start at the start of 809 * prepare_write() is the right place. 810 * 811 * Also, this function can nest inside ext4_writepage() -> 812 * block_write_full_page(). In that case, we *know* that ext4_writepage() 813 * has generated enough buffer credits to do the whole page. So we won't 814 * block on the journal in that case, which is good, because the caller may 815 * be PF_MEMALLOC. 816 * 817 * By accident, ext4 can be reentered when a transaction is open via 818 * quota file writes. If we were to commit the transaction while thus 819 * reentered, there can be a deadlock - we would be holding a quota 820 * lock, and the commit would never complete if another thread had a 821 * transaction open and was blocking on the quota lock - a ranking 822 * violation. 823 * 824 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start 825 * will _not_ run commit under these circumstances because handle->h_ref 826 * is elevated. We'll still have enough credits for the tiny quotafile 827 * write. 828 */ 829 int do_journal_get_write_access(handle_t *handle, 830 struct buffer_head *bh) 831 { 832 int dirty = buffer_dirty(bh); 833 int ret; 834 835 if (!buffer_mapped(bh) || buffer_freed(bh)) 836 return 0; 837 /* 838 * __block_write_begin() could have dirtied some buffers. Clean 839 * the dirty bit as jbd2_journal_get_write_access() could complain 840 * otherwise about fs integrity issues. Setting of the dirty bit 841 * by __block_write_begin() isn't a real problem here as we clear 842 * the bit before releasing a page lock and thus writeback cannot 843 * ever write the buffer. 844 */ 845 if (dirty) 846 clear_buffer_dirty(bh); 847 ret = ext4_journal_get_write_access(handle, bh); 848 if (!ret && dirty) 849 ret = ext4_handle_dirty_metadata(handle, NULL, bh); 850 return ret; 851 } 852 853 static int ext4_get_block_write_nolock(struct inode *inode, sector_t iblock, 854 struct buffer_head *bh_result, int create); 855 static int ext4_write_begin(struct file *file, struct address_space *mapping, 856 loff_t pos, unsigned len, unsigned flags, 857 struct page **pagep, void **fsdata) 858 { 859 struct inode *inode = mapping->host; 860 int ret, needed_blocks; 861 handle_t *handle; 862 int retries = 0; 863 struct page *page; 864 pgoff_t index; 865 unsigned from, to; 866 867 trace_ext4_write_begin(inode, pos, len, flags); 868 /* 869 * Reserve one block more for addition to orphan list in case 870 * we allocate blocks but write fails for some reason 871 */ 872 needed_blocks = ext4_writepage_trans_blocks(inode) + 1; 873 index = pos >> PAGE_CACHE_SHIFT; 874 from = pos & (PAGE_CACHE_SIZE - 1); 875 to = from + len; 876 877 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 878 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len, 879 flags, pagep); 880 if (ret < 0) 881 goto out; 882 if (ret == 1) { 883 ret = 0; 884 goto out; 885 } 886 } 887 888 retry: 889 handle = ext4_journal_start(inode, needed_blocks); 890 if (IS_ERR(handle)) { 891 ret = PTR_ERR(handle); 892 goto out; 893 } 894 895 /* We cannot recurse into the filesystem as the transaction is already 896 * started */ 897 flags |= AOP_FLAG_NOFS; 898 899 page = grab_cache_page_write_begin(mapping, index, flags); 900 if (!page) { 901 ext4_journal_stop(handle); 902 ret = -ENOMEM; 903 goto out; 904 } 905 906 *pagep = page; 907 908 if (ext4_should_dioread_nolock(inode)) 909 ret = __block_write_begin(page, pos, len, ext4_get_block_write); 910 else 911 ret = __block_write_begin(page, pos, len, ext4_get_block); 912 913 if (!ret && ext4_should_journal_data(inode)) { 914 ret = ext4_walk_page_buffers(handle, page_buffers(page), 915 from, to, NULL, 916 do_journal_get_write_access); 917 } 918 919 if (ret) { 920 unlock_page(page); 921 page_cache_release(page); 922 /* 923 * __block_write_begin may have instantiated a few blocks 924 * outside i_size. Trim these off again. Don't need 925 * i_size_read because we hold i_mutex. 926 * 927 * Add inode to orphan list in case we crash before 928 * truncate finishes 929 */ 930 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 931 ext4_orphan_add(handle, inode); 932 933 ext4_journal_stop(handle); 934 if (pos + len > inode->i_size) { 935 ext4_truncate_failed_write(inode); 936 /* 937 * If truncate failed early the inode might 938 * still be on the orphan list; we need to 939 * make sure the inode is removed from the 940 * orphan list in that case. 941 */ 942 if (inode->i_nlink) 943 ext4_orphan_del(NULL, inode); 944 } 945 } 946 947 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 948 goto retry; 949 out: 950 return ret; 951 } 952 953 /* For write_end() in data=journal mode */ 954 static int write_end_fn(handle_t *handle, struct buffer_head *bh) 955 { 956 if (!buffer_mapped(bh) || buffer_freed(bh)) 957 return 0; 958 set_buffer_uptodate(bh); 959 return ext4_handle_dirty_metadata(handle, NULL, bh); 960 } 961 962 static int ext4_generic_write_end(struct file *file, 963 struct address_space *mapping, 964 loff_t pos, unsigned len, unsigned copied, 965 struct page *page, void *fsdata) 966 { 967 int i_size_changed = 0; 968 struct inode *inode = mapping->host; 969 handle_t *handle = ext4_journal_current_handle(); 970 971 if (ext4_has_inline_data(inode)) 972 copied = ext4_write_inline_data_end(inode, pos, len, 973 copied, page); 974 else 975 copied = block_write_end(file, mapping, pos, 976 len, copied, page, fsdata); 977 978 /* 979 * No need to use i_size_read() here, the i_size 980 * cannot change under us because we hold i_mutex. 981 * 982 * But it's important to update i_size while still holding page lock: 983 * page writeout could otherwise come in and zero beyond i_size. 984 */ 985 if (pos + copied > inode->i_size) { 986 i_size_write(inode, pos + copied); 987 i_size_changed = 1; 988 } 989 990 if (pos + copied > EXT4_I(inode)->i_disksize) { 991 /* We need to mark inode dirty even if 992 * new_i_size is less that inode->i_size 993 * bu greater than i_disksize.(hint delalloc) 994 */ 995 ext4_update_i_disksize(inode, (pos + copied)); 996 i_size_changed = 1; 997 } 998 unlock_page(page); 999 page_cache_release(page); 1000 1001 /* 1002 * Don't mark the inode dirty under page lock. First, it unnecessarily 1003 * makes the holding time of page lock longer. Second, it forces lock 1004 * ordering of page lock and transaction start for journaling 1005 * filesystems. 1006 */ 1007 if (i_size_changed) 1008 ext4_mark_inode_dirty(handle, inode); 1009 1010 return copied; 1011 } 1012 1013 /* 1014 * We need to pick up the new inode size which generic_commit_write gave us 1015 * `file' can be NULL - eg, when called from page_symlink(). 1016 * 1017 * ext4 never places buffers on inode->i_mapping->private_list. metadata 1018 * buffers are managed internally. 1019 */ 1020 static int ext4_ordered_write_end(struct file *file, 1021 struct address_space *mapping, 1022 loff_t pos, unsigned len, unsigned copied, 1023 struct page *page, void *fsdata) 1024 { 1025 handle_t *handle = ext4_journal_current_handle(); 1026 struct inode *inode = mapping->host; 1027 int ret = 0, ret2; 1028 1029 trace_ext4_ordered_write_end(inode, pos, len, copied); 1030 ret = ext4_jbd2_file_inode(handle, inode); 1031 1032 if (ret == 0) { 1033 ret2 = ext4_generic_write_end(file, mapping, pos, len, copied, 1034 page, fsdata); 1035 copied = ret2; 1036 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1037 /* if we have allocated more blocks and copied 1038 * less. We will have blocks allocated outside 1039 * inode->i_size. So truncate them 1040 */ 1041 ext4_orphan_add(handle, inode); 1042 if (ret2 < 0) 1043 ret = ret2; 1044 } else { 1045 unlock_page(page); 1046 page_cache_release(page); 1047 } 1048 1049 ret2 = ext4_journal_stop(handle); 1050 if (!ret) 1051 ret = ret2; 1052 1053 if (pos + len > inode->i_size) { 1054 ext4_truncate_failed_write(inode); 1055 /* 1056 * If truncate failed early the inode might still be 1057 * on the orphan list; we need to make sure the inode 1058 * is removed from the orphan list in that case. 1059 */ 1060 if (inode->i_nlink) 1061 ext4_orphan_del(NULL, inode); 1062 } 1063 1064 1065 return ret ? ret : copied; 1066 } 1067 1068 static int ext4_writeback_write_end(struct file *file, 1069 struct address_space *mapping, 1070 loff_t pos, unsigned len, unsigned copied, 1071 struct page *page, void *fsdata) 1072 { 1073 handle_t *handle = ext4_journal_current_handle(); 1074 struct inode *inode = mapping->host; 1075 int ret = 0, ret2; 1076 1077 trace_ext4_writeback_write_end(inode, pos, len, copied); 1078 ret2 = ext4_generic_write_end(file, mapping, pos, len, copied, 1079 page, fsdata); 1080 copied = ret2; 1081 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1082 /* if we have allocated more blocks and copied 1083 * less. We will have blocks allocated outside 1084 * inode->i_size. So truncate them 1085 */ 1086 ext4_orphan_add(handle, inode); 1087 1088 if (ret2 < 0) 1089 ret = ret2; 1090 1091 ret2 = ext4_journal_stop(handle); 1092 if (!ret) 1093 ret = ret2; 1094 1095 if (pos + len > inode->i_size) { 1096 ext4_truncate_failed_write(inode); 1097 /* 1098 * If truncate failed early the inode might still be 1099 * on the orphan list; we need to make sure the inode 1100 * is removed from the orphan list in that case. 1101 */ 1102 if (inode->i_nlink) 1103 ext4_orphan_del(NULL, inode); 1104 } 1105 1106 return ret ? ret : copied; 1107 } 1108 1109 static int ext4_journalled_write_end(struct file *file, 1110 struct address_space *mapping, 1111 loff_t pos, unsigned len, unsigned copied, 1112 struct page *page, void *fsdata) 1113 { 1114 handle_t *handle = ext4_journal_current_handle(); 1115 struct inode *inode = mapping->host; 1116 int ret = 0, ret2; 1117 int partial = 0; 1118 unsigned from, to; 1119 loff_t new_i_size; 1120 1121 trace_ext4_journalled_write_end(inode, pos, len, copied); 1122 from = pos & (PAGE_CACHE_SIZE - 1); 1123 to = from + len; 1124 1125 BUG_ON(!ext4_handle_valid(handle)); 1126 1127 if (ext4_has_inline_data(inode)) 1128 copied = ext4_write_inline_data_end(inode, pos, len, 1129 copied, page); 1130 else { 1131 if (copied < len) { 1132 if (!PageUptodate(page)) 1133 copied = 0; 1134 page_zero_new_buffers(page, from+copied, to); 1135 } 1136 1137 ret = ext4_walk_page_buffers(handle, page_buffers(page), from, 1138 to, &partial, write_end_fn); 1139 if (!partial) 1140 SetPageUptodate(page); 1141 } 1142 new_i_size = pos + copied; 1143 if (new_i_size > inode->i_size) 1144 i_size_write(inode, pos+copied); 1145 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 1146 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 1147 if (new_i_size > EXT4_I(inode)->i_disksize) { 1148 ext4_update_i_disksize(inode, new_i_size); 1149 ret2 = ext4_mark_inode_dirty(handle, inode); 1150 if (!ret) 1151 ret = ret2; 1152 } 1153 1154 unlock_page(page); 1155 page_cache_release(page); 1156 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1157 /* if we have allocated more blocks and copied 1158 * less. We will have blocks allocated outside 1159 * inode->i_size. So truncate them 1160 */ 1161 ext4_orphan_add(handle, inode); 1162 1163 ret2 = ext4_journal_stop(handle); 1164 if (!ret) 1165 ret = ret2; 1166 if (pos + len > inode->i_size) { 1167 ext4_truncate_failed_write(inode); 1168 /* 1169 * If truncate failed early the inode might still be 1170 * on the orphan list; we need to make sure the inode 1171 * is removed from the orphan list in that case. 1172 */ 1173 if (inode->i_nlink) 1174 ext4_orphan_del(NULL, inode); 1175 } 1176 1177 return ret ? ret : copied; 1178 } 1179 1180 /* 1181 * Reserve a single cluster located at lblock 1182 */ 1183 static int ext4_da_reserve_space(struct inode *inode, ext4_lblk_t lblock) 1184 { 1185 int retries = 0; 1186 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1187 struct ext4_inode_info *ei = EXT4_I(inode); 1188 unsigned int md_needed; 1189 int ret; 1190 ext4_lblk_t save_last_lblock; 1191 int save_len; 1192 1193 /* 1194 * We will charge metadata quota at writeout time; this saves 1195 * us from metadata over-estimation, though we may go over by 1196 * a small amount in the end. Here we just reserve for data. 1197 */ 1198 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1)); 1199 if (ret) 1200 return ret; 1201 1202 /* 1203 * recalculate the amount of metadata blocks to reserve 1204 * in order to allocate nrblocks 1205 * worse case is one extent per block 1206 */ 1207 repeat: 1208 spin_lock(&ei->i_block_reservation_lock); 1209 /* 1210 * ext4_calc_metadata_amount() has side effects, which we have 1211 * to be prepared undo if we fail to claim space. 1212 */ 1213 save_len = ei->i_da_metadata_calc_len; 1214 save_last_lblock = ei->i_da_metadata_calc_last_lblock; 1215 md_needed = EXT4_NUM_B2C(sbi, 1216 ext4_calc_metadata_amount(inode, lblock)); 1217 trace_ext4_da_reserve_space(inode, md_needed); 1218 1219 /* 1220 * We do still charge estimated metadata to the sb though; 1221 * we cannot afford to run out of free blocks. 1222 */ 1223 if (ext4_claim_free_clusters(sbi, md_needed + 1, 0)) { 1224 ei->i_da_metadata_calc_len = save_len; 1225 ei->i_da_metadata_calc_last_lblock = save_last_lblock; 1226 spin_unlock(&ei->i_block_reservation_lock); 1227 if (ext4_should_retry_alloc(inode->i_sb, &retries)) { 1228 yield(); 1229 goto repeat; 1230 } 1231 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1)); 1232 return -ENOSPC; 1233 } 1234 ei->i_reserved_data_blocks++; 1235 ei->i_reserved_meta_blocks += md_needed; 1236 spin_unlock(&ei->i_block_reservation_lock); 1237 1238 return 0; /* success */ 1239 } 1240 1241 static void ext4_da_release_space(struct inode *inode, int to_free) 1242 { 1243 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1244 struct ext4_inode_info *ei = EXT4_I(inode); 1245 1246 if (!to_free) 1247 return; /* Nothing to release, exit */ 1248 1249 spin_lock(&EXT4_I(inode)->i_block_reservation_lock); 1250 1251 trace_ext4_da_release_space(inode, to_free); 1252 if (unlikely(to_free > ei->i_reserved_data_blocks)) { 1253 /* 1254 * if there aren't enough reserved blocks, then the 1255 * counter is messed up somewhere. Since this 1256 * function is called from invalidate page, it's 1257 * harmless to return without any action. 1258 */ 1259 ext4_msg(inode->i_sb, KERN_NOTICE, "ext4_da_release_space: " 1260 "ino %lu, to_free %d with only %d reserved " 1261 "data blocks", inode->i_ino, to_free, 1262 ei->i_reserved_data_blocks); 1263 WARN_ON(1); 1264 to_free = ei->i_reserved_data_blocks; 1265 } 1266 ei->i_reserved_data_blocks -= to_free; 1267 1268 if (ei->i_reserved_data_blocks == 0) { 1269 /* 1270 * We can release all of the reserved metadata blocks 1271 * only when we have written all of the delayed 1272 * allocation blocks. 1273 * Note that in case of bigalloc, i_reserved_meta_blocks, 1274 * i_reserved_data_blocks, etc. refer to number of clusters. 1275 */ 1276 percpu_counter_sub(&sbi->s_dirtyclusters_counter, 1277 ei->i_reserved_meta_blocks); 1278 ei->i_reserved_meta_blocks = 0; 1279 ei->i_da_metadata_calc_len = 0; 1280 } 1281 1282 /* update fs dirty data blocks counter */ 1283 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free); 1284 1285 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 1286 1287 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free)); 1288 } 1289 1290 static void ext4_da_page_release_reservation(struct page *page, 1291 unsigned long offset) 1292 { 1293 int to_release = 0; 1294 struct buffer_head *head, *bh; 1295 unsigned int curr_off = 0; 1296 struct inode *inode = page->mapping->host; 1297 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1298 int num_clusters; 1299 ext4_fsblk_t lblk; 1300 1301 head = page_buffers(page); 1302 bh = head; 1303 do { 1304 unsigned int next_off = curr_off + bh->b_size; 1305 1306 if ((offset <= curr_off) && (buffer_delay(bh))) { 1307 to_release++; 1308 clear_buffer_delay(bh); 1309 } 1310 curr_off = next_off; 1311 } while ((bh = bh->b_this_page) != head); 1312 1313 if (to_release) { 1314 lblk = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 1315 ext4_es_remove_extent(inode, lblk, to_release); 1316 } 1317 1318 /* If we have released all the blocks belonging to a cluster, then we 1319 * need to release the reserved space for that cluster. */ 1320 num_clusters = EXT4_NUM_B2C(sbi, to_release); 1321 while (num_clusters > 0) { 1322 lblk = (page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits)) + 1323 ((num_clusters - 1) << sbi->s_cluster_bits); 1324 if (sbi->s_cluster_ratio == 1 || 1325 !ext4_find_delalloc_cluster(inode, lblk)) 1326 ext4_da_release_space(inode, 1); 1327 1328 num_clusters--; 1329 } 1330 } 1331 1332 /* 1333 * Delayed allocation stuff 1334 */ 1335 1336 /* 1337 * mpage_da_submit_io - walks through extent of pages and try to write 1338 * them with writepage() call back 1339 * 1340 * @mpd->inode: inode 1341 * @mpd->first_page: first page of the extent 1342 * @mpd->next_page: page after the last page of the extent 1343 * 1344 * By the time mpage_da_submit_io() is called we expect all blocks 1345 * to be allocated. this may be wrong if allocation failed. 1346 * 1347 * As pages are already locked by write_cache_pages(), we can't use it 1348 */ 1349 static int mpage_da_submit_io(struct mpage_da_data *mpd, 1350 struct ext4_map_blocks *map) 1351 { 1352 struct pagevec pvec; 1353 unsigned long index, end; 1354 int ret = 0, err, nr_pages, i; 1355 struct inode *inode = mpd->inode; 1356 struct address_space *mapping = inode->i_mapping; 1357 loff_t size = i_size_read(inode); 1358 unsigned int len, block_start; 1359 struct buffer_head *bh, *page_bufs = NULL; 1360 int journal_data = ext4_should_journal_data(inode); 1361 sector_t pblock = 0, cur_logical = 0; 1362 struct ext4_io_submit io_submit; 1363 1364 BUG_ON(mpd->next_page <= mpd->first_page); 1365 memset(&io_submit, 0, sizeof(io_submit)); 1366 /* 1367 * We need to start from the first_page to the next_page - 1 1368 * to make sure we also write the mapped dirty buffer_heads. 1369 * If we look at mpd->b_blocknr we would only be looking 1370 * at the currently mapped buffer_heads. 1371 */ 1372 index = mpd->first_page; 1373 end = mpd->next_page - 1; 1374 1375 pagevec_init(&pvec, 0); 1376 while (index <= end) { 1377 nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE); 1378 if (nr_pages == 0) 1379 break; 1380 for (i = 0; i < nr_pages; i++) { 1381 int commit_write = 0, skip_page = 0; 1382 struct page *page = pvec.pages[i]; 1383 1384 index = page->index; 1385 if (index > end) 1386 break; 1387 1388 if (index == size >> PAGE_CACHE_SHIFT) 1389 len = size & ~PAGE_CACHE_MASK; 1390 else 1391 len = PAGE_CACHE_SIZE; 1392 if (map) { 1393 cur_logical = index << (PAGE_CACHE_SHIFT - 1394 inode->i_blkbits); 1395 pblock = map->m_pblk + (cur_logical - 1396 map->m_lblk); 1397 } 1398 index++; 1399 1400 BUG_ON(!PageLocked(page)); 1401 BUG_ON(PageWriteback(page)); 1402 1403 /* 1404 * If the page does not have buffers (for 1405 * whatever reason), try to create them using 1406 * __block_write_begin. If this fails, 1407 * skip the page and move on. 1408 */ 1409 if (!page_has_buffers(page)) { 1410 if (__block_write_begin(page, 0, len, 1411 noalloc_get_block_write)) { 1412 skip_page: 1413 unlock_page(page); 1414 continue; 1415 } 1416 commit_write = 1; 1417 } 1418 1419 bh = page_bufs = page_buffers(page); 1420 block_start = 0; 1421 do { 1422 if (!bh) 1423 goto skip_page; 1424 if (map && (cur_logical >= map->m_lblk) && 1425 (cur_logical <= (map->m_lblk + 1426 (map->m_len - 1)))) { 1427 if (buffer_delay(bh)) { 1428 clear_buffer_delay(bh); 1429 bh->b_blocknr = pblock; 1430 } 1431 if (buffer_unwritten(bh) || 1432 buffer_mapped(bh)) 1433 BUG_ON(bh->b_blocknr != pblock); 1434 if (map->m_flags & EXT4_MAP_UNINIT) 1435 set_buffer_uninit(bh); 1436 clear_buffer_unwritten(bh); 1437 } 1438 1439 /* 1440 * skip page if block allocation undone and 1441 * block is dirty 1442 */ 1443 if (ext4_bh_delay_or_unwritten(NULL, bh)) 1444 skip_page = 1; 1445 bh = bh->b_this_page; 1446 block_start += bh->b_size; 1447 cur_logical++; 1448 pblock++; 1449 } while (bh != page_bufs); 1450 1451 if (skip_page) 1452 goto skip_page; 1453 1454 if (commit_write) 1455 /* mark the buffer_heads as dirty & uptodate */ 1456 block_commit_write(page, 0, len); 1457 1458 clear_page_dirty_for_io(page); 1459 /* 1460 * Delalloc doesn't support data journalling, 1461 * but eventually maybe we'll lift this 1462 * restriction. 1463 */ 1464 if (unlikely(journal_data && PageChecked(page))) 1465 err = __ext4_journalled_writepage(page, len); 1466 else if (test_opt(inode->i_sb, MBLK_IO_SUBMIT)) 1467 err = ext4_bio_write_page(&io_submit, page, 1468 len, mpd->wbc); 1469 else if (buffer_uninit(page_bufs)) { 1470 ext4_set_bh_endio(page_bufs, inode); 1471 err = block_write_full_page_endio(page, 1472 noalloc_get_block_write, 1473 mpd->wbc, ext4_end_io_buffer_write); 1474 } else 1475 err = block_write_full_page(page, 1476 noalloc_get_block_write, mpd->wbc); 1477 1478 if (!err) 1479 mpd->pages_written++; 1480 /* 1481 * In error case, we have to continue because 1482 * remaining pages are still locked 1483 */ 1484 if (ret == 0) 1485 ret = err; 1486 } 1487 pagevec_release(&pvec); 1488 } 1489 ext4_io_submit(&io_submit); 1490 return ret; 1491 } 1492 1493 static void ext4_da_block_invalidatepages(struct mpage_da_data *mpd) 1494 { 1495 int nr_pages, i; 1496 pgoff_t index, end; 1497 struct pagevec pvec; 1498 struct inode *inode = mpd->inode; 1499 struct address_space *mapping = inode->i_mapping; 1500 ext4_lblk_t start, last; 1501 1502 index = mpd->first_page; 1503 end = mpd->next_page - 1; 1504 1505 start = index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 1506 last = end << (PAGE_CACHE_SHIFT - inode->i_blkbits); 1507 ext4_es_remove_extent(inode, start, last - start + 1); 1508 1509 pagevec_init(&pvec, 0); 1510 while (index <= end) { 1511 nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE); 1512 if (nr_pages == 0) 1513 break; 1514 for (i = 0; i < nr_pages; i++) { 1515 struct page *page = pvec.pages[i]; 1516 if (page->index > end) 1517 break; 1518 BUG_ON(!PageLocked(page)); 1519 BUG_ON(PageWriteback(page)); 1520 block_invalidatepage(page, 0); 1521 ClearPageUptodate(page); 1522 unlock_page(page); 1523 } 1524 index = pvec.pages[nr_pages - 1]->index + 1; 1525 pagevec_release(&pvec); 1526 } 1527 return; 1528 } 1529 1530 static void ext4_print_free_blocks(struct inode *inode) 1531 { 1532 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1533 struct super_block *sb = inode->i_sb; 1534 1535 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld", 1536 EXT4_C2B(EXT4_SB(inode->i_sb), 1537 ext4_count_free_clusters(inode->i_sb))); 1538 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details"); 1539 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld", 1540 (long long) EXT4_C2B(EXT4_SB(inode->i_sb), 1541 percpu_counter_sum(&sbi->s_freeclusters_counter))); 1542 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld", 1543 (long long) EXT4_C2B(EXT4_SB(inode->i_sb), 1544 percpu_counter_sum(&sbi->s_dirtyclusters_counter))); 1545 ext4_msg(sb, KERN_CRIT, "Block reservation details"); 1546 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u", 1547 EXT4_I(inode)->i_reserved_data_blocks); 1548 ext4_msg(sb, KERN_CRIT, "i_reserved_meta_blocks=%u", 1549 EXT4_I(inode)->i_reserved_meta_blocks); 1550 return; 1551 } 1552 1553 /* 1554 * mpage_da_map_and_submit - go through given space, map them 1555 * if necessary, and then submit them for I/O 1556 * 1557 * @mpd - bh describing space 1558 * 1559 * The function skips space we know is already mapped to disk blocks. 1560 * 1561 */ 1562 static void mpage_da_map_and_submit(struct mpage_da_data *mpd) 1563 { 1564 int err, blks, get_blocks_flags; 1565 struct ext4_map_blocks map, *mapp = NULL; 1566 sector_t next = mpd->b_blocknr; 1567 unsigned max_blocks = mpd->b_size >> mpd->inode->i_blkbits; 1568 loff_t disksize = EXT4_I(mpd->inode)->i_disksize; 1569 handle_t *handle = NULL; 1570 1571 /* 1572 * If the blocks are mapped already, or we couldn't accumulate 1573 * any blocks, then proceed immediately to the submission stage. 1574 */ 1575 if ((mpd->b_size == 0) || 1576 ((mpd->b_state & (1 << BH_Mapped)) && 1577 !(mpd->b_state & (1 << BH_Delay)) && 1578 !(mpd->b_state & (1 << BH_Unwritten)))) 1579 goto submit_io; 1580 1581 handle = ext4_journal_current_handle(); 1582 BUG_ON(!handle); 1583 1584 /* 1585 * Call ext4_map_blocks() to allocate any delayed allocation 1586 * blocks, or to convert an uninitialized extent to be 1587 * initialized (in the case where we have written into 1588 * one or more preallocated blocks). 1589 * 1590 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE to 1591 * indicate that we are on the delayed allocation path. This 1592 * affects functions in many different parts of the allocation 1593 * call path. This flag exists primarily because we don't 1594 * want to change *many* call functions, so ext4_map_blocks() 1595 * will set the EXT4_STATE_DELALLOC_RESERVED flag once the 1596 * inode's allocation semaphore is taken. 1597 * 1598 * If the blocks in questions were delalloc blocks, set 1599 * EXT4_GET_BLOCKS_DELALLOC_RESERVE so the delalloc accounting 1600 * variables are updated after the blocks have been allocated. 1601 */ 1602 map.m_lblk = next; 1603 map.m_len = max_blocks; 1604 get_blocks_flags = EXT4_GET_BLOCKS_CREATE; 1605 if (ext4_should_dioread_nolock(mpd->inode)) 1606 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT; 1607 if (mpd->b_state & (1 << BH_Delay)) 1608 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE; 1609 1610 blks = ext4_map_blocks(handle, mpd->inode, &map, get_blocks_flags); 1611 if (blks < 0) { 1612 struct super_block *sb = mpd->inode->i_sb; 1613 1614 err = blks; 1615 /* 1616 * If get block returns EAGAIN or ENOSPC and there 1617 * appears to be free blocks we will just let 1618 * mpage_da_submit_io() unlock all of the pages. 1619 */ 1620 if (err == -EAGAIN) 1621 goto submit_io; 1622 1623 if (err == -ENOSPC && ext4_count_free_clusters(sb)) { 1624 mpd->retval = err; 1625 goto submit_io; 1626 } 1627 1628 /* 1629 * get block failure will cause us to loop in 1630 * writepages, because a_ops->writepage won't be able 1631 * to make progress. The page will be redirtied by 1632 * writepage and writepages will again try to write 1633 * the same. 1634 */ 1635 if (!(EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)) { 1636 ext4_msg(sb, KERN_CRIT, 1637 "delayed block allocation failed for inode %lu " 1638 "at logical offset %llu with max blocks %zd " 1639 "with error %d", mpd->inode->i_ino, 1640 (unsigned long long) next, 1641 mpd->b_size >> mpd->inode->i_blkbits, err); 1642 ext4_msg(sb, KERN_CRIT, 1643 "This should not happen!! Data will be lost\n"); 1644 if (err == -ENOSPC) 1645 ext4_print_free_blocks(mpd->inode); 1646 } 1647 /* invalidate all the pages */ 1648 ext4_da_block_invalidatepages(mpd); 1649 1650 /* Mark this page range as having been completed */ 1651 mpd->io_done = 1; 1652 return; 1653 } 1654 BUG_ON(blks == 0); 1655 1656 mapp = ↦ 1657 if (map.m_flags & EXT4_MAP_NEW) { 1658 struct block_device *bdev = mpd->inode->i_sb->s_bdev; 1659 int i; 1660 1661 for (i = 0; i < map.m_len; i++) 1662 unmap_underlying_metadata(bdev, map.m_pblk + i); 1663 } 1664 1665 /* 1666 * Update on-disk size along with block allocation. 1667 */ 1668 disksize = ((loff_t) next + blks) << mpd->inode->i_blkbits; 1669 if (disksize > i_size_read(mpd->inode)) 1670 disksize = i_size_read(mpd->inode); 1671 if (disksize > EXT4_I(mpd->inode)->i_disksize) { 1672 ext4_update_i_disksize(mpd->inode, disksize); 1673 err = ext4_mark_inode_dirty(handle, mpd->inode); 1674 if (err) 1675 ext4_error(mpd->inode->i_sb, 1676 "Failed to mark inode %lu dirty", 1677 mpd->inode->i_ino); 1678 } 1679 1680 submit_io: 1681 mpage_da_submit_io(mpd, mapp); 1682 mpd->io_done = 1; 1683 } 1684 1685 #define BH_FLAGS ((1 << BH_Uptodate) | (1 << BH_Mapped) | \ 1686 (1 << BH_Delay) | (1 << BH_Unwritten)) 1687 1688 /* 1689 * mpage_add_bh_to_extent - try to add one more block to extent of blocks 1690 * 1691 * @mpd->lbh - extent of blocks 1692 * @logical - logical number of the block in the file 1693 * @bh - bh of the block (used to access block's state) 1694 * 1695 * the function is used to collect contig. blocks in same state 1696 */ 1697 static void mpage_add_bh_to_extent(struct mpage_da_data *mpd, 1698 sector_t logical, size_t b_size, 1699 unsigned long b_state) 1700 { 1701 sector_t next; 1702 int nrblocks = mpd->b_size >> mpd->inode->i_blkbits; 1703 1704 /* 1705 * XXX Don't go larger than mballoc is willing to allocate 1706 * This is a stopgap solution. We eventually need to fold 1707 * mpage_da_submit_io() into this function and then call 1708 * ext4_map_blocks() multiple times in a loop 1709 */ 1710 if (nrblocks >= 8*1024*1024/mpd->inode->i_sb->s_blocksize) 1711 goto flush_it; 1712 1713 /* check if thereserved journal credits might overflow */ 1714 if (!(ext4_test_inode_flag(mpd->inode, EXT4_INODE_EXTENTS))) { 1715 if (nrblocks >= EXT4_MAX_TRANS_DATA) { 1716 /* 1717 * With non-extent format we are limited by the journal 1718 * credit available. Total credit needed to insert 1719 * nrblocks contiguous blocks is dependent on the 1720 * nrblocks. So limit nrblocks. 1721 */ 1722 goto flush_it; 1723 } else if ((nrblocks + (b_size >> mpd->inode->i_blkbits)) > 1724 EXT4_MAX_TRANS_DATA) { 1725 /* 1726 * Adding the new buffer_head would make it cross the 1727 * allowed limit for which we have journal credit 1728 * reserved. So limit the new bh->b_size 1729 */ 1730 b_size = (EXT4_MAX_TRANS_DATA - nrblocks) << 1731 mpd->inode->i_blkbits; 1732 /* we will do mpage_da_submit_io in the next loop */ 1733 } 1734 } 1735 /* 1736 * First block in the extent 1737 */ 1738 if (mpd->b_size == 0) { 1739 mpd->b_blocknr = logical; 1740 mpd->b_size = b_size; 1741 mpd->b_state = b_state & BH_FLAGS; 1742 return; 1743 } 1744 1745 next = mpd->b_blocknr + nrblocks; 1746 /* 1747 * Can we merge the block to our big extent? 1748 */ 1749 if (logical == next && (b_state & BH_FLAGS) == mpd->b_state) { 1750 mpd->b_size += b_size; 1751 return; 1752 } 1753 1754 flush_it: 1755 /* 1756 * We couldn't merge the block to our extent, so we 1757 * need to flush current extent and start new one 1758 */ 1759 mpage_da_map_and_submit(mpd); 1760 return; 1761 } 1762 1763 static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh) 1764 { 1765 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh); 1766 } 1767 1768 /* 1769 * This function is grabs code from the very beginning of 1770 * ext4_map_blocks, but assumes that the caller is from delayed write 1771 * time. This function looks up the requested blocks and sets the 1772 * buffer delay bit under the protection of i_data_sem. 1773 */ 1774 static int ext4_da_map_blocks(struct inode *inode, sector_t iblock, 1775 struct ext4_map_blocks *map, 1776 struct buffer_head *bh) 1777 { 1778 int retval; 1779 sector_t invalid_block = ~((sector_t) 0xffff); 1780 1781 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es)) 1782 invalid_block = ~0; 1783 1784 map->m_flags = 0; 1785 ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u," 1786 "logical block %lu\n", inode->i_ino, map->m_len, 1787 (unsigned long) map->m_lblk); 1788 /* 1789 * Try to see if we can get the block without requesting a new 1790 * file system block. 1791 */ 1792 down_read((&EXT4_I(inode)->i_data_sem)); 1793 if (ext4_has_inline_data(inode)) { 1794 /* 1795 * We will soon create blocks for this page, and let 1796 * us pretend as if the blocks aren't allocated yet. 1797 * In case of clusters, we have to handle the work 1798 * of mapping from cluster so that the reserved space 1799 * is calculated properly. 1800 */ 1801 if ((EXT4_SB(inode->i_sb)->s_cluster_ratio > 1) && 1802 ext4_find_delalloc_cluster(inode, map->m_lblk)) 1803 map->m_flags |= EXT4_MAP_FROM_CLUSTER; 1804 retval = 0; 1805 } else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 1806 retval = ext4_ext_map_blocks(NULL, inode, map, 0); 1807 else 1808 retval = ext4_ind_map_blocks(NULL, inode, map, 0); 1809 1810 if (retval == 0) { 1811 /* 1812 * XXX: __block_prepare_write() unmaps passed block, 1813 * is it OK? 1814 */ 1815 /* If the block was allocated from previously allocated cluster, 1816 * then we dont need to reserve it again. */ 1817 if (!(map->m_flags & EXT4_MAP_FROM_CLUSTER)) { 1818 retval = ext4_da_reserve_space(inode, iblock); 1819 if (retval) 1820 /* not enough space to reserve */ 1821 goto out_unlock; 1822 } 1823 1824 retval = ext4_es_insert_extent(inode, map->m_lblk, map->m_len); 1825 if (retval) 1826 goto out_unlock; 1827 1828 /* Clear EXT4_MAP_FROM_CLUSTER flag since its purpose is served 1829 * and it should not appear on the bh->b_state. 1830 */ 1831 map->m_flags &= ~EXT4_MAP_FROM_CLUSTER; 1832 1833 map_bh(bh, inode->i_sb, invalid_block); 1834 set_buffer_new(bh); 1835 set_buffer_delay(bh); 1836 } 1837 1838 out_unlock: 1839 up_read((&EXT4_I(inode)->i_data_sem)); 1840 1841 return retval; 1842 } 1843 1844 /* 1845 * This is a special get_blocks_t callback which is used by 1846 * ext4_da_write_begin(). It will either return mapped block or 1847 * reserve space for a single block. 1848 * 1849 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set. 1850 * We also have b_blocknr = -1 and b_bdev initialized properly 1851 * 1852 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set. 1853 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev 1854 * initialized properly. 1855 */ 1856 int ext4_da_get_block_prep(struct inode *inode, sector_t iblock, 1857 struct buffer_head *bh, int create) 1858 { 1859 struct ext4_map_blocks map; 1860 int ret = 0; 1861 1862 BUG_ON(create == 0); 1863 BUG_ON(bh->b_size != inode->i_sb->s_blocksize); 1864 1865 map.m_lblk = iblock; 1866 map.m_len = 1; 1867 1868 /* 1869 * first, we need to know whether the block is allocated already 1870 * preallocated blocks are unmapped but should treated 1871 * the same as allocated blocks. 1872 */ 1873 ret = ext4_da_map_blocks(inode, iblock, &map, bh); 1874 if (ret <= 0) 1875 return ret; 1876 1877 map_bh(bh, inode->i_sb, map.m_pblk); 1878 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | map.m_flags; 1879 1880 if (buffer_unwritten(bh)) { 1881 /* A delayed write to unwritten bh should be marked 1882 * new and mapped. Mapped ensures that we don't do 1883 * get_block multiple times when we write to the same 1884 * offset and new ensures that we do proper zero out 1885 * for partial write. 1886 */ 1887 set_buffer_new(bh); 1888 set_buffer_mapped(bh); 1889 } 1890 return 0; 1891 } 1892 1893 /* 1894 * This function is used as a standard get_block_t calback function 1895 * when there is no desire to allocate any blocks. It is used as a 1896 * callback function for block_write_begin() and block_write_full_page(). 1897 * These functions should only try to map a single block at a time. 1898 * 1899 * Since this function doesn't do block allocations even if the caller 1900 * requests it by passing in create=1, it is critically important that 1901 * any caller checks to make sure that any buffer heads are returned 1902 * by this function are either all already mapped or marked for 1903 * delayed allocation before calling block_write_full_page(). Otherwise, 1904 * b_blocknr could be left unitialized, and the page write functions will 1905 * be taken by surprise. 1906 */ 1907 static int noalloc_get_block_write(struct inode *inode, sector_t iblock, 1908 struct buffer_head *bh_result, int create) 1909 { 1910 BUG_ON(bh_result->b_size != inode->i_sb->s_blocksize); 1911 return _ext4_get_block(inode, iblock, bh_result, 0); 1912 } 1913 1914 static int bget_one(handle_t *handle, struct buffer_head *bh) 1915 { 1916 get_bh(bh); 1917 return 0; 1918 } 1919 1920 static int bput_one(handle_t *handle, struct buffer_head *bh) 1921 { 1922 put_bh(bh); 1923 return 0; 1924 } 1925 1926 static int __ext4_journalled_writepage(struct page *page, 1927 unsigned int len) 1928 { 1929 struct address_space *mapping = page->mapping; 1930 struct inode *inode = mapping->host; 1931 struct buffer_head *page_bufs = NULL; 1932 handle_t *handle = NULL; 1933 int ret = 0, err = 0; 1934 int inline_data = ext4_has_inline_data(inode); 1935 struct buffer_head *inode_bh = NULL; 1936 1937 ClearPageChecked(page); 1938 1939 if (inline_data) { 1940 BUG_ON(page->index != 0); 1941 BUG_ON(len > ext4_get_max_inline_size(inode)); 1942 inode_bh = ext4_journalled_write_inline_data(inode, len, page); 1943 if (inode_bh == NULL) 1944 goto out; 1945 } else { 1946 page_bufs = page_buffers(page); 1947 if (!page_bufs) { 1948 BUG(); 1949 goto out; 1950 } 1951 ext4_walk_page_buffers(handle, page_bufs, 0, len, 1952 NULL, bget_one); 1953 } 1954 /* As soon as we unlock the page, it can go away, but we have 1955 * references to buffers so we are safe */ 1956 unlock_page(page); 1957 1958 handle = ext4_journal_start(inode, ext4_writepage_trans_blocks(inode)); 1959 if (IS_ERR(handle)) { 1960 ret = PTR_ERR(handle); 1961 goto out; 1962 } 1963 1964 BUG_ON(!ext4_handle_valid(handle)); 1965 1966 if (inline_data) { 1967 ret = ext4_journal_get_write_access(handle, inode_bh); 1968 1969 err = ext4_handle_dirty_metadata(handle, inode, inode_bh); 1970 1971 } else { 1972 ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL, 1973 do_journal_get_write_access); 1974 1975 err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL, 1976 write_end_fn); 1977 } 1978 if (ret == 0) 1979 ret = err; 1980 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 1981 err = ext4_journal_stop(handle); 1982 if (!ret) 1983 ret = err; 1984 1985 if (!ext4_has_inline_data(inode)) 1986 ext4_walk_page_buffers(handle, page_bufs, 0, len, 1987 NULL, bput_one); 1988 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 1989 out: 1990 brelse(inode_bh); 1991 return ret; 1992 } 1993 1994 /* 1995 * Note that we don't need to start a transaction unless we're journaling data 1996 * because we should have holes filled from ext4_page_mkwrite(). We even don't 1997 * need to file the inode to the transaction's list in ordered mode because if 1998 * we are writing back data added by write(), the inode is already there and if 1999 * we are writing back data modified via mmap(), no one guarantees in which 2000 * transaction the data will hit the disk. In case we are journaling data, we 2001 * cannot start transaction directly because transaction start ranks above page 2002 * lock so we have to do some magic. 2003 * 2004 * This function can get called via... 2005 * - ext4_da_writepages after taking page lock (have journal handle) 2006 * - journal_submit_inode_data_buffers (no journal handle) 2007 * - shrink_page_list via the kswapd/direct reclaim (no journal handle) 2008 * - grab_page_cache when doing write_begin (have journal handle) 2009 * 2010 * We don't do any block allocation in this function. If we have page with 2011 * multiple blocks we need to write those buffer_heads that are mapped. This 2012 * is important for mmaped based write. So if we do with blocksize 1K 2013 * truncate(f, 1024); 2014 * a = mmap(f, 0, 4096); 2015 * a[0] = 'a'; 2016 * truncate(f, 4096); 2017 * we have in the page first buffer_head mapped via page_mkwrite call back 2018 * but other buffer_heads would be unmapped but dirty (dirty done via the 2019 * do_wp_page). So writepage should write the first block. If we modify 2020 * the mmap area beyond 1024 we will again get a page_fault and the 2021 * page_mkwrite callback will do the block allocation and mark the 2022 * buffer_heads mapped. 2023 * 2024 * We redirty the page if we have any buffer_heads that is either delay or 2025 * unwritten in the page. 2026 * 2027 * We can get recursively called as show below. 2028 * 2029 * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() -> 2030 * ext4_writepage() 2031 * 2032 * But since we don't do any block allocation we should not deadlock. 2033 * Page also have the dirty flag cleared so we don't get recurive page_lock. 2034 */ 2035 static int ext4_writepage(struct page *page, 2036 struct writeback_control *wbc) 2037 { 2038 int ret = 0, commit_write = 0; 2039 loff_t size; 2040 unsigned int len; 2041 struct buffer_head *page_bufs = NULL; 2042 struct inode *inode = page->mapping->host; 2043 2044 trace_ext4_writepage(page); 2045 size = i_size_read(inode); 2046 if (page->index == size >> PAGE_CACHE_SHIFT) 2047 len = size & ~PAGE_CACHE_MASK; 2048 else 2049 len = PAGE_CACHE_SIZE; 2050 2051 /* 2052 * If the page does not have buffers (for whatever reason), 2053 * try to create them using __block_write_begin. If this 2054 * fails, redirty the page and move on. 2055 */ 2056 if (!page_has_buffers(page)) { 2057 if (__block_write_begin(page, 0, len, 2058 noalloc_get_block_write)) { 2059 redirty_page: 2060 redirty_page_for_writepage(wbc, page); 2061 unlock_page(page); 2062 return 0; 2063 } 2064 commit_write = 1; 2065 } 2066 page_bufs = page_buffers(page); 2067 if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL, 2068 ext4_bh_delay_or_unwritten)) { 2069 /* 2070 * We don't want to do block allocation, so redirty 2071 * the page and return. We may reach here when we do 2072 * a journal commit via journal_submit_inode_data_buffers. 2073 * We can also reach here via shrink_page_list but it 2074 * should never be for direct reclaim so warn if that 2075 * happens 2076 */ 2077 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == 2078 PF_MEMALLOC); 2079 goto redirty_page; 2080 } 2081 if (commit_write) 2082 /* now mark the buffer_heads as dirty and uptodate */ 2083 block_commit_write(page, 0, len); 2084 2085 if (PageChecked(page) && ext4_should_journal_data(inode)) 2086 /* 2087 * It's mmapped pagecache. Add buffers and journal it. There 2088 * doesn't seem much point in redirtying the page here. 2089 */ 2090 return __ext4_journalled_writepage(page, len); 2091 2092 if (buffer_uninit(page_bufs)) { 2093 ext4_set_bh_endio(page_bufs, inode); 2094 ret = block_write_full_page_endio(page, noalloc_get_block_write, 2095 wbc, ext4_end_io_buffer_write); 2096 } else 2097 ret = block_write_full_page(page, noalloc_get_block_write, 2098 wbc); 2099 2100 return ret; 2101 } 2102 2103 /* 2104 * This is called via ext4_da_writepages() to 2105 * calculate the total number of credits to reserve to fit 2106 * a single extent allocation into a single transaction, 2107 * ext4_da_writpeages() will loop calling this before 2108 * the block allocation. 2109 */ 2110 2111 static int ext4_da_writepages_trans_blocks(struct inode *inode) 2112 { 2113 int max_blocks = EXT4_I(inode)->i_reserved_data_blocks; 2114 2115 /* 2116 * With non-extent format the journal credit needed to 2117 * insert nrblocks contiguous block is dependent on 2118 * number of contiguous block. So we will limit 2119 * number of contiguous block to a sane value 2120 */ 2121 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) && 2122 (max_blocks > EXT4_MAX_TRANS_DATA)) 2123 max_blocks = EXT4_MAX_TRANS_DATA; 2124 2125 return ext4_chunk_trans_blocks(inode, max_blocks); 2126 } 2127 2128 /* 2129 * write_cache_pages_da - walk the list of dirty pages of the given 2130 * address space and accumulate pages that need writing, and call 2131 * mpage_da_map_and_submit to map a single contiguous memory region 2132 * and then write them. 2133 */ 2134 static int write_cache_pages_da(handle_t *handle, 2135 struct address_space *mapping, 2136 struct writeback_control *wbc, 2137 struct mpage_da_data *mpd, 2138 pgoff_t *done_index) 2139 { 2140 struct buffer_head *bh, *head; 2141 struct inode *inode = mapping->host; 2142 struct pagevec pvec; 2143 unsigned int nr_pages; 2144 sector_t logical; 2145 pgoff_t index, end; 2146 long nr_to_write = wbc->nr_to_write; 2147 int i, tag, ret = 0; 2148 2149 memset(mpd, 0, sizeof(struct mpage_da_data)); 2150 mpd->wbc = wbc; 2151 mpd->inode = inode; 2152 pagevec_init(&pvec, 0); 2153 index = wbc->range_start >> PAGE_CACHE_SHIFT; 2154 end = wbc->range_end >> PAGE_CACHE_SHIFT; 2155 2156 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2157 tag = PAGECACHE_TAG_TOWRITE; 2158 else 2159 tag = PAGECACHE_TAG_DIRTY; 2160 2161 *done_index = index; 2162 while (index <= end) { 2163 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 2164 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); 2165 if (nr_pages == 0) 2166 return 0; 2167 2168 for (i = 0; i < nr_pages; i++) { 2169 struct page *page = pvec.pages[i]; 2170 2171 /* 2172 * At this point, the page may be truncated or 2173 * invalidated (changing page->mapping to NULL), or 2174 * even swizzled back from swapper_space to tmpfs file 2175 * mapping. However, page->index will not change 2176 * because we have a reference on the page. 2177 */ 2178 if (page->index > end) 2179 goto out; 2180 2181 *done_index = page->index + 1; 2182 2183 /* 2184 * If we can't merge this page, and we have 2185 * accumulated an contiguous region, write it 2186 */ 2187 if ((mpd->next_page != page->index) && 2188 (mpd->next_page != mpd->first_page)) { 2189 mpage_da_map_and_submit(mpd); 2190 goto ret_extent_tail; 2191 } 2192 2193 lock_page(page); 2194 2195 /* 2196 * If the page is no longer dirty, or its 2197 * mapping no longer corresponds to inode we 2198 * are writing (which means it has been 2199 * truncated or invalidated), or the page is 2200 * already under writeback and we are not 2201 * doing a data integrity writeback, skip the page 2202 */ 2203 if (!PageDirty(page) || 2204 (PageWriteback(page) && 2205 (wbc->sync_mode == WB_SYNC_NONE)) || 2206 unlikely(page->mapping != mapping)) { 2207 unlock_page(page); 2208 continue; 2209 } 2210 2211 wait_on_page_writeback(page); 2212 BUG_ON(PageWriteback(page)); 2213 2214 /* 2215 * If we have inline data and arrive here, it means that 2216 * we will soon create the block for the 1st page, so 2217 * we'd better clear the inline data here. 2218 */ 2219 if (ext4_has_inline_data(inode)) { 2220 BUG_ON(ext4_test_inode_state(inode, 2221 EXT4_STATE_MAY_INLINE_DATA)); 2222 ext4_destroy_inline_data(handle, inode); 2223 } 2224 2225 if (mpd->next_page != page->index) 2226 mpd->first_page = page->index; 2227 mpd->next_page = page->index + 1; 2228 logical = (sector_t) page->index << 2229 (PAGE_CACHE_SHIFT - inode->i_blkbits); 2230 2231 if (!page_has_buffers(page)) { 2232 mpage_add_bh_to_extent(mpd, logical, 2233 PAGE_CACHE_SIZE, 2234 (1 << BH_Dirty) | (1 << BH_Uptodate)); 2235 if (mpd->io_done) 2236 goto ret_extent_tail; 2237 } else { 2238 /* 2239 * Page with regular buffer heads, 2240 * just add all dirty ones 2241 */ 2242 head = page_buffers(page); 2243 bh = head; 2244 do { 2245 BUG_ON(buffer_locked(bh)); 2246 /* 2247 * We need to try to allocate 2248 * unmapped blocks in the same page. 2249 * Otherwise we won't make progress 2250 * with the page in ext4_writepage 2251 */ 2252 if (ext4_bh_delay_or_unwritten(NULL, bh)) { 2253 mpage_add_bh_to_extent(mpd, logical, 2254 bh->b_size, 2255 bh->b_state); 2256 if (mpd->io_done) 2257 goto ret_extent_tail; 2258 } else if (buffer_dirty(bh) && (buffer_mapped(bh))) { 2259 /* 2260 * mapped dirty buffer. We need 2261 * to update the b_state 2262 * because we look at b_state 2263 * in mpage_da_map_blocks. We 2264 * don't update b_size because 2265 * if we find an unmapped 2266 * buffer_head later we need to 2267 * use the b_state flag of that 2268 * buffer_head. 2269 */ 2270 if (mpd->b_size == 0) 2271 mpd->b_state = bh->b_state & BH_FLAGS; 2272 } 2273 logical++; 2274 } while ((bh = bh->b_this_page) != head); 2275 } 2276 2277 if (nr_to_write > 0) { 2278 nr_to_write--; 2279 if (nr_to_write == 0 && 2280 wbc->sync_mode == WB_SYNC_NONE) 2281 /* 2282 * We stop writing back only if we are 2283 * not doing integrity sync. In case of 2284 * integrity sync we have to keep going 2285 * because someone may be concurrently 2286 * dirtying pages, and we might have 2287 * synced a lot of newly appeared dirty 2288 * pages, but have not synced all of the 2289 * old dirty pages. 2290 */ 2291 goto out; 2292 } 2293 } 2294 pagevec_release(&pvec); 2295 cond_resched(); 2296 } 2297 return 0; 2298 ret_extent_tail: 2299 ret = MPAGE_DA_EXTENT_TAIL; 2300 out: 2301 pagevec_release(&pvec); 2302 cond_resched(); 2303 return ret; 2304 } 2305 2306 2307 static int ext4_da_writepages(struct address_space *mapping, 2308 struct writeback_control *wbc) 2309 { 2310 pgoff_t index; 2311 int range_whole = 0; 2312 handle_t *handle = NULL; 2313 struct mpage_da_data mpd; 2314 struct inode *inode = mapping->host; 2315 int pages_written = 0; 2316 unsigned int max_pages; 2317 int range_cyclic, cycled = 1, io_done = 0; 2318 int needed_blocks, ret = 0; 2319 long desired_nr_to_write, nr_to_writebump = 0; 2320 loff_t range_start = wbc->range_start; 2321 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb); 2322 pgoff_t done_index = 0; 2323 pgoff_t end; 2324 struct blk_plug plug; 2325 2326 trace_ext4_da_writepages(inode, wbc); 2327 2328 /* 2329 * No pages to write? This is mainly a kludge to avoid starting 2330 * a transaction for special inodes like journal inode on last iput() 2331 * because that could violate lock ordering on umount 2332 */ 2333 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 2334 return 0; 2335 2336 /* 2337 * If the filesystem has aborted, it is read-only, so return 2338 * right away instead of dumping stack traces later on that 2339 * will obscure the real source of the problem. We test 2340 * EXT4_MF_FS_ABORTED instead of sb->s_flag's MS_RDONLY because 2341 * the latter could be true if the filesystem is mounted 2342 * read-only, and in that case, ext4_da_writepages should 2343 * *never* be called, so if that ever happens, we would want 2344 * the stack trace. 2345 */ 2346 if (unlikely(sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) 2347 return -EROFS; 2348 2349 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2350 range_whole = 1; 2351 2352 range_cyclic = wbc->range_cyclic; 2353 if (wbc->range_cyclic) { 2354 index = mapping->writeback_index; 2355 if (index) 2356 cycled = 0; 2357 wbc->range_start = index << PAGE_CACHE_SHIFT; 2358 wbc->range_end = LLONG_MAX; 2359 wbc->range_cyclic = 0; 2360 end = -1; 2361 } else { 2362 index = wbc->range_start >> PAGE_CACHE_SHIFT; 2363 end = wbc->range_end >> PAGE_CACHE_SHIFT; 2364 } 2365 2366 /* 2367 * This works around two forms of stupidity. The first is in 2368 * the writeback code, which caps the maximum number of pages 2369 * written to be 1024 pages. This is wrong on multiple 2370 * levels; different architectues have a different page size, 2371 * which changes the maximum amount of data which gets 2372 * written. Secondly, 4 megabytes is way too small. XFS 2373 * forces this value to be 16 megabytes by multiplying 2374 * nr_to_write parameter by four, and then relies on its 2375 * allocator to allocate larger extents to make them 2376 * contiguous. Unfortunately this brings us to the second 2377 * stupidity, which is that ext4's mballoc code only allocates 2378 * at most 2048 blocks. So we force contiguous writes up to 2379 * the number of dirty blocks in the inode, or 2380 * sbi->max_writeback_mb_bump whichever is smaller. 2381 */ 2382 max_pages = sbi->s_max_writeback_mb_bump << (20 - PAGE_CACHE_SHIFT); 2383 if (!range_cyclic && range_whole) { 2384 if (wbc->nr_to_write == LONG_MAX) 2385 desired_nr_to_write = wbc->nr_to_write; 2386 else 2387 desired_nr_to_write = wbc->nr_to_write * 8; 2388 } else 2389 desired_nr_to_write = ext4_num_dirty_pages(inode, index, 2390 max_pages); 2391 if (desired_nr_to_write > max_pages) 2392 desired_nr_to_write = max_pages; 2393 2394 if (wbc->nr_to_write < desired_nr_to_write) { 2395 nr_to_writebump = desired_nr_to_write - wbc->nr_to_write; 2396 wbc->nr_to_write = desired_nr_to_write; 2397 } 2398 2399 retry: 2400 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2401 tag_pages_for_writeback(mapping, index, end); 2402 2403 blk_start_plug(&plug); 2404 while (!ret && wbc->nr_to_write > 0) { 2405 2406 /* 2407 * we insert one extent at a time. So we need 2408 * credit needed for single extent allocation. 2409 * journalled mode is currently not supported 2410 * by delalloc 2411 */ 2412 BUG_ON(ext4_should_journal_data(inode)); 2413 needed_blocks = ext4_da_writepages_trans_blocks(inode); 2414 2415 /* start a new transaction*/ 2416 handle = ext4_journal_start(inode, needed_blocks); 2417 if (IS_ERR(handle)) { 2418 ret = PTR_ERR(handle); 2419 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: " 2420 "%ld pages, ino %lu; err %d", __func__, 2421 wbc->nr_to_write, inode->i_ino, ret); 2422 blk_finish_plug(&plug); 2423 goto out_writepages; 2424 } 2425 2426 /* 2427 * Now call write_cache_pages_da() to find the next 2428 * contiguous region of logical blocks that need 2429 * blocks to be allocated by ext4 and submit them. 2430 */ 2431 ret = write_cache_pages_da(handle, mapping, 2432 wbc, &mpd, &done_index); 2433 /* 2434 * If we have a contiguous extent of pages and we 2435 * haven't done the I/O yet, map the blocks and submit 2436 * them for I/O. 2437 */ 2438 if (!mpd.io_done && mpd.next_page != mpd.first_page) { 2439 mpage_da_map_and_submit(&mpd); 2440 ret = MPAGE_DA_EXTENT_TAIL; 2441 } 2442 trace_ext4_da_write_pages(inode, &mpd); 2443 wbc->nr_to_write -= mpd.pages_written; 2444 2445 ext4_journal_stop(handle); 2446 2447 if ((mpd.retval == -ENOSPC) && sbi->s_journal) { 2448 /* commit the transaction which would 2449 * free blocks released in the transaction 2450 * and try again 2451 */ 2452 jbd2_journal_force_commit_nested(sbi->s_journal); 2453 ret = 0; 2454 } else if (ret == MPAGE_DA_EXTENT_TAIL) { 2455 /* 2456 * Got one extent now try with rest of the pages. 2457 * If mpd.retval is set -EIO, journal is aborted. 2458 * So we don't need to write any more. 2459 */ 2460 pages_written += mpd.pages_written; 2461 ret = mpd.retval; 2462 io_done = 1; 2463 } else if (wbc->nr_to_write) 2464 /* 2465 * There is no more writeout needed 2466 * or we requested for a noblocking writeout 2467 * and we found the device congested 2468 */ 2469 break; 2470 } 2471 blk_finish_plug(&plug); 2472 if (!io_done && !cycled) { 2473 cycled = 1; 2474 index = 0; 2475 wbc->range_start = index << PAGE_CACHE_SHIFT; 2476 wbc->range_end = mapping->writeback_index - 1; 2477 goto retry; 2478 } 2479 2480 /* Update index */ 2481 wbc->range_cyclic = range_cyclic; 2482 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 2483 /* 2484 * set the writeback_index so that range_cyclic 2485 * mode will write it back later 2486 */ 2487 mapping->writeback_index = done_index; 2488 2489 out_writepages: 2490 wbc->nr_to_write -= nr_to_writebump; 2491 wbc->range_start = range_start; 2492 trace_ext4_da_writepages_result(inode, wbc, ret, pages_written); 2493 return ret; 2494 } 2495 2496 static int ext4_nonda_switch(struct super_block *sb) 2497 { 2498 s64 free_blocks, dirty_blocks; 2499 struct ext4_sb_info *sbi = EXT4_SB(sb); 2500 2501 /* 2502 * switch to non delalloc mode if we are running low 2503 * on free block. The free block accounting via percpu 2504 * counters can get slightly wrong with percpu_counter_batch getting 2505 * accumulated on each CPU without updating global counters 2506 * Delalloc need an accurate free block accounting. So switch 2507 * to non delalloc when we are near to error range. 2508 */ 2509 free_blocks = EXT4_C2B(sbi, 2510 percpu_counter_read_positive(&sbi->s_freeclusters_counter)); 2511 dirty_blocks = percpu_counter_read_positive(&sbi->s_dirtyclusters_counter); 2512 /* 2513 * Start pushing delalloc when 1/2 of free blocks are dirty. 2514 */ 2515 if (dirty_blocks && (free_blocks < 2 * dirty_blocks) && 2516 !writeback_in_progress(sb->s_bdi) && 2517 down_read_trylock(&sb->s_umount)) { 2518 writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE); 2519 up_read(&sb->s_umount); 2520 } 2521 2522 if (2 * free_blocks < 3 * dirty_blocks || 2523 free_blocks < (dirty_blocks + EXT4_FREECLUSTERS_WATERMARK)) { 2524 /* 2525 * free block count is less than 150% of dirty blocks 2526 * or free blocks is less than watermark 2527 */ 2528 return 1; 2529 } 2530 return 0; 2531 } 2532 2533 static int ext4_da_write_begin(struct file *file, struct address_space *mapping, 2534 loff_t pos, unsigned len, unsigned flags, 2535 struct page **pagep, void **fsdata) 2536 { 2537 int ret, retries = 0; 2538 struct page *page; 2539 pgoff_t index; 2540 struct inode *inode = mapping->host; 2541 handle_t *handle; 2542 2543 index = pos >> PAGE_CACHE_SHIFT; 2544 2545 if (ext4_nonda_switch(inode->i_sb)) { 2546 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC; 2547 return ext4_write_begin(file, mapping, pos, 2548 len, flags, pagep, fsdata); 2549 } 2550 *fsdata = (void *)0; 2551 trace_ext4_da_write_begin(inode, pos, len, flags); 2552 2553 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 2554 ret = ext4_da_write_inline_data_begin(mapping, inode, 2555 pos, len, flags, 2556 pagep, fsdata); 2557 if (ret < 0) 2558 goto out; 2559 if (ret == 1) { 2560 ret = 0; 2561 goto out; 2562 } 2563 } 2564 2565 retry: 2566 /* 2567 * With delayed allocation, we don't log the i_disksize update 2568 * if there is delayed block allocation. But we still need 2569 * to journalling the i_disksize update if writes to the end 2570 * of file which has an already mapped buffer. 2571 */ 2572 handle = ext4_journal_start(inode, 1); 2573 if (IS_ERR(handle)) { 2574 ret = PTR_ERR(handle); 2575 goto out; 2576 } 2577 /* We cannot recurse into the filesystem as the transaction is already 2578 * started */ 2579 flags |= AOP_FLAG_NOFS; 2580 2581 page = grab_cache_page_write_begin(mapping, index, flags); 2582 if (!page) { 2583 ext4_journal_stop(handle); 2584 ret = -ENOMEM; 2585 goto out; 2586 } 2587 *pagep = page; 2588 2589 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep); 2590 if (ret < 0) { 2591 unlock_page(page); 2592 ext4_journal_stop(handle); 2593 page_cache_release(page); 2594 /* 2595 * block_write_begin may have instantiated a few blocks 2596 * outside i_size. Trim these off again. Don't need 2597 * i_size_read because we hold i_mutex. 2598 */ 2599 if (pos + len > inode->i_size) 2600 ext4_truncate_failed_write(inode); 2601 } 2602 2603 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 2604 goto retry; 2605 out: 2606 return ret; 2607 } 2608 2609 /* 2610 * Check if we should update i_disksize 2611 * when write to the end of file but not require block allocation 2612 */ 2613 static int ext4_da_should_update_i_disksize(struct page *page, 2614 unsigned long offset) 2615 { 2616 struct buffer_head *bh; 2617 struct inode *inode = page->mapping->host; 2618 unsigned int idx; 2619 int i; 2620 2621 bh = page_buffers(page); 2622 idx = offset >> inode->i_blkbits; 2623 2624 for (i = 0; i < idx; i++) 2625 bh = bh->b_this_page; 2626 2627 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh)) 2628 return 0; 2629 return 1; 2630 } 2631 2632 static int ext4_da_write_end(struct file *file, 2633 struct address_space *mapping, 2634 loff_t pos, unsigned len, unsigned copied, 2635 struct page *page, void *fsdata) 2636 { 2637 struct inode *inode = mapping->host; 2638 int ret = 0, ret2; 2639 handle_t *handle = ext4_journal_current_handle(); 2640 loff_t new_i_size; 2641 unsigned long start, end; 2642 int write_mode = (int)(unsigned long)fsdata; 2643 2644 if (write_mode == FALL_BACK_TO_NONDELALLOC) { 2645 switch (ext4_inode_journal_mode(inode)) { 2646 case EXT4_INODE_ORDERED_DATA_MODE: 2647 return ext4_ordered_write_end(file, mapping, pos, 2648 len, copied, page, fsdata); 2649 case EXT4_INODE_WRITEBACK_DATA_MODE: 2650 return ext4_writeback_write_end(file, mapping, pos, 2651 len, copied, page, fsdata); 2652 default: 2653 BUG(); 2654 } 2655 } 2656 2657 trace_ext4_da_write_end(inode, pos, len, copied); 2658 start = pos & (PAGE_CACHE_SIZE - 1); 2659 end = start + copied - 1; 2660 2661 /* 2662 * generic_write_end() will run mark_inode_dirty() if i_size 2663 * changes. So let's piggyback the i_disksize mark_inode_dirty 2664 * into that. 2665 */ 2666 new_i_size = pos + copied; 2667 if (copied && new_i_size > EXT4_I(inode)->i_disksize) { 2668 if (ext4_has_inline_data(inode) || 2669 ext4_da_should_update_i_disksize(page, end)) { 2670 down_write(&EXT4_I(inode)->i_data_sem); 2671 if (new_i_size > EXT4_I(inode)->i_disksize) 2672 EXT4_I(inode)->i_disksize = new_i_size; 2673 up_write(&EXT4_I(inode)->i_data_sem); 2674 /* We need to mark inode dirty even if 2675 * new_i_size is less that inode->i_size 2676 * bu greater than i_disksize.(hint delalloc) 2677 */ 2678 ext4_mark_inode_dirty(handle, inode); 2679 } 2680 } 2681 2682 if (write_mode != CONVERT_INLINE_DATA && 2683 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && 2684 ext4_has_inline_data(inode)) 2685 ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied, 2686 page); 2687 else 2688 ret2 = generic_write_end(file, mapping, pos, len, copied, 2689 page, fsdata); 2690 2691 copied = ret2; 2692 if (ret2 < 0) 2693 ret = ret2; 2694 ret2 = ext4_journal_stop(handle); 2695 if (!ret) 2696 ret = ret2; 2697 2698 return ret ? ret : copied; 2699 } 2700 2701 static void ext4_da_invalidatepage(struct page *page, unsigned long offset) 2702 { 2703 /* 2704 * Drop reserved blocks 2705 */ 2706 BUG_ON(!PageLocked(page)); 2707 if (!page_has_buffers(page)) 2708 goto out; 2709 2710 ext4_da_page_release_reservation(page, offset); 2711 2712 out: 2713 ext4_invalidatepage(page, offset); 2714 2715 return; 2716 } 2717 2718 /* 2719 * Force all delayed allocation blocks to be allocated for a given inode. 2720 */ 2721 int ext4_alloc_da_blocks(struct inode *inode) 2722 { 2723 trace_ext4_alloc_da_blocks(inode); 2724 2725 if (!EXT4_I(inode)->i_reserved_data_blocks && 2726 !EXT4_I(inode)->i_reserved_meta_blocks) 2727 return 0; 2728 2729 /* 2730 * We do something simple for now. The filemap_flush() will 2731 * also start triggering a write of the data blocks, which is 2732 * not strictly speaking necessary (and for users of 2733 * laptop_mode, not even desirable). However, to do otherwise 2734 * would require replicating code paths in: 2735 * 2736 * ext4_da_writepages() -> 2737 * write_cache_pages() ---> (via passed in callback function) 2738 * __mpage_da_writepage() --> 2739 * mpage_add_bh_to_extent() 2740 * mpage_da_map_blocks() 2741 * 2742 * The problem is that write_cache_pages(), located in 2743 * mm/page-writeback.c, marks pages clean in preparation for 2744 * doing I/O, which is not desirable if we're not planning on 2745 * doing I/O at all. 2746 * 2747 * We could call write_cache_pages(), and then redirty all of 2748 * the pages by calling redirty_page_for_writepage() but that 2749 * would be ugly in the extreme. So instead we would need to 2750 * replicate parts of the code in the above functions, 2751 * simplifying them because we wouldn't actually intend to 2752 * write out the pages, but rather only collect contiguous 2753 * logical block extents, call the multi-block allocator, and 2754 * then update the buffer heads with the block allocations. 2755 * 2756 * For now, though, we'll cheat by calling filemap_flush(), 2757 * which will map the blocks, and start the I/O, but not 2758 * actually wait for the I/O to complete. 2759 */ 2760 return filemap_flush(inode->i_mapping); 2761 } 2762 2763 /* 2764 * bmap() is special. It gets used by applications such as lilo and by 2765 * the swapper to find the on-disk block of a specific piece of data. 2766 * 2767 * Naturally, this is dangerous if the block concerned is still in the 2768 * journal. If somebody makes a swapfile on an ext4 data-journaling 2769 * filesystem and enables swap, then they may get a nasty shock when the 2770 * data getting swapped to that swapfile suddenly gets overwritten by 2771 * the original zero's written out previously to the journal and 2772 * awaiting writeback in the kernel's buffer cache. 2773 * 2774 * So, if we see any bmap calls here on a modified, data-journaled file, 2775 * take extra steps to flush any blocks which might be in the cache. 2776 */ 2777 static sector_t ext4_bmap(struct address_space *mapping, sector_t block) 2778 { 2779 struct inode *inode = mapping->host; 2780 journal_t *journal; 2781 int err; 2782 2783 /* 2784 * We can get here for an inline file via the FIBMAP ioctl 2785 */ 2786 if (ext4_has_inline_data(inode)) 2787 return 0; 2788 2789 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && 2790 test_opt(inode->i_sb, DELALLOC)) { 2791 /* 2792 * With delalloc we want to sync the file 2793 * so that we can make sure we allocate 2794 * blocks for file 2795 */ 2796 filemap_write_and_wait(mapping); 2797 } 2798 2799 if (EXT4_JOURNAL(inode) && 2800 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) { 2801 /* 2802 * This is a REALLY heavyweight approach, but the use of 2803 * bmap on dirty files is expected to be extremely rare: 2804 * only if we run lilo or swapon on a freshly made file 2805 * do we expect this to happen. 2806 * 2807 * (bmap requires CAP_SYS_RAWIO so this does not 2808 * represent an unprivileged user DOS attack --- we'd be 2809 * in trouble if mortal users could trigger this path at 2810 * will.) 2811 * 2812 * NB. EXT4_STATE_JDATA is not set on files other than 2813 * regular files. If somebody wants to bmap a directory 2814 * or symlink and gets confused because the buffer 2815 * hasn't yet been flushed to disk, they deserve 2816 * everything they get. 2817 */ 2818 2819 ext4_clear_inode_state(inode, EXT4_STATE_JDATA); 2820 journal = EXT4_JOURNAL(inode); 2821 jbd2_journal_lock_updates(journal); 2822 err = jbd2_journal_flush(journal); 2823 jbd2_journal_unlock_updates(journal); 2824 2825 if (err) 2826 return 0; 2827 } 2828 2829 return generic_block_bmap(mapping, block, ext4_get_block); 2830 } 2831 2832 static int ext4_readpage(struct file *file, struct page *page) 2833 { 2834 int ret = -EAGAIN; 2835 struct inode *inode = page->mapping->host; 2836 2837 trace_ext4_readpage(page); 2838 2839 if (ext4_has_inline_data(inode)) 2840 ret = ext4_readpage_inline(inode, page); 2841 2842 if (ret == -EAGAIN) 2843 return mpage_readpage(page, ext4_get_block); 2844 2845 return ret; 2846 } 2847 2848 static int 2849 ext4_readpages(struct file *file, struct address_space *mapping, 2850 struct list_head *pages, unsigned nr_pages) 2851 { 2852 struct inode *inode = mapping->host; 2853 2854 /* If the file has inline data, no need to do readpages. */ 2855 if (ext4_has_inline_data(inode)) 2856 return 0; 2857 2858 return mpage_readpages(mapping, pages, nr_pages, ext4_get_block); 2859 } 2860 2861 static void ext4_invalidatepage_free_endio(struct page *page, unsigned long offset) 2862 { 2863 struct buffer_head *head, *bh; 2864 unsigned int curr_off = 0; 2865 2866 if (!page_has_buffers(page)) 2867 return; 2868 head = bh = page_buffers(page); 2869 do { 2870 if (offset <= curr_off && test_clear_buffer_uninit(bh) 2871 && bh->b_private) { 2872 ext4_free_io_end(bh->b_private); 2873 bh->b_private = NULL; 2874 bh->b_end_io = NULL; 2875 } 2876 curr_off = curr_off + bh->b_size; 2877 bh = bh->b_this_page; 2878 } while (bh != head); 2879 } 2880 2881 static void ext4_invalidatepage(struct page *page, unsigned long offset) 2882 { 2883 trace_ext4_invalidatepage(page, offset); 2884 2885 /* 2886 * free any io_end structure allocated for buffers to be discarded 2887 */ 2888 if (ext4_should_dioread_nolock(page->mapping->host)) 2889 ext4_invalidatepage_free_endio(page, offset); 2890 2891 /* No journalling happens on data buffers when this function is used */ 2892 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page))); 2893 2894 block_invalidatepage(page, offset); 2895 } 2896 2897 static int __ext4_journalled_invalidatepage(struct page *page, 2898 unsigned long offset) 2899 { 2900 journal_t *journal = EXT4_JOURNAL(page->mapping->host); 2901 2902 trace_ext4_journalled_invalidatepage(page, offset); 2903 2904 /* 2905 * If it's a full truncate we just forget about the pending dirtying 2906 */ 2907 if (offset == 0) 2908 ClearPageChecked(page); 2909 2910 return jbd2_journal_invalidatepage(journal, page, offset); 2911 } 2912 2913 /* Wrapper for aops... */ 2914 static void ext4_journalled_invalidatepage(struct page *page, 2915 unsigned long offset) 2916 { 2917 WARN_ON(__ext4_journalled_invalidatepage(page, offset) < 0); 2918 } 2919 2920 static int ext4_releasepage(struct page *page, gfp_t wait) 2921 { 2922 journal_t *journal = EXT4_JOURNAL(page->mapping->host); 2923 2924 trace_ext4_releasepage(page); 2925 2926 WARN_ON(PageChecked(page)); 2927 if (!page_has_buffers(page)) 2928 return 0; 2929 if (journal) 2930 return jbd2_journal_try_to_free_buffers(journal, page, wait); 2931 else 2932 return try_to_free_buffers(page); 2933 } 2934 2935 /* 2936 * ext4_get_block used when preparing for a DIO write or buffer write. 2937 * We allocate an uinitialized extent if blocks haven't been allocated. 2938 * The extent will be converted to initialized after the IO is complete. 2939 */ 2940 int ext4_get_block_write(struct inode *inode, sector_t iblock, 2941 struct buffer_head *bh_result, int create) 2942 { 2943 ext4_debug("ext4_get_block_write: inode %lu, create flag %d\n", 2944 inode->i_ino, create); 2945 return _ext4_get_block(inode, iblock, bh_result, 2946 EXT4_GET_BLOCKS_IO_CREATE_EXT); 2947 } 2948 2949 static int ext4_get_block_write_nolock(struct inode *inode, sector_t iblock, 2950 struct buffer_head *bh_result, int create) 2951 { 2952 ext4_debug("ext4_get_block_write_nolock: inode %lu, create flag %d\n", 2953 inode->i_ino, create); 2954 return _ext4_get_block(inode, iblock, bh_result, 2955 EXT4_GET_BLOCKS_NO_LOCK); 2956 } 2957 2958 static void ext4_end_io_dio(struct kiocb *iocb, loff_t offset, 2959 ssize_t size, void *private, int ret, 2960 bool is_async) 2961 { 2962 struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; 2963 ext4_io_end_t *io_end = iocb->private; 2964 2965 /* if not async direct IO or dio with 0 bytes write, just return */ 2966 if (!io_end || !size) 2967 goto out; 2968 2969 ext_debug("ext4_end_io_dio(): io_end 0x%p " 2970 "for inode %lu, iocb 0x%p, offset %llu, size %zd\n", 2971 iocb->private, io_end->inode->i_ino, iocb, offset, 2972 size); 2973 2974 iocb->private = NULL; 2975 2976 /* if not aio dio with unwritten extents, just free io and return */ 2977 if (!(io_end->flag & EXT4_IO_END_UNWRITTEN)) { 2978 ext4_free_io_end(io_end); 2979 out: 2980 if (is_async) 2981 aio_complete(iocb, ret, 0); 2982 inode_dio_done(inode); 2983 return; 2984 } 2985 2986 io_end->offset = offset; 2987 io_end->size = size; 2988 if (is_async) { 2989 io_end->iocb = iocb; 2990 io_end->result = ret; 2991 } 2992 2993 ext4_add_complete_io(io_end); 2994 } 2995 2996 static void ext4_end_io_buffer_write(struct buffer_head *bh, int uptodate) 2997 { 2998 ext4_io_end_t *io_end = bh->b_private; 2999 struct inode *inode; 3000 3001 if (!test_clear_buffer_uninit(bh) || !io_end) 3002 goto out; 3003 3004 if (!(io_end->inode->i_sb->s_flags & MS_ACTIVE)) { 3005 ext4_msg(io_end->inode->i_sb, KERN_INFO, 3006 "sb umounted, discard end_io request for inode %lu", 3007 io_end->inode->i_ino); 3008 ext4_free_io_end(io_end); 3009 goto out; 3010 } 3011 3012 /* 3013 * It may be over-defensive here to check EXT4_IO_END_UNWRITTEN now, 3014 * but being more careful is always safe for the future change. 3015 */ 3016 inode = io_end->inode; 3017 ext4_set_io_unwritten_flag(inode, io_end); 3018 ext4_add_complete_io(io_end); 3019 out: 3020 bh->b_private = NULL; 3021 bh->b_end_io = NULL; 3022 clear_buffer_uninit(bh); 3023 end_buffer_async_write(bh, uptodate); 3024 } 3025 3026 static int ext4_set_bh_endio(struct buffer_head *bh, struct inode *inode) 3027 { 3028 ext4_io_end_t *io_end; 3029 struct page *page = bh->b_page; 3030 loff_t offset = (sector_t)page->index << PAGE_CACHE_SHIFT; 3031 size_t size = bh->b_size; 3032 3033 retry: 3034 io_end = ext4_init_io_end(inode, GFP_ATOMIC); 3035 if (!io_end) { 3036 pr_warn_ratelimited("%s: allocation fail\n", __func__); 3037 schedule(); 3038 goto retry; 3039 } 3040 io_end->offset = offset; 3041 io_end->size = size; 3042 /* 3043 * We need to hold a reference to the page to make sure it 3044 * doesn't get evicted before ext4_end_io_work() has a chance 3045 * to convert the extent from written to unwritten. 3046 */ 3047 io_end->page = page; 3048 get_page(io_end->page); 3049 3050 bh->b_private = io_end; 3051 bh->b_end_io = ext4_end_io_buffer_write; 3052 return 0; 3053 } 3054 3055 /* 3056 * For ext4 extent files, ext4 will do direct-io write to holes, 3057 * preallocated extents, and those write extend the file, no need to 3058 * fall back to buffered IO. 3059 * 3060 * For holes, we fallocate those blocks, mark them as uninitialized 3061 * If those blocks were preallocated, we mark sure they are split, but 3062 * still keep the range to write as uninitialized. 3063 * 3064 * The unwritten extents will be converted to written when DIO is completed. 3065 * For async direct IO, since the IO may still pending when return, we 3066 * set up an end_io call back function, which will do the conversion 3067 * when async direct IO completed. 3068 * 3069 * If the O_DIRECT write will extend the file then add this inode to the 3070 * orphan list. So recovery will truncate it back to the original size 3071 * if the machine crashes during the write. 3072 * 3073 */ 3074 static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb, 3075 const struct iovec *iov, loff_t offset, 3076 unsigned long nr_segs) 3077 { 3078 struct file *file = iocb->ki_filp; 3079 struct inode *inode = file->f_mapping->host; 3080 ssize_t ret; 3081 size_t count = iov_length(iov, nr_segs); 3082 int overwrite = 0; 3083 get_block_t *get_block_func = NULL; 3084 int dio_flags = 0; 3085 loff_t final_size = offset + count; 3086 3087 /* Use the old path for reads and writes beyond i_size. */ 3088 if (rw != WRITE || final_size > inode->i_size) 3089 return ext4_ind_direct_IO(rw, iocb, iov, offset, nr_segs); 3090 3091 BUG_ON(iocb->private == NULL); 3092 3093 /* If we do a overwrite dio, i_mutex locking can be released */ 3094 overwrite = *((int *)iocb->private); 3095 3096 if (overwrite) { 3097 atomic_inc(&inode->i_dio_count); 3098 down_read(&EXT4_I(inode)->i_data_sem); 3099 mutex_unlock(&inode->i_mutex); 3100 } 3101 3102 /* 3103 * We could direct write to holes and fallocate. 3104 * 3105 * Allocated blocks to fill the hole are marked as 3106 * uninitialized to prevent parallel buffered read to expose 3107 * the stale data before DIO complete the data IO. 3108 * 3109 * As to previously fallocated extents, ext4 get_block will 3110 * just simply mark the buffer mapped but still keep the 3111 * extents uninitialized. 3112 * 3113 * For non AIO case, we will convert those unwritten extents 3114 * to written after return back from blockdev_direct_IO. 3115 * 3116 * For async DIO, the conversion needs to be deferred when the 3117 * IO is completed. The ext4 end_io callback function will be 3118 * called to take care of the conversion work. Here for async 3119 * case, we allocate an io_end structure to hook to the iocb. 3120 */ 3121 iocb->private = NULL; 3122 ext4_inode_aio_set(inode, NULL); 3123 if (!is_sync_kiocb(iocb)) { 3124 ext4_io_end_t *io_end = ext4_init_io_end(inode, GFP_NOFS); 3125 if (!io_end) { 3126 ret = -ENOMEM; 3127 goto retake_lock; 3128 } 3129 io_end->flag |= EXT4_IO_END_DIRECT; 3130 iocb->private = io_end; 3131 /* 3132 * we save the io structure for current async direct 3133 * IO, so that later ext4_map_blocks() could flag the 3134 * io structure whether there is a unwritten extents 3135 * needs to be converted when IO is completed. 3136 */ 3137 ext4_inode_aio_set(inode, io_end); 3138 } 3139 3140 if (overwrite) { 3141 get_block_func = ext4_get_block_write_nolock; 3142 } else { 3143 get_block_func = ext4_get_block_write; 3144 dio_flags = DIO_LOCKING; 3145 } 3146 ret = __blockdev_direct_IO(rw, iocb, inode, 3147 inode->i_sb->s_bdev, iov, 3148 offset, nr_segs, 3149 get_block_func, 3150 ext4_end_io_dio, 3151 NULL, 3152 dio_flags); 3153 3154 if (iocb->private) 3155 ext4_inode_aio_set(inode, NULL); 3156 /* 3157 * The io_end structure takes a reference to the inode, that 3158 * structure needs to be destroyed and the reference to the 3159 * inode need to be dropped, when IO is complete, even with 0 3160 * byte write, or failed. 3161 * 3162 * In the successful AIO DIO case, the io_end structure will 3163 * be destroyed and the reference to the inode will be dropped 3164 * after the end_io call back function is called. 3165 * 3166 * In the case there is 0 byte write, or error case, since VFS 3167 * direct IO won't invoke the end_io call back function, we 3168 * need to free the end_io structure here. 3169 */ 3170 if (ret != -EIOCBQUEUED && ret <= 0 && iocb->private) { 3171 ext4_free_io_end(iocb->private); 3172 iocb->private = NULL; 3173 } else if (ret > 0 && !overwrite && ext4_test_inode_state(inode, 3174 EXT4_STATE_DIO_UNWRITTEN)) { 3175 int err; 3176 /* 3177 * for non AIO case, since the IO is already 3178 * completed, we could do the conversion right here 3179 */ 3180 err = ext4_convert_unwritten_extents(inode, 3181 offset, ret); 3182 if (err < 0) 3183 ret = err; 3184 ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN); 3185 } 3186 3187 retake_lock: 3188 /* take i_mutex locking again if we do a ovewrite dio */ 3189 if (overwrite) { 3190 inode_dio_done(inode); 3191 up_read(&EXT4_I(inode)->i_data_sem); 3192 mutex_lock(&inode->i_mutex); 3193 } 3194 3195 return ret; 3196 } 3197 3198 static ssize_t ext4_direct_IO(int rw, struct kiocb *iocb, 3199 const struct iovec *iov, loff_t offset, 3200 unsigned long nr_segs) 3201 { 3202 struct file *file = iocb->ki_filp; 3203 struct inode *inode = file->f_mapping->host; 3204 ssize_t ret; 3205 3206 /* 3207 * If we are doing data journalling we don't support O_DIRECT 3208 */ 3209 if (ext4_should_journal_data(inode)) 3210 return 0; 3211 3212 /* Let buffer I/O handle the inline data case. */ 3213 if (ext4_has_inline_data(inode)) 3214 return 0; 3215 3216 trace_ext4_direct_IO_enter(inode, offset, iov_length(iov, nr_segs), rw); 3217 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 3218 ret = ext4_ext_direct_IO(rw, iocb, iov, offset, nr_segs); 3219 else 3220 ret = ext4_ind_direct_IO(rw, iocb, iov, offset, nr_segs); 3221 trace_ext4_direct_IO_exit(inode, offset, 3222 iov_length(iov, nr_segs), rw, ret); 3223 return ret; 3224 } 3225 3226 /* 3227 * Pages can be marked dirty completely asynchronously from ext4's journalling 3228 * activity. By filemap_sync_pte(), try_to_unmap_one(), etc. We cannot do 3229 * much here because ->set_page_dirty is called under VFS locks. The page is 3230 * not necessarily locked. 3231 * 3232 * We cannot just dirty the page and leave attached buffers clean, because the 3233 * buffers' dirty state is "definitive". We cannot just set the buffers dirty 3234 * or jbddirty because all the journalling code will explode. 3235 * 3236 * So what we do is to mark the page "pending dirty" and next time writepage 3237 * is called, propagate that into the buffers appropriately. 3238 */ 3239 static int ext4_journalled_set_page_dirty(struct page *page) 3240 { 3241 SetPageChecked(page); 3242 return __set_page_dirty_nobuffers(page); 3243 } 3244 3245 static const struct address_space_operations ext4_ordered_aops = { 3246 .readpage = ext4_readpage, 3247 .readpages = ext4_readpages, 3248 .writepage = ext4_writepage, 3249 .write_begin = ext4_write_begin, 3250 .write_end = ext4_ordered_write_end, 3251 .bmap = ext4_bmap, 3252 .invalidatepage = ext4_invalidatepage, 3253 .releasepage = ext4_releasepage, 3254 .direct_IO = ext4_direct_IO, 3255 .migratepage = buffer_migrate_page, 3256 .is_partially_uptodate = block_is_partially_uptodate, 3257 .error_remove_page = generic_error_remove_page, 3258 }; 3259 3260 static const struct address_space_operations ext4_writeback_aops = { 3261 .readpage = ext4_readpage, 3262 .readpages = ext4_readpages, 3263 .writepage = ext4_writepage, 3264 .write_begin = ext4_write_begin, 3265 .write_end = ext4_writeback_write_end, 3266 .bmap = ext4_bmap, 3267 .invalidatepage = ext4_invalidatepage, 3268 .releasepage = ext4_releasepage, 3269 .direct_IO = ext4_direct_IO, 3270 .migratepage = buffer_migrate_page, 3271 .is_partially_uptodate = block_is_partially_uptodate, 3272 .error_remove_page = generic_error_remove_page, 3273 }; 3274 3275 static const struct address_space_operations ext4_journalled_aops = { 3276 .readpage = ext4_readpage, 3277 .readpages = ext4_readpages, 3278 .writepage = ext4_writepage, 3279 .write_begin = ext4_write_begin, 3280 .write_end = ext4_journalled_write_end, 3281 .set_page_dirty = ext4_journalled_set_page_dirty, 3282 .bmap = ext4_bmap, 3283 .invalidatepage = ext4_journalled_invalidatepage, 3284 .releasepage = ext4_releasepage, 3285 .direct_IO = ext4_direct_IO, 3286 .is_partially_uptodate = block_is_partially_uptodate, 3287 .error_remove_page = generic_error_remove_page, 3288 }; 3289 3290 static const struct address_space_operations ext4_da_aops = { 3291 .readpage = ext4_readpage, 3292 .readpages = ext4_readpages, 3293 .writepage = ext4_writepage, 3294 .writepages = ext4_da_writepages, 3295 .write_begin = ext4_da_write_begin, 3296 .write_end = ext4_da_write_end, 3297 .bmap = ext4_bmap, 3298 .invalidatepage = ext4_da_invalidatepage, 3299 .releasepage = ext4_releasepage, 3300 .direct_IO = ext4_direct_IO, 3301 .migratepage = buffer_migrate_page, 3302 .is_partially_uptodate = block_is_partially_uptodate, 3303 .error_remove_page = generic_error_remove_page, 3304 }; 3305 3306 void ext4_set_aops(struct inode *inode) 3307 { 3308 switch (ext4_inode_journal_mode(inode)) { 3309 case EXT4_INODE_ORDERED_DATA_MODE: 3310 if (test_opt(inode->i_sb, DELALLOC)) 3311 inode->i_mapping->a_ops = &ext4_da_aops; 3312 else 3313 inode->i_mapping->a_ops = &ext4_ordered_aops; 3314 break; 3315 case EXT4_INODE_WRITEBACK_DATA_MODE: 3316 if (test_opt(inode->i_sb, DELALLOC)) 3317 inode->i_mapping->a_ops = &ext4_da_aops; 3318 else 3319 inode->i_mapping->a_ops = &ext4_writeback_aops; 3320 break; 3321 case EXT4_INODE_JOURNAL_DATA_MODE: 3322 inode->i_mapping->a_ops = &ext4_journalled_aops; 3323 break; 3324 default: 3325 BUG(); 3326 } 3327 } 3328 3329 3330 /* 3331 * ext4_discard_partial_page_buffers() 3332 * Wrapper function for ext4_discard_partial_page_buffers_no_lock. 3333 * This function finds and locks the page containing the offset 3334 * "from" and passes it to ext4_discard_partial_page_buffers_no_lock. 3335 * Calling functions that already have the page locked should call 3336 * ext4_discard_partial_page_buffers_no_lock directly. 3337 */ 3338 int ext4_discard_partial_page_buffers(handle_t *handle, 3339 struct address_space *mapping, loff_t from, 3340 loff_t length, int flags) 3341 { 3342 struct inode *inode = mapping->host; 3343 struct page *page; 3344 int err = 0; 3345 3346 page = find_or_create_page(mapping, from >> PAGE_CACHE_SHIFT, 3347 mapping_gfp_mask(mapping) & ~__GFP_FS); 3348 if (!page) 3349 return -ENOMEM; 3350 3351 err = ext4_discard_partial_page_buffers_no_lock(handle, inode, page, 3352 from, length, flags); 3353 3354 unlock_page(page); 3355 page_cache_release(page); 3356 return err; 3357 } 3358 3359 /* 3360 * ext4_discard_partial_page_buffers_no_lock() 3361 * Zeros a page range of length 'length' starting from offset 'from'. 3362 * Buffer heads that correspond to the block aligned regions of the 3363 * zeroed range will be unmapped. Unblock aligned regions 3364 * will have the corresponding buffer head mapped if needed so that 3365 * that region of the page can be updated with the partial zero out. 3366 * 3367 * This function assumes that the page has already been locked. The 3368 * The range to be discarded must be contained with in the given page. 3369 * If the specified range exceeds the end of the page it will be shortened 3370 * to the end of the page that corresponds to 'from'. This function is 3371 * appropriate for updating a page and it buffer heads to be unmapped and 3372 * zeroed for blocks that have been either released, or are going to be 3373 * released. 3374 * 3375 * handle: The journal handle 3376 * inode: The files inode 3377 * page: A locked page that contains the offset "from" 3378 * from: The starting byte offset (from the beginning of the file) 3379 * to begin discarding 3380 * len: The length of bytes to discard 3381 * flags: Optional flags that may be used: 3382 * 3383 * EXT4_DISCARD_PARTIAL_PG_ZERO_UNMAPPED 3384 * Only zero the regions of the page whose buffer heads 3385 * have already been unmapped. This flag is appropriate 3386 * for updating the contents of a page whose blocks may 3387 * have already been released, and we only want to zero 3388 * out the regions that correspond to those released blocks. 3389 * 3390 * Returns zero on success or negative on failure. 3391 */ 3392 static int ext4_discard_partial_page_buffers_no_lock(handle_t *handle, 3393 struct inode *inode, struct page *page, loff_t from, 3394 loff_t length, int flags) 3395 { 3396 ext4_fsblk_t index = from >> PAGE_CACHE_SHIFT; 3397 unsigned int offset = from & (PAGE_CACHE_SIZE-1); 3398 unsigned int blocksize, max, pos; 3399 ext4_lblk_t iblock; 3400 struct buffer_head *bh; 3401 int err = 0; 3402 3403 blocksize = inode->i_sb->s_blocksize; 3404 max = PAGE_CACHE_SIZE - offset; 3405 3406 if (index != page->index) 3407 return -EINVAL; 3408 3409 /* 3410 * correct length if it does not fall between 3411 * 'from' and the end of the page 3412 */ 3413 if (length > max || length < 0) 3414 length = max; 3415 3416 iblock = index << (PAGE_CACHE_SHIFT - inode->i_sb->s_blocksize_bits); 3417 3418 if (!page_has_buffers(page)) 3419 create_empty_buffers(page, blocksize, 0); 3420 3421 /* Find the buffer that contains "offset" */ 3422 bh = page_buffers(page); 3423 pos = blocksize; 3424 while (offset >= pos) { 3425 bh = bh->b_this_page; 3426 iblock++; 3427 pos += blocksize; 3428 } 3429 3430 pos = offset; 3431 while (pos < offset + length) { 3432 unsigned int end_of_block, range_to_discard; 3433 3434 err = 0; 3435 3436 /* The length of space left to zero and unmap */ 3437 range_to_discard = offset + length - pos; 3438 3439 /* The length of space until the end of the block */ 3440 end_of_block = blocksize - (pos & (blocksize-1)); 3441 3442 /* 3443 * Do not unmap or zero past end of block 3444 * for this buffer head 3445 */ 3446 if (range_to_discard > end_of_block) 3447 range_to_discard = end_of_block; 3448 3449 3450 /* 3451 * Skip this buffer head if we are only zeroing unampped 3452 * regions of the page 3453 */ 3454 if (flags & EXT4_DISCARD_PARTIAL_PG_ZERO_UNMAPPED && 3455 buffer_mapped(bh)) 3456 goto next; 3457 3458 /* If the range is block aligned, unmap */ 3459 if (range_to_discard == blocksize) { 3460 clear_buffer_dirty(bh); 3461 bh->b_bdev = NULL; 3462 clear_buffer_mapped(bh); 3463 clear_buffer_req(bh); 3464 clear_buffer_new(bh); 3465 clear_buffer_delay(bh); 3466 clear_buffer_unwritten(bh); 3467 clear_buffer_uptodate(bh); 3468 zero_user(page, pos, range_to_discard); 3469 BUFFER_TRACE(bh, "Buffer discarded"); 3470 goto next; 3471 } 3472 3473 /* 3474 * If this block is not completely contained in the range 3475 * to be discarded, then it is not going to be released. Because 3476 * we need to keep this block, we need to make sure this part 3477 * of the page is uptodate before we modify it by writeing 3478 * partial zeros on it. 3479 */ 3480 if (!buffer_mapped(bh)) { 3481 /* 3482 * Buffer head must be mapped before we can read 3483 * from the block 3484 */ 3485 BUFFER_TRACE(bh, "unmapped"); 3486 ext4_get_block(inode, iblock, bh, 0); 3487 /* unmapped? It's a hole - nothing to do */ 3488 if (!buffer_mapped(bh)) { 3489 BUFFER_TRACE(bh, "still unmapped"); 3490 goto next; 3491 } 3492 } 3493 3494 /* Ok, it's mapped. Make sure it's up-to-date */ 3495 if (PageUptodate(page)) 3496 set_buffer_uptodate(bh); 3497 3498 if (!buffer_uptodate(bh)) { 3499 err = -EIO; 3500 ll_rw_block(READ, 1, &bh); 3501 wait_on_buffer(bh); 3502 /* Uhhuh. Read error. Complain and punt.*/ 3503 if (!buffer_uptodate(bh)) 3504 goto next; 3505 } 3506 3507 if (ext4_should_journal_data(inode)) { 3508 BUFFER_TRACE(bh, "get write access"); 3509 err = ext4_journal_get_write_access(handle, bh); 3510 if (err) 3511 goto next; 3512 } 3513 3514 zero_user(page, pos, range_to_discard); 3515 3516 err = 0; 3517 if (ext4_should_journal_data(inode)) { 3518 err = ext4_handle_dirty_metadata(handle, inode, bh); 3519 } else 3520 mark_buffer_dirty(bh); 3521 3522 BUFFER_TRACE(bh, "Partial buffer zeroed"); 3523 next: 3524 bh = bh->b_this_page; 3525 iblock++; 3526 pos += range_to_discard; 3527 } 3528 3529 return err; 3530 } 3531 3532 int ext4_can_truncate(struct inode *inode) 3533 { 3534 if (S_ISREG(inode->i_mode)) 3535 return 1; 3536 if (S_ISDIR(inode->i_mode)) 3537 return 1; 3538 if (S_ISLNK(inode->i_mode)) 3539 return !ext4_inode_is_fast_symlink(inode); 3540 return 0; 3541 } 3542 3543 /* 3544 * ext4_punch_hole: punches a hole in a file by releaseing the blocks 3545 * associated with the given offset and length 3546 * 3547 * @inode: File inode 3548 * @offset: The offset where the hole will begin 3549 * @len: The length of the hole 3550 * 3551 * Returns: 0 on success or negative on failure 3552 */ 3553 3554 int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) 3555 { 3556 struct inode *inode = file->f_path.dentry->d_inode; 3557 if (!S_ISREG(inode->i_mode)) 3558 return -EOPNOTSUPP; 3559 3560 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 3561 /* TODO: Add support for non extent hole punching */ 3562 return -EOPNOTSUPP; 3563 } 3564 3565 if (EXT4_SB(inode->i_sb)->s_cluster_ratio > 1) { 3566 /* TODO: Add support for bigalloc file systems */ 3567 return -EOPNOTSUPP; 3568 } 3569 3570 return ext4_ext_punch_hole(file, offset, length); 3571 } 3572 3573 /* 3574 * ext4_truncate() 3575 * 3576 * We block out ext4_get_block() block instantiations across the entire 3577 * transaction, and VFS/VM ensures that ext4_truncate() cannot run 3578 * simultaneously on behalf of the same inode. 3579 * 3580 * As we work through the truncate and commit bits of it to the journal there 3581 * is one core, guiding principle: the file's tree must always be consistent on 3582 * disk. We must be able to restart the truncate after a crash. 3583 * 3584 * The file's tree may be transiently inconsistent in memory (although it 3585 * probably isn't), but whenever we close off and commit a journal transaction, 3586 * the contents of (the filesystem + the journal) must be consistent and 3587 * restartable. It's pretty simple, really: bottom up, right to left (although 3588 * left-to-right works OK too). 3589 * 3590 * Note that at recovery time, journal replay occurs *before* the restart of 3591 * truncate against the orphan inode list. 3592 * 3593 * The committed inode has the new, desired i_size (which is the same as 3594 * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see 3595 * that this inode's truncate did not complete and it will again call 3596 * ext4_truncate() to have another go. So there will be instantiated blocks 3597 * to the right of the truncation point in a crashed ext4 filesystem. But 3598 * that's fine - as long as they are linked from the inode, the post-crash 3599 * ext4_truncate() run will find them and release them. 3600 */ 3601 void ext4_truncate(struct inode *inode) 3602 { 3603 trace_ext4_truncate_enter(inode); 3604 3605 if (!ext4_can_truncate(inode)) 3606 return; 3607 3608 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS); 3609 3610 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC)) 3611 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE); 3612 3613 if (ext4_has_inline_data(inode)) { 3614 int has_inline = 1; 3615 3616 ext4_inline_data_truncate(inode, &has_inline); 3617 if (has_inline) 3618 return; 3619 } 3620 3621 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 3622 ext4_ext_truncate(inode); 3623 else 3624 ext4_ind_truncate(inode); 3625 3626 trace_ext4_truncate_exit(inode); 3627 } 3628 3629 /* 3630 * ext4_get_inode_loc returns with an extra refcount against the inode's 3631 * underlying buffer_head on success. If 'in_mem' is true, we have all 3632 * data in memory that is needed to recreate the on-disk version of this 3633 * inode. 3634 */ 3635 static int __ext4_get_inode_loc(struct inode *inode, 3636 struct ext4_iloc *iloc, int in_mem) 3637 { 3638 struct ext4_group_desc *gdp; 3639 struct buffer_head *bh; 3640 struct super_block *sb = inode->i_sb; 3641 ext4_fsblk_t block; 3642 int inodes_per_block, inode_offset; 3643 3644 iloc->bh = NULL; 3645 if (!ext4_valid_inum(sb, inode->i_ino)) 3646 return -EIO; 3647 3648 iloc->block_group = (inode->i_ino - 1) / EXT4_INODES_PER_GROUP(sb); 3649 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); 3650 if (!gdp) 3651 return -EIO; 3652 3653 /* 3654 * Figure out the offset within the block group inode table 3655 */ 3656 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 3657 inode_offset = ((inode->i_ino - 1) % 3658 EXT4_INODES_PER_GROUP(sb)); 3659 block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block); 3660 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); 3661 3662 bh = sb_getblk(sb, block); 3663 if (!bh) { 3664 EXT4_ERROR_INODE_BLOCK(inode, block, 3665 "unable to read itable block"); 3666 return -EIO; 3667 } 3668 if (!buffer_uptodate(bh)) { 3669 lock_buffer(bh); 3670 3671 /* 3672 * If the buffer has the write error flag, we have failed 3673 * to write out another inode in the same block. In this 3674 * case, we don't have to read the block because we may 3675 * read the old inode data successfully. 3676 */ 3677 if (buffer_write_io_error(bh) && !buffer_uptodate(bh)) 3678 set_buffer_uptodate(bh); 3679 3680 if (buffer_uptodate(bh)) { 3681 /* someone brought it uptodate while we waited */ 3682 unlock_buffer(bh); 3683 goto has_buffer; 3684 } 3685 3686 /* 3687 * If we have all information of the inode in memory and this 3688 * is the only valid inode in the block, we need not read the 3689 * block. 3690 */ 3691 if (in_mem) { 3692 struct buffer_head *bitmap_bh; 3693 int i, start; 3694 3695 start = inode_offset & ~(inodes_per_block - 1); 3696 3697 /* Is the inode bitmap in cache? */ 3698 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp)); 3699 if (!bitmap_bh) 3700 goto make_io; 3701 3702 /* 3703 * If the inode bitmap isn't in cache then the 3704 * optimisation may end up performing two reads instead 3705 * of one, so skip it. 3706 */ 3707 if (!buffer_uptodate(bitmap_bh)) { 3708 brelse(bitmap_bh); 3709 goto make_io; 3710 } 3711 for (i = start; i < start + inodes_per_block; i++) { 3712 if (i == inode_offset) 3713 continue; 3714 if (ext4_test_bit(i, bitmap_bh->b_data)) 3715 break; 3716 } 3717 brelse(bitmap_bh); 3718 if (i == start + inodes_per_block) { 3719 /* all other inodes are free, so skip I/O */ 3720 memset(bh->b_data, 0, bh->b_size); 3721 set_buffer_uptodate(bh); 3722 unlock_buffer(bh); 3723 goto has_buffer; 3724 } 3725 } 3726 3727 make_io: 3728 /* 3729 * If we need to do any I/O, try to pre-readahead extra 3730 * blocks from the inode table. 3731 */ 3732 if (EXT4_SB(sb)->s_inode_readahead_blks) { 3733 ext4_fsblk_t b, end, table; 3734 unsigned num; 3735 3736 table = ext4_inode_table(sb, gdp); 3737 /* s_inode_readahead_blks is always a power of 2 */ 3738 b = block & ~(EXT4_SB(sb)->s_inode_readahead_blks-1); 3739 if (table > b) 3740 b = table; 3741 end = b + EXT4_SB(sb)->s_inode_readahead_blks; 3742 num = EXT4_INODES_PER_GROUP(sb); 3743 if (ext4_has_group_desc_csum(sb)) 3744 num -= ext4_itable_unused_count(sb, gdp); 3745 table += num / inodes_per_block; 3746 if (end > table) 3747 end = table; 3748 while (b <= end) 3749 sb_breadahead(sb, b++); 3750 } 3751 3752 /* 3753 * There are other valid inodes in the buffer, this inode 3754 * has in-inode xattrs, or we don't have this inode in memory. 3755 * Read the block from disk. 3756 */ 3757 trace_ext4_load_inode(inode); 3758 get_bh(bh); 3759 bh->b_end_io = end_buffer_read_sync; 3760 submit_bh(READ | REQ_META | REQ_PRIO, bh); 3761 wait_on_buffer(bh); 3762 if (!buffer_uptodate(bh)) { 3763 EXT4_ERROR_INODE_BLOCK(inode, block, 3764 "unable to read itable block"); 3765 brelse(bh); 3766 return -EIO; 3767 } 3768 } 3769 has_buffer: 3770 iloc->bh = bh; 3771 return 0; 3772 } 3773 3774 int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) 3775 { 3776 /* We have all inode data except xattrs in memory here. */ 3777 return __ext4_get_inode_loc(inode, iloc, 3778 !ext4_test_inode_state(inode, EXT4_STATE_XATTR)); 3779 } 3780 3781 void ext4_set_inode_flags(struct inode *inode) 3782 { 3783 unsigned int flags = EXT4_I(inode)->i_flags; 3784 3785 inode->i_flags &= ~(S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC); 3786 if (flags & EXT4_SYNC_FL) 3787 inode->i_flags |= S_SYNC; 3788 if (flags & EXT4_APPEND_FL) 3789 inode->i_flags |= S_APPEND; 3790 if (flags & EXT4_IMMUTABLE_FL) 3791 inode->i_flags |= S_IMMUTABLE; 3792 if (flags & EXT4_NOATIME_FL) 3793 inode->i_flags |= S_NOATIME; 3794 if (flags & EXT4_DIRSYNC_FL) 3795 inode->i_flags |= S_DIRSYNC; 3796 } 3797 3798 /* Propagate flags from i_flags to EXT4_I(inode)->i_flags */ 3799 void ext4_get_inode_flags(struct ext4_inode_info *ei) 3800 { 3801 unsigned int vfs_fl; 3802 unsigned long old_fl, new_fl; 3803 3804 do { 3805 vfs_fl = ei->vfs_inode.i_flags; 3806 old_fl = ei->i_flags; 3807 new_fl = old_fl & ~(EXT4_SYNC_FL|EXT4_APPEND_FL| 3808 EXT4_IMMUTABLE_FL|EXT4_NOATIME_FL| 3809 EXT4_DIRSYNC_FL); 3810 if (vfs_fl & S_SYNC) 3811 new_fl |= EXT4_SYNC_FL; 3812 if (vfs_fl & S_APPEND) 3813 new_fl |= EXT4_APPEND_FL; 3814 if (vfs_fl & S_IMMUTABLE) 3815 new_fl |= EXT4_IMMUTABLE_FL; 3816 if (vfs_fl & S_NOATIME) 3817 new_fl |= EXT4_NOATIME_FL; 3818 if (vfs_fl & S_DIRSYNC) 3819 new_fl |= EXT4_DIRSYNC_FL; 3820 } while (cmpxchg(&ei->i_flags, old_fl, new_fl) != old_fl); 3821 } 3822 3823 static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode, 3824 struct ext4_inode_info *ei) 3825 { 3826 blkcnt_t i_blocks ; 3827 struct inode *inode = &(ei->vfs_inode); 3828 struct super_block *sb = inode->i_sb; 3829 3830 if (EXT4_HAS_RO_COMPAT_FEATURE(sb, 3831 EXT4_FEATURE_RO_COMPAT_HUGE_FILE)) { 3832 /* we are using combined 48 bit field */ 3833 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 | 3834 le32_to_cpu(raw_inode->i_blocks_lo); 3835 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) { 3836 /* i_blocks represent file system block size */ 3837 return i_blocks << (inode->i_blkbits - 9); 3838 } else { 3839 return i_blocks; 3840 } 3841 } else { 3842 return le32_to_cpu(raw_inode->i_blocks_lo); 3843 } 3844 } 3845 3846 static inline void ext4_iget_extra_inode(struct inode *inode, 3847 struct ext4_inode *raw_inode, 3848 struct ext4_inode_info *ei) 3849 { 3850 __le32 *magic = (void *)raw_inode + 3851 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize; 3852 if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC)) { 3853 ext4_set_inode_state(inode, EXT4_STATE_XATTR); 3854 ext4_find_inline_data_nolock(inode); 3855 } else 3856 EXT4_I(inode)->i_inline_off = 0; 3857 } 3858 3859 struct inode *ext4_iget(struct super_block *sb, unsigned long ino) 3860 { 3861 struct ext4_iloc iloc; 3862 struct ext4_inode *raw_inode; 3863 struct ext4_inode_info *ei; 3864 struct inode *inode; 3865 journal_t *journal = EXT4_SB(sb)->s_journal; 3866 long ret; 3867 int block; 3868 uid_t i_uid; 3869 gid_t i_gid; 3870 3871 inode = iget_locked(sb, ino); 3872 if (!inode) 3873 return ERR_PTR(-ENOMEM); 3874 if (!(inode->i_state & I_NEW)) 3875 return inode; 3876 3877 ei = EXT4_I(inode); 3878 iloc.bh = NULL; 3879 3880 ret = __ext4_get_inode_loc(inode, &iloc, 0); 3881 if (ret < 0) 3882 goto bad_inode; 3883 raw_inode = ext4_raw_inode(&iloc); 3884 3885 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 3886 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize); 3887 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > 3888 EXT4_INODE_SIZE(inode->i_sb)) { 3889 EXT4_ERROR_INODE(inode, "bad extra_isize (%u != %u)", 3890 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize, 3891 EXT4_INODE_SIZE(inode->i_sb)); 3892 ret = -EIO; 3893 goto bad_inode; 3894 } 3895 } else 3896 ei->i_extra_isize = 0; 3897 3898 /* Precompute checksum seed for inode metadata */ 3899 if (EXT4_HAS_RO_COMPAT_FEATURE(sb, 3900 EXT4_FEATURE_RO_COMPAT_METADATA_CSUM)) { 3901 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 3902 __u32 csum; 3903 __le32 inum = cpu_to_le32(inode->i_ino); 3904 __le32 gen = raw_inode->i_generation; 3905 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum, 3906 sizeof(inum)); 3907 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen, 3908 sizeof(gen)); 3909 } 3910 3911 if (!ext4_inode_csum_verify(inode, raw_inode, ei)) { 3912 EXT4_ERROR_INODE(inode, "checksum invalid"); 3913 ret = -EIO; 3914 goto bad_inode; 3915 } 3916 3917 inode->i_mode = le16_to_cpu(raw_inode->i_mode); 3918 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low); 3919 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low); 3920 if (!(test_opt(inode->i_sb, NO_UID32))) { 3921 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16; 3922 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16; 3923 } 3924 i_uid_write(inode, i_uid); 3925 i_gid_write(inode, i_gid); 3926 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); 3927 3928 ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */ 3929 ei->i_inline_off = 0; 3930 ei->i_dir_start_lookup = 0; 3931 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); 3932 /* We now have enough fields to check if the inode was active or not. 3933 * This is needed because nfsd might try to access dead inodes 3934 * the test is that same one that e2fsck uses 3935 * NeilBrown 1999oct15 3936 */ 3937 if (inode->i_nlink == 0) { 3938 if (inode->i_mode == 0 || 3939 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) { 3940 /* this inode is deleted */ 3941 ret = -ESTALE; 3942 goto bad_inode; 3943 } 3944 /* The only unlinked inodes we let through here have 3945 * valid i_mode and are being read by the orphan 3946 * recovery code: that's fine, we're about to complete 3947 * the process of deleting those. */ 3948 } 3949 ei->i_flags = le32_to_cpu(raw_inode->i_flags); 3950 inode->i_blocks = ext4_inode_blocks(raw_inode, ei); 3951 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo); 3952 if (EXT4_HAS_INCOMPAT_FEATURE(sb, EXT4_FEATURE_INCOMPAT_64BIT)) 3953 ei->i_file_acl |= 3954 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; 3955 inode->i_size = ext4_isize(raw_inode); 3956 ei->i_disksize = inode->i_size; 3957 #ifdef CONFIG_QUOTA 3958 ei->i_reserved_quota = 0; 3959 #endif 3960 inode->i_generation = le32_to_cpu(raw_inode->i_generation); 3961 ei->i_block_group = iloc.block_group; 3962 ei->i_last_alloc_group = ~0; 3963 /* 3964 * NOTE! The in-memory inode i_data array is in little-endian order 3965 * even on big-endian machines: we do NOT byteswap the block numbers! 3966 */ 3967 for (block = 0; block < EXT4_N_BLOCKS; block++) 3968 ei->i_data[block] = raw_inode->i_block[block]; 3969 INIT_LIST_HEAD(&ei->i_orphan); 3970 3971 /* 3972 * Set transaction id's of transactions that have to be committed 3973 * to finish f[data]sync. We set them to currently running transaction 3974 * as we cannot be sure that the inode or some of its metadata isn't 3975 * part of the transaction - the inode could have been reclaimed and 3976 * now it is reread from disk. 3977 */ 3978 if (journal) { 3979 transaction_t *transaction; 3980 tid_t tid; 3981 3982 read_lock(&journal->j_state_lock); 3983 if (journal->j_running_transaction) 3984 transaction = journal->j_running_transaction; 3985 else 3986 transaction = journal->j_committing_transaction; 3987 if (transaction) 3988 tid = transaction->t_tid; 3989 else 3990 tid = journal->j_commit_sequence; 3991 read_unlock(&journal->j_state_lock); 3992 ei->i_sync_tid = tid; 3993 ei->i_datasync_tid = tid; 3994 } 3995 3996 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 3997 if (ei->i_extra_isize == 0) { 3998 /* The extra space is currently unused. Use it. */ 3999 ei->i_extra_isize = sizeof(struct ext4_inode) - 4000 EXT4_GOOD_OLD_INODE_SIZE; 4001 } else { 4002 ext4_iget_extra_inode(inode, raw_inode, ei); 4003 } 4004 } 4005 4006 EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode); 4007 EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode); 4008 EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode); 4009 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode); 4010 4011 inode->i_version = le32_to_cpu(raw_inode->i_disk_version); 4012 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 4013 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 4014 inode->i_version |= 4015 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32; 4016 } 4017 4018 ret = 0; 4019 if (ei->i_file_acl && 4020 !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) { 4021 EXT4_ERROR_INODE(inode, "bad extended attribute block %llu", 4022 ei->i_file_acl); 4023 ret = -EIO; 4024 goto bad_inode; 4025 } else if (!ext4_has_inline_data(inode)) { 4026 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 4027 if ((S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || 4028 (S_ISLNK(inode->i_mode) && 4029 !ext4_inode_is_fast_symlink(inode)))) 4030 /* Validate extent which is part of inode */ 4031 ret = ext4_ext_check_inode(inode); 4032 } else if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || 4033 (S_ISLNK(inode->i_mode) && 4034 !ext4_inode_is_fast_symlink(inode))) { 4035 /* Validate block references which are part of inode */ 4036 ret = ext4_ind_check_inode(inode); 4037 } 4038 } 4039 if (ret) 4040 goto bad_inode; 4041 4042 if (S_ISREG(inode->i_mode)) { 4043 inode->i_op = &ext4_file_inode_operations; 4044 inode->i_fop = &ext4_file_operations; 4045 ext4_set_aops(inode); 4046 } else if (S_ISDIR(inode->i_mode)) { 4047 inode->i_op = &ext4_dir_inode_operations; 4048 inode->i_fop = &ext4_dir_operations; 4049 } else if (S_ISLNK(inode->i_mode)) { 4050 if (ext4_inode_is_fast_symlink(inode)) { 4051 inode->i_op = &ext4_fast_symlink_inode_operations; 4052 nd_terminate_link(ei->i_data, inode->i_size, 4053 sizeof(ei->i_data) - 1); 4054 } else { 4055 inode->i_op = &ext4_symlink_inode_operations; 4056 ext4_set_aops(inode); 4057 } 4058 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || 4059 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { 4060 inode->i_op = &ext4_special_inode_operations; 4061 if (raw_inode->i_block[0]) 4062 init_special_inode(inode, inode->i_mode, 4063 old_decode_dev(le32_to_cpu(raw_inode->i_block[0]))); 4064 else 4065 init_special_inode(inode, inode->i_mode, 4066 new_decode_dev(le32_to_cpu(raw_inode->i_block[1]))); 4067 } else { 4068 ret = -EIO; 4069 EXT4_ERROR_INODE(inode, "bogus i_mode (%o)", inode->i_mode); 4070 goto bad_inode; 4071 } 4072 brelse(iloc.bh); 4073 ext4_set_inode_flags(inode); 4074 unlock_new_inode(inode); 4075 return inode; 4076 4077 bad_inode: 4078 brelse(iloc.bh); 4079 iget_failed(inode); 4080 return ERR_PTR(ret); 4081 } 4082 4083 static int ext4_inode_blocks_set(handle_t *handle, 4084 struct ext4_inode *raw_inode, 4085 struct ext4_inode_info *ei) 4086 { 4087 struct inode *inode = &(ei->vfs_inode); 4088 u64 i_blocks = inode->i_blocks; 4089 struct super_block *sb = inode->i_sb; 4090 4091 if (i_blocks <= ~0U) { 4092 /* 4093 * i_blocks can be represented in a 32 bit variable 4094 * as multiple of 512 bytes 4095 */ 4096 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4097 raw_inode->i_blocks_high = 0; 4098 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4099 return 0; 4100 } 4101 if (!EXT4_HAS_RO_COMPAT_FEATURE(sb, EXT4_FEATURE_RO_COMPAT_HUGE_FILE)) 4102 return -EFBIG; 4103 4104 if (i_blocks <= 0xffffffffffffULL) { 4105 /* 4106 * i_blocks can be represented in a 48 bit variable 4107 * as multiple of 512 bytes 4108 */ 4109 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4110 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 4111 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4112 } else { 4113 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4114 /* i_block is stored in file system block size */ 4115 i_blocks = i_blocks >> (inode->i_blkbits - 9); 4116 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4117 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 4118 } 4119 return 0; 4120 } 4121 4122 /* 4123 * Post the struct inode info into an on-disk inode location in the 4124 * buffer-cache. This gobbles the caller's reference to the 4125 * buffer_head in the inode location struct. 4126 * 4127 * The caller must have write access to iloc->bh. 4128 */ 4129 static int ext4_do_update_inode(handle_t *handle, 4130 struct inode *inode, 4131 struct ext4_iloc *iloc) 4132 { 4133 struct ext4_inode *raw_inode = ext4_raw_inode(iloc); 4134 struct ext4_inode_info *ei = EXT4_I(inode); 4135 struct buffer_head *bh = iloc->bh; 4136 int err = 0, rc, block; 4137 int need_datasync = 0; 4138 uid_t i_uid; 4139 gid_t i_gid; 4140 4141 /* For fields not not tracking in the in-memory inode, 4142 * initialise them to zero for new inodes. */ 4143 if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) 4144 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size); 4145 4146 ext4_get_inode_flags(ei); 4147 raw_inode->i_mode = cpu_to_le16(inode->i_mode); 4148 i_uid = i_uid_read(inode); 4149 i_gid = i_gid_read(inode); 4150 if (!(test_opt(inode->i_sb, NO_UID32))) { 4151 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid)); 4152 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid)); 4153 /* 4154 * Fix up interoperability with old kernels. Otherwise, old inodes get 4155 * re-used with the upper 16 bits of the uid/gid intact 4156 */ 4157 if (!ei->i_dtime) { 4158 raw_inode->i_uid_high = 4159 cpu_to_le16(high_16_bits(i_uid)); 4160 raw_inode->i_gid_high = 4161 cpu_to_le16(high_16_bits(i_gid)); 4162 } else { 4163 raw_inode->i_uid_high = 0; 4164 raw_inode->i_gid_high = 0; 4165 } 4166 } else { 4167 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid)); 4168 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid)); 4169 raw_inode->i_uid_high = 0; 4170 raw_inode->i_gid_high = 0; 4171 } 4172 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink); 4173 4174 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode); 4175 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode); 4176 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode); 4177 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode); 4178 4179 if (ext4_inode_blocks_set(handle, raw_inode, ei)) 4180 goto out_brelse; 4181 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime); 4182 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF); 4183 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 4184 cpu_to_le32(EXT4_OS_HURD)) 4185 raw_inode->i_file_acl_high = 4186 cpu_to_le16(ei->i_file_acl >> 32); 4187 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl); 4188 if (ei->i_disksize != ext4_isize(raw_inode)) { 4189 ext4_isize_set(raw_inode, ei->i_disksize); 4190 need_datasync = 1; 4191 } 4192 if (ei->i_disksize > 0x7fffffffULL) { 4193 struct super_block *sb = inode->i_sb; 4194 if (!EXT4_HAS_RO_COMPAT_FEATURE(sb, 4195 EXT4_FEATURE_RO_COMPAT_LARGE_FILE) || 4196 EXT4_SB(sb)->s_es->s_rev_level == 4197 cpu_to_le32(EXT4_GOOD_OLD_REV)) { 4198 /* If this is the first large file 4199 * created, add a flag to the superblock. 4200 */ 4201 err = ext4_journal_get_write_access(handle, 4202 EXT4_SB(sb)->s_sbh); 4203 if (err) 4204 goto out_brelse; 4205 ext4_update_dynamic_rev(sb); 4206 EXT4_SET_RO_COMPAT_FEATURE(sb, 4207 EXT4_FEATURE_RO_COMPAT_LARGE_FILE); 4208 ext4_handle_sync(handle); 4209 err = ext4_handle_dirty_super(handle, sb); 4210 } 4211 } 4212 raw_inode->i_generation = cpu_to_le32(inode->i_generation); 4213 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { 4214 if (old_valid_dev(inode->i_rdev)) { 4215 raw_inode->i_block[0] = 4216 cpu_to_le32(old_encode_dev(inode->i_rdev)); 4217 raw_inode->i_block[1] = 0; 4218 } else { 4219 raw_inode->i_block[0] = 0; 4220 raw_inode->i_block[1] = 4221 cpu_to_le32(new_encode_dev(inode->i_rdev)); 4222 raw_inode->i_block[2] = 0; 4223 } 4224 } else if (!ext4_has_inline_data(inode)) { 4225 for (block = 0; block < EXT4_N_BLOCKS; block++) 4226 raw_inode->i_block[block] = ei->i_data[block]; 4227 } 4228 4229 raw_inode->i_disk_version = cpu_to_le32(inode->i_version); 4230 if (ei->i_extra_isize) { 4231 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 4232 raw_inode->i_version_hi = 4233 cpu_to_le32(inode->i_version >> 32); 4234 raw_inode->i_extra_isize = cpu_to_le16(ei->i_extra_isize); 4235 } 4236 4237 ext4_inode_csum_set(inode, raw_inode, ei); 4238 4239 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 4240 rc = ext4_handle_dirty_metadata(handle, NULL, bh); 4241 if (!err) 4242 err = rc; 4243 ext4_clear_inode_state(inode, EXT4_STATE_NEW); 4244 4245 ext4_update_inode_fsync_trans(handle, inode, need_datasync); 4246 out_brelse: 4247 brelse(bh); 4248 ext4_std_error(inode->i_sb, err); 4249 return err; 4250 } 4251 4252 /* 4253 * ext4_write_inode() 4254 * 4255 * We are called from a few places: 4256 * 4257 * - Within generic_file_write() for O_SYNC files. 4258 * Here, there will be no transaction running. We wait for any running 4259 * transaction to commit. 4260 * 4261 * - Within sys_sync(), kupdate and such. 4262 * We wait on commit, if tol to. 4263 * 4264 * - Within prune_icache() (PF_MEMALLOC == true) 4265 * Here we simply return. We can't afford to block kswapd on the 4266 * journal commit. 4267 * 4268 * In all cases it is actually safe for us to return without doing anything, 4269 * because the inode has been copied into a raw inode buffer in 4270 * ext4_mark_inode_dirty(). This is a correctness thing for O_SYNC and for 4271 * knfsd. 4272 * 4273 * Note that we are absolutely dependent upon all inode dirtiers doing the 4274 * right thing: they *must* call mark_inode_dirty() after dirtying info in 4275 * which we are interested. 4276 * 4277 * It would be a bug for them to not do this. The code: 4278 * 4279 * mark_inode_dirty(inode) 4280 * stuff(); 4281 * inode->i_size = expr; 4282 * 4283 * is in error because a kswapd-driven write_inode() could occur while 4284 * `stuff()' is running, and the new i_size will be lost. Plus the inode 4285 * will no longer be on the superblock's dirty inode list. 4286 */ 4287 int ext4_write_inode(struct inode *inode, struct writeback_control *wbc) 4288 { 4289 int err; 4290 4291 if (current->flags & PF_MEMALLOC) 4292 return 0; 4293 4294 if (EXT4_SB(inode->i_sb)->s_journal) { 4295 if (ext4_journal_current_handle()) { 4296 jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n"); 4297 dump_stack(); 4298 return -EIO; 4299 } 4300 4301 if (wbc->sync_mode != WB_SYNC_ALL) 4302 return 0; 4303 4304 err = ext4_force_commit(inode->i_sb); 4305 } else { 4306 struct ext4_iloc iloc; 4307 4308 err = __ext4_get_inode_loc(inode, &iloc, 0); 4309 if (err) 4310 return err; 4311 if (wbc->sync_mode == WB_SYNC_ALL) 4312 sync_dirty_buffer(iloc.bh); 4313 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) { 4314 EXT4_ERROR_INODE_BLOCK(inode, iloc.bh->b_blocknr, 4315 "IO error syncing inode"); 4316 err = -EIO; 4317 } 4318 brelse(iloc.bh); 4319 } 4320 return err; 4321 } 4322 4323 /* 4324 * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate 4325 * buffers that are attached to a page stradding i_size and are undergoing 4326 * commit. In that case we have to wait for commit to finish and try again. 4327 */ 4328 static void ext4_wait_for_tail_page_commit(struct inode *inode) 4329 { 4330 struct page *page; 4331 unsigned offset; 4332 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 4333 tid_t commit_tid = 0; 4334 int ret; 4335 4336 offset = inode->i_size & (PAGE_CACHE_SIZE - 1); 4337 /* 4338 * All buffers in the last page remain valid? Then there's nothing to 4339 * do. We do the check mainly to optimize the common PAGE_CACHE_SIZE == 4340 * blocksize case 4341 */ 4342 if (offset > PAGE_CACHE_SIZE - (1 << inode->i_blkbits)) 4343 return; 4344 while (1) { 4345 page = find_lock_page(inode->i_mapping, 4346 inode->i_size >> PAGE_CACHE_SHIFT); 4347 if (!page) 4348 return; 4349 ret = __ext4_journalled_invalidatepage(page, offset); 4350 unlock_page(page); 4351 page_cache_release(page); 4352 if (ret != -EBUSY) 4353 return; 4354 commit_tid = 0; 4355 read_lock(&journal->j_state_lock); 4356 if (journal->j_committing_transaction) 4357 commit_tid = journal->j_committing_transaction->t_tid; 4358 read_unlock(&journal->j_state_lock); 4359 if (commit_tid) 4360 jbd2_log_wait_commit(journal, commit_tid); 4361 } 4362 } 4363 4364 /* 4365 * ext4_setattr() 4366 * 4367 * Called from notify_change. 4368 * 4369 * We want to trap VFS attempts to truncate the file as soon as 4370 * possible. In particular, we want to make sure that when the VFS 4371 * shrinks i_size, we put the inode on the orphan list and modify 4372 * i_disksize immediately, so that during the subsequent flushing of 4373 * dirty pages and freeing of disk blocks, we can guarantee that any 4374 * commit will leave the blocks being flushed in an unused state on 4375 * disk. (On recovery, the inode will get truncated and the blocks will 4376 * be freed, so we have a strong guarantee that no future commit will 4377 * leave these blocks visible to the user.) 4378 * 4379 * Another thing we have to assure is that if we are in ordered mode 4380 * and inode is still attached to the committing transaction, we must 4381 * we start writeout of all the dirty pages which are being truncated. 4382 * This way we are sure that all the data written in the previous 4383 * transaction are already on disk (truncate waits for pages under 4384 * writeback). 4385 * 4386 * Called with inode->i_mutex down. 4387 */ 4388 int ext4_setattr(struct dentry *dentry, struct iattr *attr) 4389 { 4390 struct inode *inode = dentry->d_inode; 4391 int error, rc = 0; 4392 int orphan = 0; 4393 const unsigned int ia_valid = attr->ia_valid; 4394 4395 error = inode_change_ok(inode, attr); 4396 if (error) 4397 return error; 4398 4399 if (is_quota_modification(inode, attr)) 4400 dquot_initialize(inode); 4401 if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) || 4402 (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) { 4403 handle_t *handle; 4404 4405 /* (user+group)*(old+new) structure, inode write (sb, 4406 * inode block, ? - but truncate inode update has it) */ 4407 handle = ext4_journal_start(inode, (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb)+ 4408 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb))+3); 4409 if (IS_ERR(handle)) { 4410 error = PTR_ERR(handle); 4411 goto err_out; 4412 } 4413 error = dquot_transfer(inode, attr); 4414 if (error) { 4415 ext4_journal_stop(handle); 4416 return error; 4417 } 4418 /* Update corresponding info in inode so that everything is in 4419 * one transaction */ 4420 if (attr->ia_valid & ATTR_UID) 4421 inode->i_uid = attr->ia_uid; 4422 if (attr->ia_valid & ATTR_GID) 4423 inode->i_gid = attr->ia_gid; 4424 error = ext4_mark_inode_dirty(handle, inode); 4425 ext4_journal_stop(handle); 4426 } 4427 4428 if (attr->ia_valid & ATTR_SIZE) { 4429 4430 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 4431 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 4432 4433 if (attr->ia_size > sbi->s_bitmap_maxbytes) 4434 return -EFBIG; 4435 } 4436 } 4437 4438 if (S_ISREG(inode->i_mode) && 4439 attr->ia_valid & ATTR_SIZE && 4440 (attr->ia_size < inode->i_size)) { 4441 handle_t *handle; 4442 4443 handle = ext4_journal_start(inode, 3); 4444 if (IS_ERR(handle)) { 4445 error = PTR_ERR(handle); 4446 goto err_out; 4447 } 4448 if (ext4_handle_valid(handle)) { 4449 error = ext4_orphan_add(handle, inode); 4450 orphan = 1; 4451 } 4452 EXT4_I(inode)->i_disksize = attr->ia_size; 4453 rc = ext4_mark_inode_dirty(handle, inode); 4454 if (!error) 4455 error = rc; 4456 ext4_journal_stop(handle); 4457 4458 if (ext4_should_order_data(inode)) { 4459 error = ext4_begin_ordered_truncate(inode, 4460 attr->ia_size); 4461 if (error) { 4462 /* Do as much error cleanup as possible */ 4463 handle = ext4_journal_start(inode, 3); 4464 if (IS_ERR(handle)) { 4465 ext4_orphan_del(NULL, inode); 4466 goto err_out; 4467 } 4468 ext4_orphan_del(handle, inode); 4469 orphan = 0; 4470 ext4_journal_stop(handle); 4471 goto err_out; 4472 } 4473 } 4474 } 4475 4476 if (attr->ia_valid & ATTR_SIZE) { 4477 if (attr->ia_size != inode->i_size) { 4478 loff_t oldsize = inode->i_size; 4479 4480 i_size_write(inode, attr->ia_size); 4481 /* 4482 * Blocks are going to be removed from the inode. Wait 4483 * for dio in flight. Temporarily disable 4484 * dioread_nolock to prevent livelock. 4485 */ 4486 if (orphan) { 4487 if (!ext4_should_journal_data(inode)) { 4488 ext4_inode_block_unlocked_dio(inode); 4489 inode_dio_wait(inode); 4490 ext4_inode_resume_unlocked_dio(inode); 4491 } else 4492 ext4_wait_for_tail_page_commit(inode); 4493 } 4494 /* 4495 * Truncate pagecache after we've waited for commit 4496 * in data=journal mode to make pages freeable. 4497 */ 4498 truncate_pagecache(inode, oldsize, inode->i_size); 4499 } 4500 ext4_truncate(inode); 4501 } 4502 4503 if (!rc) { 4504 setattr_copy(inode, attr); 4505 mark_inode_dirty(inode); 4506 } 4507 4508 /* 4509 * If the call to ext4_truncate failed to get a transaction handle at 4510 * all, we need to clean up the in-core orphan list manually. 4511 */ 4512 if (orphan && inode->i_nlink) 4513 ext4_orphan_del(NULL, inode); 4514 4515 if (!rc && (ia_valid & ATTR_MODE)) 4516 rc = ext4_acl_chmod(inode); 4517 4518 err_out: 4519 ext4_std_error(inode->i_sb, error); 4520 if (!error) 4521 error = rc; 4522 return error; 4523 } 4524 4525 int ext4_getattr(struct vfsmount *mnt, struct dentry *dentry, 4526 struct kstat *stat) 4527 { 4528 struct inode *inode; 4529 unsigned long delalloc_blocks; 4530 4531 inode = dentry->d_inode; 4532 generic_fillattr(inode, stat); 4533 4534 /* 4535 * We can't update i_blocks if the block allocation is delayed 4536 * otherwise in the case of system crash before the real block 4537 * allocation is done, we will have i_blocks inconsistent with 4538 * on-disk file blocks. 4539 * We always keep i_blocks updated together with real 4540 * allocation. But to not confuse with user, stat 4541 * will return the blocks that include the delayed allocation 4542 * blocks for this file. 4543 */ 4544 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb), 4545 EXT4_I(inode)->i_reserved_data_blocks); 4546 4547 stat->blocks += (delalloc_blocks << inode->i_sb->s_blocksize_bits)>>9; 4548 return 0; 4549 } 4550 4551 static int ext4_index_trans_blocks(struct inode *inode, int nrblocks, int chunk) 4552 { 4553 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 4554 return ext4_ind_trans_blocks(inode, nrblocks, chunk); 4555 return ext4_ext_index_trans_blocks(inode, nrblocks, chunk); 4556 } 4557 4558 /* 4559 * Account for index blocks, block groups bitmaps and block group 4560 * descriptor blocks if modify datablocks and index blocks 4561 * worse case, the indexs blocks spread over different block groups 4562 * 4563 * If datablocks are discontiguous, they are possible to spread over 4564 * different block groups too. If they are contiguous, with flexbg, 4565 * they could still across block group boundary. 4566 * 4567 * Also account for superblock, inode, quota and xattr blocks 4568 */ 4569 static int ext4_meta_trans_blocks(struct inode *inode, int nrblocks, int chunk) 4570 { 4571 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); 4572 int gdpblocks; 4573 int idxblocks; 4574 int ret = 0; 4575 4576 /* 4577 * How many index blocks need to touch to modify nrblocks? 4578 * The "Chunk" flag indicating whether the nrblocks is 4579 * physically contiguous on disk 4580 * 4581 * For Direct IO and fallocate, they calls get_block to allocate 4582 * one single extent at a time, so they could set the "Chunk" flag 4583 */ 4584 idxblocks = ext4_index_trans_blocks(inode, nrblocks, chunk); 4585 4586 ret = idxblocks; 4587 4588 /* 4589 * Now let's see how many group bitmaps and group descriptors need 4590 * to account 4591 */ 4592 groups = idxblocks; 4593 if (chunk) 4594 groups += 1; 4595 else 4596 groups += nrblocks; 4597 4598 gdpblocks = groups; 4599 if (groups > ngroups) 4600 groups = ngroups; 4601 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count) 4602 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count; 4603 4604 /* bitmaps and block group descriptor blocks */ 4605 ret += groups + gdpblocks; 4606 4607 /* Blocks for super block, inode, quota and xattr blocks */ 4608 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb); 4609 4610 return ret; 4611 } 4612 4613 /* 4614 * Calculate the total number of credits to reserve to fit 4615 * the modification of a single pages into a single transaction, 4616 * which may include multiple chunks of block allocations. 4617 * 4618 * This could be called via ext4_write_begin() 4619 * 4620 * We need to consider the worse case, when 4621 * one new block per extent. 4622 */ 4623 int ext4_writepage_trans_blocks(struct inode *inode) 4624 { 4625 int bpp = ext4_journal_blocks_per_page(inode); 4626 int ret; 4627 4628 ret = ext4_meta_trans_blocks(inode, bpp, 0); 4629 4630 /* Account for data blocks for journalled mode */ 4631 if (ext4_should_journal_data(inode)) 4632 ret += bpp; 4633 return ret; 4634 } 4635 4636 /* 4637 * Calculate the journal credits for a chunk of data modification. 4638 * 4639 * This is called from DIO, fallocate or whoever calling 4640 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks. 4641 * 4642 * journal buffers for data blocks are not included here, as DIO 4643 * and fallocate do no need to journal data buffers. 4644 */ 4645 int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) 4646 { 4647 return ext4_meta_trans_blocks(inode, nrblocks, 1); 4648 } 4649 4650 /* 4651 * The caller must have previously called ext4_reserve_inode_write(). 4652 * Give this, we know that the caller already has write access to iloc->bh. 4653 */ 4654 int ext4_mark_iloc_dirty(handle_t *handle, 4655 struct inode *inode, struct ext4_iloc *iloc) 4656 { 4657 int err = 0; 4658 4659 if (IS_I_VERSION(inode)) 4660 inode_inc_iversion(inode); 4661 4662 /* the do_update_inode consumes one bh->b_count */ 4663 get_bh(iloc->bh); 4664 4665 /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */ 4666 err = ext4_do_update_inode(handle, inode, iloc); 4667 put_bh(iloc->bh); 4668 return err; 4669 } 4670 4671 /* 4672 * On success, We end up with an outstanding reference count against 4673 * iloc->bh. This _must_ be cleaned up later. 4674 */ 4675 4676 int 4677 ext4_reserve_inode_write(handle_t *handle, struct inode *inode, 4678 struct ext4_iloc *iloc) 4679 { 4680 int err; 4681 4682 err = ext4_get_inode_loc(inode, iloc); 4683 if (!err) { 4684 BUFFER_TRACE(iloc->bh, "get_write_access"); 4685 err = ext4_journal_get_write_access(handle, iloc->bh); 4686 if (err) { 4687 brelse(iloc->bh); 4688 iloc->bh = NULL; 4689 } 4690 } 4691 ext4_std_error(inode->i_sb, err); 4692 return err; 4693 } 4694 4695 /* 4696 * Expand an inode by new_extra_isize bytes. 4697 * Returns 0 on success or negative error number on failure. 4698 */ 4699 static int ext4_expand_extra_isize(struct inode *inode, 4700 unsigned int new_extra_isize, 4701 struct ext4_iloc iloc, 4702 handle_t *handle) 4703 { 4704 struct ext4_inode *raw_inode; 4705 struct ext4_xattr_ibody_header *header; 4706 4707 if (EXT4_I(inode)->i_extra_isize >= new_extra_isize) 4708 return 0; 4709 4710 raw_inode = ext4_raw_inode(&iloc); 4711 4712 header = IHDR(inode, raw_inode); 4713 4714 /* No extended attributes present */ 4715 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) || 4716 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) { 4717 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE, 0, 4718 new_extra_isize); 4719 EXT4_I(inode)->i_extra_isize = new_extra_isize; 4720 return 0; 4721 } 4722 4723 /* try to expand with EAs present */ 4724 return ext4_expand_extra_isize_ea(inode, new_extra_isize, 4725 raw_inode, handle); 4726 } 4727 4728 /* 4729 * What we do here is to mark the in-core inode as clean with respect to inode 4730 * dirtiness (it may still be data-dirty). 4731 * This means that the in-core inode may be reaped by prune_icache 4732 * without having to perform any I/O. This is a very good thing, 4733 * because *any* task may call prune_icache - even ones which 4734 * have a transaction open against a different journal. 4735 * 4736 * Is this cheating? Not really. Sure, we haven't written the 4737 * inode out, but prune_icache isn't a user-visible syncing function. 4738 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync) 4739 * we start and wait on commits. 4740 */ 4741 int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode) 4742 { 4743 struct ext4_iloc iloc; 4744 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 4745 static unsigned int mnt_count; 4746 int err, ret; 4747 4748 might_sleep(); 4749 trace_ext4_mark_inode_dirty(inode, _RET_IP_); 4750 err = ext4_reserve_inode_write(handle, inode, &iloc); 4751 if (ext4_handle_valid(handle) && 4752 EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize && 4753 !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) { 4754 /* 4755 * We need extra buffer credits since we may write into EA block 4756 * with this same handle. If journal_extend fails, then it will 4757 * only result in a minor loss of functionality for that inode. 4758 * If this is felt to be critical, then e2fsck should be run to 4759 * force a large enough s_min_extra_isize. 4760 */ 4761 if ((jbd2_journal_extend(handle, 4762 EXT4_DATA_TRANS_BLOCKS(inode->i_sb))) == 0) { 4763 ret = ext4_expand_extra_isize(inode, 4764 sbi->s_want_extra_isize, 4765 iloc, handle); 4766 if (ret) { 4767 ext4_set_inode_state(inode, 4768 EXT4_STATE_NO_EXPAND); 4769 if (mnt_count != 4770 le16_to_cpu(sbi->s_es->s_mnt_count)) { 4771 ext4_warning(inode->i_sb, 4772 "Unable to expand inode %lu. Delete" 4773 " some EAs or run e2fsck.", 4774 inode->i_ino); 4775 mnt_count = 4776 le16_to_cpu(sbi->s_es->s_mnt_count); 4777 } 4778 } 4779 } 4780 } 4781 if (!err) 4782 err = ext4_mark_iloc_dirty(handle, inode, &iloc); 4783 return err; 4784 } 4785 4786 /* 4787 * ext4_dirty_inode() is called from __mark_inode_dirty() 4788 * 4789 * We're really interested in the case where a file is being extended. 4790 * i_size has been changed by generic_commit_write() and we thus need 4791 * to include the updated inode in the current transaction. 4792 * 4793 * Also, dquot_alloc_block() will always dirty the inode when blocks 4794 * are allocated to the file. 4795 * 4796 * If the inode is marked synchronous, we don't honour that here - doing 4797 * so would cause a commit on atime updates, which we don't bother doing. 4798 * We handle synchronous inodes at the highest possible level. 4799 */ 4800 void ext4_dirty_inode(struct inode *inode, int flags) 4801 { 4802 handle_t *handle; 4803 4804 handle = ext4_journal_start(inode, 2); 4805 if (IS_ERR(handle)) 4806 goto out; 4807 4808 ext4_mark_inode_dirty(handle, inode); 4809 4810 ext4_journal_stop(handle); 4811 out: 4812 return; 4813 } 4814 4815 #if 0 4816 /* 4817 * Bind an inode's backing buffer_head into this transaction, to prevent 4818 * it from being flushed to disk early. Unlike 4819 * ext4_reserve_inode_write, this leaves behind no bh reference and 4820 * returns no iloc structure, so the caller needs to repeat the iloc 4821 * lookup to mark the inode dirty later. 4822 */ 4823 static int ext4_pin_inode(handle_t *handle, struct inode *inode) 4824 { 4825 struct ext4_iloc iloc; 4826 4827 int err = 0; 4828 if (handle) { 4829 err = ext4_get_inode_loc(inode, &iloc); 4830 if (!err) { 4831 BUFFER_TRACE(iloc.bh, "get_write_access"); 4832 err = jbd2_journal_get_write_access(handle, iloc.bh); 4833 if (!err) 4834 err = ext4_handle_dirty_metadata(handle, 4835 NULL, 4836 iloc.bh); 4837 brelse(iloc.bh); 4838 } 4839 } 4840 ext4_std_error(inode->i_sb, err); 4841 return err; 4842 } 4843 #endif 4844 4845 int ext4_change_inode_journal_flag(struct inode *inode, int val) 4846 { 4847 journal_t *journal; 4848 handle_t *handle; 4849 int err; 4850 4851 /* 4852 * We have to be very careful here: changing a data block's 4853 * journaling status dynamically is dangerous. If we write a 4854 * data block to the journal, change the status and then delete 4855 * that block, we risk forgetting to revoke the old log record 4856 * from the journal and so a subsequent replay can corrupt data. 4857 * So, first we make sure that the journal is empty and that 4858 * nobody is changing anything. 4859 */ 4860 4861 journal = EXT4_JOURNAL(inode); 4862 if (!journal) 4863 return 0; 4864 if (is_journal_aborted(journal)) 4865 return -EROFS; 4866 /* We have to allocate physical blocks for delalloc blocks 4867 * before flushing journal. otherwise delalloc blocks can not 4868 * be allocated any more. even more truncate on delalloc blocks 4869 * could trigger BUG by flushing delalloc blocks in journal. 4870 * There is no delalloc block in non-journal data mode. 4871 */ 4872 if (val && test_opt(inode->i_sb, DELALLOC)) { 4873 err = ext4_alloc_da_blocks(inode); 4874 if (err < 0) 4875 return err; 4876 } 4877 4878 /* Wait for all existing dio workers */ 4879 ext4_inode_block_unlocked_dio(inode); 4880 inode_dio_wait(inode); 4881 4882 jbd2_journal_lock_updates(journal); 4883 4884 /* 4885 * OK, there are no updates running now, and all cached data is 4886 * synced to disk. We are now in a completely consistent state 4887 * which doesn't have anything in the journal, and we know that 4888 * no filesystem updates are running, so it is safe to modify 4889 * the inode's in-core data-journaling state flag now. 4890 */ 4891 4892 if (val) 4893 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 4894 else { 4895 jbd2_journal_flush(journal); 4896 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 4897 } 4898 ext4_set_aops(inode); 4899 4900 jbd2_journal_unlock_updates(journal); 4901 ext4_inode_resume_unlocked_dio(inode); 4902 4903 /* Finally we can mark the inode as dirty. */ 4904 4905 handle = ext4_journal_start(inode, 1); 4906 if (IS_ERR(handle)) 4907 return PTR_ERR(handle); 4908 4909 err = ext4_mark_inode_dirty(handle, inode); 4910 ext4_handle_sync(handle); 4911 ext4_journal_stop(handle); 4912 ext4_std_error(inode->i_sb, err); 4913 4914 return err; 4915 } 4916 4917 static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh) 4918 { 4919 return !buffer_mapped(bh); 4920 } 4921 4922 int ext4_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 4923 { 4924 struct page *page = vmf->page; 4925 loff_t size; 4926 unsigned long len; 4927 int ret; 4928 struct file *file = vma->vm_file; 4929 struct inode *inode = file->f_path.dentry->d_inode; 4930 struct address_space *mapping = inode->i_mapping; 4931 handle_t *handle; 4932 get_block_t *get_block; 4933 int retries = 0; 4934 4935 sb_start_pagefault(inode->i_sb); 4936 file_update_time(vma->vm_file); 4937 /* Delalloc case is easy... */ 4938 if (test_opt(inode->i_sb, DELALLOC) && 4939 !ext4_should_journal_data(inode) && 4940 !ext4_nonda_switch(inode->i_sb)) { 4941 do { 4942 ret = __block_page_mkwrite(vma, vmf, 4943 ext4_da_get_block_prep); 4944 } while (ret == -ENOSPC && 4945 ext4_should_retry_alloc(inode->i_sb, &retries)); 4946 goto out_ret; 4947 } 4948 4949 lock_page(page); 4950 size = i_size_read(inode); 4951 /* Page got truncated from under us? */ 4952 if (page->mapping != mapping || page_offset(page) > size) { 4953 unlock_page(page); 4954 ret = VM_FAULT_NOPAGE; 4955 goto out; 4956 } 4957 4958 if (page->index == size >> PAGE_CACHE_SHIFT) 4959 len = size & ~PAGE_CACHE_MASK; 4960 else 4961 len = PAGE_CACHE_SIZE; 4962 /* 4963 * Return if we have all the buffers mapped. This avoids the need to do 4964 * journal_start/journal_stop which can block and take a long time 4965 */ 4966 if (page_has_buffers(page)) { 4967 if (!ext4_walk_page_buffers(NULL, page_buffers(page), 4968 0, len, NULL, 4969 ext4_bh_unmapped)) { 4970 /* Wait so that we don't change page under IO */ 4971 wait_on_page_writeback(page); 4972 ret = VM_FAULT_LOCKED; 4973 goto out; 4974 } 4975 } 4976 unlock_page(page); 4977 /* OK, we need to fill the hole... */ 4978 if (ext4_should_dioread_nolock(inode)) 4979 get_block = ext4_get_block_write; 4980 else 4981 get_block = ext4_get_block; 4982 retry_alloc: 4983 handle = ext4_journal_start(inode, ext4_writepage_trans_blocks(inode)); 4984 if (IS_ERR(handle)) { 4985 ret = VM_FAULT_SIGBUS; 4986 goto out; 4987 } 4988 ret = __block_page_mkwrite(vma, vmf, get_block); 4989 if (!ret && ext4_should_journal_data(inode)) { 4990 if (ext4_walk_page_buffers(handle, page_buffers(page), 0, 4991 PAGE_CACHE_SIZE, NULL, do_journal_get_write_access)) { 4992 unlock_page(page); 4993 ret = VM_FAULT_SIGBUS; 4994 ext4_journal_stop(handle); 4995 goto out; 4996 } 4997 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 4998 } 4999 ext4_journal_stop(handle); 5000 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 5001 goto retry_alloc; 5002 out_ret: 5003 ret = block_page_mkwrite_return(ret); 5004 out: 5005 sb_end_pagefault(inode->i_sb); 5006 return ret; 5007 } 5008