1 /* 2 * linux/fs/ext4/inode.c 3 * 4 * Copyright (C) 1992, 1993, 1994, 1995 5 * Remy Card (card@masi.ibp.fr) 6 * Laboratoire MASI - Institut Blaise Pascal 7 * Universite Pierre et Marie Curie (Paris VI) 8 * 9 * from 10 * 11 * linux/fs/minix/inode.c 12 * 13 * Copyright (C) 1991, 1992 Linus Torvalds 14 * 15 * 64-bit file support on 64-bit platforms by Jakub Jelinek 16 * (jj@sunsite.ms.mff.cuni.cz) 17 * 18 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000 19 */ 20 21 #include <linux/fs.h> 22 #include <linux/time.h> 23 #include <linux/highuid.h> 24 #include <linux/pagemap.h> 25 #include <linux/dax.h> 26 #include <linux/quotaops.h> 27 #include <linux/string.h> 28 #include <linux/buffer_head.h> 29 #include <linux/writeback.h> 30 #include <linux/pagevec.h> 31 #include <linux/mpage.h> 32 #include <linux/namei.h> 33 #include <linux/uio.h> 34 #include <linux/bio.h> 35 #include <linux/workqueue.h> 36 #include <linux/kernel.h> 37 #include <linux/printk.h> 38 #include <linux/slab.h> 39 #include <linux/bitops.h> 40 41 #include "ext4_jbd2.h" 42 #include "xattr.h" 43 #include "acl.h" 44 #include "truncate.h" 45 46 #include <trace/events/ext4.h> 47 48 #define MPAGE_DA_EXTENT_TAIL 0x01 49 50 static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw, 51 struct ext4_inode_info *ei) 52 { 53 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 54 __u32 csum; 55 __u16 dummy_csum = 0; 56 int offset = offsetof(struct ext4_inode, i_checksum_lo); 57 unsigned int csum_size = sizeof(dummy_csum); 58 59 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset); 60 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size); 61 offset += csum_size; 62 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset, 63 EXT4_GOOD_OLD_INODE_SIZE - offset); 64 65 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 66 offset = offsetof(struct ext4_inode, i_checksum_hi); 67 csum = ext4_chksum(sbi, csum, (__u8 *)raw + 68 EXT4_GOOD_OLD_INODE_SIZE, 69 offset - EXT4_GOOD_OLD_INODE_SIZE); 70 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) { 71 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, 72 csum_size); 73 offset += csum_size; 74 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset, 75 EXT4_INODE_SIZE(inode->i_sb) - 76 offset); 77 } 78 } 79 80 return csum; 81 } 82 83 static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw, 84 struct ext4_inode_info *ei) 85 { 86 __u32 provided, calculated; 87 88 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 89 cpu_to_le32(EXT4_OS_LINUX) || 90 !ext4_has_metadata_csum(inode->i_sb)) 91 return 1; 92 93 provided = le16_to_cpu(raw->i_checksum_lo); 94 calculated = ext4_inode_csum(inode, raw, ei); 95 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 96 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 97 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16; 98 else 99 calculated &= 0xFFFF; 100 101 return provided == calculated; 102 } 103 104 static void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, 105 struct ext4_inode_info *ei) 106 { 107 __u32 csum; 108 109 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 110 cpu_to_le32(EXT4_OS_LINUX) || 111 !ext4_has_metadata_csum(inode->i_sb)) 112 return; 113 114 csum = ext4_inode_csum(inode, raw, ei); 115 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF); 116 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 117 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 118 raw->i_checksum_hi = cpu_to_le16(csum >> 16); 119 } 120 121 static inline int ext4_begin_ordered_truncate(struct inode *inode, 122 loff_t new_size) 123 { 124 trace_ext4_begin_ordered_truncate(inode, new_size); 125 /* 126 * If jinode is zero, then we never opened the file for 127 * writing, so there's no need to call 128 * jbd2_journal_begin_ordered_truncate() since there's no 129 * outstanding writes we need to flush. 130 */ 131 if (!EXT4_I(inode)->jinode) 132 return 0; 133 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), 134 EXT4_I(inode)->jinode, 135 new_size); 136 } 137 138 static void ext4_invalidatepage(struct page *page, unsigned int offset, 139 unsigned int length); 140 static int __ext4_journalled_writepage(struct page *page, unsigned int len); 141 static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh); 142 static int ext4_meta_trans_blocks(struct inode *inode, int lblocks, 143 int pextents); 144 145 /* 146 * Test whether an inode is a fast symlink. 147 */ 148 int ext4_inode_is_fast_symlink(struct inode *inode) 149 { 150 int ea_blocks = EXT4_I(inode)->i_file_acl ? 151 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0; 152 153 if (ext4_has_inline_data(inode)) 154 return 0; 155 156 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0); 157 } 158 159 /* 160 * Restart the transaction associated with *handle. This does a commit, 161 * so before we call here everything must be consistently dirtied against 162 * this transaction. 163 */ 164 int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode, 165 int nblocks) 166 { 167 int ret; 168 169 /* 170 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this 171 * moment, get_block can be called only for blocks inside i_size since 172 * page cache has been already dropped and writes are blocked by 173 * i_mutex. So we can safely drop the i_data_sem here. 174 */ 175 BUG_ON(EXT4_JOURNAL(inode) == NULL); 176 jbd_debug(2, "restarting handle %p\n", handle); 177 up_write(&EXT4_I(inode)->i_data_sem); 178 ret = ext4_journal_restart(handle, nblocks); 179 down_write(&EXT4_I(inode)->i_data_sem); 180 ext4_discard_preallocations(inode); 181 182 return ret; 183 } 184 185 /* 186 * Called at the last iput() if i_nlink is zero. 187 */ 188 void ext4_evict_inode(struct inode *inode) 189 { 190 handle_t *handle; 191 int err; 192 193 trace_ext4_evict_inode(inode); 194 195 if (inode->i_nlink) { 196 /* 197 * When journalling data dirty buffers are tracked only in the 198 * journal. So although mm thinks everything is clean and 199 * ready for reaping the inode might still have some pages to 200 * write in the running transaction or waiting to be 201 * checkpointed. Thus calling jbd2_journal_invalidatepage() 202 * (via truncate_inode_pages()) to discard these buffers can 203 * cause data loss. Also even if we did not discard these 204 * buffers, we would have no way to find them after the inode 205 * is reaped and thus user could see stale data if he tries to 206 * read them before the transaction is checkpointed. So be 207 * careful and force everything to disk here... We use 208 * ei->i_datasync_tid to store the newest transaction 209 * containing inode's data. 210 * 211 * Note that directories do not have this problem because they 212 * don't use page cache. 213 */ 214 if (inode->i_ino != EXT4_JOURNAL_INO && 215 ext4_should_journal_data(inode) && 216 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode))) { 217 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 218 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid; 219 220 jbd2_complete_transaction(journal, commit_tid); 221 filemap_write_and_wait(&inode->i_data); 222 } 223 truncate_inode_pages_final(&inode->i_data); 224 225 goto no_delete; 226 } 227 228 if (is_bad_inode(inode)) 229 goto no_delete; 230 dquot_initialize(inode); 231 232 if (ext4_should_order_data(inode)) 233 ext4_begin_ordered_truncate(inode, 0); 234 truncate_inode_pages_final(&inode->i_data); 235 236 /* 237 * Protect us against freezing - iput() caller didn't have to have any 238 * protection against it 239 */ 240 sb_start_intwrite(inode->i_sb); 241 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, 242 ext4_blocks_for_truncate(inode)+3); 243 if (IS_ERR(handle)) { 244 ext4_std_error(inode->i_sb, PTR_ERR(handle)); 245 /* 246 * If we're going to skip the normal cleanup, we still need to 247 * make sure that the in-core orphan linked list is properly 248 * cleaned up. 249 */ 250 ext4_orphan_del(NULL, inode); 251 sb_end_intwrite(inode->i_sb); 252 goto no_delete; 253 } 254 255 if (IS_SYNC(inode)) 256 ext4_handle_sync(handle); 257 inode->i_size = 0; 258 err = ext4_mark_inode_dirty(handle, inode); 259 if (err) { 260 ext4_warning(inode->i_sb, 261 "couldn't mark inode dirty (err %d)", err); 262 goto stop_handle; 263 } 264 if (inode->i_blocks) 265 ext4_truncate(inode); 266 267 /* 268 * ext4_ext_truncate() doesn't reserve any slop when it 269 * restarts journal transactions; therefore there may not be 270 * enough credits left in the handle to remove the inode from 271 * the orphan list and set the dtime field. 272 */ 273 if (!ext4_handle_has_enough_credits(handle, 3)) { 274 err = ext4_journal_extend(handle, 3); 275 if (err > 0) 276 err = ext4_journal_restart(handle, 3); 277 if (err != 0) { 278 ext4_warning(inode->i_sb, 279 "couldn't extend journal (err %d)", err); 280 stop_handle: 281 ext4_journal_stop(handle); 282 ext4_orphan_del(NULL, inode); 283 sb_end_intwrite(inode->i_sb); 284 goto no_delete; 285 } 286 } 287 288 /* 289 * Kill off the orphan record which ext4_truncate created. 290 * AKPM: I think this can be inside the above `if'. 291 * Note that ext4_orphan_del() has to be able to cope with the 292 * deletion of a non-existent orphan - this is because we don't 293 * know if ext4_truncate() actually created an orphan record. 294 * (Well, we could do this if we need to, but heck - it works) 295 */ 296 ext4_orphan_del(handle, inode); 297 EXT4_I(inode)->i_dtime = get_seconds(); 298 299 /* 300 * One subtle ordering requirement: if anything has gone wrong 301 * (transaction abort, IO errors, whatever), then we can still 302 * do these next steps (the fs will already have been marked as 303 * having errors), but we can't free the inode if the mark_dirty 304 * fails. 305 */ 306 if (ext4_mark_inode_dirty(handle, inode)) 307 /* If that failed, just do the required in-core inode clear. */ 308 ext4_clear_inode(inode); 309 else 310 ext4_free_inode(handle, inode); 311 ext4_journal_stop(handle); 312 sb_end_intwrite(inode->i_sb); 313 return; 314 no_delete: 315 ext4_clear_inode(inode); /* We must guarantee clearing of inode... */ 316 } 317 318 #ifdef CONFIG_QUOTA 319 qsize_t *ext4_get_reserved_space(struct inode *inode) 320 { 321 return &EXT4_I(inode)->i_reserved_quota; 322 } 323 #endif 324 325 /* 326 * Called with i_data_sem down, which is important since we can call 327 * ext4_discard_preallocations() from here. 328 */ 329 void ext4_da_update_reserve_space(struct inode *inode, 330 int used, int quota_claim) 331 { 332 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 333 struct ext4_inode_info *ei = EXT4_I(inode); 334 335 spin_lock(&ei->i_block_reservation_lock); 336 trace_ext4_da_update_reserve_space(inode, used, quota_claim); 337 if (unlikely(used > ei->i_reserved_data_blocks)) { 338 ext4_warning(inode->i_sb, "%s: ino %lu, used %d " 339 "with only %d reserved data blocks", 340 __func__, inode->i_ino, used, 341 ei->i_reserved_data_blocks); 342 WARN_ON(1); 343 used = ei->i_reserved_data_blocks; 344 } 345 346 /* Update per-inode reservations */ 347 ei->i_reserved_data_blocks -= used; 348 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used); 349 350 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 351 352 /* Update quota subsystem for data blocks */ 353 if (quota_claim) 354 dquot_claim_block(inode, EXT4_C2B(sbi, used)); 355 else { 356 /* 357 * We did fallocate with an offset that is already delayed 358 * allocated. So on delayed allocated writeback we should 359 * not re-claim the quota for fallocated blocks. 360 */ 361 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used)); 362 } 363 364 /* 365 * If we have done all the pending block allocations and if 366 * there aren't any writers on the inode, we can discard the 367 * inode's preallocations. 368 */ 369 if ((ei->i_reserved_data_blocks == 0) && 370 (atomic_read(&inode->i_writecount) == 0)) 371 ext4_discard_preallocations(inode); 372 } 373 374 static int __check_block_validity(struct inode *inode, const char *func, 375 unsigned int line, 376 struct ext4_map_blocks *map) 377 { 378 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), map->m_pblk, 379 map->m_len)) { 380 ext4_error_inode(inode, func, line, map->m_pblk, 381 "lblock %lu mapped to illegal pblock " 382 "(length %d)", (unsigned long) map->m_lblk, 383 map->m_len); 384 return -EFSCORRUPTED; 385 } 386 return 0; 387 } 388 389 int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, 390 ext4_lblk_t len) 391 { 392 int ret; 393 394 if (ext4_encrypted_inode(inode)) 395 return fscrypt_zeroout_range(inode, lblk, pblk, len); 396 397 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS); 398 if (ret > 0) 399 ret = 0; 400 401 return ret; 402 } 403 404 #define check_block_validity(inode, map) \ 405 __check_block_validity((inode), __func__, __LINE__, (map)) 406 407 #ifdef ES_AGGRESSIVE_TEST 408 static void ext4_map_blocks_es_recheck(handle_t *handle, 409 struct inode *inode, 410 struct ext4_map_blocks *es_map, 411 struct ext4_map_blocks *map, 412 int flags) 413 { 414 int retval; 415 416 map->m_flags = 0; 417 /* 418 * There is a race window that the result is not the same. 419 * e.g. xfstests #223 when dioread_nolock enables. The reason 420 * is that we lookup a block mapping in extent status tree with 421 * out taking i_data_sem. So at the time the unwritten extent 422 * could be converted. 423 */ 424 down_read(&EXT4_I(inode)->i_data_sem); 425 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 426 retval = ext4_ext_map_blocks(handle, inode, map, flags & 427 EXT4_GET_BLOCKS_KEEP_SIZE); 428 } else { 429 retval = ext4_ind_map_blocks(handle, inode, map, flags & 430 EXT4_GET_BLOCKS_KEEP_SIZE); 431 } 432 up_read((&EXT4_I(inode)->i_data_sem)); 433 434 /* 435 * We don't check m_len because extent will be collpased in status 436 * tree. So the m_len might not equal. 437 */ 438 if (es_map->m_lblk != map->m_lblk || 439 es_map->m_flags != map->m_flags || 440 es_map->m_pblk != map->m_pblk) { 441 printk("ES cache assertion failed for inode: %lu " 442 "es_cached ex [%d/%d/%llu/%x] != " 443 "found ex [%d/%d/%llu/%x] retval %d flags %x\n", 444 inode->i_ino, es_map->m_lblk, es_map->m_len, 445 es_map->m_pblk, es_map->m_flags, map->m_lblk, 446 map->m_len, map->m_pblk, map->m_flags, 447 retval, flags); 448 } 449 } 450 #endif /* ES_AGGRESSIVE_TEST */ 451 452 /* 453 * The ext4_map_blocks() function tries to look up the requested blocks, 454 * and returns if the blocks are already mapped. 455 * 456 * Otherwise it takes the write lock of the i_data_sem and allocate blocks 457 * and store the allocated blocks in the result buffer head and mark it 458 * mapped. 459 * 460 * If file type is extents based, it will call ext4_ext_map_blocks(), 461 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping 462 * based files 463 * 464 * On success, it returns the number of blocks being mapped or allocated. if 465 * create==0 and the blocks are pre-allocated and unwritten, the resulting @map 466 * is marked as unwritten. If the create == 1, it will mark @map as mapped. 467 * 468 * It returns 0 if plain look up failed (blocks have not been allocated), in 469 * that case, @map is returned as unmapped but we still do fill map->m_len to 470 * indicate the length of a hole starting at map->m_lblk. 471 * 472 * It returns the error in case of allocation failure. 473 */ 474 int ext4_map_blocks(handle_t *handle, struct inode *inode, 475 struct ext4_map_blocks *map, int flags) 476 { 477 struct extent_status es; 478 int retval; 479 int ret = 0; 480 #ifdef ES_AGGRESSIVE_TEST 481 struct ext4_map_blocks orig_map; 482 483 memcpy(&orig_map, map, sizeof(*map)); 484 #endif 485 486 map->m_flags = 0; 487 ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u," 488 "logical block %lu\n", inode->i_ino, flags, map->m_len, 489 (unsigned long) map->m_lblk); 490 491 /* 492 * ext4_map_blocks returns an int, and m_len is an unsigned int 493 */ 494 if (unlikely(map->m_len > INT_MAX)) 495 map->m_len = INT_MAX; 496 497 /* We can handle the block number less than EXT_MAX_BLOCKS */ 498 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS)) 499 return -EFSCORRUPTED; 500 501 /* Lookup extent status tree firstly */ 502 if (ext4_es_lookup_extent(inode, map->m_lblk, &es)) { 503 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) { 504 map->m_pblk = ext4_es_pblock(&es) + 505 map->m_lblk - es.es_lblk; 506 map->m_flags |= ext4_es_is_written(&es) ? 507 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN; 508 retval = es.es_len - (map->m_lblk - es.es_lblk); 509 if (retval > map->m_len) 510 retval = map->m_len; 511 map->m_len = retval; 512 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) { 513 map->m_pblk = 0; 514 retval = es.es_len - (map->m_lblk - es.es_lblk); 515 if (retval > map->m_len) 516 retval = map->m_len; 517 map->m_len = retval; 518 retval = 0; 519 } else { 520 BUG_ON(1); 521 } 522 #ifdef ES_AGGRESSIVE_TEST 523 ext4_map_blocks_es_recheck(handle, inode, map, 524 &orig_map, flags); 525 #endif 526 goto found; 527 } 528 529 /* 530 * Try to see if we can get the block without requesting a new 531 * file system block. 532 */ 533 down_read(&EXT4_I(inode)->i_data_sem); 534 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 535 retval = ext4_ext_map_blocks(handle, inode, map, flags & 536 EXT4_GET_BLOCKS_KEEP_SIZE); 537 } else { 538 retval = ext4_ind_map_blocks(handle, inode, map, flags & 539 EXT4_GET_BLOCKS_KEEP_SIZE); 540 } 541 if (retval > 0) { 542 unsigned int status; 543 544 if (unlikely(retval != map->m_len)) { 545 ext4_warning(inode->i_sb, 546 "ES len assertion failed for inode " 547 "%lu: retval %d != map->m_len %d", 548 inode->i_ino, retval, map->m_len); 549 WARN_ON(1); 550 } 551 552 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 553 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 554 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) && 555 !(status & EXTENT_STATUS_WRITTEN) && 556 ext4_find_delalloc_range(inode, map->m_lblk, 557 map->m_lblk + map->m_len - 1)) 558 status |= EXTENT_STATUS_DELAYED; 559 ret = ext4_es_insert_extent(inode, map->m_lblk, 560 map->m_len, map->m_pblk, status); 561 if (ret < 0) 562 retval = ret; 563 } 564 up_read((&EXT4_I(inode)->i_data_sem)); 565 566 found: 567 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 568 ret = check_block_validity(inode, map); 569 if (ret != 0) 570 return ret; 571 } 572 573 /* If it is only a block(s) look up */ 574 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) 575 return retval; 576 577 /* 578 * Returns if the blocks have already allocated 579 * 580 * Note that if blocks have been preallocated 581 * ext4_ext_get_block() returns the create = 0 582 * with buffer head unmapped. 583 */ 584 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) 585 /* 586 * If we need to convert extent to unwritten 587 * we continue and do the actual work in 588 * ext4_ext_map_blocks() 589 */ 590 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) 591 return retval; 592 593 /* 594 * Here we clear m_flags because after allocating an new extent, 595 * it will be set again. 596 */ 597 map->m_flags &= ~EXT4_MAP_FLAGS; 598 599 /* 600 * New blocks allocate and/or writing to unwritten extent 601 * will possibly result in updating i_data, so we take 602 * the write lock of i_data_sem, and call get_block() 603 * with create == 1 flag. 604 */ 605 down_write(&EXT4_I(inode)->i_data_sem); 606 607 /* 608 * We need to check for EXT4 here because migrate 609 * could have changed the inode type in between 610 */ 611 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 612 retval = ext4_ext_map_blocks(handle, inode, map, flags); 613 } else { 614 retval = ext4_ind_map_blocks(handle, inode, map, flags); 615 616 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) { 617 /* 618 * We allocated new blocks which will result in 619 * i_data's format changing. Force the migrate 620 * to fail by clearing migrate flags 621 */ 622 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE); 623 } 624 625 /* 626 * Update reserved blocks/metadata blocks after successful 627 * block allocation which had been deferred till now. We don't 628 * support fallocate for non extent files. So we can update 629 * reserve space here. 630 */ 631 if ((retval > 0) && 632 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)) 633 ext4_da_update_reserve_space(inode, retval, 1); 634 } 635 636 if (retval > 0) { 637 unsigned int status; 638 639 if (unlikely(retval != map->m_len)) { 640 ext4_warning(inode->i_sb, 641 "ES len assertion failed for inode " 642 "%lu: retval %d != map->m_len %d", 643 inode->i_ino, retval, map->m_len); 644 WARN_ON(1); 645 } 646 647 /* 648 * We have to zeroout blocks before inserting them into extent 649 * status tree. Otherwise someone could look them up there and 650 * use them before they are really zeroed. We also have to 651 * unmap metadata before zeroing as otherwise writeback can 652 * overwrite zeros with stale data from block device. 653 */ 654 if (flags & EXT4_GET_BLOCKS_ZERO && 655 map->m_flags & EXT4_MAP_MAPPED && 656 map->m_flags & EXT4_MAP_NEW) { 657 ext4_lblk_t i; 658 659 for (i = 0; i < map->m_len; i++) { 660 unmap_underlying_metadata(inode->i_sb->s_bdev, 661 map->m_pblk + i); 662 } 663 ret = ext4_issue_zeroout(inode, map->m_lblk, 664 map->m_pblk, map->m_len); 665 if (ret) { 666 retval = ret; 667 goto out_sem; 668 } 669 } 670 671 /* 672 * If the extent has been zeroed out, we don't need to update 673 * extent status tree. 674 */ 675 if ((flags & EXT4_GET_BLOCKS_PRE_IO) && 676 ext4_es_lookup_extent(inode, map->m_lblk, &es)) { 677 if (ext4_es_is_written(&es)) 678 goto out_sem; 679 } 680 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 681 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 682 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) && 683 !(status & EXTENT_STATUS_WRITTEN) && 684 ext4_find_delalloc_range(inode, map->m_lblk, 685 map->m_lblk + map->m_len - 1)) 686 status |= EXTENT_STATUS_DELAYED; 687 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len, 688 map->m_pblk, status); 689 if (ret < 0) { 690 retval = ret; 691 goto out_sem; 692 } 693 } 694 695 out_sem: 696 up_write((&EXT4_I(inode)->i_data_sem)); 697 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 698 ret = check_block_validity(inode, map); 699 if (ret != 0) 700 return ret; 701 702 /* 703 * Inodes with freshly allocated blocks where contents will be 704 * visible after transaction commit must be on transaction's 705 * ordered data list. 706 */ 707 if (map->m_flags & EXT4_MAP_NEW && 708 !(map->m_flags & EXT4_MAP_UNWRITTEN) && 709 !(flags & EXT4_GET_BLOCKS_ZERO) && 710 !IS_NOQUOTA(inode) && 711 ext4_should_order_data(inode)) { 712 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT) 713 ret = ext4_jbd2_inode_add_wait(handle, inode); 714 else 715 ret = ext4_jbd2_inode_add_write(handle, inode); 716 if (ret) 717 return ret; 718 } 719 } 720 return retval; 721 } 722 723 /* 724 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages 725 * we have to be careful as someone else may be manipulating b_state as well. 726 */ 727 static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags) 728 { 729 unsigned long old_state; 730 unsigned long new_state; 731 732 flags &= EXT4_MAP_FLAGS; 733 734 /* Dummy buffer_head? Set non-atomically. */ 735 if (!bh->b_page) { 736 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags; 737 return; 738 } 739 /* 740 * Someone else may be modifying b_state. Be careful! This is ugly but 741 * once we get rid of using bh as a container for mapping information 742 * to pass to / from get_block functions, this can go away. 743 */ 744 do { 745 old_state = READ_ONCE(bh->b_state); 746 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags; 747 } while (unlikely( 748 cmpxchg(&bh->b_state, old_state, new_state) != old_state)); 749 } 750 751 static int _ext4_get_block(struct inode *inode, sector_t iblock, 752 struct buffer_head *bh, int flags) 753 { 754 struct ext4_map_blocks map; 755 int ret = 0; 756 757 if (ext4_has_inline_data(inode)) 758 return -ERANGE; 759 760 map.m_lblk = iblock; 761 map.m_len = bh->b_size >> inode->i_blkbits; 762 763 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map, 764 flags); 765 if (ret > 0) { 766 map_bh(bh, inode->i_sb, map.m_pblk); 767 ext4_update_bh_state(bh, map.m_flags); 768 bh->b_size = inode->i_sb->s_blocksize * map.m_len; 769 ret = 0; 770 } 771 return ret; 772 } 773 774 int ext4_get_block(struct inode *inode, sector_t iblock, 775 struct buffer_head *bh, int create) 776 { 777 return _ext4_get_block(inode, iblock, bh, 778 create ? EXT4_GET_BLOCKS_CREATE : 0); 779 } 780 781 /* 782 * Get block function used when preparing for buffered write if we require 783 * creating an unwritten extent if blocks haven't been allocated. The extent 784 * will be converted to written after the IO is complete. 785 */ 786 int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, 787 struct buffer_head *bh_result, int create) 788 { 789 ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n", 790 inode->i_ino, create); 791 return _ext4_get_block(inode, iblock, bh_result, 792 EXT4_GET_BLOCKS_IO_CREATE_EXT); 793 } 794 795 /* Maximum number of blocks we map for direct IO at once. */ 796 #define DIO_MAX_BLOCKS 4096 797 798 /* 799 * Get blocks function for the cases that need to start a transaction - 800 * generally difference cases of direct IO and DAX IO. It also handles retries 801 * in case of ENOSPC. 802 */ 803 static int ext4_get_block_trans(struct inode *inode, sector_t iblock, 804 struct buffer_head *bh_result, int flags) 805 { 806 int dio_credits; 807 handle_t *handle; 808 int retries = 0; 809 int ret; 810 811 /* Trim mapping request to maximum we can map at once for DIO */ 812 if (bh_result->b_size >> inode->i_blkbits > DIO_MAX_BLOCKS) 813 bh_result->b_size = DIO_MAX_BLOCKS << inode->i_blkbits; 814 dio_credits = ext4_chunk_trans_blocks(inode, 815 bh_result->b_size >> inode->i_blkbits); 816 retry: 817 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits); 818 if (IS_ERR(handle)) 819 return PTR_ERR(handle); 820 821 ret = _ext4_get_block(inode, iblock, bh_result, flags); 822 ext4_journal_stop(handle); 823 824 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 825 goto retry; 826 return ret; 827 } 828 829 /* Get block function for DIO reads and writes to inodes without extents */ 830 int ext4_dio_get_block(struct inode *inode, sector_t iblock, 831 struct buffer_head *bh, int create) 832 { 833 /* We don't expect handle for direct IO */ 834 WARN_ON_ONCE(ext4_journal_current_handle()); 835 836 if (!create) 837 return _ext4_get_block(inode, iblock, bh, 0); 838 return ext4_get_block_trans(inode, iblock, bh, EXT4_GET_BLOCKS_CREATE); 839 } 840 841 /* 842 * Get block function for AIO DIO writes when we create unwritten extent if 843 * blocks are not allocated yet. The extent will be converted to written 844 * after IO is complete. 845 */ 846 static int ext4_dio_get_block_unwritten_async(struct inode *inode, 847 sector_t iblock, struct buffer_head *bh_result, int create) 848 { 849 int ret; 850 851 /* We don't expect handle for direct IO */ 852 WARN_ON_ONCE(ext4_journal_current_handle()); 853 854 ret = ext4_get_block_trans(inode, iblock, bh_result, 855 EXT4_GET_BLOCKS_IO_CREATE_EXT); 856 857 /* 858 * When doing DIO using unwritten extents, we need io_end to convert 859 * unwritten extents to written on IO completion. We allocate io_end 860 * once we spot unwritten extent and store it in b_private. Generic 861 * DIO code keeps b_private set and furthermore passes the value to 862 * our completion callback in 'private' argument. 863 */ 864 if (!ret && buffer_unwritten(bh_result)) { 865 if (!bh_result->b_private) { 866 ext4_io_end_t *io_end; 867 868 io_end = ext4_init_io_end(inode, GFP_KERNEL); 869 if (!io_end) 870 return -ENOMEM; 871 bh_result->b_private = io_end; 872 ext4_set_io_unwritten_flag(inode, io_end); 873 } 874 set_buffer_defer_completion(bh_result); 875 } 876 877 return ret; 878 } 879 880 /* 881 * Get block function for non-AIO DIO writes when we create unwritten extent if 882 * blocks are not allocated yet. The extent will be converted to written 883 * after IO is complete from ext4_ext_direct_IO() function. 884 */ 885 static int ext4_dio_get_block_unwritten_sync(struct inode *inode, 886 sector_t iblock, struct buffer_head *bh_result, int create) 887 { 888 int ret; 889 890 /* We don't expect handle for direct IO */ 891 WARN_ON_ONCE(ext4_journal_current_handle()); 892 893 ret = ext4_get_block_trans(inode, iblock, bh_result, 894 EXT4_GET_BLOCKS_IO_CREATE_EXT); 895 896 /* 897 * Mark inode as having pending DIO writes to unwritten extents. 898 * ext4_ext_direct_IO() checks this flag and converts extents to 899 * written. 900 */ 901 if (!ret && buffer_unwritten(bh_result)) 902 ext4_set_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN); 903 904 return ret; 905 } 906 907 static int ext4_dio_get_block_overwrite(struct inode *inode, sector_t iblock, 908 struct buffer_head *bh_result, int create) 909 { 910 int ret; 911 912 ext4_debug("ext4_dio_get_block_overwrite: inode %lu, create flag %d\n", 913 inode->i_ino, create); 914 /* We don't expect handle for direct IO */ 915 WARN_ON_ONCE(ext4_journal_current_handle()); 916 917 ret = _ext4_get_block(inode, iblock, bh_result, 0); 918 /* 919 * Blocks should have been preallocated! ext4_file_write_iter() checks 920 * that. 921 */ 922 WARN_ON_ONCE(!buffer_mapped(bh_result) || buffer_unwritten(bh_result)); 923 924 return ret; 925 } 926 927 928 /* 929 * `handle' can be NULL if create is zero 930 */ 931 struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode, 932 ext4_lblk_t block, int map_flags) 933 { 934 struct ext4_map_blocks map; 935 struct buffer_head *bh; 936 int create = map_flags & EXT4_GET_BLOCKS_CREATE; 937 int err; 938 939 J_ASSERT(handle != NULL || create == 0); 940 941 map.m_lblk = block; 942 map.m_len = 1; 943 err = ext4_map_blocks(handle, inode, &map, map_flags); 944 945 if (err == 0) 946 return create ? ERR_PTR(-ENOSPC) : NULL; 947 if (err < 0) 948 return ERR_PTR(err); 949 950 bh = sb_getblk(inode->i_sb, map.m_pblk); 951 if (unlikely(!bh)) 952 return ERR_PTR(-ENOMEM); 953 if (map.m_flags & EXT4_MAP_NEW) { 954 J_ASSERT(create != 0); 955 J_ASSERT(handle != NULL); 956 957 /* 958 * Now that we do not always journal data, we should 959 * keep in mind whether this should always journal the 960 * new buffer as metadata. For now, regular file 961 * writes use ext4_get_block instead, so it's not a 962 * problem. 963 */ 964 lock_buffer(bh); 965 BUFFER_TRACE(bh, "call get_create_access"); 966 err = ext4_journal_get_create_access(handle, bh); 967 if (unlikely(err)) { 968 unlock_buffer(bh); 969 goto errout; 970 } 971 if (!buffer_uptodate(bh)) { 972 memset(bh->b_data, 0, inode->i_sb->s_blocksize); 973 set_buffer_uptodate(bh); 974 } 975 unlock_buffer(bh); 976 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 977 err = ext4_handle_dirty_metadata(handle, inode, bh); 978 if (unlikely(err)) 979 goto errout; 980 } else 981 BUFFER_TRACE(bh, "not a new buffer"); 982 return bh; 983 errout: 984 brelse(bh); 985 return ERR_PTR(err); 986 } 987 988 struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode, 989 ext4_lblk_t block, int map_flags) 990 { 991 struct buffer_head *bh; 992 993 bh = ext4_getblk(handle, inode, block, map_flags); 994 if (IS_ERR(bh)) 995 return bh; 996 if (!bh || buffer_uptodate(bh)) 997 return bh; 998 ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh); 999 wait_on_buffer(bh); 1000 if (buffer_uptodate(bh)) 1001 return bh; 1002 put_bh(bh); 1003 return ERR_PTR(-EIO); 1004 } 1005 1006 int ext4_walk_page_buffers(handle_t *handle, 1007 struct buffer_head *head, 1008 unsigned from, 1009 unsigned to, 1010 int *partial, 1011 int (*fn)(handle_t *handle, 1012 struct buffer_head *bh)) 1013 { 1014 struct buffer_head *bh; 1015 unsigned block_start, block_end; 1016 unsigned blocksize = head->b_size; 1017 int err, ret = 0; 1018 struct buffer_head *next; 1019 1020 for (bh = head, block_start = 0; 1021 ret == 0 && (bh != head || !block_start); 1022 block_start = block_end, bh = next) { 1023 next = bh->b_this_page; 1024 block_end = block_start + blocksize; 1025 if (block_end <= from || block_start >= to) { 1026 if (partial && !buffer_uptodate(bh)) 1027 *partial = 1; 1028 continue; 1029 } 1030 err = (*fn)(handle, bh); 1031 if (!ret) 1032 ret = err; 1033 } 1034 return ret; 1035 } 1036 1037 /* 1038 * To preserve ordering, it is essential that the hole instantiation and 1039 * the data write be encapsulated in a single transaction. We cannot 1040 * close off a transaction and start a new one between the ext4_get_block() 1041 * and the commit_write(). So doing the jbd2_journal_start at the start of 1042 * prepare_write() is the right place. 1043 * 1044 * Also, this function can nest inside ext4_writepage(). In that case, we 1045 * *know* that ext4_writepage() has generated enough buffer credits to do the 1046 * whole page. So we won't block on the journal in that case, which is good, 1047 * because the caller may be PF_MEMALLOC. 1048 * 1049 * By accident, ext4 can be reentered when a transaction is open via 1050 * quota file writes. If we were to commit the transaction while thus 1051 * reentered, there can be a deadlock - we would be holding a quota 1052 * lock, and the commit would never complete if another thread had a 1053 * transaction open and was blocking on the quota lock - a ranking 1054 * violation. 1055 * 1056 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start 1057 * will _not_ run commit under these circumstances because handle->h_ref 1058 * is elevated. We'll still have enough credits for the tiny quotafile 1059 * write. 1060 */ 1061 int do_journal_get_write_access(handle_t *handle, 1062 struct buffer_head *bh) 1063 { 1064 int dirty = buffer_dirty(bh); 1065 int ret; 1066 1067 if (!buffer_mapped(bh) || buffer_freed(bh)) 1068 return 0; 1069 /* 1070 * __block_write_begin() could have dirtied some buffers. Clean 1071 * the dirty bit as jbd2_journal_get_write_access() could complain 1072 * otherwise about fs integrity issues. Setting of the dirty bit 1073 * by __block_write_begin() isn't a real problem here as we clear 1074 * the bit before releasing a page lock and thus writeback cannot 1075 * ever write the buffer. 1076 */ 1077 if (dirty) 1078 clear_buffer_dirty(bh); 1079 BUFFER_TRACE(bh, "get write access"); 1080 ret = ext4_journal_get_write_access(handle, bh); 1081 if (!ret && dirty) 1082 ret = ext4_handle_dirty_metadata(handle, NULL, bh); 1083 return ret; 1084 } 1085 1086 #ifdef CONFIG_EXT4_FS_ENCRYPTION 1087 static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, 1088 get_block_t *get_block) 1089 { 1090 unsigned from = pos & (PAGE_SIZE - 1); 1091 unsigned to = from + len; 1092 struct inode *inode = page->mapping->host; 1093 unsigned block_start, block_end; 1094 sector_t block; 1095 int err = 0; 1096 unsigned blocksize = inode->i_sb->s_blocksize; 1097 unsigned bbits; 1098 struct buffer_head *bh, *head, *wait[2], **wait_bh = wait; 1099 bool decrypt = false; 1100 1101 BUG_ON(!PageLocked(page)); 1102 BUG_ON(from > PAGE_SIZE); 1103 BUG_ON(to > PAGE_SIZE); 1104 BUG_ON(from > to); 1105 1106 if (!page_has_buffers(page)) 1107 create_empty_buffers(page, blocksize, 0); 1108 head = page_buffers(page); 1109 bbits = ilog2(blocksize); 1110 block = (sector_t)page->index << (PAGE_SHIFT - bbits); 1111 1112 for (bh = head, block_start = 0; bh != head || !block_start; 1113 block++, block_start = block_end, bh = bh->b_this_page) { 1114 block_end = block_start + blocksize; 1115 if (block_end <= from || block_start >= to) { 1116 if (PageUptodate(page)) { 1117 if (!buffer_uptodate(bh)) 1118 set_buffer_uptodate(bh); 1119 } 1120 continue; 1121 } 1122 if (buffer_new(bh)) 1123 clear_buffer_new(bh); 1124 if (!buffer_mapped(bh)) { 1125 WARN_ON(bh->b_size != blocksize); 1126 err = get_block(inode, block, bh, 1); 1127 if (err) 1128 break; 1129 if (buffer_new(bh)) { 1130 unmap_underlying_metadata(bh->b_bdev, 1131 bh->b_blocknr); 1132 if (PageUptodate(page)) { 1133 clear_buffer_new(bh); 1134 set_buffer_uptodate(bh); 1135 mark_buffer_dirty(bh); 1136 continue; 1137 } 1138 if (block_end > to || block_start < from) 1139 zero_user_segments(page, to, block_end, 1140 block_start, from); 1141 continue; 1142 } 1143 } 1144 if (PageUptodate(page)) { 1145 if (!buffer_uptodate(bh)) 1146 set_buffer_uptodate(bh); 1147 continue; 1148 } 1149 if (!buffer_uptodate(bh) && !buffer_delay(bh) && 1150 !buffer_unwritten(bh) && 1151 (block_start < from || block_end > to)) { 1152 ll_rw_block(REQ_OP_READ, 0, 1, &bh); 1153 *wait_bh++ = bh; 1154 decrypt = ext4_encrypted_inode(inode) && 1155 S_ISREG(inode->i_mode); 1156 } 1157 } 1158 /* 1159 * If we issued read requests, let them complete. 1160 */ 1161 while (wait_bh > wait) { 1162 wait_on_buffer(*--wait_bh); 1163 if (!buffer_uptodate(*wait_bh)) 1164 err = -EIO; 1165 } 1166 if (unlikely(err)) 1167 page_zero_new_buffers(page, from, to); 1168 else if (decrypt) 1169 err = fscrypt_decrypt_page(page); 1170 return err; 1171 } 1172 #endif 1173 1174 static int ext4_write_begin(struct file *file, struct address_space *mapping, 1175 loff_t pos, unsigned len, unsigned flags, 1176 struct page **pagep, void **fsdata) 1177 { 1178 struct inode *inode = mapping->host; 1179 int ret, needed_blocks; 1180 handle_t *handle; 1181 int retries = 0; 1182 struct page *page; 1183 pgoff_t index; 1184 unsigned from, to; 1185 1186 trace_ext4_write_begin(inode, pos, len, flags); 1187 /* 1188 * Reserve one block more for addition to orphan list in case 1189 * we allocate blocks but write fails for some reason 1190 */ 1191 needed_blocks = ext4_writepage_trans_blocks(inode) + 1; 1192 index = pos >> PAGE_SHIFT; 1193 from = pos & (PAGE_SIZE - 1); 1194 to = from + len; 1195 1196 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 1197 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len, 1198 flags, pagep); 1199 if (ret < 0) 1200 return ret; 1201 if (ret == 1) 1202 return 0; 1203 } 1204 1205 /* 1206 * grab_cache_page_write_begin() can take a long time if the 1207 * system is thrashing due to memory pressure, or if the page 1208 * is being written back. So grab it first before we start 1209 * the transaction handle. This also allows us to allocate 1210 * the page (if needed) without using GFP_NOFS. 1211 */ 1212 retry_grab: 1213 page = grab_cache_page_write_begin(mapping, index, flags); 1214 if (!page) 1215 return -ENOMEM; 1216 unlock_page(page); 1217 1218 retry_journal: 1219 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); 1220 if (IS_ERR(handle)) { 1221 put_page(page); 1222 return PTR_ERR(handle); 1223 } 1224 1225 lock_page(page); 1226 if (page->mapping != mapping) { 1227 /* The page got truncated from under us */ 1228 unlock_page(page); 1229 put_page(page); 1230 ext4_journal_stop(handle); 1231 goto retry_grab; 1232 } 1233 /* In case writeback began while the page was unlocked */ 1234 wait_for_stable_page(page); 1235 1236 #ifdef CONFIG_EXT4_FS_ENCRYPTION 1237 if (ext4_should_dioread_nolock(inode)) 1238 ret = ext4_block_write_begin(page, pos, len, 1239 ext4_get_block_unwritten); 1240 else 1241 ret = ext4_block_write_begin(page, pos, len, 1242 ext4_get_block); 1243 #else 1244 if (ext4_should_dioread_nolock(inode)) 1245 ret = __block_write_begin(page, pos, len, 1246 ext4_get_block_unwritten); 1247 else 1248 ret = __block_write_begin(page, pos, len, ext4_get_block); 1249 #endif 1250 if (!ret && ext4_should_journal_data(inode)) { 1251 ret = ext4_walk_page_buffers(handle, page_buffers(page), 1252 from, to, NULL, 1253 do_journal_get_write_access); 1254 } 1255 1256 if (ret) { 1257 unlock_page(page); 1258 /* 1259 * __block_write_begin may have instantiated a few blocks 1260 * outside i_size. Trim these off again. Don't need 1261 * i_size_read because we hold i_mutex. 1262 * 1263 * Add inode to orphan list in case we crash before 1264 * truncate finishes 1265 */ 1266 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1267 ext4_orphan_add(handle, inode); 1268 1269 ext4_journal_stop(handle); 1270 if (pos + len > inode->i_size) { 1271 ext4_truncate_failed_write(inode); 1272 /* 1273 * If truncate failed early the inode might 1274 * still be on the orphan list; we need to 1275 * make sure the inode is removed from the 1276 * orphan list in that case. 1277 */ 1278 if (inode->i_nlink) 1279 ext4_orphan_del(NULL, inode); 1280 } 1281 1282 if (ret == -ENOSPC && 1283 ext4_should_retry_alloc(inode->i_sb, &retries)) 1284 goto retry_journal; 1285 put_page(page); 1286 return ret; 1287 } 1288 *pagep = page; 1289 return ret; 1290 } 1291 1292 /* For write_end() in data=journal mode */ 1293 static int write_end_fn(handle_t *handle, struct buffer_head *bh) 1294 { 1295 int ret; 1296 if (!buffer_mapped(bh) || buffer_freed(bh)) 1297 return 0; 1298 set_buffer_uptodate(bh); 1299 ret = ext4_handle_dirty_metadata(handle, NULL, bh); 1300 clear_buffer_meta(bh); 1301 clear_buffer_prio(bh); 1302 return ret; 1303 } 1304 1305 /* 1306 * We need to pick up the new inode size which generic_commit_write gave us 1307 * `file' can be NULL - eg, when called from page_symlink(). 1308 * 1309 * ext4 never places buffers on inode->i_mapping->private_list. metadata 1310 * buffers are managed internally. 1311 */ 1312 static int ext4_write_end(struct file *file, 1313 struct address_space *mapping, 1314 loff_t pos, unsigned len, unsigned copied, 1315 struct page *page, void *fsdata) 1316 { 1317 handle_t *handle = ext4_journal_current_handle(); 1318 struct inode *inode = mapping->host; 1319 loff_t old_size = inode->i_size; 1320 int ret = 0, ret2; 1321 int i_size_changed = 0; 1322 1323 trace_ext4_write_end(inode, pos, len, copied); 1324 if (ext4_has_inline_data(inode)) { 1325 ret = ext4_write_inline_data_end(inode, pos, len, 1326 copied, page); 1327 if (ret < 0) 1328 goto errout; 1329 copied = ret; 1330 } else 1331 copied = block_write_end(file, mapping, pos, 1332 len, copied, page, fsdata); 1333 /* 1334 * it's important to update i_size while still holding page lock: 1335 * page writeout could otherwise come in and zero beyond i_size. 1336 */ 1337 i_size_changed = ext4_update_inode_size(inode, pos + copied); 1338 unlock_page(page); 1339 put_page(page); 1340 1341 if (old_size < pos) 1342 pagecache_isize_extended(inode, old_size, pos); 1343 /* 1344 * Don't mark the inode dirty under page lock. First, it unnecessarily 1345 * makes the holding time of page lock longer. Second, it forces lock 1346 * ordering of page lock and transaction start for journaling 1347 * filesystems. 1348 */ 1349 if (i_size_changed) 1350 ext4_mark_inode_dirty(handle, inode); 1351 1352 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1353 /* if we have allocated more blocks and copied 1354 * less. We will have blocks allocated outside 1355 * inode->i_size. So truncate them 1356 */ 1357 ext4_orphan_add(handle, inode); 1358 errout: 1359 ret2 = ext4_journal_stop(handle); 1360 if (!ret) 1361 ret = ret2; 1362 1363 if (pos + len > inode->i_size) { 1364 ext4_truncate_failed_write(inode); 1365 /* 1366 * If truncate failed early the inode might still be 1367 * on the orphan list; we need to make sure the inode 1368 * is removed from the orphan list in that case. 1369 */ 1370 if (inode->i_nlink) 1371 ext4_orphan_del(NULL, inode); 1372 } 1373 1374 return ret ? ret : copied; 1375 } 1376 1377 /* 1378 * This is a private version of page_zero_new_buffers() which doesn't 1379 * set the buffer to be dirty, since in data=journalled mode we need 1380 * to call ext4_handle_dirty_metadata() instead. 1381 */ 1382 static void zero_new_buffers(struct page *page, unsigned from, unsigned to) 1383 { 1384 unsigned int block_start = 0, block_end; 1385 struct buffer_head *head, *bh; 1386 1387 bh = head = page_buffers(page); 1388 do { 1389 block_end = block_start + bh->b_size; 1390 if (buffer_new(bh)) { 1391 if (block_end > from && block_start < to) { 1392 if (!PageUptodate(page)) { 1393 unsigned start, size; 1394 1395 start = max(from, block_start); 1396 size = min(to, block_end) - start; 1397 1398 zero_user(page, start, size); 1399 set_buffer_uptodate(bh); 1400 } 1401 clear_buffer_new(bh); 1402 } 1403 } 1404 block_start = block_end; 1405 bh = bh->b_this_page; 1406 } while (bh != head); 1407 } 1408 1409 static int ext4_journalled_write_end(struct file *file, 1410 struct address_space *mapping, 1411 loff_t pos, unsigned len, unsigned copied, 1412 struct page *page, void *fsdata) 1413 { 1414 handle_t *handle = ext4_journal_current_handle(); 1415 struct inode *inode = mapping->host; 1416 loff_t old_size = inode->i_size; 1417 int ret = 0, ret2; 1418 int partial = 0; 1419 unsigned from, to; 1420 int size_changed = 0; 1421 1422 trace_ext4_journalled_write_end(inode, pos, len, copied); 1423 from = pos & (PAGE_SIZE - 1); 1424 to = from + len; 1425 1426 BUG_ON(!ext4_handle_valid(handle)); 1427 1428 if (ext4_has_inline_data(inode)) 1429 copied = ext4_write_inline_data_end(inode, pos, len, 1430 copied, page); 1431 else { 1432 if (copied < len) { 1433 if (!PageUptodate(page)) 1434 copied = 0; 1435 zero_new_buffers(page, from+copied, to); 1436 } 1437 1438 ret = ext4_walk_page_buffers(handle, page_buffers(page), from, 1439 to, &partial, write_end_fn); 1440 if (!partial) 1441 SetPageUptodate(page); 1442 } 1443 size_changed = ext4_update_inode_size(inode, pos + copied); 1444 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 1445 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 1446 unlock_page(page); 1447 put_page(page); 1448 1449 if (old_size < pos) 1450 pagecache_isize_extended(inode, old_size, pos); 1451 1452 if (size_changed) { 1453 ret2 = ext4_mark_inode_dirty(handle, inode); 1454 if (!ret) 1455 ret = ret2; 1456 } 1457 1458 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1459 /* if we have allocated more blocks and copied 1460 * less. We will have blocks allocated outside 1461 * inode->i_size. So truncate them 1462 */ 1463 ext4_orphan_add(handle, inode); 1464 1465 ret2 = ext4_journal_stop(handle); 1466 if (!ret) 1467 ret = ret2; 1468 if (pos + len > inode->i_size) { 1469 ext4_truncate_failed_write(inode); 1470 /* 1471 * If truncate failed early the inode might still be 1472 * on the orphan list; we need to make sure the inode 1473 * is removed from the orphan list in that case. 1474 */ 1475 if (inode->i_nlink) 1476 ext4_orphan_del(NULL, inode); 1477 } 1478 1479 return ret ? ret : copied; 1480 } 1481 1482 /* 1483 * Reserve space for a single cluster 1484 */ 1485 static int ext4_da_reserve_space(struct inode *inode) 1486 { 1487 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1488 struct ext4_inode_info *ei = EXT4_I(inode); 1489 int ret; 1490 1491 /* 1492 * We will charge metadata quota at writeout time; this saves 1493 * us from metadata over-estimation, though we may go over by 1494 * a small amount in the end. Here we just reserve for data. 1495 */ 1496 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1)); 1497 if (ret) 1498 return ret; 1499 1500 spin_lock(&ei->i_block_reservation_lock); 1501 if (ext4_claim_free_clusters(sbi, 1, 0)) { 1502 spin_unlock(&ei->i_block_reservation_lock); 1503 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1)); 1504 return -ENOSPC; 1505 } 1506 ei->i_reserved_data_blocks++; 1507 trace_ext4_da_reserve_space(inode); 1508 spin_unlock(&ei->i_block_reservation_lock); 1509 1510 return 0; /* success */ 1511 } 1512 1513 static void ext4_da_release_space(struct inode *inode, int to_free) 1514 { 1515 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1516 struct ext4_inode_info *ei = EXT4_I(inode); 1517 1518 if (!to_free) 1519 return; /* Nothing to release, exit */ 1520 1521 spin_lock(&EXT4_I(inode)->i_block_reservation_lock); 1522 1523 trace_ext4_da_release_space(inode, to_free); 1524 if (unlikely(to_free > ei->i_reserved_data_blocks)) { 1525 /* 1526 * if there aren't enough reserved blocks, then the 1527 * counter is messed up somewhere. Since this 1528 * function is called from invalidate page, it's 1529 * harmless to return without any action. 1530 */ 1531 ext4_warning(inode->i_sb, "ext4_da_release_space: " 1532 "ino %lu, to_free %d with only %d reserved " 1533 "data blocks", inode->i_ino, to_free, 1534 ei->i_reserved_data_blocks); 1535 WARN_ON(1); 1536 to_free = ei->i_reserved_data_blocks; 1537 } 1538 ei->i_reserved_data_blocks -= to_free; 1539 1540 /* update fs dirty data blocks counter */ 1541 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free); 1542 1543 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 1544 1545 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free)); 1546 } 1547 1548 static void ext4_da_page_release_reservation(struct page *page, 1549 unsigned int offset, 1550 unsigned int length) 1551 { 1552 int to_release = 0, contiguous_blks = 0; 1553 struct buffer_head *head, *bh; 1554 unsigned int curr_off = 0; 1555 struct inode *inode = page->mapping->host; 1556 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1557 unsigned int stop = offset + length; 1558 int num_clusters; 1559 ext4_fsblk_t lblk; 1560 1561 BUG_ON(stop > PAGE_SIZE || stop < length); 1562 1563 head = page_buffers(page); 1564 bh = head; 1565 do { 1566 unsigned int next_off = curr_off + bh->b_size; 1567 1568 if (next_off > stop) 1569 break; 1570 1571 if ((offset <= curr_off) && (buffer_delay(bh))) { 1572 to_release++; 1573 contiguous_blks++; 1574 clear_buffer_delay(bh); 1575 } else if (contiguous_blks) { 1576 lblk = page->index << 1577 (PAGE_SHIFT - inode->i_blkbits); 1578 lblk += (curr_off >> inode->i_blkbits) - 1579 contiguous_blks; 1580 ext4_es_remove_extent(inode, lblk, contiguous_blks); 1581 contiguous_blks = 0; 1582 } 1583 curr_off = next_off; 1584 } while ((bh = bh->b_this_page) != head); 1585 1586 if (contiguous_blks) { 1587 lblk = page->index << (PAGE_SHIFT - inode->i_blkbits); 1588 lblk += (curr_off >> inode->i_blkbits) - contiguous_blks; 1589 ext4_es_remove_extent(inode, lblk, contiguous_blks); 1590 } 1591 1592 /* If we have released all the blocks belonging to a cluster, then we 1593 * need to release the reserved space for that cluster. */ 1594 num_clusters = EXT4_NUM_B2C(sbi, to_release); 1595 while (num_clusters > 0) { 1596 lblk = (page->index << (PAGE_SHIFT - inode->i_blkbits)) + 1597 ((num_clusters - 1) << sbi->s_cluster_bits); 1598 if (sbi->s_cluster_ratio == 1 || 1599 !ext4_find_delalloc_cluster(inode, lblk)) 1600 ext4_da_release_space(inode, 1); 1601 1602 num_clusters--; 1603 } 1604 } 1605 1606 /* 1607 * Delayed allocation stuff 1608 */ 1609 1610 struct mpage_da_data { 1611 struct inode *inode; 1612 struct writeback_control *wbc; 1613 1614 pgoff_t first_page; /* The first page to write */ 1615 pgoff_t next_page; /* Current page to examine */ 1616 pgoff_t last_page; /* Last page to examine */ 1617 /* 1618 * Extent to map - this can be after first_page because that can be 1619 * fully mapped. We somewhat abuse m_flags to store whether the extent 1620 * is delalloc or unwritten. 1621 */ 1622 struct ext4_map_blocks map; 1623 struct ext4_io_submit io_submit; /* IO submission data */ 1624 }; 1625 1626 static void mpage_release_unused_pages(struct mpage_da_data *mpd, 1627 bool invalidate) 1628 { 1629 int nr_pages, i; 1630 pgoff_t index, end; 1631 struct pagevec pvec; 1632 struct inode *inode = mpd->inode; 1633 struct address_space *mapping = inode->i_mapping; 1634 1635 /* This is necessary when next_page == 0. */ 1636 if (mpd->first_page >= mpd->next_page) 1637 return; 1638 1639 index = mpd->first_page; 1640 end = mpd->next_page - 1; 1641 if (invalidate) { 1642 ext4_lblk_t start, last; 1643 start = index << (PAGE_SHIFT - inode->i_blkbits); 1644 last = end << (PAGE_SHIFT - inode->i_blkbits); 1645 ext4_es_remove_extent(inode, start, last - start + 1); 1646 } 1647 1648 pagevec_init(&pvec, 0); 1649 while (index <= end) { 1650 nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE); 1651 if (nr_pages == 0) 1652 break; 1653 for (i = 0; i < nr_pages; i++) { 1654 struct page *page = pvec.pages[i]; 1655 if (page->index > end) 1656 break; 1657 BUG_ON(!PageLocked(page)); 1658 BUG_ON(PageWriteback(page)); 1659 if (invalidate) { 1660 if (page_mapped(page)) 1661 clear_page_dirty_for_io(page); 1662 block_invalidatepage(page, 0, PAGE_SIZE); 1663 ClearPageUptodate(page); 1664 } 1665 unlock_page(page); 1666 } 1667 index = pvec.pages[nr_pages - 1]->index + 1; 1668 pagevec_release(&pvec); 1669 } 1670 } 1671 1672 static void ext4_print_free_blocks(struct inode *inode) 1673 { 1674 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1675 struct super_block *sb = inode->i_sb; 1676 struct ext4_inode_info *ei = EXT4_I(inode); 1677 1678 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld", 1679 EXT4_C2B(EXT4_SB(inode->i_sb), 1680 ext4_count_free_clusters(sb))); 1681 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details"); 1682 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld", 1683 (long long) EXT4_C2B(EXT4_SB(sb), 1684 percpu_counter_sum(&sbi->s_freeclusters_counter))); 1685 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld", 1686 (long long) EXT4_C2B(EXT4_SB(sb), 1687 percpu_counter_sum(&sbi->s_dirtyclusters_counter))); 1688 ext4_msg(sb, KERN_CRIT, "Block reservation details"); 1689 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u", 1690 ei->i_reserved_data_blocks); 1691 return; 1692 } 1693 1694 static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh) 1695 { 1696 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh); 1697 } 1698 1699 /* 1700 * This function is grabs code from the very beginning of 1701 * ext4_map_blocks, but assumes that the caller is from delayed write 1702 * time. This function looks up the requested blocks and sets the 1703 * buffer delay bit under the protection of i_data_sem. 1704 */ 1705 static int ext4_da_map_blocks(struct inode *inode, sector_t iblock, 1706 struct ext4_map_blocks *map, 1707 struct buffer_head *bh) 1708 { 1709 struct extent_status es; 1710 int retval; 1711 sector_t invalid_block = ~((sector_t) 0xffff); 1712 #ifdef ES_AGGRESSIVE_TEST 1713 struct ext4_map_blocks orig_map; 1714 1715 memcpy(&orig_map, map, sizeof(*map)); 1716 #endif 1717 1718 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es)) 1719 invalid_block = ~0; 1720 1721 map->m_flags = 0; 1722 ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u," 1723 "logical block %lu\n", inode->i_ino, map->m_len, 1724 (unsigned long) map->m_lblk); 1725 1726 /* Lookup extent status tree firstly */ 1727 if (ext4_es_lookup_extent(inode, iblock, &es)) { 1728 if (ext4_es_is_hole(&es)) { 1729 retval = 0; 1730 down_read(&EXT4_I(inode)->i_data_sem); 1731 goto add_delayed; 1732 } 1733 1734 /* 1735 * Delayed extent could be allocated by fallocate. 1736 * So we need to check it. 1737 */ 1738 if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) { 1739 map_bh(bh, inode->i_sb, invalid_block); 1740 set_buffer_new(bh); 1741 set_buffer_delay(bh); 1742 return 0; 1743 } 1744 1745 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk; 1746 retval = es.es_len - (iblock - es.es_lblk); 1747 if (retval > map->m_len) 1748 retval = map->m_len; 1749 map->m_len = retval; 1750 if (ext4_es_is_written(&es)) 1751 map->m_flags |= EXT4_MAP_MAPPED; 1752 else if (ext4_es_is_unwritten(&es)) 1753 map->m_flags |= EXT4_MAP_UNWRITTEN; 1754 else 1755 BUG_ON(1); 1756 1757 #ifdef ES_AGGRESSIVE_TEST 1758 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0); 1759 #endif 1760 return retval; 1761 } 1762 1763 /* 1764 * Try to see if we can get the block without requesting a new 1765 * file system block. 1766 */ 1767 down_read(&EXT4_I(inode)->i_data_sem); 1768 if (ext4_has_inline_data(inode)) 1769 retval = 0; 1770 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 1771 retval = ext4_ext_map_blocks(NULL, inode, map, 0); 1772 else 1773 retval = ext4_ind_map_blocks(NULL, inode, map, 0); 1774 1775 add_delayed: 1776 if (retval == 0) { 1777 int ret; 1778 /* 1779 * XXX: __block_prepare_write() unmaps passed block, 1780 * is it OK? 1781 */ 1782 /* 1783 * If the block was allocated from previously allocated cluster, 1784 * then we don't need to reserve it again. However we still need 1785 * to reserve metadata for every block we're going to write. 1786 */ 1787 if (EXT4_SB(inode->i_sb)->s_cluster_ratio == 1 || 1788 !ext4_find_delalloc_cluster(inode, map->m_lblk)) { 1789 ret = ext4_da_reserve_space(inode); 1790 if (ret) { 1791 /* not enough space to reserve */ 1792 retval = ret; 1793 goto out_unlock; 1794 } 1795 } 1796 1797 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len, 1798 ~0, EXTENT_STATUS_DELAYED); 1799 if (ret) { 1800 retval = ret; 1801 goto out_unlock; 1802 } 1803 1804 map_bh(bh, inode->i_sb, invalid_block); 1805 set_buffer_new(bh); 1806 set_buffer_delay(bh); 1807 } else if (retval > 0) { 1808 int ret; 1809 unsigned int status; 1810 1811 if (unlikely(retval != map->m_len)) { 1812 ext4_warning(inode->i_sb, 1813 "ES len assertion failed for inode " 1814 "%lu: retval %d != map->m_len %d", 1815 inode->i_ino, retval, map->m_len); 1816 WARN_ON(1); 1817 } 1818 1819 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 1820 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 1821 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len, 1822 map->m_pblk, status); 1823 if (ret != 0) 1824 retval = ret; 1825 } 1826 1827 out_unlock: 1828 up_read((&EXT4_I(inode)->i_data_sem)); 1829 1830 return retval; 1831 } 1832 1833 /* 1834 * This is a special get_block_t callback which is used by 1835 * ext4_da_write_begin(). It will either return mapped block or 1836 * reserve space for a single block. 1837 * 1838 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set. 1839 * We also have b_blocknr = -1 and b_bdev initialized properly 1840 * 1841 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set. 1842 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev 1843 * initialized properly. 1844 */ 1845 int ext4_da_get_block_prep(struct inode *inode, sector_t iblock, 1846 struct buffer_head *bh, int create) 1847 { 1848 struct ext4_map_blocks map; 1849 int ret = 0; 1850 1851 BUG_ON(create == 0); 1852 BUG_ON(bh->b_size != inode->i_sb->s_blocksize); 1853 1854 map.m_lblk = iblock; 1855 map.m_len = 1; 1856 1857 /* 1858 * first, we need to know whether the block is allocated already 1859 * preallocated blocks are unmapped but should treated 1860 * the same as allocated blocks. 1861 */ 1862 ret = ext4_da_map_blocks(inode, iblock, &map, bh); 1863 if (ret <= 0) 1864 return ret; 1865 1866 map_bh(bh, inode->i_sb, map.m_pblk); 1867 ext4_update_bh_state(bh, map.m_flags); 1868 1869 if (buffer_unwritten(bh)) { 1870 /* A delayed write to unwritten bh should be marked 1871 * new and mapped. Mapped ensures that we don't do 1872 * get_block multiple times when we write to the same 1873 * offset and new ensures that we do proper zero out 1874 * for partial write. 1875 */ 1876 set_buffer_new(bh); 1877 set_buffer_mapped(bh); 1878 } 1879 return 0; 1880 } 1881 1882 static int bget_one(handle_t *handle, struct buffer_head *bh) 1883 { 1884 get_bh(bh); 1885 return 0; 1886 } 1887 1888 static int bput_one(handle_t *handle, struct buffer_head *bh) 1889 { 1890 put_bh(bh); 1891 return 0; 1892 } 1893 1894 static int __ext4_journalled_writepage(struct page *page, 1895 unsigned int len) 1896 { 1897 struct address_space *mapping = page->mapping; 1898 struct inode *inode = mapping->host; 1899 struct buffer_head *page_bufs = NULL; 1900 handle_t *handle = NULL; 1901 int ret = 0, err = 0; 1902 int inline_data = ext4_has_inline_data(inode); 1903 struct buffer_head *inode_bh = NULL; 1904 1905 ClearPageChecked(page); 1906 1907 if (inline_data) { 1908 BUG_ON(page->index != 0); 1909 BUG_ON(len > ext4_get_max_inline_size(inode)); 1910 inode_bh = ext4_journalled_write_inline_data(inode, len, page); 1911 if (inode_bh == NULL) 1912 goto out; 1913 } else { 1914 page_bufs = page_buffers(page); 1915 if (!page_bufs) { 1916 BUG(); 1917 goto out; 1918 } 1919 ext4_walk_page_buffers(handle, page_bufs, 0, len, 1920 NULL, bget_one); 1921 } 1922 /* 1923 * We need to release the page lock before we start the 1924 * journal, so grab a reference so the page won't disappear 1925 * out from under us. 1926 */ 1927 get_page(page); 1928 unlock_page(page); 1929 1930 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, 1931 ext4_writepage_trans_blocks(inode)); 1932 if (IS_ERR(handle)) { 1933 ret = PTR_ERR(handle); 1934 put_page(page); 1935 goto out_no_pagelock; 1936 } 1937 BUG_ON(!ext4_handle_valid(handle)); 1938 1939 lock_page(page); 1940 put_page(page); 1941 if (page->mapping != mapping) { 1942 /* The page got truncated from under us */ 1943 ext4_journal_stop(handle); 1944 ret = 0; 1945 goto out; 1946 } 1947 1948 if (inline_data) { 1949 BUFFER_TRACE(inode_bh, "get write access"); 1950 ret = ext4_journal_get_write_access(handle, inode_bh); 1951 1952 err = ext4_handle_dirty_metadata(handle, inode, inode_bh); 1953 1954 } else { 1955 ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL, 1956 do_journal_get_write_access); 1957 1958 err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL, 1959 write_end_fn); 1960 } 1961 if (ret == 0) 1962 ret = err; 1963 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 1964 err = ext4_journal_stop(handle); 1965 if (!ret) 1966 ret = err; 1967 1968 if (!ext4_has_inline_data(inode)) 1969 ext4_walk_page_buffers(NULL, page_bufs, 0, len, 1970 NULL, bput_one); 1971 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 1972 out: 1973 unlock_page(page); 1974 out_no_pagelock: 1975 brelse(inode_bh); 1976 return ret; 1977 } 1978 1979 /* 1980 * Note that we don't need to start a transaction unless we're journaling data 1981 * because we should have holes filled from ext4_page_mkwrite(). We even don't 1982 * need to file the inode to the transaction's list in ordered mode because if 1983 * we are writing back data added by write(), the inode is already there and if 1984 * we are writing back data modified via mmap(), no one guarantees in which 1985 * transaction the data will hit the disk. In case we are journaling data, we 1986 * cannot start transaction directly because transaction start ranks above page 1987 * lock so we have to do some magic. 1988 * 1989 * This function can get called via... 1990 * - ext4_writepages after taking page lock (have journal handle) 1991 * - journal_submit_inode_data_buffers (no journal handle) 1992 * - shrink_page_list via the kswapd/direct reclaim (no journal handle) 1993 * - grab_page_cache when doing write_begin (have journal handle) 1994 * 1995 * We don't do any block allocation in this function. If we have page with 1996 * multiple blocks we need to write those buffer_heads that are mapped. This 1997 * is important for mmaped based write. So if we do with blocksize 1K 1998 * truncate(f, 1024); 1999 * a = mmap(f, 0, 4096); 2000 * a[0] = 'a'; 2001 * truncate(f, 4096); 2002 * we have in the page first buffer_head mapped via page_mkwrite call back 2003 * but other buffer_heads would be unmapped but dirty (dirty done via the 2004 * do_wp_page). So writepage should write the first block. If we modify 2005 * the mmap area beyond 1024 we will again get a page_fault and the 2006 * page_mkwrite callback will do the block allocation and mark the 2007 * buffer_heads mapped. 2008 * 2009 * We redirty the page if we have any buffer_heads that is either delay or 2010 * unwritten in the page. 2011 * 2012 * We can get recursively called as show below. 2013 * 2014 * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() -> 2015 * ext4_writepage() 2016 * 2017 * But since we don't do any block allocation we should not deadlock. 2018 * Page also have the dirty flag cleared so we don't get recurive page_lock. 2019 */ 2020 static int ext4_writepage(struct page *page, 2021 struct writeback_control *wbc) 2022 { 2023 int ret = 0; 2024 loff_t size; 2025 unsigned int len; 2026 struct buffer_head *page_bufs = NULL; 2027 struct inode *inode = page->mapping->host; 2028 struct ext4_io_submit io_submit; 2029 bool keep_towrite = false; 2030 2031 trace_ext4_writepage(page); 2032 size = i_size_read(inode); 2033 if (page->index == size >> PAGE_SHIFT) 2034 len = size & ~PAGE_MASK; 2035 else 2036 len = PAGE_SIZE; 2037 2038 page_bufs = page_buffers(page); 2039 /* 2040 * We cannot do block allocation or other extent handling in this 2041 * function. If there are buffers needing that, we have to redirty 2042 * the page. But we may reach here when we do a journal commit via 2043 * journal_submit_inode_data_buffers() and in that case we must write 2044 * allocated buffers to achieve data=ordered mode guarantees. 2045 * 2046 * Also, if there is only one buffer per page (the fs block 2047 * size == the page size), if one buffer needs block 2048 * allocation or needs to modify the extent tree to clear the 2049 * unwritten flag, we know that the page can't be written at 2050 * all, so we might as well refuse the write immediately. 2051 * Unfortunately if the block size != page size, we can't as 2052 * easily detect this case using ext4_walk_page_buffers(), but 2053 * for the extremely common case, this is an optimization that 2054 * skips a useless round trip through ext4_bio_write_page(). 2055 */ 2056 if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL, 2057 ext4_bh_delay_or_unwritten)) { 2058 redirty_page_for_writepage(wbc, page); 2059 if ((current->flags & PF_MEMALLOC) || 2060 (inode->i_sb->s_blocksize == PAGE_SIZE)) { 2061 /* 2062 * For memory cleaning there's no point in writing only 2063 * some buffers. So just bail out. Warn if we came here 2064 * from direct reclaim. 2065 */ 2066 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) 2067 == PF_MEMALLOC); 2068 unlock_page(page); 2069 return 0; 2070 } 2071 keep_towrite = true; 2072 } 2073 2074 if (PageChecked(page) && ext4_should_journal_data(inode)) 2075 /* 2076 * It's mmapped pagecache. Add buffers and journal it. There 2077 * doesn't seem much point in redirtying the page here. 2078 */ 2079 return __ext4_journalled_writepage(page, len); 2080 2081 ext4_io_submit_init(&io_submit, wbc); 2082 io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS); 2083 if (!io_submit.io_end) { 2084 redirty_page_for_writepage(wbc, page); 2085 unlock_page(page); 2086 return -ENOMEM; 2087 } 2088 ret = ext4_bio_write_page(&io_submit, page, len, wbc, keep_towrite); 2089 ext4_io_submit(&io_submit); 2090 /* Drop io_end reference we got from init */ 2091 ext4_put_io_end_defer(io_submit.io_end); 2092 return ret; 2093 } 2094 2095 static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page) 2096 { 2097 int len; 2098 loff_t size = i_size_read(mpd->inode); 2099 int err; 2100 2101 BUG_ON(page->index != mpd->first_page); 2102 if (page->index == size >> PAGE_SHIFT) 2103 len = size & ~PAGE_MASK; 2104 else 2105 len = PAGE_SIZE; 2106 clear_page_dirty_for_io(page); 2107 err = ext4_bio_write_page(&mpd->io_submit, page, len, mpd->wbc, false); 2108 if (!err) 2109 mpd->wbc->nr_to_write--; 2110 mpd->first_page++; 2111 2112 return err; 2113 } 2114 2115 #define BH_FLAGS ((1 << BH_Unwritten) | (1 << BH_Delay)) 2116 2117 /* 2118 * mballoc gives us at most this number of blocks... 2119 * XXX: That seems to be only a limitation of ext4_mb_normalize_request(). 2120 * The rest of mballoc seems to handle chunks up to full group size. 2121 */ 2122 #define MAX_WRITEPAGES_EXTENT_LEN 2048 2123 2124 /* 2125 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map 2126 * 2127 * @mpd - extent of blocks 2128 * @lblk - logical number of the block in the file 2129 * @bh - buffer head we want to add to the extent 2130 * 2131 * The function is used to collect contig. blocks in the same state. If the 2132 * buffer doesn't require mapping for writeback and we haven't started the 2133 * extent of buffers to map yet, the function returns 'true' immediately - the 2134 * caller can write the buffer right away. Otherwise the function returns true 2135 * if the block has been added to the extent, false if the block couldn't be 2136 * added. 2137 */ 2138 static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk, 2139 struct buffer_head *bh) 2140 { 2141 struct ext4_map_blocks *map = &mpd->map; 2142 2143 /* Buffer that doesn't need mapping for writeback? */ 2144 if (!buffer_dirty(bh) || !buffer_mapped(bh) || 2145 (!buffer_delay(bh) && !buffer_unwritten(bh))) { 2146 /* So far no extent to map => we write the buffer right away */ 2147 if (map->m_len == 0) 2148 return true; 2149 return false; 2150 } 2151 2152 /* First block in the extent? */ 2153 if (map->m_len == 0) { 2154 map->m_lblk = lblk; 2155 map->m_len = 1; 2156 map->m_flags = bh->b_state & BH_FLAGS; 2157 return true; 2158 } 2159 2160 /* Don't go larger than mballoc is willing to allocate */ 2161 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN) 2162 return false; 2163 2164 /* Can we merge the block to our big extent? */ 2165 if (lblk == map->m_lblk + map->m_len && 2166 (bh->b_state & BH_FLAGS) == map->m_flags) { 2167 map->m_len++; 2168 return true; 2169 } 2170 return false; 2171 } 2172 2173 /* 2174 * mpage_process_page_bufs - submit page buffers for IO or add them to extent 2175 * 2176 * @mpd - extent of blocks for mapping 2177 * @head - the first buffer in the page 2178 * @bh - buffer we should start processing from 2179 * @lblk - logical number of the block in the file corresponding to @bh 2180 * 2181 * Walk through page buffers from @bh upto @head (exclusive) and either submit 2182 * the page for IO if all buffers in this page were mapped and there's no 2183 * accumulated extent of buffers to map or add buffers in the page to the 2184 * extent of buffers to map. The function returns 1 if the caller can continue 2185 * by processing the next page, 0 if it should stop adding buffers to the 2186 * extent to map because we cannot extend it anymore. It can also return value 2187 * < 0 in case of error during IO submission. 2188 */ 2189 static int mpage_process_page_bufs(struct mpage_da_data *mpd, 2190 struct buffer_head *head, 2191 struct buffer_head *bh, 2192 ext4_lblk_t lblk) 2193 { 2194 struct inode *inode = mpd->inode; 2195 int err; 2196 ext4_lblk_t blocks = (i_size_read(inode) + (1 << inode->i_blkbits) - 1) 2197 >> inode->i_blkbits; 2198 2199 do { 2200 BUG_ON(buffer_locked(bh)); 2201 2202 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) { 2203 /* Found extent to map? */ 2204 if (mpd->map.m_len) 2205 return 0; 2206 /* Everything mapped so far and we hit EOF */ 2207 break; 2208 } 2209 } while (lblk++, (bh = bh->b_this_page) != head); 2210 /* So far everything mapped? Submit the page for IO. */ 2211 if (mpd->map.m_len == 0) { 2212 err = mpage_submit_page(mpd, head->b_page); 2213 if (err < 0) 2214 return err; 2215 } 2216 return lblk < blocks; 2217 } 2218 2219 /* 2220 * mpage_map_buffers - update buffers corresponding to changed extent and 2221 * submit fully mapped pages for IO 2222 * 2223 * @mpd - description of extent to map, on return next extent to map 2224 * 2225 * Scan buffers corresponding to changed extent (we expect corresponding pages 2226 * to be already locked) and update buffer state according to new extent state. 2227 * We map delalloc buffers to their physical location, clear unwritten bits, 2228 * and mark buffers as uninit when we perform writes to unwritten extents 2229 * and do extent conversion after IO is finished. If the last page is not fully 2230 * mapped, we update @map to the next extent in the last page that needs 2231 * mapping. Otherwise we submit the page for IO. 2232 */ 2233 static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd) 2234 { 2235 struct pagevec pvec; 2236 int nr_pages, i; 2237 struct inode *inode = mpd->inode; 2238 struct buffer_head *head, *bh; 2239 int bpp_bits = PAGE_SHIFT - inode->i_blkbits; 2240 pgoff_t start, end; 2241 ext4_lblk_t lblk; 2242 sector_t pblock; 2243 int err; 2244 2245 start = mpd->map.m_lblk >> bpp_bits; 2246 end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits; 2247 lblk = start << bpp_bits; 2248 pblock = mpd->map.m_pblk; 2249 2250 pagevec_init(&pvec, 0); 2251 while (start <= end) { 2252 nr_pages = pagevec_lookup(&pvec, inode->i_mapping, start, 2253 PAGEVEC_SIZE); 2254 if (nr_pages == 0) 2255 break; 2256 for (i = 0; i < nr_pages; i++) { 2257 struct page *page = pvec.pages[i]; 2258 2259 if (page->index > end) 2260 break; 2261 /* Up to 'end' pages must be contiguous */ 2262 BUG_ON(page->index != start); 2263 bh = head = page_buffers(page); 2264 do { 2265 if (lblk < mpd->map.m_lblk) 2266 continue; 2267 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) { 2268 /* 2269 * Buffer after end of mapped extent. 2270 * Find next buffer in the page to map. 2271 */ 2272 mpd->map.m_len = 0; 2273 mpd->map.m_flags = 0; 2274 /* 2275 * FIXME: If dioread_nolock supports 2276 * blocksize < pagesize, we need to make 2277 * sure we add size mapped so far to 2278 * io_end->size as the following call 2279 * can submit the page for IO. 2280 */ 2281 err = mpage_process_page_bufs(mpd, head, 2282 bh, lblk); 2283 pagevec_release(&pvec); 2284 if (err > 0) 2285 err = 0; 2286 return err; 2287 } 2288 if (buffer_delay(bh)) { 2289 clear_buffer_delay(bh); 2290 bh->b_blocknr = pblock++; 2291 } 2292 clear_buffer_unwritten(bh); 2293 } while (lblk++, (bh = bh->b_this_page) != head); 2294 2295 /* 2296 * FIXME: This is going to break if dioread_nolock 2297 * supports blocksize < pagesize as we will try to 2298 * convert potentially unmapped parts of inode. 2299 */ 2300 mpd->io_submit.io_end->size += PAGE_SIZE; 2301 /* Page fully mapped - let IO run! */ 2302 err = mpage_submit_page(mpd, page); 2303 if (err < 0) { 2304 pagevec_release(&pvec); 2305 return err; 2306 } 2307 start++; 2308 } 2309 pagevec_release(&pvec); 2310 } 2311 /* Extent fully mapped and matches with page boundary. We are done. */ 2312 mpd->map.m_len = 0; 2313 mpd->map.m_flags = 0; 2314 return 0; 2315 } 2316 2317 static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd) 2318 { 2319 struct inode *inode = mpd->inode; 2320 struct ext4_map_blocks *map = &mpd->map; 2321 int get_blocks_flags; 2322 int err, dioread_nolock; 2323 2324 trace_ext4_da_write_pages_extent(inode, map); 2325 /* 2326 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or 2327 * to convert an unwritten extent to be initialized (in the case 2328 * where we have written into one or more preallocated blocks). It is 2329 * possible that we're going to need more metadata blocks than 2330 * previously reserved. However we must not fail because we're in 2331 * writeback and there is nothing we can do about it so it might result 2332 * in data loss. So use reserved blocks to allocate metadata if 2333 * possible. 2334 * 2335 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if 2336 * the blocks in question are delalloc blocks. This indicates 2337 * that the blocks and quotas has already been checked when 2338 * the data was copied into the page cache. 2339 */ 2340 get_blocks_flags = EXT4_GET_BLOCKS_CREATE | 2341 EXT4_GET_BLOCKS_METADATA_NOFAIL | 2342 EXT4_GET_BLOCKS_IO_SUBMIT; 2343 dioread_nolock = ext4_should_dioread_nolock(inode); 2344 if (dioread_nolock) 2345 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT; 2346 if (map->m_flags & (1 << BH_Delay)) 2347 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE; 2348 2349 err = ext4_map_blocks(handle, inode, map, get_blocks_flags); 2350 if (err < 0) 2351 return err; 2352 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) { 2353 if (!mpd->io_submit.io_end->handle && 2354 ext4_handle_valid(handle)) { 2355 mpd->io_submit.io_end->handle = handle->h_rsv_handle; 2356 handle->h_rsv_handle = NULL; 2357 } 2358 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end); 2359 } 2360 2361 BUG_ON(map->m_len == 0); 2362 if (map->m_flags & EXT4_MAP_NEW) { 2363 struct block_device *bdev = inode->i_sb->s_bdev; 2364 int i; 2365 2366 for (i = 0; i < map->m_len; i++) 2367 unmap_underlying_metadata(bdev, map->m_pblk + i); 2368 } 2369 return 0; 2370 } 2371 2372 /* 2373 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length 2374 * mpd->len and submit pages underlying it for IO 2375 * 2376 * @handle - handle for journal operations 2377 * @mpd - extent to map 2378 * @give_up_on_write - we set this to true iff there is a fatal error and there 2379 * is no hope of writing the data. The caller should discard 2380 * dirty pages to avoid infinite loops. 2381 * 2382 * The function maps extent starting at mpd->lblk of length mpd->len. If it is 2383 * delayed, blocks are allocated, if it is unwritten, we may need to convert 2384 * them to initialized or split the described range from larger unwritten 2385 * extent. Note that we need not map all the described range since allocation 2386 * can return less blocks or the range is covered by more unwritten extents. We 2387 * cannot map more because we are limited by reserved transaction credits. On 2388 * the other hand we always make sure that the last touched page is fully 2389 * mapped so that it can be written out (and thus forward progress is 2390 * guaranteed). After mapping we submit all mapped pages for IO. 2391 */ 2392 static int mpage_map_and_submit_extent(handle_t *handle, 2393 struct mpage_da_data *mpd, 2394 bool *give_up_on_write) 2395 { 2396 struct inode *inode = mpd->inode; 2397 struct ext4_map_blocks *map = &mpd->map; 2398 int err; 2399 loff_t disksize; 2400 int progress = 0; 2401 2402 mpd->io_submit.io_end->offset = 2403 ((loff_t)map->m_lblk) << inode->i_blkbits; 2404 do { 2405 err = mpage_map_one_extent(handle, mpd); 2406 if (err < 0) { 2407 struct super_block *sb = inode->i_sb; 2408 2409 if (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED) 2410 goto invalidate_dirty_pages; 2411 /* 2412 * Let the uper layers retry transient errors. 2413 * In the case of ENOSPC, if ext4_count_free_blocks() 2414 * is non-zero, a commit should free up blocks. 2415 */ 2416 if ((err == -ENOMEM) || 2417 (err == -ENOSPC && ext4_count_free_clusters(sb))) { 2418 if (progress) 2419 goto update_disksize; 2420 return err; 2421 } 2422 ext4_msg(sb, KERN_CRIT, 2423 "Delayed block allocation failed for " 2424 "inode %lu at logical offset %llu with" 2425 " max blocks %u with error %d", 2426 inode->i_ino, 2427 (unsigned long long)map->m_lblk, 2428 (unsigned)map->m_len, -err); 2429 ext4_msg(sb, KERN_CRIT, 2430 "This should not happen!! Data will " 2431 "be lost\n"); 2432 if (err == -ENOSPC) 2433 ext4_print_free_blocks(inode); 2434 invalidate_dirty_pages: 2435 *give_up_on_write = true; 2436 return err; 2437 } 2438 progress = 1; 2439 /* 2440 * Update buffer state, submit mapped pages, and get us new 2441 * extent to map 2442 */ 2443 err = mpage_map_and_submit_buffers(mpd); 2444 if (err < 0) 2445 goto update_disksize; 2446 } while (map->m_len); 2447 2448 update_disksize: 2449 /* 2450 * Update on-disk size after IO is submitted. Races with 2451 * truncate are avoided by checking i_size under i_data_sem. 2452 */ 2453 disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT; 2454 if (disksize > EXT4_I(inode)->i_disksize) { 2455 int err2; 2456 loff_t i_size; 2457 2458 down_write(&EXT4_I(inode)->i_data_sem); 2459 i_size = i_size_read(inode); 2460 if (disksize > i_size) 2461 disksize = i_size; 2462 if (disksize > EXT4_I(inode)->i_disksize) 2463 EXT4_I(inode)->i_disksize = disksize; 2464 err2 = ext4_mark_inode_dirty(handle, inode); 2465 up_write(&EXT4_I(inode)->i_data_sem); 2466 if (err2) 2467 ext4_error(inode->i_sb, 2468 "Failed to mark inode %lu dirty", 2469 inode->i_ino); 2470 if (!err) 2471 err = err2; 2472 } 2473 return err; 2474 } 2475 2476 /* 2477 * Calculate the total number of credits to reserve for one writepages 2478 * iteration. This is called from ext4_writepages(). We map an extent of 2479 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping 2480 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN + 2481 * bpp - 1 blocks in bpp different extents. 2482 */ 2483 static int ext4_da_writepages_trans_blocks(struct inode *inode) 2484 { 2485 int bpp = ext4_journal_blocks_per_page(inode); 2486 2487 return ext4_meta_trans_blocks(inode, 2488 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp); 2489 } 2490 2491 /* 2492 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages 2493 * and underlying extent to map 2494 * 2495 * @mpd - where to look for pages 2496 * 2497 * Walk dirty pages in the mapping. If they are fully mapped, submit them for 2498 * IO immediately. When we find a page which isn't mapped we start accumulating 2499 * extent of buffers underlying these pages that needs mapping (formed by 2500 * either delayed or unwritten buffers). We also lock the pages containing 2501 * these buffers. The extent found is returned in @mpd structure (starting at 2502 * mpd->lblk with length mpd->len blocks). 2503 * 2504 * Note that this function can attach bios to one io_end structure which are 2505 * neither logically nor physically contiguous. Although it may seem as an 2506 * unnecessary complication, it is actually inevitable in blocksize < pagesize 2507 * case as we need to track IO to all buffers underlying a page in one io_end. 2508 */ 2509 static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) 2510 { 2511 struct address_space *mapping = mpd->inode->i_mapping; 2512 struct pagevec pvec; 2513 unsigned int nr_pages; 2514 long left = mpd->wbc->nr_to_write; 2515 pgoff_t index = mpd->first_page; 2516 pgoff_t end = mpd->last_page; 2517 int tag; 2518 int i, err = 0; 2519 int blkbits = mpd->inode->i_blkbits; 2520 ext4_lblk_t lblk; 2521 struct buffer_head *head; 2522 2523 if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages) 2524 tag = PAGECACHE_TAG_TOWRITE; 2525 else 2526 tag = PAGECACHE_TAG_DIRTY; 2527 2528 pagevec_init(&pvec, 0); 2529 mpd->map.m_len = 0; 2530 mpd->next_page = index; 2531 while (index <= end) { 2532 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 2533 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); 2534 if (nr_pages == 0) 2535 goto out; 2536 2537 for (i = 0; i < nr_pages; i++) { 2538 struct page *page = pvec.pages[i]; 2539 2540 /* 2541 * At this point, the page may be truncated or 2542 * invalidated (changing page->mapping to NULL), or 2543 * even swizzled back from swapper_space to tmpfs file 2544 * mapping. However, page->index will not change 2545 * because we have a reference on the page. 2546 */ 2547 if (page->index > end) 2548 goto out; 2549 2550 /* 2551 * Accumulated enough dirty pages? This doesn't apply 2552 * to WB_SYNC_ALL mode. For integrity sync we have to 2553 * keep going because someone may be concurrently 2554 * dirtying pages, and we might have synced a lot of 2555 * newly appeared dirty pages, but have not synced all 2556 * of the old dirty pages. 2557 */ 2558 if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0) 2559 goto out; 2560 2561 /* If we can't merge this page, we are done. */ 2562 if (mpd->map.m_len > 0 && mpd->next_page != page->index) 2563 goto out; 2564 2565 lock_page(page); 2566 /* 2567 * If the page is no longer dirty, or its mapping no 2568 * longer corresponds to inode we are writing (which 2569 * means it has been truncated or invalidated), or the 2570 * page is already under writeback and we are not doing 2571 * a data integrity writeback, skip the page 2572 */ 2573 if (!PageDirty(page) || 2574 (PageWriteback(page) && 2575 (mpd->wbc->sync_mode == WB_SYNC_NONE)) || 2576 unlikely(page->mapping != mapping)) { 2577 unlock_page(page); 2578 continue; 2579 } 2580 2581 wait_on_page_writeback(page); 2582 BUG_ON(PageWriteback(page)); 2583 2584 if (mpd->map.m_len == 0) 2585 mpd->first_page = page->index; 2586 mpd->next_page = page->index + 1; 2587 /* Add all dirty buffers to mpd */ 2588 lblk = ((ext4_lblk_t)page->index) << 2589 (PAGE_SHIFT - blkbits); 2590 head = page_buffers(page); 2591 err = mpage_process_page_bufs(mpd, head, head, lblk); 2592 if (err <= 0) 2593 goto out; 2594 err = 0; 2595 left--; 2596 } 2597 pagevec_release(&pvec); 2598 cond_resched(); 2599 } 2600 return 0; 2601 out: 2602 pagevec_release(&pvec); 2603 return err; 2604 } 2605 2606 static int __writepage(struct page *page, struct writeback_control *wbc, 2607 void *data) 2608 { 2609 struct address_space *mapping = data; 2610 int ret = ext4_writepage(page, wbc); 2611 mapping_set_error(mapping, ret); 2612 return ret; 2613 } 2614 2615 static int ext4_writepages(struct address_space *mapping, 2616 struct writeback_control *wbc) 2617 { 2618 pgoff_t writeback_index = 0; 2619 long nr_to_write = wbc->nr_to_write; 2620 int range_whole = 0; 2621 int cycled = 1; 2622 handle_t *handle = NULL; 2623 struct mpage_da_data mpd; 2624 struct inode *inode = mapping->host; 2625 int needed_blocks, rsv_blocks = 0, ret = 0; 2626 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb); 2627 bool done; 2628 struct blk_plug plug; 2629 bool give_up_on_write = false; 2630 2631 percpu_down_read(&sbi->s_journal_flag_rwsem); 2632 trace_ext4_writepages(inode, wbc); 2633 2634 if (dax_mapping(mapping)) { 2635 ret = dax_writeback_mapping_range(mapping, inode->i_sb->s_bdev, 2636 wbc); 2637 goto out_writepages; 2638 } 2639 2640 /* 2641 * No pages to write? This is mainly a kludge to avoid starting 2642 * a transaction for special inodes like journal inode on last iput() 2643 * because that could violate lock ordering on umount 2644 */ 2645 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 2646 goto out_writepages; 2647 2648 if (ext4_should_journal_data(inode)) { 2649 struct blk_plug plug; 2650 2651 blk_start_plug(&plug); 2652 ret = write_cache_pages(mapping, wbc, __writepage, mapping); 2653 blk_finish_plug(&plug); 2654 goto out_writepages; 2655 } 2656 2657 /* 2658 * If the filesystem has aborted, it is read-only, so return 2659 * right away instead of dumping stack traces later on that 2660 * will obscure the real source of the problem. We test 2661 * EXT4_MF_FS_ABORTED instead of sb->s_flag's MS_RDONLY because 2662 * the latter could be true if the filesystem is mounted 2663 * read-only, and in that case, ext4_writepages should 2664 * *never* be called, so if that ever happens, we would want 2665 * the stack trace. 2666 */ 2667 if (unlikely(sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) { 2668 ret = -EROFS; 2669 goto out_writepages; 2670 } 2671 2672 if (ext4_should_dioread_nolock(inode)) { 2673 /* 2674 * We may need to convert up to one extent per block in 2675 * the page and we may dirty the inode. 2676 */ 2677 rsv_blocks = 1 + (PAGE_SIZE >> inode->i_blkbits); 2678 } 2679 2680 /* 2681 * If we have inline data and arrive here, it means that 2682 * we will soon create the block for the 1st page, so 2683 * we'd better clear the inline data here. 2684 */ 2685 if (ext4_has_inline_data(inode)) { 2686 /* Just inode will be modified... */ 2687 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 2688 if (IS_ERR(handle)) { 2689 ret = PTR_ERR(handle); 2690 goto out_writepages; 2691 } 2692 BUG_ON(ext4_test_inode_state(inode, 2693 EXT4_STATE_MAY_INLINE_DATA)); 2694 ext4_destroy_inline_data(handle, inode); 2695 ext4_journal_stop(handle); 2696 } 2697 2698 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2699 range_whole = 1; 2700 2701 if (wbc->range_cyclic) { 2702 writeback_index = mapping->writeback_index; 2703 if (writeback_index) 2704 cycled = 0; 2705 mpd.first_page = writeback_index; 2706 mpd.last_page = -1; 2707 } else { 2708 mpd.first_page = wbc->range_start >> PAGE_SHIFT; 2709 mpd.last_page = wbc->range_end >> PAGE_SHIFT; 2710 } 2711 2712 mpd.inode = inode; 2713 mpd.wbc = wbc; 2714 ext4_io_submit_init(&mpd.io_submit, wbc); 2715 retry: 2716 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2717 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page); 2718 done = false; 2719 blk_start_plug(&plug); 2720 while (!done && mpd.first_page <= mpd.last_page) { 2721 /* For each extent of pages we use new io_end */ 2722 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL); 2723 if (!mpd.io_submit.io_end) { 2724 ret = -ENOMEM; 2725 break; 2726 } 2727 2728 /* 2729 * We have two constraints: We find one extent to map and we 2730 * must always write out whole page (makes a difference when 2731 * blocksize < pagesize) so that we don't block on IO when we 2732 * try to write out the rest of the page. Journalled mode is 2733 * not supported by delalloc. 2734 */ 2735 BUG_ON(ext4_should_journal_data(inode)); 2736 needed_blocks = ext4_da_writepages_trans_blocks(inode); 2737 2738 /* start a new transaction */ 2739 handle = ext4_journal_start_with_reserve(inode, 2740 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks); 2741 if (IS_ERR(handle)) { 2742 ret = PTR_ERR(handle); 2743 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: " 2744 "%ld pages, ino %lu; err %d", __func__, 2745 wbc->nr_to_write, inode->i_ino, ret); 2746 /* Release allocated io_end */ 2747 ext4_put_io_end(mpd.io_submit.io_end); 2748 break; 2749 } 2750 2751 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc); 2752 ret = mpage_prepare_extent_to_map(&mpd); 2753 if (!ret) { 2754 if (mpd.map.m_len) 2755 ret = mpage_map_and_submit_extent(handle, &mpd, 2756 &give_up_on_write); 2757 else { 2758 /* 2759 * We scanned the whole range (or exhausted 2760 * nr_to_write), submitted what was mapped and 2761 * didn't find anything needing mapping. We are 2762 * done. 2763 */ 2764 done = true; 2765 } 2766 } 2767 /* 2768 * Caution: If the handle is synchronous, 2769 * ext4_journal_stop() can wait for transaction commit 2770 * to finish which may depend on writeback of pages to 2771 * complete or on page lock to be released. In that 2772 * case, we have to wait until after after we have 2773 * submitted all the IO, released page locks we hold, 2774 * and dropped io_end reference (for extent conversion 2775 * to be able to complete) before stopping the handle. 2776 */ 2777 if (!ext4_handle_valid(handle) || handle->h_sync == 0) { 2778 ext4_journal_stop(handle); 2779 handle = NULL; 2780 } 2781 /* Submit prepared bio */ 2782 ext4_io_submit(&mpd.io_submit); 2783 /* Unlock pages we didn't use */ 2784 mpage_release_unused_pages(&mpd, give_up_on_write); 2785 /* 2786 * Drop our io_end reference we got from init. We have 2787 * to be careful and use deferred io_end finishing if 2788 * we are still holding the transaction as we can 2789 * release the last reference to io_end which may end 2790 * up doing unwritten extent conversion. 2791 */ 2792 if (handle) { 2793 ext4_put_io_end_defer(mpd.io_submit.io_end); 2794 ext4_journal_stop(handle); 2795 } else 2796 ext4_put_io_end(mpd.io_submit.io_end); 2797 2798 if (ret == -ENOSPC && sbi->s_journal) { 2799 /* 2800 * Commit the transaction which would 2801 * free blocks released in the transaction 2802 * and try again 2803 */ 2804 jbd2_journal_force_commit_nested(sbi->s_journal); 2805 ret = 0; 2806 continue; 2807 } 2808 /* Fatal error - ENOMEM, EIO... */ 2809 if (ret) 2810 break; 2811 } 2812 blk_finish_plug(&plug); 2813 if (!ret && !cycled && wbc->nr_to_write > 0) { 2814 cycled = 1; 2815 mpd.last_page = writeback_index - 1; 2816 mpd.first_page = 0; 2817 goto retry; 2818 } 2819 2820 /* Update index */ 2821 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 2822 /* 2823 * Set the writeback_index so that range_cyclic 2824 * mode will write it back later 2825 */ 2826 mapping->writeback_index = mpd.first_page; 2827 2828 out_writepages: 2829 trace_ext4_writepages_result(inode, wbc, ret, 2830 nr_to_write - wbc->nr_to_write); 2831 percpu_up_read(&sbi->s_journal_flag_rwsem); 2832 return ret; 2833 } 2834 2835 static int ext4_nonda_switch(struct super_block *sb) 2836 { 2837 s64 free_clusters, dirty_clusters; 2838 struct ext4_sb_info *sbi = EXT4_SB(sb); 2839 2840 /* 2841 * switch to non delalloc mode if we are running low 2842 * on free block. The free block accounting via percpu 2843 * counters can get slightly wrong with percpu_counter_batch getting 2844 * accumulated on each CPU without updating global counters 2845 * Delalloc need an accurate free block accounting. So switch 2846 * to non delalloc when we are near to error range. 2847 */ 2848 free_clusters = 2849 percpu_counter_read_positive(&sbi->s_freeclusters_counter); 2850 dirty_clusters = 2851 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter); 2852 /* 2853 * Start pushing delalloc when 1/2 of free blocks are dirty. 2854 */ 2855 if (dirty_clusters && (free_clusters < 2 * dirty_clusters)) 2856 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE); 2857 2858 if (2 * free_clusters < 3 * dirty_clusters || 2859 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) { 2860 /* 2861 * free block count is less than 150% of dirty blocks 2862 * or free blocks is less than watermark 2863 */ 2864 return 1; 2865 } 2866 return 0; 2867 } 2868 2869 /* We always reserve for an inode update; the superblock could be there too */ 2870 static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len) 2871 { 2872 if (likely(ext4_has_feature_large_file(inode->i_sb))) 2873 return 1; 2874 2875 if (pos + len <= 0x7fffffffULL) 2876 return 1; 2877 2878 /* We might need to update the superblock to set LARGE_FILE */ 2879 return 2; 2880 } 2881 2882 static int ext4_da_write_begin(struct file *file, struct address_space *mapping, 2883 loff_t pos, unsigned len, unsigned flags, 2884 struct page **pagep, void **fsdata) 2885 { 2886 int ret, retries = 0; 2887 struct page *page; 2888 pgoff_t index; 2889 struct inode *inode = mapping->host; 2890 handle_t *handle; 2891 2892 index = pos >> PAGE_SHIFT; 2893 2894 if (ext4_nonda_switch(inode->i_sb)) { 2895 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC; 2896 return ext4_write_begin(file, mapping, pos, 2897 len, flags, pagep, fsdata); 2898 } 2899 *fsdata = (void *)0; 2900 trace_ext4_da_write_begin(inode, pos, len, flags); 2901 2902 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 2903 ret = ext4_da_write_inline_data_begin(mapping, inode, 2904 pos, len, flags, 2905 pagep, fsdata); 2906 if (ret < 0) 2907 return ret; 2908 if (ret == 1) 2909 return 0; 2910 } 2911 2912 /* 2913 * grab_cache_page_write_begin() can take a long time if the 2914 * system is thrashing due to memory pressure, or if the page 2915 * is being written back. So grab it first before we start 2916 * the transaction handle. This also allows us to allocate 2917 * the page (if needed) without using GFP_NOFS. 2918 */ 2919 retry_grab: 2920 page = grab_cache_page_write_begin(mapping, index, flags); 2921 if (!page) 2922 return -ENOMEM; 2923 unlock_page(page); 2924 2925 /* 2926 * With delayed allocation, we don't log the i_disksize update 2927 * if there is delayed block allocation. But we still need 2928 * to journalling the i_disksize update if writes to the end 2929 * of file which has an already mapped buffer. 2930 */ 2931 retry_journal: 2932 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, 2933 ext4_da_write_credits(inode, pos, len)); 2934 if (IS_ERR(handle)) { 2935 put_page(page); 2936 return PTR_ERR(handle); 2937 } 2938 2939 lock_page(page); 2940 if (page->mapping != mapping) { 2941 /* The page got truncated from under us */ 2942 unlock_page(page); 2943 put_page(page); 2944 ext4_journal_stop(handle); 2945 goto retry_grab; 2946 } 2947 /* In case writeback began while the page was unlocked */ 2948 wait_for_stable_page(page); 2949 2950 #ifdef CONFIG_EXT4_FS_ENCRYPTION 2951 ret = ext4_block_write_begin(page, pos, len, 2952 ext4_da_get_block_prep); 2953 #else 2954 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep); 2955 #endif 2956 if (ret < 0) { 2957 unlock_page(page); 2958 ext4_journal_stop(handle); 2959 /* 2960 * block_write_begin may have instantiated a few blocks 2961 * outside i_size. Trim these off again. Don't need 2962 * i_size_read because we hold i_mutex. 2963 */ 2964 if (pos + len > inode->i_size) 2965 ext4_truncate_failed_write(inode); 2966 2967 if (ret == -ENOSPC && 2968 ext4_should_retry_alloc(inode->i_sb, &retries)) 2969 goto retry_journal; 2970 2971 put_page(page); 2972 return ret; 2973 } 2974 2975 *pagep = page; 2976 return ret; 2977 } 2978 2979 /* 2980 * Check if we should update i_disksize 2981 * when write to the end of file but not require block allocation 2982 */ 2983 static int ext4_da_should_update_i_disksize(struct page *page, 2984 unsigned long offset) 2985 { 2986 struct buffer_head *bh; 2987 struct inode *inode = page->mapping->host; 2988 unsigned int idx; 2989 int i; 2990 2991 bh = page_buffers(page); 2992 idx = offset >> inode->i_blkbits; 2993 2994 for (i = 0; i < idx; i++) 2995 bh = bh->b_this_page; 2996 2997 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh)) 2998 return 0; 2999 return 1; 3000 } 3001 3002 static int ext4_da_write_end(struct file *file, 3003 struct address_space *mapping, 3004 loff_t pos, unsigned len, unsigned copied, 3005 struct page *page, void *fsdata) 3006 { 3007 struct inode *inode = mapping->host; 3008 int ret = 0, ret2; 3009 handle_t *handle = ext4_journal_current_handle(); 3010 loff_t new_i_size; 3011 unsigned long start, end; 3012 int write_mode = (int)(unsigned long)fsdata; 3013 3014 if (write_mode == FALL_BACK_TO_NONDELALLOC) 3015 return ext4_write_end(file, mapping, pos, 3016 len, copied, page, fsdata); 3017 3018 trace_ext4_da_write_end(inode, pos, len, copied); 3019 start = pos & (PAGE_SIZE - 1); 3020 end = start + copied - 1; 3021 3022 /* 3023 * generic_write_end() will run mark_inode_dirty() if i_size 3024 * changes. So let's piggyback the i_disksize mark_inode_dirty 3025 * into that. 3026 */ 3027 new_i_size = pos + copied; 3028 if (copied && new_i_size > EXT4_I(inode)->i_disksize) { 3029 if (ext4_has_inline_data(inode) || 3030 ext4_da_should_update_i_disksize(page, end)) { 3031 ext4_update_i_disksize(inode, new_i_size); 3032 /* We need to mark inode dirty even if 3033 * new_i_size is less that inode->i_size 3034 * bu greater than i_disksize.(hint delalloc) 3035 */ 3036 ext4_mark_inode_dirty(handle, inode); 3037 } 3038 } 3039 3040 if (write_mode != CONVERT_INLINE_DATA && 3041 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && 3042 ext4_has_inline_data(inode)) 3043 ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied, 3044 page); 3045 else 3046 ret2 = generic_write_end(file, mapping, pos, len, copied, 3047 page, fsdata); 3048 3049 copied = ret2; 3050 if (ret2 < 0) 3051 ret = ret2; 3052 ret2 = ext4_journal_stop(handle); 3053 if (!ret) 3054 ret = ret2; 3055 3056 return ret ? ret : copied; 3057 } 3058 3059 static void ext4_da_invalidatepage(struct page *page, unsigned int offset, 3060 unsigned int length) 3061 { 3062 /* 3063 * Drop reserved blocks 3064 */ 3065 BUG_ON(!PageLocked(page)); 3066 if (!page_has_buffers(page)) 3067 goto out; 3068 3069 ext4_da_page_release_reservation(page, offset, length); 3070 3071 out: 3072 ext4_invalidatepage(page, offset, length); 3073 3074 return; 3075 } 3076 3077 /* 3078 * Force all delayed allocation blocks to be allocated for a given inode. 3079 */ 3080 int ext4_alloc_da_blocks(struct inode *inode) 3081 { 3082 trace_ext4_alloc_da_blocks(inode); 3083 3084 if (!EXT4_I(inode)->i_reserved_data_blocks) 3085 return 0; 3086 3087 /* 3088 * We do something simple for now. The filemap_flush() will 3089 * also start triggering a write of the data blocks, which is 3090 * not strictly speaking necessary (and for users of 3091 * laptop_mode, not even desirable). However, to do otherwise 3092 * would require replicating code paths in: 3093 * 3094 * ext4_writepages() -> 3095 * write_cache_pages() ---> (via passed in callback function) 3096 * __mpage_da_writepage() --> 3097 * mpage_add_bh_to_extent() 3098 * mpage_da_map_blocks() 3099 * 3100 * The problem is that write_cache_pages(), located in 3101 * mm/page-writeback.c, marks pages clean in preparation for 3102 * doing I/O, which is not desirable if we're not planning on 3103 * doing I/O at all. 3104 * 3105 * We could call write_cache_pages(), and then redirty all of 3106 * the pages by calling redirty_page_for_writepage() but that 3107 * would be ugly in the extreme. So instead we would need to 3108 * replicate parts of the code in the above functions, 3109 * simplifying them because we wouldn't actually intend to 3110 * write out the pages, but rather only collect contiguous 3111 * logical block extents, call the multi-block allocator, and 3112 * then update the buffer heads with the block allocations. 3113 * 3114 * For now, though, we'll cheat by calling filemap_flush(), 3115 * which will map the blocks, and start the I/O, but not 3116 * actually wait for the I/O to complete. 3117 */ 3118 return filemap_flush(inode->i_mapping); 3119 } 3120 3121 /* 3122 * bmap() is special. It gets used by applications such as lilo and by 3123 * the swapper to find the on-disk block of a specific piece of data. 3124 * 3125 * Naturally, this is dangerous if the block concerned is still in the 3126 * journal. If somebody makes a swapfile on an ext4 data-journaling 3127 * filesystem and enables swap, then they may get a nasty shock when the 3128 * data getting swapped to that swapfile suddenly gets overwritten by 3129 * the original zero's written out previously to the journal and 3130 * awaiting writeback in the kernel's buffer cache. 3131 * 3132 * So, if we see any bmap calls here on a modified, data-journaled file, 3133 * take extra steps to flush any blocks which might be in the cache. 3134 */ 3135 static sector_t ext4_bmap(struct address_space *mapping, sector_t block) 3136 { 3137 struct inode *inode = mapping->host; 3138 journal_t *journal; 3139 int err; 3140 3141 /* 3142 * We can get here for an inline file via the FIBMAP ioctl 3143 */ 3144 if (ext4_has_inline_data(inode)) 3145 return 0; 3146 3147 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && 3148 test_opt(inode->i_sb, DELALLOC)) { 3149 /* 3150 * With delalloc we want to sync the file 3151 * so that we can make sure we allocate 3152 * blocks for file 3153 */ 3154 filemap_write_and_wait(mapping); 3155 } 3156 3157 if (EXT4_JOURNAL(inode) && 3158 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) { 3159 /* 3160 * This is a REALLY heavyweight approach, but the use of 3161 * bmap on dirty files is expected to be extremely rare: 3162 * only if we run lilo or swapon on a freshly made file 3163 * do we expect this to happen. 3164 * 3165 * (bmap requires CAP_SYS_RAWIO so this does not 3166 * represent an unprivileged user DOS attack --- we'd be 3167 * in trouble if mortal users could trigger this path at 3168 * will.) 3169 * 3170 * NB. EXT4_STATE_JDATA is not set on files other than 3171 * regular files. If somebody wants to bmap a directory 3172 * or symlink and gets confused because the buffer 3173 * hasn't yet been flushed to disk, they deserve 3174 * everything they get. 3175 */ 3176 3177 ext4_clear_inode_state(inode, EXT4_STATE_JDATA); 3178 journal = EXT4_JOURNAL(inode); 3179 jbd2_journal_lock_updates(journal); 3180 err = jbd2_journal_flush(journal); 3181 jbd2_journal_unlock_updates(journal); 3182 3183 if (err) 3184 return 0; 3185 } 3186 3187 return generic_block_bmap(mapping, block, ext4_get_block); 3188 } 3189 3190 static int ext4_readpage(struct file *file, struct page *page) 3191 { 3192 int ret = -EAGAIN; 3193 struct inode *inode = page->mapping->host; 3194 3195 trace_ext4_readpage(page); 3196 3197 if (ext4_has_inline_data(inode)) 3198 ret = ext4_readpage_inline(inode, page); 3199 3200 if (ret == -EAGAIN) 3201 return ext4_mpage_readpages(page->mapping, NULL, page, 1); 3202 3203 return ret; 3204 } 3205 3206 static int 3207 ext4_readpages(struct file *file, struct address_space *mapping, 3208 struct list_head *pages, unsigned nr_pages) 3209 { 3210 struct inode *inode = mapping->host; 3211 3212 /* If the file has inline data, no need to do readpages. */ 3213 if (ext4_has_inline_data(inode)) 3214 return 0; 3215 3216 return ext4_mpage_readpages(mapping, pages, NULL, nr_pages); 3217 } 3218 3219 static void ext4_invalidatepage(struct page *page, unsigned int offset, 3220 unsigned int length) 3221 { 3222 trace_ext4_invalidatepage(page, offset, length); 3223 3224 /* No journalling happens on data buffers when this function is used */ 3225 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page))); 3226 3227 block_invalidatepage(page, offset, length); 3228 } 3229 3230 static int __ext4_journalled_invalidatepage(struct page *page, 3231 unsigned int offset, 3232 unsigned int length) 3233 { 3234 journal_t *journal = EXT4_JOURNAL(page->mapping->host); 3235 3236 trace_ext4_journalled_invalidatepage(page, offset, length); 3237 3238 /* 3239 * If it's a full truncate we just forget about the pending dirtying 3240 */ 3241 if (offset == 0 && length == PAGE_SIZE) 3242 ClearPageChecked(page); 3243 3244 return jbd2_journal_invalidatepage(journal, page, offset, length); 3245 } 3246 3247 /* Wrapper for aops... */ 3248 static void ext4_journalled_invalidatepage(struct page *page, 3249 unsigned int offset, 3250 unsigned int length) 3251 { 3252 WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0); 3253 } 3254 3255 static int ext4_releasepage(struct page *page, gfp_t wait) 3256 { 3257 journal_t *journal = EXT4_JOURNAL(page->mapping->host); 3258 3259 trace_ext4_releasepage(page); 3260 3261 /* Page has dirty journalled data -> cannot release */ 3262 if (PageChecked(page)) 3263 return 0; 3264 if (journal) 3265 return jbd2_journal_try_to_free_buffers(journal, page, wait); 3266 else 3267 return try_to_free_buffers(page); 3268 } 3269 3270 #ifdef CONFIG_FS_DAX 3271 /* 3272 * Get block function for DAX IO and mmap faults. It takes care of converting 3273 * unwritten extents to written ones and initializes new / converted blocks 3274 * to zeros. 3275 */ 3276 int ext4_dax_get_block(struct inode *inode, sector_t iblock, 3277 struct buffer_head *bh_result, int create) 3278 { 3279 int ret; 3280 3281 ext4_debug("inode %lu, create flag %d\n", inode->i_ino, create); 3282 if (!create) 3283 return _ext4_get_block(inode, iblock, bh_result, 0); 3284 3285 ret = ext4_get_block_trans(inode, iblock, bh_result, 3286 EXT4_GET_BLOCKS_PRE_IO | 3287 EXT4_GET_BLOCKS_CREATE_ZERO); 3288 if (ret < 0) 3289 return ret; 3290 3291 if (buffer_unwritten(bh_result)) { 3292 /* 3293 * We are protected by i_mmap_sem or i_mutex so we know block 3294 * cannot go away from under us even though we dropped 3295 * i_data_sem. Convert extent to written and write zeros there. 3296 */ 3297 ret = ext4_get_block_trans(inode, iblock, bh_result, 3298 EXT4_GET_BLOCKS_CONVERT | 3299 EXT4_GET_BLOCKS_CREATE_ZERO); 3300 if (ret < 0) 3301 return ret; 3302 } 3303 /* 3304 * At least for now we have to clear BH_New so that DAX code 3305 * doesn't attempt to zero blocks again in a racy way. 3306 */ 3307 clear_buffer_new(bh_result); 3308 return 0; 3309 } 3310 #else 3311 /* Just define empty function, it will never get called. */ 3312 int ext4_dax_get_block(struct inode *inode, sector_t iblock, 3313 struct buffer_head *bh_result, int create) 3314 { 3315 BUG(); 3316 return 0; 3317 } 3318 #endif 3319 3320 static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset, 3321 ssize_t size, void *private) 3322 { 3323 ext4_io_end_t *io_end = private; 3324 3325 /* if not async direct IO just return */ 3326 if (!io_end) 3327 return 0; 3328 3329 ext_debug("ext4_end_io_dio(): io_end 0x%p " 3330 "for inode %lu, iocb 0x%p, offset %llu, size %zd\n", 3331 io_end, io_end->inode->i_ino, iocb, offset, size); 3332 3333 /* 3334 * Error during AIO DIO. We cannot convert unwritten extents as the 3335 * data was not written. Just clear the unwritten flag and drop io_end. 3336 */ 3337 if (size <= 0) { 3338 ext4_clear_io_unwritten_flag(io_end); 3339 size = 0; 3340 } 3341 io_end->offset = offset; 3342 io_end->size = size; 3343 ext4_put_io_end(io_end); 3344 3345 return 0; 3346 } 3347 3348 /* 3349 * Handling of direct IO writes. 3350 * 3351 * For ext4 extent files, ext4 will do direct-io write even to holes, 3352 * preallocated extents, and those write extend the file, no need to 3353 * fall back to buffered IO. 3354 * 3355 * For holes, we fallocate those blocks, mark them as unwritten 3356 * If those blocks were preallocated, we mark sure they are split, but 3357 * still keep the range to write as unwritten. 3358 * 3359 * The unwritten extents will be converted to written when DIO is completed. 3360 * For async direct IO, since the IO may still pending when return, we 3361 * set up an end_io call back function, which will do the conversion 3362 * when async direct IO completed. 3363 * 3364 * If the O_DIRECT write will extend the file then add this inode to the 3365 * orphan list. So recovery will truncate it back to the original size 3366 * if the machine crashes during the write. 3367 * 3368 */ 3369 static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter) 3370 { 3371 struct file *file = iocb->ki_filp; 3372 struct inode *inode = file->f_mapping->host; 3373 struct ext4_inode_info *ei = EXT4_I(inode); 3374 ssize_t ret; 3375 loff_t offset = iocb->ki_pos; 3376 size_t count = iov_iter_count(iter); 3377 int overwrite = 0; 3378 get_block_t *get_block_func = NULL; 3379 int dio_flags = 0; 3380 loff_t final_size = offset + count; 3381 int orphan = 0; 3382 handle_t *handle; 3383 3384 if (final_size > inode->i_size) { 3385 /* Credits for sb + inode write */ 3386 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 3387 if (IS_ERR(handle)) { 3388 ret = PTR_ERR(handle); 3389 goto out; 3390 } 3391 ret = ext4_orphan_add(handle, inode); 3392 if (ret) { 3393 ext4_journal_stop(handle); 3394 goto out; 3395 } 3396 orphan = 1; 3397 ei->i_disksize = inode->i_size; 3398 ext4_journal_stop(handle); 3399 } 3400 3401 BUG_ON(iocb->private == NULL); 3402 3403 /* 3404 * Make all waiters for direct IO properly wait also for extent 3405 * conversion. This also disallows race between truncate() and 3406 * overwrite DIO as i_dio_count needs to be incremented under i_mutex. 3407 */ 3408 inode_dio_begin(inode); 3409 3410 /* If we do a overwrite dio, i_mutex locking can be released */ 3411 overwrite = *((int *)iocb->private); 3412 3413 if (overwrite) 3414 inode_unlock(inode); 3415 3416 /* 3417 * For extent mapped files we could direct write to holes and fallocate. 3418 * 3419 * Allocated blocks to fill the hole are marked as unwritten to prevent 3420 * parallel buffered read to expose the stale data before DIO complete 3421 * the data IO. 3422 * 3423 * As to previously fallocated extents, ext4 get_block will just simply 3424 * mark the buffer mapped but still keep the extents unwritten. 3425 * 3426 * For non AIO case, we will convert those unwritten extents to written 3427 * after return back from blockdev_direct_IO. That way we save us from 3428 * allocating io_end structure and also the overhead of offloading 3429 * the extent convertion to a workqueue. 3430 * 3431 * For async DIO, the conversion needs to be deferred when the 3432 * IO is completed. The ext4 end_io callback function will be 3433 * called to take care of the conversion work. Here for async 3434 * case, we allocate an io_end structure to hook to the iocb. 3435 */ 3436 iocb->private = NULL; 3437 if (overwrite) 3438 get_block_func = ext4_dio_get_block_overwrite; 3439 else if (IS_DAX(inode)) { 3440 /* 3441 * We can avoid zeroing for aligned DAX writes beyond EOF. Other 3442 * writes need zeroing either because they can race with page 3443 * faults or because they use partial blocks. 3444 */ 3445 if (round_down(offset, 1<<inode->i_blkbits) >= inode->i_size && 3446 ext4_aligned_io(inode, offset, count)) 3447 get_block_func = ext4_dio_get_block; 3448 else 3449 get_block_func = ext4_dax_get_block; 3450 dio_flags = DIO_LOCKING; 3451 } else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) || 3452 round_down(offset, 1 << inode->i_blkbits) >= inode->i_size) { 3453 get_block_func = ext4_dio_get_block; 3454 dio_flags = DIO_LOCKING | DIO_SKIP_HOLES; 3455 } else if (is_sync_kiocb(iocb)) { 3456 get_block_func = ext4_dio_get_block_unwritten_sync; 3457 dio_flags = DIO_LOCKING; 3458 } else { 3459 get_block_func = ext4_dio_get_block_unwritten_async; 3460 dio_flags = DIO_LOCKING; 3461 } 3462 #ifdef CONFIG_EXT4_FS_ENCRYPTION 3463 BUG_ON(ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)); 3464 #endif 3465 if (IS_DAX(inode)) { 3466 ret = dax_do_io(iocb, inode, iter, get_block_func, 3467 ext4_end_io_dio, dio_flags); 3468 } else 3469 ret = __blockdev_direct_IO(iocb, inode, 3470 inode->i_sb->s_bdev, iter, 3471 get_block_func, 3472 ext4_end_io_dio, NULL, dio_flags); 3473 3474 if (ret > 0 && !overwrite && ext4_test_inode_state(inode, 3475 EXT4_STATE_DIO_UNWRITTEN)) { 3476 int err; 3477 /* 3478 * for non AIO case, since the IO is already 3479 * completed, we could do the conversion right here 3480 */ 3481 err = ext4_convert_unwritten_extents(NULL, inode, 3482 offset, ret); 3483 if (err < 0) 3484 ret = err; 3485 ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN); 3486 } 3487 3488 inode_dio_end(inode); 3489 /* take i_mutex locking again if we do a ovewrite dio */ 3490 if (overwrite) 3491 inode_lock(inode); 3492 3493 if (ret < 0 && final_size > inode->i_size) 3494 ext4_truncate_failed_write(inode); 3495 3496 /* Handle extending of i_size after direct IO write */ 3497 if (orphan) { 3498 int err; 3499 3500 /* Credits for sb + inode write */ 3501 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 3502 if (IS_ERR(handle)) { 3503 /* This is really bad luck. We've written the data 3504 * but cannot extend i_size. Bail out and pretend 3505 * the write failed... */ 3506 ret = PTR_ERR(handle); 3507 if (inode->i_nlink) 3508 ext4_orphan_del(NULL, inode); 3509 3510 goto out; 3511 } 3512 if (inode->i_nlink) 3513 ext4_orphan_del(handle, inode); 3514 if (ret > 0) { 3515 loff_t end = offset + ret; 3516 if (end > inode->i_size) { 3517 ei->i_disksize = end; 3518 i_size_write(inode, end); 3519 /* 3520 * We're going to return a positive `ret' 3521 * here due to non-zero-length I/O, so there's 3522 * no way of reporting error returns from 3523 * ext4_mark_inode_dirty() to userspace. So 3524 * ignore it. 3525 */ 3526 ext4_mark_inode_dirty(handle, inode); 3527 } 3528 } 3529 err = ext4_journal_stop(handle); 3530 if (ret == 0) 3531 ret = err; 3532 } 3533 out: 3534 return ret; 3535 } 3536 3537 static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter) 3538 { 3539 struct address_space *mapping = iocb->ki_filp->f_mapping; 3540 struct inode *inode = mapping->host; 3541 ssize_t ret; 3542 3543 /* 3544 * Shared inode_lock is enough for us - it protects against concurrent 3545 * writes & truncates and since we take care of writing back page cache, 3546 * we are protected against page writeback as well. 3547 */ 3548 inode_lock_shared(inode); 3549 if (IS_DAX(inode)) { 3550 ret = dax_do_io(iocb, inode, iter, ext4_dio_get_block, NULL, 0); 3551 } else { 3552 size_t count = iov_iter_count(iter); 3553 3554 ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, 3555 iocb->ki_pos + count); 3556 if (ret) 3557 goto out_unlock; 3558 ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, 3559 iter, ext4_dio_get_block, 3560 NULL, NULL, 0); 3561 } 3562 out_unlock: 3563 inode_unlock_shared(inode); 3564 return ret; 3565 } 3566 3567 static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 3568 { 3569 struct file *file = iocb->ki_filp; 3570 struct inode *inode = file->f_mapping->host; 3571 size_t count = iov_iter_count(iter); 3572 loff_t offset = iocb->ki_pos; 3573 ssize_t ret; 3574 3575 #ifdef CONFIG_EXT4_FS_ENCRYPTION 3576 if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 3577 return 0; 3578 #endif 3579 3580 /* 3581 * If we are doing data journalling we don't support O_DIRECT 3582 */ 3583 if (ext4_should_journal_data(inode)) 3584 return 0; 3585 3586 /* Let buffer I/O handle the inline data case. */ 3587 if (ext4_has_inline_data(inode)) 3588 return 0; 3589 3590 trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); 3591 if (iov_iter_rw(iter) == READ) 3592 ret = ext4_direct_IO_read(iocb, iter); 3593 else 3594 ret = ext4_direct_IO_write(iocb, iter); 3595 trace_ext4_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), ret); 3596 return ret; 3597 } 3598 3599 /* 3600 * Pages can be marked dirty completely asynchronously from ext4's journalling 3601 * activity. By filemap_sync_pte(), try_to_unmap_one(), etc. We cannot do 3602 * much here because ->set_page_dirty is called under VFS locks. The page is 3603 * not necessarily locked. 3604 * 3605 * We cannot just dirty the page and leave attached buffers clean, because the 3606 * buffers' dirty state is "definitive". We cannot just set the buffers dirty 3607 * or jbddirty because all the journalling code will explode. 3608 * 3609 * So what we do is to mark the page "pending dirty" and next time writepage 3610 * is called, propagate that into the buffers appropriately. 3611 */ 3612 static int ext4_journalled_set_page_dirty(struct page *page) 3613 { 3614 SetPageChecked(page); 3615 return __set_page_dirty_nobuffers(page); 3616 } 3617 3618 static const struct address_space_operations ext4_aops = { 3619 .readpage = ext4_readpage, 3620 .readpages = ext4_readpages, 3621 .writepage = ext4_writepage, 3622 .writepages = ext4_writepages, 3623 .write_begin = ext4_write_begin, 3624 .write_end = ext4_write_end, 3625 .bmap = ext4_bmap, 3626 .invalidatepage = ext4_invalidatepage, 3627 .releasepage = ext4_releasepage, 3628 .direct_IO = ext4_direct_IO, 3629 .migratepage = buffer_migrate_page, 3630 .is_partially_uptodate = block_is_partially_uptodate, 3631 .error_remove_page = generic_error_remove_page, 3632 }; 3633 3634 static const struct address_space_operations ext4_journalled_aops = { 3635 .readpage = ext4_readpage, 3636 .readpages = ext4_readpages, 3637 .writepage = ext4_writepage, 3638 .writepages = ext4_writepages, 3639 .write_begin = ext4_write_begin, 3640 .write_end = ext4_journalled_write_end, 3641 .set_page_dirty = ext4_journalled_set_page_dirty, 3642 .bmap = ext4_bmap, 3643 .invalidatepage = ext4_journalled_invalidatepage, 3644 .releasepage = ext4_releasepage, 3645 .direct_IO = ext4_direct_IO, 3646 .is_partially_uptodate = block_is_partially_uptodate, 3647 .error_remove_page = generic_error_remove_page, 3648 }; 3649 3650 static const struct address_space_operations ext4_da_aops = { 3651 .readpage = ext4_readpage, 3652 .readpages = ext4_readpages, 3653 .writepage = ext4_writepage, 3654 .writepages = ext4_writepages, 3655 .write_begin = ext4_da_write_begin, 3656 .write_end = ext4_da_write_end, 3657 .bmap = ext4_bmap, 3658 .invalidatepage = ext4_da_invalidatepage, 3659 .releasepage = ext4_releasepage, 3660 .direct_IO = ext4_direct_IO, 3661 .migratepage = buffer_migrate_page, 3662 .is_partially_uptodate = block_is_partially_uptodate, 3663 .error_remove_page = generic_error_remove_page, 3664 }; 3665 3666 void ext4_set_aops(struct inode *inode) 3667 { 3668 switch (ext4_inode_journal_mode(inode)) { 3669 case EXT4_INODE_ORDERED_DATA_MODE: 3670 case EXT4_INODE_WRITEBACK_DATA_MODE: 3671 break; 3672 case EXT4_INODE_JOURNAL_DATA_MODE: 3673 inode->i_mapping->a_ops = &ext4_journalled_aops; 3674 return; 3675 default: 3676 BUG(); 3677 } 3678 if (test_opt(inode->i_sb, DELALLOC)) 3679 inode->i_mapping->a_ops = &ext4_da_aops; 3680 else 3681 inode->i_mapping->a_ops = &ext4_aops; 3682 } 3683 3684 static int __ext4_block_zero_page_range(handle_t *handle, 3685 struct address_space *mapping, loff_t from, loff_t length) 3686 { 3687 ext4_fsblk_t index = from >> PAGE_SHIFT; 3688 unsigned offset = from & (PAGE_SIZE-1); 3689 unsigned blocksize, pos; 3690 ext4_lblk_t iblock; 3691 struct inode *inode = mapping->host; 3692 struct buffer_head *bh; 3693 struct page *page; 3694 int err = 0; 3695 3696 page = find_or_create_page(mapping, from >> PAGE_SHIFT, 3697 mapping_gfp_constraint(mapping, ~__GFP_FS)); 3698 if (!page) 3699 return -ENOMEM; 3700 3701 blocksize = inode->i_sb->s_blocksize; 3702 3703 iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits); 3704 3705 if (!page_has_buffers(page)) 3706 create_empty_buffers(page, blocksize, 0); 3707 3708 /* Find the buffer that contains "offset" */ 3709 bh = page_buffers(page); 3710 pos = blocksize; 3711 while (offset >= pos) { 3712 bh = bh->b_this_page; 3713 iblock++; 3714 pos += blocksize; 3715 } 3716 if (buffer_freed(bh)) { 3717 BUFFER_TRACE(bh, "freed: skip"); 3718 goto unlock; 3719 } 3720 if (!buffer_mapped(bh)) { 3721 BUFFER_TRACE(bh, "unmapped"); 3722 ext4_get_block(inode, iblock, bh, 0); 3723 /* unmapped? It's a hole - nothing to do */ 3724 if (!buffer_mapped(bh)) { 3725 BUFFER_TRACE(bh, "still unmapped"); 3726 goto unlock; 3727 } 3728 } 3729 3730 /* Ok, it's mapped. Make sure it's up-to-date */ 3731 if (PageUptodate(page)) 3732 set_buffer_uptodate(bh); 3733 3734 if (!buffer_uptodate(bh)) { 3735 err = -EIO; 3736 ll_rw_block(REQ_OP_READ, 0, 1, &bh); 3737 wait_on_buffer(bh); 3738 /* Uhhuh. Read error. Complain and punt. */ 3739 if (!buffer_uptodate(bh)) 3740 goto unlock; 3741 if (S_ISREG(inode->i_mode) && 3742 ext4_encrypted_inode(inode)) { 3743 /* We expect the key to be set. */ 3744 BUG_ON(!fscrypt_has_encryption_key(inode)); 3745 BUG_ON(blocksize != PAGE_SIZE); 3746 WARN_ON_ONCE(fscrypt_decrypt_page(page)); 3747 } 3748 } 3749 if (ext4_should_journal_data(inode)) { 3750 BUFFER_TRACE(bh, "get write access"); 3751 err = ext4_journal_get_write_access(handle, bh); 3752 if (err) 3753 goto unlock; 3754 } 3755 zero_user(page, offset, length); 3756 BUFFER_TRACE(bh, "zeroed end of block"); 3757 3758 if (ext4_should_journal_data(inode)) { 3759 err = ext4_handle_dirty_metadata(handle, inode, bh); 3760 } else { 3761 err = 0; 3762 mark_buffer_dirty(bh); 3763 if (ext4_should_order_data(inode)) 3764 err = ext4_jbd2_inode_add_write(handle, inode); 3765 } 3766 3767 unlock: 3768 unlock_page(page); 3769 put_page(page); 3770 return err; 3771 } 3772 3773 /* 3774 * ext4_block_zero_page_range() zeros out a mapping of length 'length' 3775 * starting from file offset 'from'. The range to be zero'd must 3776 * be contained with in one block. If the specified range exceeds 3777 * the end of the block it will be shortened to end of the block 3778 * that cooresponds to 'from' 3779 */ 3780 static int ext4_block_zero_page_range(handle_t *handle, 3781 struct address_space *mapping, loff_t from, loff_t length) 3782 { 3783 struct inode *inode = mapping->host; 3784 unsigned offset = from & (PAGE_SIZE-1); 3785 unsigned blocksize = inode->i_sb->s_blocksize; 3786 unsigned max = blocksize - (offset & (blocksize - 1)); 3787 3788 /* 3789 * correct length if it does not fall between 3790 * 'from' and the end of the block 3791 */ 3792 if (length > max || length < 0) 3793 length = max; 3794 3795 if (IS_DAX(inode)) 3796 return dax_zero_page_range(inode, from, length, ext4_get_block); 3797 return __ext4_block_zero_page_range(handle, mapping, from, length); 3798 } 3799 3800 /* 3801 * ext4_block_truncate_page() zeroes out a mapping from file offset `from' 3802 * up to the end of the block which corresponds to `from'. 3803 * This required during truncate. We need to physically zero the tail end 3804 * of that block so it doesn't yield old data if the file is later grown. 3805 */ 3806 static int ext4_block_truncate_page(handle_t *handle, 3807 struct address_space *mapping, loff_t from) 3808 { 3809 unsigned offset = from & (PAGE_SIZE-1); 3810 unsigned length; 3811 unsigned blocksize; 3812 struct inode *inode = mapping->host; 3813 3814 blocksize = inode->i_sb->s_blocksize; 3815 length = blocksize - (offset & (blocksize - 1)); 3816 3817 return ext4_block_zero_page_range(handle, mapping, from, length); 3818 } 3819 3820 int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode, 3821 loff_t lstart, loff_t length) 3822 { 3823 struct super_block *sb = inode->i_sb; 3824 struct address_space *mapping = inode->i_mapping; 3825 unsigned partial_start, partial_end; 3826 ext4_fsblk_t start, end; 3827 loff_t byte_end = (lstart + length - 1); 3828 int err = 0; 3829 3830 partial_start = lstart & (sb->s_blocksize - 1); 3831 partial_end = byte_end & (sb->s_blocksize - 1); 3832 3833 start = lstart >> sb->s_blocksize_bits; 3834 end = byte_end >> sb->s_blocksize_bits; 3835 3836 /* Handle partial zero within the single block */ 3837 if (start == end && 3838 (partial_start || (partial_end != sb->s_blocksize - 1))) { 3839 err = ext4_block_zero_page_range(handle, mapping, 3840 lstart, length); 3841 return err; 3842 } 3843 /* Handle partial zero out on the start of the range */ 3844 if (partial_start) { 3845 err = ext4_block_zero_page_range(handle, mapping, 3846 lstart, sb->s_blocksize); 3847 if (err) 3848 return err; 3849 } 3850 /* Handle partial zero out on the end of the range */ 3851 if (partial_end != sb->s_blocksize - 1) 3852 err = ext4_block_zero_page_range(handle, mapping, 3853 byte_end - partial_end, 3854 partial_end + 1); 3855 return err; 3856 } 3857 3858 int ext4_can_truncate(struct inode *inode) 3859 { 3860 if (S_ISREG(inode->i_mode)) 3861 return 1; 3862 if (S_ISDIR(inode->i_mode)) 3863 return 1; 3864 if (S_ISLNK(inode->i_mode)) 3865 return !ext4_inode_is_fast_symlink(inode); 3866 return 0; 3867 } 3868 3869 /* 3870 * We have to make sure i_disksize gets properly updated before we truncate 3871 * page cache due to hole punching or zero range. Otherwise i_disksize update 3872 * can get lost as it may have been postponed to submission of writeback but 3873 * that will never happen after we truncate page cache. 3874 */ 3875 int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, 3876 loff_t len) 3877 { 3878 handle_t *handle; 3879 loff_t size = i_size_read(inode); 3880 3881 WARN_ON(!inode_is_locked(inode)); 3882 if (offset > size || offset + len < size) 3883 return 0; 3884 3885 if (EXT4_I(inode)->i_disksize >= size) 3886 return 0; 3887 3888 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); 3889 if (IS_ERR(handle)) 3890 return PTR_ERR(handle); 3891 ext4_update_i_disksize(inode, size); 3892 ext4_mark_inode_dirty(handle, inode); 3893 ext4_journal_stop(handle); 3894 3895 return 0; 3896 } 3897 3898 /* 3899 * ext4_punch_hole: punches a hole in a file by releasing the blocks 3900 * associated with the given offset and length 3901 * 3902 * @inode: File inode 3903 * @offset: The offset where the hole will begin 3904 * @len: The length of the hole 3905 * 3906 * Returns: 0 on success or negative on failure 3907 */ 3908 3909 int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length) 3910 { 3911 struct super_block *sb = inode->i_sb; 3912 ext4_lblk_t first_block, stop_block; 3913 struct address_space *mapping = inode->i_mapping; 3914 loff_t first_block_offset, last_block_offset; 3915 handle_t *handle; 3916 unsigned int credits; 3917 int ret = 0; 3918 3919 if (!S_ISREG(inode->i_mode)) 3920 return -EOPNOTSUPP; 3921 3922 trace_ext4_punch_hole(inode, offset, length, 0); 3923 3924 /* 3925 * Write out all dirty pages to avoid race conditions 3926 * Then release them. 3927 */ 3928 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) { 3929 ret = filemap_write_and_wait_range(mapping, offset, 3930 offset + length - 1); 3931 if (ret) 3932 return ret; 3933 } 3934 3935 inode_lock(inode); 3936 3937 /* No need to punch hole beyond i_size */ 3938 if (offset >= inode->i_size) 3939 goto out_mutex; 3940 3941 /* 3942 * If the hole extends beyond i_size, set the hole 3943 * to end after the page that contains i_size 3944 */ 3945 if (offset + length > inode->i_size) { 3946 length = inode->i_size + 3947 PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) - 3948 offset; 3949 } 3950 3951 if (offset & (sb->s_blocksize - 1) || 3952 (offset + length) & (sb->s_blocksize - 1)) { 3953 /* 3954 * Attach jinode to inode for jbd2 if we do any zeroing of 3955 * partial block 3956 */ 3957 ret = ext4_inode_attach_jinode(inode); 3958 if (ret < 0) 3959 goto out_mutex; 3960 3961 } 3962 3963 /* Wait all existing dio workers, newcomers will block on i_mutex */ 3964 ext4_inode_block_unlocked_dio(inode); 3965 inode_dio_wait(inode); 3966 3967 /* 3968 * Prevent page faults from reinstantiating pages we have released from 3969 * page cache. 3970 */ 3971 down_write(&EXT4_I(inode)->i_mmap_sem); 3972 first_block_offset = round_up(offset, sb->s_blocksize); 3973 last_block_offset = round_down((offset + length), sb->s_blocksize) - 1; 3974 3975 /* Now release the pages and zero block aligned part of pages*/ 3976 if (last_block_offset > first_block_offset) { 3977 ret = ext4_update_disksize_before_punch(inode, offset, length); 3978 if (ret) 3979 goto out_dio; 3980 truncate_pagecache_range(inode, first_block_offset, 3981 last_block_offset); 3982 } 3983 3984 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 3985 credits = ext4_writepage_trans_blocks(inode); 3986 else 3987 credits = ext4_blocks_for_truncate(inode); 3988 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 3989 if (IS_ERR(handle)) { 3990 ret = PTR_ERR(handle); 3991 ext4_std_error(sb, ret); 3992 goto out_dio; 3993 } 3994 3995 ret = ext4_zero_partial_blocks(handle, inode, offset, 3996 length); 3997 if (ret) 3998 goto out_stop; 3999 4000 first_block = (offset + sb->s_blocksize - 1) >> 4001 EXT4_BLOCK_SIZE_BITS(sb); 4002 stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb); 4003 4004 /* If there are no blocks to remove, return now */ 4005 if (first_block >= stop_block) 4006 goto out_stop; 4007 4008 down_write(&EXT4_I(inode)->i_data_sem); 4009 ext4_discard_preallocations(inode); 4010 4011 ret = ext4_es_remove_extent(inode, first_block, 4012 stop_block - first_block); 4013 if (ret) { 4014 up_write(&EXT4_I(inode)->i_data_sem); 4015 goto out_stop; 4016 } 4017 4018 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 4019 ret = ext4_ext_remove_space(inode, first_block, 4020 stop_block - 1); 4021 else 4022 ret = ext4_ind_remove_space(handle, inode, first_block, 4023 stop_block); 4024 4025 up_write(&EXT4_I(inode)->i_data_sem); 4026 if (IS_SYNC(inode)) 4027 ext4_handle_sync(handle); 4028 4029 inode->i_mtime = inode->i_ctime = ext4_current_time(inode); 4030 ext4_mark_inode_dirty(handle, inode); 4031 out_stop: 4032 ext4_journal_stop(handle); 4033 out_dio: 4034 up_write(&EXT4_I(inode)->i_mmap_sem); 4035 ext4_inode_resume_unlocked_dio(inode); 4036 out_mutex: 4037 inode_unlock(inode); 4038 return ret; 4039 } 4040 4041 int ext4_inode_attach_jinode(struct inode *inode) 4042 { 4043 struct ext4_inode_info *ei = EXT4_I(inode); 4044 struct jbd2_inode *jinode; 4045 4046 if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal) 4047 return 0; 4048 4049 jinode = jbd2_alloc_inode(GFP_KERNEL); 4050 spin_lock(&inode->i_lock); 4051 if (!ei->jinode) { 4052 if (!jinode) { 4053 spin_unlock(&inode->i_lock); 4054 return -ENOMEM; 4055 } 4056 ei->jinode = jinode; 4057 jbd2_journal_init_jbd_inode(ei->jinode, inode); 4058 jinode = NULL; 4059 } 4060 spin_unlock(&inode->i_lock); 4061 if (unlikely(jinode != NULL)) 4062 jbd2_free_inode(jinode); 4063 return 0; 4064 } 4065 4066 /* 4067 * ext4_truncate() 4068 * 4069 * We block out ext4_get_block() block instantiations across the entire 4070 * transaction, and VFS/VM ensures that ext4_truncate() cannot run 4071 * simultaneously on behalf of the same inode. 4072 * 4073 * As we work through the truncate and commit bits of it to the journal there 4074 * is one core, guiding principle: the file's tree must always be consistent on 4075 * disk. We must be able to restart the truncate after a crash. 4076 * 4077 * The file's tree may be transiently inconsistent in memory (although it 4078 * probably isn't), but whenever we close off and commit a journal transaction, 4079 * the contents of (the filesystem + the journal) must be consistent and 4080 * restartable. It's pretty simple, really: bottom up, right to left (although 4081 * left-to-right works OK too). 4082 * 4083 * Note that at recovery time, journal replay occurs *before* the restart of 4084 * truncate against the orphan inode list. 4085 * 4086 * The committed inode has the new, desired i_size (which is the same as 4087 * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see 4088 * that this inode's truncate did not complete and it will again call 4089 * ext4_truncate() to have another go. So there will be instantiated blocks 4090 * to the right of the truncation point in a crashed ext4 filesystem. But 4091 * that's fine - as long as they are linked from the inode, the post-crash 4092 * ext4_truncate() run will find them and release them. 4093 */ 4094 void ext4_truncate(struct inode *inode) 4095 { 4096 struct ext4_inode_info *ei = EXT4_I(inode); 4097 unsigned int credits; 4098 handle_t *handle; 4099 struct address_space *mapping = inode->i_mapping; 4100 4101 /* 4102 * There is a possibility that we're either freeing the inode 4103 * or it's a completely new inode. In those cases we might not 4104 * have i_mutex locked because it's not necessary. 4105 */ 4106 if (!(inode->i_state & (I_NEW|I_FREEING))) 4107 WARN_ON(!inode_is_locked(inode)); 4108 trace_ext4_truncate_enter(inode); 4109 4110 if (!ext4_can_truncate(inode)) 4111 return; 4112 4113 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS); 4114 4115 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC)) 4116 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE); 4117 4118 if (ext4_has_inline_data(inode)) { 4119 int has_inline = 1; 4120 4121 ext4_inline_data_truncate(inode, &has_inline); 4122 if (has_inline) 4123 return; 4124 } 4125 4126 /* If we zero-out tail of the page, we have to create jinode for jbd2 */ 4127 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) { 4128 if (ext4_inode_attach_jinode(inode) < 0) 4129 return; 4130 } 4131 4132 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 4133 credits = ext4_writepage_trans_blocks(inode); 4134 else 4135 credits = ext4_blocks_for_truncate(inode); 4136 4137 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 4138 if (IS_ERR(handle)) { 4139 ext4_std_error(inode->i_sb, PTR_ERR(handle)); 4140 return; 4141 } 4142 4143 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) 4144 ext4_block_truncate_page(handle, mapping, inode->i_size); 4145 4146 /* 4147 * We add the inode to the orphan list, so that if this 4148 * truncate spans multiple transactions, and we crash, we will 4149 * resume the truncate when the filesystem recovers. It also 4150 * marks the inode dirty, to catch the new size. 4151 * 4152 * Implication: the file must always be in a sane, consistent 4153 * truncatable state while each transaction commits. 4154 */ 4155 if (ext4_orphan_add(handle, inode)) 4156 goto out_stop; 4157 4158 down_write(&EXT4_I(inode)->i_data_sem); 4159 4160 ext4_discard_preallocations(inode); 4161 4162 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 4163 ext4_ext_truncate(handle, inode); 4164 else 4165 ext4_ind_truncate(handle, inode); 4166 4167 up_write(&ei->i_data_sem); 4168 4169 if (IS_SYNC(inode)) 4170 ext4_handle_sync(handle); 4171 4172 out_stop: 4173 /* 4174 * If this was a simple ftruncate() and the file will remain alive, 4175 * then we need to clear up the orphan record which we created above. 4176 * However, if this was a real unlink then we were called by 4177 * ext4_evict_inode(), and we allow that function to clean up the 4178 * orphan info for us. 4179 */ 4180 if (inode->i_nlink) 4181 ext4_orphan_del(handle, inode); 4182 4183 inode->i_mtime = inode->i_ctime = ext4_current_time(inode); 4184 ext4_mark_inode_dirty(handle, inode); 4185 ext4_journal_stop(handle); 4186 4187 trace_ext4_truncate_exit(inode); 4188 } 4189 4190 /* 4191 * ext4_get_inode_loc returns with an extra refcount against the inode's 4192 * underlying buffer_head on success. If 'in_mem' is true, we have all 4193 * data in memory that is needed to recreate the on-disk version of this 4194 * inode. 4195 */ 4196 static int __ext4_get_inode_loc(struct inode *inode, 4197 struct ext4_iloc *iloc, int in_mem) 4198 { 4199 struct ext4_group_desc *gdp; 4200 struct buffer_head *bh; 4201 struct super_block *sb = inode->i_sb; 4202 ext4_fsblk_t block; 4203 int inodes_per_block, inode_offset; 4204 4205 iloc->bh = NULL; 4206 if (!ext4_valid_inum(sb, inode->i_ino)) 4207 return -EFSCORRUPTED; 4208 4209 iloc->block_group = (inode->i_ino - 1) / EXT4_INODES_PER_GROUP(sb); 4210 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); 4211 if (!gdp) 4212 return -EIO; 4213 4214 /* 4215 * Figure out the offset within the block group inode table 4216 */ 4217 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 4218 inode_offset = ((inode->i_ino - 1) % 4219 EXT4_INODES_PER_GROUP(sb)); 4220 block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block); 4221 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); 4222 4223 bh = sb_getblk(sb, block); 4224 if (unlikely(!bh)) 4225 return -ENOMEM; 4226 if (!buffer_uptodate(bh)) { 4227 lock_buffer(bh); 4228 4229 /* 4230 * If the buffer has the write error flag, we have failed 4231 * to write out another inode in the same block. In this 4232 * case, we don't have to read the block because we may 4233 * read the old inode data successfully. 4234 */ 4235 if (buffer_write_io_error(bh) && !buffer_uptodate(bh)) 4236 set_buffer_uptodate(bh); 4237 4238 if (buffer_uptodate(bh)) { 4239 /* someone brought it uptodate while we waited */ 4240 unlock_buffer(bh); 4241 goto has_buffer; 4242 } 4243 4244 /* 4245 * If we have all information of the inode in memory and this 4246 * is the only valid inode in the block, we need not read the 4247 * block. 4248 */ 4249 if (in_mem) { 4250 struct buffer_head *bitmap_bh; 4251 int i, start; 4252 4253 start = inode_offset & ~(inodes_per_block - 1); 4254 4255 /* Is the inode bitmap in cache? */ 4256 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp)); 4257 if (unlikely(!bitmap_bh)) 4258 goto make_io; 4259 4260 /* 4261 * If the inode bitmap isn't in cache then the 4262 * optimisation may end up performing two reads instead 4263 * of one, so skip it. 4264 */ 4265 if (!buffer_uptodate(bitmap_bh)) { 4266 brelse(bitmap_bh); 4267 goto make_io; 4268 } 4269 for (i = start; i < start + inodes_per_block; i++) { 4270 if (i == inode_offset) 4271 continue; 4272 if (ext4_test_bit(i, bitmap_bh->b_data)) 4273 break; 4274 } 4275 brelse(bitmap_bh); 4276 if (i == start + inodes_per_block) { 4277 /* all other inodes are free, so skip I/O */ 4278 memset(bh->b_data, 0, bh->b_size); 4279 set_buffer_uptodate(bh); 4280 unlock_buffer(bh); 4281 goto has_buffer; 4282 } 4283 } 4284 4285 make_io: 4286 /* 4287 * If we need to do any I/O, try to pre-readahead extra 4288 * blocks from the inode table. 4289 */ 4290 if (EXT4_SB(sb)->s_inode_readahead_blks) { 4291 ext4_fsblk_t b, end, table; 4292 unsigned num; 4293 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks; 4294 4295 table = ext4_inode_table(sb, gdp); 4296 /* s_inode_readahead_blks is always a power of 2 */ 4297 b = block & ~((ext4_fsblk_t) ra_blks - 1); 4298 if (table > b) 4299 b = table; 4300 end = b + ra_blks; 4301 num = EXT4_INODES_PER_GROUP(sb); 4302 if (ext4_has_group_desc_csum(sb)) 4303 num -= ext4_itable_unused_count(sb, gdp); 4304 table += num / inodes_per_block; 4305 if (end > table) 4306 end = table; 4307 while (b <= end) 4308 sb_breadahead(sb, b++); 4309 } 4310 4311 /* 4312 * There are other valid inodes in the buffer, this inode 4313 * has in-inode xattrs, or we don't have this inode in memory. 4314 * Read the block from disk. 4315 */ 4316 trace_ext4_load_inode(inode); 4317 get_bh(bh); 4318 bh->b_end_io = end_buffer_read_sync; 4319 submit_bh(REQ_OP_READ, REQ_META | REQ_PRIO, bh); 4320 wait_on_buffer(bh); 4321 if (!buffer_uptodate(bh)) { 4322 EXT4_ERROR_INODE_BLOCK(inode, block, 4323 "unable to read itable block"); 4324 brelse(bh); 4325 return -EIO; 4326 } 4327 } 4328 has_buffer: 4329 iloc->bh = bh; 4330 return 0; 4331 } 4332 4333 int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) 4334 { 4335 /* We have all inode data except xattrs in memory here. */ 4336 return __ext4_get_inode_loc(inode, iloc, 4337 !ext4_test_inode_state(inode, EXT4_STATE_XATTR)); 4338 } 4339 4340 void ext4_set_inode_flags(struct inode *inode) 4341 { 4342 unsigned int flags = EXT4_I(inode)->i_flags; 4343 unsigned int new_fl = 0; 4344 4345 if (flags & EXT4_SYNC_FL) 4346 new_fl |= S_SYNC; 4347 if (flags & EXT4_APPEND_FL) 4348 new_fl |= S_APPEND; 4349 if (flags & EXT4_IMMUTABLE_FL) 4350 new_fl |= S_IMMUTABLE; 4351 if (flags & EXT4_NOATIME_FL) 4352 new_fl |= S_NOATIME; 4353 if (flags & EXT4_DIRSYNC_FL) 4354 new_fl |= S_DIRSYNC; 4355 if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode)) 4356 new_fl |= S_DAX; 4357 inode_set_flags(inode, new_fl, 4358 S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX); 4359 } 4360 4361 /* Propagate flags from i_flags to EXT4_I(inode)->i_flags */ 4362 void ext4_get_inode_flags(struct ext4_inode_info *ei) 4363 { 4364 unsigned int vfs_fl; 4365 unsigned long old_fl, new_fl; 4366 4367 do { 4368 vfs_fl = ei->vfs_inode.i_flags; 4369 old_fl = ei->i_flags; 4370 new_fl = old_fl & ~(EXT4_SYNC_FL|EXT4_APPEND_FL| 4371 EXT4_IMMUTABLE_FL|EXT4_NOATIME_FL| 4372 EXT4_DIRSYNC_FL); 4373 if (vfs_fl & S_SYNC) 4374 new_fl |= EXT4_SYNC_FL; 4375 if (vfs_fl & S_APPEND) 4376 new_fl |= EXT4_APPEND_FL; 4377 if (vfs_fl & S_IMMUTABLE) 4378 new_fl |= EXT4_IMMUTABLE_FL; 4379 if (vfs_fl & S_NOATIME) 4380 new_fl |= EXT4_NOATIME_FL; 4381 if (vfs_fl & S_DIRSYNC) 4382 new_fl |= EXT4_DIRSYNC_FL; 4383 } while (cmpxchg(&ei->i_flags, old_fl, new_fl) != old_fl); 4384 } 4385 4386 static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode, 4387 struct ext4_inode_info *ei) 4388 { 4389 blkcnt_t i_blocks ; 4390 struct inode *inode = &(ei->vfs_inode); 4391 struct super_block *sb = inode->i_sb; 4392 4393 if (ext4_has_feature_huge_file(sb)) { 4394 /* we are using combined 48 bit field */ 4395 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 | 4396 le32_to_cpu(raw_inode->i_blocks_lo); 4397 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) { 4398 /* i_blocks represent file system block size */ 4399 return i_blocks << (inode->i_blkbits - 9); 4400 } else { 4401 return i_blocks; 4402 } 4403 } else { 4404 return le32_to_cpu(raw_inode->i_blocks_lo); 4405 } 4406 } 4407 4408 static inline void ext4_iget_extra_inode(struct inode *inode, 4409 struct ext4_inode *raw_inode, 4410 struct ext4_inode_info *ei) 4411 { 4412 __le32 *magic = (void *)raw_inode + 4413 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize; 4414 if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC)) { 4415 ext4_set_inode_state(inode, EXT4_STATE_XATTR); 4416 ext4_find_inline_data_nolock(inode); 4417 } else 4418 EXT4_I(inode)->i_inline_off = 0; 4419 } 4420 4421 int ext4_get_projid(struct inode *inode, kprojid_t *projid) 4422 { 4423 if (!ext4_has_feature_project(inode->i_sb)) 4424 return -EOPNOTSUPP; 4425 *projid = EXT4_I(inode)->i_projid; 4426 return 0; 4427 } 4428 4429 struct inode *ext4_iget(struct super_block *sb, unsigned long ino) 4430 { 4431 struct ext4_iloc iloc; 4432 struct ext4_inode *raw_inode; 4433 struct ext4_inode_info *ei; 4434 struct inode *inode; 4435 journal_t *journal = EXT4_SB(sb)->s_journal; 4436 long ret; 4437 int block; 4438 uid_t i_uid; 4439 gid_t i_gid; 4440 projid_t i_projid; 4441 4442 inode = iget_locked(sb, ino); 4443 if (!inode) 4444 return ERR_PTR(-ENOMEM); 4445 if (!(inode->i_state & I_NEW)) 4446 return inode; 4447 4448 ei = EXT4_I(inode); 4449 iloc.bh = NULL; 4450 4451 ret = __ext4_get_inode_loc(inode, &iloc, 0); 4452 if (ret < 0) 4453 goto bad_inode; 4454 raw_inode = ext4_raw_inode(&iloc); 4455 4456 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 4457 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize); 4458 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > 4459 EXT4_INODE_SIZE(inode->i_sb)) { 4460 EXT4_ERROR_INODE(inode, "bad extra_isize (%u != %u)", 4461 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize, 4462 EXT4_INODE_SIZE(inode->i_sb)); 4463 ret = -EFSCORRUPTED; 4464 goto bad_inode; 4465 } 4466 } else 4467 ei->i_extra_isize = 0; 4468 4469 /* Precompute checksum seed for inode metadata */ 4470 if (ext4_has_metadata_csum(sb)) { 4471 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 4472 __u32 csum; 4473 __le32 inum = cpu_to_le32(inode->i_ino); 4474 __le32 gen = raw_inode->i_generation; 4475 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum, 4476 sizeof(inum)); 4477 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen, 4478 sizeof(gen)); 4479 } 4480 4481 if (!ext4_inode_csum_verify(inode, raw_inode, ei)) { 4482 EXT4_ERROR_INODE(inode, "checksum invalid"); 4483 ret = -EFSBADCRC; 4484 goto bad_inode; 4485 } 4486 4487 inode->i_mode = le16_to_cpu(raw_inode->i_mode); 4488 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low); 4489 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low); 4490 if (ext4_has_feature_project(sb) && 4491 EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE && 4492 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) 4493 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid); 4494 else 4495 i_projid = EXT4_DEF_PROJID; 4496 4497 if (!(test_opt(inode->i_sb, NO_UID32))) { 4498 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16; 4499 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16; 4500 } 4501 i_uid_write(inode, i_uid); 4502 i_gid_write(inode, i_gid); 4503 ei->i_projid = make_kprojid(&init_user_ns, i_projid); 4504 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); 4505 4506 ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */ 4507 ei->i_inline_off = 0; 4508 ei->i_dir_start_lookup = 0; 4509 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); 4510 /* We now have enough fields to check if the inode was active or not. 4511 * This is needed because nfsd might try to access dead inodes 4512 * the test is that same one that e2fsck uses 4513 * NeilBrown 1999oct15 4514 */ 4515 if (inode->i_nlink == 0) { 4516 if ((inode->i_mode == 0 || 4517 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) && 4518 ino != EXT4_BOOT_LOADER_INO) { 4519 /* this inode is deleted */ 4520 ret = -ESTALE; 4521 goto bad_inode; 4522 } 4523 /* The only unlinked inodes we let through here have 4524 * valid i_mode and are being read by the orphan 4525 * recovery code: that's fine, we're about to complete 4526 * the process of deleting those. 4527 * OR it is the EXT4_BOOT_LOADER_INO which is 4528 * not initialized on a new filesystem. */ 4529 } 4530 ei->i_flags = le32_to_cpu(raw_inode->i_flags); 4531 inode->i_blocks = ext4_inode_blocks(raw_inode, ei); 4532 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo); 4533 if (ext4_has_feature_64bit(sb)) 4534 ei->i_file_acl |= 4535 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; 4536 inode->i_size = ext4_isize(raw_inode); 4537 ei->i_disksize = inode->i_size; 4538 #ifdef CONFIG_QUOTA 4539 ei->i_reserved_quota = 0; 4540 #endif 4541 inode->i_generation = le32_to_cpu(raw_inode->i_generation); 4542 ei->i_block_group = iloc.block_group; 4543 ei->i_last_alloc_group = ~0; 4544 /* 4545 * NOTE! The in-memory inode i_data array is in little-endian order 4546 * even on big-endian machines: we do NOT byteswap the block numbers! 4547 */ 4548 for (block = 0; block < EXT4_N_BLOCKS; block++) 4549 ei->i_data[block] = raw_inode->i_block[block]; 4550 INIT_LIST_HEAD(&ei->i_orphan); 4551 4552 /* 4553 * Set transaction id's of transactions that have to be committed 4554 * to finish f[data]sync. We set them to currently running transaction 4555 * as we cannot be sure that the inode or some of its metadata isn't 4556 * part of the transaction - the inode could have been reclaimed and 4557 * now it is reread from disk. 4558 */ 4559 if (journal) { 4560 transaction_t *transaction; 4561 tid_t tid; 4562 4563 read_lock(&journal->j_state_lock); 4564 if (journal->j_running_transaction) 4565 transaction = journal->j_running_transaction; 4566 else 4567 transaction = journal->j_committing_transaction; 4568 if (transaction) 4569 tid = transaction->t_tid; 4570 else 4571 tid = journal->j_commit_sequence; 4572 read_unlock(&journal->j_state_lock); 4573 ei->i_sync_tid = tid; 4574 ei->i_datasync_tid = tid; 4575 } 4576 4577 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 4578 if (ei->i_extra_isize == 0) { 4579 /* The extra space is currently unused. Use it. */ 4580 ei->i_extra_isize = sizeof(struct ext4_inode) - 4581 EXT4_GOOD_OLD_INODE_SIZE; 4582 } else { 4583 ext4_iget_extra_inode(inode, raw_inode, ei); 4584 } 4585 } 4586 4587 EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode); 4588 EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode); 4589 EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode); 4590 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode); 4591 4592 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { 4593 inode->i_version = le32_to_cpu(raw_inode->i_disk_version); 4594 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 4595 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 4596 inode->i_version |= 4597 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32; 4598 } 4599 } 4600 4601 ret = 0; 4602 if (ei->i_file_acl && 4603 !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) { 4604 EXT4_ERROR_INODE(inode, "bad extended attribute block %llu", 4605 ei->i_file_acl); 4606 ret = -EFSCORRUPTED; 4607 goto bad_inode; 4608 } else if (!ext4_has_inline_data(inode)) { 4609 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 4610 if ((S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || 4611 (S_ISLNK(inode->i_mode) && 4612 !ext4_inode_is_fast_symlink(inode)))) 4613 /* Validate extent which is part of inode */ 4614 ret = ext4_ext_check_inode(inode); 4615 } else if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || 4616 (S_ISLNK(inode->i_mode) && 4617 !ext4_inode_is_fast_symlink(inode))) { 4618 /* Validate block references which are part of inode */ 4619 ret = ext4_ind_check_inode(inode); 4620 } 4621 } 4622 if (ret) 4623 goto bad_inode; 4624 4625 if (S_ISREG(inode->i_mode)) { 4626 inode->i_op = &ext4_file_inode_operations; 4627 inode->i_fop = &ext4_file_operations; 4628 ext4_set_aops(inode); 4629 } else if (S_ISDIR(inode->i_mode)) { 4630 inode->i_op = &ext4_dir_inode_operations; 4631 inode->i_fop = &ext4_dir_operations; 4632 } else if (S_ISLNK(inode->i_mode)) { 4633 if (ext4_encrypted_inode(inode)) { 4634 inode->i_op = &ext4_encrypted_symlink_inode_operations; 4635 ext4_set_aops(inode); 4636 } else if (ext4_inode_is_fast_symlink(inode)) { 4637 inode->i_link = (char *)ei->i_data; 4638 inode->i_op = &ext4_fast_symlink_inode_operations; 4639 nd_terminate_link(ei->i_data, inode->i_size, 4640 sizeof(ei->i_data) - 1); 4641 } else { 4642 inode->i_op = &ext4_symlink_inode_operations; 4643 ext4_set_aops(inode); 4644 } 4645 inode_nohighmem(inode); 4646 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || 4647 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { 4648 inode->i_op = &ext4_special_inode_operations; 4649 if (raw_inode->i_block[0]) 4650 init_special_inode(inode, inode->i_mode, 4651 old_decode_dev(le32_to_cpu(raw_inode->i_block[0]))); 4652 else 4653 init_special_inode(inode, inode->i_mode, 4654 new_decode_dev(le32_to_cpu(raw_inode->i_block[1]))); 4655 } else if (ino == EXT4_BOOT_LOADER_INO) { 4656 make_bad_inode(inode); 4657 } else { 4658 ret = -EFSCORRUPTED; 4659 EXT4_ERROR_INODE(inode, "bogus i_mode (%o)", inode->i_mode); 4660 goto bad_inode; 4661 } 4662 brelse(iloc.bh); 4663 ext4_set_inode_flags(inode); 4664 unlock_new_inode(inode); 4665 return inode; 4666 4667 bad_inode: 4668 brelse(iloc.bh); 4669 iget_failed(inode); 4670 return ERR_PTR(ret); 4671 } 4672 4673 struct inode *ext4_iget_normal(struct super_block *sb, unsigned long ino) 4674 { 4675 if (ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) 4676 return ERR_PTR(-EFSCORRUPTED); 4677 return ext4_iget(sb, ino); 4678 } 4679 4680 static int ext4_inode_blocks_set(handle_t *handle, 4681 struct ext4_inode *raw_inode, 4682 struct ext4_inode_info *ei) 4683 { 4684 struct inode *inode = &(ei->vfs_inode); 4685 u64 i_blocks = inode->i_blocks; 4686 struct super_block *sb = inode->i_sb; 4687 4688 if (i_blocks <= ~0U) { 4689 /* 4690 * i_blocks can be represented in a 32 bit variable 4691 * as multiple of 512 bytes 4692 */ 4693 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4694 raw_inode->i_blocks_high = 0; 4695 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4696 return 0; 4697 } 4698 if (!ext4_has_feature_huge_file(sb)) 4699 return -EFBIG; 4700 4701 if (i_blocks <= 0xffffffffffffULL) { 4702 /* 4703 * i_blocks can be represented in a 48 bit variable 4704 * as multiple of 512 bytes 4705 */ 4706 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4707 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 4708 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4709 } else { 4710 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4711 /* i_block is stored in file system block size */ 4712 i_blocks = i_blocks >> (inode->i_blkbits - 9); 4713 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4714 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 4715 } 4716 return 0; 4717 } 4718 4719 struct other_inode { 4720 unsigned long orig_ino; 4721 struct ext4_inode *raw_inode; 4722 }; 4723 4724 static int other_inode_match(struct inode * inode, unsigned long ino, 4725 void *data) 4726 { 4727 struct other_inode *oi = (struct other_inode *) data; 4728 4729 if ((inode->i_ino != ino) || 4730 (inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW | 4731 I_DIRTY_SYNC | I_DIRTY_DATASYNC)) || 4732 ((inode->i_state & I_DIRTY_TIME) == 0)) 4733 return 0; 4734 spin_lock(&inode->i_lock); 4735 if (((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW | 4736 I_DIRTY_SYNC | I_DIRTY_DATASYNC)) == 0) && 4737 (inode->i_state & I_DIRTY_TIME)) { 4738 struct ext4_inode_info *ei = EXT4_I(inode); 4739 4740 inode->i_state &= ~(I_DIRTY_TIME | I_DIRTY_TIME_EXPIRED); 4741 spin_unlock(&inode->i_lock); 4742 4743 spin_lock(&ei->i_raw_lock); 4744 EXT4_INODE_SET_XTIME(i_ctime, inode, oi->raw_inode); 4745 EXT4_INODE_SET_XTIME(i_mtime, inode, oi->raw_inode); 4746 EXT4_INODE_SET_XTIME(i_atime, inode, oi->raw_inode); 4747 ext4_inode_csum_set(inode, oi->raw_inode, ei); 4748 spin_unlock(&ei->i_raw_lock); 4749 trace_ext4_other_inode_update_time(inode, oi->orig_ino); 4750 return -1; 4751 } 4752 spin_unlock(&inode->i_lock); 4753 return -1; 4754 } 4755 4756 /* 4757 * Opportunistically update the other time fields for other inodes in 4758 * the same inode table block. 4759 */ 4760 static void ext4_update_other_inodes_time(struct super_block *sb, 4761 unsigned long orig_ino, char *buf) 4762 { 4763 struct other_inode oi; 4764 unsigned long ino; 4765 int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 4766 int inode_size = EXT4_INODE_SIZE(sb); 4767 4768 oi.orig_ino = orig_ino; 4769 /* 4770 * Calculate the first inode in the inode table block. Inode 4771 * numbers are one-based. That is, the first inode in a block 4772 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1). 4773 */ 4774 ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1; 4775 for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) { 4776 if (ino == orig_ino) 4777 continue; 4778 oi.raw_inode = (struct ext4_inode *) buf; 4779 (void) find_inode_nowait(sb, ino, other_inode_match, &oi); 4780 } 4781 } 4782 4783 /* 4784 * Post the struct inode info into an on-disk inode location in the 4785 * buffer-cache. This gobbles the caller's reference to the 4786 * buffer_head in the inode location struct. 4787 * 4788 * The caller must have write access to iloc->bh. 4789 */ 4790 static int ext4_do_update_inode(handle_t *handle, 4791 struct inode *inode, 4792 struct ext4_iloc *iloc) 4793 { 4794 struct ext4_inode *raw_inode = ext4_raw_inode(iloc); 4795 struct ext4_inode_info *ei = EXT4_I(inode); 4796 struct buffer_head *bh = iloc->bh; 4797 struct super_block *sb = inode->i_sb; 4798 int err = 0, rc, block; 4799 int need_datasync = 0, set_large_file = 0; 4800 uid_t i_uid; 4801 gid_t i_gid; 4802 projid_t i_projid; 4803 4804 spin_lock(&ei->i_raw_lock); 4805 4806 /* For fields not tracked in the in-memory inode, 4807 * initialise them to zero for new inodes. */ 4808 if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) 4809 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size); 4810 4811 ext4_get_inode_flags(ei); 4812 raw_inode->i_mode = cpu_to_le16(inode->i_mode); 4813 i_uid = i_uid_read(inode); 4814 i_gid = i_gid_read(inode); 4815 i_projid = from_kprojid(&init_user_ns, ei->i_projid); 4816 if (!(test_opt(inode->i_sb, NO_UID32))) { 4817 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid)); 4818 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid)); 4819 /* 4820 * Fix up interoperability with old kernels. Otherwise, old inodes get 4821 * re-used with the upper 16 bits of the uid/gid intact 4822 */ 4823 if (ei->i_dtime && list_empty(&ei->i_orphan)) { 4824 raw_inode->i_uid_high = 0; 4825 raw_inode->i_gid_high = 0; 4826 } else { 4827 raw_inode->i_uid_high = 4828 cpu_to_le16(high_16_bits(i_uid)); 4829 raw_inode->i_gid_high = 4830 cpu_to_le16(high_16_bits(i_gid)); 4831 } 4832 } else { 4833 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid)); 4834 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid)); 4835 raw_inode->i_uid_high = 0; 4836 raw_inode->i_gid_high = 0; 4837 } 4838 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink); 4839 4840 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode); 4841 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode); 4842 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode); 4843 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode); 4844 4845 err = ext4_inode_blocks_set(handle, raw_inode, ei); 4846 if (err) { 4847 spin_unlock(&ei->i_raw_lock); 4848 goto out_brelse; 4849 } 4850 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime); 4851 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF); 4852 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) 4853 raw_inode->i_file_acl_high = 4854 cpu_to_le16(ei->i_file_acl >> 32); 4855 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl); 4856 if (ei->i_disksize != ext4_isize(raw_inode)) { 4857 ext4_isize_set(raw_inode, ei->i_disksize); 4858 need_datasync = 1; 4859 } 4860 if (ei->i_disksize > 0x7fffffffULL) { 4861 if (!ext4_has_feature_large_file(sb) || 4862 EXT4_SB(sb)->s_es->s_rev_level == 4863 cpu_to_le32(EXT4_GOOD_OLD_REV)) 4864 set_large_file = 1; 4865 } 4866 raw_inode->i_generation = cpu_to_le32(inode->i_generation); 4867 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { 4868 if (old_valid_dev(inode->i_rdev)) { 4869 raw_inode->i_block[0] = 4870 cpu_to_le32(old_encode_dev(inode->i_rdev)); 4871 raw_inode->i_block[1] = 0; 4872 } else { 4873 raw_inode->i_block[0] = 0; 4874 raw_inode->i_block[1] = 4875 cpu_to_le32(new_encode_dev(inode->i_rdev)); 4876 raw_inode->i_block[2] = 0; 4877 } 4878 } else if (!ext4_has_inline_data(inode)) { 4879 for (block = 0; block < EXT4_N_BLOCKS; block++) 4880 raw_inode->i_block[block] = ei->i_data[block]; 4881 } 4882 4883 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { 4884 raw_inode->i_disk_version = cpu_to_le32(inode->i_version); 4885 if (ei->i_extra_isize) { 4886 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 4887 raw_inode->i_version_hi = 4888 cpu_to_le32(inode->i_version >> 32); 4889 raw_inode->i_extra_isize = 4890 cpu_to_le16(ei->i_extra_isize); 4891 } 4892 } 4893 4894 BUG_ON(!ext4_has_feature_project(inode->i_sb) && 4895 i_projid != EXT4_DEF_PROJID); 4896 4897 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 4898 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) 4899 raw_inode->i_projid = cpu_to_le32(i_projid); 4900 4901 ext4_inode_csum_set(inode, raw_inode, ei); 4902 spin_unlock(&ei->i_raw_lock); 4903 if (inode->i_sb->s_flags & MS_LAZYTIME) 4904 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino, 4905 bh->b_data); 4906 4907 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 4908 rc = ext4_handle_dirty_metadata(handle, NULL, bh); 4909 if (!err) 4910 err = rc; 4911 ext4_clear_inode_state(inode, EXT4_STATE_NEW); 4912 if (set_large_file) { 4913 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access"); 4914 err = ext4_journal_get_write_access(handle, EXT4_SB(sb)->s_sbh); 4915 if (err) 4916 goto out_brelse; 4917 ext4_update_dynamic_rev(sb); 4918 ext4_set_feature_large_file(sb); 4919 ext4_handle_sync(handle); 4920 err = ext4_handle_dirty_super(handle, sb); 4921 } 4922 ext4_update_inode_fsync_trans(handle, inode, need_datasync); 4923 out_brelse: 4924 brelse(bh); 4925 ext4_std_error(inode->i_sb, err); 4926 return err; 4927 } 4928 4929 /* 4930 * ext4_write_inode() 4931 * 4932 * We are called from a few places: 4933 * 4934 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files. 4935 * Here, there will be no transaction running. We wait for any running 4936 * transaction to commit. 4937 * 4938 * - Within flush work (sys_sync(), kupdate and such). 4939 * We wait on commit, if told to. 4940 * 4941 * - Within iput_final() -> write_inode_now() 4942 * We wait on commit, if told to. 4943 * 4944 * In all cases it is actually safe for us to return without doing anything, 4945 * because the inode has been copied into a raw inode buffer in 4946 * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL 4947 * writeback. 4948 * 4949 * Note that we are absolutely dependent upon all inode dirtiers doing the 4950 * right thing: they *must* call mark_inode_dirty() after dirtying info in 4951 * which we are interested. 4952 * 4953 * It would be a bug for them to not do this. The code: 4954 * 4955 * mark_inode_dirty(inode) 4956 * stuff(); 4957 * inode->i_size = expr; 4958 * 4959 * is in error because write_inode() could occur while `stuff()' is running, 4960 * and the new i_size will be lost. Plus the inode will no longer be on the 4961 * superblock's dirty inode list. 4962 */ 4963 int ext4_write_inode(struct inode *inode, struct writeback_control *wbc) 4964 { 4965 int err; 4966 4967 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC)) 4968 return 0; 4969 4970 if (EXT4_SB(inode->i_sb)->s_journal) { 4971 if (ext4_journal_current_handle()) { 4972 jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n"); 4973 dump_stack(); 4974 return -EIO; 4975 } 4976 4977 /* 4978 * No need to force transaction in WB_SYNC_NONE mode. Also 4979 * ext4_sync_fs() will force the commit after everything is 4980 * written. 4981 */ 4982 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) 4983 return 0; 4984 4985 err = ext4_force_commit(inode->i_sb); 4986 } else { 4987 struct ext4_iloc iloc; 4988 4989 err = __ext4_get_inode_loc(inode, &iloc, 0); 4990 if (err) 4991 return err; 4992 /* 4993 * sync(2) will flush the whole buffer cache. No need to do 4994 * it here separately for each inode. 4995 */ 4996 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) 4997 sync_dirty_buffer(iloc.bh); 4998 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) { 4999 EXT4_ERROR_INODE_BLOCK(inode, iloc.bh->b_blocknr, 5000 "IO error syncing inode"); 5001 err = -EIO; 5002 } 5003 brelse(iloc.bh); 5004 } 5005 return err; 5006 } 5007 5008 /* 5009 * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate 5010 * buffers that are attached to a page stradding i_size and are undergoing 5011 * commit. In that case we have to wait for commit to finish and try again. 5012 */ 5013 static void ext4_wait_for_tail_page_commit(struct inode *inode) 5014 { 5015 struct page *page; 5016 unsigned offset; 5017 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 5018 tid_t commit_tid = 0; 5019 int ret; 5020 5021 offset = inode->i_size & (PAGE_SIZE - 1); 5022 /* 5023 * All buffers in the last page remain valid? Then there's nothing to 5024 * do. We do the check mainly to optimize the common PAGE_SIZE == 5025 * blocksize case 5026 */ 5027 if (offset > PAGE_SIZE - (1 << inode->i_blkbits)) 5028 return; 5029 while (1) { 5030 page = find_lock_page(inode->i_mapping, 5031 inode->i_size >> PAGE_SHIFT); 5032 if (!page) 5033 return; 5034 ret = __ext4_journalled_invalidatepage(page, offset, 5035 PAGE_SIZE - offset); 5036 unlock_page(page); 5037 put_page(page); 5038 if (ret != -EBUSY) 5039 return; 5040 commit_tid = 0; 5041 read_lock(&journal->j_state_lock); 5042 if (journal->j_committing_transaction) 5043 commit_tid = journal->j_committing_transaction->t_tid; 5044 read_unlock(&journal->j_state_lock); 5045 if (commit_tid) 5046 jbd2_log_wait_commit(journal, commit_tid); 5047 } 5048 } 5049 5050 /* 5051 * ext4_setattr() 5052 * 5053 * Called from notify_change. 5054 * 5055 * We want to trap VFS attempts to truncate the file as soon as 5056 * possible. In particular, we want to make sure that when the VFS 5057 * shrinks i_size, we put the inode on the orphan list and modify 5058 * i_disksize immediately, so that during the subsequent flushing of 5059 * dirty pages and freeing of disk blocks, we can guarantee that any 5060 * commit will leave the blocks being flushed in an unused state on 5061 * disk. (On recovery, the inode will get truncated and the blocks will 5062 * be freed, so we have a strong guarantee that no future commit will 5063 * leave these blocks visible to the user.) 5064 * 5065 * Another thing we have to assure is that if we are in ordered mode 5066 * and inode is still attached to the committing transaction, we must 5067 * we start writeout of all the dirty pages which are being truncated. 5068 * This way we are sure that all the data written in the previous 5069 * transaction are already on disk (truncate waits for pages under 5070 * writeback). 5071 * 5072 * Called with inode->i_mutex down. 5073 */ 5074 int ext4_setattr(struct dentry *dentry, struct iattr *attr) 5075 { 5076 struct inode *inode = d_inode(dentry); 5077 int error, rc = 0; 5078 int orphan = 0; 5079 const unsigned int ia_valid = attr->ia_valid; 5080 5081 error = setattr_prepare(dentry, attr); 5082 if (error) 5083 return error; 5084 5085 if (is_quota_modification(inode, attr)) { 5086 error = dquot_initialize(inode); 5087 if (error) 5088 return error; 5089 } 5090 if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) || 5091 (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) { 5092 handle_t *handle; 5093 5094 /* (user+group)*(old+new) structure, inode write (sb, 5095 * inode block, ? - but truncate inode update has it) */ 5096 handle = ext4_journal_start(inode, EXT4_HT_QUOTA, 5097 (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) + 5098 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3); 5099 if (IS_ERR(handle)) { 5100 error = PTR_ERR(handle); 5101 goto err_out; 5102 } 5103 error = dquot_transfer(inode, attr); 5104 if (error) { 5105 ext4_journal_stop(handle); 5106 return error; 5107 } 5108 /* Update corresponding info in inode so that everything is in 5109 * one transaction */ 5110 if (attr->ia_valid & ATTR_UID) 5111 inode->i_uid = attr->ia_uid; 5112 if (attr->ia_valid & ATTR_GID) 5113 inode->i_gid = attr->ia_gid; 5114 error = ext4_mark_inode_dirty(handle, inode); 5115 ext4_journal_stop(handle); 5116 } 5117 5118 if (attr->ia_valid & ATTR_SIZE) { 5119 handle_t *handle; 5120 loff_t oldsize = inode->i_size; 5121 int shrink = (attr->ia_size <= inode->i_size); 5122 5123 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 5124 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 5125 5126 if (attr->ia_size > sbi->s_bitmap_maxbytes) 5127 return -EFBIG; 5128 } 5129 if (!S_ISREG(inode->i_mode)) 5130 return -EINVAL; 5131 5132 if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size) 5133 inode_inc_iversion(inode); 5134 5135 if (ext4_should_order_data(inode) && 5136 (attr->ia_size < inode->i_size)) { 5137 error = ext4_begin_ordered_truncate(inode, 5138 attr->ia_size); 5139 if (error) 5140 goto err_out; 5141 } 5142 if (attr->ia_size != inode->i_size) { 5143 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); 5144 if (IS_ERR(handle)) { 5145 error = PTR_ERR(handle); 5146 goto err_out; 5147 } 5148 if (ext4_handle_valid(handle) && shrink) { 5149 error = ext4_orphan_add(handle, inode); 5150 orphan = 1; 5151 } 5152 /* 5153 * Update c/mtime on truncate up, ext4_truncate() will 5154 * update c/mtime in shrink case below 5155 */ 5156 if (!shrink) { 5157 inode->i_mtime = ext4_current_time(inode); 5158 inode->i_ctime = inode->i_mtime; 5159 } 5160 down_write(&EXT4_I(inode)->i_data_sem); 5161 EXT4_I(inode)->i_disksize = attr->ia_size; 5162 rc = ext4_mark_inode_dirty(handle, inode); 5163 if (!error) 5164 error = rc; 5165 /* 5166 * We have to update i_size under i_data_sem together 5167 * with i_disksize to avoid races with writeback code 5168 * running ext4_wb_update_i_disksize(). 5169 */ 5170 if (!error) 5171 i_size_write(inode, attr->ia_size); 5172 up_write(&EXT4_I(inode)->i_data_sem); 5173 ext4_journal_stop(handle); 5174 if (error) { 5175 if (orphan) 5176 ext4_orphan_del(NULL, inode); 5177 goto err_out; 5178 } 5179 } 5180 if (!shrink) 5181 pagecache_isize_extended(inode, oldsize, inode->i_size); 5182 5183 /* 5184 * Blocks are going to be removed from the inode. Wait 5185 * for dio in flight. Temporarily disable 5186 * dioread_nolock to prevent livelock. 5187 */ 5188 if (orphan) { 5189 if (!ext4_should_journal_data(inode)) { 5190 ext4_inode_block_unlocked_dio(inode); 5191 inode_dio_wait(inode); 5192 ext4_inode_resume_unlocked_dio(inode); 5193 } else 5194 ext4_wait_for_tail_page_commit(inode); 5195 } 5196 down_write(&EXT4_I(inode)->i_mmap_sem); 5197 /* 5198 * Truncate pagecache after we've waited for commit 5199 * in data=journal mode to make pages freeable. 5200 */ 5201 truncate_pagecache(inode, inode->i_size); 5202 if (shrink) 5203 ext4_truncate(inode); 5204 up_write(&EXT4_I(inode)->i_mmap_sem); 5205 } 5206 5207 if (!rc) { 5208 setattr_copy(inode, attr); 5209 mark_inode_dirty(inode); 5210 } 5211 5212 /* 5213 * If the call to ext4_truncate failed to get a transaction handle at 5214 * all, we need to clean up the in-core orphan list manually. 5215 */ 5216 if (orphan && inode->i_nlink) 5217 ext4_orphan_del(NULL, inode); 5218 5219 if (!rc && (ia_valid & ATTR_MODE)) 5220 rc = posix_acl_chmod(inode, inode->i_mode); 5221 5222 err_out: 5223 ext4_std_error(inode->i_sb, error); 5224 if (!error) 5225 error = rc; 5226 return error; 5227 } 5228 5229 int ext4_getattr(struct vfsmount *mnt, struct dentry *dentry, 5230 struct kstat *stat) 5231 { 5232 struct inode *inode; 5233 unsigned long long delalloc_blocks; 5234 5235 inode = d_inode(dentry); 5236 generic_fillattr(inode, stat); 5237 5238 /* 5239 * If there is inline data in the inode, the inode will normally not 5240 * have data blocks allocated (it may have an external xattr block). 5241 * Report at least one sector for such files, so tools like tar, rsync, 5242 * others doen't incorrectly think the file is completely sparse. 5243 */ 5244 if (unlikely(ext4_has_inline_data(inode))) 5245 stat->blocks += (stat->size + 511) >> 9; 5246 5247 /* 5248 * We can't update i_blocks if the block allocation is delayed 5249 * otherwise in the case of system crash before the real block 5250 * allocation is done, we will have i_blocks inconsistent with 5251 * on-disk file blocks. 5252 * We always keep i_blocks updated together with real 5253 * allocation. But to not confuse with user, stat 5254 * will return the blocks that include the delayed allocation 5255 * blocks for this file. 5256 */ 5257 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb), 5258 EXT4_I(inode)->i_reserved_data_blocks); 5259 stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9); 5260 return 0; 5261 } 5262 5263 static int ext4_index_trans_blocks(struct inode *inode, int lblocks, 5264 int pextents) 5265 { 5266 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 5267 return ext4_ind_trans_blocks(inode, lblocks); 5268 return ext4_ext_index_trans_blocks(inode, pextents); 5269 } 5270 5271 /* 5272 * Account for index blocks, block groups bitmaps and block group 5273 * descriptor blocks if modify datablocks and index blocks 5274 * worse case, the indexs blocks spread over different block groups 5275 * 5276 * If datablocks are discontiguous, they are possible to spread over 5277 * different block groups too. If they are contiguous, with flexbg, 5278 * they could still across block group boundary. 5279 * 5280 * Also account for superblock, inode, quota and xattr blocks 5281 */ 5282 static int ext4_meta_trans_blocks(struct inode *inode, int lblocks, 5283 int pextents) 5284 { 5285 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); 5286 int gdpblocks; 5287 int idxblocks; 5288 int ret = 0; 5289 5290 /* 5291 * How many index blocks need to touch to map @lblocks logical blocks 5292 * to @pextents physical extents? 5293 */ 5294 idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents); 5295 5296 ret = idxblocks; 5297 5298 /* 5299 * Now let's see how many group bitmaps and group descriptors need 5300 * to account 5301 */ 5302 groups = idxblocks + pextents; 5303 gdpblocks = groups; 5304 if (groups > ngroups) 5305 groups = ngroups; 5306 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count) 5307 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count; 5308 5309 /* bitmaps and block group descriptor blocks */ 5310 ret += groups + gdpblocks; 5311 5312 /* Blocks for super block, inode, quota and xattr blocks */ 5313 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb); 5314 5315 return ret; 5316 } 5317 5318 /* 5319 * Calculate the total number of credits to reserve to fit 5320 * the modification of a single pages into a single transaction, 5321 * which may include multiple chunks of block allocations. 5322 * 5323 * This could be called via ext4_write_begin() 5324 * 5325 * We need to consider the worse case, when 5326 * one new block per extent. 5327 */ 5328 int ext4_writepage_trans_blocks(struct inode *inode) 5329 { 5330 int bpp = ext4_journal_blocks_per_page(inode); 5331 int ret; 5332 5333 ret = ext4_meta_trans_blocks(inode, bpp, bpp); 5334 5335 /* Account for data blocks for journalled mode */ 5336 if (ext4_should_journal_data(inode)) 5337 ret += bpp; 5338 return ret; 5339 } 5340 5341 /* 5342 * Calculate the journal credits for a chunk of data modification. 5343 * 5344 * This is called from DIO, fallocate or whoever calling 5345 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks. 5346 * 5347 * journal buffers for data blocks are not included here, as DIO 5348 * and fallocate do no need to journal data buffers. 5349 */ 5350 int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) 5351 { 5352 return ext4_meta_trans_blocks(inode, nrblocks, 1); 5353 } 5354 5355 /* 5356 * The caller must have previously called ext4_reserve_inode_write(). 5357 * Give this, we know that the caller already has write access to iloc->bh. 5358 */ 5359 int ext4_mark_iloc_dirty(handle_t *handle, 5360 struct inode *inode, struct ext4_iloc *iloc) 5361 { 5362 int err = 0; 5363 5364 if (IS_I_VERSION(inode)) 5365 inode_inc_iversion(inode); 5366 5367 /* the do_update_inode consumes one bh->b_count */ 5368 get_bh(iloc->bh); 5369 5370 /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */ 5371 err = ext4_do_update_inode(handle, inode, iloc); 5372 put_bh(iloc->bh); 5373 return err; 5374 } 5375 5376 /* 5377 * On success, We end up with an outstanding reference count against 5378 * iloc->bh. This _must_ be cleaned up later. 5379 */ 5380 5381 int 5382 ext4_reserve_inode_write(handle_t *handle, struct inode *inode, 5383 struct ext4_iloc *iloc) 5384 { 5385 int err; 5386 5387 err = ext4_get_inode_loc(inode, iloc); 5388 if (!err) { 5389 BUFFER_TRACE(iloc->bh, "get_write_access"); 5390 err = ext4_journal_get_write_access(handle, iloc->bh); 5391 if (err) { 5392 brelse(iloc->bh); 5393 iloc->bh = NULL; 5394 } 5395 } 5396 ext4_std_error(inode->i_sb, err); 5397 return err; 5398 } 5399 5400 /* 5401 * Expand an inode by new_extra_isize bytes. 5402 * Returns 0 on success or negative error number on failure. 5403 */ 5404 static int ext4_expand_extra_isize(struct inode *inode, 5405 unsigned int new_extra_isize, 5406 struct ext4_iloc iloc, 5407 handle_t *handle) 5408 { 5409 struct ext4_inode *raw_inode; 5410 struct ext4_xattr_ibody_header *header; 5411 5412 if (EXT4_I(inode)->i_extra_isize >= new_extra_isize) 5413 return 0; 5414 5415 raw_inode = ext4_raw_inode(&iloc); 5416 5417 header = IHDR(inode, raw_inode); 5418 5419 /* No extended attributes present */ 5420 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) || 5421 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) { 5422 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE, 0, 5423 new_extra_isize); 5424 EXT4_I(inode)->i_extra_isize = new_extra_isize; 5425 return 0; 5426 } 5427 5428 /* try to expand with EAs present */ 5429 return ext4_expand_extra_isize_ea(inode, new_extra_isize, 5430 raw_inode, handle); 5431 } 5432 5433 /* 5434 * What we do here is to mark the in-core inode as clean with respect to inode 5435 * dirtiness (it may still be data-dirty). 5436 * This means that the in-core inode may be reaped by prune_icache 5437 * without having to perform any I/O. This is a very good thing, 5438 * because *any* task may call prune_icache - even ones which 5439 * have a transaction open against a different journal. 5440 * 5441 * Is this cheating? Not really. Sure, we haven't written the 5442 * inode out, but prune_icache isn't a user-visible syncing function. 5443 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync) 5444 * we start and wait on commits. 5445 */ 5446 int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode) 5447 { 5448 struct ext4_iloc iloc; 5449 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 5450 static unsigned int mnt_count; 5451 int err, ret; 5452 5453 might_sleep(); 5454 trace_ext4_mark_inode_dirty(inode, _RET_IP_); 5455 err = ext4_reserve_inode_write(handle, inode, &iloc); 5456 if (err) 5457 return err; 5458 if (ext4_handle_valid(handle) && 5459 EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize && 5460 !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) { 5461 /* 5462 * We need extra buffer credits since we may write into EA block 5463 * with this same handle. If journal_extend fails, then it will 5464 * only result in a minor loss of functionality for that inode. 5465 * If this is felt to be critical, then e2fsck should be run to 5466 * force a large enough s_min_extra_isize. 5467 */ 5468 if ((jbd2_journal_extend(handle, 5469 EXT4_DATA_TRANS_BLOCKS(inode->i_sb))) == 0) { 5470 ret = ext4_expand_extra_isize(inode, 5471 sbi->s_want_extra_isize, 5472 iloc, handle); 5473 if (ret) { 5474 if (mnt_count != 5475 le16_to_cpu(sbi->s_es->s_mnt_count)) { 5476 ext4_warning(inode->i_sb, 5477 "Unable to expand inode %lu. Delete" 5478 " some EAs or run e2fsck.", 5479 inode->i_ino); 5480 mnt_count = 5481 le16_to_cpu(sbi->s_es->s_mnt_count); 5482 } 5483 } 5484 } 5485 } 5486 return ext4_mark_iloc_dirty(handle, inode, &iloc); 5487 } 5488 5489 /* 5490 * ext4_dirty_inode() is called from __mark_inode_dirty() 5491 * 5492 * We're really interested in the case where a file is being extended. 5493 * i_size has been changed by generic_commit_write() and we thus need 5494 * to include the updated inode in the current transaction. 5495 * 5496 * Also, dquot_alloc_block() will always dirty the inode when blocks 5497 * are allocated to the file. 5498 * 5499 * If the inode is marked synchronous, we don't honour that here - doing 5500 * so would cause a commit on atime updates, which we don't bother doing. 5501 * We handle synchronous inodes at the highest possible level. 5502 * 5503 * If only the I_DIRTY_TIME flag is set, we can skip everything. If 5504 * I_DIRTY_TIME and I_DIRTY_SYNC is set, the only inode fields we need 5505 * to copy into the on-disk inode structure are the timestamp files. 5506 */ 5507 void ext4_dirty_inode(struct inode *inode, int flags) 5508 { 5509 handle_t *handle; 5510 5511 if (flags == I_DIRTY_TIME) 5512 return; 5513 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 5514 if (IS_ERR(handle)) 5515 goto out; 5516 5517 ext4_mark_inode_dirty(handle, inode); 5518 5519 ext4_journal_stop(handle); 5520 out: 5521 return; 5522 } 5523 5524 #if 0 5525 /* 5526 * Bind an inode's backing buffer_head into this transaction, to prevent 5527 * it from being flushed to disk early. Unlike 5528 * ext4_reserve_inode_write, this leaves behind no bh reference and 5529 * returns no iloc structure, so the caller needs to repeat the iloc 5530 * lookup to mark the inode dirty later. 5531 */ 5532 static int ext4_pin_inode(handle_t *handle, struct inode *inode) 5533 { 5534 struct ext4_iloc iloc; 5535 5536 int err = 0; 5537 if (handle) { 5538 err = ext4_get_inode_loc(inode, &iloc); 5539 if (!err) { 5540 BUFFER_TRACE(iloc.bh, "get_write_access"); 5541 err = jbd2_journal_get_write_access(handle, iloc.bh); 5542 if (!err) 5543 err = ext4_handle_dirty_metadata(handle, 5544 NULL, 5545 iloc.bh); 5546 brelse(iloc.bh); 5547 } 5548 } 5549 ext4_std_error(inode->i_sb, err); 5550 return err; 5551 } 5552 #endif 5553 5554 int ext4_change_inode_journal_flag(struct inode *inode, int val) 5555 { 5556 journal_t *journal; 5557 handle_t *handle; 5558 int err; 5559 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 5560 5561 /* 5562 * We have to be very careful here: changing a data block's 5563 * journaling status dynamically is dangerous. If we write a 5564 * data block to the journal, change the status and then delete 5565 * that block, we risk forgetting to revoke the old log record 5566 * from the journal and so a subsequent replay can corrupt data. 5567 * So, first we make sure that the journal is empty and that 5568 * nobody is changing anything. 5569 */ 5570 5571 journal = EXT4_JOURNAL(inode); 5572 if (!journal) 5573 return 0; 5574 if (is_journal_aborted(journal)) 5575 return -EROFS; 5576 5577 /* Wait for all existing dio workers */ 5578 ext4_inode_block_unlocked_dio(inode); 5579 inode_dio_wait(inode); 5580 5581 /* 5582 * Before flushing the journal and switching inode's aops, we have 5583 * to flush all dirty data the inode has. There can be outstanding 5584 * delayed allocations, there can be unwritten extents created by 5585 * fallocate or buffered writes in dioread_nolock mode covered by 5586 * dirty data which can be converted only after flushing the dirty 5587 * data (and journalled aops don't know how to handle these cases). 5588 */ 5589 if (val) { 5590 down_write(&EXT4_I(inode)->i_mmap_sem); 5591 err = filemap_write_and_wait(inode->i_mapping); 5592 if (err < 0) { 5593 up_write(&EXT4_I(inode)->i_mmap_sem); 5594 ext4_inode_resume_unlocked_dio(inode); 5595 return err; 5596 } 5597 } 5598 5599 percpu_down_write(&sbi->s_journal_flag_rwsem); 5600 jbd2_journal_lock_updates(journal); 5601 5602 /* 5603 * OK, there are no updates running now, and all cached data is 5604 * synced to disk. We are now in a completely consistent state 5605 * which doesn't have anything in the journal, and we know that 5606 * no filesystem updates are running, so it is safe to modify 5607 * the inode's in-core data-journaling state flag now. 5608 */ 5609 5610 if (val) 5611 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 5612 else { 5613 err = jbd2_journal_flush(journal); 5614 if (err < 0) { 5615 jbd2_journal_unlock_updates(journal); 5616 percpu_up_write(&sbi->s_journal_flag_rwsem); 5617 ext4_inode_resume_unlocked_dio(inode); 5618 return err; 5619 } 5620 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 5621 } 5622 ext4_set_aops(inode); 5623 5624 jbd2_journal_unlock_updates(journal); 5625 percpu_up_write(&sbi->s_journal_flag_rwsem); 5626 5627 if (val) 5628 up_write(&EXT4_I(inode)->i_mmap_sem); 5629 ext4_inode_resume_unlocked_dio(inode); 5630 5631 /* Finally we can mark the inode as dirty. */ 5632 5633 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 5634 if (IS_ERR(handle)) 5635 return PTR_ERR(handle); 5636 5637 err = ext4_mark_inode_dirty(handle, inode); 5638 ext4_handle_sync(handle); 5639 ext4_journal_stop(handle); 5640 ext4_std_error(inode->i_sb, err); 5641 5642 return err; 5643 } 5644 5645 static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh) 5646 { 5647 return !buffer_mapped(bh); 5648 } 5649 5650 int ext4_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 5651 { 5652 struct page *page = vmf->page; 5653 loff_t size; 5654 unsigned long len; 5655 int ret; 5656 struct file *file = vma->vm_file; 5657 struct inode *inode = file_inode(file); 5658 struct address_space *mapping = inode->i_mapping; 5659 handle_t *handle; 5660 get_block_t *get_block; 5661 int retries = 0; 5662 5663 sb_start_pagefault(inode->i_sb); 5664 file_update_time(vma->vm_file); 5665 5666 down_read(&EXT4_I(inode)->i_mmap_sem); 5667 /* Delalloc case is easy... */ 5668 if (test_opt(inode->i_sb, DELALLOC) && 5669 !ext4_should_journal_data(inode) && 5670 !ext4_nonda_switch(inode->i_sb)) { 5671 do { 5672 ret = block_page_mkwrite(vma, vmf, 5673 ext4_da_get_block_prep); 5674 } while (ret == -ENOSPC && 5675 ext4_should_retry_alloc(inode->i_sb, &retries)); 5676 goto out_ret; 5677 } 5678 5679 lock_page(page); 5680 size = i_size_read(inode); 5681 /* Page got truncated from under us? */ 5682 if (page->mapping != mapping || page_offset(page) > size) { 5683 unlock_page(page); 5684 ret = VM_FAULT_NOPAGE; 5685 goto out; 5686 } 5687 5688 if (page->index == size >> PAGE_SHIFT) 5689 len = size & ~PAGE_MASK; 5690 else 5691 len = PAGE_SIZE; 5692 /* 5693 * Return if we have all the buffers mapped. This avoids the need to do 5694 * journal_start/journal_stop which can block and take a long time 5695 */ 5696 if (page_has_buffers(page)) { 5697 if (!ext4_walk_page_buffers(NULL, page_buffers(page), 5698 0, len, NULL, 5699 ext4_bh_unmapped)) { 5700 /* Wait so that we don't change page under IO */ 5701 wait_for_stable_page(page); 5702 ret = VM_FAULT_LOCKED; 5703 goto out; 5704 } 5705 } 5706 unlock_page(page); 5707 /* OK, we need to fill the hole... */ 5708 if (ext4_should_dioread_nolock(inode)) 5709 get_block = ext4_get_block_unwritten; 5710 else 5711 get_block = ext4_get_block; 5712 retry_alloc: 5713 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, 5714 ext4_writepage_trans_blocks(inode)); 5715 if (IS_ERR(handle)) { 5716 ret = VM_FAULT_SIGBUS; 5717 goto out; 5718 } 5719 ret = block_page_mkwrite(vma, vmf, get_block); 5720 if (!ret && ext4_should_journal_data(inode)) { 5721 if (ext4_walk_page_buffers(handle, page_buffers(page), 0, 5722 PAGE_SIZE, NULL, do_journal_get_write_access)) { 5723 unlock_page(page); 5724 ret = VM_FAULT_SIGBUS; 5725 ext4_journal_stop(handle); 5726 goto out; 5727 } 5728 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 5729 } 5730 ext4_journal_stop(handle); 5731 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 5732 goto retry_alloc; 5733 out_ret: 5734 ret = block_page_mkwrite_return(ret); 5735 out: 5736 up_read(&EXT4_I(inode)->i_mmap_sem); 5737 sb_end_pagefault(inode->i_sb); 5738 return ret; 5739 } 5740 5741 int ext4_filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 5742 { 5743 struct inode *inode = file_inode(vma->vm_file); 5744 int err; 5745 5746 down_read(&EXT4_I(inode)->i_mmap_sem); 5747 err = filemap_fault(vma, vmf); 5748 up_read(&EXT4_I(inode)->i_mmap_sem); 5749 5750 return err; 5751 } 5752 5753 /* 5754 * Find the first extent at or after @lblk in an inode that is not a hole. 5755 * Search for @map_len blocks at most. The extent is returned in @result. 5756 * 5757 * The function returns 1 if we found an extent. The function returns 0 in 5758 * case there is no extent at or after @lblk and in that case also sets 5759 * @result->es_len to 0. In case of error, the error code is returned. 5760 */ 5761 int ext4_get_next_extent(struct inode *inode, ext4_lblk_t lblk, 5762 unsigned int map_len, struct extent_status *result) 5763 { 5764 struct ext4_map_blocks map; 5765 struct extent_status es = {}; 5766 int ret; 5767 5768 map.m_lblk = lblk; 5769 map.m_len = map_len; 5770 5771 /* 5772 * For non-extent based files this loop may iterate several times since 5773 * we do not determine full hole size. 5774 */ 5775 while (map.m_len > 0) { 5776 ret = ext4_map_blocks(NULL, inode, &map, 0); 5777 if (ret < 0) 5778 return ret; 5779 /* There's extent covering m_lblk? Just return it. */ 5780 if (ret > 0) { 5781 int status; 5782 5783 ext4_es_store_pblock(result, map.m_pblk); 5784 result->es_lblk = map.m_lblk; 5785 result->es_len = map.m_len; 5786 if (map.m_flags & EXT4_MAP_UNWRITTEN) 5787 status = EXTENT_STATUS_UNWRITTEN; 5788 else 5789 status = EXTENT_STATUS_WRITTEN; 5790 ext4_es_store_status(result, status); 5791 return 1; 5792 } 5793 ext4_es_find_delayed_extent_range(inode, map.m_lblk, 5794 map.m_lblk + map.m_len - 1, 5795 &es); 5796 /* Is delalloc data before next block in extent tree? */ 5797 if (es.es_len && es.es_lblk < map.m_lblk + map.m_len) { 5798 ext4_lblk_t offset = 0; 5799 5800 if (es.es_lblk < lblk) 5801 offset = lblk - es.es_lblk; 5802 result->es_lblk = es.es_lblk + offset; 5803 ext4_es_store_pblock(result, 5804 ext4_es_pblock(&es) + offset); 5805 result->es_len = es.es_len - offset; 5806 ext4_es_store_status(result, ext4_es_status(&es)); 5807 5808 return 1; 5809 } 5810 /* There's a hole at m_lblk, advance us after it */ 5811 map.m_lblk += map.m_len; 5812 map_len -= map.m_len; 5813 map.m_len = map_len; 5814 cond_resched(); 5815 } 5816 result->es_len = 0; 5817 return 0; 5818 } 5819