1 /* 2 * linux/fs/ext4/inode.c 3 * 4 * Copyright (C) 1992, 1993, 1994, 1995 5 * Remy Card (card@masi.ibp.fr) 6 * Laboratoire MASI - Institut Blaise Pascal 7 * Universite Pierre et Marie Curie (Paris VI) 8 * 9 * from 10 * 11 * linux/fs/minix/inode.c 12 * 13 * Copyright (C) 1991, 1992 Linus Torvalds 14 * 15 * 64-bit file support on 64-bit platforms by Jakub Jelinek 16 * (jj@sunsite.ms.mff.cuni.cz) 17 * 18 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000 19 */ 20 21 #include <linux/fs.h> 22 #include <linux/time.h> 23 #include <linux/highuid.h> 24 #include <linux/pagemap.h> 25 #include <linux/dax.h> 26 #include <linux/quotaops.h> 27 #include <linux/string.h> 28 #include <linux/buffer_head.h> 29 #include <linux/writeback.h> 30 #include <linux/pagevec.h> 31 #include <linux/mpage.h> 32 #include <linux/namei.h> 33 #include <linux/uio.h> 34 #include <linux/bio.h> 35 #include <linux/workqueue.h> 36 #include <linux/kernel.h> 37 #include <linux/printk.h> 38 #include <linux/slab.h> 39 #include <linux/bitops.h> 40 #include <linux/iomap.h> 41 42 #include "ext4_jbd2.h" 43 #include "xattr.h" 44 #include "acl.h" 45 #include "truncate.h" 46 47 #include <trace/events/ext4.h> 48 49 #define MPAGE_DA_EXTENT_TAIL 0x01 50 51 static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw, 52 struct ext4_inode_info *ei) 53 { 54 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 55 __u32 csum; 56 __u16 dummy_csum = 0; 57 int offset = offsetof(struct ext4_inode, i_checksum_lo); 58 unsigned int csum_size = sizeof(dummy_csum); 59 60 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset); 61 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size); 62 offset += csum_size; 63 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset, 64 EXT4_GOOD_OLD_INODE_SIZE - offset); 65 66 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 67 offset = offsetof(struct ext4_inode, i_checksum_hi); 68 csum = ext4_chksum(sbi, csum, (__u8 *)raw + 69 EXT4_GOOD_OLD_INODE_SIZE, 70 offset - EXT4_GOOD_OLD_INODE_SIZE); 71 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) { 72 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, 73 csum_size); 74 offset += csum_size; 75 } 76 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset, 77 EXT4_INODE_SIZE(inode->i_sb) - offset); 78 } 79 80 return csum; 81 } 82 83 static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw, 84 struct ext4_inode_info *ei) 85 { 86 __u32 provided, calculated; 87 88 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 89 cpu_to_le32(EXT4_OS_LINUX) || 90 !ext4_has_metadata_csum(inode->i_sb)) 91 return 1; 92 93 provided = le16_to_cpu(raw->i_checksum_lo); 94 calculated = ext4_inode_csum(inode, raw, ei); 95 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 96 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 97 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16; 98 else 99 calculated &= 0xFFFF; 100 101 return provided == calculated; 102 } 103 104 static void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, 105 struct ext4_inode_info *ei) 106 { 107 __u32 csum; 108 109 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 110 cpu_to_le32(EXT4_OS_LINUX) || 111 !ext4_has_metadata_csum(inode->i_sb)) 112 return; 113 114 csum = ext4_inode_csum(inode, raw, ei); 115 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF); 116 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 117 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 118 raw->i_checksum_hi = cpu_to_le16(csum >> 16); 119 } 120 121 static inline int ext4_begin_ordered_truncate(struct inode *inode, 122 loff_t new_size) 123 { 124 trace_ext4_begin_ordered_truncate(inode, new_size); 125 /* 126 * If jinode is zero, then we never opened the file for 127 * writing, so there's no need to call 128 * jbd2_journal_begin_ordered_truncate() since there's no 129 * outstanding writes we need to flush. 130 */ 131 if (!EXT4_I(inode)->jinode) 132 return 0; 133 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), 134 EXT4_I(inode)->jinode, 135 new_size); 136 } 137 138 static void ext4_invalidatepage(struct page *page, unsigned int offset, 139 unsigned int length); 140 static int __ext4_journalled_writepage(struct page *page, unsigned int len); 141 static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh); 142 static int ext4_meta_trans_blocks(struct inode *inode, int lblocks, 143 int pextents); 144 145 /* 146 * Test whether an inode is a fast symlink. 147 */ 148 int ext4_inode_is_fast_symlink(struct inode *inode) 149 { 150 int ea_blocks = EXT4_I(inode)->i_file_acl ? 151 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0; 152 153 if (ext4_has_inline_data(inode)) 154 return 0; 155 156 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0); 157 } 158 159 /* 160 * Restart the transaction associated with *handle. This does a commit, 161 * so before we call here everything must be consistently dirtied against 162 * this transaction. 163 */ 164 int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode, 165 int nblocks) 166 { 167 int ret; 168 169 /* 170 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this 171 * moment, get_block can be called only for blocks inside i_size since 172 * page cache has been already dropped and writes are blocked by 173 * i_mutex. So we can safely drop the i_data_sem here. 174 */ 175 BUG_ON(EXT4_JOURNAL(inode) == NULL); 176 jbd_debug(2, "restarting handle %p\n", handle); 177 up_write(&EXT4_I(inode)->i_data_sem); 178 ret = ext4_journal_restart(handle, nblocks); 179 down_write(&EXT4_I(inode)->i_data_sem); 180 ext4_discard_preallocations(inode); 181 182 return ret; 183 } 184 185 /* 186 * Called at the last iput() if i_nlink is zero. 187 */ 188 void ext4_evict_inode(struct inode *inode) 189 { 190 handle_t *handle; 191 int err; 192 193 trace_ext4_evict_inode(inode); 194 195 if (inode->i_nlink) { 196 /* 197 * When journalling data dirty buffers are tracked only in the 198 * journal. So although mm thinks everything is clean and 199 * ready for reaping the inode might still have some pages to 200 * write in the running transaction or waiting to be 201 * checkpointed. Thus calling jbd2_journal_invalidatepage() 202 * (via truncate_inode_pages()) to discard these buffers can 203 * cause data loss. Also even if we did not discard these 204 * buffers, we would have no way to find them after the inode 205 * is reaped and thus user could see stale data if he tries to 206 * read them before the transaction is checkpointed. So be 207 * careful and force everything to disk here... We use 208 * ei->i_datasync_tid to store the newest transaction 209 * containing inode's data. 210 * 211 * Note that directories do not have this problem because they 212 * don't use page cache. 213 */ 214 if (inode->i_ino != EXT4_JOURNAL_INO && 215 ext4_should_journal_data(inode) && 216 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode))) { 217 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 218 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid; 219 220 jbd2_complete_transaction(journal, commit_tid); 221 filemap_write_and_wait(&inode->i_data); 222 } 223 truncate_inode_pages_final(&inode->i_data); 224 225 goto no_delete; 226 } 227 228 if (is_bad_inode(inode)) 229 goto no_delete; 230 dquot_initialize(inode); 231 232 if (ext4_should_order_data(inode)) 233 ext4_begin_ordered_truncate(inode, 0); 234 truncate_inode_pages_final(&inode->i_data); 235 236 /* 237 * Protect us against freezing - iput() caller didn't have to have any 238 * protection against it 239 */ 240 sb_start_intwrite(inode->i_sb); 241 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, 242 ext4_blocks_for_truncate(inode)+3); 243 if (IS_ERR(handle)) { 244 ext4_std_error(inode->i_sb, PTR_ERR(handle)); 245 /* 246 * If we're going to skip the normal cleanup, we still need to 247 * make sure that the in-core orphan linked list is properly 248 * cleaned up. 249 */ 250 ext4_orphan_del(NULL, inode); 251 sb_end_intwrite(inode->i_sb); 252 goto no_delete; 253 } 254 255 if (IS_SYNC(inode)) 256 ext4_handle_sync(handle); 257 inode->i_size = 0; 258 err = ext4_mark_inode_dirty(handle, inode); 259 if (err) { 260 ext4_warning(inode->i_sb, 261 "couldn't mark inode dirty (err %d)", err); 262 goto stop_handle; 263 } 264 if (inode->i_blocks) { 265 err = ext4_truncate(inode); 266 if (err) { 267 ext4_error(inode->i_sb, 268 "couldn't truncate inode %lu (err %d)", 269 inode->i_ino, err); 270 goto stop_handle; 271 } 272 } 273 274 /* 275 * ext4_ext_truncate() doesn't reserve any slop when it 276 * restarts journal transactions; therefore there may not be 277 * enough credits left in the handle to remove the inode from 278 * the orphan list and set the dtime field. 279 */ 280 if (!ext4_handle_has_enough_credits(handle, 3)) { 281 err = ext4_journal_extend(handle, 3); 282 if (err > 0) 283 err = ext4_journal_restart(handle, 3); 284 if (err != 0) { 285 ext4_warning(inode->i_sb, 286 "couldn't extend journal (err %d)", err); 287 stop_handle: 288 ext4_journal_stop(handle); 289 ext4_orphan_del(NULL, inode); 290 sb_end_intwrite(inode->i_sb); 291 goto no_delete; 292 } 293 } 294 295 /* 296 * Kill off the orphan record which ext4_truncate created. 297 * AKPM: I think this can be inside the above `if'. 298 * Note that ext4_orphan_del() has to be able to cope with the 299 * deletion of a non-existent orphan - this is because we don't 300 * know if ext4_truncate() actually created an orphan record. 301 * (Well, we could do this if we need to, but heck - it works) 302 */ 303 ext4_orphan_del(handle, inode); 304 EXT4_I(inode)->i_dtime = get_seconds(); 305 306 /* 307 * One subtle ordering requirement: if anything has gone wrong 308 * (transaction abort, IO errors, whatever), then we can still 309 * do these next steps (the fs will already have been marked as 310 * having errors), but we can't free the inode if the mark_dirty 311 * fails. 312 */ 313 if (ext4_mark_inode_dirty(handle, inode)) 314 /* If that failed, just do the required in-core inode clear. */ 315 ext4_clear_inode(inode); 316 else 317 ext4_free_inode(handle, inode); 318 ext4_journal_stop(handle); 319 sb_end_intwrite(inode->i_sb); 320 return; 321 no_delete: 322 ext4_clear_inode(inode); /* We must guarantee clearing of inode... */ 323 } 324 325 #ifdef CONFIG_QUOTA 326 qsize_t *ext4_get_reserved_space(struct inode *inode) 327 { 328 return &EXT4_I(inode)->i_reserved_quota; 329 } 330 #endif 331 332 /* 333 * Called with i_data_sem down, which is important since we can call 334 * ext4_discard_preallocations() from here. 335 */ 336 void ext4_da_update_reserve_space(struct inode *inode, 337 int used, int quota_claim) 338 { 339 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 340 struct ext4_inode_info *ei = EXT4_I(inode); 341 342 spin_lock(&ei->i_block_reservation_lock); 343 trace_ext4_da_update_reserve_space(inode, used, quota_claim); 344 if (unlikely(used > ei->i_reserved_data_blocks)) { 345 ext4_warning(inode->i_sb, "%s: ino %lu, used %d " 346 "with only %d reserved data blocks", 347 __func__, inode->i_ino, used, 348 ei->i_reserved_data_blocks); 349 WARN_ON(1); 350 used = ei->i_reserved_data_blocks; 351 } 352 353 /* Update per-inode reservations */ 354 ei->i_reserved_data_blocks -= used; 355 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used); 356 357 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 358 359 /* Update quota subsystem for data blocks */ 360 if (quota_claim) 361 dquot_claim_block(inode, EXT4_C2B(sbi, used)); 362 else { 363 /* 364 * We did fallocate with an offset that is already delayed 365 * allocated. So on delayed allocated writeback we should 366 * not re-claim the quota for fallocated blocks. 367 */ 368 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used)); 369 } 370 371 /* 372 * If we have done all the pending block allocations and if 373 * there aren't any writers on the inode, we can discard the 374 * inode's preallocations. 375 */ 376 if ((ei->i_reserved_data_blocks == 0) && 377 (atomic_read(&inode->i_writecount) == 0)) 378 ext4_discard_preallocations(inode); 379 } 380 381 static int __check_block_validity(struct inode *inode, const char *func, 382 unsigned int line, 383 struct ext4_map_blocks *map) 384 { 385 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), map->m_pblk, 386 map->m_len)) { 387 ext4_error_inode(inode, func, line, map->m_pblk, 388 "lblock %lu mapped to illegal pblock " 389 "(length %d)", (unsigned long) map->m_lblk, 390 map->m_len); 391 return -EFSCORRUPTED; 392 } 393 return 0; 394 } 395 396 int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, 397 ext4_lblk_t len) 398 { 399 int ret; 400 401 if (ext4_encrypted_inode(inode)) 402 return fscrypt_zeroout_range(inode, lblk, pblk, len); 403 404 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS); 405 if (ret > 0) 406 ret = 0; 407 408 return ret; 409 } 410 411 #define check_block_validity(inode, map) \ 412 __check_block_validity((inode), __func__, __LINE__, (map)) 413 414 #ifdef ES_AGGRESSIVE_TEST 415 static void ext4_map_blocks_es_recheck(handle_t *handle, 416 struct inode *inode, 417 struct ext4_map_blocks *es_map, 418 struct ext4_map_blocks *map, 419 int flags) 420 { 421 int retval; 422 423 map->m_flags = 0; 424 /* 425 * There is a race window that the result is not the same. 426 * e.g. xfstests #223 when dioread_nolock enables. The reason 427 * is that we lookup a block mapping in extent status tree with 428 * out taking i_data_sem. So at the time the unwritten extent 429 * could be converted. 430 */ 431 down_read(&EXT4_I(inode)->i_data_sem); 432 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 433 retval = ext4_ext_map_blocks(handle, inode, map, flags & 434 EXT4_GET_BLOCKS_KEEP_SIZE); 435 } else { 436 retval = ext4_ind_map_blocks(handle, inode, map, flags & 437 EXT4_GET_BLOCKS_KEEP_SIZE); 438 } 439 up_read((&EXT4_I(inode)->i_data_sem)); 440 441 /* 442 * We don't check m_len because extent will be collpased in status 443 * tree. So the m_len might not equal. 444 */ 445 if (es_map->m_lblk != map->m_lblk || 446 es_map->m_flags != map->m_flags || 447 es_map->m_pblk != map->m_pblk) { 448 printk("ES cache assertion failed for inode: %lu " 449 "es_cached ex [%d/%d/%llu/%x] != " 450 "found ex [%d/%d/%llu/%x] retval %d flags %x\n", 451 inode->i_ino, es_map->m_lblk, es_map->m_len, 452 es_map->m_pblk, es_map->m_flags, map->m_lblk, 453 map->m_len, map->m_pblk, map->m_flags, 454 retval, flags); 455 } 456 } 457 #endif /* ES_AGGRESSIVE_TEST */ 458 459 /* 460 * The ext4_map_blocks() function tries to look up the requested blocks, 461 * and returns if the blocks are already mapped. 462 * 463 * Otherwise it takes the write lock of the i_data_sem and allocate blocks 464 * and store the allocated blocks in the result buffer head and mark it 465 * mapped. 466 * 467 * If file type is extents based, it will call ext4_ext_map_blocks(), 468 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping 469 * based files 470 * 471 * On success, it returns the number of blocks being mapped or allocated. if 472 * create==0 and the blocks are pre-allocated and unwritten, the resulting @map 473 * is marked as unwritten. If the create == 1, it will mark @map as mapped. 474 * 475 * It returns 0 if plain look up failed (blocks have not been allocated), in 476 * that case, @map is returned as unmapped but we still do fill map->m_len to 477 * indicate the length of a hole starting at map->m_lblk. 478 * 479 * It returns the error in case of allocation failure. 480 */ 481 int ext4_map_blocks(handle_t *handle, struct inode *inode, 482 struct ext4_map_blocks *map, int flags) 483 { 484 struct extent_status es; 485 int retval; 486 int ret = 0; 487 #ifdef ES_AGGRESSIVE_TEST 488 struct ext4_map_blocks orig_map; 489 490 memcpy(&orig_map, map, sizeof(*map)); 491 #endif 492 493 map->m_flags = 0; 494 ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u," 495 "logical block %lu\n", inode->i_ino, flags, map->m_len, 496 (unsigned long) map->m_lblk); 497 498 /* 499 * ext4_map_blocks returns an int, and m_len is an unsigned int 500 */ 501 if (unlikely(map->m_len > INT_MAX)) 502 map->m_len = INT_MAX; 503 504 /* We can handle the block number less than EXT_MAX_BLOCKS */ 505 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS)) 506 return -EFSCORRUPTED; 507 508 /* Lookup extent status tree firstly */ 509 if (ext4_es_lookup_extent(inode, map->m_lblk, &es)) { 510 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) { 511 map->m_pblk = ext4_es_pblock(&es) + 512 map->m_lblk - es.es_lblk; 513 map->m_flags |= ext4_es_is_written(&es) ? 514 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN; 515 retval = es.es_len - (map->m_lblk - es.es_lblk); 516 if (retval > map->m_len) 517 retval = map->m_len; 518 map->m_len = retval; 519 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) { 520 map->m_pblk = 0; 521 retval = es.es_len - (map->m_lblk - es.es_lblk); 522 if (retval > map->m_len) 523 retval = map->m_len; 524 map->m_len = retval; 525 retval = 0; 526 } else { 527 BUG_ON(1); 528 } 529 #ifdef ES_AGGRESSIVE_TEST 530 ext4_map_blocks_es_recheck(handle, inode, map, 531 &orig_map, flags); 532 #endif 533 goto found; 534 } 535 536 /* 537 * Try to see if we can get the block without requesting a new 538 * file system block. 539 */ 540 down_read(&EXT4_I(inode)->i_data_sem); 541 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 542 retval = ext4_ext_map_blocks(handle, inode, map, flags & 543 EXT4_GET_BLOCKS_KEEP_SIZE); 544 } else { 545 retval = ext4_ind_map_blocks(handle, inode, map, flags & 546 EXT4_GET_BLOCKS_KEEP_SIZE); 547 } 548 if (retval > 0) { 549 unsigned int status; 550 551 if (unlikely(retval != map->m_len)) { 552 ext4_warning(inode->i_sb, 553 "ES len assertion failed for inode " 554 "%lu: retval %d != map->m_len %d", 555 inode->i_ino, retval, map->m_len); 556 WARN_ON(1); 557 } 558 559 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 560 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 561 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) && 562 !(status & EXTENT_STATUS_WRITTEN) && 563 ext4_find_delalloc_range(inode, map->m_lblk, 564 map->m_lblk + map->m_len - 1)) 565 status |= EXTENT_STATUS_DELAYED; 566 ret = ext4_es_insert_extent(inode, map->m_lblk, 567 map->m_len, map->m_pblk, status); 568 if (ret < 0) 569 retval = ret; 570 } 571 up_read((&EXT4_I(inode)->i_data_sem)); 572 573 found: 574 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 575 ret = check_block_validity(inode, map); 576 if (ret != 0) 577 return ret; 578 } 579 580 /* If it is only a block(s) look up */ 581 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) 582 return retval; 583 584 /* 585 * Returns if the blocks have already allocated 586 * 587 * Note that if blocks have been preallocated 588 * ext4_ext_get_block() returns the create = 0 589 * with buffer head unmapped. 590 */ 591 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) 592 /* 593 * If we need to convert extent to unwritten 594 * we continue and do the actual work in 595 * ext4_ext_map_blocks() 596 */ 597 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) 598 return retval; 599 600 /* 601 * Here we clear m_flags because after allocating an new extent, 602 * it will be set again. 603 */ 604 map->m_flags &= ~EXT4_MAP_FLAGS; 605 606 /* 607 * New blocks allocate and/or writing to unwritten extent 608 * will possibly result in updating i_data, so we take 609 * the write lock of i_data_sem, and call get_block() 610 * with create == 1 flag. 611 */ 612 down_write(&EXT4_I(inode)->i_data_sem); 613 614 /* 615 * We need to check for EXT4 here because migrate 616 * could have changed the inode type in between 617 */ 618 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 619 retval = ext4_ext_map_blocks(handle, inode, map, flags); 620 } else { 621 retval = ext4_ind_map_blocks(handle, inode, map, flags); 622 623 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) { 624 /* 625 * We allocated new blocks which will result in 626 * i_data's format changing. Force the migrate 627 * to fail by clearing migrate flags 628 */ 629 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE); 630 } 631 632 /* 633 * Update reserved blocks/metadata blocks after successful 634 * block allocation which had been deferred till now. We don't 635 * support fallocate for non extent files. So we can update 636 * reserve space here. 637 */ 638 if ((retval > 0) && 639 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)) 640 ext4_da_update_reserve_space(inode, retval, 1); 641 } 642 643 if (retval > 0) { 644 unsigned int status; 645 646 if (unlikely(retval != map->m_len)) { 647 ext4_warning(inode->i_sb, 648 "ES len assertion failed for inode " 649 "%lu: retval %d != map->m_len %d", 650 inode->i_ino, retval, map->m_len); 651 WARN_ON(1); 652 } 653 654 /* 655 * We have to zeroout blocks before inserting them into extent 656 * status tree. Otherwise someone could look them up there and 657 * use them before they are really zeroed. We also have to 658 * unmap metadata before zeroing as otherwise writeback can 659 * overwrite zeros with stale data from block device. 660 */ 661 if (flags & EXT4_GET_BLOCKS_ZERO && 662 map->m_flags & EXT4_MAP_MAPPED && 663 map->m_flags & EXT4_MAP_NEW) { 664 clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk, 665 map->m_len); 666 ret = ext4_issue_zeroout(inode, map->m_lblk, 667 map->m_pblk, map->m_len); 668 if (ret) { 669 retval = ret; 670 goto out_sem; 671 } 672 } 673 674 /* 675 * If the extent has been zeroed out, we don't need to update 676 * extent status tree. 677 */ 678 if ((flags & EXT4_GET_BLOCKS_PRE_IO) && 679 ext4_es_lookup_extent(inode, map->m_lblk, &es)) { 680 if (ext4_es_is_written(&es)) 681 goto out_sem; 682 } 683 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 684 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 685 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) && 686 !(status & EXTENT_STATUS_WRITTEN) && 687 ext4_find_delalloc_range(inode, map->m_lblk, 688 map->m_lblk + map->m_len - 1)) 689 status |= EXTENT_STATUS_DELAYED; 690 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len, 691 map->m_pblk, status); 692 if (ret < 0) { 693 retval = ret; 694 goto out_sem; 695 } 696 } 697 698 out_sem: 699 up_write((&EXT4_I(inode)->i_data_sem)); 700 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 701 ret = check_block_validity(inode, map); 702 if (ret != 0) 703 return ret; 704 705 /* 706 * Inodes with freshly allocated blocks where contents will be 707 * visible after transaction commit must be on transaction's 708 * ordered data list. 709 */ 710 if (map->m_flags & EXT4_MAP_NEW && 711 !(map->m_flags & EXT4_MAP_UNWRITTEN) && 712 !(flags & EXT4_GET_BLOCKS_ZERO) && 713 !IS_NOQUOTA(inode) && 714 ext4_should_order_data(inode)) { 715 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT) 716 ret = ext4_jbd2_inode_add_wait(handle, inode); 717 else 718 ret = ext4_jbd2_inode_add_write(handle, inode); 719 if (ret) 720 return ret; 721 } 722 } 723 return retval; 724 } 725 726 /* 727 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages 728 * we have to be careful as someone else may be manipulating b_state as well. 729 */ 730 static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags) 731 { 732 unsigned long old_state; 733 unsigned long new_state; 734 735 flags &= EXT4_MAP_FLAGS; 736 737 /* Dummy buffer_head? Set non-atomically. */ 738 if (!bh->b_page) { 739 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags; 740 return; 741 } 742 /* 743 * Someone else may be modifying b_state. Be careful! This is ugly but 744 * once we get rid of using bh as a container for mapping information 745 * to pass to / from get_block functions, this can go away. 746 */ 747 do { 748 old_state = READ_ONCE(bh->b_state); 749 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags; 750 } while (unlikely( 751 cmpxchg(&bh->b_state, old_state, new_state) != old_state)); 752 } 753 754 static int _ext4_get_block(struct inode *inode, sector_t iblock, 755 struct buffer_head *bh, int flags) 756 { 757 struct ext4_map_blocks map; 758 int ret = 0; 759 760 if (ext4_has_inline_data(inode)) 761 return -ERANGE; 762 763 map.m_lblk = iblock; 764 map.m_len = bh->b_size >> inode->i_blkbits; 765 766 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map, 767 flags); 768 if (ret > 0) { 769 map_bh(bh, inode->i_sb, map.m_pblk); 770 ext4_update_bh_state(bh, map.m_flags); 771 bh->b_size = inode->i_sb->s_blocksize * map.m_len; 772 ret = 0; 773 } else if (ret == 0) { 774 /* hole case, need to fill in bh->b_size */ 775 bh->b_size = inode->i_sb->s_blocksize * map.m_len; 776 } 777 return ret; 778 } 779 780 int ext4_get_block(struct inode *inode, sector_t iblock, 781 struct buffer_head *bh, int create) 782 { 783 return _ext4_get_block(inode, iblock, bh, 784 create ? EXT4_GET_BLOCKS_CREATE : 0); 785 } 786 787 /* 788 * Get block function used when preparing for buffered write if we require 789 * creating an unwritten extent if blocks haven't been allocated. The extent 790 * will be converted to written after the IO is complete. 791 */ 792 int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, 793 struct buffer_head *bh_result, int create) 794 { 795 ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n", 796 inode->i_ino, create); 797 return _ext4_get_block(inode, iblock, bh_result, 798 EXT4_GET_BLOCKS_IO_CREATE_EXT); 799 } 800 801 /* Maximum number of blocks we map for direct IO at once. */ 802 #define DIO_MAX_BLOCKS 4096 803 804 /* 805 * Get blocks function for the cases that need to start a transaction - 806 * generally difference cases of direct IO and DAX IO. It also handles retries 807 * in case of ENOSPC. 808 */ 809 static int ext4_get_block_trans(struct inode *inode, sector_t iblock, 810 struct buffer_head *bh_result, int flags) 811 { 812 int dio_credits; 813 handle_t *handle; 814 int retries = 0; 815 int ret; 816 817 /* Trim mapping request to maximum we can map at once for DIO */ 818 if (bh_result->b_size >> inode->i_blkbits > DIO_MAX_BLOCKS) 819 bh_result->b_size = DIO_MAX_BLOCKS << inode->i_blkbits; 820 dio_credits = ext4_chunk_trans_blocks(inode, 821 bh_result->b_size >> inode->i_blkbits); 822 retry: 823 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits); 824 if (IS_ERR(handle)) 825 return PTR_ERR(handle); 826 827 ret = _ext4_get_block(inode, iblock, bh_result, flags); 828 ext4_journal_stop(handle); 829 830 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 831 goto retry; 832 return ret; 833 } 834 835 /* Get block function for DIO reads and writes to inodes without extents */ 836 int ext4_dio_get_block(struct inode *inode, sector_t iblock, 837 struct buffer_head *bh, int create) 838 { 839 /* We don't expect handle for direct IO */ 840 WARN_ON_ONCE(ext4_journal_current_handle()); 841 842 if (!create) 843 return _ext4_get_block(inode, iblock, bh, 0); 844 return ext4_get_block_trans(inode, iblock, bh, EXT4_GET_BLOCKS_CREATE); 845 } 846 847 /* 848 * Get block function for AIO DIO writes when we create unwritten extent if 849 * blocks are not allocated yet. The extent will be converted to written 850 * after IO is complete. 851 */ 852 static int ext4_dio_get_block_unwritten_async(struct inode *inode, 853 sector_t iblock, struct buffer_head *bh_result, int create) 854 { 855 int ret; 856 857 /* We don't expect handle for direct IO */ 858 WARN_ON_ONCE(ext4_journal_current_handle()); 859 860 ret = ext4_get_block_trans(inode, iblock, bh_result, 861 EXT4_GET_BLOCKS_IO_CREATE_EXT); 862 863 /* 864 * When doing DIO using unwritten extents, we need io_end to convert 865 * unwritten extents to written on IO completion. We allocate io_end 866 * once we spot unwritten extent and store it in b_private. Generic 867 * DIO code keeps b_private set and furthermore passes the value to 868 * our completion callback in 'private' argument. 869 */ 870 if (!ret && buffer_unwritten(bh_result)) { 871 if (!bh_result->b_private) { 872 ext4_io_end_t *io_end; 873 874 io_end = ext4_init_io_end(inode, GFP_KERNEL); 875 if (!io_end) 876 return -ENOMEM; 877 bh_result->b_private = io_end; 878 ext4_set_io_unwritten_flag(inode, io_end); 879 } 880 set_buffer_defer_completion(bh_result); 881 } 882 883 return ret; 884 } 885 886 /* 887 * Get block function for non-AIO DIO writes when we create unwritten extent if 888 * blocks are not allocated yet. The extent will be converted to written 889 * after IO is complete from ext4_ext_direct_IO() function. 890 */ 891 static int ext4_dio_get_block_unwritten_sync(struct inode *inode, 892 sector_t iblock, struct buffer_head *bh_result, int create) 893 { 894 int ret; 895 896 /* We don't expect handle for direct IO */ 897 WARN_ON_ONCE(ext4_journal_current_handle()); 898 899 ret = ext4_get_block_trans(inode, iblock, bh_result, 900 EXT4_GET_BLOCKS_IO_CREATE_EXT); 901 902 /* 903 * Mark inode as having pending DIO writes to unwritten extents. 904 * ext4_ext_direct_IO() checks this flag and converts extents to 905 * written. 906 */ 907 if (!ret && buffer_unwritten(bh_result)) 908 ext4_set_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN); 909 910 return ret; 911 } 912 913 static int ext4_dio_get_block_overwrite(struct inode *inode, sector_t iblock, 914 struct buffer_head *bh_result, int create) 915 { 916 int ret; 917 918 ext4_debug("ext4_dio_get_block_overwrite: inode %lu, create flag %d\n", 919 inode->i_ino, create); 920 /* We don't expect handle for direct IO */ 921 WARN_ON_ONCE(ext4_journal_current_handle()); 922 923 ret = _ext4_get_block(inode, iblock, bh_result, 0); 924 /* 925 * Blocks should have been preallocated! ext4_file_write_iter() checks 926 * that. 927 */ 928 WARN_ON_ONCE(!buffer_mapped(bh_result) || buffer_unwritten(bh_result)); 929 930 return ret; 931 } 932 933 934 /* 935 * `handle' can be NULL if create is zero 936 */ 937 struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode, 938 ext4_lblk_t block, int map_flags) 939 { 940 struct ext4_map_blocks map; 941 struct buffer_head *bh; 942 int create = map_flags & EXT4_GET_BLOCKS_CREATE; 943 int err; 944 945 J_ASSERT(handle != NULL || create == 0); 946 947 map.m_lblk = block; 948 map.m_len = 1; 949 err = ext4_map_blocks(handle, inode, &map, map_flags); 950 951 if (err == 0) 952 return create ? ERR_PTR(-ENOSPC) : NULL; 953 if (err < 0) 954 return ERR_PTR(err); 955 956 bh = sb_getblk(inode->i_sb, map.m_pblk); 957 if (unlikely(!bh)) 958 return ERR_PTR(-ENOMEM); 959 if (map.m_flags & EXT4_MAP_NEW) { 960 J_ASSERT(create != 0); 961 J_ASSERT(handle != NULL); 962 963 /* 964 * Now that we do not always journal data, we should 965 * keep in mind whether this should always journal the 966 * new buffer as metadata. For now, regular file 967 * writes use ext4_get_block instead, so it's not a 968 * problem. 969 */ 970 lock_buffer(bh); 971 BUFFER_TRACE(bh, "call get_create_access"); 972 err = ext4_journal_get_create_access(handle, bh); 973 if (unlikely(err)) { 974 unlock_buffer(bh); 975 goto errout; 976 } 977 if (!buffer_uptodate(bh)) { 978 memset(bh->b_data, 0, inode->i_sb->s_blocksize); 979 set_buffer_uptodate(bh); 980 } 981 unlock_buffer(bh); 982 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 983 err = ext4_handle_dirty_metadata(handle, inode, bh); 984 if (unlikely(err)) 985 goto errout; 986 } else 987 BUFFER_TRACE(bh, "not a new buffer"); 988 return bh; 989 errout: 990 brelse(bh); 991 return ERR_PTR(err); 992 } 993 994 struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode, 995 ext4_lblk_t block, int map_flags) 996 { 997 struct buffer_head *bh; 998 999 bh = ext4_getblk(handle, inode, block, map_flags); 1000 if (IS_ERR(bh)) 1001 return bh; 1002 if (!bh || buffer_uptodate(bh)) 1003 return bh; 1004 ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh); 1005 wait_on_buffer(bh); 1006 if (buffer_uptodate(bh)) 1007 return bh; 1008 put_bh(bh); 1009 return ERR_PTR(-EIO); 1010 } 1011 1012 int ext4_walk_page_buffers(handle_t *handle, 1013 struct buffer_head *head, 1014 unsigned from, 1015 unsigned to, 1016 int *partial, 1017 int (*fn)(handle_t *handle, 1018 struct buffer_head *bh)) 1019 { 1020 struct buffer_head *bh; 1021 unsigned block_start, block_end; 1022 unsigned blocksize = head->b_size; 1023 int err, ret = 0; 1024 struct buffer_head *next; 1025 1026 for (bh = head, block_start = 0; 1027 ret == 0 && (bh != head || !block_start); 1028 block_start = block_end, bh = next) { 1029 next = bh->b_this_page; 1030 block_end = block_start + blocksize; 1031 if (block_end <= from || block_start >= to) { 1032 if (partial && !buffer_uptodate(bh)) 1033 *partial = 1; 1034 continue; 1035 } 1036 err = (*fn)(handle, bh); 1037 if (!ret) 1038 ret = err; 1039 } 1040 return ret; 1041 } 1042 1043 /* 1044 * To preserve ordering, it is essential that the hole instantiation and 1045 * the data write be encapsulated in a single transaction. We cannot 1046 * close off a transaction and start a new one between the ext4_get_block() 1047 * and the commit_write(). So doing the jbd2_journal_start at the start of 1048 * prepare_write() is the right place. 1049 * 1050 * Also, this function can nest inside ext4_writepage(). In that case, we 1051 * *know* that ext4_writepage() has generated enough buffer credits to do the 1052 * whole page. So we won't block on the journal in that case, which is good, 1053 * because the caller may be PF_MEMALLOC. 1054 * 1055 * By accident, ext4 can be reentered when a transaction is open via 1056 * quota file writes. If we were to commit the transaction while thus 1057 * reentered, there can be a deadlock - we would be holding a quota 1058 * lock, and the commit would never complete if another thread had a 1059 * transaction open and was blocking on the quota lock - a ranking 1060 * violation. 1061 * 1062 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start 1063 * will _not_ run commit under these circumstances because handle->h_ref 1064 * is elevated. We'll still have enough credits for the tiny quotafile 1065 * write. 1066 */ 1067 int do_journal_get_write_access(handle_t *handle, 1068 struct buffer_head *bh) 1069 { 1070 int dirty = buffer_dirty(bh); 1071 int ret; 1072 1073 if (!buffer_mapped(bh) || buffer_freed(bh)) 1074 return 0; 1075 /* 1076 * __block_write_begin() could have dirtied some buffers. Clean 1077 * the dirty bit as jbd2_journal_get_write_access() could complain 1078 * otherwise about fs integrity issues. Setting of the dirty bit 1079 * by __block_write_begin() isn't a real problem here as we clear 1080 * the bit before releasing a page lock and thus writeback cannot 1081 * ever write the buffer. 1082 */ 1083 if (dirty) 1084 clear_buffer_dirty(bh); 1085 BUFFER_TRACE(bh, "get write access"); 1086 ret = ext4_journal_get_write_access(handle, bh); 1087 if (!ret && dirty) 1088 ret = ext4_handle_dirty_metadata(handle, NULL, bh); 1089 return ret; 1090 } 1091 1092 #ifdef CONFIG_EXT4_FS_ENCRYPTION 1093 static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, 1094 get_block_t *get_block) 1095 { 1096 unsigned from = pos & (PAGE_SIZE - 1); 1097 unsigned to = from + len; 1098 struct inode *inode = page->mapping->host; 1099 unsigned block_start, block_end; 1100 sector_t block; 1101 int err = 0; 1102 unsigned blocksize = inode->i_sb->s_blocksize; 1103 unsigned bbits; 1104 struct buffer_head *bh, *head, *wait[2], **wait_bh = wait; 1105 bool decrypt = false; 1106 1107 BUG_ON(!PageLocked(page)); 1108 BUG_ON(from > PAGE_SIZE); 1109 BUG_ON(to > PAGE_SIZE); 1110 BUG_ON(from > to); 1111 1112 if (!page_has_buffers(page)) 1113 create_empty_buffers(page, blocksize, 0); 1114 head = page_buffers(page); 1115 bbits = ilog2(blocksize); 1116 block = (sector_t)page->index << (PAGE_SHIFT - bbits); 1117 1118 for (bh = head, block_start = 0; bh != head || !block_start; 1119 block++, block_start = block_end, bh = bh->b_this_page) { 1120 block_end = block_start + blocksize; 1121 if (block_end <= from || block_start >= to) { 1122 if (PageUptodate(page)) { 1123 if (!buffer_uptodate(bh)) 1124 set_buffer_uptodate(bh); 1125 } 1126 continue; 1127 } 1128 if (buffer_new(bh)) 1129 clear_buffer_new(bh); 1130 if (!buffer_mapped(bh)) { 1131 WARN_ON(bh->b_size != blocksize); 1132 err = get_block(inode, block, bh, 1); 1133 if (err) 1134 break; 1135 if (buffer_new(bh)) { 1136 clean_bdev_bh_alias(bh); 1137 if (PageUptodate(page)) { 1138 clear_buffer_new(bh); 1139 set_buffer_uptodate(bh); 1140 mark_buffer_dirty(bh); 1141 continue; 1142 } 1143 if (block_end > to || block_start < from) 1144 zero_user_segments(page, to, block_end, 1145 block_start, from); 1146 continue; 1147 } 1148 } 1149 if (PageUptodate(page)) { 1150 if (!buffer_uptodate(bh)) 1151 set_buffer_uptodate(bh); 1152 continue; 1153 } 1154 if (!buffer_uptodate(bh) && !buffer_delay(bh) && 1155 !buffer_unwritten(bh) && 1156 (block_start < from || block_end > to)) { 1157 ll_rw_block(REQ_OP_READ, 0, 1, &bh); 1158 *wait_bh++ = bh; 1159 decrypt = ext4_encrypted_inode(inode) && 1160 S_ISREG(inode->i_mode); 1161 } 1162 } 1163 /* 1164 * If we issued read requests, let them complete. 1165 */ 1166 while (wait_bh > wait) { 1167 wait_on_buffer(*--wait_bh); 1168 if (!buffer_uptodate(*wait_bh)) 1169 err = -EIO; 1170 } 1171 if (unlikely(err)) 1172 page_zero_new_buffers(page, from, to); 1173 else if (decrypt) 1174 err = fscrypt_decrypt_page(page->mapping->host, page, 1175 PAGE_SIZE, 0, page->index); 1176 return err; 1177 } 1178 #endif 1179 1180 static int ext4_write_begin(struct file *file, struct address_space *mapping, 1181 loff_t pos, unsigned len, unsigned flags, 1182 struct page **pagep, void **fsdata) 1183 { 1184 struct inode *inode = mapping->host; 1185 int ret, needed_blocks; 1186 handle_t *handle; 1187 int retries = 0; 1188 struct page *page; 1189 pgoff_t index; 1190 unsigned from, to; 1191 1192 trace_ext4_write_begin(inode, pos, len, flags); 1193 /* 1194 * Reserve one block more for addition to orphan list in case 1195 * we allocate blocks but write fails for some reason 1196 */ 1197 needed_blocks = ext4_writepage_trans_blocks(inode) + 1; 1198 index = pos >> PAGE_SHIFT; 1199 from = pos & (PAGE_SIZE - 1); 1200 to = from + len; 1201 1202 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 1203 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len, 1204 flags, pagep); 1205 if (ret < 0) 1206 return ret; 1207 if (ret == 1) 1208 return 0; 1209 } 1210 1211 /* 1212 * grab_cache_page_write_begin() can take a long time if the 1213 * system is thrashing due to memory pressure, or if the page 1214 * is being written back. So grab it first before we start 1215 * the transaction handle. This also allows us to allocate 1216 * the page (if needed) without using GFP_NOFS. 1217 */ 1218 retry_grab: 1219 page = grab_cache_page_write_begin(mapping, index, flags); 1220 if (!page) 1221 return -ENOMEM; 1222 unlock_page(page); 1223 1224 retry_journal: 1225 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); 1226 if (IS_ERR(handle)) { 1227 put_page(page); 1228 return PTR_ERR(handle); 1229 } 1230 1231 lock_page(page); 1232 if (page->mapping != mapping) { 1233 /* The page got truncated from under us */ 1234 unlock_page(page); 1235 put_page(page); 1236 ext4_journal_stop(handle); 1237 goto retry_grab; 1238 } 1239 /* In case writeback began while the page was unlocked */ 1240 wait_for_stable_page(page); 1241 1242 #ifdef CONFIG_EXT4_FS_ENCRYPTION 1243 if (ext4_should_dioread_nolock(inode)) 1244 ret = ext4_block_write_begin(page, pos, len, 1245 ext4_get_block_unwritten); 1246 else 1247 ret = ext4_block_write_begin(page, pos, len, 1248 ext4_get_block); 1249 #else 1250 if (ext4_should_dioread_nolock(inode)) 1251 ret = __block_write_begin(page, pos, len, 1252 ext4_get_block_unwritten); 1253 else 1254 ret = __block_write_begin(page, pos, len, ext4_get_block); 1255 #endif 1256 if (!ret && ext4_should_journal_data(inode)) { 1257 ret = ext4_walk_page_buffers(handle, page_buffers(page), 1258 from, to, NULL, 1259 do_journal_get_write_access); 1260 } 1261 1262 if (ret) { 1263 unlock_page(page); 1264 /* 1265 * __block_write_begin may have instantiated a few blocks 1266 * outside i_size. Trim these off again. Don't need 1267 * i_size_read because we hold i_mutex. 1268 * 1269 * Add inode to orphan list in case we crash before 1270 * truncate finishes 1271 */ 1272 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1273 ext4_orphan_add(handle, inode); 1274 1275 ext4_journal_stop(handle); 1276 if (pos + len > inode->i_size) { 1277 ext4_truncate_failed_write(inode); 1278 /* 1279 * If truncate failed early the inode might 1280 * still be on the orphan list; we need to 1281 * make sure the inode is removed from the 1282 * orphan list in that case. 1283 */ 1284 if (inode->i_nlink) 1285 ext4_orphan_del(NULL, inode); 1286 } 1287 1288 if (ret == -ENOSPC && 1289 ext4_should_retry_alloc(inode->i_sb, &retries)) 1290 goto retry_journal; 1291 put_page(page); 1292 return ret; 1293 } 1294 *pagep = page; 1295 return ret; 1296 } 1297 1298 /* For write_end() in data=journal mode */ 1299 static int write_end_fn(handle_t *handle, struct buffer_head *bh) 1300 { 1301 int ret; 1302 if (!buffer_mapped(bh) || buffer_freed(bh)) 1303 return 0; 1304 set_buffer_uptodate(bh); 1305 ret = ext4_handle_dirty_metadata(handle, NULL, bh); 1306 clear_buffer_meta(bh); 1307 clear_buffer_prio(bh); 1308 return ret; 1309 } 1310 1311 /* 1312 * We need to pick up the new inode size which generic_commit_write gave us 1313 * `file' can be NULL - eg, when called from page_symlink(). 1314 * 1315 * ext4 never places buffers on inode->i_mapping->private_list. metadata 1316 * buffers are managed internally. 1317 */ 1318 static int ext4_write_end(struct file *file, 1319 struct address_space *mapping, 1320 loff_t pos, unsigned len, unsigned copied, 1321 struct page *page, void *fsdata) 1322 { 1323 handle_t *handle = ext4_journal_current_handle(); 1324 struct inode *inode = mapping->host; 1325 loff_t old_size = inode->i_size; 1326 int ret = 0, ret2; 1327 int i_size_changed = 0; 1328 1329 trace_ext4_write_end(inode, pos, len, copied); 1330 if (ext4_has_inline_data(inode)) { 1331 ret = ext4_write_inline_data_end(inode, pos, len, 1332 copied, page); 1333 if (ret < 0) 1334 goto errout; 1335 copied = ret; 1336 } else 1337 copied = block_write_end(file, mapping, pos, 1338 len, copied, page, fsdata); 1339 /* 1340 * it's important to update i_size while still holding page lock: 1341 * page writeout could otherwise come in and zero beyond i_size. 1342 */ 1343 i_size_changed = ext4_update_inode_size(inode, pos + copied); 1344 unlock_page(page); 1345 put_page(page); 1346 1347 if (old_size < pos) 1348 pagecache_isize_extended(inode, old_size, pos); 1349 /* 1350 * Don't mark the inode dirty under page lock. First, it unnecessarily 1351 * makes the holding time of page lock longer. Second, it forces lock 1352 * ordering of page lock and transaction start for journaling 1353 * filesystems. 1354 */ 1355 if (i_size_changed) 1356 ext4_mark_inode_dirty(handle, inode); 1357 1358 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1359 /* if we have allocated more blocks and copied 1360 * less. We will have blocks allocated outside 1361 * inode->i_size. So truncate them 1362 */ 1363 ext4_orphan_add(handle, inode); 1364 errout: 1365 ret2 = ext4_journal_stop(handle); 1366 if (!ret) 1367 ret = ret2; 1368 1369 if (pos + len > inode->i_size) { 1370 ext4_truncate_failed_write(inode); 1371 /* 1372 * If truncate failed early the inode might still be 1373 * on the orphan list; we need to make sure the inode 1374 * is removed from the orphan list in that case. 1375 */ 1376 if (inode->i_nlink) 1377 ext4_orphan_del(NULL, inode); 1378 } 1379 1380 return ret ? ret : copied; 1381 } 1382 1383 /* 1384 * This is a private version of page_zero_new_buffers() which doesn't 1385 * set the buffer to be dirty, since in data=journalled mode we need 1386 * to call ext4_handle_dirty_metadata() instead. 1387 */ 1388 static void zero_new_buffers(struct page *page, unsigned from, unsigned to) 1389 { 1390 unsigned int block_start = 0, block_end; 1391 struct buffer_head *head, *bh; 1392 1393 bh = head = page_buffers(page); 1394 do { 1395 block_end = block_start + bh->b_size; 1396 if (buffer_new(bh)) { 1397 if (block_end > from && block_start < to) { 1398 if (!PageUptodate(page)) { 1399 unsigned start, size; 1400 1401 start = max(from, block_start); 1402 size = min(to, block_end) - start; 1403 1404 zero_user(page, start, size); 1405 set_buffer_uptodate(bh); 1406 } 1407 clear_buffer_new(bh); 1408 } 1409 } 1410 block_start = block_end; 1411 bh = bh->b_this_page; 1412 } while (bh != head); 1413 } 1414 1415 static int ext4_journalled_write_end(struct file *file, 1416 struct address_space *mapping, 1417 loff_t pos, unsigned len, unsigned copied, 1418 struct page *page, void *fsdata) 1419 { 1420 handle_t *handle = ext4_journal_current_handle(); 1421 struct inode *inode = mapping->host; 1422 loff_t old_size = inode->i_size; 1423 int ret = 0, ret2; 1424 int partial = 0; 1425 unsigned from, to; 1426 int size_changed = 0; 1427 1428 trace_ext4_journalled_write_end(inode, pos, len, copied); 1429 from = pos & (PAGE_SIZE - 1); 1430 to = from + len; 1431 1432 BUG_ON(!ext4_handle_valid(handle)); 1433 1434 if (ext4_has_inline_data(inode)) 1435 copied = ext4_write_inline_data_end(inode, pos, len, 1436 copied, page); 1437 else { 1438 if (copied < len) { 1439 if (!PageUptodate(page)) 1440 copied = 0; 1441 zero_new_buffers(page, from+copied, to); 1442 } 1443 1444 ret = ext4_walk_page_buffers(handle, page_buffers(page), from, 1445 to, &partial, write_end_fn); 1446 if (!partial) 1447 SetPageUptodate(page); 1448 } 1449 size_changed = ext4_update_inode_size(inode, pos + copied); 1450 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 1451 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 1452 unlock_page(page); 1453 put_page(page); 1454 1455 if (old_size < pos) 1456 pagecache_isize_extended(inode, old_size, pos); 1457 1458 if (size_changed) { 1459 ret2 = ext4_mark_inode_dirty(handle, inode); 1460 if (!ret) 1461 ret = ret2; 1462 } 1463 1464 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 1465 /* if we have allocated more blocks and copied 1466 * less. We will have blocks allocated outside 1467 * inode->i_size. So truncate them 1468 */ 1469 ext4_orphan_add(handle, inode); 1470 1471 ret2 = ext4_journal_stop(handle); 1472 if (!ret) 1473 ret = ret2; 1474 if (pos + len > inode->i_size) { 1475 ext4_truncate_failed_write(inode); 1476 /* 1477 * If truncate failed early the inode might still be 1478 * on the orphan list; we need to make sure the inode 1479 * is removed from the orphan list in that case. 1480 */ 1481 if (inode->i_nlink) 1482 ext4_orphan_del(NULL, inode); 1483 } 1484 1485 return ret ? ret : copied; 1486 } 1487 1488 /* 1489 * Reserve space for a single cluster 1490 */ 1491 static int ext4_da_reserve_space(struct inode *inode) 1492 { 1493 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1494 struct ext4_inode_info *ei = EXT4_I(inode); 1495 int ret; 1496 1497 /* 1498 * We will charge metadata quota at writeout time; this saves 1499 * us from metadata over-estimation, though we may go over by 1500 * a small amount in the end. Here we just reserve for data. 1501 */ 1502 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1)); 1503 if (ret) 1504 return ret; 1505 1506 spin_lock(&ei->i_block_reservation_lock); 1507 if (ext4_claim_free_clusters(sbi, 1, 0)) { 1508 spin_unlock(&ei->i_block_reservation_lock); 1509 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1)); 1510 return -ENOSPC; 1511 } 1512 ei->i_reserved_data_blocks++; 1513 trace_ext4_da_reserve_space(inode); 1514 spin_unlock(&ei->i_block_reservation_lock); 1515 1516 return 0; /* success */ 1517 } 1518 1519 static void ext4_da_release_space(struct inode *inode, int to_free) 1520 { 1521 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1522 struct ext4_inode_info *ei = EXT4_I(inode); 1523 1524 if (!to_free) 1525 return; /* Nothing to release, exit */ 1526 1527 spin_lock(&EXT4_I(inode)->i_block_reservation_lock); 1528 1529 trace_ext4_da_release_space(inode, to_free); 1530 if (unlikely(to_free > ei->i_reserved_data_blocks)) { 1531 /* 1532 * if there aren't enough reserved blocks, then the 1533 * counter is messed up somewhere. Since this 1534 * function is called from invalidate page, it's 1535 * harmless to return without any action. 1536 */ 1537 ext4_warning(inode->i_sb, "ext4_da_release_space: " 1538 "ino %lu, to_free %d with only %d reserved " 1539 "data blocks", inode->i_ino, to_free, 1540 ei->i_reserved_data_blocks); 1541 WARN_ON(1); 1542 to_free = ei->i_reserved_data_blocks; 1543 } 1544 ei->i_reserved_data_blocks -= to_free; 1545 1546 /* update fs dirty data blocks counter */ 1547 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free); 1548 1549 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 1550 1551 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free)); 1552 } 1553 1554 static void ext4_da_page_release_reservation(struct page *page, 1555 unsigned int offset, 1556 unsigned int length) 1557 { 1558 int to_release = 0, contiguous_blks = 0; 1559 struct buffer_head *head, *bh; 1560 unsigned int curr_off = 0; 1561 struct inode *inode = page->mapping->host; 1562 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1563 unsigned int stop = offset + length; 1564 int num_clusters; 1565 ext4_fsblk_t lblk; 1566 1567 BUG_ON(stop > PAGE_SIZE || stop < length); 1568 1569 head = page_buffers(page); 1570 bh = head; 1571 do { 1572 unsigned int next_off = curr_off + bh->b_size; 1573 1574 if (next_off > stop) 1575 break; 1576 1577 if ((offset <= curr_off) && (buffer_delay(bh))) { 1578 to_release++; 1579 contiguous_blks++; 1580 clear_buffer_delay(bh); 1581 } else if (contiguous_blks) { 1582 lblk = page->index << 1583 (PAGE_SHIFT - inode->i_blkbits); 1584 lblk += (curr_off >> inode->i_blkbits) - 1585 contiguous_blks; 1586 ext4_es_remove_extent(inode, lblk, contiguous_blks); 1587 contiguous_blks = 0; 1588 } 1589 curr_off = next_off; 1590 } while ((bh = bh->b_this_page) != head); 1591 1592 if (contiguous_blks) { 1593 lblk = page->index << (PAGE_SHIFT - inode->i_blkbits); 1594 lblk += (curr_off >> inode->i_blkbits) - contiguous_blks; 1595 ext4_es_remove_extent(inode, lblk, contiguous_blks); 1596 } 1597 1598 /* If we have released all the blocks belonging to a cluster, then we 1599 * need to release the reserved space for that cluster. */ 1600 num_clusters = EXT4_NUM_B2C(sbi, to_release); 1601 while (num_clusters > 0) { 1602 lblk = (page->index << (PAGE_SHIFT - inode->i_blkbits)) + 1603 ((num_clusters - 1) << sbi->s_cluster_bits); 1604 if (sbi->s_cluster_ratio == 1 || 1605 !ext4_find_delalloc_cluster(inode, lblk)) 1606 ext4_da_release_space(inode, 1); 1607 1608 num_clusters--; 1609 } 1610 } 1611 1612 /* 1613 * Delayed allocation stuff 1614 */ 1615 1616 struct mpage_da_data { 1617 struct inode *inode; 1618 struct writeback_control *wbc; 1619 1620 pgoff_t first_page; /* The first page to write */ 1621 pgoff_t next_page; /* Current page to examine */ 1622 pgoff_t last_page; /* Last page to examine */ 1623 /* 1624 * Extent to map - this can be after first_page because that can be 1625 * fully mapped. We somewhat abuse m_flags to store whether the extent 1626 * is delalloc or unwritten. 1627 */ 1628 struct ext4_map_blocks map; 1629 struct ext4_io_submit io_submit; /* IO submission data */ 1630 }; 1631 1632 static void mpage_release_unused_pages(struct mpage_da_data *mpd, 1633 bool invalidate) 1634 { 1635 int nr_pages, i; 1636 pgoff_t index, end; 1637 struct pagevec pvec; 1638 struct inode *inode = mpd->inode; 1639 struct address_space *mapping = inode->i_mapping; 1640 1641 /* This is necessary when next_page == 0. */ 1642 if (mpd->first_page >= mpd->next_page) 1643 return; 1644 1645 index = mpd->first_page; 1646 end = mpd->next_page - 1; 1647 if (invalidate) { 1648 ext4_lblk_t start, last; 1649 start = index << (PAGE_SHIFT - inode->i_blkbits); 1650 last = end << (PAGE_SHIFT - inode->i_blkbits); 1651 ext4_es_remove_extent(inode, start, last - start + 1); 1652 } 1653 1654 pagevec_init(&pvec, 0); 1655 while (index <= end) { 1656 nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE); 1657 if (nr_pages == 0) 1658 break; 1659 for (i = 0; i < nr_pages; i++) { 1660 struct page *page = pvec.pages[i]; 1661 if (page->index > end) 1662 break; 1663 BUG_ON(!PageLocked(page)); 1664 BUG_ON(PageWriteback(page)); 1665 if (invalidate) { 1666 if (page_mapped(page)) 1667 clear_page_dirty_for_io(page); 1668 block_invalidatepage(page, 0, PAGE_SIZE); 1669 ClearPageUptodate(page); 1670 } 1671 unlock_page(page); 1672 } 1673 index = pvec.pages[nr_pages - 1]->index + 1; 1674 pagevec_release(&pvec); 1675 } 1676 } 1677 1678 static void ext4_print_free_blocks(struct inode *inode) 1679 { 1680 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1681 struct super_block *sb = inode->i_sb; 1682 struct ext4_inode_info *ei = EXT4_I(inode); 1683 1684 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld", 1685 EXT4_C2B(EXT4_SB(inode->i_sb), 1686 ext4_count_free_clusters(sb))); 1687 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details"); 1688 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld", 1689 (long long) EXT4_C2B(EXT4_SB(sb), 1690 percpu_counter_sum(&sbi->s_freeclusters_counter))); 1691 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld", 1692 (long long) EXT4_C2B(EXT4_SB(sb), 1693 percpu_counter_sum(&sbi->s_dirtyclusters_counter))); 1694 ext4_msg(sb, KERN_CRIT, "Block reservation details"); 1695 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u", 1696 ei->i_reserved_data_blocks); 1697 return; 1698 } 1699 1700 static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh) 1701 { 1702 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh); 1703 } 1704 1705 /* 1706 * This function is grabs code from the very beginning of 1707 * ext4_map_blocks, but assumes that the caller is from delayed write 1708 * time. This function looks up the requested blocks and sets the 1709 * buffer delay bit under the protection of i_data_sem. 1710 */ 1711 static int ext4_da_map_blocks(struct inode *inode, sector_t iblock, 1712 struct ext4_map_blocks *map, 1713 struct buffer_head *bh) 1714 { 1715 struct extent_status es; 1716 int retval; 1717 sector_t invalid_block = ~((sector_t) 0xffff); 1718 #ifdef ES_AGGRESSIVE_TEST 1719 struct ext4_map_blocks orig_map; 1720 1721 memcpy(&orig_map, map, sizeof(*map)); 1722 #endif 1723 1724 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es)) 1725 invalid_block = ~0; 1726 1727 map->m_flags = 0; 1728 ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u," 1729 "logical block %lu\n", inode->i_ino, map->m_len, 1730 (unsigned long) map->m_lblk); 1731 1732 /* Lookup extent status tree firstly */ 1733 if (ext4_es_lookup_extent(inode, iblock, &es)) { 1734 if (ext4_es_is_hole(&es)) { 1735 retval = 0; 1736 down_read(&EXT4_I(inode)->i_data_sem); 1737 goto add_delayed; 1738 } 1739 1740 /* 1741 * Delayed extent could be allocated by fallocate. 1742 * So we need to check it. 1743 */ 1744 if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) { 1745 map_bh(bh, inode->i_sb, invalid_block); 1746 set_buffer_new(bh); 1747 set_buffer_delay(bh); 1748 return 0; 1749 } 1750 1751 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk; 1752 retval = es.es_len - (iblock - es.es_lblk); 1753 if (retval > map->m_len) 1754 retval = map->m_len; 1755 map->m_len = retval; 1756 if (ext4_es_is_written(&es)) 1757 map->m_flags |= EXT4_MAP_MAPPED; 1758 else if (ext4_es_is_unwritten(&es)) 1759 map->m_flags |= EXT4_MAP_UNWRITTEN; 1760 else 1761 BUG_ON(1); 1762 1763 #ifdef ES_AGGRESSIVE_TEST 1764 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0); 1765 #endif 1766 return retval; 1767 } 1768 1769 /* 1770 * Try to see if we can get the block without requesting a new 1771 * file system block. 1772 */ 1773 down_read(&EXT4_I(inode)->i_data_sem); 1774 if (ext4_has_inline_data(inode)) 1775 retval = 0; 1776 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 1777 retval = ext4_ext_map_blocks(NULL, inode, map, 0); 1778 else 1779 retval = ext4_ind_map_blocks(NULL, inode, map, 0); 1780 1781 add_delayed: 1782 if (retval == 0) { 1783 int ret; 1784 /* 1785 * XXX: __block_prepare_write() unmaps passed block, 1786 * is it OK? 1787 */ 1788 /* 1789 * If the block was allocated from previously allocated cluster, 1790 * then we don't need to reserve it again. However we still need 1791 * to reserve metadata for every block we're going to write. 1792 */ 1793 if (EXT4_SB(inode->i_sb)->s_cluster_ratio == 1 || 1794 !ext4_find_delalloc_cluster(inode, map->m_lblk)) { 1795 ret = ext4_da_reserve_space(inode); 1796 if (ret) { 1797 /* not enough space to reserve */ 1798 retval = ret; 1799 goto out_unlock; 1800 } 1801 } 1802 1803 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len, 1804 ~0, EXTENT_STATUS_DELAYED); 1805 if (ret) { 1806 retval = ret; 1807 goto out_unlock; 1808 } 1809 1810 map_bh(bh, inode->i_sb, invalid_block); 1811 set_buffer_new(bh); 1812 set_buffer_delay(bh); 1813 } else if (retval > 0) { 1814 int ret; 1815 unsigned int status; 1816 1817 if (unlikely(retval != map->m_len)) { 1818 ext4_warning(inode->i_sb, 1819 "ES len assertion failed for inode " 1820 "%lu: retval %d != map->m_len %d", 1821 inode->i_ino, retval, map->m_len); 1822 WARN_ON(1); 1823 } 1824 1825 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 1826 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 1827 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len, 1828 map->m_pblk, status); 1829 if (ret != 0) 1830 retval = ret; 1831 } 1832 1833 out_unlock: 1834 up_read((&EXT4_I(inode)->i_data_sem)); 1835 1836 return retval; 1837 } 1838 1839 /* 1840 * This is a special get_block_t callback which is used by 1841 * ext4_da_write_begin(). It will either return mapped block or 1842 * reserve space for a single block. 1843 * 1844 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set. 1845 * We also have b_blocknr = -1 and b_bdev initialized properly 1846 * 1847 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set. 1848 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev 1849 * initialized properly. 1850 */ 1851 int ext4_da_get_block_prep(struct inode *inode, sector_t iblock, 1852 struct buffer_head *bh, int create) 1853 { 1854 struct ext4_map_blocks map; 1855 int ret = 0; 1856 1857 BUG_ON(create == 0); 1858 BUG_ON(bh->b_size != inode->i_sb->s_blocksize); 1859 1860 map.m_lblk = iblock; 1861 map.m_len = 1; 1862 1863 /* 1864 * first, we need to know whether the block is allocated already 1865 * preallocated blocks are unmapped but should treated 1866 * the same as allocated blocks. 1867 */ 1868 ret = ext4_da_map_blocks(inode, iblock, &map, bh); 1869 if (ret <= 0) 1870 return ret; 1871 1872 map_bh(bh, inode->i_sb, map.m_pblk); 1873 ext4_update_bh_state(bh, map.m_flags); 1874 1875 if (buffer_unwritten(bh)) { 1876 /* A delayed write to unwritten bh should be marked 1877 * new and mapped. Mapped ensures that we don't do 1878 * get_block multiple times when we write to the same 1879 * offset and new ensures that we do proper zero out 1880 * for partial write. 1881 */ 1882 set_buffer_new(bh); 1883 set_buffer_mapped(bh); 1884 } 1885 return 0; 1886 } 1887 1888 static int bget_one(handle_t *handle, struct buffer_head *bh) 1889 { 1890 get_bh(bh); 1891 return 0; 1892 } 1893 1894 static int bput_one(handle_t *handle, struct buffer_head *bh) 1895 { 1896 put_bh(bh); 1897 return 0; 1898 } 1899 1900 static int __ext4_journalled_writepage(struct page *page, 1901 unsigned int len) 1902 { 1903 struct address_space *mapping = page->mapping; 1904 struct inode *inode = mapping->host; 1905 struct buffer_head *page_bufs = NULL; 1906 handle_t *handle = NULL; 1907 int ret = 0, err = 0; 1908 int inline_data = ext4_has_inline_data(inode); 1909 struct buffer_head *inode_bh = NULL; 1910 1911 ClearPageChecked(page); 1912 1913 if (inline_data) { 1914 BUG_ON(page->index != 0); 1915 BUG_ON(len > ext4_get_max_inline_size(inode)); 1916 inode_bh = ext4_journalled_write_inline_data(inode, len, page); 1917 if (inode_bh == NULL) 1918 goto out; 1919 } else { 1920 page_bufs = page_buffers(page); 1921 if (!page_bufs) { 1922 BUG(); 1923 goto out; 1924 } 1925 ext4_walk_page_buffers(handle, page_bufs, 0, len, 1926 NULL, bget_one); 1927 } 1928 /* 1929 * We need to release the page lock before we start the 1930 * journal, so grab a reference so the page won't disappear 1931 * out from under us. 1932 */ 1933 get_page(page); 1934 unlock_page(page); 1935 1936 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, 1937 ext4_writepage_trans_blocks(inode)); 1938 if (IS_ERR(handle)) { 1939 ret = PTR_ERR(handle); 1940 put_page(page); 1941 goto out_no_pagelock; 1942 } 1943 BUG_ON(!ext4_handle_valid(handle)); 1944 1945 lock_page(page); 1946 put_page(page); 1947 if (page->mapping != mapping) { 1948 /* The page got truncated from under us */ 1949 ext4_journal_stop(handle); 1950 ret = 0; 1951 goto out; 1952 } 1953 1954 if (inline_data) { 1955 BUFFER_TRACE(inode_bh, "get write access"); 1956 ret = ext4_journal_get_write_access(handle, inode_bh); 1957 1958 err = ext4_handle_dirty_metadata(handle, inode, inode_bh); 1959 1960 } else { 1961 ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL, 1962 do_journal_get_write_access); 1963 1964 err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL, 1965 write_end_fn); 1966 } 1967 if (ret == 0) 1968 ret = err; 1969 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 1970 err = ext4_journal_stop(handle); 1971 if (!ret) 1972 ret = err; 1973 1974 if (!ext4_has_inline_data(inode)) 1975 ext4_walk_page_buffers(NULL, page_bufs, 0, len, 1976 NULL, bput_one); 1977 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 1978 out: 1979 unlock_page(page); 1980 out_no_pagelock: 1981 brelse(inode_bh); 1982 return ret; 1983 } 1984 1985 /* 1986 * Note that we don't need to start a transaction unless we're journaling data 1987 * because we should have holes filled from ext4_page_mkwrite(). We even don't 1988 * need to file the inode to the transaction's list in ordered mode because if 1989 * we are writing back data added by write(), the inode is already there and if 1990 * we are writing back data modified via mmap(), no one guarantees in which 1991 * transaction the data will hit the disk. In case we are journaling data, we 1992 * cannot start transaction directly because transaction start ranks above page 1993 * lock so we have to do some magic. 1994 * 1995 * This function can get called via... 1996 * - ext4_writepages after taking page lock (have journal handle) 1997 * - journal_submit_inode_data_buffers (no journal handle) 1998 * - shrink_page_list via the kswapd/direct reclaim (no journal handle) 1999 * - grab_page_cache when doing write_begin (have journal handle) 2000 * 2001 * We don't do any block allocation in this function. If we have page with 2002 * multiple blocks we need to write those buffer_heads that are mapped. This 2003 * is important for mmaped based write. So if we do with blocksize 1K 2004 * truncate(f, 1024); 2005 * a = mmap(f, 0, 4096); 2006 * a[0] = 'a'; 2007 * truncate(f, 4096); 2008 * we have in the page first buffer_head mapped via page_mkwrite call back 2009 * but other buffer_heads would be unmapped but dirty (dirty done via the 2010 * do_wp_page). So writepage should write the first block. If we modify 2011 * the mmap area beyond 1024 we will again get a page_fault and the 2012 * page_mkwrite callback will do the block allocation and mark the 2013 * buffer_heads mapped. 2014 * 2015 * We redirty the page if we have any buffer_heads that is either delay or 2016 * unwritten in the page. 2017 * 2018 * We can get recursively called as show below. 2019 * 2020 * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() -> 2021 * ext4_writepage() 2022 * 2023 * But since we don't do any block allocation we should not deadlock. 2024 * Page also have the dirty flag cleared so we don't get recurive page_lock. 2025 */ 2026 static int ext4_writepage(struct page *page, 2027 struct writeback_control *wbc) 2028 { 2029 int ret = 0; 2030 loff_t size; 2031 unsigned int len; 2032 struct buffer_head *page_bufs = NULL; 2033 struct inode *inode = page->mapping->host; 2034 struct ext4_io_submit io_submit; 2035 bool keep_towrite = false; 2036 2037 trace_ext4_writepage(page); 2038 size = i_size_read(inode); 2039 if (page->index == size >> PAGE_SHIFT) 2040 len = size & ~PAGE_MASK; 2041 else 2042 len = PAGE_SIZE; 2043 2044 page_bufs = page_buffers(page); 2045 /* 2046 * We cannot do block allocation or other extent handling in this 2047 * function. If there are buffers needing that, we have to redirty 2048 * the page. But we may reach here when we do a journal commit via 2049 * journal_submit_inode_data_buffers() and in that case we must write 2050 * allocated buffers to achieve data=ordered mode guarantees. 2051 * 2052 * Also, if there is only one buffer per page (the fs block 2053 * size == the page size), if one buffer needs block 2054 * allocation or needs to modify the extent tree to clear the 2055 * unwritten flag, we know that the page can't be written at 2056 * all, so we might as well refuse the write immediately. 2057 * Unfortunately if the block size != page size, we can't as 2058 * easily detect this case using ext4_walk_page_buffers(), but 2059 * for the extremely common case, this is an optimization that 2060 * skips a useless round trip through ext4_bio_write_page(). 2061 */ 2062 if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL, 2063 ext4_bh_delay_or_unwritten)) { 2064 redirty_page_for_writepage(wbc, page); 2065 if ((current->flags & PF_MEMALLOC) || 2066 (inode->i_sb->s_blocksize == PAGE_SIZE)) { 2067 /* 2068 * For memory cleaning there's no point in writing only 2069 * some buffers. So just bail out. Warn if we came here 2070 * from direct reclaim. 2071 */ 2072 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) 2073 == PF_MEMALLOC); 2074 unlock_page(page); 2075 return 0; 2076 } 2077 keep_towrite = true; 2078 } 2079 2080 if (PageChecked(page) && ext4_should_journal_data(inode)) 2081 /* 2082 * It's mmapped pagecache. Add buffers and journal it. There 2083 * doesn't seem much point in redirtying the page here. 2084 */ 2085 return __ext4_journalled_writepage(page, len); 2086 2087 ext4_io_submit_init(&io_submit, wbc); 2088 io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS); 2089 if (!io_submit.io_end) { 2090 redirty_page_for_writepage(wbc, page); 2091 unlock_page(page); 2092 return -ENOMEM; 2093 } 2094 ret = ext4_bio_write_page(&io_submit, page, len, wbc, keep_towrite); 2095 ext4_io_submit(&io_submit); 2096 /* Drop io_end reference we got from init */ 2097 ext4_put_io_end_defer(io_submit.io_end); 2098 return ret; 2099 } 2100 2101 static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page) 2102 { 2103 int len; 2104 loff_t size = i_size_read(mpd->inode); 2105 int err; 2106 2107 BUG_ON(page->index != mpd->first_page); 2108 if (page->index == size >> PAGE_SHIFT) 2109 len = size & ~PAGE_MASK; 2110 else 2111 len = PAGE_SIZE; 2112 clear_page_dirty_for_io(page); 2113 err = ext4_bio_write_page(&mpd->io_submit, page, len, mpd->wbc, false); 2114 if (!err) 2115 mpd->wbc->nr_to_write--; 2116 mpd->first_page++; 2117 2118 return err; 2119 } 2120 2121 #define BH_FLAGS ((1 << BH_Unwritten) | (1 << BH_Delay)) 2122 2123 /* 2124 * mballoc gives us at most this number of blocks... 2125 * XXX: That seems to be only a limitation of ext4_mb_normalize_request(). 2126 * The rest of mballoc seems to handle chunks up to full group size. 2127 */ 2128 #define MAX_WRITEPAGES_EXTENT_LEN 2048 2129 2130 /* 2131 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map 2132 * 2133 * @mpd - extent of blocks 2134 * @lblk - logical number of the block in the file 2135 * @bh - buffer head we want to add to the extent 2136 * 2137 * The function is used to collect contig. blocks in the same state. If the 2138 * buffer doesn't require mapping for writeback and we haven't started the 2139 * extent of buffers to map yet, the function returns 'true' immediately - the 2140 * caller can write the buffer right away. Otherwise the function returns true 2141 * if the block has been added to the extent, false if the block couldn't be 2142 * added. 2143 */ 2144 static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk, 2145 struct buffer_head *bh) 2146 { 2147 struct ext4_map_blocks *map = &mpd->map; 2148 2149 /* Buffer that doesn't need mapping for writeback? */ 2150 if (!buffer_dirty(bh) || !buffer_mapped(bh) || 2151 (!buffer_delay(bh) && !buffer_unwritten(bh))) { 2152 /* So far no extent to map => we write the buffer right away */ 2153 if (map->m_len == 0) 2154 return true; 2155 return false; 2156 } 2157 2158 /* First block in the extent? */ 2159 if (map->m_len == 0) { 2160 map->m_lblk = lblk; 2161 map->m_len = 1; 2162 map->m_flags = bh->b_state & BH_FLAGS; 2163 return true; 2164 } 2165 2166 /* Don't go larger than mballoc is willing to allocate */ 2167 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN) 2168 return false; 2169 2170 /* Can we merge the block to our big extent? */ 2171 if (lblk == map->m_lblk + map->m_len && 2172 (bh->b_state & BH_FLAGS) == map->m_flags) { 2173 map->m_len++; 2174 return true; 2175 } 2176 return false; 2177 } 2178 2179 /* 2180 * mpage_process_page_bufs - submit page buffers for IO or add them to extent 2181 * 2182 * @mpd - extent of blocks for mapping 2183 * @head - the first buffer in the page 2184 * @bh - buffer we should start processing from 2185 * @lblk - logical number of the block in the file corresponding to @bh 2186 * 2187 * Walk through page buffers from @bh upto @head (exclusive) and either submit 2188 * the page for IO if all buffers in this page were mapped and there's no 2189 * accumulated extent of buffers to map or add buffers in the page to the 2190 * extent of buffers to map. The function returns 1 if the caller can continue 2191 * by processing the next page, 0 if it should stop adding buffers to the 2192 * extent to map because we cannot extend it anymore. It can also return value 2193 * < 0 in case of error during IO submission. 2194 */ 2195 static int mpage_process_page_bufs(struct mpage_da_data *mpd, 2196 struct buffer_head *head, 2197 struct buffer_head *bh, 2198 ext4_lblk_t lblk) 2199 { 2200 struct inode *inode = mpd->inode; 2201 int err; 2202 ext4_lblk_t blocks = (i_size_read(inode) + (1 << inode->i_blkbits) - 1) 2203 >> inode->i_blkbits; 2204 2205 do { 2206 BUG_ON(buffer_locked(bh)); 2207 2208 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) { 2209 /* Found extent to map? */ 2210 if (mpd->map.m_len) 2211 return 0; 2212 /* Everything mapped so far and we hit EOF */ 2213 break; 2214 } 2215 } while (lblk++, (bh = bh->b_this_page) != head); 2216 /* So far everything mapped? Submit the page for IO. */ 2217 if (mpd->map.m_len == 0) { 2218 err = mpage_submit_page(mpd, head->b_page); 2219 if (err < 0) 2220 return err; 2221 } 2222 return lblk < blocks; 2223 } 2224 2225 /* 2226 * mpage_map_buffers - update buffers corresponding to changed extent and 2227 * submit fully mapped pages for IO 2228 * 2229 * @mpd - description of extent to map, on return next extent to map 2230 * 2231 * Scan buffers corresponding to changed extent (we expect corresponding pages 2232 * to be already locked) and update buffer state according to new extent state. 2233 * We map delalloc buffers to their physical location, clear unwritten bits, 2234 * and mark buffers as uninit when we perform writes to unwritten extents 2235 * and do extent conversion after IO is finished. If the last page is not fully 2236 * mapped, we update @map to the next extent in the last page that needs 2237 * mapping. Otherwise we submit the page for IO. 2238 */ 2239 static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd) 2240 { 2241 struct pagevec pvec; 2242 int nr_pages, i; 2243 struct inode *inode = mpd->inode; 2244 struct buffer_head *head, *bh; 2245 int bpp_bits = PAGE_SHIFT - inode->i_blkbits; 2246 pgoff_t start, end; 2247 ext4_lblk_t lblk; 2248 sector_t pblock; 2249 int err; 2250 2251 start = mpd->map.m_lblk >> bpp_bits; 2252 end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits; 2253 lblk = start << bpp_bits; 2254 pblock = mpd->map.m_pblk; 2255 2256 pagevec_init(&pvec, 0); 2257 while (start <= end) { 2258 nr_pages = pagevec_lookup(&pvec, inode->i_mapping, start, 2259 PAGEVEC_SIZE); 2260 if (nr_pages == 0) 2261 break; 2262 for (i = 0; i < nr_pages; i++) { 2263 struct page *page = pvec.pages[i]; 2264 2265 if (page->index > end) 2266 break; 2267 /* Up to 'end' pages must be contiguous */ 2268 BUG_ON(page->index != start); 2269 bh = head = page_buffers(page); 2270 do { 2271 if (lblk < mpd->map.m_lblk) 2272 continue; 2273 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) { 2274 /* 2275 * Buffer after end of mapped extent. 2276 * Find next buffer in the page to map. 2277 */ 2278 mpd->map.m_len = 0; 2279 mpd->map.m_flags = 0; 2280 /* 2281 * FIXME: If dioread_nolock supports 2282 * blocksize < pagesize, we need to make 2283 * sure we add size mapped so far to 2284 * io_end->size as the following call 2285 * can submit the page for IO. 2286 */ 2287 err = mpage_process_page_bufs(mpd, head, 2288 bh, lblk); 2289 pagevec_release(&pvec); 2290 if (err > 0) 2291 err = 0; 2292 return err; 2293 } 2294 if (buffer_delay(bh)) { 2295 clear_buffer_delay(bh); 2296 bh->b_blocknr = pblock++; 2297 } 2298 clear_buffer_unwritten(bh); 2299 } while (lblk++, (bh = bh->b_this_page) != head); 2300 2301 /* 2302 * FIXME: This is going to break if dioread_nolock 2303 * supports blocksize < pagesize as we will try to 2304 * convert potentially unmapped parts of inode. 2305 */ 2306 mpd->io_submit.io_end->size += PAGE_SIZE; 2307 /* Page fully mapped - let IO run! */ 2308 err = mpage_submit_page(mpd, page); 2309 if (err < 0) { 2310 pagevec_release(&pvec); 2311 return err; 2312 } 2313 start++; 2314 } 2315 pagevec_release(&pvec); 2316 } 2317 /* Extent fully mapped and matches with page boundary. We are done. */ 2318 mpd->map.m_len = 0; 2319 mpd->map.m_flags = 0; 2320 return 0; 2321 } 2322 2323 static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd) 2324 { 2325 struct inode *inode = mpd->inode; 2326 struct ext4_map_blocks *map = &mpd->map; 2327 int get_blocks_flags; 2328 int err, dioread_nolock; 2329 2330 trace_ext4_da_write_pages_extent(inode, map); 2331 /* 2332 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or 2333 * to convert an unwritten extent to be initialized (in the case 2334 * where we have written into one or more preallocated blocks). It is 2335 * possible that we're going to need more metadata blocks than 2336 * previously reserved. However we must not fail because we're in 2337 * writeback and there is nothing we can do about it so it might result 2338 * in data loss. So use reserved blocks to allocate metadata if 2339 * possible. 2340 * 2341 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if 2342 * the blocks in question are delalloc blocks. This indicates 2343 * that the blocks and quotas has already been checked when 2344 * the data was copied into the page cache. 2345 */ 2346 get_blocks_flags = EXT4_GET_BLOCKS_CREATE | 2347 EXT4_GET_BLOCKS_METADATA_NOFAIL | 2348 EXT4_GET_BLOCKS_IO_SUBMIT; 2349 dioread_nolock = ext4_should_dioread_nolock(inode); 2350 if (dioread_nolock) 2351 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT; 2352 if (map->m_flags & (1 << BH_Delay)) 2353 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE; 2354 2355 err = ext4_map_blocks(handle, inode, map, get_blocks_flags); 2356 if (err < 0) 2357 return err; 2358 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) { 2359 if (!mpd->io_submit.io_end->handle && 2360 ext4_handle_valid(handle)) { 2361 mpd->io_submit.io_end->handle = handle->h_rsv_handle; 2362 handle->h_rsv_handle = NULL; 2363 } 2364 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end); 2365 } 2366 2367 BUG_ON(map->m_len == 0); 2368 if (map->m_flags & EXT4_MAP_NEW) { 2369 clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk, 2370 map->m_len); 2371 } 2372 return 0; 2373 } 2374 2375 /* 2376 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length 2377 * mpd->len and submit pages underlying it for IO 2378 * 2379 * @handle - handle for journal operations 2380 * @mpd - extent to map 2381 * @give_up_on_write - we set this to true iff there is a fatal error and there 2382 * is no hope of writing the data. The caller should discard 2383 * dirty pages to avoid infinite loops. 2384 * 2385 * The function maps extent starting at mpd->lblk of length mpd->len. If it is 2386 * delayed, blocks are allocated, if it is unwritten, we may need to convert 2387 * them to initialized or split the described range from larger unwritten 2388 * extent. Note that we need not map all the described range since allocation 2389 * can return less blocks or the range is covered by more unwritten extents. We 2390 * cannot map more because we are limited by reserved transaction credits. On 2391 * the other hand we always make sure that the last touched page is fully 2392 * mapped so that it can be written out (and thus forward progress is 2393 * guaranteed). After mapping we submit all mapped pages for IO. 2394 */ 2395 static int mpage_map_and_submit_extent(handle_t *handle, 2396 struct mpage_da_data *mpd, 2397 bool *give_up_on_write) 2398 { 2399 struct inode *inode = mpd->inode; 2400 struct ext4_map_blocks *map = &mpd->map; 2401 int err; 2402 loff_t disksize; 2403 int progress = 0; 2404 2405 mpd->io_submit.io_end->offset = 2406 ((loff_t)map->m_lblk) << inode->i_blkbits; 2407 do { 2408 err = mpage_map_one_extent(handle, mpd); 2409 if (err < 0) { 2410 struct super_block *sb = inode->i_sb; 2411 2412 if (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED) 2413 goto invalidate_dirty_pages; 2414 /* 2415 * Let the uper layers retry transient errors. 2416 * In the case of ENOSPC, if ext4_count_free_blocks() 2417 * is non-zero, a commit should free up blocks. 2418 */ 2419 if ((err == -ENOMEM) || 2420 (err == -ENOSPC && ext4_count_free_clusters(sb))) { 2421 if (progress) 2422 goto update_disksize; 2423 return err; 2424 } 2425 ext4_msg(sb, KERN_CRIT, 2426 "Delayed block allocation failed for " 2427 "inode %lu at logical offset %llu with" 2428 " max blocks %u with error %d", 2429 inode->i_ino, 2430 (unsigned long long)map->m_lblk, 2431 (unsigned)map->m_len, -err); 2432 ext4_msg(sb, KERN_CRIT, 2433 "This should not happen!! Data will " 2434 "be lost\n"); 2435 if (err == -ENOSPC) 2436 ext4_print_free_blocks(inode); 2437 invalidate_dirty_pages: 2438 *give_up_on_write = true; 2439 return err; 2440 } 2441 progress = 1; 2442 /* 2443 * Update buffer state, submit mapped pages, and get us new 2444 * extent to map 2445 */ 2446 err = mpage_map_and_submit_buffers(mpd); 2447 if (err < 0) 2448 goto update_disksize; 2449 } while (map->m_len); 2450 2451 update_disksize: 2452 /* 2453 * Update on-disk size after IO is submitted. Races with 2454 * truncate are avoided by checking i_size under i_data_sem. 2455 */ 2456 disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT; 2457 if (disksize > EXT4_I(inode)->i_disksize) { 2458 int err2; 2459 loff_t i_size; 2460 2461 down_write(&EXT4_I(inode)->i_data_sem); 2462 i_size = i_size_read(inode); 2463 if (disksize > i_size) 2464 disksize = i_size; 2465 if (disksize > EXT4_I(inode)->i_disksize) 2466 EXT4_I(inode)->i_disksize = disksize; 2467 err2 = ext4_mark_inode_dirty(handle, inode); 2468 up_write(&EXT4_I(inode)->i_data_sem); 2469 if (err2) 2470 ext4_error(inode->i_sb, 2471 "Failed to mark inode %lu dirty", 2472 inode->i_ino); 2473 if (!err) 2474 err = err2; 2475 } 2476 return err; 2477 } 2478 2479 /* 2480 * Calculate the total number of credits to reserve for one writepages 2481 * iteration. This is called from ext4_writepages(). We map an extent of 2482 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping 2483 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN + 2484 * bpp - 1 blocks in bpp different extents. 2485 */ 2486 static int ext4_da_writepages_trans_blocks(struct inode *inode) 2487 { 2488 int bpp = ext4_journal_blocks_per_page(inode); 2489 2490 return ext4_meta_trans_blocks(inode, 2491 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp); 2492 } 2493 2494 /* 2495 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages 2496 * and underlying extent to map 2497 * 2498 * @mpd - where to look for pages 2499 * 2500 * Walk dirty pages in the mapping. If they are fully mapped, submit them for 2501 * IO immediately. When we find a page which isn't mapped we start accumulating 2502 * extent of buffers underlying these pages that needs mapping (formed by 2503 * either delayed or unwritten buffers). We also lock the pages containing 2504 * these buffers. The extent found is returned in @mpd structure (starting at 2505 * mpd->lblk with length mpd->len blocks). 2506 * 2507 * Note that this function can attach bios to one io_end structure which are 2508 * neither logically nor physically contiguous. Although it may seem as an 2509 * unnecessary complication, it is actually inevitable in blocksize < pagesize 2510 * case as we need to track IO to all buffers underlying a page in one io_end. 2511 */ 2512 static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) 2513 { 2514 struct address_space *mapping = mpd->inode->i_mapping; 2515 struct pagevec pvec; 2516 unsigned int nr_pages; 2517 long left = mpd->wbc->nr_to_write; 2518 pgoff_t index = mpd->first_page; 2519 pgoff_t end = mpd->last_page; 2520 int tag; 2521 int i, err = 0; 2522 int blkbits = mpd->inode->i_blkbits; 2523 ext4_lblk_t lblk; 2524 struct buffer_head *head; 2525 2526 if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages) 2527 tag = PAGECACHE_TAG_TOWRITE; 2528 else 2529 tag = PAGECACHE_TAG_DIRTY; 2530 2531 pagevec_init(&pvec, 0); 2532 mpd->map.m_len = 0; 2533 mpd->next_page = index; 2534 while (index <= end) { 2535 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 2536 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); 2537 if (nr_pages == 0) 2538 goto out; 2539 2540 for (i = 0; i < nr_pages; i++) { 2541 struct page *page = pvec.pages[i]; 2542 2543 /* 2544 * At this point, the page may be truncated or 2545 * invalidated (changing page->mapping to NULL), or 2546 * even swizzled back from swapper_space to tmpfs file 2547 * mapping. However, page->index will not change 2548 * because we have a reference on the page. 2549 */ 2550 if (page->index > end) 2551 goto out; 2552 2553 /* 2554 * Accumulated enough dirty pages? This doesn't apply 2555 * to WB_SYNC_ALL mode. For integrity sync we have to 2556 * keep going because someone may be concurrently 2557 * dirtying pages, and we might have synced a lot of 2558 * newly appeared dirty pages, but have not synced all 2559 * of the old dirty pages. 2560 */ 2561 if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0) 2562 goto out; 2563 2564 /* If we can't merge this page, we are done. */ 2565 if (mpd->map.m_len > 0 && mpd->next_page != page->index) 2566 goto out; 2567 2568 lock_page(page); 2569 /* 2570 * If the page is no longer dirty, or its mapping no 2571 * longer corresponds to inode we are writing (which 2572 * means it has been truncated or invalidated), or the 2573 * page is already under writeback and we are not doing 2574 * a data integrity writeback, skip the page 2575 */ 2576 if (!PageDirty(page) || 2577 (PageWriteback(page) && 2578 (mpd->wbc->sync_mode == WB_SYNC_NONE)) || 2579 unlikely(page->mapping != mapping)) { 2580 unlock_page(page); 2581 continue; 2582 } 2583 2584 wait_on_page_writeback(page); 2585 BUG_ON(PageWriteback(page)); 2586 2587 if (mpd->map.m_len == 0) 2588 mpd->first_page = page->index; 2589 mpd->next_page = page->index + 1; 2590 /* Add all dirty buffers to mpd */ 2591 lblk = ((ext4_lblk_t)page->index) << 2592 (PAGE_SHIFT - blkbits); 2593 head = page_buffers(page); 2594 err = mpage_process_page_bufs(mpd, head, head, lblk); 2595 if (err <= 0) 2596 goto out; 2597 err = 0; 2598 left--; 2599 } 2600 pagevec_release(&pvec); 2601 cond_resched(); 2602 } 2603 return 0; 2604 out: 2605 pagevec_release(&pvec); 2606 return err; 2607 } 2608 2609 static int __writepage(struct page *page, struct writeback_control *wbc, 2610 void *data) 2611 { 2612 struct address_space *mapping = data; 2613 int ret = ext4_writepage(page, wbc); 2614 mapping_set_error(mapping, ret); 2615 return ret; 2616 } 2617 2618 static int ext4_writepages(struct address_space *mapping, 2619 struct writeback_control *wbc) 2620 { 2621 pgoff_t writeback_index = 0; 2622 long nr_to_write = wbc->nr_to_write; 2623 int range_whole = 0; 2624 int cycled = 1; 2625 handle_t *handle = NULL; 2626 struct mpage_da_data mpd; 2627 struct inode *inode = mapping->host; 2628 int needed_blocks, rsv_blocks = 0, ret = 0; 2629 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb); 2630 bool done; 2631 struct blk_plug plug; 2632 bool give_up_on_write = false; 2633 2634 percpu_down_read(&sbi->s_journal_flag_rwsem); 2635 trace_ext4_writepages(inode, wbc); 2636 2637 if (dax_mapping(mapping)) { 2638 ret = dax_writeback_mapping_range(mapping, inode->i_sb->s_bdev, 2639 wbc); 2640 goto out_writepages; 2641 } 2642 2643 /* 2644 * No pages to write? This is mainly a kludge to avoid starting 2645 * a transaction for special inodes like journal inode on last iput() 2646 * because that could violate lock ordering on umount 2647 */ 2648 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 2649 goto out_writepages; 2650 2651 if (ext4_should_journal_data(inode)) { 2652 struct blk_plug plug; 2653 2654 blk_start_plug(&plug); 2655 ret = write_cache_pages(mapping, wbc, __writepage, mapping); 2656 blk_finish_plug(&plug); 2657 goto out_writepages; 2658 } 2659 2660 /* 2661 * If the filesystem has aborted, it is read-only, so return 2662 * right away instead of dumping stack traces later on that 2663 * will obscure the real source of the problem. We test 2664 * EXT4_MF_FS_ABORTED instead of sb->s_flag's MS_RDONLY because 2665 * the latter could be true if the filesystem is mounted 2666 * read-only, and in that case, ext4_writepages should 2667 * *never* be called, so if that ever happens, we would want 2668 * the stack trace. 2669 */ 2670 if (unlikely(sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) { 2671 ret = -EROFS; 2672 goto out_writepages; 2673 } 2674 2675 if (ext4_should_dioread_nolock(inode)) { 2676 /* 2677 * We may need to convert up to one extent per block in 2678 * the page and we may dirty the inode. 2679 */ 2680 rsv_blocks = 1 + (PAGE_SIZE >> inode->i_blkbits); 2681 } 2682 2683 /* 2684 * If we have inline data and arrive here, it means that 2685 * we will soon create the block for the 1st page, so 2686 * we'd better clear the inline data here. 2687 */ 2688 if (ext4_has_inline_data(inode)) { 2689 /* Just inode will be modified... */ 2690 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 2691 if (IS_ERR(handle)) { 2692 ret = PTR_ERR(handle); 2693 goto out_writepages; 2694 } 2695 BUG_ON(ext4_test_inode_state(inode, 2696 EXT4_STATE_MAY_INLINE_DATA)); 2697 ext4_destroy_inline_data(handle, inode); 2698 ext4_journal_stop(handle); 2699 } 2700 2701 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2702 range_whole = 1; 2703 2704 if (wbc->range_cyclic) { 2705 writeback_index = mapping->writeback_index; 2706 if (writeback_index) 2707 cycled = 0; 2708 mpd.first_page = writeback_index; 2709 mpd.last_page = -1; 2710 } else { 2711 mpd.first_page = wbc->range_start >> PAGE_SHIFT; 2712 mpd.last_page = wbc->range_end >> PAGE_SHIFT; 2713 } 2714 2715 mpd.inode = inode; 2716 mpd.wbc = wbc; 2717 ext4_io_submit_init(&mpd.io_submit, wbc); 2718 retry: 2719 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2720 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page); 2721 done = false; 2722 blk_start_plug(&plug); 2723 while (!done && mpd.first_page <= mpd.last_page) { 2724 /* For each extent of pages we use new io_end */ 2725 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL); 2726 if (!mpd.io_submit.io_end) { 2727 ret = -ENOMEM; 2728 break; 2729 } 2730 2731 /* 2732 * We have two constraints: We find one extent to map and we 2733 * must always write out whole page (makes a difference when 2734 * blocksize < pagesize) so that we don't block on IO when we 2735 * try to write out the rest of the page. Journalled mode is 2736 * not supported by delalloc. 2737 */ 2738 BUG_ON(ext4_should_journal_data(inode)); 2739 needed_blocks = ext4_da_writepages_trans_blocks(inode); 2740 2741 /* start a new transaction */ 2742 handle = ext4_journal_start_with_reserve(inode, 2743 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks); 2744 if (IS_ERR(handle)) { 2745 ret = PTR_ERR(handle); 2746 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: " 2747 "%ld pages, ino %lu; err %d", __func__, 2748 wbc->nr_to_write, inode->i_ino, ret); 2749 /* Release allocated io_end */ 2750 ext4_put_io_end(mpd.io_submit.io_end); 2751 break; 2752 } 2753 2754 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc); 2755 ret = mpage_prepare_extent_to_map(&mpd); 2756 if (!ret) { 2757 if (mpd.map.m_len) 2758 ret = mpage_map_and_submit_extent(handle, &mpd, 2759 &give_up_on_write); 2760 else { 2761 /* 2762 * We scanned the whole range (or exhausted 2763 * nr_to_write), submitted what was mapped and 2764 * didn't find anything needing mapping. We are 2765 * done. 2766 */ 2767 done = true; 2768 } 2769 } 2770 /* 2771 * Caution: If the handle is synchronous, 2772 * ext4_journal_stop() can wait for transaction commit 2773 * to finish which may depend on writeback of pages to 2774 * complete or on page lock to be released. In that 2775 * case, we have to wait until after after we have 2776 * submitted all the IO, released page locks we hold, 2777 * and dropped io_end reference (for extent conversion 2778 * to be able to complete) before stopping the handle. 2779 */ 2780 if (!ext4_handle_valid(handle) || handle->h_sync == 0) { 2781 ext4_journal_stop(handle); 2782 handle = NULL; 2783 } 2784 /* Submit prepared bio */ 2785 ext4_io_submit(&mpd.io_submit); 2786 /* Unlock pages we didn't use */ 2787 mpage_release_unused_pages(&mpd, give_up_on_write); 2788 /* 2789 * Drop our io_end reference we got from init. We have 2790 * to be careful and use deferred io_end finishing if 2791 * we are still holding the transaction as we can 2792 * release the last reference to io_end which may end 2793 * up doing unwritten extent conversion. 2794 */ 2795 if (handle) { 2796 ext4_put_io_end_defer(mpd.io_submit.io_end); 2797 ext4_journal_stop(handle); 2798 } else 2799 ext4_put_io_end(mpd.io_submit.io_end); 2800 2801 if (ret == -ENOSPC && sbi->s_journal) { 2802 /* 2803 * Commit the transaction which would 2804 * free blocks released in the transaction 2805 * and try again 2806 */ 2807 jbd2_journal_force_commit_nested(sbi->s_journal); 2808 ret = 0; 2809 continue; 2810 } 2811 /* Fatal error - ENOMEM, EIO... */ 2812 if (ret) 2813 break; 2814 } 2815 blk_finish_plug(&plug); 2816 if (!ret && !cycled && wbc->nr_to_write > 0) { 2817 cycled = 1; 2818 mpd.last_page = writeback_index - 1; 2819 mpd.first_page = 0; 2820 goto retry; 2821 } 2822 2823 /* Update index */ 2824 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 2825 /* 2826 * Set the writeback_index so that range_cyclic 2827 * mode will write it back later 2828 */ 2829 mapping->writeback_index = mpd.first_page; 2830 2831 out_writepages: 2832 trace_ext4_writepages_result(inode, wbc, ret, 2833 nr_to_write - wbc->nr_to_write); 2834 percpu_up_read(&sbi->s_journal_flag_rwsem); 2835 return ret; 2836 } 2837 2838 static int ext4_nonda_switch(struct super_block *sb) 2839 { 2840 s64 free_clusters, dirty_clusters; 2841 struct ext4_sb_info *sbi = EXT4_SB(sb); 2842 2843 /* 2844 * switch to non delalloc mode if we are running low 2845 * on free block. The free block accounting via percpu 2846 * counters can get slightly wrong with percpu_counter_batch getting 2847 * accumulated on each CPU without updating global counters 2848 * Delalloc need an accurate free block accounting. So switch 2849 * to non delalloc when we are near to error range. 2850 */ 2851 free_clusters = 2852 percpu_counter_read_positive(&sbi->s_freeclusters_counter); 2853 dirty_clusters = 2854 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter); 2855 /* 2856 * Start pushing delalloc when 1/2 of free blocks are dirty. 2857 */ 2858 if (dirty_clusters && (free_clusters < 2 * dirty_clusters)) 2859 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE); 2860 2861 if (2 * free_clusters < 3 * dirty_clusters || 2862 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) { 2863 /* 2864 * free block count is less than 150% of dirty blocks 2865 * or free blocks is less than watermark 2866 */ 2867 return 1; 2868 } 2869 return 0; 2870 } 2871 2872 /* We always reserve for an inode update; the superblock could be there too */ 2873 static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len) 2874 { 2875 if (likely(ext4_has_feature_large_file(inode->i_sb))) 2876 return 1; 2877 2878 if (pos + len <= 0x7fffffffULL) 2879 return 1; 2880 2881 /* We might need to update the superblock to set LARGE_FILE */ 2882 return 2; 2883 } 2884 2885 static int ext4_da_write_begin(struct file *file, struct address_space *mapping, 2886 loff_t pos, unsigned len, unsigned flags, 2887 struct page **pagep, void **fsdata) 2888 { 2889 int ret, retries = 0; 2890 struct page *page; 2891 pgoff_t index; 2892 struct inode *inode = mapping->host; 2893 handle_t *handle; 2894 2895 index = pos >> PAGE_SHIFT; 2896 2897 if (ext4_nonda_switch(inode->i_sb) || 2898 S_ISLNK(inode->i_mode)) { 2899 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC; 2900 return ext4_write_begin(file, mapping, pos, 2901 len, flags, pagep, fsdata); 2902 } 2903 *fsdata = (void *)0; 2904 trace_ext4_da_write_begin(inode, pos, len, flags); 2905 2906 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 2907 ret = ext4_da_write_inline_data_begin(mapping, inode, 2908 pos, len, flags, 2909 pagep, fsdata); 2910 if (ret < 0) 2911 return ret; 2912 if (ret == 1) 2913 return 0; 2914 } 2915 2916 /* 2917 * grab_cache_page_write_begin() can take a long time if the 2918 * system is thrashing due to memory pressure, or if the page 2919 * is being written back. So grab it first before we start 2920 * the transaction handle. This also allows us to allocate 2921 * the page (if needed) without using GFP_NOFS. 2922 */ 2923 retry_grab: 2924 page = grab_cache_page_write_begin(mapping, index, flags); 2925 if (!page) 2926 return -ENOMEM; 2927 unlock_page(page); 2928 2929 /* 2930 * With delayed allocation, we don't log the i_disksize update 2931 * if there is delayed block allocation. But we still need 2932 * to journalling the i_disksize update if writes to the end 2933 * of file which has an already mapped buffer. 2934 */ 2935 retry_journal: 2936 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, 2937 ext4_da_write_credits(inode, pos, len)); 2938 if (IS_ERR(handle)) { 2939 put_page(page); 2940 return PTR_ERR(handle); 2941 } 2942 2943 lock_page(page); 2944 if (page->mapping != mapping) { 2945 /* The page got truncated from under us */ 2946 unlock_page(page); 2947 put_page(page); 2948 ext4_journal_stop(handle); 2949 goto retry_grab; 2950 } 2951 /* In case writeback began while the page was unlocked */ 2952 wait_for_stable_page(page); 2953 2954 #ifdef CONFIG_EXT4_FS_ENCRYPTION 2955 ret = ext4_block_write_begin(page, pos, len, 2956 ext4_da_get_block_prep); 2957 #else 2958 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep); 2959 #endif 2960 if (ret < 0) { 2961 unlock_page(page); 2962 ext4_journal_stop(handle); 2963 /* 2964 * block_write_begin may have instantiated a few blocks 2965 * outside i_size. Trim these off again. Don't need 2966 * i_size_read because we hold i_mutex. 2967 */ 2968 if (pos + len > inode->i_size) 2969 ext4_truncate_failed_write(inode); 2970 2971 if (ret == -ENOSPC && 2972 ext4_should_retry_alloc(inode->i_sb, &retries)) 2973 goto retry_journal; 2974 2975 put_page(page); 2976 return ret; 2977 } 2978 2979 *pagep = page; 2980 return ret; 2981 } 2982 2983 /* 2984 * Check if we should update i_disksize 2985 * when write to the end of file but not require block allocation 2986 */ 2987 static int ext4_da_should_update_i_disksize(struct page *page, 2988 unsigned long offset) 2989 { 2990 struct buffer_head *bh; 2991 struct inode *inode = page->mapping->host; 2992 unsigned int idx; 2993 int i; 2994 2995 bh = page_buffers(page); 2996 idx = offset >> inode->i_blkbits; 2997 2998 for (i = 0; i < idx; i++) 2999 bh = bh->b_this_page; 3000 3001 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh)) 3002 return 0; 3003 return 1; 3004 } 3005 3006 static int ext4_da_write_end(struct file *file, 3007 struct address_space *mapping, 3008 loff_t pos, unsigned len, unsigned copied, 3009 struct page *page, void *fsdata) 3010 { 3011 struct inode *inode = mapping->host; 3012 int ret = 0, ret2; 3013 handle_t *handle = ext4_journal_current_handle(); 3014 loff_t new_i_size; 3015 unsigned long start, end; 3016 int write_mode = (int)(unsigned long)fsdata; 3017 3018 if (write_mode == FALL_BACK_TO_NONDELALLOC) 3019 return ext4_write_end(file, mapping, pos, 3020 len, copied, page, fsdata); 3021 3022 trace_ext4_da_write_end(inode, pos, len, copied); 3023 start = pos & (PAGE_SIZE - 1); 3024 end = start + copied - 1; 3025 3026 /* 3027 * generic_write_end() will run mark_inode_dirty() if i_size 3028 * changes. So let's piggyback the i_disksize mark_inode_dirty 3029 * into that. 3030 */ 3031 new_i_size = pos + copied; 3032 if (copied && new_i_size > EXT4_I(inode)->i_disksize) { 3033 if (ext4_has_inline_data(inode) || 3034 ext4_da_should_update_i_disksize(page, end)) { 3035 ext4_update_i_disksize(inode, new_i_size); 3036 /* We need to mark inode dirty even if 3037 * new_i_size is less that inode->i_size 3038 * bu greater than i_disksize.(hint delalloc) 3039 */ 3040 ext4_mark_inode_dirty(handle, inode); 3041 } 3042 } 3043 3044 if (write_mode != CONVERT_INLINE_DATA && 3045 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && 3046 ext4_has_inline_data(inode)) 3047 ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied, 3048 page); 3049 else 3050 ret2 = generic_write_end(file, mapping, pos, len, copied, 3051 page, fsdata); 3052 3053 copied = ret2; 3054 if (ret2 < 0) 3055 ret = ret2; 3056 ret2 = ext4_journal_stop(handle); 3057 if (!ret) 3058 ret = ret2; 3059 3060 return ret ? ret : copied; 3061 } 3062 3063 static void ext4_da_invalidatepage(struct page *page, unsigned int offset, 3064 unsigned int length) 3065 { 3066 /* 3067 * Drop reserved blocks 3068 */ 3069 BUG_ON(!PageLocked(page)); 3070 if (!page_has_buffers(page)) 3071 goto out; 3072 3073 ext4_da_page_release_reservation(page, offset, length); 3074 3075 out: 3076 ext4_invalidatepage(page, offset, length); 3077 3078 return; 3079 } 3080 3081 /* 3082 * Force all delayed allocation blocks to be allocated for a given inode. 3083 */ 3084 int ext4_alloc_da_blocks(struct inode *inode) 3085 { 3086 trace_ext4_alloc_da_blocks(inode); 3087 3088 if (!EXT4_I(inode)->i_reserved_data_blocks) 3089 return 0; 3090 3091 /* 3092 * We do something simple for now. The filemap_flush() will 3093 * also start triggering a write of the data blocks, which is 3094 * not strictly speaking necessary (and for users of 3095 * laptop_mode, not even desirable). However, to do otherwise 3096 * would require replicating code paths in: 3097 * 3098 * ext4_writepages() -> 3099 * write_cache_pages() ---> (via passed in callback function) 3100 * __mpage_da_writepage() --> 3101 * mpage_add_bh_to_extent() 3102 * mpage_da_map_blocks() 3103 * 3104 * The problem is that write_cache_pages(), located in 3105 * mm/page-writeback.c, marks pages clean in preparation for 3106 * doing I/O, which is not desirable if we're not planning on 3107 * doing I/O at all. 3108 * 3109 * We could call write_cache_pages(), and then redirty all of 3110 * the pages by calling redirty_page_for_writepage() but that 3111 * would be ugly in the extreme. So instead we would need to 3112 * replicate parts of the code in the above functions, 3113 * simplifying them because we wouldn't actually intend to 3114 * write out the pages, but rather only collect contiguous 3115 * logical block extents, call the multi-block allocator, and 3116 * then update the buffer heads with the block allocations. 3117 * 3118 * For now, though, we'll cheat by calling filemap_flush(), 3119 * which will map the blocks, and start the I/O, but not 3120 * actually wait for the I/O to complete. 3121 */ 3122 return filemap_flush(inode->i_mapping); 3123 } 3124 3125 /* 3126 * bmap() is special. It gets used by applications such as lilo and by 3127 * the swapper to find the on-disk block of a specific piece of data. 3128 * 3129 * Naturally, this is dangerous if the block concerned is still in the 3130 * journal. If somebody makes a swapfile on an ext4 data-journaling 3131 * filesystem and enables swap, then they may get a nasty shock when the 3132 * data getting swapped to that swapfile suddenly gets overwritten by 3133 * the original zero's written out previously to the journal and 3134 * awaiting writeback in the kernel's buffer cache. 3135 * 3136 * So, if we see any bmap calls here on a modified, data-journaled file, 3137 * take extra steps to flush any blocks which might be in the cache. 3138 */ 3139 static sector_t ext4_bmap(struct address_space *mapping, sector_t block) 3140 { 3141 struct inode *inode = mapping->host; 3142 journal_t *journal; 3143 int err; 3144 3145 /* 3146 * We can get here for an inline file via the FIBMAP ioctl 3147 */ 3148 if (ext4_has_inline_data(inode)) 3149 return 0; 3150 3151 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && 3152 test_opt(inode->i_sb, DELALLOC)) { 3153 /* 3154 * With delalloc we want to sync the file 3155 * so that we can make sure we allocate 3156 * blocks for file 3157 */ 3158 filemap_write_and_wait(mapping); 3159 } 3160 3161 if (EXT4_JOURNAL(inode) && 3162 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) { 3163 /* 3164 * This is a REALLY heavyweight approach, but the use of 3165 * bmap on dirty files is expected to be extremely rare: 3166 * only if we run lilo or swapon on a freshly made file 3167 * do we expect this to happen. 3168 * 3169 * (bmap requires CAP_SYS_RAWIO so this does not 3170 * represent an unprivileged user DOS attack --- we'd be 3171 * in trouble if mortal users could trigger this path at 3172 * will.) 3173 * 3174 * NB. EXT4_STATE_JDATA is not set on files other than 3175 * regular files. If somebody wants to bmap a directory 3176 * or symlink and gets confused because the buffer 3177 * hasn't yet been flushed to disk, they deserve 3178 * everything they get. 3179 */ 3180 3181 ext4_clear_inode_state(inode, EXT4_STATE_JDATA); 3182 journal = EXT4_JOURNAL(inode); 3183 jbd2_journal_lock_updates(journal); 3184 err = jbd2_journal_flush(journal); 3185 jbd2_journal_unlock_updates(journal); 3186 3187 if (err) 3188 return 0; 3189 } 3190 3191 return generic_block_bmap(mapping, block, ext4_get_block); 3192 } 3193 3194 static int ext4_readpage(struct file *file, struct page *page) 3195 { 3196 int ret = -EAGAIN; 3197 struct inode *inode = page->mapping->host; 3198 3199 trace_ext4_readpage(page); 3200 3201 if (ext4_has_inline_data(inode)) 3202 ret = ext4_readpage_inline(inode, page); 3203 3204 if (ret == -EAGAIN) 3205 return ext4_mpage_readpages(page->mapping, NULL, page, 1); 3206 3207 return ret; 3208 } 3209 3210 static int 3211 ext4_readpages(struct file *file, struct address_space *mapping, 3212 struct list_head *pages, unsigned nr_pages) 3213 { 3214 struct inode *inode = mapping->host; 3215 3216 /* If the file has inline data, no need to do readpages. */ 3217 if (ext4_has_inline_data(inode)) 3218 return 0; 3219 3220 return ext4_mpage_readpages(mapping, pages, NULL, nr_pages); 3221 } 3222 3223 static void ext4_invalidatepage(struct page *page, unsigned int offset, 3224 unsigned int length) 3225 { 3226 trace_ext4_invalidatepage(page, offset, length); 3227 3228 /* No journalling happens on data buffers when this function is used */ 3229 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page))); 3230 3231 block_invalidatepage(page, offset, length); 3232 } 3233 3234 static int __ext4_journalled_invalidatepage(struct page *page, 3235 unsigned int offset, 3236 unsigned int length) 3237 { 3238 journal_t *journal = EXT4_JOURNAL(page->mapping->host); 3239 3240 trace_ext4_journalled_invalidatepage(page, offset, length); 3241 3242 /* 3243 * If it's a full truncate we just forget about the pending dirtying 3244 */ 3245 if (offset == 0 && length == PAGE_SIZE) 3246 ClearPageChecked(page); 3247 3248 return jbd2_journal_invalidatepage(journal, page, offset, length); 3249 } 3250 3251 /* Wrapper for aops... */ 3252 static void ext4_journalled_invalidatepage(struct page *page, 3253 unsigned int offset, 3254 unsigned int length) 3255 { 3256 WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0); 3257 } 3258 3259 static int ext4_releasepage(struct page *page, gfp_t wait) 3260 { 3261 journal_t *journal = EXT4_JOURNAL(page->mapping->host); 3262 3263 trace_ext4_releasepage(page); 3264 3265 /* Page has dirty journalled data -> cannot release */ 3266 if (PageChecked(page)) 3267 return 0; 3268 if (journal) 3269 return jbd2_journal_try_to_free_buffers(journal, page, wait); 3270 else 3271 return try_to_free_buffers(page); 3272 } 3273 3274 #ifdef CONFIG_FS_DAX 3275 static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, 3276 unsigned flags, struct iomap *iomap) 3277 { 3278 unsigned int blkbits = inode->i_blkbits; 3279 unsigned long first_block = offset >> blkbits; 3280 unsigned long last_block = (offset + length - 1) >> blkbits; 3281 struct ext4_map_blocks map; 3282 int ret; 3283 3284 if (WARN_ON_ONCE(ext4_has_inline_data(inode))) 3285 return -ERANGE; 3286 3287 map.m_lblk = first_block; 3288 map.m_len = last_block - first_block + 1; 3289 3290 if (!(flags & IOMAP_WRITE)) { 3291 ret = ext4_map_blocks(NULL, inode, &map, 0); 3292 } else { 3293 int dio_credits; 3294 handle_t *handle; 3295 int retries = 0; 3296 3297 /* Trim mapping request to maximum we can map at once for DIO */ 3298 if (map.m_len > DIO_MAX_BLOCKS) 3299 map.m_len = DIO_MAX_BLOCKS; 3300 dio_credits = ext4_chunk_trans_blocks(inode, map.m_len); 3301 retry: 3302 /* 3303 * Either we allocate blocks and then we don't get unwritten 3304 * extent so we have reserved enough credits, or the blocks 3305 * are already allocated and unwritten and in that case 3306 * extent conversion fits in the credits as well. 3307 */ 3308 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, 3309 dio_credits); 3310 if (IS_ERR(handle)) 3311 return PTR_ERR(handle); 3312 3313 ret = ext4_map_blocks(handle, inode, &map, 3314 EXT4_GET_BLOCKS_CREATE_ZERO); 3315 if (ret < 0) { 3316 ext4_journal_stop(handle); 3317 if (ret == -ENOSPC && 3318 ext4_should_retry_alloc(inode->i_sb, &retries)) 3319 goto retry; 3320 return ret; 3321 } 3322 3323 /* 3324 * If we added blocks beyond i_size, we need to make sure they 3325 * will get truncated if we crash before updating i_size in 3326 * ext4_iomap_end(). For faults we don't need to do that (and 3327 * even cannot because for orphan list operations inode_lock is 3328 * required) - if we happen to instantiate block beyond i_size, 3329 * it is because we race with truncate which has already added 3330 * the inode to the orphan list. 3331 */ 3332 if (!(flags & IOMAP_FAULT) && first_block + map.m_len > 3333 (i_size_read(inode) + (1 << blkbits) - 1) >> blkbits) { 3334 int err; 3335 3336 err = ext4_orphan_add(handle, inode); 3337 if (err < 0) { 3338 ext4_journal_stop(handle); 3339 return err; 3340 } 3341 } 3342 ext4_journal_stop(handle); 3343 } 3344 3345 iomap->flags = 0; 3346 iomap->bdev = inode->i_sb->s_bdev; 3347 iomap->offset = first_block << blkbits; 3348 3349 if (ret == 0) { 3350 iomap->type = IOMAP_HOLE; 3351 iomap->blkno = IOMAP_NULL_BLOCK; 3352 iomap->length = (u64)map.m_len << blkbits; 3353 } else { 3354 if (map.m_flags & EXT4_MAP_MAPPED) { 3355 iomap->type = IOMAP_MAPPED; 3356 } else if (map.m_flags & EXT4_MAP_UNWRITTEN) { 3357 iomap->type = IOMAP_UNWRITTEN; 3358 } else { 3359 WARN_ON_ONCE(1); 3360 return -EIO; 3361 } 3362 iomap->blkno = (sector_t)map.m_pblk << (blkbits - 9); 3363 iomap->length = (u64)map.m_len << blkbits; 3364 } 3365 3366 if (map.m_flags & EXT4_MAP_NEW) 3367 iomap->flags |= IOMAP_F_NEW; 3368 return 0; 3369 } 3370 3371 static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length, 3372 ssize_t written, unsigned flags, struct iomap *iomap) 3373 { 3374 int ret = 0; 3375 handle_t *handle; 3376 int blkbits = inode->i_blkbits; 3377 bool truncate = false; 3378 3379 if (!(flags & IOMAP_WRITE) || (flags & IOMAP_FAULT)) 3380 return 0; 3381 3382 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 3383 if (IS_ERR(handle)) { 3384 ret = PTR_ERR(handle); 3385 goto orphan_del; 3386 } 3387 if (ext4_update_inode_size(inode, offset + written)) 3388 ext4_mark_inode_dirty(handle, inode); 3389 /* 3390 * We may need to truncate allocated but not written blocks beyond EOF. 3391 */ 3392 if (iomap->offset + iomap->length > 3393 ALIGN(inode->i_size, 1 << blkbits)) { 3394 ext4_lblk_t written_blk, end_blk; 3395 3396 written_blk = (offset + written) >> blkbits; 3397 end_blk = (offset + length) >> blkbits; 3398 if (written_blk < end_blk && ext4_can_truncate(inode)) 3399 truncate = true; 3400 } 3401 /* 3402 * Remove inode from orphan list if we were extending a inode and 3403 * everything went fine. 3404 */ 3405 if (!truncate && inode->i_nlink && 3406 !list_empty(&EXT4_I(inode)->i_orphan)) 3407 ext4_orphan_del(handle, inode); 3408 ext4_journal_stop(handle); 3409 if (truncate) { 3410 ext4_truncate_failed_write(inode); 3411 orphan_del: 3412 /* 3413 * If truncate failed early the inode might still be on the 3414 * orphan list; we need to make sure the inode is removed from 3415 * the orphan list in that case. 3416 */ 3417 if (inode->i_nlink) 3418 ext4_orphan_del(NULL, inode); 3419 } 3420 return ret; 3421 } 3422 3423 struct iomap_ops ext4_iomap_ops = { 3424 .iomap_begin = ext4_iomap_begin, 3425 .iomap_end = ext4_iomap_end, 3426 }; 3427 3428 #endif 3429 3430 static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset, 3431 ssize_t size, void *private) 3432 { 3433 ext4_io_end_t *io_end = private; 3434 3435 /* if not async direct IO just return */ 3436 if (!io_end) 3437 return 0; 3438 3439 ext_debug("ext4_end_io_dio(): io_end 0x%p " 3440 "for inode %lu, iocb 0x%p, offset %llu, size %zd\n", 3441 io_end, io_end->inode->i_ino, iocb, offset, size); 3442 3443 /* 3444 * Error during AIO DIO. We cannot convert unwritten extents as the 3445 * data was not written. Just clear the unwritten flag and drop io_end. 3446 */ 3447 if (size <= 0) { 3448 ext4_clear_io_unwritten_flag(io_end); 3449 size = 0; 3450 } 3451 io_end->offset = offset; 3452 io_end->size = size; 3453 ext4_put_io_end(io_end); 3454 3455 return 0; 3456 } 3457 3458 /* 3459 * Handling of direct IO writes. 3460 * 3461 * For ext4 extent files, ext4 will do direct-io write even to holes, 3462 * preallocated extents, and those write extend the file, no need to 3463 * fall back to buffered IO. 3464 * 3465 * For holes, we fallocate those blocks, mark them as unwritten 3466 * If those blocks were preallocated, we mark sure they are split, but 3467 * still keep the range to write as unwritten. 3468 * 3469 * The unwritten extents will be converted to written when DIO is completed. 3470 * For async direct IO, since the IO may still pending when return, we 3471 * set up an end_io call back function, which will do the conversion 3472 * when async direct IO completed. 3473 * 3474 * If the O_DIRECT write will extend the file then add this inode to the 3475 * orphan list. So recovery will truncate it back to the original size 3476 * if the machine crashes during the write. 3477 * 3478 */ 3479 static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter) 3480 { 3481 struct file *file = iocb->ki_filp; 3482 struct inode *inode = file->f_mapping->host; 3483 struct ext4_inode_info *ei = EXT4_I(inode); 3484 ssize_t ret; 3485 loff_t offset = iocb->ki_pos; 3486 size_t count = iov_iter_count(iter); 3487 int overwrite = 0; 3488 get_block_t *get_block_func = NULL; 3489 int dio_flags = 0; 3490 loff_t final_size = offset + count; 3491 int orphan = 0; 3492 handle_t *handle; 3493 3494 if (final_size > inode->i_size) { 3495 /* Credits for sb + inode write */ 3496 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 3497 if (IS_ERR(handle)) { 3498 ret = PTR_ERR(handle); 3499 goto out; 3500 } 3501 ret = ext4_orphan_add(handle, inode); 3502 if (ret) { 3503 ext4_journal_stop(handle); 3504 goto out; 3505 } 3506 orphan = 1; 3507 ei->i_disksize = inode->i_size; 3508 ext4_journal_stop(handle); 3509 } 3510 3511 BUG_ON(iocb->private == NULL); 3512 3513 /* 3514 * Make all waiters for direct IO properly wait also for extent 3515 * conversion. This also disallows race between truncate() and 3516 * overwrite DIO as i_dio_count needs to be incremented under i_mutex. 3517 */ 3518 inode_dio_begin(inode); 3519 3520 /* If we do a overwrite dio, i_mutex locking can be released */ 3521 overwrite = *((int *)iocb->private); 3522 3523 if (overwrite) 3524 inode_unlock(inode); 3525 3526 /* 3527 * For extent mapped files we could direct write to holes and fallocate. 3528 * 3529 * Allocated blocks to fill the hole are marked as unwritten to prevent 3530 * parallel buffered read to expose the stale data before DIO complete 3531 * the data IO. 3532 * 3533 * As to previously fallocated extents, ext4 get_block will just simply 3534 * mark the buffer mapped but still keep the extents unwritten. 3535 * 3536 * For non AIO case, we will convert those unwritten extents to written 3537 * after return back from blockdev_direct_IO. That way we save us from 3538 * allocating io_end structure and also the overhead of offloading 3539 * the extent convertion to a workqueue. 3540 * 3541 * For async DIO, the conversion needs to be deferred when the 3542 * IO is completed. The ext4 end_io callback function will be 3543 * called to take care of the conversion work. Here for async 3544 * case, we allocate an io_end structure to hook to the iocb. 3545 */ 3546 iocb->private = NULL; 3547 if (overwrite) 3548 get_block_func = ext4_dio_get_block_overwrite; 3549 else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) || 3550 round_down(offset, 1 << inode->i_blkbits) >= inode->i_size) { 3551 get_block_func = ext4_dio_get_block; 3552 dio_flags = DIO_LOCKING | DIO_SKIP_HOLES; 3553 } else if (is_sync_kiocb(iocb)) { 3554 get_block_func = ext4_dio_get_block_unwritten_sync; 3555 dio_flags = DIO_LOCKING; 3556 } else { 3557 get_block_func = ext4_dio_get_block_unwritten_async; 3558 dio_flags = DIO_LOCKING; 3559 } 3560 #ifdef CONFIG_EXT4_FS_ENCRYPTION 3561 BUG_ON(ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)); 3562 #endif 3563 ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter, 3564 get_block_func, ext4_end_io_dio, NULL, 3565 dio_flags); 3566 3567 if (ret > 0 && !overwrite && ext4_test_inode_state(inode, 3568 EXT4_STATE_DIO_UNWRITTEN)) { 3569 int err; 3570 /* 3571 * for non AIO case, since the IO is already 3572 * completed, we could do the conversion right here 3573 */ 3574 err = ext4_convert_unwritten_extents(NULL, inode, 3575 offset, ret); 3576 if (err < 0) 3577 ret = err; 3578 ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN); 3579 } 3580 3581 inode_dio_end(inode); 3582 /* take i_mutex locking again if we do a ovewrite dio */ 3583 if (overwrite) 3584 inode_lock(inode); 3585 3586 if (ret < 0 && final_size > inode->i_size) 3587 ext4_truncate_failed_write(inode); 3588 3589 /* Handle extending of i_size after direct IO write */ 3590 if (orphan) { 3591 int err; 3592 3593 /* Credits for sb + inode write */ 3594 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 3595 if (IS_ERR(handle)) { 3596 /* This is really bad luck. We've written the data 3597 * but cannot extend i_size. Bail out and pretend 3598 * the write failed... */ 3599 ret = PTR_ERR(handle); 3600 if (inode->i_nlink) 3601 ext4_orphan_del(NULL, inode); 3602 3603 goto out; 3604 } 3605 if (inode->i_nlink) 3606 ext4_orphan_del(handle, inode); 3607 if (ret > 0) { 3608 loff_t end = offset + ret; 3609 if (end > inode->i_size) { 3610 ei->i_disksize = end; 3611 i_size_write(inode, end); 3612 /* 3613 * We're going to return a positive `ret' 3614 * here due to non-zero-length I/O, so there's 3615 * no way of reporting error returns from 3616 * ext4_mark_inode_dirty() to userspace. So 3617 * ignore it. 3618 */ 3619 ext4_mark_inode_dirty(handle, inode); 3620 } 3621 } 3622 err = ext4_journal_stop(handle); 3623 if (ret == 0) 3624 ret = err; 3625 } 3626 out: 3627 return ret; 3628 } 3629 3630 static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter) 3631 { 3632 struct address_space *mapping = iocb->ki_filp->f_mapping; 3633 struct inode *inode = mapping->host; 3634 size_t count = iov_iter_count(iter); 3635 ssize_t ret; 3636 3637 /* 3638 * Shared inode_lock is enough for us - it protects against concurrent 3639 * writes & truncates and since we take care of writing back page cache, 3640 * we are protected against page writeback as well. 3641 */ 3642 inode_lock_shared(inode); 3643 ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, 3644 iocb->ki_pos + count); 3645 if (ret) 3646 goto out_unlock; 3647 ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, 3648 iter, ext4_dio_get_block, NULL, NULL, 0); 3649 out_unlock: 3650 inode_unlock_shared(inode); 3651 return ret; 3652 } 3653 3654 static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 3655 { 3656 struct file *file = iocb->ki_filp; 3657 struct inode *inode = file->f_mapping->host; 3658 size_t count = iov_iter_count(iter); 3659 loff_t offset = iocb->ki_pos; 3660 ssize_t ret; 3661 3662 #ifdef CONFIG_EXT4_FS_ENCRYPTION 3663 if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 3664 return 0; 3665 #endif 3666 3667 /* 3668 * If we are doing data journalling we don't support O_DIRECT 3669 */ 3670 if (ext4_should_journal_data(inode)) 3671 return 0; 3672 3673 /* Let buffer I/O handle the inline data case. */ 3674 if (ext4_has_inline_data(inode)) 3675 return 0; 3676 3677 /* DAX uses iomap path now */ 3678 if (WARN_ON_ONCE(IS_DAX(inode))) 3679 return 0; 3680 3681 trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); 3682 if (iov_iter_rw(iter) == READ) 3683 ret = ext4_direct_IO_read(iocb, iter); 3684 else 3685 ret = ext4_direct_IO_write(iocb, iter); 3686 trace_ext4_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), ret); 3687 return ret; 3688 } 3689 3690 /* 3691 * Pages can be marked dirty completely asynchronously from ext4's journalling 3692 * activity. By filemap_sync_pte(), try_to_unmap_one(), etc. We cannot do 3693 * much here because ->set_page_dirty is called under VFS locks. The page is 3694 * not necessarily locked. 3695 * 3696 * We cannot just dirty the page and leave attached buffers clean, because the 3697 * buffers' dirty state is "definitive". We cannot just set the buffers dirty 3698 * or jbddirty because all the journalling code will explode. 3699 * 3700 * So what we do is to mark the page "pending dirty" and next time writepage 3701 * is called, propagate that into the buffers appropriately. 3702 */ 3703 static int ext4_journalled_set_page_dirty(struct page *page) 3704 { 3705 SetPageChecked(page); 3706 return __set_page_dirty_nobuffers(page); 3707 } 3708 3709 static int ext4_set_page_dirty(struct page *page) 3710 { 3711 WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page)); 3712 WARN_ON_ONCE(!page_has_buffers(page)); 3713 return __set_page_dirty_buffers(page); 3714 } 3715 3716 static const struct address_space_operations ext4_aops = { 3717 .readpage = ext4_readpage, 3718 .readpages = ext4_readpages, 3719 .writepage = ext4_writepage, 3720 .writepages = ext4_writepages, 3721 .write_begin = ext4_write_begin, 3722 .write_end = ext4_write_end, 3723 .set_page_dirty = ext4_set_page_dirty, 3724 .bmap = ext4_bmap, 3725 .invalidatepage = ext4_invalidatepage, 3726 .releasepage = ext4_releasepage, 3727 .direct_IO = ext4_direct_IO, 3728 .migratepage = buffer_migrate_page, 3729 .is_partially_uptodate = block_is_partially_uptodate, 3730 .error_remove_page = generic_error_remove_page, 3731 }; 3732 3733 static const struct address_space_operations ext4_journalled_aops = { 3734 .readpage = ext4_readpage, 3735 .readpages = ext4_readpages, 3736 .writepage = ext4_writepage, 3737 .writepages = ext4_writepages, 3738 .write_begin = ext4_write_begin, 3739 .write_end = ext4_journalled_write_end, 3740 .set_page_dirty = ext4_journalled_set_page_dirty, 3741 .bmap = ext4_bmap, 3742 .invalidatepage = ext4_journalled_invalidatepage, 3743 .releasepage = ext4_releasepage, 3744 .direct_IO = ext4_direct_IO, 3745 .is_partially_uptodate = block_is_partially_uptodate, 3746 .error_remove_page = generic_error_remove_page, 3747 }; 3748 3749 static const struct address_space_operations ext4_da_aops = { 3750 .readpage = ext4_readpage, 3751 .readpages = ext4_readpages, 3752 .writepage = ext4_writepage, 3753 .writepages = ext4_writepages, 3754 .write_begin = ext4_da_write_begin, 3755 .write_end = ext4_da_write_end, 3756 .set_page_dirty = ext4_set_page_dirty, 3757 .bmap = ext4_bmap, 3758 .invalidatepage = ext4_da_invalidatepage, 3759 .releasepage = ext4_releasepage, 3760 .direct_IO = ext4_direct_IO, 3761 .migratepage = buffer_migrate_page, 3762 .is_partially_uptodate = block_is_partially_uptodate, 3763 .error_remove_page = generic_error_remove_page, 3764 }; 3765 3766 void ext4_set_aops(struct inode *inode) 3767 { 3768 switch (ext4_inode_journal_mode(inode)) { 3769 case EXT4_INODE_ORDERED_DATA_MODE: 3770 case EXT4_INODE_WRITEBACK_DATA_MODE: 3771 break; 3772 case EXT4_INODE_JOURNAL_DATA_MODE: 3773 inode->i_mapping->a_ops = &ext4_journalled_aops; 3774 return; 3775 default: 3776 BUG(); 3777 } 3778 if (test_opt(inode->i_sb, DELALLOC)) 3779 inode->i_mapping->a_ops = &ext4_da_aops; 3780 else 3781 inode->i_mapping->a_ops = &ext4_aops; 3782 } 3783 3784 static int __ext4_block_zero_page_range(handle_t *handle, 3785 struct address_space *mapping, loff_t from, loff_t length) 3786 { 3787 ext4_fsblk_t index = from >> PAGE_SHIFT; 3788 unsigned offset = from & (PAGE_SIZE-1); 3789 unsigned blocksize, pos; 3790 ext4_lblk_t iblock; 3791 struct inode *inode = mapping->host; 3792 struct buffer_head *bh; 3793 struct page *page; 3794 int err = 0; 3795 3796 page = find_or_create_page(mapping, from >> PAGE_SHIFT, 3797 mapping_gfp_constraint(mapping, ~__GFP_FS)); 3798 if (!page) 3799 return -ENOMEM; 3800 3801 blocksize = inode->i_sb->s_blocksize; 3802 3803 iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits); 3804 3805 if (!page_has_buffers(page)) 3806 create_empty_buffers(page, blocksize, 0); 3807 3808 /* Find the buffer that contains "offset" */ 3809 bh = page_buffers(page); 3810 pos = blocksize; 3811 while (offset >= pos) { 3812 bh = bh->b_this_page; 3813 iblock++; 3814 pos += blocksize; 3815 } 3816 if (buffer_freed(bh)) { 3817 BUFFER_TRACE(bh, "freed: skip"); 3818 goto unlock; 3819 } 3820 if (!buffer_mapped(bh)) { 3821 BUFFER_TRACE(bh, "unmapped"); 3822 ext4_get_block(inode, iblock, bh, 0); 3823 /* unmapped? It's a hole - nothing to do */ 3824 if (!buffer_mapped(bh)) { 3825 BUFFER_TRACE(bh, "still unmapped"); 3826 goto unlock; 3827 } 3828 } 3829 3830 /* Ok, it's mapped. Make sure it's up-to-date */ 3831 if (PageUptodate(page)) 3832 set_buffer_uptodate(bh); 3833 3834 if (!buffer_uptodate(bh)) { 3835 err = -EIO; 3836 ll_rw_block(REQ_OP_READ, 0, 1, &bh); 3837 wait_on_buffer(bh); 3838 /* Uhhuh. Read error. Complain and punt. */ 3839 if (!buffer_uptodate(bh)) 3840 goto unlock; 3841 if (S_ISREG(inode->i_mode) && 3842 ext4_encrypted_inode(inode)) { 3843 /* We expect the key to be set. */ 3844 BUG_ON(!fscrypt_has_encryption_key(inode)); 3845 BUG_ON(blocksize != PAGE_SIZE); 3846 WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host, 3847 page, PAGE_SIZE, 0, page->index)); 3848 } 3849 } 3850 if (ext4_should_journal_data(inode)) { 3851 BUFFER_TRACE(bh, "get write access"); 3852 err = ext4_journal_get_write_access(handle, bh); 3853 if (err) 3854 goto unlock; 3855 } 3856 zero_user(page, offset, length); 3857 BUFFER_TRACE(bh, "zeroed end of block"); 3858 3859 if (ext4_should_journal_data(inode)) { 3860 err = ext4_handle_dirty_metadata(handle, inode, bh); 3861 } else { 3862 err = 0; 3863 mark_buffer_dirty(bh); 3864 if (ext4_should_order_data(inode)) 3865 err = ext4_jbd2_inode_add_write(handle, inode); 3866 } 3867 3868 unlock: 3869 unlock_page(page); 3870 put_page(page); 3871 return err; 3872 } 3873 3874 /* 3875 * ext4_block_zero_page_range() zeros out a mapping of length 'length' 3876 * starting from file offset 'from'. The range to be zero'd must 3877 * be contained with in one block. If the specified range exceeds 3878 * the end of the block it will be shortened to end of the block 3879 * that cooresponds to 'from' 3880 */ 3881 static int ext4_block_zero_page_range(handle_t *handle, 3882 struct address_space *mapping, loff_t from, loff_t length) 3883 { 3884 struct inode *inode = mapping->host; 3885 unsigned offset = from & (PAGE_SIZE-1); 3886 unsigned blocksize = inode->i_sb->s_blocksize; 3887 unsigned max = blocksize - (offset & (blocksize - 1)); 3888 3889 /* 3890 * correct length if it does not fall between 3891 * 'from' and the end of the block 3892 */ 3893 if (length > max || length < 0) 3894 length = max; 3895 3896 if (IS_DAX(inode)) { 3897 return iomap_zero_range(inode, from, length, NULL, 3898 &ext4_iomap_ops); 3899 } 3900 return __ext4_block_zero_page_range(handle, mapping, from, length); 3901 } 3902 3903 /* 3904 * ext4_block_truncate_page() zeroes out a mapping from file offset `from' 3905 * up to the end of the block which corresponds to `from'. 3906 * This required during truncate. We need to physically zero the tail end 3907 * of that block so it doesn't yield old data if the file is later grown. 3908 */ 3909 static int ext4_block_truncate_page(handle_t *handle, 3910 struct address_space *mapping, loff_t from) 3911 { 3912 unsigned offset = from & (PAGE_SIZE-1); 3913 unsigned length; 3914 unsigned blocksize; 3915 struct inode *inode = mapping->host; 3916 3917 blocksize = inode->i_sb->s_blocksize; 3918 length = blocksize - (offset & (blocksize - 1)); 3919 3920 return ext4_block_zero_page_range(handle, mapping, from, length); 3921 } 3922 3923 int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode, 3924 loff_t lstart, loff_t length) 3925 { 3926 struct super_block *sb = inode->i_sb; 3927 struct address_space *mapping = inode->i_mapping; 3928 unsigned partial_start, partial_end; 3929 ext4_fsblk_t start, end; 3930 loff_t byte_end = (lstart + length - 1); 3931 int err = 0; 3932 3933 partial_start = lstart & (sb->s_blocksize - 1); 3934 partial_end = byte_end & (sb->s_blocksize - 1); 3935 3936 start = lstart >> sb->s_blocksize_bits; 3937 end = byte_end >> sb->s_blocksize_bits; 3938 3939 /* Handle partial zero within the single block */ 3940 if (start == end && 3941 (partial_start || (partial_end != sb->s_blocksize - 1))) { 3942 err = ext4_block_zero_page_range(handle, mapping, 3943 lstart, length); 3944 return err; 3945 } 3946 /* Handle partial zero out on the start of the range */ 3947 if (partial_start) { 3948 err = ext4_block_zero_page_range(handle, mapping, 3949 lstart, sb->s_blocksize); 3950 if (err) 3951 return err; 3952 } 3953 /* Handle partial zero out on the end of the range */ 3954 if (partial_end != sb->s_blocksize - 1) 3955 err = ext4_block_zero_page_range(handle, mapping, 3956 byte_end - partial_end, 3957 partial_end + 1); 3958 return err; 3959 } 3960 3961 int ext4_can_truncate(struct inode *inode) 3962 { 3963 if (S_ISREG(inode->i_mode)) 3964 return 1; 3965 if (S_ISDIR(inode->i_mode)) 3966 return 1; 3967 if (S_ISLNK(inode->i_mode)) 3968 return !ext4_inode_is_fast_symlink(inode); 3969 return 0; 3970 } 3971 3972 /* 3973 * We have to make sure i_disksize gets properly updated before we truncate 3974 * page cache due to hole punching or zero range. Otherwise i_disksize update 3975 * can get lost as it may have been postponed to submission of writeback but 3976 * that will never happen after we truncate page cache. 3977 */ 3978 int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, 3979 loff_t len) 3980 { 3981 handle_t *handle; 3982 loff_t size = i_size_read(inode); 3983 3984 WARN_ON(!inode_is_locked(inode)); 3985 if (offset > size || offset + len < size) 3986 return 0; 3987 3988 if (EXT4_I(inode)->i_disksize >= size) 3989 return 0; 3990 3991 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); 3992 if (IS_ERR(handle)) 3993 return PTR_ERR(handle); 3994 ext4_update_i_disksize(inode, size); 3995 ext4_mark_inode_dirty(handle, inode); 3996 ext4_journal_stop(handle); 3997 3998 return 0; 3999 } 4000 4001 /* 4002 * ext4_punch_hole: punches a hole in a file by releasing the blocks 4003 * associated with the given offset and length 4004 * 4005 * @inode: File inode 4006 * @offset: The offset where the hole will begin 4007 * @len: The length of the hole 4008 * 4009 * Returns: 0 on success or negative on failure 4010 */ 4011 4012 int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length) 4013 { 4014 struct super_block *sb = inode->i_sb; 4015 ext4_lblk_t first_block, stop_block; 4016 struct address_space *mapping = inode->i_mapping; 4017 loff_t first_block_offset, last_block_offset; 4018 handle_t *handle; 4019 unsigned int credits; 4020 int ret = 0; 4021 4022 if (!S_ISREG(inode->i_mode)) 4023 return -EOPNOTSUPP; 4024 4025 trace_ext4_punch_hole(inode, offset, length, 0); 4026 4027 /* 4028 * Write out all dirty pages to avoid race conditions 4029 * Then release them. 4030 */ 4031 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) { 4032 ret = filemap_write_and_wait_range(mapping, offset, 4033 offset + length - 1); 4034 if (ret) 4035 return ret; 4036 } 4037 4038 inode_lock(inode); 4039 4040 /* No need to punch hole beyond i_size */ 4041 if (offset >= inode->i_size) 4042 goto out_mutex; 4043 4044 /* 4045 * If the hole extends beyond i_size, set the hole 4046 * to end after the page that contains i_size 4047 */ 4048 if (offset + length > inode->i_size) { 4049 length = inode->i_size + 4050 PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) - 4051 offset; 4052 } 4053 4054 if (offset & (sb->s_blocksize - 1) || 4055 (offset + length) & (sb->s_blocksize - 1)) { 4056 /* 4057 * Attach jinode to inode for jbd2 if we do any zeroing of 4058 * partial block 4059 */ 4060 ret = ext4_inode_attach_jinode(inode); 4061 if (ret < 0) 4062 goto out_mutex; 4063 4064 } 4065 4066 /* Wait all existing dio workers, newcomers will block on i_mutex */ 4067 ext4_inode_block_unlocked_dio(inode); 4068 inode_dio_wait(inode); 4069 4070 /* 4071 * Prevent page faults from reinstantiating pages we have released from 4072 * page cache. 4073 */ 4074 down_write(&EXT4_I(inode)->i_mmap_sem); 4075 first_block_offset = round_up(offset, sb->s_blocksize); 4076 last_block_offset = round_down((offset + length), sb->s_blocksize) - 1; 4077 4078 /* Now release the pages and zero block aligned part of pages*/ 4079 if (last_block_offset > first_block_offset) { 4080 ret = ext4_update_disksize_before_punch(inode, offset, length); 4081 if (ret) 4082 goto out_dio; 4083 truncate_pagecache_range(inode, first_block_offset, 4084 last_block_offset); 4085 } 4086 4087 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 4088 credits = ext4_writepage_trans_blocks(inode); 4089 else 4090 credits = ext4_blocks_for_truncate(inode); 4091 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 4092 if (IS_ERR(handle)) { 4093 ret = PTR_ERR(handle); 4094 ext4_std_error(sb, ret); 4095 goto out_dio; 4096 } 4097 4098 ret = ext4_zero_partial_blocks(handle, inode, offset, 4099 length); 4100 if (ret) 4101 goto out_stop; 4102 4103 first_block = (offset + sb->s_blocksize - 1) >> 4104 EXT4_BLOCK_SIZE_BITS(sb); 4105 stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb); 4106 4107 /* If there are no blocks to remove, return now */ 4108 if (first_block >= stop_block) 4109 goto out_stop; 4110 4111 down_write(&EXT4_I(inode)->i_data_sem); 4112 ext4_discard_preallocations(inode); 4113 4114 ret = ext4_es_remove_extent(inode, first_block, 4115 stop_block - first_block); 4116 if (ret) { 4117 up_write(&EXT4_I(inode)->i_data_sem); 4118 goto out_stop; 4119 } 4120 4121 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 4122 ret = ext4_ext_remove_space(inode, first_block, 4123 stop_block - 1); 4124 else 4125 ret = ext4_ind_remove_space(handle, inode, first_block, 4126 stop_block); 4127 4128 up_write(&EXT4_I(inode)->i_data_sem); 4129 if (IS_SYNC(inode)) 4130 ext4_handle_sync(handle); 4131 4132 inode->i_mtime = inode->i_ctime = current_time(inode); 4133 ext4_mark_inode_dirty(handle, inode); 4134 out_stop: 4135 ext4_journal_stop(handle); 4136 out_dio: 4137 up_write(&EXT4_I(inode)->i_mmap_sem); 4138 ext4_inode_resume_unlocked_dio(inode); 4139 out_mutex: 4140 inode_unlock(inode); 4141 return ret; 4142 } 4143 4144 int ext4_inode_attach_jinode(struct inode *inode) 4145 { 4146 struct ext4_inode_info *ei = EXT4_I(inode); 4147 struct jbd2_inode *jinode; 4148 4149 if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal) 4150 return 0; 4151 4152 jinode = jbd2_alloc_inode(GFP_KERNEL); 4153 spin_lock(&inode->i_lock); 4154 if (!ei->jinode) { 4155 if (!jinode) { 4156 spin_unlock(&inode->i_lock); 4157 return -ENOMEM; 4158 } 4159 ei->jinode = jinode; 4160 jbd2_journal_init_jbd_inode(ei->jinode, inode); 4161 jinode = NULL; 4162 } 4163 spin_unlock(&inode->i_lock); 4164 if (unlikely(jinode != NULL)) 4165 jbd2_free_inode(jinode); 4166 return 0; 4167 } 4168 4169 /* 4170 * ext4_truncate() 4171 * 4172 * We block out ext4_get_block() block instantiations across the entire 4173 * transaction, and VFS/VM ensures that ext4_truncate() cannot run 4174 * simultaneously on behalf of the same inode. 4175 * 4176 * As we work through the truncate and commit bits of it to the journal there 4177 * is one core, guiding principle: the file's tree must always be consistent on 4178 * disk. We must be able to restart the truncate after a crash. 4179 * 4180 * The file's tree may be transiently inconsistent in memory (although it 4181 * probably isn't), but whenever we close off and commit a journal transaction, 4182 * the contents of (the filesystem + the journal) must be consistent and 4183 * restartable. It's pretty simple, really: bottom up, right to left (although 4184 * left-to-right works OK too). 4185 * 4186 * Note that at recovery time, journal replay occurs *before* the restart of 4187 * truncate against the orphan inode list. 4188 * 4189 * The committed inode has the new, desired i_size (which is the same as 4190 * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see 4191 * that this inode's truncate did not complete and it will again call 4192 * ext4_truncate() to have another go. So there will be instantiated blocks 4193 * to the right of the truncation point in a crashed ext4 filesystem. But 4194 * that's fine - as long as they are linked from the inode, the post-crash 4195 * ext4_truncate() run will find them and release them. 4196 */ 4197 int ext4_truncate(struct inode *inode) 4198 { 4199 struct ext4_inode_info *ei = EXT4_I(inode); 4200 unsigned int credits; 4201 int err = 0; 4202 handle_t *handle; 4203 struct address_space *mapping = inode->i_mapping; 4204 4205 /* 4206 * There is a possibility that we're either freeing the inode 4207 * or it's a completely new inode. In those cases we might not 4208 * have i_mutex locked because it's not necessary. 4209 */ 4210 if (!(inode->i_state & (I_NEW|I_FREEING))) 4211 WARN_ON(!inode_is_locked(inode)); 4212 trace_ext4_truncate_enter(inode); 4213 4214 if (!ext4_can_truncate(inode)) 4215 return 0; 4216 4217 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS); 4218 4219 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC)) 4220 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE); 4221 4222 if (ext4_has_inline_data(inode)) { 4223 int has_inline = 1; 4224 4225 ext4_inline_data_truncate(inode, &has_inline); 4226 if (has_inline) 4227 return 0; 4228 } 4229 4230 /* If we zero-out tail of the page, we have to create jinode for jbd2 */ 4231 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) { 4232 if (ext4_inode_attach_jinode(inode) < 0) 4233 return 0; 4234 } 4235 4236 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 4237 credits = ext4_writepage_trans_blocks(inode); 4238 else 4239 credits = ext4_blocks_for_truncate(inode); 4240 4241 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 4242 if (IS_ERR(handle)) 4243 return PTR_ERR(handle); 4244 4245 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) 4246 ext4_block_truncate_page(handle, mapping, inode->i_size); 4247 4248 /* 4249 * We add the inode to the orphan list, so that if this 4250 * truncate spans multiple transactions, and we crash, we will 4251 * resume the truncate when the filesystem recovers. It also 4252 * marks the inode dirty, to catch the new size. 4253 * 4254 * Implication: the file must always be in a sane, consistent 4255 * truncatable state while each transaction commits. 4256 */ 4257 err = ext4_orphan_add(handle, inode); 4258 if (err) 4259 goto out_stop; 4260 4261 down_write(&EXT4_I(inode)->i_data_sem); 4262 4263 ext4_discard_preallocations(inode); 4264 4265 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 4266 err = ext4_ext_truncate(handle, inode); 4267 else 4268 ext4_ind_truncate(handle, inode); 4269 4270 up_write(&ei->i_data_sem); 4271 if (err) 4272 goto out_stop; 4273 4274 if (IS_SYNC(inode)) 4275 ext4_handle_sync(handle); 4276 4277 out_stop: 4278 /* 4279 * If this was a simple ftruncate() and the file will remain alive, 4280 * then we need to clear up the orphan record which we created above. 4281 * However, if this was a real unlink then we were called by 4282 * ext4_evict_inode(), and we allow that function to clean up the 4283 * orphan info for us. 4284 */ 4285 if (inode->i_nlink) 4286 ext4_orphan_del(handle, inode); 4287 4288 inode->i_mtime = inode->i_ctime = current_time(inode); 4289 ext4_mark_inode_dirty(handle, inode); 4290 ext4_journal_stop(handle); 4291 4292 trace_ext4_truncate_exit(inode); 4293 return err; 4294 } 4295 4296 /* 4297 * ext4_get_inode_loc returns with an extra refcount against the inode's 4298 * underlying buffer_head on success. If 'in_mem' is true, we have all 4299 * data in memory that is needed to recreate the on-disk version of this 4300 * inode. 4301 */ 4302 static int __ext4_get_inode_loc(struct inode *inode, 4303 struct ext4_iloc *iloc, int in_mem) 4304 { 4305 struct ext4_group_desc *gdp; 4306 struct buffer_head *bh; 4307 struct super_block *sb = inode->i_sb; 4308 ext4_fsblk_t block; 4309 int inodes_per_block, inode_offset; 4310 4311 iloc->bh = NULL; 4312 if (!ext4_valid_inum(sb, inode->i_ino)) 4313 return -EFSCORRUPTED; 4314 4315 iloc->block_group = (inode->i_ino - 1) / EXT4_INODES_PER_GROUP(sb); 4316 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); 4317 if (!gdp) 4318 return -EIO; 4319 4320 /* 4321 * Figure out the offset within the block group inode table 4322 */ 4323 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 4324 inode_offset = ((inode->i_ino - 1) % 4325 EXT4_INODES_PER_GROUP(sb)); 4326 block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block); 4327 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); 4328 4329 bh = sb_getblk(sb, block); 4330 if (unlikely(!bh)) 4331 return -ENOMEM; 4332 if (!buffer_uptodate(bh)) { 4333 lock_buffer(bh); 4334 4335 /* 4336 * If the buffer has the write error flag, we have failed 4337 * to write out another inode in the same block. In this 4338 * case, we don't have to read the block because we may 4339 * read the old inode data successfully. 4340 */ 4341 if (buffer_write_io_error(bh) && !buffer_uptodate(bh)) 4342 set_buffer_uptodate(bh); 4343 4344 if (buffer_uptodate(bh)) { 4345 /* someone brought it uptodate while we waited */ 4346 unlock_buffer(bh); 4347 goto has_buffer; 4348 } 4349 4350 /* 4351 * If we have all information of the inode in memory and this 4352 * is the only valid inode in the block, we need not read the 4353 * block. 4354 */ 4355 if (in_mem) { 4356 struct buffer_head *bitmap_bh; 4357 int i, start; 4358 4359 start = inode_offset & ~(inodes_per_block - 1); 4360 4361 /* Is the inode bitmap in cache? */ 4362 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp)); 4363 if (unlikely(!bitmap_bh)) 4364 goto make_io; 4365 4366 /* 4367 * If the inode bitmap isn't in cache then the 4368 * optimisation may end up performing two reads instead 4369 * of one, so skip it. 4370 */ 4371 if (!buffer_uptodate(bitmap_bh)) { 4372 brelse(bitmap_bh); 4373 goto make_io; 4374 } 4375 for (i = start; i < start + inodes_per_block; i++) { 4376 if (i == inode_offset) 4377 continue; 4378 if (ext4_test_bit(i, bitmap_bh->b_data)) 4379 break; 4380 } 4381 brelse(bitmap_bh); 4382 if (i == start + inodes_per_block) { 4383 /* all other inodes are free, so skip I/O */ 4384 memset(bh->b_data, 0, bh->b_size); 4385 set_buffer_uptodate(bh); 4386 unlock_buffer(bh); 4387 goto has_buffer; 4388 } 4389 } 4390 4391 make_io: 4392 /* 4393 * If we need to do any I/O, try to pre-readahead extra 4394 * blocks from the inode table. 4395 */ 4396 if (EXT4_SB(sb)->s_inode_readahead_blks) { 4397 ext4_fsblk_t b, end, table; 4398 unsigned num; 4399 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks; 4400 4401 table = ext4_inode_table(sb, gdp); 4402 /* s_inode_readahead_blks is always a power of 2 */ 4403 b = block & ~((ext4_fsblk_t) ra_blks - 1); 4404 if (table > b) 4405 b = table; 4406 end = b + ra_blks; 4407 num = EXT4_INODES_PER_GROUP(sb); 4408 if (ext4_has_group_desc_csum(sb)) 4409 num -= ext4_itable_unused_count(sb, gdp); 4410 table += num / inodes_per_block; 4411 if (end > table) 4412 end = table; 4413 while (b <= end) 4414 sb_breadahead(sb, b++); 4415 } 4416 4417 /* 4418 * There are other valid inodes in the buffer, this inode 4419 * has in-inode xattrs, or we don't have this inode in memory. 4420 * Read the block from disk. 4421 */ 4422 trace_ext4_load_inode(inode); 4423 get_bh(bh); 4424 bh->b_end_io = end_buffer_read_sync; 4425 submit_bh(REQ_OP_READ, REQ_META | REQ_PRIO, bh); 4426 wait_on_buffer(bh); 4427 if (!buffer_uptodate(bh)) { 4428 EXT4_ERROR_INODE_BLOCK(inode, block, 4429 "unable to read itable block"); 4430 brelse(bh); 4431 return -EIO; 4432 } 4433 } 4434 has_buffer: 4435 iloc->bh = bh; 4436 return 0; 4437 } 4438 4439 int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) 4440 { 4441 /* We have all inode data except xattrs in memory here. */ 4442 return __ext4_get_inode_loc(inode, iloc, 4443 !ext4_test_inode_state(inode, EXT4_STATE_XATTR)); 4444 } 4445 4446 void ext4_set_inode_flags(struct inode *inode) 4447 { 4448 unsigned int flags = EXT4_I(inode)->i_flags; 4449 unsigned int new_fl = 0; 4450 4451 if (flags & EXT4_SYNC_FL) 4452 new_fl |= S_SYNC; 4453 if (flags & EXT4_APPEND_FL) 4454 new_fl |= S_APPEND; 4455 if (flags & EXT4_IMMUTABLE_FL) 4456 new_fl |= S_IMMUTABLE; 4457 if (flags & EXT4_NOATIME_FL) 4458 new_fl |= S_NOATIME; 4459 if (flags & EXT4_DIRSYNC_FL) 4460 new_fl |= S_DIRSYNC; 4461 if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode) && 4462 !ext4_should_journal_data(inode) && !ext4_has_inline_data(inode) && 4463 !ext4_encrypted_inode(inode)) 4464 new_fl |= S_DAX; 4465 inode_set_flags(inode, new_fl, 4466 S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX); 4467 } 4468 4469 /* Propagate flags from i_flags to EXT4_I(inode)->i_flags */ 4470 void ext4_get_inode_flags(struct ext4_inode_info *ei) 4471 { 4472 unsigned int vfs_fl; 4473 unsigned long old_fl, new_fl; 4474 4475 do { 4476 vfs_fl = ei->vfs_inode.i_flags; 4477 old_fl = ei->i_flags; 4478 new_fl = old_fl & ~(EXT4_SYNC_FL|EXT4_APPEND_FL| 4479 EXT4_IMMUTABLE_FL|EXT4_NOATIME_FL| 4480 EXT4_DIRSYNC_FL); 4481 if (vfs_fl & S_SYNC) 4482 new_fl |= EXT4_SYNC_FL; 4483 if (vfs_fl & S_APPEND) 4484 new_fl |= EXT4_APPEND_FL; 4485 if (vfs_fl & S_IMMUTABLE) 4486 new_fl |= EXT4_IMMUTABLE_FL; 4487 if (vfs_fl & S_NOATIME) 4488 new_fl |= EXT4_NOATIME_FL; 4489 if (vfs_fl & S_DIRSYNC) 4490 new_fl |= EXT4_DIRSYNC_FL; 4491 } while (cmpxchg(&ei->i_flags, old_fl, new_fl) != old_fl); 4492 } 4493 4494 static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode, 4495 struct ext4_inode_info *ei) 4496 { 4497 blkcnt_t i_blocks ; 4498 struct inode *inode = &(ei->vfs_inode); 4499 struct super_block *sb = inode->i_sb; 4500 4501 if (ext4_has_feature_huge_file(sb)) { 4502 /* we are using combined 48 bit field */ 4503 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 | 4504 le32_to_cpu(raw_inode->i_blocks_lo); 4505 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) { 4506 /* i_blocks represent file system block size */ 4507 return i_blocks << (inode->i_blkbits - 9); 4508 } else { 4509 return i_blocks; 4510 } 4511 } else { 4512 return le32_to_cpu(raw_inode->i_blocks_lo); 4513 } 4514 } 4515 4516 static inline void ext4_iget_extra_inode(struct inode *inode, 4517 struct ext4_inode *raw_inode, 4518 struct ext4_inode_info *ei) 4519 { 4520 __le32 *magic = (void *)raw_inode + 4521 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize; 4522 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <= 4523 EXT4_INODE_SIZE(inode->i_sb) && 4524 *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) { 4525 ext4_set_inode_state(inode, EXT4_STATE_XATTR); 4526 ext4_find_inline_data_nolock(inode); 4527 } else 4528 EXT4_I(inode)->i_inline_off = 0; 4529 } 4530 4531 int ext4_get_projid(struct inode *inode, kprojid_t *projid) 4532 { 4533 if (!ext4_has_feature_project(inode->i_sb)) 4534 return -EOPNOTSUPP; 4535 *projid = EXT4_I(inode)->i_projid; 4536 return 0; 4537 } 4538 4539 struct inode *ext4_iget(struct super_block *sb, unsigned long ino) 4540 { 4541 struct ext4_iloc iloc; 4542 struct ext4_inode *raw_inode; 4543 struct ext4_inode_info *ei; 4544 struct inode *inode; 4545 journal_t *journal = EXT4_SB(sb)->s_journal; 4546 long ret; 4547 loff_t size; 4548 int block; 4549 uid_t i_uid; 4550 gid_t i_gid; 4551 projid_t i_projid; 4552 4553 inode = iget_locked(sb, ino); 4554 if (!inode) 4555 return ERR_PTR(-ENOMEM); 4556 if (!(inode->i_state & I_NEW)) 4557 return inode; 4558 4559 ei = EXT4_I(inode); 4560 iloc.bh = NULL; 4561 4562 ret = __ext4_get_inode_loc(inode, &iloc, 0); 4563 if (ret < 0) 4564 goto bad_inode; 4565 raw_inode = ext4_raw_inode(&iloc); 4566 4567 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 4568 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize); 4569 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > 4570 EXT4_INODE_SIZE(inode->i_sb) || 4571 (ei->i_extra_isize & 3)) { 4572 EXT4_ERROR_INODE(inode, 4573 "bad extra_isize %u (inode size %u)", 4574 ei->i_extra_isize, 4575 EXT4_INODE_SIZE(inode->i_sb)); 4576 ret = -EFSCORRUPTED; 4577 goto bad_inode; 4578 } 4579 } else 4580 ei->i_extra_isize = 0; 4581 4582 /* Precompute checksum seed for inode metadata */ 4583 if (ext4_has_metadata_csum(sb)) { 4584 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 4585 __u32 csum; 4586 __le32 inum = cpu_to_le32(inode->i_ino); 4587 __le32 gen = raw_inode->i_generation; 4588 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum, 4589 sizeof(inum)); 4590 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen, 4591 sizeof(gen)); 4592 } 4593 4594 if (!ext4_inode_csum_verify(inode, raw_inode, ei)) { 4595 EXT4_ERROR_INODE(inode, "checksum invalid"); 4596 ret = -EFSBADCRC; 4597 goto bad_inode; 4598 } 4599 4600 inode->i_mode = le16_to_cpu(raw_inode->i_mode); 4601 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low); 4602 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low); 4603 if (ext4_has_feature_project(sb) && 4604 EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE && 4605 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) 4606 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid); 4607 else 4608 i_projid = EXT4_DEF_PROJID; 4609 4610 if (!(test_opt(inode->i_sb, NO_UID32))) { 4611 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16; 4612 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16; 4613 } 4614 i_uid_write(inode, i_uid); 4615 i_gid_write(inode, i_gid); 4616 ei->i_projid = make_kprojid(&init_user_ns, i_projid); 4617 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); 4618 4619 ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */ 4620 ei->i_inline_off = 0; 4621 ei->i_dir_start_lookup = 0; 4622 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); 4623 /* We now have enough fields to check if the inode was active or not. 4624 * This is needed because nfsd might try to access dead inodes 4625 * the test is that same one that e2fsck uses 4626 * NeilBrown 1999oct15 4627 */ 4628 if (inode->i_nlink == 0) { 4629 if ((inode->i_mode == 0 || 4630 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) && 4631 ino != EXT4_BOOT_LOADER_INO) { 4632 /* this inode is deleted */ 4633 ret = -ESTALE; 4634 goto bad_inode; 4635 } 4636 /* The only unlinked inodes we let through here have 4637 * valid i_mode and are being read by the orphan 4638 * recovery code: that's fine, we're about to complete 4639 * the process of deleting those. 4640 * OR it is the EXT4_BOOT_LOADER_INO which is 4641 * not initialized on a new filesystem. */ 4642 } 4643 ei->i_flags = le32_to_cpu(raw_inode->i_flags); 4644 inode->i_blocks = ext4_inode_blocks(raw_inode, ei); 4645 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo); 4646 if (ext4_has_feature_64bit(sb)) 4647 ei->i_file_acl |= 4648 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; 4649 inode->i_size = ext4_isize(raw_inode); 4650 if ((size = i_size_read(inode)) < 0) { 4651 EXT4_ERROR_INODE(inode, "bad i_size value: %lld", size); 4652 ret = -EFSCORRUPTED; 4653 goto bad_inode; 4654 } 4655 ei->i_disksize = inode->i_size; 4656 #ifdef CONFIG_QUOTA 4657 ei->i_reserved_quota = 0; 4658 #endif 4659 inode->i_generation = le32_to_cpu(raw_inode->i_generation); 4660 ei->i_block_group = iloc.block_group; 4661 ei->i_last_alloc_group = ~0; 4662 /* 4663 * NOTE! The in-memory inode i_data array is in little-endian order 4664 * even on big-endian machines: we do NOT byteswap the block numbers! 4665 */ 4666 for (block = 0; block < EXT4_N_BLOCKS; block++) 4667 ei->i_data[block] = raw_inode->i_block[block]; 4668 INIT_LIST_HEAD(&ei->i_orphan); 4669 4670 /* 4671 * Set transaction id's of transactions that have to be committed 4672 * to finish f[data]sync. We set them to currently running transaction 4673 * as we cannot be sure that the inode or some of its metadata isn't 4674 * part of the transaction - the inode could have been reclaimed and 4675 * now it is reread from disk. 4676 */ 4677 if (journal) { 4678 transaction_t *transaction; 4679 tid_t tid; 4680 4681 read_lock(&journal->j_state_lock); 4682 if (journal->j_running_transaction) 4683 transaction = journal->j_running_transaction; 4684 else 4685 transaction = journal->j_committing_transaction; 4686 if (transaction) 4687 tid = transaction->t_tid; 4688 else 4689 tid = journal->j_commit_sequence; 4690 read_unlock(&journal->j_state_lock); 4691 ei->i_sync_tid = tid; 4692 ei->i_datasync_tid = tid; 4693 } 4694 4695 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 4696 if (ei->i_extra_isize == 0) { 4697 /* The extra space is currently unused. Use it. */ 4698 BUILD_BUG_ON(sizeof(struct ext4_inode) & 3); 4699 ei->i_extra_isize = sizeof(struct ext4_inode) - 4700 EXT4_GOOD_OLD_INODE_SIZE; 4701 } else { 4702 ext4_iget_extra_inode(inode, raw_inode, ei); 4703 } 4704 } 4705 4706 EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode); 4707 EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode); 4708 EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode); 4709 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode); 4710 4711 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { 4712 inode->i_version = le32_to_cpu(raw_inode->i_disk_version); 4713 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 4714 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 4715 inode->i_version |= 4716 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32; 4717 } 4718 } 4719 4720 ret = 0; 4721 if (ei->i_file_acl && 4722 !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) { 4723 EXT4_ERROR_INODE(inode, "bad extended attribute block %llu", 4724 ei->i_file_acl); 4725 ret = -EFSCORRUPTED; 4726 goto bad_inode; 4727 } else if (!ext4_has_inline_data(inode)) { 4728 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 4729 if ((S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || 4730 (S_ISLNK(inode->i_mode) && 4731 !ext4_inode_is_fast_symlink(inode)))) 4732 /* Validate extent which is part of inode */ 4733 ret = ext4_ext_check_inode(inode); 4734 } else if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || 4735 (S_ISLNK(inode->i_mode) && 4736 !ext4_inode_is_fast_symlink(inode))) { 4737 /* Validate block references which are part of inode */ 4738 ret = ext4_ind_check_inode(inode); 4739 } 4740 } 4741 if (ret) 4742 goto bad_inode; 4743 4744 if (S_ISREG(inode->i_mode)) { 4745 inode->i_op = &ext4_file_inode_operations; 4746 inode->i_fop = &ext4_file_operations; 4747 ext4_set_aops(inode); 4748 } else if (S_ISDIR(inode->i_mode)) { 4749 inode->i_op = &ext4_dir_inode_operations; 4750 inode->i_fop = &ext4_dir_operations; 4751 } else if (S_ISLNK(inode->i_mode)) { 4752 if (ext4_encrypted_inode(inode)) { 4753 inode->i_op = &ext4_encrypted_symlink_inode_operations; 4754 ext4_set_aops(inode); 4755 } else if (ext4_inode_is_fast_symlink(inode)) { 4756 inode->i_link = (char *)ei->i_data; 4757 inode->i_op = &ext4_fast_symlink_inode_operations; 4758 nd_terminate_link(ei->i_data, inode->i_size, 4759 sizeof(ei->i_data) - 1); 4760 } else { 4761 inode->i_op = &ext4_symlink_inode_operations; 4762 ext4_set_aops(inode); 4763 } 4764 inode_nohighmem(inode); 4765 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || 4766 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { 4767 inode->i_op = &ext4_special_inode_operations; 4768 if (raw_inode->i_block[0]) 4769 init_special_inode(inode, inode->i_mode, 4770 old_decode_dev(le32_to_cpu(raw_inode->i_block[0]))); 4771 else 4772 init_special_inode(inode, inode->i_mode, 4773 new_decode_dev(le32_to_cpu(raw_inode->i_block[1]))); 4774 } else if (ino == EXT4_BOOT_LOADER_INO) { 4775 make_bad_inode(inode); 4776 } else { 4777 ret = -EFSCORRUPTED; 4778 EXT4_ERROR_INODE(inode, "bogus i_mode (%o)", inode->i_mode); 4779 goto bad_inode; 4780 } 4781 brelse(iloc.bh); 4782 ext4_set_inode_flags(inode); 4783 unlock_new_inode(inode); 4784 return inode; 4785 4786 bad_inode: 4787 brelse(iloc.bh); 4788 iget_failed(inode); 4789 return ERR_PTR(ret); 4790 } 4791 4792 struct inode *ext4_iget_normal(struct super_block *sb, unsigned long ino) 4793 { 4794 if (ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) 4795 return ERR_PTR(-EFSCORRUPTED); 4796 return ext4_iget(sb, ino); 4797 } 4798 4799 static int ext4_inode_blocks_set(handle_t *handle, 4800 struct ext4_inode *raw_inode, 4801 struct ext4_inode_info *ei) 4802 { 4803 struct inode *inode = &(ei->vfs_inode); 4804 u64 i_blocks = inode->i_blocks; 4805 struct super_block *sb = inode->i_sb; 4806 4807 if (i_blocks <= ~0U) { 4808 /* 4809 * i_blocks can be represented in a 32 bit variable 4810 * as multiple of 512 bytes 4811 */ 4812 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4813 raw_inode->i_blocks_high = 0; 4814 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4815 return 0; 4816 } 4817 if (!ext4_has_feature_huge_file(sb)) 4818 return -EFBIG; 4819 4820 if (i_blocks <= 0xffffffffffffULL) { 4821 /* 4822 * i_blocks can be represented in a 48 bit variable 4823 * as multiple of 512 bytes 4824 */ 4825 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4826 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 4827 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4828 } else { 4829 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE); 4830 /* i_block is stored in file system block size */ 4831 i_blocks = i_blocks >> (inode->i_blkbits - 9); 4832 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 4833 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 4834 } 4835 return 0; 4836 } 4837 4838 struct other_inode { 4839 unsigned long orig_ino; 4840 struct ext4_inode *raw_inode; 4841 }; 4842 4843 static int other_inode_match(struct inode * inode, unsigned long ino, 4844 void *data) 4845 { 4846 struct other_inode *oi = (struct other_inode *) data; 4847 4848 if ((inode->i_ino != ino) || 4849 (inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW | 4850 I_DIRTY_SYNC | I_DIRTY_DATASYNC)) || 4851 ((inode->i_state & I_DIRTY_TIME) == 0)) 4852 return 0; 4853 spin_lock(&inode->i_lock); 4854 if (((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW | 4855 I_DIRTY_SYNC | I_DIRTY_DATASYNC)) == 0) && 4856 (inode->i_state & I_DIRTY_TIME)) { 4857 struct ext4_inode_info *ei = EXT4_I(inode); 4858 4859 inode->i_state &= ~(I_DIRTY_TIME | I_DIRTY_TIME_EXPIRED); 4860 spin_unlock(&inode->i_lock); 4861 4862 spin_lock(&ei->i_raw_lock); 4863 EXT4_INODE_SET_XTIME(i_ctime, inode, oi->raw_inode); 4864 EXT4_INODE_SET_XTIME(i_mtime, inode, oi->raw_inode); 4865 EXT4_INODE_SET_XTIME(i_atime, inode, oi->raw_inode); 4866 ext4_inode_csum_set(inode, oi->raw_inode, ei); 4867 spin_unlock(&ei->i_raw_lock); 4868 trace_ext4_other_inode_update_time(inode, oi->orig_ino); 4869 return -1; 4870 } 4871 spin_unlock(&inode->i_lock); 4872 return -1; 4873 } 4874 4875 /* 4876 * Opportunistically update the other time fields for other inodes in 4877 * the same inode table block. 4878 */ 4879 static void ext4_update_other_inodes_time(struct super_block *sb, 4880 unsigned long orig_ino, char *buf) 4881 { 4882 struct other_inode oi; 4883 unsigned long ino; 4884 int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 4885 int inode_size = EXT4_INODE_SIZE(sb); 4886 4887 oi.orig_ino = orig_ino; 4888 /* 4889 * Calculate the first inode in the inode table block. Inode 4890 * numbers are one-based. That is, the first inode in a block 4891 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1). 4892 */ 4893 ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1; 4894 for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) { 4895 if (ino == orig_ino) 4896 continue; 4897 oi.raw_inode = (struct ext4_inode *) buf; 4898 (void) find_inode_nowait(sb, ino, other_inode_match, &oi); 4899 } 4900 } 4901 4902 /* 4903 * Post the struct inode info into an on-disk inode location in the 4904 * buffer-cache. This gobbles the caller's reference to the 4905 * buffer_head in the inode location struct. 4906 * 4907 * The caller must have write access to iloc->bh. 4908 */ 4909 static int ext4_do_update_inode(handle_t *handle, 4910 struct inode *inode, 4911 struct ext4_iloc *iloc) 4912 { 4913 struct ext4_inode *raw_inode = ext4_raw_inode(iloc); 4914 struct ext4_inode_info *ei = EXT4_I(inode); 4915 struct buffer_head *bh = iloc->bh; 4916 struct super_block *sb = inode->i_sb; 4917 int err = 0, rc, block; 4918 int need_datasync = 0, set_large_file = 0; 4919 uid_t i_uid; 4920 gid_t i_gid; 4921 projid_t i_projid; 4922 4923 spin_lock(&ei->i_raw_lock); 4924 4925 /* For fields not tracked in the in-memory inode, 4926 * initialise them to zero for new inodes. */ 4927 if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) 4928 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size); 4929 4930 ext4_get_inode_flags(ei); 4931 raw_inode->i_mode = cpu_to_le16(inode->i_mode); 4932 i_uid = i_uid_read(inode); 4933 i_gid = i_gid_read(inode); 4934 i_projid = from_kprojid(&init_user_ns, ei->i_projid); 4935 if (!(test_opt(inode->i_sb, NO_UID32))) { 4936 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid)); 4937 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid)); 4938 /* 4939 * Fix up interoperability with old kernels. Otherwise, old inodes get 4940 * re-used with the upper 16 bits of the uid/gid intact 4941 */ 4942 if (ei->i_dtime && list_empty(&ei->i_orphan)) { 4943 raw_inode->i_uid_high = 0; 4944 raw_inode->i_gid_high = 0; 4945 } else { 4946 raw_inode->i_uid_high = 4947 cpu_to_le16(high_16_bits(i_uid)); 4948 raw_inode->i_gid_high = 4949 cpu_to_le16(high_16_bits(i_gid)); 4950 } 4951 } else { 4952 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid)); 4953 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid)); 4954 raw_inode->i_uid_high = 0; 4955 raw_inode->i_gid_high = 0; 4956 } 4957 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink); 4958 4959 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode); 4960 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode); 4961 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode); 4962 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode); 4963 4964 err = ext4_inode_blocks_set(handle, raw_inode, ei); 4965 if (err) { 4966 spin_unlock(&ei->i_raw_lock); 4967 goto out_brelse; 4968 } 4969 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime); 4970 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF); 4971 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) 4972 raw_inode->i_file_acl_high = 4973 cpu_to_le16(ei->i_file_acl >> 32); 4974 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl); 4975 if (ei->i_disksize != ext4_isize(raw_inode)) { 4976 ext4_isize_set(raw_inode, ei->i_disksize); 4977 need_datasync = 1; 4978 } 4979 if (ei->i_disksize > 0x7fffffffULL) { 4980 if (!ext4_has_feature_large_file(sb) || 4981 EXT4_SB(sb)->s_es->s_rev_level == 4982 cpu_to_le32(EXT4_GOOD_OLD_REV)) 4983 set_large_file = 1; 4984 } 4985 raw_inode->i_generation = cpu_to_le32(inode->i_generation); 4986 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { 4987 if (old_valid_dev(inode->i_rdev)) { 4988 raw_inode->i_block[0] = 4989 cpu_to_le32(old_encode_dev(inode->i_rdev)); 4990 raw_inode->i_block[1] = 0; 4991 } else { 4992 raw_inode->i_block[0] = 0; 4993 raw_inode->i_block[1] = 4994 cpu_to_le32(new_encode_dev(inode->i_rdev)); 4995 raw_inode->i_block[2] = 0; 4996 } 4997 } else if (!ext4_has_inline_data(inode)) { 4998 for (block = 0; block < EXT4_N_BLOCKS; block++) 4999 raw_inode->i_block[block] = ei->i_data[block]; 5000 } 5001 5002 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { 5003 raw_inode->i_disk_version = cpu_to_le32(inode->i_version); 5004 if (ei->i_extra_isize) { 5005 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 5006 raw_inode->i_version_hi = 5007 cpu_to_le32(inode->i_version >> 32); 5008 raw_inode->i_extra_isize = 5009 cpu_to_le16(ei->i_extra_isize); 5010 } 5011 } 5012 5013 BUG_ON(!ext4_has_feature_project(inode->i_sb) && 5014 i_projid != EXT4_DEF_PROJID); 5015 5016 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 5017 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) 5018 raw_inode->i_projid = cpu_to_le32(i_projid); 5019 5020 ext4_inode_csum_set(inode, raw_inode, ei); 5021 spin_unlock(&ei->i_raw_lock); 5022 if (inode->i_sb->s_flags & MS_LAZYTIME) 5023 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino, 5024 bh->b_data); 5025 5026 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 5027 rc = ext4_handle_dirty_metadata(handle, NULL, bh); 5028 if (!err) 5029 err = rc; 5030 ext4_clear_inode_state(inode, EXT4_STATE_NEW); 5031 if (set_large_file) { 5032 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access"); 5033 err = ext4_journal_get_write_access(handle, EXT4_SB(sb)->s_sbh); 5034 if (err) 5035 goto out_brelse; 5036 ext4_update_dynamic_rev(sb); 5037 ext4_set_feature_large_file(sb); 5038 ext4_handle_sync(handle); 5039 err = ext4_handle_dirty_super(handle, sb); 5040 } 5041 ext4_update_inode_fsync_trans(handle, inode, need_datasync); 5042 out_brelse: 5043 brelse(bh); 5044 ext4_std_error(inode->i_sb, err); 5045 return err; 5046 } 5047 5048 /* 5049 * ext4_write_inode() 5050 * 5051 * We are called from a few places: 5052 * 5053 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files. 5054 * Here, there will be no transaction running. We wait for any running 5055 * transaction to commit. 5056 * 5057 * - Within flush work (sys_sync(), kupdate and such). 5058 * We wait on commit, if told to. 5059 * 5060 * - Within iput_final() -> write_inode_now() 5061 * We wait on commit, if told to. 5062 * 5063 * In all cases it is actually safe for us to return without doing anything, 5064 * because the inode has been copied into a raw inode buffer in 5065 * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL 5066 * writeback. 5067 * 5068 * Note that we are absolutely dependent upon all inode dirtiers doing the 5069 * right thing: they *must* call mark_inode_dirty() after dirtying info in 5070 * which we are interested. 5071 * 5072 * It would be a bug for them to not do this. The code: 5073 * 5074 * mark_inode_dirty(inode) 5075 * stuff(); 5076 * inode->i_size = expr; 5077 * 5078 * is in error because write_inode() could occur while `stuff()' is running, 5079 * and the new i_size will be lost. Plus the inode will no longer be on the 5080 * superblock's dirty inode list. 5081 */ 5082 int ext4_write_inode(struct inode *inode, struct writeback_control *wbc) 5083 { 5084 int err; 5085 5086 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC)) 5087 return 0; 5088 5089 if (EXT4_SB(inode->i_sb)->s_journal) { 5090 if (ext4_journal_current_handle()) { 5091 jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n"); 5092 dump_stack(); 5093 return -EIO; 5094 } 5095 5096 /* 5097 * No need to force transaction in WB_SYNC_NONE mode. Also 5098 * ext4_sync_fs() will force the commit after everything is 5099 * written. 5100 */ 5101 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) 5102 return 0; 5103 5104 err = ext4_force_commit(inode->i_sb); 5105 } else { 5106 struct ext4_iloc iloc; 5107 5108 err = __ext4_get_inode_loc(inode, &iloc, 0); 5109 if (err) 5110 return err; 5111 /* 5112 * sync(2) will flush the whole buffer cache. No need to do 5113 * it here separately for each inode. 5114 */ 5115 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) 5116 sync_dirty_buffer(iloc.bh); 5117 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) { 5118 EXT4_ERROR_INODE_BLOCK(inode, iloc.bh->b_blocknr, 5119 "IO error syncing inode"); 5120 err = -EIO; 5121 } 5122 brelse(iloc.bh); 5123 } 5124 return err; 5125 } 5126 5127 /* 5128 * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate 5129 * buffers that are attached to a page stradding i_size and are undergoing 5130 * commit. In that case we have to wait for commit to finish and try again. 5131 */ 5132 static void ext4_wait_for_tail_page_commit(struct inode *inode) 5133 { 5134 struct page *page; 5135 unsigned offset; 5136 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 5137 tid_t commit_tid = 0; 5138 int ret; 5139 5140 offset = inode->i_size & (PAGE_SIZE - 1); 5141 /* 5142 * All buffers in the last page remain valid? Then there's nothing to 5143 * do. We do the check mainly to optimize the common PAGE_SIZE == 5144 * blocksize case 5145 */ 5146 if (offset > PAGE_SIZE - (1 << inode->i_blkbits)) 5147 return; 5148 while (1) { 5149 page = find_lock_page(inode->i_mapping, 5150 inode->i_size >> PAGE_SHIFT); 5151 if (!page) 5152 return; 5153 ret = __ext4_journalled_invalidatepage(page, offset, 5154 PAGE_SIZE - offset); 5155 unlock_page(page); 5156 put_page(page); 5157 if (ret != -EBUSY) 5158 return; 5159 commit_tid = 0; 5160 read_lock(&journal->j_state_lock); 5161 if (journal->j_committing_transaction) 5162 commit_tid = journal->j_committing_transaction->t_tid; 5163 read_unlock(&journal->j_state_lock); 5164 if (commit_tid) 5165 jbd2_log_wait_commit(journal, commit_tid); 5166 } 5167 } 5168 5169 /* 5170 * ext4_setattr() 5171 * 5172 * Called from notify_change. 5173 * 5174 * We want to trap VFS attempts to truncate the file as soon as 5175 * possible. In particular, we want to make sure that when the VFS 5176 * shrinks i_size, we put the inode on the orphan list and modify 5177 * i_disksize immediately, so that during the subsequent flushing of 5178 * dirty pages and freeing of disk blocks, we can guarantee that any 5179 * commit will leave the blocks being flushed in an unused state on 5180 * disk. (On recovery, the inode will get truncated and the blocks will 5181 * be freed, so we have a strong guarantee that no future commit will 5182 * leave these blocks visible to the user.) 5183 * 5184 * Another thing we have to assure is that if we are in ordered mode 5185 * and inode is still attached to the committing transaction, we must 5186 * we start writeout of all the dirty pages which are being truncated. 5187 * This way we are sure that all the data written in the previous 5188 * transaction are already on disk (truncate waits for pages under 5189 * writeback). 5190 * 5191 * Called with inode->i_mutex down. 5192 */ 5193 int ext4_setattr(struct dentry *dentry, struct iattr *attr) 5194 { 5195 struct inode *inode = d_inode(dentry); 5196 int error, rc = 0; 5197 int orphan = 0; 5198 const unsigned int ia_valid = attr->ia_valid; 5199 5200 error = setattr_prepare(dentry, attr); 5201 if (error) 5202 return error; 5203 5204 if (is_quota_modification(inode, attr)) { 5205 error = dquot_initialize(inode); 5206 if (error) 5207 return error; 5208 } 5209 if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) || 5210 (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) { 5211 handle_t *handle; 5212 5213 /* (user+group)*(old+new) structure, inode write (sb, 5214 * inode block, ? - but truncate inode update has it) */ 5215 handle = ext4_journal_start(inode, EXT4_HT_QUOTA, 5216 (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) + 5217 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3); 5218 if (IS_ERR(handle)) { 5219 error = PTR_ERR(handle); 5220 goto err_out; 5221 } 5222 error = dquot_transfer(inode, attr); 5223 if (error) { 5224 ext4_journal_stop(handle); 5225 return error; 5226 } 5227 /* Update corresponding info in inode so that everything is in 5228 * one transaction */ 5229 if (attr->ia_valid & ATTR_UID) 5230 inode->i_uid = attr->ia_uid; 5231 if (attr->ia_valid & ATTR_GID) 5232 inode->i_gid = attr->ia_gid; 5233 error = ext4_mark_inode_dirty(handle, inode); 5234 ext4_journal_stop(handle); 5235 } 5236 5237 if (attr->ia_valid & ATTR_SIZE) { 5238 handle_t *handle; 5239 loff_t oldsize = inode->i_size; 5240 int shrink = (attr->ia_size <= inode->i_size); 5241 5242 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 5243 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 5244 5245 if (attr->ia_size > sbi->s_bitmap_maxbytes) 5246 return -EFBIG; 5247 } 5248 if (!S_ISREG(inode->i_mode)) 5249 return -EINVAL; 5250 5251 if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size) 5252 inode_inc_iversion(inode); 5253 5254 if (ext4_should_order_data(inode) && 5255 (attr->ia_size < inode->i_size)) { 5256 error = ext4_begin_ordered_truncate(inode, 5257 attr->ia_size); 5258 if (error) 5259 goto err_out; 5260 } 5261 if (attr->ia_size != inode->i_size) { 5262 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); 5263 if (IS_ERR(handle)) { 5264 error = PTR_ERR(handle); 5265 goto err_out; 5266 } 5267 if (ext4_handle_valid(handle) && shrink) { 5268 error = ext4_orphan_add(handle, inode); 5269 orphan = 1; 5270 } 5271 /* 5272 * Update c/mtime on truncate up, ext4_truncate() will 5273 * update c/mtime in shrink case below 5274 */ 5275 if (!shrink) { 5276 inode->i_mtime = current_time(inode); 5277 inode->i_ctime = inode->i_mtime; 5278 } 5279 down_write(&EXT4_I(inode)->i_data_sem); 5280 EXT4_I(inode)->i_disksize = attr->ia_size; 5281 rc = ext4_mark_inode_dirty(handle, inode); 5282 if (!error) 5283 error = rc; 5284 /* 5285 * We have to update i_size under i_data_sem together 5286 * with i_disksize to avoid races with writeback code 5287 * running ext4_wb_update_i_disksize(). 5288 */ 5289 if (!error) 5290 i_size_write(inode, attr->ia_size); 5291 up_write(&EXT4_I(inode)->i_data_sem); 5292 ext4_journal_stop(handle); 5293 if (error) { 5294 if (orphan) 5295 ext4_orphan_del(NULL, inode); 5296 goto err_out; 5297 } 5298 } 5299 if (!shrink) 5300 pagecache_isize_extended(inode, oldsize, inode->i_size); 5301 5302 /* 5303 * Blocks are going to be removed from the inode. Wait 5304 * for dio in flight. Temporarily disable 5305 * dioread_nolock to prevent livelock. 5306 */ 5307 if (orphan) { 5308 if (!ext4_should_journal_data(inode)) { 5309 ext4_inode_block_unlocked_dio(inode); 5310 inode_dio_wait(inode); 5311 ext4_inode_resume_unlocked_dio(inode); 5312 } else 5313 ext4_wait_for_tail_page_commit(inode); 5314 } 5315 down_write(&EXT4_I(inode)->i_mmap_sem); 5316 /* 5317 * Truncate pagecache after we've waited for commit 5318 * in data=journal mode to make pages freeable. 5319 */ 5320 truncate_pagecache(inode, inode->i_size); 5321 if (shrink) { 5322 rc = ext4_truncate(inode); 5323 if (rc) 5324 error = rc; 5325 } 5326 up_write(&EXT4_I(inode)->i_mmap_sem); 5327 } 5328 5329 if (!error) { 5330 setattr_copy(inode, attr); 5331 mark_inode_dirty(inode); 5332 } 5333 5334 /* 5335 * If the call to ext4_truncate failed to get a transaction handle at 5336 * all, we need to clean up the in-core orphan list manually. 5337 */ 5338 if (orphan && inode->i_nlink) 5339 ext4_orphan_del(NULL, inode); 5340 5341 if (!error && (ia_valid & ATTR_MODE)) 5342 rc = posix_acl_chmod(inode, inode->i_mode); 5343 5344 err_out: 5345 ext4_std_error(inode->i_sb, error); 5346 if (!error) 5347 error = rc; 5348 return error; 5349 } 5350 5351 int ext4_getattr(struct vfsmount *mnt, struct dentry *dentry, 5352 struct kstat *stat) 5353 { 5354 struct inode *inode; 5355 unsigned long long delalloc_blocks; 5356 5357 inode = d_inode(dentry); 5358 generic_fillattr(inode, stat); 5359 5360 /* 5361 * If there is inline data in the inode, the inode will normally not 5362 * have data blocks allocated (it may have an external xattr block). 5363 * Report at least one sector for such files, so tools like tar, rsync, 5364 * others doen't incorrectly think the file is completely sparse. 5365 */ 5366 if (unlikely(ext4_has_inline_data(inode))) 5367 stat->blocks += (stat->size + 511) >> 9; 5368 5369 /* 5370 * We can't update i_blocks if the block allocation is delayed 5371 * otherwise in the case of system crash before the real block 5372 * allocation is done, we will have i_blocks inconsistent with 5373 * on-disk file blocks. 5374 * We always keep i_blocks updated together with real 5375 * allocation. But to not confuse with user, stat 5376 * will return the blocks that include the delayed allocation 5377 * blocks for this file. 5378 */ 5379 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb), 5380 EXT4_I(inode)->i_reserved_data_blocks); 5381 stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9); 5382 return 0; 5383 } 5384 5385 static int ext4_index_trans_blocks(struct inode *inode, int lblocks, 5386 int pextents) 5387 { 5388 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 5389 return ext4_ind_trans_blocks(inode, lblocks); 5390 return ext4_ext_index_trans_blocks(inode, pextents); 5391 } 5392 5393 /* 5394 * Account for index blocks, block groups bitmaps and block group 5395 * descriptor blocks if modify datablocks and index blocks 5396 * worse case, the indexs blocks spread over different block groups 5397 * 5398 * If datablocks are discontiguous, they are possible to spread over 5399 * different block groups too. If they are contiguous, with flexbg, 5400 * they could still across block group boundary. 5401 * 5402 * Also account for superblock, inode, quota and xattr blocks 5403 */ 5404 static int ext4_meta_trans_blocks(struct inode *inode, int lblocks, 5405 int pextents) 5406 { 5407 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); 5408 int gdpblocks; 5409 int idxblocks; 5410 int ret = 0; 5411 5412 /* 5413 * How many index blocks need to touch to map @lblocks logical blocks 5414 * to @pextents physical extents? 5415 */ 5416 idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents); 5417 5418 ret = idxblocks; 5419 5420 /* 5421 * Now let's see how many group bitmaps and group descriptors need 5422 * to account 5423 */ 5424 groups = idxblocks + pextents; 5425 gdpblocks = groups; 5426 if (groups > ngroups) 5427 groups = ngroups; 5428 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count) 5429 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count; 5430 5431 /* bitmaps and block group descriptor blocks */ 5432 ret += groups + gdpblocks; 5433 5434 /* Blocks for super block, inode, quota and xattr blocks */ 5435 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb); 5436 5437 return ret; 5438 } 5439 5440 /* 5441 * Calculate the total number of credits to reserve to fit 5442 * the modification of a single pages into a single transaction, 5443 * which may include multiple chunks of block allocations. 5444 * 5445 * This could be called via ext4_write_begin() 5446 * 5447 * We need to consider the worse case, when 5448 * one new block per extent. 5449 */ 5450 int ext4_writepage_trans_blocks(struct inode *inode) 5451 { 5452 int bpp = ext4_journal_blocks_per_page(inode); 5453 int ret; 5454 5455 ret = ext4_meta_trans_blocks(inode, bpp, bpp); 5456 5457 /* Account for data blocks for journalled mode */ 5458 if (ext4_should_journal_data(inode)) 5459 ret += bpp; 5460 return ret; 5461 } 5462 5463 /* 5464 * Calculate the journal credits for a chunk of data modification. 5465 * 5466 * This is called from DIO, fallocate or whoever calling 5467 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks. 5468 * 5469 * journal buffers for data blocks are not included here, as DIO 5470 * and fallocate do no need to journal data buffers. 5471 */ 5472 int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) 5473 { 5474 return ext4_meta_trans_blocks(inode, nrblocks, 1); 5475 } 5476 5477 /* 5478 * The caller must have previously called ext4_reserve_inode_write(). 5479 * Give this, we know that the caller already has write access to iloc->bh. 5480 */ 5481 int ext4_mark_iloc_dirty(handle_t *handle, 5482 struct inode *inode, struct ext4_iloc *iloc) 5483 { 5484 int err = 0; 5485 5486 if (IS_I_VERSION(inode)) 5487 inode_inc_iversion(inode); 5488 5489 /* the do_update_inode consumes one bh->b_count */ 5490 get_bh(iloc->bh); 5491 5492 /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */ 5493 err = ext4_do_update_inode(handle, inode, iloc); 5494 put_bh(iloc->bh); 5495 return err; 5496 } 5497 5498 /* 5499 * On success, We end up with an outstanding reference count against 5500 * iloc->bh. This _must_ be cleaned up later. 5501 */ 5502 5503 int 5504 ext4_reserve_inode_write(handle_t *handle, struct inode *inode, 5505 struct ext4_iloc *iloc) 5506 { 5507 int err; 5508 5509 err = ext4_get_inode_loc(inode, iloc); 5510 if (!err) { 5511 BUFFER_TRACE(iloc->bh, "get_write_access"); 5512 err = ext4_journal_get_write_access(handle, iloc->bh); 5513 if (err) { 5514 brelse(iloc->bh); 5515 iloc->bh = NULL; 5516 } 5517 } 5518 ext4_std_error(inode->i_sb, err); 5519 return err; 5520 } 5521 5522 /* 5523 * Expand an inode by new_extra_isize bytes. 5524 * Returns 0 on success or negative error number on failure. 5525 */ 5526 static int ext4_expand_extra_isize(struct inode *inode, 5527 unsigned int new_extra_isize, 5528 struct ext4_iloc iloc, 5529 handle_t *handle) 5530 { 5531 struct ext4_inode *raw_inode; 5532 struct ext4_xattr_ibody_header *header; 5533 5534 if (EXT4_I(inode)->i_extra_isize >= new_extra_isize) 5535 return 0; 5536 5537 raw_inode = ext4_raw_inode(&iloc); 5538 5539 header = IHDR(inode, raw_inode); 5540 5541 /* No extended attributes present */ 5542 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) || 5543 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) { 5544 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE, 0, 5545 new_extra_isize); 5546 EXT4_I(inode)->i_extra_isize = new_extra_isize; 5547 return 0; 5548 } 5549 5550 /* try to expand with EAs present */ 5551 return ext4_expand_extra_isize_ea(inode, new_extra_isize, 5552 raw_inode, handle); 5553 } 5554 5555 /* 5556 * What we do here is to mark the in-core inode as clean with respect to inode 5557 * dirtiness (it may still be data-dirty). 5558 * This means that the in-core inode may be reaped by prune_icache 5559 * without having to perform any I/O. This is a very good thing, 5560 * because *any* task may call prune_icache - even ones which 5561 * have a transaction open against a different journal. 5562 * 5563 * Is this cheating? Not really. Sure, we haven't written the 5564 * inode out, but prune_icache isn't a user-visible syncing function. 5565 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync) 5566 * we start and wait on commits. 5567 */ 5568 int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode) 5569 { 5570 struct ext4_iloc iloc; 5571 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 5572 static unsigned int mnt_count; 5573 int err, ret; 5574 5575 might_sleep(); 5576 trace_ext4_mark_inode_dirty(inode, _RET_IP_); 5577 err = ext4_reserve_inode_write(handle, inode, &iloc); 5578 if (err) 5579 return err; 5580 if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize && 5581 !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) { 5582 /* 5583 * In nojournal mode, we can immediately attempt to expand 5584 * the inode. When journaled, we first need to obtain extra 5585 * buffer credits since we may write into the EA block 5586 * with this same handle. If journal_extend fails, then it will 5587 * only result in a minor loss of functionality for that inode. 5588 * If this is felt to be critical, then e2fsck should be run to 5589 * force a large enough s_min_extra_isize. 5590 */ 5591 if (!ext4_handle_valid(handle) || 5592 jbd2_journal_extend(handle, 5593 EXT4_DATA_TRANS_BLOCKS(inode->i_sb)) == 0) { 5594 ret = ext4_expand_extra_isize(inode, 5595 sbi->s_want_extra_isize, 5596 iloc, handle); 5597 if (ret) { 5598 if (mnt_count != 5599 le16_to_cpu(sbi->s_es->s_mnt_count)) { 5600 ext4_warning(inode->i_sb, 5601 "Unable to expand inode %lu. Delete" 5602 " some EAs or run e2fsck.", 5603 inode->i_ino); 5604 mnt_count = 5605 le16_to_cpu(sbi->s_es->s_mnt_count); 5606 } 5607 } 5608 } 5609 } 5610 return ext4_mark_iloc_dirty(handle, inode, &iloc); 5611 } 5612 5613 /* 5614 * ext4_dirty_inode() is called from __mark_inode_dirty() 5615 * 5616 * We're really interested in the case where a file is being extended. 5617 * i_size has been changed by generic_commit_write() and we thus need 5618 * to include the updated inode in the current transaction. 5619 * 5620 * Also, dquot_alloc_block() will always dirty the inode when blocks 5621 * are allocated to the file. 5622 * 5623 * If the inode is marked synchronous, we don't honour that here - doing 5624 * so would cause a commit on atime updates, which we don't bother doing. 5625 * We handle synchronous inodes at the highest possible level. 5626 * 5627 * If only the I_DIRTY_TIME flag is set, we can skip everything. If 5628 * I_DIRTY_TIME and I_DIRTY_SYNC is set, the only inode fields we need 5629 * to copy into the on-disk inode structure are the timestamp files. 5630 */ 5631 void ext4_dirty_inode(struct inode *inode, int flags) 5632 { 5633 handle_t *handle; 5634 5635 if (flags == I_DIRTY_TIME) 5636 return; 5637 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 5638 if (IS_ERR(handle)) 5639 goto out; 5640 5641 ext4_mark_inode_dirty(handle, inode); 5642 5643 ext4_journal_stop(handle); 5644 out: 5645 return; 5646 } 5647 5648 #if 0 5649 /* 5650 * Bind an inode's backing buffer_head into this transaction, to prevent 5651 * it from being flushed to disk early. Unlike 5652 * ext4_reserve_inode_write, this leaves behind no bh reference and 5653 * returns no iloc structure, so the caller needs to repeat the iloc 5654 * lookup to mark the inode dirty later. 5655 */ 5656 static int ext4_pin_inode(handle_t *handle, struct inode *inode) 5657 { 5658 struct ext4_iloc iloc; 5659 5660 int err = 0; 5661 if (handle) { 5662 err = ext4_get_inode_loc(inode, &iloc); 5663 if (!err) { 5664 BUFFER_TRACE(iloc.bh, "get_write_access"); 5665 err = jbd2_journal_get_write_access(handle, iloc.bh); 5666 if (!err) 5667 err = ext4_handle_dirty_metadata(handle, 5668 NULL, 5669 iloc.bh); 5670 brelse(iloc.bh); 5671 } 5672 } 5673 ext4_std_error(inode->i_sb, err); 5674 return err; 5675 } 5676 #endif 5677 5678 int ext4_change_inode_journal_flag(struct inode *inode, int val) 5679 { 5680 journal_t *journal; 5681 handle_t *handle; 5682 int err; 5683 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 5684 5685 /* 5686 * We have to be very careful here: changing a data block's 5687 * journaling status dynamically is dangerous. If we write a 5688 * data block to the journal, change the status and then delete 5689 * that block, we risk forgetting to revoke the old log record 5690 * from the journal and so a subsequent replay can corrupt data. 5691 * So, first we make sure that the journal is empty and that 5692 * nobody is changing anything. 5693 */ 5694 5695 journal = EXT4_JOURNAL(inode); 5696 if (!journal) 5697 return 0; 5698 if (is_journal_aborted(journal)) 5699 return -EROFS; 5700 5701 /* Wait for all existing dio workers */ 5702 ext4_inode_block_unlocked_dio(inode); 5703 inode_dio_wait(inode); 5704 5705 /* 5706 * Before flushing the journal and switching inode's aops, we have 5707 * to flush all dirty data the inode has. There can be outstanding 5708 * delayed allocations, there can be unwritten extents created by 5709 * fallocate or buffered writes in dioread_nolock mode covered by 5710 * dirty data which can be converted only after flushing the dirty 5711 * data (and journalled aops don't know how to handle these cases). 5712 */ 5713 if (val) { 5714 down_write(&EXT4_I(inode)->i_mmap_sem); 5715 err = filemap_write_and_wait(inode->i_mapping); 5716 if (err < 0) { 5717 up_write(&EXT4_I(inode)->i_mmap_sem); 5718 ext4_inode_resume_unlocked_dio(inode); 5719 return err; 5720 } 5721 } 5722 5723 percpu_down_write(&sbi->s_journal_flag_rwsem); 5724 jbd2_journal_lock_updates(journal); 5725 5726 /* 5727 * OK, there are no updates running now, and all cached data is 5728 * synced to disk. We are now in a completely consistent state 5729 * which doesn't have anything in the journal, and we know that 5730 * no filesystem updates are running, so it is safe to modify 5731 * the inode's in-core data-journaling state flag now. 5732 */ 5733 5734 if (val) 5735 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 5736 else { 5737 err = jbd2_journal_flush(journal); 5738 if (err < 0) { 5739 jbd2_journal_unlock_updates(journal); 5740 percpu_up_write(&sbi->s_journal_flag_rwsem); 5741 ext4_inode_resume_unlocked_dio(inode); 5742 return err; 5743 } 5744 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 5745 } 5746 ext4_set_aops(inode); 5747 /* 5748 * Update inode->i_flags after EXT4_INODE_JOURNAL_DATA was updated. 5749 * E.g. S_DAX may get cleared / set. 5750 */ 5751 ext4_set_inode_flags(inode); 5752 5753 jbd2_journal_unlock_updates(journal); 5754 percpu_up_write(&sbi->s_journal_flag_rwsem); 5755 5756 if (val) 5757 up_write(&EXT4_I(inode)->i_mmap_sem); 5758 ext4_inode_resume_unlocked_dio(inode); 5759 5760 /* Finally we can mark the inode as dirty. */ 5761 5762 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 5763 if (IS_ERR(handle)) 5764 return PTR_ERR(handle); 5765 5766 err = ext4_mark_inode_dirty(handle, inode); 5767 ext4_handle_sync(handle); 5768 ext4_journal_stop(handle); 5769 ext4_std_error(inode->i_sb, err); 5770 5771 return err; 5772 } 5773 5774 static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh) 5775 { 5776 return !buffer_mapped(bh); 5777 } 5778 5779 int ext4_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 5780 { 5781 struct page *page = vmf->page; 5782 loff_t size; 5783 unsigned long len; 5784 int ret; 5785 struct file *file = vma->vm_file; 5786 struct inode *inode = file_inode(file); 5787 struct address_space *mapping = inode->i_mapping; 5788 handle_t *handle; 5789 get_block_t *get_block; 5790 int retries = 0; 5791 5792 sb_start_pagefault(inode->i_sb); 5793 file_update_time(vma->vm_file); 5794 5795 down_read(&EXT4_I(inode)->i_mmap_sem); 5796 /* Delalloc case is easy... */ 5797 if (test_opt(inode->i_sb, DELALLOC) && 5798 !ext4_should_journal_data(inode) && 5799 !ext4_nonda_switch(inode->i_sb)) { 5800 do { 5801 ret = block_page_mkwrite(vma, vmf, 5802 ext4_da_get_block_prep); 5803 } while (ret == -ENOSPC && 5804 ext4_should_retry_alloc(inode->i_sb, &retries)); 5805 goto out_ret; 5806 } 5807 5808 lock_page(page); 5809 size = i_size_read(inode); 5810 /* Page got truncated from under us? */ 5811 if (page->mapping != mapping || page_offset(page) > size) { 5812 unlock_page(page); 5813 ret = VM_FAULT_NOPAGE; 5814 goto out; 5815 } 5816 5817 if (page->index == size >> PAGE_SHIFT) 5818 len = size & ~PAGE_MASK; 5819 else 5820 len = PAGE_SIZE; 5821 /* 5822 * Return if we have all the buffers mapped. This avoids the need to do 5823 * journal_start/journal_stop which can block and take a long time 5824 */ 5825 if (page_has_buffers(page)) { 5826 if (!ext4_walk_page_buffers(NULL, page_buffers(page), 5827 0, len, NULL, 5828 ext4_bh_unmapped)) { 5829 /* Wait so that we don't change page under IO */ 5830 wait_for_stable_page(page); 5831 ret = VM_FAULT_LOCKED; 5832 goto out; 5833 } 5834 } 5835 unlock_page(page); 5836 /* OK, we need to fill the hole... */ 5837 if (ext4_should_dioread_nolock(inode)) 5838 get_block = ext4_get_block_unwritten; 5839 else 5840 get_block = ext4_get_block; 5841 retry_alloc: 5842 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, 5843 ext4_writepage_trans_blocks(inode)); 5844 if (IS_ERR(handle)) { 5845 ret = VM_FAULT_SIGBUS; 5846 goto out; 5847 } 5848 ret = block_page_mkwrite(vma, vmf, get_block); 5849 if (!ret && ext4_should_journal_data(inode)) { 5850 if (ext4_walk_page_buffers(handle, page_buffers(page), 0, 5851 PAGE_SIZE, NULL, do_journal_get_write_access)) { 5852 unlock_page(page); 5853 ret = VM_FAULT_SIGBUS; 5854 ext4_journal_stop(handle); 5855 goto out; 5856 } 5857 ext4_set_inode_state(inode, EXT4_STATE_JDATA); 5858 } 5859 ext4_journal_stop(handle); 5860 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 5861 goto retry_alloc; 5862 out_ret: 5863 ret = block_page_mkwrite_return(ret); 5864 out: 5865 up_read(&EXT4_I(inode)->i_mmap_sem); 5866 sb_end_pagefault(inode->i_sb); 5867 return ret; 5868 } 5869 5870 int ext4_filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 5871 { 5872 struct inode *inode = file_inode(vma->vm_file); 5873 int err; 5874 5875 down_read(&EXT4_I(inode)->i_mmap_sem); 5876 err = filemap_fault(vma, vmf); 5877 up_read(&EXT4_I(inode)->i_mmap_sem); 5878 5879 return err; 5880 } 5881 5882 /* 5883 * Find the first extent at or after @lblk in an inode that is not a hole. 5884 * Search for @map_len blocks at most. The extent is returned in @result. 5885 * 5886 * The function returns 1 if we found an extent. The function returns 0 in 5887 * case there is no extent at or after @lblk and in that case also sets 5888 * @result->es_len to 0. In case of error, the error code is returned. 5889 */ 5890 int ext4_get_next_extent(struct inode *inode, ext4_lblk_t lblk, 5891 unsigned int map_len, struct extent_status *result) 5892 { 5893 struct ext4_map_blocks map; 5894 struct extent_status es = {}; 5895 int ret; 5896 5897 map.m_lblk = lblk; 5898 map.m_len = map_len; 5899 5900 /* 5901 * For non-extent based files this loop may iterate several times since 5902 * we do not determine full hole size. 5903 */ 5904 while (map.m_len > 0) { 5905 ret = ext4_map_blocks(NULL, inode, &map, 0); 5906 if (ret < 0) 5907 return ret; 5908 /* There's extent covering m_lblk? Just return it. */ 5909 if (ret > 0) { 5910 int status; 5911 5912 ext4_es_store_pblock(result, map.m_pblk); 5913 result->es_lblk = map.m_lblk; 5914 result->es_len = map.m_len; 5915 if (map.m_flags & EXT4_MAP_UNWRITTEN) 5916 status = EXTENT_STATUS_UNWRITTEN; 5917 else 5918 status = EXTENT_STATUS_WRITTEN; 5919 ext4_es_store_status(result, status); 5920 return 1; 5921 } 5922 ext4_es_find_delayed_extent_range(inode, map.m_lblk, 5923 map.m_lblk + map.m_len - 1, 5924 &es); 5925 /* Is delalloc data before next block in extent tree? */ 5926 if (es.es_len && es.es_lblk < map.m_lblk + map.m_len) { 5927 ext4_lblk_t offset = 0; 5928 5929 if (es.es_lblk < lblk) 5930 offset = lblk - es.es_lblk; 5931 result->es_lblk = es.es_lblk + offset; 5932 ext4_es_store_pblock(result, 5933 ext4_es_pblock(&es) + offset); 5934 result->es_len = es.es_len - offset; 5935 ext4_es_store_status(result, ext4_es_status(&es)); 5936 5937 return 1; 5938 } 5939 /* There's a hole at m_lblk, advance us after it */ 5940 map.m_lblk += map.m_len; 5941 map_len -= map.m_len; 5942 map.m_len = map_len; 5943 cond_resched(); 5944 } 5945 result->es_len = 0; 5946 return 0; 5947 } 5948