1 /* 2 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 3 * Written by Alex Tomas <alex@clusterfs.com> 4 * 5 * Architecture independence: 6 * Copyright (c) 2005, Bull S.A. 7 * Written by Pierre Peiffer <pierre.peiffer@bull.net> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program; if not, write to the Free Software 20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- 21 */ 22 23 /* 24 * Extents support for EXT4 25 * 26 * TODO: 27 * - ext4*_error() should be used in some situations 28 * - analyze all BUG()/BUG_ON(), use -EIO where appropriate 29 * - smart tree reduction 30 */ 31 32 #include <linux/fs.h> 33 #include <linux/time.h> 34 #include <linux/jbd2.h> 35 #include <linux/highuid.h> 36 #include <linux/pagemap.h> 37 #include <linux/quotaops.h> 38 #include <linux/string.h> 39 #include <linux/slab.h> 40 #include <asm/uaccess.h> 41 #include <linux/fiemap.h> 42 #include <linux/backing-dev.h> 43 #include "ext4_jbd2.h" 44 #include "ext4_extents.h" 45 #include "xattr.h" 46 47 #include <trace/events/ext4.h> 48 49 /* 50 * used by extent splitting. 51 */ 52 #define EXT4_EXT_MAY_ZEROOUT 0x1 /* safe to zeroout if split fails \ 53 due to ENOSPC */ 54 #define EXT4_EXT_MARK_UNWRIT1 0x2 /* mark first half unwritten */ 55 #define EXT4_EXT_MARK_UNWRIT2 0x4 /* mark second half unwritten */ 56 57 #define EXT4_EXT_DATA_VALID1 0x8 /* first half contains valid data */ 58 #define EXT4_EXT_DATA_VALID2 0x10 /* second half contains valid data */ 59 60 static __le32 ext4_extent_block_csum(struct inode *inode, 61 struct ext4_extent_header *eh) 62 { 63 struct ext4_inode_info *ei = EXT4_I(inode); 64 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 65 __u32 csum; 66 67 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)eh, 68 EXT4_EXTENT_TAIL_OFFSET(eh)); 69 return cpu_to_le32(csum); 70 } 71 72 static int ext4_extent_block_csum_verify(struct inode *inode, 73 struct ext4_extent_header *eh) 74 { 75 struct ext4_extent_tail *et; 76 77 if (!ext4_has_metadata_csum(inode->i_sb)) 78 return 1; 79 80 et = find_ext4_extent_tail(eh); 81 if (et->et_checksum != ext4_extent_block_csum(inode, eh)) 82 return 0; 83 return 1; 84 } 85 86 static void ext4_extent_block_csum_set(struct inode *inode, 87 struct ext4_extent_header *eh) 88 { 89 struct ext4_extent_tail *et; 90 91 if (!ext4_has_metadata_csum(inode->i_sb)) 92 return; 93 94 et = find_ext4_extent_tail(eh); 95 et->et_checksum = ext4_extent_block_csum(inode, eh); 96 } 97 98 static int ext4_split_extent(handle_t *handle, 99 struct inode *inode, 100 struct ext4_ext_path **ppath, 101 struct ext4_map_blocks *map, 102 int split_flag, 103 int flags); 104 105 static int ext4_split_extent_at(handle_t *handle, 106 struct inode *inode, 107 struct ext4_ext_path **ppath, 108 ext4_lblk_t split, 109 int split_flag, 110 int flags); 111 112 static int ext4_find_delayed_extent(struct inode *inode, 113 struct extent_status *newes); 114 115 static int ext4_ext_truncate_extend_restart(handle_t *handle, 116 struct inode *inode, 117 int needed) 118 { 119 int err; 120 121 if (!ext4_handle_valid(handle)) 122 return 0; 123 if (handle->h_buffer_credits >= needed) 124 return 0; 125 /* 126 * If we need to extend the journal get a few extra blocks 127 * while we're at it for efficiency's sake. 128 */ 129 needed += 3; 130 err = ext4_journal_extend(handle, needed - handle->h_buffer_credits); 131 if (err <= 0) 132 return err; 133 err = ext4_truncate_restart_trans(handle, inode, needed); 134 if (err == 0) 135 err = -EAGAIN; 136 137 return err; 138 } 139 140 /* 141 * could return: 142 * - EROFS 143 * - ENOMEM 144 */ 145 static int ext4_ext_get_access(handle_t *handle, struct inode *inode, 146 struct ext4_ext_path *path) 147 { 148 if (path->p_bh) { 149 /* path points to block */ 150 BUFFER_TRACE(path->p_bh, "get_write_access"); 151 return ext4_journal_get_write_access(handle, path->p_bh); 152 } 153 /* path points to leaf/index in inode body */ 154 /* we use in-core data, no need to protect them */ 155 return 0; 156 } 157 158 /* 159 * could return: 160 * - EROFS 161 * - ENOMEM 162 * - EIO 163 */ 164 int __ext4_ext_dirty(const char *where, unsigned int line, handle_t *handle, 165 struct inode *inode, struct ext4_ext_path *path) 166 { 167 int err; 168 169 WARN_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem)); 170 if (path->p_bh) { 171 ext4_extent_block_csum_set(inode, ext_block_hdr(path->p_bh)); 172 /* path points to block */ 173 err = __ext4_handle_dirty_metadata(where, line, handle, 174 inode, path->p_bh); 175 } else { 176 /* path points to leaf/index in inode body */ 177 err = ext4_mark_inode_dirty(handle, inode); 178 } 179 return err; 180 } 181 182 static ext4_fsblk_t ext4_ext_find_goal(struct inode *inode, 183 struct ext4_ext_path *path, 184 ext4_lblk_t block) 185 { 186 if (path) { 187 int depth = path->p_depth; 188 struct ext4_extent *ex; 189 190 /* 191 * Try to predict block placement assuming that we are 192 * filling in a file which will eventually be 193 * non-sparse --- i.e., in the case of libbfd writing 194 * an ELF object sections out-of-order but in a way 195 * the eventually results in a contiguous object or 196 * executable file, or some database extending a table 197 * space file. However, this is actually somewhat 198 * non-ideal if we are writing a sparse file such as 199 * qemu or KVM writing a raw image file that is going 200 * to stay fairly sparse, since it will end up 201 * fragmenting the file system's free space. Maybe we 202 * should have some hueristics or some way to allow 203 * userspace to pass a hint to file system, 204 * especially if the latter case turns out to be 205 * common. 206 */ 207 ex = path[depth].p_ext; 208 if (ex) { 209 ext4_fsblk_t ext_pblk = ext4_ext_pblock(ex); 210 ext4_lblk_t ext_block = le32_to_cpu(ex->ee_block); 211 212 if (block > ext_block) 213 return ext_pblk + (block - ext_block); 214 else 215 return ext_pblk - (ext_block - block); 216 } 217 218 /* it looks like index is empty; 219 * try to find starting block from index itself */ 220 if (path[depth].p_bh) 221 return path[depth].p_bh->b_blocknr; 222 } 223 224 /* OK. use inode's group */ 225 return ext4_inode_to_goal_block(inode); 226 } 227 228 /* 229 * Allocation for a meta data block 230 */ 231 static ext4_fsblk_t 232 ext4_ext_new_meta_block(handle_t *handle, struct inode *inode, 233 struct ext4_ext_path *path, 234 struct ext4_extent *ex, int *err, unsigned int flags) 235 { 236 ext4_fsblk_t goal, newblock; 237 238 goal = ext4_ext_find_goal(inode, path, le32_to_cpu(ex->ee_block)); 239 newblock = ext4_new_meta_blocks(handle, inode, goal, flags, 240 NULL, err); 241 return newblock; 242 } 243 244 static inline int ext4_ext_space_block(struct inode *inode, int check) 245 { 246 int size; 247 248 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header)) 249 / sizeof(struct ext4_extent); 250 #ifdef AGGRESSIVE_TEST 251 if (!check && size > 6) 252 size = 6; 253 #endif 254 return size; 255 } 256 257 static inline int ext4_ext_space_block_idx(struct inode *inode, int check) 258 { 259 int size; 260 261 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header)) 262 / sizeof(struct ext4_extent_idx); 263 #ifdef AGGRESSIVE_TEST 264 if (!check && size > 5) 265 size = 5; 266 #endif 267 return size; 268 } 269 270 static inline int ext4_ext_space_root(struct inode *inode, int check) 271 { 272 int size; 273 274 size = sizeof(EXT4_I(inode)->i_data); 275 size -= sizeof(struct ext4_extent_header); 276 size /= sizeof(struct ext4_extent); 277 #ifdef AGGRESSIVE_TEST 278 if (!check && size > 3) 279 size = 3; 280 #endif 281 return size; 282 } 283 284 static inline int ext4_ext_space_root_idx(struct inode *inode, int check) 285 { 286 int size; 287 288 size = sizeof(EXT4_I(inode)->i_data); 289 size -= sizeof(struct ext4_extent_header); 290 size /= sizeof(struct ext4_extent_idx); 291 #ifdef AGGRESSIVE_TEST 292 if (!check && size > 4) 293 size = 4; 294 #endif 295 return size; 296 } 297 298 static inline int 299 ext4_force_split_extent_at(handle_t *handle, struct inode *inode, 300 struct ext4_ext_path **ppath, ext4_lblk_t lblk, 301 int nofail) 302 { 303 struct ext4_ext_path *path = *ppath; 304 int unwritten = ext4_ext_is_unwritten(path[path->p_depth].p_ext); 305 306 return ext4_split_extent_at(handle, inode, ppath, lblk, unwritten ? 307 EXT4_EXT_MARK_UNWRIT1|EXT4_EXT_MARK_UNWRIT2 : 0, 308 EXT4_EX_NOCACHE | EXT4_GET_BLOCKS_PRE_IO | 309 (nofail ? EXT4_GET_BLOCKS_METADATA_NOFAIL:0)); 310 } 311 312 /* 313 * Calculate the number of metadata blocks needed 314 * to allocate @blocks 315 * Worse case is one block per extent 316 */ 317 int ext4_ext_calc_metadata_amount(struct inode *inode, ext4_lblk_t lblock) 318 { 319 struct ext4_inode_info *ei = EXT4_I(inode); 320 int idxs; 321 322 idxs = ((inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header)) 323 / sizeof(struct ext4_extent_idx)); 324 325 /* 326 * If the new delayed allocation block is contiguous with the 327 * previous da block, it can share index blocks with the 328 * previous block, so we only need to allocate a new index 329 * block every idxs leaf blocks. At ldxs**2 blocks, we need 330 * an additional index block, and at ldxs**3 blocks, yet 331 * another index blocks. 332 */ 333 if (ei->i_da_metadata_calc_len && 334 ei->i_da_metadata_calc_last_lblock+1 == lblock) { 335 int num = 0; 336 337 if ((ei->i_da_metadata_calc_len % idxs) == 0) 338 num++; 339 if ((ei->i_da_metadata_calc_len % (idxs*idxs)) == 0) 340 num++; 341 if ((ei->i_da_metadata_calc_len % (idxs*idxs*idxs)) == 0) { 342 num++; 343 ei->i_da_metadata_calc_len = 0; 344 } else 345 ei->i_da_metadata_calc_len++; 346 ei->i_da_metadata_calc_last_lblock++; 347 return num; 348 } 349 350 /* 351 * In the worst case we need a new set of index blocks at 352 * every level of the inode's extent tree. 353 */ 354 ei->i_da_metadata_calc_len = 1; 355 ei->i_da_metadata_calc_last_lblock = lblock; 356 return ext_depth(inode) + 1; 357 } 358 359 static int 360 ext4_ext_max_entries(struct inode *inode, int depth) 361 { 362 int max; 363 364 if (depth == ext_depth(inode)) { 365 if (depth == 0) 366 max = ext4_ext_space_root(inode, 1); 367 else 368 max = ext4_ext_space_root_idx(inode, 1); 369 } else { 370 if (depth == 0) 371 max = ext4_ext_space_block(inode, 1); 372 else 373 max = ext4_ext_space_block_idx(inode, 1); 374 } 375 376 return max; 377 } 378 379 static int ext4_valid_extent(struct inode *inode, struct ext4_extent *ext) 380 { 381 ext4_fsblk_t block = ext4_ext_pblock(ext); 382 int len = ext4_ext_get_actual_len(ext); 383 ext4_lblk_t lblock = le32_to_cpu(ext->ee_block); 384 385 /* 386 * We allow neither: 387 * - zero length 388 * - overflow/wrap-around 389 */ 390 if (lblock + len <= lblock) 391 return 0; 392 return ext4_data_block_valid(EXT4_SB(inode->i_sb), block, len); 393 } 394 395 static int ext4_valid_extent_idx(struct inode *inode, 396 struct ext4_extent_idx *ext_idx) 397 { 398 ext4_fsblk_t block = ext4_idx_pblock(ext_idx); 399 400 return ext4_data_block_valid(EXT4_SB(inode->i_sb), block, 1); 401 } 402 403 static int ext4_valid_extent_entries(struct inode *inode, 404 struct ext4_extent_header *eh, 405 int depth) 406 { 407 unsigned short entries; 408 if (eh->eh_entries == 0) 409 return 1; 410 411 entries = le16_to_cpu(eh->eh_entries); 412 413 if (depth == 0) { 414 /* leaf entries */ 415 struct ext4_extent *ext = EXT_FIRST_EXTENT(eh); 416 struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es; 417 ext4_fsblk_t pblock = 0; 418 ext4_lblk_t lblock = 0; 419 ext4_lblk_t prev = 0; 420 int len = 0; 421 while (entries) { 422 if (!ext4_valid_extent(inode, ext)) 423 return 0; 424 425 /* Check for overlapping extents */ 426 lblock = le32_to_cpu(ext->ee_block); 427 len = ext4_ext_get_actual_len(ext); 428 if ((lblock <= prev) && prev) { 429 pblock = ext4_ext_pblock(ext); 430 es->s_last_error_block = cpu_to_le64(pblock); 431 return 0; 432 } 433 ext++; 434 entries--; 435 prev = lblock + len - 1; 436 } 437 } else { 438 struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh); 439 while (entries) { 440 if (!ext4_valid_extent_idx(inode, ext_idx)) 441 return 0; 442 ext_idx++; 443 entries--; 444 } 445 } 446 return 1; 447 } 448 449 static int __ext4_ext_check(const char *function, unsigned int line, 450 struct inode *inode, struct ext4_extent_header *eh, 451 int depth, ext4_fsblk_t pblk) 452 { 453 const char *error_msg; 454 int max = 0, err = -EFSCORRUPTED; 455 456 if (unlikely(eh->eh_magic != EXT4_EXT_MAGIC)) { 457 error_msg = "invalid magic"; 458 goto corrupted; 459 } 460 if (unlikely(le16_to_cpu(eh->eh_depth) != depth)) { 461 error_msg = "unexpected eh_depth"; 462 goto corrupted; 463 } 464 if (unlikely(eh->eh_max == 0)) { 465 error_msg = "invalid eh_max"; 466 goto corrupted; 467 } 468 max = ext4_ext_max_entries(inode, depth); 469 if (unlikely(le16_to_cpu(eh->eh_max) > max)) { 470 error_msg = "too large eh_max"; 471 goto corrupted; 472 } 473 if (unlikely(le16_to_cpu(eh->eh_entries) > le16_to_cpu(eh->eh_max))) { 474 error_msg = "invalid eh_entries"; 475 goto corrupted; 476 } 477 if (!ext4_valid_extent_entries(inode, eh, depth)) { 478 error_msg = "invalid extent entries"; 479 goto corrupted; 480 } 481 if (unlikely(depth > 32)) { 482 error_msg = "too large eh_depth"; 483 goto corrupted; 484 } 485 /* Verify checksum on non-root extent tree nodes */ 486 if (ext_depth(inode) != depth && 487 !ext4_extent_block_csum_verify(inode, eh)) { 488 error_msg = "extent tree corrupted"; 489 err = -EFSBADCRC; 490 goto corrupted; 491 } 492 return 0; 493 494 corrupted: 495 ext4_error_inode(inode, function, line, 0, 496 "pblk %llu bad header/extent: %s - magic %x, " 497 "entries %u, max %u(%u), depth %u(%u)", 498 (unsigned long long) pblk, error_msg, 499 le16_to_cpu(eh->eh_magic), 500 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max), 501 max, le16_to_cpu(eh->eh_depth), depth); 502 return err; 503 } 504 505 #define ext4_ext_check(inode, eh, depth, pblk) \ 506 __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk)) 507 508 int ext4_ext_check_inode(struct inode *inode) 509 { 510 return ext4_ext_check(inode, ext_inode_hdr(inode), ext_depth(inode), 0); 511 } 512 513 static struct buffer_head * 514 __read_extent_tree_block(const char *function, unsigned int line, 515 struct inode *inode, ext4_fsblk_t pblk, int depth, 516 int flags) 517 { 518 struct buffer_head *bh; 519 int err; 520 521 bh = sb_getblk_gfp(inode->i_sb, pblk, __GFP_MOVABLE | GFP_NOFS); 522 if (unlikely(!bh)) 523 return ERR_PTR(-ENOMEM); 524 525 if (!bh_uptodate_or_lock(bh)) { 526 trace_ext4_ext_load_extent(inode, pblk, _RET_IP_); 527 err = bh_submit_read(bh); 528 if (err < 0) 529 goto errout; 530 } 531 if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE)) 532 return bh; 533 err = __ext4_ext_check(function, line, inode, 534 ext_block_hdr(bh), depth, pblk); 535 if (err) 536 goto errout; 537 set_buffer_verified(bh); 538 /* 539 * If this is a leaf block, cache all of its entries 540 */ 541 if (!(flags & EXT4_EX_NOCACHE) && depth == 0) { 542 struct ext4_extent_header *eh = ext_block_hdr(bh); 543 struct ext4_extent *ex = EXT_FIRST_EXTENT(eh); 544 ext4_lblk_t prev = 0; 545 int i; 546 547 for (i = le16_to_cpu(eh->eh_entries); i > 0; i--, ex++) { 548 unsigned int status = EXTENT_STATUS_WRITTEN; 549 ext4_lblk_t lblk = le32_to_cpu(ex->ee_block); 550 int len = ext4_ext_get_actual_len(ex); 551 552 if (prev && (prev != lblk)) 553 ext4_es_cache_extent(inode, prev, 554 lblk - prev, ~0, 555 EXTENT_STATUS_HOLE); 556 557 if (ext4_ext_is_unwritten(ex)) 558 status = EXTENT_STATUS_UNWRITTEN; 559 ext4_es_cache_extent(inode, lblk, len, 560 ext4_ext_pblock(ex), status); 561 prev = lblk + len; 562 } 563 } 564 return bh; 565 errout: 566 put_bh(bh); 567 return ERR_PTR(err); 568 569 } 570 571 #define read_extent_tree_block(inode, pblk, depth, flags) \ 572 __read_extent_tree_block(__func__, __LINE__, (inode), (pblk), \ 573 (depth), (flags)) 574 575 /* 576 * This function is called to cache a file's extent information in the 577 * extent status tree 578 */ 579 int ext4_ext_precache(struct inode *inode) 580 { 581 struct ext4_inode_info *ei = EXT4_I(inode); 582 struct ext4_ext_path *path = NULL; 583 struct buffer_head *bh; 584 int i = 0, depth, ret = 0; 585 586 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 587 return 0; /* not an extent-mapped inode */ 588 589 down_read(&ei->i_data_sem); 590 depth = ext_depth(inode); 591 592 path = kzalloc(sizeof(struct ext4_ext_path) * (depth + 1), 593 GFP_NOFS); 594 if (path == NULL) { 595 up_read(&ei->i_data_sem); 596 return -ENOMEM; 597 } 598 599 /* Don't cache anything if there are no external extent blocks */ 600 if (depth == 0) 601 goto out; 602 path[0].p_hdr = ext_inode_hdr(inode); 603 ret = ext4_ext_check(inode, path[0].p_hdr, depth, 0); 604 if (ret) 605 goto out; 606 path[0].p_idx = EXT_FIRST_INDEX(path[0].p_hdr); 607 while (i >= 0) { 608 /* 609 * If this is a leaf block or we've reached the end of 610 * the index block, go up 611 */ 612 if ((i == depth) || 613 path[i].p_idx > EXT_LAST_INDEX(path[i].p_hdr)) { 614 brelse(path[i].p_bh); 615 path[i].p_bh = NULL; 616 i--; 617 continue; 618 } 619 bh = read_extent_tree_block(inode, 620 ext4_idx_pblock(path[i].p_idx++), 621 depth - i - 1, 622 EXT4_EX_FORCE_CACHE); 623 if (IS_ERR(bh)) { 624 ret = PTR_ERR(bh); 625 break; 626 } 627 i++; 628 path[i].p_bh = bh; 629 path[i].p_hdr = ext_block_hdr(bh); 630 path[i].p_idx = EXT_FIRST_INDEX(path[i].p_hdr); 631 } 632 ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED); 633 out: 634 up_read(&ei->i_data_sem); 635 ext4_ext_drop_refs(path); 636 kfree(path); 637 return ret; 638 } 639 640 #ifdef EXT_DEBUG 641 static void ext4_ext_show_path(struct inode *inode, struct ext4_ext_path *path) 642 { 643 int k, l = path->p_depth; 644 645 ext_debug("path:"); 646 for (k = 0; k <= l; k++, path++) { 647 if (path->p_idx) { 648 ext_debug(" %d->%llu", le32_to_cpu(path->p_idx->ei_block), 649 ext4_idx_pblock(path->p_idx)); 650 } else if (path->p_ext) { 651 ext_debug(" %d:[%d]%d:%llu ", 652 le32_to_cpu(path->p_ext->ee_block), 653 ext4_ext_is_unwritten(path->p_ext), 654 ext4_ext_get_actual_len(path->p_ext), 655 ext4_ext_pblock(path->p_ext)); 656 } else 657 ext_debug(" []"); 658 } 659 ext_debug("\n"); 660 } 661 662 static void ext4_ext_show_leaf(struct inode *inode, struct ext4_ext_path *path) 663 { 664 int depth = ext_depth(inode); 665 struct ext4_extent_header *eh; 666 struct ext4_extent *ex; 667 int i; 668 669 if (!path) 670 return; 671 672 eh = path[depth].p_hdr; 673 ex = EXT_FIRST_EXTENT(eh); 674 675 ext_debug("Displaying leaf extents for inode %lu\n", inode->i_ino); 676 677 for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ex++) { 678 ext_debug("%d:[%d]%d:%llu ", le32_to_cpu(ex->ee_block), 679 ext4_ext_is_unwritten(ex), 680 ext4_ext_get_actual_len(ex), ext4_ext_pblock(ex)); 681 } 682 ext_debug("\n"); 683 } 684 685 static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path, 686 ext4_fsblk_t newblock, int level) 687 { 688 int depth = ext_depth(inode); 689 struct ext4_extent *ex; 690 691 if (depth != level) { 692 struct ext4_extent_idx *idx; 693 idx = path[level].p_idx; 694 while (idx <= EXT_MAX_INDEX(path[level].p_hdr)) { 695 ext_debug("%d: move %d:%llu in new index %llu\n", level, 696 le32_to_cpu(idx->ei_block), 697 ext4_idx_pblock(idx), 698 newblock); 699 idx++; 700 } 701 702 return; 703 } 704 705 ex = path[depth].p_ext; 706 while (ex <= EXT_MAX_EXTENT(path[depth].p_hdr)) { 707 ext_debug("move %d:%llu:[%d]%d in new leaf %llu\n", 708 le32_to_cpu(ex->ee_block), 709 ext4_ext_pblock(ex), 710 ext4_ext_is_unwritten(ex), 711 ext4_ext_get_actual_len(ex), 712 newblock); 713 ex++; 714 } 715 } 716 717 #else 718 #define ext4_ext_show_path(inode, path) 719 #define ext4_ext_show_leaf(inode, path) 720 #define ext4_ext_show_move(inode, path, newblock, level) 721 #endif 722 723 void ext4_ext_drop_refs(struct ext4_ext_path *path) 724 { 725 int depth, i; 726 727 if (!path) 728 return; 729 depth = path->p_depth; 730 for (i = 0; i <= depth; i++, path++) 731 if (path->p_bh) { 732 brelse(path->p_bh); 733 path->p_bh = NULL; 734 } 735 } 736 737 /* 738 * ext4_ext_binsearch_idx: 739 * binary search for the closest index of the given block 740 * the header must be checked before calling this 741 */ 742 static void 743 ext4_ext_binsearch_idx(struct inode *inode, 744 struct ext4_ext_path *path, ext4_lblk_t block) 745 { 746 struct ext4_extent_header *eh = path->p_hdr; 747 struct ext4_extent_idx *r, *l, *m; 748 749 750 ext_debug("binsearch for %u(idx): ", block); 751 752 l = EXT_FIRST_INDEX(eh) + 1; 753 r = EXT_LAST_INDEX(eh); 754 while (l <= r) { 755 m = l + (r - l) / 2; 756 if (block < le32_to_cpu(m->ei_block)) 757 r = m - 1; 758 else 759 l = m + 1; 760 ext_debug("%p(%u):%p(%u):%p(%u) ", l, le32_to_cpu(l->ei_block), 761 m, le32_to_cpu(m->ei_block), 762 r, le32_to_cpu(r->ei_block)); 763 } 764 765 path->p_idx = l - 1; 766 ext_debug(" -> %u->%lld ", le32_to_cpu(path->p_idx->ei_block), 767 ext4_idx_pblock(path->p_idx)); 768 769 #ifdef CHECK_BINSEARCH 770 { 771 struct ext4_extent_idx *chix, *ix; 772 int k; 773 774 chix = ix = EXT_FIRST_INDEX(eh); 775 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ix++) { 776 if (k != 0 && 777 le32_to_cpu(ix->ei_block) <= le32_to_cpu(ix[-1].ei_block)) { 778 printk(KERN_DEBUG "k=%d, ix=0x%p, " 779 "first=0x%p\n", k, 780 ix, EXT_FIRST_INDEX(eh)); 781 printk(KERN_DEBUG "%u <= %u\n", 782 le32_to_cpu(ix->ei_block), 783 le32_to_cpu(ix[-1].ei_block)); 784 } 785 BUG_ON(k && le32_to_cpu(ix->ei_block) 786 <= le32_to_cpu(ix[-1].ei_block)); 787 if (block < le32_to_cpu(ix->ei_block)) 788 break; 789 chix = ix; 790 } 791 BUG_ON(chix != path->p_idx); 792 } 793 #endif 794 795 } 796 797 /* 798 * ext4_ext_binsearch: 799 * binary search for closest extent of the given block 800 * the header must be checked before calling this 801 */ 802 static void 803 ext4_ext_binsearch(struct inode *inode, 804 struct ext4_ext_path *path, ext4_lblk_t block) 805 { 806 struct ext4_extent_header *eh = path->p_hdr; 807 struct ext4_extent *r, *l, *m; 808 809 if (eh->eh_entries == 0) { 810 /* 811 * this leaf is empty: 812 * we get such a leaf in split/add case 813 */ 814 return; 815 } 816 817 ext_debug("binsearch for %u: ", block); 818 819 l = EXT_FIRST_EXTENT(eh) + 1; 820 r = EXT_LAST_EXTENT(eh); 821 822 while (l <= r) { 823 m = l + (r - l) / 2; 824 if (block < le32_to_cpu(m->ee_block)) 825 r = m - 1; 826 else 827 l = m + 1; 828 ext_debug("%p(%u):%p(%u):%p(%u) ", l, le32_to_cpu(l->ee_block), 829 m, le32_to_cpu(m->ee_block), 830 r, le32_to_cpu(r->ee_block)); 831 } 832 833 path->p_ext = l - 1; 834 ext_debug(" -> %d:%llu:[%d]%d ", 835 le32_to_cpu(path->p_ext->ee_block), 836 ext4_ext_pblock(path->p_ext), 837 ext4_ext_is_unwritten(path->p_ext), 838 ext4_ext_get_actual_len(path->p_ext)); 839 840 #ifdef CHECK_BINSEARCH 841 { 842 struct ext4_extent *chex, *ex; 843 int k; 844 845 chex = ex = EXT_FIRST_EXTENT(eh); 846 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ex++) { 847 BUG_ON(k && le32_to_cpu(ex->ee_block) 848 <= le32_to_cpu(ex[-1].ee_block)); 849 if (block < le32_to_cpu(ex->ee_block)) 850 break; 851 chex = ex; 852 } 853 BUG_ON(chex != path->p_ext); 854 } 855 #endif 856 857 } 858 859 int ext4_ext_tree_init(handle_t *handle, struct inode *inode) 860 { 861 struct ext4_extent_header *eh; 862 863 eh = ext_inode_hdr(inode); 864 eh->eh_depth = 0; 865 eh->eh_entries = 0; 866 eh->eh_magic = EXT4_EXT_MAGIC; 867 eh->eh_max = cpu_to_le16(ext4_ext_space_root(inode, 0)); 868 ext4_mark_inode_dirty(handle, inode); 869 return 0; 870 } 871 872 struct ext4_ext_path * 873 ext4_find_extent(struct inode *inode, ext4_lblk_t block, 874 struct ext4_ext_path **orig_path, int flags) 875 { 876 struct ext4_extent_header *eh; 877 struct buffer_head *bh; 878 struct ext4_ext_path *path = orig_path ? *orig_path : NULL; 879 short int depth, i, ppos = 0; 880 int ret; 881 882 eh = ext_inode_hdr(inode); 883 depth = ext_depth(inode); 884 885 if (path) { 886 ext4_ext_drop_refs(path); 887 if (depth > path[0].p_maxdepth) { 888 kfree(path); 889 *orig_path = path = NULL; 890 } 891 } 892 if (!path) { 893 /* account possible depth increase */ 894 path = kzalloc(sizeof(struct ext4_ext_path) * (depth + 2), 895 GFP_NOFS); 896 if (unlikely(!path)) 897 return ERR_PTR(-ENOMEM); 898 path[0].p_maxdepth = depth + 1; 899 } 900 path[0].p_hdr = eh; 901 path[0].p_bh = NULL; 902 903 i = depth; 904 /* walk through the tree */ 905 while (i) { 906 ext_debug("depth %d: num %d, max %d\n", 907 ppos, le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max)); 908 909 ext4_ext_binsearch_idx(inode, path + ppos, block); 910 path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx); 911 path[ppos].p_depth = i; 912 path[ppos].p_ext = NULL; 913 914 bh = read_extent_tree_block(inode, path[ppos].p_block, --i, 915 flags); 916 if (IS_ERR(bh)) { 917 ret = PTR_ERR(bh); 918 goto err; 919 } 920 921 eh = ext_block_hdr(bh); 922 ppos++; 923 path[ppos].p_bh = bh; 924 path[ppos].p_hdr = eh; 925 } 926 927 path[ppos].p_depth = i; 928 path[ppos].p_ext = NULL; 929 path[ppos].p_idx = NULL; 930 931 /* find extent */ 932 ext4_ext_binsearch(inode, path + ppos, block); 933 /* if not an empty leaf */ 934 if (path[ppos].p_ext) 935 path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext); 936 937 ext4_ext_show_path(inode, path); 938 939 return path; 940 941 err: 942 ext4_ext_drop_refs(path); 943 kfree(path); 944 if (orig_path) 945 *orig_path = NULL; 946 return ERR_PTR(ret); 947 } 948 949 /* 950 * ext4_ext_insert_index: 951 * insert new index [@logical;@ptr] into the block at @curp; 952 * check where to insert: before @curp or after @curp 953 */ 954 static int ext4_ext_insert_index(handle_t *handle, struct inode *inode, 955 struct ext4_ext_path *curp, 956 int logical, ext4_fsblk_t ptr) 957 { 958 struct ext4_extent_idx *ix; 959 int len, err; 960 961 err = ext4_ext_get_access(handle, inode, curp); 962 if (err) 963 return err; 964 965 if (unlikely(logical == le32_to_cpu(curp->p_idx->ei_block))) { 966 EXT4_ERROR_INODE(inode, 967 "logical %d == ei_block %d!", 968 logical, le32_to_cpu(curp->p_idx->ei_block)); 969 return -EFSCORRUPTED; 970 } 971 972 if (unlikely(le16_to_cpu(curp->p_hdr->eh_entries) 973 >= le16_to_cpu(curp->p_hdr->eh_max))) { 974 EXT4_ERROR_INODE(inode, 975 "eh_entries %d >= eh_max %d!", 976 le16_to_cpu(curp->p_hdr->eh_entries), 977 le16_to_cpu(curp->p_hdr->eh_max)); 978 return -EFSCORRUPTED; 979 } 980 981 if (logical > le32_to_cpu(curp->p_idx->ei_block)) { 982 /* insert after */ 983 ext_debug("insert new index %d after: %llu\n", logical, ptr); 984 ix = curp->p_idx + 1; 985 } else { 986 /* insert before */ 987 ext_debug("insert new index %d before: %llu\n", logical, ptr); 988 ix = curp->p_idx; 989 } 990 991 len = EXT_LAST_INDEX(curp->p_hdr) - ix + 1; 992 BUG_ON(len < 0); 993 if (len > 0) { 994 ext_debug("insert new index %d: " 995 "move %d indices from 0x%p to 0x%p\n", 996 logical, len, ix, ix + 1); 997 memmove(ix + 1, ix, len * sizeof(struct ext4_extent_idx)); 998 } 999 1000 if (unlikely(ix > EXT_MAX_INDEX(curp->p_hdr))) { 1001 EXT4_ERROR_INODE(inode, "ix > EXT_MAX_INDEX!"); 1002 return -EFSCORRUPTED; 1003 } 1004 1005 ix->ei_block = cpu_to_le32(logical); 1006 ext4_idx_store_pblock(ix, ptr); 1007 le16_add_cpu(&curp->p_hdr->eh_entries, 1); 1008 1009 if (unlikely(ix > EXT_LAST_INDEX(curp->p_hdr))) { 1010 EXT4_ERROR_INODE(inode, "ix > EXT_LAST_INDEX!"); 1011 return -EFSCORRUPTED; 1012 } 1013 1014 err = ext4_ext_dirty(handle, inode, curp); 1015 ext4_std_error(inode->i_sb, err); 1016 1017 return err; 1018 } 1019 1020 /* 1021 * ext4_ext_split: 1022 * inserts new subtree into the path, using free index entry 1023 * at depth @at: 1024 * - allocates all needed blocks (new leaf and all intermediate index blocks) 1025 * - makes decision where to split 1026 * - moves remaining extents and index entries (right to the split point) 1027 * into the newly allocated blocks 1028 * - initializes subtree 1029 */ 1030 static int ext4_ext_split(handle_t *handle, struct inode *inode, 1031 unsigned int flags, 1032 struct ext4_ext_path *path, 1033 struct ext4_extent *newext, int at) 1034 { 1035 struct buffer_head *bh = NULL; 1036 int depth = ext_depth(inode); 1037 struct ext4_extent_header *neh; 1038 struct ext4_extent_idx *fidx; 1039 int i = at, k, m, a; 1040 ext4_fsblk_t newblock, oldblock; 1041 __le32 border; 1042 ext4_fsblk_t *ablocks = NULL; /* array of allocated blocks */ 1043 int err = 0; 1044 1045 /* make decision: where to split? */ 1046 /* FIXME: now decision is simplest: at current extent */ 1047 1048 /* if current leaf will be split, then we should use 1049 * border from split point */ 1050 if (unlikely(path[depth].p_ext > EXT_MAX_EXTENT(path[depth].p_hdr))) { 1051 EXT4_ERROR_INODE(inode, "p_ext > EXT_MAX_EXTENT!"); 1052 return -EFSCORRUPTED; 1053 } 1054 if (path[depth].p_ext != EXT_MAX_EXTENT(path[depth].p_hdr)) { 1055 border = path[depth].p_ext[1].ee_block; 1056 ext_debug("leaf will be split." 1057 " next leaf starts at %d\n", 1058 le32_to_cpu(border)); 1059 } else { 1060 border = newext->ee_block; 1061 ext_debug("leaf will be added." 1062 " next leaf starts at %d\n", 1063 le32_to_cpu(border)); 1064 } 1065 1066 /* 1067 * If error occurs, then we break processing 1068 * and mark filesystem read-only. index won't 1069 * be inserted and tree will be in consistent 1070 * state. Next mount will repair buffers too. 1071 */ 1072 1073 /* 1074 * Get array to track all allocated blocks. 1075 * We need this to handle errors and free blocks 1076 * upon them. 1077 */ 1078 ablocks = kzalloc(sizeof(ext4_fsblk_t) * depth, GFP_NOFS); 1079 if (!ablocks) 1080 return -ENOMEM; 1081 1082 /* allocate all needed blocks */ 1083 ext_debug("allocate %d blocks for indexes/leaf\n", depth - at); 1084 for (a = 0; a < depth - at; a++) { 1085 newblock = ext4_ext_new_meta_block(handle, inode, path, 1086 newext, &err, flags); 1087 if (newblock == 0) 1088 goto cleanup; 1089 ablocks[a] = newblock; 1090 } 1091 1092 /* initialize new leaf */ 1093 newblock = ablocks[--a]; 1094 if (unlikely(newblock == 0)) { 1095 EXT4_ERROR_INODE(inode, "newblock == 0!"); 1096 err = -EFSCORRUPTED; 1097 goto cleanup; 1098 } 1099 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS); 1100 if (unlikely(!bh)) { 1101 err = -ENOMEM; 1102 goto cleanup; 1103 } 1104 lock_buffer(bh); 1105 1106 err = ext4_journal_get_create_access(handle, bh); 1107 if (err) 1108 goto cleanup; 1109 1110 neh = ext_block_hdr(bh); 1111 neh->eh_entries = 0; 1112 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0)); 1113 neh->eh_magic = EXT4_EXT_MAGIC; 1114 neh->eh_depth = 0; 1115 1116 /* move remainder of path[depth] to the new leaf */ 1117 if (unlikely(path[depth].p_hdr->eh_entries != 1118 path[depth].p_hdr->eh_max)) { 1119 EXT4_ERROR_INODE(inode, "eh_entries %d != eh_max %d!", 1120 path[depth].p_hdr->eh_entries, 1121 path[depth].p_hdr->eh_max); 1122 err = -EFSCORRUPTED; 1123 goto cleanup; 1124 } 1125 /* start copy from next extent */ 1126 m = EXT_MAX_EXTENT(path[depth].p_hdr) - path[depth].p_ext++; 1127 ext4_ext_show_move(inode, path, newblock, depth); 1128 if (m) { 1129 struct ext4_extent *ex; 1130 ex = EXT_FIRST_EXTENT(neh); 1131 memmove(ex, path[depth].p_ext, sizeof(struct ext4_extent) * m); 1132 le16_add_cpu(&neh->eh_entries, m); 1133 } 1134 1135 ext4_extent_block_csum_set(inode, neh); 1136 set_buffer_uptodate(bh); 1137 unlock_buffer(bh); 1138 1139 err = ext4_handle_dirty_metadata(handle, inode, bh); 1140 if (err) 1141 goto cleanup; 1142 brelse(bh); 1143 bh = NULL; 1144 1145 /* correct old leaf */ 1146 if (m) { 1147 err = ext4_ext_get_access(handle, inode, path + depth); 1148 if (err) 1149 goto cleanup; 1150 le16_add_cpu(&path[depth].p_hdr->eh_entries, -m); 1151 err = ext4_ext_dirty(handle, inode, path + depth); 1152 if (err) 1153 goto cleanup; 1154 1155 } 1156 1157 /* create intermediate indexes */ 1158 k = depth - at - 1; 1159 if (unlikely(k < 0)) { 1160 EXT4_ERROR_INODE(inode, "k %d < 0!", k); 1161 err = -EFSCORRUPTED; 1162 goto cleanup; 1163 } 1164 if (k) 1165 ext_debug("create %d intermediate indices\n", k); 1166 /* insert new index into current index block */ 1167 /* current depth stored in i var */ 1168 i = depth - 1; 1169 while (k--) { 1170 oldblock = newblock; 1171 newblock = ablocks[--a]; 1172 bh = sb_getblk(inode->i_sb, newblock); 1173 if (unlikely(!bh)) { 1174 err = -ENOMEM; 1175 goto cleanup; 1176 } 1177 lock_buffer(bh); 1178 1179 err = ext4_journal_get_create_access(handle, bh); 1180 if (err) 1181 goto cleanup; 1182 1183 neh = ext_block_hdr(bh); 1184 neh->eh_entries = cpu_to_le16(1); 1185 neh->eh_magic = EXT4_EXT_MAGIC; 1186 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0)); 1187 neh->eh_depth = cpu_to_le16(depth - i); 1188 fidx = EXT_FIRST_INDEX(neh); 1189 fidx->ei_block = border; 1190 ext4_idx_store_pblock(fidx, oldblock); 1191 1192 ext_debug("int.index at %d (block %llu): %u -> %llu\n", 1193 i, newblock, le32_to_cpu(border), oldblock); 1194 1195 /* move remainder of path[i] to the new index block */ 1196 if (unlikely(EXT_MAX_INDEX(path[i].p_hdr) != 1197 EXT_LAST_INDEX(path[i].p_hdr))) { 1198 EXT4_ERROR_INODE(inode, 1199 "EXT_MAX_INDEX != EXT_LAST_INDEX ee_block %d!", 1200 le32_to_cpu(path[i].p_ext->ee_block)); 1201 err = -EFSCORRUPTED; 1202 goto cleanup; 1203 } 1204 /* start copy indexes */ 1205 m = EXT_MAX_INDEX(path[i].p_hdr) - path[i].p_idx++; 1206 ext_debug("cur 0x%p, last 0x%p\n", path[i].p_idx, 1207 EXT_MAX_INDEX(path[i].p_hdr)); 1208 ext4_ext_show_move(inode, path, newblock, i); 1209 if (m) { 1210 memmove(++fidx, path[i].p_idx, 1211 sizeof(struct ext4_extent_idx) * m); 1212 le16_add_cpu(&neh->eh_entries, m); 1213 } 1214 ext4_extent_block_csum_set(inode, neh); 1215 set_buffer_uptodate(bh); 1216 unlock_buffer(bh); 1217 1218 err = ext4_handle_dirty_metadata(handle, inode, bh); 1219 if (err) 1220 goto cleanup; 1221 brelse(bh); 1222 bh = NULL; 1223 1224 /* correct old index */ 1225 if (m) { 1226 err = ext4_ext_get_access(handle, inode, path + i); 1227 if (err) 1228 goto cleanup; 1229 le16_add_cpu(&path[i].p_hdr->eh_entries, -m); 1230 err = ext4_ext_dirty(handle, inode, path + i); 1231 if (err) 1232 goto cleanup; 1233 } 1234 1235 i--; 1236 } 1237 1238 /* insert new index */ 1239 err = ext4_ext_insert_index(handle, inode, path + at, 1240 le32_to_cpu(border), newblock); 1241 1242 cleanup: 1243 if (bh) { 1244 if (buffer_locked(bh)) 1245 unlock_buffer(bh); 1246 brelse(bh); 1247 } 1248 1249 if (err) { 1250 /* free all allocated blocks in error case */ 1251 for (i = 0; i < depth; i++) { 1252 if (!ablocks[i]) 1253 continue; 1254 ext4_free_blocks(handle, inode, NULL, ablocks[i], 1, 1255 EXT4_FREE_BLOCKS_METADATA); 1256 } 1257 } 1258 kfree(ablocks); 1259 1260 return err; 1261 } 1262 1263 /* 1264 * ext4_ext_grow_indepth: 1265 * implements tree growing procedure: 1266 * - allocates new block 1267 * - moves top-level data (index block or leaf) into the new block 1268 * - initializes new top-level, creating index that points to the 1269 * just created block 1270 */ 1271 static int ext4_ext_grow_indepth(handle_t *handle, struct inode *inode, 1272 unsigned int flags) 1273 { 1274 struct ext4_extent_header *neh; 1275 struct buffer_head *bh; 1276 ext4_fsblk_t newblock, goal = 0; 1277 struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es; 1278 int err = 0; 1279 1280 /* Try to prepend new index to old one */ 1281 if (ext_depth(inode)) 1282 goal = ext4_idx_pblock(EXT_FIRST_INDEX(ext_inode_hdr(inode))); 1283 if (goal > le32_to_cpu(es->s_first_data_block)) { 1284 flags |= EXT4_MB_HINT_TRY_GOAL; 1285 goal--; 1286 } else 1287 goal = ext4_inode_to_goal_block(inode); 1288 newblock = ext4_new_meta_blocks(handle, inode, goal, flags, 1289 NULL, &err); 1290 if (newblock == 0) 1291 return err; 1292 1293 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS); 1294 if (unlikely(!bh)) 1295 return -ENOMEM; 1296 lock_buffer(bh); 1297 1298 err = ext4_journal_get_create_access(handle, bh); 1299 if (err) { 1300 unlock_buffer(bh); 1301 goto out; 1302 } 1303 1304 /* move top-level index/leaf into new block */ 1305 memmove(bh->b_data, EXT4_I(inode)->i_data, 1306 sizeof(EXT4_I(inode)->i_data)); 1307 1308 /* set size of new block */ 1309 neh = ext_block_hdr(bh); 1310 /* old root could have indexes or leaves 1311 * so calculate e_max right way */ 1312 if (ext_depth(inode)) 1313 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0)); 1314 else 1315 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0)); 1316 neh->eh_magic = EXT4_EXT_MAGIC; 1317 ext4_extent_block_csum_set(inode, neh); 1318 set_buffer_uptodate(bh); 1319 unlock_buffer(bh); 1320 1321 err = ext4_handle_dirty_metadata(handle, inode, bh); 1322 if (err) 1323 goto out; 1324 1325 /* Update top-level index: num,max,pointer */ 1326 neh = ext_inode_hdr(inode); 1327 neh->eh_entries = cpu_to_le16(1); 1328 ext4_idx_store_pblock(EXT_FIRST_INDEX(neh), newblock); 1329 if (neh->eh_depth == 0) { 1330 /* Root extent block becomes index block */ 1331 neh->eh_max = cpu_to_le16(ext4_ext_space_root_idx(inode, 0)); 1332 EXT_FIRST_INDEX(neh)->ei_block = 1333 EXT_FIRST_EXTENT(neh)->ee_block; 1334 } 1335 ext_debug("new root: num %d(%d), lblock %d, ptr %llu\n", 1336 le16_to_cpu(neh->eh_entries), le16_to_cpu(neh->eh_max), 1337 le32_to_cpu(EXT_FIRST_INDEX(neh)->ei_block), 1338 ext4_idx_pblock(EXT_FIRST_INDEX(neh))); 1339 1340 le16_add_cpu(&neh->eh_depth, 1); 1341 ext4_mark_inode_dirty(handle, inode); 1342 out: 1343 brelse(bh); 1344 1345 return err; 1346 } 1347 1348 /* 1349 * ext4_ext_create_new_leaf: 1350 * finds empty index and adds new leaf. 1351 * if no free index is found, then it requests in-depth growing. 1352 */ 1353 static int ext4_ext_create_new_leaf(handle_t *handle, struct inode *inode, 1354 unsigned int mb_flags, 1355 unsigned int gb_flags, 1356 struct ext4_ext_path **ppath, 1357 struct ext4_extent *newext) 1358 { 1359 struct ext4_ext_path *path = *ppath; 1360 struct ext4_ext_path *curp; 1361 int depth, i, err = 0; 1362 1363 repeat: 1364 i = depth = ext_depth(inode); 1365 1366 /* walk up to the tree and look for free index entry */ 1367 curp = path + depth; 1368 while (i > 0 && !EXT_HAS_FREE_INDEX(curp)) { 1369 i--; 1370 curp--; 1371 } 1372 1373 /* we use already allocated block for index block, 1374 * so subsequent data blocks should be contiguous */ 1375 if (EXT_HAS_FREE_INDEX(curp)) { 1376 /* if we found index with free entry, then use that 1377 * entry: create all needed subtree and add new leaf */ 1378 err = ext4_ext_split(handle, inode, mb_flags, path, newext, i); 1379 if (err) 1380 goto out; 1381 1382 /* refill path */ 1383 path = ext4_find_extent(inode, 1384 (ext4_lblk_t)le32_to_cpu(newext->ee_block), 1385 ppath, gb_flags); 1386 if (IS_ERR(path)) 1387 err = PTR_ERR(path); 1388 } else { 1389 /* tree is full, time to grow in depth */ 1390 err = ext4_ext_grow_indepth(handle, inode, mb_flags); 1391 if (err) 1392 goto out; 1393 1394 /* refill path */ 1395 path = ext4_find_extent(inode, 1396 (ext4_lblk_t)le32_to_cpu(newext->ee_block), 1397 ppath, gb_flags); 1398 if (IS_ERR(path)) { 1399 err = PTR_ERR(path); 1400 goto out; 1401 } 1402 1403 /* 1404 * only first (depth 0 -> 1) produces free space; 1405 * in all other cases we have to split the grown tree 1406 */ 1407 depth = ext_depth(inode); 1408 if (path[depth].p_hdr->eh_entries == path[depth].p_hdr->eh_max) { 1409 /* now we need to split */ 1410 goto repeat; 1411 } 1412 } 1413 1414 out: 1415 return err; 1416 } 1417 1418 /* 1419 * search the closest allocated block to the left for *logical 1420 * and returns it at @logical + it's physical address at @phys 1421 * if *logical is the smallest allocated block, the function 1422 * returns 0 at @phys 1423 * return value contains 0 (success) or error code 1424 */ 1425 static int ext4_ext_search_left(struct inode *inode, 1426 struct ext4_ext_path *path, 1427 ext4_lblk_t *logical, ext4_fsblk_t *phys) 1428 { 1429 struct ext4_extent_idx *ix; 1430 struct ext4_extent *ex; 1431 int depth, ee_len; 1432 1433 if (unlikely(path == NULL)) { 1434 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical); 1435 return -EFSCORRUPTED; 1436 } 1437 depth = path->p_depth; 1438 *phys = 0; 1439 1440 if (depth == 0 && path->p_ext == NULL) 1441 return 0; 1442 1443 /* usually extent in the path covers blocks smaller 1444 * then *logical, but it can be that extent is the 1445 * first one in the file */ 1446 1447 ex = path[depth].p_ext; 1448 ee_len = ext4_ext_get_actual_len(ex); 1449 if (*logical < le32_to_cpu(ex->ee_block)) { 1450 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) { 1451 EXT4_ERROR_INODE(inode, 1452 "EXT_FIRST_EXTENT != ex *logical %d ee_block %d!", 1453 *logical, le32_to_cpu(ex->ee_block)); 1454 return -EFSCORRUPTED; 1455 } 1456 while (--depth >= 0) { 1457 ix = path[depth].p_idx; 1458 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) { 1459 EXT4_ERROR_INODE(inode, 1460 "ix (%d) != EXT_FIRST_INDEX (%d) (depth %d)!", 1461 ix != NULL ? le32_to_cpu(ix->ei_block) : 0, 1462 EXT_FIRST_INDEX(path[depth].p_hdr) != NULL ? 1463 le32_to_cpu(EXT_FIRST_INDEX(path[depth].p_hdr)->ei_block) : 0, 1464 depth); 1465 return -EFSCORRUPTED; 1466 } 1467 } 1468 return 0; 1469 } 1470 1471 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) { 1472 EXT4_ERROR_INODE(inode, 1473 "logical %d < ee_block %d + ee_len %d!", 1474 *logical, le32_to_cpu(ex->ee_block), ee_len); 1475 return -EFSCORRUPTED; 1476 } 1477 1478 *logical = le32_to_cpu(ex->ee_block) + ee_len - 1; 1479 *phys = ext4_ext_pblock(ex) + ee_len - 1; 1480 return 0; 1481 } 1482 1483 /* 1484 * search the closest allocated block to the right for *logical 1485 * and returns it at @logical + it's physical address at @phys 1486 * if *logical is the largest allocated block, the function 1487 * returns 0 at @phys 1488 * return value contains 0 (success) or error code 1489 */ 1490 static int ext4_ext_search_right(struct inode *inode, 1491 struct ext4_ext_path *path, 1492 ext4_lblk_t *logical, ext4_fsblk_t *phys, 1493 struct ext4_extent **ret_ex) 1494 { 1495 struct buffer_head *bh = NULL; 1496 struct ext4_extent_header *eh; 1497 struct ext4_extent_idx *ix; 1498 struct ext4_extent *ex; 1499 ext4_fsblk_t block; 1500 int depth; /* Note, NOT eh_depth; depth from top of tree */ 1501 int ee_len; 1502 1503 if (unlikely(path == NULL)) { 1504 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical); 1505 return -EFSCORRUPTED; 1506 } 1507 depth = path->p_depth; 1508 *phys = 0; 1509 1510 if (depth == 0 && path->p_ext == NULL) 1511 return 0; 1512 1513 /* usually extent in the path covers blocks smaller 1514 * then *logical, but it can be that extent is the 1515 * first one in the file */ 1516 1517 ex = path[depth].p_ext; 1518 ee_len = ext4_ext_get_actual_len(ex); 1519 if (*logical < le32_to_cpu(ex->ee_block)) { 1520 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) { 1521 EXT4_ERROR_INODE(inode, 1522 "first_extent(path[%d].p_hdr) != ex", 1523 depth); 1524 return -EFSCORRUPTED; 1525 } 1526 while (--depth >= 0) { 1527 ix = path[depth].p_idx; 1528 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) { 1529 EXT4_ERROR_INODE(inode, 1530 "ix != EXT_FIRST_INDEX *logical %d!", 1531 *logical); 1532 return -EFSCORRUPTED; 1533 } 1534 } 1535 goto found_extent; 1536 } 1537 1538 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) { 1539 EXT4_ERROR_INODE(inode, 1540 "logical %d < ee_block %d + ee_len %d!", 1541 *logical, le32_to_cpu(ex->ee_block), ee_len); 1542 return -EFSCORRUPTED; 1543 } 1544 1545 if (ex != EXT_LAST_EXTENT(path[depth].p_hdr)) { 1546 /* next allocated block in this leaf */ 1547 ex++; 1548 goto found_extent; 1549 } 1550 1551 /* go up and search for index to the right */ 1552 while (--depth >= 0) { 1553 ix = path[depth].p_idx; 1554 if (ix != EXT_LAST_INDEX(path[depth].p_hdr)) 1555 goto got_index; 1556 } 1557 1558 /* we've gone up to the root and found no index to the right */ 1559 return 0; 1560 1561 got_index: 1562 /* we've found index to the right, let's 1563 * follow it and find the closest allocated 1564 * block to the right */ 1565 ix++; 1566 block = ext4_idx_pblock(ix); 1567 while (++depth < path->p_depth) { 1568 /* subtract from p_depth to get proper eh_depth */ 1569 bh = read_extent_tree_block(inode, block, 1570 path->p_depth - depth, 0); 1571 if (IS_ERR(bh)) 1572 return PTR_ERR(bh); 1573 eh = ext_block_hdr(bh); 1574 ix = EXT_FIRST_INDEX(eh); 1575 block = ext4_idx_pblock(ix); 1576 put_bh(bh); 1577 } 1578 1579 bh = read_extent_tree_block(inode, block, path->p_depth - depth, 0); 1580 if (IS_ERR(bh)) 1581 return PTR_ERR(bh); 1582 eh = ext_block_hdr(bh); 1583 ex = EXT_FIRST_EXTENT(eh); 1584 found_extent: 1585 *logical = le32_to_cpu(ex->ee_block); 1586 *phys = ext4_ext_pblock(ex); 1587 *ret_ex = ex; 1588 if (bh) 1589 put_bh(bh); 1590 return 0; 1591 } 1592 1593 /* 1594 * ext4_ext_next_allocated_block: 1595 * returns allocated block in subsequent extent or EXT_MAX_BLOCKS. 1596 * NOTE: it considers block number from index entry as 1597 * allocated block. Thus, index entries have to be consistent 1598 * with leaves. 1599 */ 1600 ext4_lblk_t 1601 ext4_ext_next_allocated_block(struct ext4_ext_path *path) 1602 { 1603 int depth; 1604 1605 BUG_ON(path == NULL); 1606 depth = path->p_depth; 1607 1608 if (depth == 0 && path->p_ext == NULL) 1609 return EXT_MAX_BLOCKS; 1610 1611 while (depth >= 0) { 1612 if (depth == path->p_depth) { 1613 /* leaf */ 1614 if (path[depth].p_ext && 1615 path[depth].p_ext != 1616 EXT_LAST_EXTENT(path[depth].p_hdr)) 1617 return le32_to_cpu(path[depth].p_ext[1].ee_block); 1618 } else { 1619 /* index */ 1620 if (path[depth].p_idx != 1621 EXT_LAST_INDEX(path[depth].p_hdr)) 1622 return le32_to_cpu(path[depth].p_idx[1].ei_block); 1623 } 1624 depth--; 1625 } 1626 1627 return EXT_MAX_BLOCKS; 1628 } 1629 1630 /* 1631 * ext4_ext_next_leaf_block: 1632 * returns first allocated block from next leaf or EXT_MAX_BLOCKS 1633 */ 1634 static ext4_lblk_t ext4_ext_next_leaf_block(struct ext4_ext_path *path) 1635 { 1636 int depth; 1637 1638 BUG_ON(path == NULL); 1639 depth = path->p_depth; 1640 1641 /* zero-tree has no leaf blocks at all */ 1642 if (depth == 0) 1643 return EXT_MAX_BLOCKS; 1644 1645 /* go to index block */ 1646 depth--; 1647 1648 while (depth >= 0) { 1649 if (path[depth].p_idx != 1650 EXT_LAST_INDEX(path[depth].p_hdr)) 1651 return (ext4_lblk_t) 1652 le32_to_cpu(path[depth].p_idx[1].ei_block); 1653 depth--; 1654 } 1655 1656 return EXT_MAX_BLOCKS; 1657 } 1658 1659 /* 1660 * ext4_ext_correct_indexes: 1661 * if leaf gets modified and modified extent is first in the leaf, 1662 * then we have to correct all indexes above. 1663 * TODO: do we need to correct tree in all cases? 1664 */ 1665 static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, 1666 struct ext4_ext_path *path) 1667 { 1668 struct ext4_extent_header *eh; 1669 int depth = ext_depth(inode); 1670 struct ext4_extent *ex; 1671 __le32 border; 1672 int k, err = 0; 1673 1674 eh = path[depth].p_hdr; 1675 ex = path[depth].p_ext; 1676 1677 if (unlikely(ex == NULL || eh == NULL)) { 1678 EXT4_ERROR_INODE(inode, 1679 "ex %p == NULL or eh %p == NULL", ex, eh); 1680 return -EFSCORRUPTED; 1681 } 1682 1683 if (depth == 0) { 1684 /* there is no tree at all */ 1685 return 0; 1686 } 1687 1688 if (ex != EXT_FIRST_EXTENT(eh)) { 1689 /* we correct tree if first leaf got modified only */ 1690 return 0; 1691 } 1692 1693 /* 1694 * TODO: we need correction if border is smaller than current one 1695 */ 1696 k = depth - 1; 1697 border = path[depth].p_ext->ee_block; 1698 err = ext4_ext_get_access(handle, inode, path + k); 1699 if (err) 1700 return err; 1701 path[k].p_idx->ei_block = border; 1702 err = ext4_ext_dirty(handle, inode, path + k); 1703 if (err) 1704 return err; 1705 1706 while (k--) { 1707 /* change all left-side indexes */ 1708 if (path[k+1].p_idx != EXT_FIRST_INDEX(path[k+1].p_hdr)) 1709 break; 1710 err = ext4_ext_get_access(handle, inode, path + k); 1711 if (err) 1712 break; 1713 path[k].p_idx->ei_block = border; 1714 err = ext4_ext_dirty(handle, inode, path + k); 1715 if (err) 1716 break; 1717 } 1718 1719 return err; 1720 } 1721 1722 int 1723 ext4_can_extents_be_merged(struct inode *inode, struct ext4_extent *ex1, 1724 struct ext4_extent *ex2) 1725 { 1726 unsigned short ext1_ee_len, ext2_ee_len; 1727 1728 if (ext4_ext_is_unwritten(ex1) != ext4_ext_is_unwritten(ex2)) 1729 return 0; 1730 1731 ext1_ee_len = ext4_ext_get_actual_len(ex1); 1732 ext2_ee_len = ext4_ext_get_actual_len(ex2); 1733 1734 if (le32_to_cpu(ex1->ee_block) + ext1_ee_len != 1735 le32_to_cpu(ex2->ee_block)) 1736 return 0; 1737 1738 /* 1739 * To allow future support for preallocated extents to be added 1740 * as an RO_COMPAT feature, refuse to merge to extents if 1741 * this can result in the top bit of ee_len being set. 1742 */ 1743 if (ext1_ee_len + ext2_ee_len > EXT_INIT_MAX_LEN) 1744 return 0; 1745 /* 1746 * The check for IO to unwritten extent is somewhat racy as we 1747 * increment i_unwritten / set EXT4_STATE_DIO_UNWRITTEN only after 1748 * dropping i_data_sem. But reserved blocks should save us in that 1749 * case. 1750 */ 1751 if (ext4_ext_is_unwritten(ex1) && 1752 (ext4_test_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN) || 1753 atomic_read(&EXT4_I(inode)->i_unwritten) || 1754 (ext1_ee_len + ext2_ee_len > EXT_UNWRITTEN_MAX_LEN))) 1755 return 0; 1756 #ifdef AGGRESSIVE_TEST 1757 if (ext1_ee_len >= 4) 1758 return 0; 1759 #endif 1760 1761 if (ext4_ext_pblock(ex1) + ext1_ee_len == ext4_ext_pblock(ex2)) 1762 return 1; 1763 return 0; 1764 } 1765 1766 /* 1767 * This function tries to merge the "ex" extent to the next extent in the tree. 1768 * It always tries to merge towards right. If you want to merge towards 1769 * left, pass "ex - 1" as argument instead of "ex". 1770 * Returns 0 if the extents (ex and ex+1) were _not_ merged and returns 1771 * 1 if they got merged. 1772 */ 1773 static int ext4_ext_try_to_merge_right(struct inode *inode, 1774 struct ext4_ext_path *path, 1775 struct ext4_extent *ex) 1776 { 1777 struct ext4_extent_header *eh; 1778 unsigned int depth, len; 1779 int merge_done = 0, unwritten; 1780 1781 depth = ext_depth(inode); 1782 BUG_ON(path[depth].p_hdr == NULL); 1783 eh = path[depth].p_hdr; 1784 1785 while (ex < EXT_LAST_EXTENT(eh)) { 1786 if (!ext4_can_extents_be_merged(inode, ex, ex + 1)) 1787 break; 1788 /* merge with next extent! */ 1789 unwritten = ext4_ext_is_unwritten(ex); 1790 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex) 1791 + ext4_ext_get_actual_len(ex + 1)); 1792 if (unwritten) 1793 ext4_ext_mark_unwritten(ex); 1794 1795 if (ex + 1 < EXT_LAST_EXTENT(eh)) { 1796 len = (EXT_LAST_EXTENT(eh) - ex - 1) 1797 * sizeof(struct ext4_extent); 1798 memmove(ex + 1, ex + 2, len); 1799 } 1800 le16_add_cpu(&eh->eh_entries, -1); 1801 merge_done = 1; 1802 WARN_ON(eh->eh_entries == 0); 1803 if (!eh->eh_entries) 1804 EXT4_ERROR_INODE(inode, "eh->eh_entries = 0!"); 1805 } 1806 1807 return merge_done; 1808 } 1809 1810 /* 1811 * This function does a very simple check to see if we can collapse 1812 * an extent tree with a single extent tree leaf block into the inode. 1813 */ 1814 static void ext4_ext_try_to_merge_up(handle_t *handle, 1815 struct inode *inode, 1816 struct ext4_ext_path *path) 1817 { 1818 size_t s; 1819 unsigned max_root = ext4_ext_space_root(inode, 0); 1820 ext4_fsblk_t blk; 1821 1822 if ((path[0].p_depth != 1) || 1823 (le16_to_cpu(path[0].p_hdr->eh_entries) != 1) || 1824 (le16_to_cpu(path[1].p_hdr->eh_entries) > max_root)) 1825 return; 1826 1827 /* 1828 * We need to modify the block allocation bitmap and the block 1829 * group descriptor to release the extent tree block. If we 1830 * can't get the journal credits, give up. 1831 */ 1832 if (ext4_journal_extend(handle, 2)) 1833 return; 1834 1835 /* 1836 * Copy the extent data up to the inode 1837 */ 1838 blk = ext4_idx_pblock(path[0].p_idx); 1839 s = le16_to_cpu(path[1].p_hdr->eh_entries) * 1840 sizeof(struct ext4_extent_idx); 1841 s += sizeof(struct ext4_extent_header); 1842 1843 path[1].p_maxdepth = path[0].p_maxdepth; 1844 memcpy(path[0].p_hdr, path[1].p_hdr, s); 1845 path[0].p_depth = 0; 1846 path[0].p_ext = EXT_FIRST_EXTENT(path[0].p_hdr) + 1847 (path[1].p_ext - EXT_FIRST_EXTENT(path[1].p_hdr)); 1848 path[0].p_hdr->eh_max = cpu_to_le16(max_root); 1849 1850 brelse(path[1].p_bh); 1851 ext4_free_blocks(handle, inode, NULL, blk, 1, 1852 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET); 1853 } 1854 1855 /* 1856 * This function tries to merge the @ex extent to neighbours in the tree. 1857 * return 1 if merge left else 0. 1858 */ 1859 static void ext4_ext_try_to_merge(handle_t *handle, 1860 struct inode *inode, 1861 struct ext4_ext_path *path, 1862 struct ext4_extent *ex) { 1863 struct ext4_extent_header *eh; 1864 unsigned int depth; 1865 int merge_done = 0; 1866 1867 depth = ext_depth(inode); 1868 BUG_ON(path[depth].p_hdr == NULL); 1869 eh = path[depth].p_hdr; 1870 1871 if (ex > EXT_FIRST_EXTENT(eh)) 1872 merge_done = ext4_ext_try_to_merge_right(inode, path, ex - 1); 1873 1874 if (!merge_done) 1875 (void) ext4_ext_try_to_merge_right(inode, path, ex); 1876 1877 ext4_ext_try_to_merge_up(handle, inode, path); 1878 } 1879 1880 /* 1881 * check if a portion of the "newext" extent overlaps with an 1882 * existing extent. 1883 * 1884 * If there is an overlap discovered, it updates the length of the newext 1885 * such that there will be no overlap, and then returns 1. 1886 * If there is no overlap found, it returns 0. 1887 */ 1888 static unsigned int ext4_ext_check_overlap(struct ext4_sb_info *sbi, 1889 struct inode *inode, 1890 struct ext4_extent *newext, 1891 struct ext4_ext_path *path) 1892 { 1893 ext4_lblk_t b1, b2; 1894 unsigned int depth, len1; 1895 unsigned int ret = 0; 1896 1897 b1 = le32_to_cpu(newext->ee_block); 1898 len1 = ext4_ext_get_actual_len(newext); 1899 depth = ext_depth(inode); 1900 if (!path[depth].p_ext) 1901 goto out; 1902 b2 = EXT4_LBLK_CMASK(sbi, le32_to_cpu(path[depth].p_ext->ee_block)); 1903 1904 /* 1905 * get the next allocated block if the extent in the path 1906 * is before the requested block(s) 1907 */ 1908 if (b2 < b1) { 1909 b2 = ext4_ext_next_allocated_block(path); 1910 if (b2 == EXT_MAX_BLOCKS) 1911 goto out; 1912 b2 = EXT4_LBLK_CMASK(sbi, b2); 1913 } 1914 1915 /* check for wrap through zero on extent logical start block*/ 1916 if (b1 + len1 < b1) { 1917 len1 = EXT_MAX_BLOCKS - b1; 1918 newext->ee_len = cpu_to_le16(len1); 1919 ret = 1; 1920 } 1921 1922 /* check for overlap */ 1923 if (b1 + len1 > b2) { 1924 newext->ee_len = cpu_to_le16(b2 - b1); 1925 ret = 1; 1926 } 1927 out: 1928 return ret; 1929 } 1930 1931 /* 1932 * ext4_ext_insert_extent: 1933 * tries to merge requsted extent into the existing extent or 1934 * inserts requested extent as new one into the tree, 1935 * creating new leaf in the no-space case. 1936 */ 1937 int ext4_ext_insert_extent(handle_t *handle, struct inode *inode, 1938 struct ext4_ext_path **ppath, 1939 struct ext4_extent *newext, int gb_flags) 1940 { 1941 struct ext4_ext_path *path = *ppath; 1942 struct ext4_extent_header *eh; 1943 struct ext4_extent *ex, *fex; 1944 struct ext4_extent *nearex; /* nearest extent */ 1945 struct ext4_ext_path *npath = NULL; 1946 int depth, len, err; 1947 ext4_lblk_t next; 1948 int mb_flags = 0, unwritten; 1949 1950 if (gb_flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) 1951 mb_flags |= EXT4_MB_DELALLOC_RESERVED; 1952 if (unlikely(ext4_ext_get_actual_len(newext) == 0)) { 1953 EXT4_ERROR_INODE(inode, "ext4_ext_get_actual_len(newext) == 0"); 1954 return -EFSCORRUPTED; 1955 } 1956 depth = ext_depth(inode); 1957 ex = path[depth].p_ext; 1958 eh = path[depth].p_hdr; 1959 if (unlikely(path[depth].p_hdr == NULL)) { 1960 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth); 1961 return -EFSCORRUPTED; 1962 } 1963 1964 /* try to insert block into found extent and return */ 1965 if (ex && !(gb_flags & EXT4_GET_BLOCKS_PRE_IO)) { 1966 1967 /* 1968 * Try to see whether we should rather test the extent on 1969 * right from ex, or from the left of ex. This is because 1970 * ext4_find_extent() can return either extent on the 1971 * left, or on the right from the searched position. This 1972 * will make merging more effective. 1973 */ 1974 if (ex < EXT_LAST_EXTENT(eh) && 1975 (le32_to_cpu(ex->ee_block) + 1976 ext4_ext_get_actual_len(ex) < 1977 le32_to_cpu(newext->ee_block))) { 1978 ex += 1; 1979 goto prepend; 1980 } else if ((ex > EXT_FIRST_EXTENT(eh)) && 1981 (le32_to_cpu(newext->ee_block) + 1982 ext4_ext_get_actual_len(newext) < 1983 le32_to_cpu(ex->ee_block))) 1984 ex -= 1; 1985 1986 /* Try to append newex to the ex */ 1987 if (ext4_can_extents_be_merged(inode, ex, newext)) { 1988 ext_debug("append [%d]%d block to %u:[%d]%d" 1989 "(from %llu)\n", 1990 ext4_ext_is_unwritten(newext), 1991 ext4_ext_get_actual_len(newext), 1992 le32_to_cpu(ex->ee_block), 1993 ext4_ext_is_unwritten(ex), 1994 ext4_ext_get_actual_len(ex), 1995 ext4_ext_pblock(ex)); 1996 err = ext4_ext_get_access(handle, inode, 1997 path + depth); 1998 if (err) 1999 return err; 2000 unwritten = ext4_ext_is_unwritten(ex); 2001 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex) 2002 + ext4_ext_get_actual_len(newext)); 2003 if (unwritten) 2004 ext4_ext_mark_unwritten(ex); 2005 eh = path[depth].p_hdr; 2006 nearex = ex; 2007 goto merge; 2008 } 2009 2010 prepend: 2011 /* Try to prepend newex to the ex */ 2012 if (ext4_can_extents_be_merged(inode, newext, ex)) { 2013 ext_debug("prepend %u[%d]%d block to %u:[%d]%d" 2014 "(from %llu)\n", 2015 le32_to_cpu(newext->ee_block), 2016 ext4_ext_is_unwritten(newext), 2017 ext4_ext_get_actual_len(newext), 2018 le32_to_cpu(ex->ee_block), 2019 ext4_ext_is_unwritten(ex), 2020 ext4_ext_get_actual_len(ex), 2021 ext4_ext_pblock(ex)); 2022 err = ext4_ext_get_access(handle, inode, 2023 path + depth); 2024 if (err) 2025 return err; 2026 2027 unwritten = ext4_ext_is_unwritten(ex); 2028 ex->ee_block = newext->ee_block; 2029 ext4_ext_store_pblock(ex, ext4_ext_pblock(newext)); 2030 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex) 2031 + ext4_ext_get_actual_len(newext)); 2032 if (unwritten) 2033 ext4_ext_mark_unwritten(ex); 2034 eh = path[depth].p_hdr; 2035 nearex = ex; 2036 goto merge; 2037 } 2038 } 2039 2040 depth = ext_depth(inode); 2041 eh = path[depth].p_hdr; 2042 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) 2043 goto has_space; 2044 2045 /* probably next leaf has space for us? */ 2046 fex = EXT_LAST_EXTENT(eh); 2047 next = EXT_MAX_BLOCKS; 2048 if (le32_to_cpu(newext->ee_block) > le32_to_cpu(fex->ee_block)) 2049 next = ext4_ext_next_leaf_block(path); 2050 if (next != EXT_MAX_BLOCKS) { 2051 ext_debug("next leaf block - %u\n", next); 2052 BUG_ON(npath != NULL); 2053 npath = ext4_find_extent(inode, next, NULL, 0); 2054 if (IS_ERR(npath)) 2055 return PTR_ERR(npath); 2056 BUG_ON(npath->p_depth != path->p_depth); 2057 eh = npath[depth].p_hdr; 2058 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) { 2059 ext_debug("next leaf isn't full(%d)\n", 2060 le16_to_cpu(eh->eh_entries)); 2061 path = npath; 2062 goto has_space; 2063 } 2064 ext_debug("next leaf has no free space(%d,%d)\n", 2065 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max)); 2066 } 2067 2068 /* 2069 * There is no free space in the found leaf. 2070 * We're gonna add a new leaf in the tree. 2071 */ 2072 if (gb_flags & EXT4_GET_BLOCKS_METADATA_NOFAIL) 2073 mb_flags |= EXT4_MB_USE_RESERVED; 2074 err = ext4_ext_create_new_leaf(handle, inode, mb_flags, gb_flags, 2075 ppath, newext); 2076 if (err) 2077 goto cleanup; 2078 depth = ext_depth(inode); 2079 eh = path[depth].p_hdr; 2080 2081 has_space: 2082 nearex = path[depth].p_ext; 2083 2084 err = ext4_ext_get_access(handle, inode, path + depth); 2085 if (err) 2086 goto cleanup; 2087 2088 if (!nearex) { 2089 /* there is no extent in this leaf, create first one */ 2090 ext_debug("first extent in the leaf: %u:%llu:[%d]%d\n", 2091 le32_to_cpu(newext->ee_block), 2092 ext4_ext_pblock(newext), 2093 ext4_ext_is_unwritten(newext), 2094 ext4_ext_get_actual_len(newext)); 2095 nearex = EXT_FIRST_EXTENT(eh); 2096 } else { 2097 if (le32_to_cpu(newext->ee_block) 2098 > le32_to_cpu(nearex->ee_block)) { 2099 /* Insert after */ 2100 ext_debug("insert %u:%llu:[%d]%d before: " 2101 "nearest %p\n", 2102 le32_to_cpu(newext->ee_block), 2103 ext4_ext_pblock(newext), 2104 ext4_ext_is_unwritten(newext), 2105 ext4_ext_get_actual_len(newext), 2106 nearex); 2107 nearex++; 2108 } else { 2109 /* Insert before */ 2110 BUG_ON(newext->ee_block == nearex->ee_block); 2111 ext_debug("insert %u:%llu:[%d]%d after: " 2112 "nearest %p\n", 2113 le32_to_cpu(newext->ee_block), 2114 ext4_ext_pblock(newext), 2115 ext4_ext_is_unwritten(newext), 2116 ext4_ext_get_actual_len(newext), 2117 nearex); 2118 } 2119 len = EXT_LAST_EXTENT(eh) - nearex + 1; 2120 if (len > 0) { 2121 ext_debug("insert %u:%llu:[%d]%d: " 2122 "move %d extents from 0x%p to 0x%p\n", 2123 le32_to_cpu(newext->ee_block), 2124 ext4_ext_pblock(newext), 2125 ext4_ext_is_unwritten(newext), 2126 ext4_ext_get_actual_len(newext), 2127 len, nearex, nearex + 1); 2128 memmove(nearex + 1, nearex, 2129 len * sizeof(struct ext4_extent)); 2130 } 2131 } 2132 2133 le16_add_cpu(&eh->eh_entries, 1); 2134 path[depth].p_ext = nearex; 2135 nearex->ee_block = newext->ee_block; 2136 ext4_ext_store_pblock(nearex, ext4_ext_pblock(newext)); 2137 nearex->ee_len = newext->ee_len; 2138 2139 merge: 2140 /* try to merge extents */ 2141 if (!(gb_flags & EXT4_GET_BLOCKS_PRE_IO)) 2142 ext4_ext_try_to_merge(handle, inode, path, nearex); 2143 2144 2145 /* time to correct all indexes above */ 2146 err = ext4_ext_correct_indexes(handle, inode, path); 2147 if (err) 2148 goto cleanup; 2149 2150 err = ext4_ext_dirty(handle, inode, path + path->p_depth); 2151 2152 cleanup: 2153 ext4_ext_drop_refs(npath); 2154 kfree(npath); 2155 return err; 2156 } 2157 2158 static int ext4_fill_fiemap_extents(struct inode *inode, 2159 ext4_lblk_t block, ext4_lblk_t num, 2160 struct fiemap_extent_info *fieinfo) 2161 { 2162 struct ext4_ext_path *path = NULL; 2163 struct ext4_extent *ex; 2164 struct extent_status es; 2165 ext4_lblk_t next, next_del, start = 0, end = 0; 2166 ext4_lblk_t last = block + num; 2167 int exists, depth = 0, err = 0; 2168 unsigned int flags = 0; 2169 unsigned char blksize_bits = inode->i_sb->s_blocksize_bits; 2170 2171 while (block < last && block != EXT_MAX_BLOCKS) { 2172 num = last - block; 2173 /* find extent for this block */ 2174 down_read(&EXT4_I(inode)->i_data_sem); 2175 2176 path = ext4_find_extent(inode, block, &path, 0); 2177 if (IS_ERR(path)) { 2178 up_read(&EXT4_I(inode)->i_data_sem); 2179 err = PTR_ERR(path); 2180 path = NULL; 2181 break; 2182 } 2183 2184 depth = ext_depth(inode); 2185 if (unlikely(path[depth].p_hdr == NULL)) { 2186 up_read(&EXT4_I(inode)->i_data_sem); 2187 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth); 2188 err = -EFSCORRUPTED; 2189 break; 2190 } 2191 ex = path[depth].p_ext; 2192 next = ext4_ext_next_allocated_block(path); 2193 2194 flags = 0; 2195 exists = 0; 2196 if (!ex) { 2197 /* there is no extent yet, so try to allocate 2198 * all requested space */ 2199 start = block; 2200 end = block + num; 2201 } else if (le32_to_cpu(ex->ee_block) > block) { 2202 /* need to allocate space before found extent */ 2203 start = block; 2204 end = le32_to_cpu(ex->ee_block); 2205 if (block + num < end) 2206 end = block + num; 2207 } else if (block >= le32_to_cpu(ex->ee_block) 2208 + ext4_ext_get_actual_len(ex)) { 2209 /* need to allocate space after found extent */ 2210 start = block; 2211 end = block + num; 2212 if (end >= next) 2213 end = next; 2214 } else if (block >= le32_to_cpu(ex->ee_block)) { 2215 /* 2216 * some part of requested space is covered 2217 * by found extent 2218 */ 2219 start = block; 2220 end = le32_to_cpu(ex->ee_block) 2221 + ext4_ext_get_actual_len(ex); 2222 if (block + num < end) 2223 end = block + num; 2224 exists = 1; 2225 } else { 2226 BUG(); 2227 } 2228 BUG_ON(end <= start); 2229 2230 if (!exists) { 2231 es.es_lblk = start; 2232 es.es_len = end - start; 2233 es.es_pblk = 0; 2234 } else { 2235 es.es_lblk = le32_to_cpu(ex->ee_block); 2236 es.es_len = ext4_ext_get_actual_len(ex); 2237 es.es_pblk = ext4_ext_pblock(ex); 2238 if (ext4_ext_is_unwritten(ex)) 2239 flags |= FIEMAP_EXTENT_UNWRITTEN; 2240 } 2241 2242 /* 2243 * Find delayed extent and update es accordingly. We call 2244 * it even in !exists case to find out whether es is the 2245 * last existing extent or not. 2246 */ 2247 next_del = ext4_find_delayed_extent(inode, &es); 2248 if (!exists && next_del) { 2249 exists = 1; 2250 flags |= (FIEMAP_EXTENT_DELALLOC | 2251 FIEMAP_EXTENT_UNKNOWN); 2252 } 2253 up_read(&EXT4_I(inode)->i_data_sem); 2254 2255 if (unlikely(es.es_len == 0)) { 2256 EXT4_ERROR_INODE(inode, "es.es_len == 0"); 2257 err = -EFSCORRUPTED; 2258 break; 2259 } 2260 2261 /* 2262 * This is possible iff next == next_del == EXT_MAX_BLOCKS. 2263 * we need to check next == EXT_MAX_BLOCKS because it is 2264 * possible that an extent is with unwritten and delayed 2265 * status due to when an extent is delayed allocated and 2266 * is allocated by fallocate status tree will track both of 2267 * them in a extent. 2268 * 2269 * So we could return a unwritten and delayed extent, and 2270 * its block is equal to 'next'. 2271 */ 2272 if (next == next_del && next == EXT_MAX_BLOCKS) { 2273 flags |= FIEMAP_EXTENT_LAST; 2274 if (unlikely(next_del != EXT_MAX_BLOCKS || 2275 next != EXT_MAX_BLOCKS)) { 2276 EXT4_ERROR_INODE(inode, 2277 "next extent == %u, next " 2278 "delalloc extent = %u", 2279 next, next_del); 2280 err = -EFSCORRUPTED; 2281 break; 2282 } 2283 } 2284 2285 if (exists) { 2286 err = fiemap_fill_next_extent(fieinfo, 2287 (__u64)es.es_lblk << blksize_bits, 2288 (__u64)es.es_pblk << blksize_bits, 2289 (__u64)es.es_len << blksize_bits, 2290 flags); 2291 if (err < 0) 2292 break; 2293 if (err == 1) { 2294 err = 0; 2295 break; 2296 } 2297 } 2298 2299 block = es.es_lblk + es.es_len; 2300 } 2301 2302 ext4_ext_drop_refs(path); 2303 kfree(path); 2304 return err; 2305 } 2306 2307 /* 2308 * ext4_ext_determine_hole - determine hole around given block 2309 * @inode: inode we lookup in 2310 * @path: path in extent tree to @lblk 2311 * @lblk: pointer to logical block around which we want to determine hole 2312 * 2313 * Determine hole length (and start if easily possible) around given logical 2314 * block. We don't try too hard to find the beginning of the hole but @path 2315 * actually points to extent before @lblk, we provide it. 2316 * 2317 * The function returns the length of a hole starting at @lblk. We update @lblk 2318 * to the beginning of the hole if we managed to find it. 2319 */ 2320 static ext4_lblk_t ext4_ext_determine_hole(struct inode *inode, 2321 struct ext4_ext_path *path, 2322 ext4_lblk_t *lblk) 2323 { 2324 int depth = ext_depth(inode); 2325 struct ext4_extent *ex; 2326 ext4_lblk_t len; 2327 2328 ex = path[depth].p_ext; 2329 if (ex == NULL) { 2330 /* there is no extent yet, so gap is [0;-] */ 2331 *lblk = 0; 2332 len = EXT_MAX_BLOCKS; 2333 } else if (*lblk < le32_to_cpu(ex->ee_block)) { 2334 len = le32_to_cpu(ex->ee_block) - *lblk; 2335 } else if (*lblk >= le32_to_cpu(ex->ee_block) 2336 + ext4_ext_get_actual_len(ex)) { 2337 ext4_lblk_t next; 2338 2339 *lblk = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); 2340 next = ext4_ext_next_allocated_block(path); 2341 BUG_ON(next == *lblk); 2342 len = next - *lblk; 2343 } else { 2344 BUG(); 2345 } 2346 return len; 2347 } 2348 2349 /* 2350 * ext4_ext_put_gap_in_cache: 2351 * calculate boundaries of the gap that the requested block fits into 2352 * and cache this gap 2353 */ 2354 static void 2355 ext4_ext_put_gap_in_cache(struct inode *inode, ext4_lblk_t hole_start, 2356 ext4_lblk_t hole_len) 2357 { 2358 struct extent_status es; 2359 2360 ext4_es_find_delayed_extent_range(inode, hole_start, 2361 hole_start + hole_len - 1, &es); 2362 if (es.es_len) { 2363 /* There's delayed extent containing lblock? */ 2364 if (es.es_lblk <= hole_start) 2365 return; 2366 hole_len = min(es.es_lblk - hole_start, hole_len); 2367 } 2368 ext_debug(" -> %u:%u\n", hole_start, hole_len); 2369 ext4_es_insert_extent(inode, hole_start, hole_len, ~0, 2370 EXTENT_STATUS_HOLE); 2371 } 2372 2373 /* 2374 * ext4_ext_rm_idx: 2375 * removes index from the index block. 2376 */ 2377 static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode, 2378 struct ext4_ext_path *path, int depth) 2379 { 2380 int err; 2381 ext4_fsblk_t leaf; 2382 2383 /* free index block */ 2384 depth--; 2385 path = path + depth; 2386 leaf = ext4_idx_pblock(path->p_idx); 2387 if (unlikely(path->p_hdr->eh_entries == 0)) { 2388 EXT4_ERROR_INODE(inode, "path->p_hdr->eh_entries == 0"); 2389 return -EFSCORRUPTED; 2390 } 2391 err = ext4_ext_get_access(handle, inode, path); 2392 if (err) 2393 return err; 2394 2395 if (path->p_idx != EXT_LAST_INDEX(path->p_hdr)) { 2396 int len = EXT_LAST_INDEX(path->p_hdr) - path->p_idx; 2397 len *= sizeof(struct ext4_extent_idx); 2398 memmove(path->p_idx, path->p_idx + 1, len); 2399 } 2400 2401 le16_add_cpu(&path->p_hdr->eh_entries, -1); 2402 err = ext4_ext_dirty(handle, inode, path); 2403 if (err) 2404 return err; 2405 ext_debug("index is empty, remove it, free block %llu\n", leaf); 2406 trace_ext4_ext_rm_idx(inode, leaf); 2407 2408 ext4_free_blocks(handle, inode, NULL, leaf, 1, 2409 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET); 2410 2411 while (--depth >= 0) { 2412 if (path->p_idx != EXT_FIRST_INDEX(path->p_hdr)) 2413 break; 2414 path--; 2415 err = ext4_ext_get_access(handle, inode, path); 2416 if (err) 2417 break; 2418 path->p_idx->ei_block = (path+1)->p_idx->ei_block; 2419 err = ext4_ext_dirty(handle, inode, path); 2420 if (err) 2421 break; 2422 } 2423 return err; 2424 } 2425 2426 /* 2427 * ext4_ext_calc_credits_for_single_extent: 2428 * This routine returns max. credits that needed to insert an extent 2429 * to the extent tree. 2430 * When pass the actual path, the caller should calculate credits 2431 * under i_data_sem. 2432 */ 2433 int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks, 2434 struct ext4_ext_path *path) 2435 { 2436 if (path) { 2437 int depth = ext_depth(inode); 2438 int ret = 0; 2439 2440 /* probably there is space in leaf? */ 2441 if (le16_to_cpu(path[depth].p_hdr->eh_entries) 2442 < le16_to_cpu(path[depth].p_hdr->eh_max)) { 2443 2444 /* 2445 * There are some space in the leaf tree, no 2446 * need to account for leaf block credit 2447 * 2448 * bitmaps and block group descriptor blocks 2449 * and other metadata blocks still need to be 2450 * accounted. 2451 */ 2452 /* 1 bitmap, 1 block group descriptor */ 2453 ret = 2 + EXT4_META_TRANS_BLOCKS(inode->i_sb); 2454 return ret; 2455 } 2456 } 2457 2458 return ext4_chunk_trans_blocks(inode, nrblocks); 2459 } 2460 2461 /* 2462 * How many index/leaf blocks need to change/allocate to add @extents extents? 2463 * 2464 * If we add a single extent, then in the worse case, each tree level 2465 * index/leaf need to be changed in case of the tree split. 2466 * 2467 * If more extents are inserted, they could cause the whole tree split more 2468 * than once, but this is really rare. 2469 */ 2470 int ext4_ext_index_trans_blocks(struct inode *inode, int extents) 2471 { 2472 int index; 2473 int depth; 2474 2475 /* If we are converting the inline data, only one is needed here. */ 2476 if (ext4_has_inline_data(inode)) 2477 return 1; 2478 2479 depth = ext_depth(inode); 2480 2481 if (extents <= 1) 2482 index = depth * 2; 2483 else 2484 index = depth * 3; 2485 2486 return index; 2487 } 2488 2489 static inline int get_default_free_blocks_flags(struct inode *inode) 2490 { 2491 if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) 2492 return EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET; 2493 else if (ext4_should_journal_data(inode)) 2494 return EXT4_FREE_BLOCKS_FORGET; 2495 return 0; 2496 } 2497 2498 static int ext4_remove_blocks(handle_t *handle, struct inode *inode, 2499 struct ext4_extent *ex, 2500 long long *partial_cluster, 2501 ext4_lblk_t from, ext4_lblk_t to) 2502 { 2503 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 2504 unsigned short ee_len = ext4_ext_get_actual_len(ex); 2505 ext4_fsblk_t pblk; 2506 int flags = get_default_free_blocks_flags(inode); 2507 2508 /* 2509 * For bigalloc file systems, we never free a partial cluster 2510 * at the beginning of the extent. Instead, we make a note 2511 * that we tried freeing the cluster, and check to see if we 2512 * need to free it on a subsequent call to ext4_remove_blocks, 2513 * or at the end of ext4_ext_rm_leaf or ext4_ext_remove_space. 2514 */ 2515 flags |= EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER; 2516 2517 trace_ext4_remove_blocks(inode, ex, from, to, *partial_cluster); 2518 /* 2519 * If we have a partial cluster, and it's different from the 2520 * cluster of the last block, we need to explicitly free the 2521 * partial cluster here. 2522 */ 2523 pblk = ext4_ext_pblock(ex) + ee_len - 1; 2524 if (*partial_cluster > 0 && 2525 *partial_cluster != (long long) EXT4_B2C(sbi, pblk)) { 2526 ext4_free_blocks(handle, inode, NULL, 2527 EXT4_C2B(sbi, *partial_cluster), 2528 sbi->s_cluster_ratio, flags); 2529 *partial_cluster = 0; 2530 } 2531 2532 #ifdef EXTENTS_STATS 2533 { 2534 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 2535 spin_lock(&sbi->s_ext_stats_lock); 2536 sbi->s_ext_blocks += ee_len; 2537 sbi->s_ext_extents++; 2538 if (ee_len < sbi->s_ext_min) 2539 sbi->s_ext_min = ee_len; 2540 if (ee_len > sbi->s_ext_max) 2541 sbi->s_ext_max = ee_len; 2542 if (ext_depth(inode) > sbi->s_depth_max) 2543 sbi->s_depth_max = ext_depth(inode); 2544 spin_unlock(&sbi->s_ext_stats_lock); 2545 } 2546 #endif 2547 if (from >= le32_to_cpu(ex->ee_block) 2548 && to == le32_to_cpu(ex->ee_block) + ee_len - 1) { 2549 /* tail removal */ 2550 ext4_lblk_t num; 2551 long long first_cluster; 2552 2553 num = le32_to_cpu(ex->ee_block) + ee_len - from; 2554 pblk = ext4_ext_pblock(ex) + ee_len - num; 2555 /* 2556 * Usually we want to free partial cluster at the end of the 2557 * extent, except for the situation when the cluster is still 2558 * used by any other extent (partial_cluster is negative). 2559 */ 2560 if (*partial_cluster < 0 && 2561 *partial_cluster == -(long long) EXT4_B2C(sbi, pblk+num-1)) 2562 flags |= EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER; 2563 2564 ext_debug("free last %u blocks starting %llu partial %lld\n", 2565 num, pblk, *partial_cluster); 2566 ext4_free_blocks(handle, inode, NULL, pblk, num, flags); 2567 /* 2568 * If the block range to be freed didn't start at the 2569 * beginning of a cluster, and we removed the entire 2570 * extent and the cluster is not used by any other extent, 2571 * save the partial cluster here, since we might need to 2572 * delete if we determine that the truncate or punch hole 2573 * operation has removed all of the blocks in the cluster. 2574 * If that cluster is used by another extent, preserve its 2575 * negative value so it isn't freed later on. 2576 * 2577 * If the whole extent wasn't freed, we've reached the 2578 * start of the truncated/punched region and have finished 2579 * removing blocks. If there's a partial cluster here it's 2580 * shared with the remainder of the extent and is no longer 2581 * a candidate for removal. 2582 */ 2583 if (EXT4_PBLK_COFF(sbi, pblk) && ee_len == num) { 2584 first_cluster = (long long) EXT4_B2C(sbi, pblk); 2585 if (first_cluster != -*partial_cluster) 2586 *partial_cluster = first_cluster; 2587 } else { 2588 *partial_cluster = 0; 2589 } 2590 } else 2591 ext4_error(sbi->s_sb, "strange request: removal(2) " 2592 "%u-%u from %u:%u", 2593 from, to, le32_to_cpu(ex->ee_block), ee_len); 2594 return 0; 2595 } 2596 2597 2598 /* 2599 * ext4_ext_rm_leaf() Removes the extents associated with the 2600 * blocks appearing between "start" and "end". Both "start" 2601 * and "end" must appear in the same extent or EIO is returned. 2602 * 2603 * @handle: The journal handle 2604 * @inode: The files inode 2605 * @path: The path to the leaf 2606 * @partial_cluster: The cluster which we'll have to free if all extents 2607 * has been released from it. However, if this value is 2608 * negative, it's a cluster just to the right of the 2609 * punched region and it must not be freed. 2610 * @start: The first block to remove 2611 * @end: The last block to remove 2612 */ 2613 static int 2614 ext4_ext_rm_leaf(handle_t *handle, struct inode *inode, 2615 struct ext4_ext_path *path, 2616 long long *partial_cluster, 2617 ext4_lblk_t start, ext4_lblk_t end) 2618 { 2619 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 2620 int err = 0, correct_index = 0; 2621 int depth = ext_depth(inode), credits; 2622 struct ext4_extent_header *eh; 2623 ext4_lblk_t a, b; 2624 unsigned num; 2625 ext4_lblk_t ex_ee_block; 2626 unsigned short ex_ee_len; 2627 unsigned unwritten = 0; 2628 struct ext4_extent *ex; 2629 ext4_fsblk_t pblk; 2630 2631 /* the header must be checked already in ext4_ext_remove_space() */ 2632 ext_debug("truncate since %u in leaf to %u\n", start, end); 2633 if (!path[depth].p_hdr) 2634 path[depth].p_hdr = ext_block_hdr(path[depth].p_bh); 2635 eh = path[depth].p_hdr; 2636 if (unlikely(path[depth].p_hdr == NULL)) { 2637 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth); 2638 return -EFSCORRUPTED; 2639 } 2640 /* find where to start removing */ 2641 ex = path[depth].p_ext; 2642 if (!ex) 2643 ex = EXT_LAST_EXTENT(eh); 2644 2645 ex_ee_block = le32_to_cpu(ex->ee_block); 2646 ex_ee_len = ext4_ext_get_actual_len(ex); 2647 2648 trace_ext4_ext_rm_leaf(inode, start, ex, *partial_cluster); 2649 2650 while (ex >= EXT_FIRST_EXTENT(eh) && 2651 ex_ee_block + ex_ee_len > start) { 2652 2653 if (ext4_ext_is_unwritten(ex)) 2654 unwritten = 1; 2655 else 2656 unwritten = 0; 2657 2658 ext_debug("remove ext %u:[%d]%d\n", ex_ee_block, 2659 unwritten, ex_ee_len); 2660 path[depth].p_ext = ex; 2661 2662 a = ex_ee_block > start ? ex_ee_block : start; 2663 b = ex_ee_block+ex_ee_len - 1 < end ? 2664 ex_ee_block+ex_ee_len - 1 : end; 2665 2666 ext_debug(" border %u:%u\n", a, b); 2667 2668 /* If this extent is beyond the end of the hole, skip it */ 2669 if (end < ex_ee_block) { 2670 /* 2671 * We're going to skip this extent and move to another, 2672 * so note that its first cluster is in use to avoid 2673 * freeing it when removing blocks. Eventually, the 2674 * right edge of the truncated/punched region will 2675 * be just to the left. 2676 */ 2677 if (sbi->s_cluster_ratio > 1) { 2678 pblk = ext4_ext_pblock(ex); 2679 *partial_cluster = 2680 -(long long) EXT4_B2C(sbi, pblk); 2681 } 2682 ex--; 2683 ex_ee_block = le32_to_cpu(ex->ee_block); 2684 ex_ee_len = ext4_ext_get_actual_len(ex); 2685 continue; 2686 } else if (b != ex_ee_block + ex_ee_len - 1) { 2687 EXT4_ERROR_INODE(inode, 2688 "can not handle truncate %u:%u " 2689 "on extent %u:%u", 2690 start, end, ex_ee_block, 2691 ex_ee_block + ex_ee_len - 1); 2692 err = -EFSCORRUPTED; 2693 goto out; 2694 } else if (a != ex_ee_block) { 2695 /* remove tail of the extent */ 2696 num = a - ex_ee_block; 2697 } else { 2698 /* remove whole extent: excellent! */ 2699 num = 0; 2700 } 2701 /* 2702 * 3 for leaf, sb, and inode plus 2 (bmap and group 2703 * descriptor) for each block group; assume two block 2704 * groups plus ex_ee_len/blocks_per_block_group for 2705 * the worst case 2706 */ 2707 credits = 7 + 2*(ex_ee_len/EXT4_BLOCKS_PER_GROUP(inode->i_sb)); 2708 if (ex == EXT_FIRST_EXTENT(eh)) { 2709 correct_index = 1; 2710 credits += (ext_depth(inode)) + 1; 2711 } 2712 credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb); 2713 2714 err = ext4_ext_truncate_extend_restart(handle, inode, credits); 2715 if (err) 2716 goto out; 2717 2718 err = ext4_ext_get_access(handle, inode, path + depth); 2719 if (err) 2720 goto out; 2721 2722 err = ext4_remove_blocks(handle, inode, ex, partial_cluster, 2723 a, b); 2724 if (err) 2725 goto out; 2726 2727 if (num == 0) 2728 /* this extent is removed; mark slot entirely unused */ 2729 ext4_ext_store_pblock(ex, 0); 2730 2731 ex->ee_len = cpu_to_le16(num); 2732 /* 2733 * Do not mark unwritten if all the blocks in the 2734 * extent have been removed. 2735 */ 2736 if (unwritten && num) 2737 ext4_ext_mark_unwritten(ex); 2738 /* 2739 * If the extent was completely released, 2740 * we need to remove it from the leaf 2741 */ 2742 if (num == 0) { 2743 if (end != EXT_MAX_BLOCKS - 1) { 2744 /* 2745 * For hole punching, we need to scoot all the 2746 * extents up when an extent is removed so that 2747 * we dont have blank extents in the middle 2748 */ 2749 memmove(ex, ex+1, (EXT_LAST_EXTENT(eh) - ex) * 2750 sizeof(struct ext4_extent)); 2751 2752 /* Now get rid of the one at the end */ 2753 memset(EXT_LAST_EXTENT(eh), 0, 2754 sizeof(struct ext4_extent)); 2755 } 2756 le16_add_cpu(&eh->eh_entries, -1); 2757 } 2758 2759 err = ext4_ext_dirty(handle, inode, path + depth); 2760 if (err) 2761 goto out; 2762 2763 ext_debug("new extent: %u:%u:%llu\n", ex_ee_block, num, 2764 ext4_ext_pblock(ex)); 2765 ex--; 2766 ex_ee_block = le32_to_cpu(ex->ee_block); 2767 ex_ee_len = ext4_ext_get_actual_len(ex); 2768 } 2769 2770 if (correct_index && eh->eh_entries) 2771 err = ext4_ext_correct_indexes(handle, inode, path); 2772 2773 /* 2774 * If there's a partial cluster and at least one extent remains in 2775 * the leaf, free the partial cluster if it isn't shared with the 2776 * current extent. If it is shared with the current extent 2777 * we zero partial_cluster because we've reached the start of the 2778 * truncated/punched region and we're done removing blocks. 2779 */ 2780 if (*partial_cluster > 0 && ex >= EXT_FIRST_EXTENT(eh)) { 2781 pblk = ext4_ext_pblock(ex) + ex_ee_len - 1; 2782 if (*partial_cluster != (long long) EXT4_B2C(sbi, pblk)) { 2783 ext4_free_blocks(handle, inode, NULL, 2784 EXT4_C2B(sbi, *partial_cluster), 2785 sbi->s_cluster_ratio, 2786 get_default_free_blocks_flags(inode)); 2787 } 2788 *partial_cluster = 0; 2789 } 2790 2791 /* if this leaf is free, then we should 2792 * remove it from index block above */ 2793 if (err == 0 && eh->eh_entries == 0 && path[depth].p_bh != NULL) 2794 err = ext4_ext_rm_idx(handle, inode, path, depth); 2795 2796 out: 2797 return err; 2798 } 2799 2800 /* 2801 * ext4_ext_more_to_rm: 2802 * returns 1 if current index has to be freed (even partial) 2803 */ 2804 static int 2805 ext4_ext_more_to_rm(struct ext4_ext_path *path) 2806 { 2807 BUG_ON(path->p_idx == NULL); 2808 2809 if (path->p_idx < EXT_FIRST_INDEX(path->p_hdr)) 2810 return 0; 2811 2812 /* 2813 * if truncate on deeper level happened, it wasn't partial, 2814 * so we have to consider current index for truncation 2815 */ 2816 if (le16_to_cpu(path->p_hdr->eh_entries) == path->p_block) 2817 return 0; 2818 return 1; 2819 } 2820 2821 int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start, 2822 ext4_lblk_t end) 2823 { 2824 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 2825 int depth = ext_depth(inode); 2826 struct ext4_ext_path *path = NULL; 2827 long long partial_cluster = 0; 2828 handle_t *handle; 2829 int i = 0, err = 0; 2830 2831 ext_debug("truncate since %u to %u\n", start, end); 2832 2833 /* probably first extent we're gonna free will be last in block */ 2834 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, depth + 1); 2835 if (IS_ERR(handle)) 2836 return PTR_ERR(handle); 2837 2838 again: 2839 trace_ext4_ext_remove_space(inode, start, end, depth); 2840 2841 /* 2842 * Check if we are removing extents inside the extent tree. If that 2843 * is the case, we are going to punch a hole inside the extent tree 2844 * so we have to check whether we need to split the extent covering 2845 * the last block to remove so we can easily remove the part of it 2846 * in ext4_ext_rm_leaf(). 2847 */ 2848 if (end < EXT_MAX_BLOCKS - 1) { 2849 struct ext4_extent *ex; 2850 ext4_lblk_t ee_block, ex_end, lblk; 2851 ext4_fsblk_t pblk; 2852 2853 /* find extent for or closest extent to this block */ 2854 path = ext4_find_extent(inode, end, NULL, EXT4_EX_NOCACHE); 2855 if (IS_ERR(path)) { 2856 ext4_journal_stop(handle); 2857 return PTR_ERR(path); 2858 } 2859 depth = ext_depth(inode); 2860 /* Leaf not may not exist only if inode has no blocks at all */ 2861 ex = path[depth].p_ext; 2862 if (!ex) { 2863 if (depth) { 2864 EXT4_ERROR_INODE(inode, 2865 "path[%d].p_hdr == NULL", 2866 depth); 2867 err = -EFSCORRUPTED; 2868 } 2869 goto out; 2870 } 2871 2872 ee_block = le32_to_cpu(ex->ee_block); 2873 ex_end = ee_block + ext4_ext_get_actual_len(ex) - 1; 2874 2875 /* 2876 * See if the last block is inside the extent, if so split 2877 * the extent at 'end' block so we can easily remove the 2878 * tail of the first part of the split extent in 2879 * ext4_ext_rm_leaf(). 2880 */ 2881 if (end >= ee_block && end < ex_end) { 2882 2883 /* 2884 * If we're going to split the extent, note that 2885 * the cluster containing the block after 'end' is 2886 * in use to avoid freeing it when removing blocks. 2887 */ 2888 if (sbi->s_cluster_ratio > 1) { 2889 pblk = ext4_ext_pblock(ex) + end - ee_block + 2; 2890 partial_cluster = 2891 -(long long) EXT4_B2C(sbi, pblk); 2892 } 2893 2894 /* 2895 * Split the extent in two so that 'end' is the last 2896 * block in the first new extent. Also we should not 2897 * fail removing space due to ENOSPC so try to use 2898 * reserved block if that happens. 2899 */ 2900 err = ext4_force_split_extent_at(handle, inode, &path, 2901 end + 1, 1); 2902 if (err < 0) 2903 goto out; 2904 2905 } else if (sbi->s_cluster_ratio > 1 && end >= ex_end) { 2906 /* 2907 * If there's an extent to the right its first cluster 2908 * contains the immediate right boundary of the 2909 * truncated/punched region. Set partial_cluster to 2910 * its negative value so it won't be freed if shared 2911 * with the current extent. The end < ee_block case 2912 * is handled in ext4_ext_rm_leaf(). 2913 */ 2914 lblk = ex_end + 1; 2915 err = ext4_ext_search_right(inode, path, &lblk, &pblk, 2916 &ex); 2917 if (err) 2918 goto out; 2919 if (pblk) 2920 partial_cluster = 2921 -(long long) EXT4_B2C(sbi, pblk); 2922 } 2923 } 2924 /* 2925 * We start scanning from right side, freeing all the blocks 2926 * after i_size and walking into the tree depth-wise. 2927 */ 2928 depth = ext_depth(inode); 2929 if (path) { 2930 int k = i = depth; 2931 while (--k > 0) 2932 path[k].p_block = 2933 le16_to_cpu(path[k].p_hdr->eh_entries)+1; 2934 } else { 2935 path = kzalloc(sizeof(struct ext4_ext_path) * (depth + 1), 2936 GFP_NOFS); 2937 if (path == NULL) { 2938 ext4_journal_stop(handle); 2939 return -ENOMEM; 2940 } 2941 path[0].p_maxdepth = path[0].p_depth = depth; 2942 path[0].p_hdr = ext_inode_hdr(inode); 2943 i = 0; 2944 2945 if (ext4_ext_check(inode, path[0].p_hdr, depth, 0)) { 2946 err = -EFSCORRUPTED; 2947 goto out; 2948 } 2949 } 2950 err = 0; 2951 2952 while (i >= 0 && err == 0) { 2953 if (i == depth) { 2954 /* this is leaf block */ 2955 err = ext4_ext_rm_leaf(handle, inode, path, 2956 &partial_cluster, start, 2957 end); 2958 /* root level has p_bh == NULL, brelse() eats this */ 2959 brelse(path[i].p_bh); 2960 path[i].p_bh = NULL; 2961 i--; 2962 continue; 2963 } 2964 2965 /* this is index block */ 2966 if (!path[i].p_hdr) { 2967 ext_debug("initialize header\n"); 2968 path[i].p_hdr = ext_block_hdr(path[i].p_bh); 2969 } 2970 2971 if (!path[i].p_idx) { 2972 /* this level hasn't been touched yet */ 2973 path[i].p_idx = EXT_LAST_INDEX(path[i].p_hdr); 2974 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries)+1; 2975 ext_debug("init index ptr: hdr 0x%p, num %d\n", 2976 path[i].p_hdr, 2977 le16_to_cpu(path[i].p_hdr->eh_entries)); 2978 } else { 2979 /* we were already here, see at next index */ 2980 path[i].p_idx--; 2981 } 2982 2983 ext_debug("level %d - index, first 0x%p, cur 0x%p\n", 2984 i, EXT_FIRST_INDEX(path[i].p_hdr), 2985 path[i].p_idx); 2986 if (ext4_ext_more_to_rm(path + i)) { 2987 struct buffer_head *bh; 2988 /* go to the next level */ 2989 ext_debug("move to level %d (block %llu)\n", 2990 i + 1, ext4_idx_pblock(path[i].p_idx)); 2991 memset(path + i + 1, 0, sizeof(*path)); 2992 bh = read_extent_tree_block(inode, 2993 ext4_idx_pblock(path[i].p_idx), depth - i - 1, 2994 EXT4_EX_NOCACHE); 2995 if (IS_ERR(bh)) { 2996 /* should we reset i_size? */ 2997 err = PTR_ERR(bh); 2998 break; 2999 } 3000 /* Yield here to deal with large extent trees. 3001 * Should be a no-op if we did IO above. */ 3002 cond_resched(); 3003 if (WARN_ON(i + 1 > depth)) { 3004 err = -EFSCORRUPTED; 3005 break; 3006 } 3007 path[i + 1].p_bh = bh; 3008 3009 /* save actual number of indexes since this 3010 * number is changed at the next iteration */ 3011 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries); 3012 i++; 3013 } else { 3014 /* we finished processing this index, go up */ 3015 if (path[i].p_hdr->eh_entries == 0 && i > 0) { 3016 /* index is empty, remove it; 3017 * handle must be already prepared by the 3018 * truncatei_leaf() */ 3019 err = ext4_ext_rm_idx(handle, inode, path, i); 3020 } 3021 /* root level has p_bh == NULL, brelse() eats this */ 3022 brelse(path[i].p_bh); 3023 path[i].p_bh = NULL; 3024 i--; 3025 ext_debug("return to level %d\n", i); 3026 } 3027 } 3028 3029 trace_ext4_ext_remove_space_done(inode, start, end, depth, 3030 partial_cluster, path->p_hdr->eh_entries); 3031 3032 /* 3033 * If we still have something in the partial cluster and we have removed 3034 * even the first extent, then we should free the blocks in the partial 3035 * cluster as well. (This code will only run when there are no leaves 3036 * to the immediate left of the truncated/punched region.) 3037 */ 3038 if (partial_cluster > 0 && err == 0) { 3039 /* don't zero partial_cluster since it's not used afterwards */ 3040 ext4_free_blocks(handle, inode, NULL, 3041 EXT4_C2B(sbi, partial_cluster), 3042 sbi->s_cluster_ratio, 3043 get_default_free_blocks_flags(inode)); 3044 } 3045 3046 /* TODO: flexible tree reduction should be here */ 3047 if (path->p_hdr->eh_entries == 0) { 3048 /* 3049 * truncate to zero freed all the tree, 3050 * so we need to correct eh_depth 3051 */ 3052 err = ext4_ext_get_access(handle, inode, path); 3053 if (err == 0) { 3054 ext_inode_hdr(inode)->eh_depth = 0; 3055 ext_inode_hdr(inode)->eh_max = 3056 cpu_to_le16(ext4_ext_space_root(inode, 0)); 3057 err = ext4_ext_dirty(handle, inode, path); 3058 } 3059 } 3060 out: 3061 ext4_ext_drop_refs(path); 3062 kfree(path); 3063 path = NULL; 3064 if (err == -EAGAIN) 3065 goto again; 3066 ext4_journal_stop(handle); 3067 3068 return err; 3069 } 3070 3071 /* 3072 * called at mount time 3073 */ 3074 void ext4_ext_init(struct super_block *sb) 3075 { 3076 /* 3077 * possible initialization would be here 3078 */ 3079 3080 if (ext4_has_feature_extents(sb)) { 3081 #if defined(AGGRESSIVE_TEST) || defined(CHECK_BINSEARCH) || defined(EXTENTS_STATS) 3082 printk(KERN_INFO "EXT4-fs: file extents enabled" 3083 #ifdef AGGRESSIVE_TEST 3084 ", aggressive tests" 3085 #endif 3086 #ifdef CHECK_BINSEARCH 3087 ", check binsearch" 3088 #endif 3089 #ifdef EXTENTS_STATS 3090 ", stats" 3091 #endif 3092 "\n"); 3093 #endif 3094 #ifdef EXTENTS_STATS 3095 spin_lock_init(&EXT4_SB(sb)->s_ext_stats_lock); 3096 EXT4_SB(sb)->s_ext_min = 1 << 30; 3097 EXT4_SB(sb)->s_ext_max = 0; 3098 #endif 3099 } 3100 } 3101 3102 /* 3103 * called at umount time 3104 */ 3105 void ext4_ext_release(struct super_block *sb) 3106 { 3107 if (!ext4_has_feature_extents(sb)) 3108 return; 3109 3110 #ifdef EXTENTS_STATS 3111 if (EXT4_SB(sb)->s_ext_blocks && EXT4_SB(sb)->s_ext_extents) { 3112 struct ext4_sb_info *sbi = EXT4_SB(sb); 3113 printk(KERN_ERR "EXT4-fs: %lu blocks in %lu extents (%lu ave)\n", 3114 sbi->s_ext_blocks, sbi->s_ext_extents, 3115 sbi->s_ext_blocks / sbi->s_ext_extents); 3116 printk(KERN_ERR "EXT4-fs: extents: %lu min, %lu max, max depth %lu\n", 3117 sbi->s_ext_min, sbi->s_ext_max, sbi->s_depth_max); 3118 } 3119 #endif 3120 } 3121 3122 static int ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex) 3123 { 3124 ext4_lblk_t ee_block; 3125 ext4_fsblk_t ee_pblock; 3126 unsigned int ee_len; 3127 3128 ee_block = le32_to_cpu(ex->ee_block); 3129 ee_len = ext4_ext_get_actual_len(ex); 3130 ee_pblock = ext4_ext_pblock(ex); 3131 3132 if (ee_len == 0) 3133 return 0; 3134 3135 return ext4_es_insert_extent(inode, ee_block, ee_len, ee_pblock, 3136 EXTENT_STATUS_WRITTEN); 3137 } 3138 3139 /* FIXME!! we need to try to merge to left or right after zero-out */ 3140 static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex) 3141 { 3142 ext4_fsblk_t ee_pblock; 3143 unsigned int ee_len; 3144 3145 ee_len = ext4_ext_get_actual_len(ex); 3146 ee_pblock = ext4_ext_pblock(ex); 3147 return ext4_issue_zeroout(inode, le32_to_cpu(ex->ee_block), ee_pblock, 3148 ee_len); 3149 } 3150 3151 /* 3152 * ext4_split_extent_at() splits an extent at given block. 3153 * 3154 * @handle: the journal handle 3155 * @inode: the file inode 3156 * @path: the path to the extent 3157 * @split: the logical block where the extent is splitted. 3158 * @split_flags: indicates if the extent could be zeroout if split fails, and 3159 * the states(init or unwritten) of new extents. 3160 * @flags: flags used to insert new extent to extent tree. 3161 * 3162 * 3163 * Splits extent [a, b] into two extents [a, @split) and [@split, b], states 3164 * of which are deterimined by split_flag. 3165 * 3166 * There are two cases: 3167 * a> the extent are splitted into two extent. 3168 * b> split is not needed, and just mark the extent. 3169 * 3170 * return 0 on success. 3171 */ 3172 static int ext4_split_extent_at(handle_t *handle, 3173 struct inode *inode, 3174 struct ext4_ext_path **ppath, 3175 ext4_lblk_t split, 3176 int split_flag, 3177 int flags) 3178 { 3179 struct ext4_ext_path *path = *ppath; 3180 ext4_fsblk_t newblock; 3181 ext4_lblk_t ee_block; 3182 struct ext4_extent *ex, newex, orig_ex, zero_ex; 3183 struct ext4_extent *ex2 = NULL; 3184 unsigned int ee_len, depth; 3185 int err = 0; 3186 3187 BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) == 3188 (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)); 3189 3190 ext_debug("ext4_split_extents_at: inode %lu, logical" 3191 "block %llu\n", inode->i_ino, (unsigned long long)split); 3192 3193 ext4_ext_show_leaf(inode, path); 3194 3195 depth = ext_depth(inode); 3196 ex = path[depth].p_ext; 3197 ee_block = le32_to_cpu(ex->ee_block); 3198 ee_len = ext4_ext_get_actual_len(ex); 3199 newblock = split - ee_block + ext4_ext_pblock(ex); 3200 3201 BUG_ON(split < ee_block || split >= (ee_block + ee_len)); 3202 BUG_ON(!ext4_ext_is_unwritten(ex) && 3203 split_flag & (EXT4_EXT_MAY_ZEROOUT | 3204 EXT4_EXT_MARK_UNWRIT1 | 3205 EXT4_EXT_MARK_UNWRIT2)); 3206 3207 err = ext4_ext_get_access(handle, inode, path + depth); 3208 if (err) 3209 goto out; 3210 3211 if (split == ee_block) { 3212 /* 3213 * case b: block @split is the block that the extent begins with 3214 * then we just change the state of the extent, and splitting 3215 * is not needed. 3216 */ 3217 if (split_flag & EXT4_EXT_MARK_UNWRIT2) 3218 ext4_ext_mark_unwritten(ex); 3219 else 3220 ext4_ext_mark_initialized(ex); 3221 3222 if (!(flags & EXT4_GET_BLOCKS_PRE_IO)) 3223 ext4_ext_try_to_merge(handle, inode, path, ex); 3224 3225 err = ext4_ext_dirty(handle, inode, path + path->p_depth); 3226 goto out; 3227 } 3228 3229 /* case a */ 3230 memcpy(&orig_ex, ex, sizeof(orig_ex)); 3231 ex->ee_len = cpu_to_le16(split - ee_block); 3232 if (split_flag & EXT4_EXT_MARK_UNWRIT1) 3233 ext4_ext_mark_unwritten(ex); 3234 3235 /* 3236 * path may lead to new leaf, not to original leaf any more 3237 * after ext4_ext_insert_extent() returns, 3238 */ 3239 err = ext4_ext_dirty(handle, inode, path + depth); 3240 if (err) 3241 goto fix_extent_len; 3242 3243 ex2 = &newex; 3244 ex2->ee_block = cpu_to_le32(split); 3245 ex2->ee_len = cpu_to_le16(ee_len - (split - ee_block)); 3246 ext4_ext_store_pblock(ex2, newblock); 3247 if (split_flag & EXT4_EXT_MARK_UNWRIT2) 3248 ext4_ext_mark_unwritten(ex2); 3249 3250 err = ext4_ext_insert_extent(handle, inode, ppath, &newex, flags); 3251 if (err == -ENOSPC && (EXT4_EXT_MAY_ZEROOUT & split_flag)) { 3252 if (split_flag & (EXT4_EXT_DATA_VALID1|EXT4_EXT_DATA_VALID2)) { 3253 if (split_flag & EXT4_EXT_DATA_VALID1) { 3254 err = ext4_ext_zeroout(inode, ex2); 3255 zero_ex.ee_block = ex2->ee_block; 3256 zero_ex.ee_len = cpu_to_le16( 3257 ext4_ext_get_actual_len(ex2)); 3258 ext4_ext_store_pblock(&zero_ex, 3259 ext4_ext_pblock(ex2)); 3260 } else { 3261 err = ext4_ext_zeroout(inode, ex); 3262 zero_ex.ee_block = ex->ee_block; 3263 zero_ex.ee_len = cpu_to_le16( 3264 ext4_ext_get_actual_len(ex)); 3265 ext4_ext_store_pblock(&zero_ex, 3266 ext4_ext_pblock(ex)); 3267 } 3268 } else { 3269 err = ext4_ext_zeroout(inode, &orig_ex); 3270 zero_ex.ee_block = orig_ex.ee_block; 3271 zero_ex.ee_len = cpu_to_le16( 3272 ext4_ext_get_actual_len(&orig_ex)); 3273 ext4_ext_store_pblock(&zero_ex, 3274 ext4_ext_pblock(&orig_ex)); 3275 } 3276 3277 if (err) 3278 goto fix_extent_len; 3279 /* update the extent length and mark as initialized */ 3280 ex->ee_len = cpu_to_le16(ee_len); 3281 ext4_ext_try_to_merge(handle, inode, path, ex); 3282 err = ext4_ext_dirty(handle, inode, path + path->p_depth); 3283 if (err) 3284 goto fix_extent_len; 3285 3286 /* update extent status tree */ 3287 err = ext4_zeroout_es(inode, &zero_ex); 3288 3289 goto out; 3290 } else if (err) 3291 goto fix_extent_len; 3292 3293 out: 3294 ext4_ext_show_leaf(inode, path); 3295 return err; 3296 3297 fix_extent_len: 3298 ex->ee_len = orig_ex.ee_len; 3299 ext4_ext_dirty(handle, inode, path + path->p_depth); 3300 return err; 3301 } 3302 3303 /* 3304 * ext4_split_extents() splits an extent and mark extent which is covered 3305 * by @map as split_flags indicates 3306 * 3307 * It may result in splitting the extent into multiple extents (up to three) 3308 * There are three possibilities: 3309 * a> There is no split required 3310 * b> Splits in two extents: Split is happening at either end of the extent 3311 * c> Splits in three extents: Somone is splitting in middle of the extent 3312 * 3313 */ 3314 static int ext4_split_extent(handle_t *handle, 3315 struct inode *inode, 3316 struct ext4_ext_path **ppath, 3317 struct ext4_map_blocks *map, 3318 int split_flag, 3319 int flags) 3320 { 3321 struct ext4_ext_path *path = *ppath; 3322 ext4_lblk_t ee_block; 3323 struct ext4_extent *ex; 3324 unsigned int ee_len, depth; 3325 int err = 0; 3326 int unwritten; 3327 int split_flag1, flags1; 3328 int allocated = map->m_len; 3329 3330 depth = ext_depth(inode); 3331 ex = path[depth].p_ext; 3332 ee_block = le32_to_cpu(ex->ee_block); 3333 ee_len = ext4_ext_get_actual_len(ex); 3334 unwritten = ext4_ext_is_unwritten(ex); 3335 3336 if (map->m_lblk + map->m_len < ee_block + ee_len) { 3337 split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT; 3338 flags1 = flags | EXT4_GET_BLOCKS_PRE_IO; 3339 if (unwritten) 3340 split_flag1 |= EXT4_EXT_MARK_UNWRIT1 | 3341 EXT4_EXT_MARK_UNWRIT2; 3342 if (split_flag & EXT4_EXT_DATA_VALID2) 3343 split_flag1 |= EXT4_EXT_DATA_VALID1; 3344 err = ext4_split_extent_at(handle, inode, ppath, 3345 map->m_lblk + map->m_len, split_flag1, flags1); 3346 if (err) 3347 goto out; 3348 } else { 3349 allocated = ee_len - (map->m_lblk - ee_block); 3350 } 3351 /* 3352 * Update path is required because previous ext4_split_extent_at() may 3353 * result in split of original leaf or extent zeroout. 3354 */ 3355 path = ext4_find_extent(inode, map->m_lblk, ppath, 0); 3356 if (IS_ERR(path)) 3357 return PTR_ERR(path); 3358 depth = ext_depth(inode); 3359 ex = path[depth].p_ext; 3360 if (!ex) { 3361 EXT4_ERROR_INODE(inode, "unexpected hole at %lu", 3362 (unsigned long) map->m_lblk); 3363 return -EFSCORRUPTED; 3364 } 3365 unwritten = ext4_ext_is_unwritten(ex); 3366 split_flag1 = 0; 3367 3368 if (map->m_lblk >= ee_block) { 3369 split_flag1 = split_flag & EXT4_EXT_DATA_VALID2; 3370 if (unwritten) { 3371 split_flag1 |= EXT4_EXT_MARK_UNWRIT1; 3372 split_flag1 |= split_flag & (EXT4_EXT_MAY_ZEROOUT | 3373 EXT4_EXT_MARK_UNWRIT2); 3374 } 3375 err = ext4_split_extent_at(handle, inode, ppath, 3376 map->m_lblk, split_flag1, flags); 3377 if (err) 3378 goto out; 3379 } 3380 3381 ext4_ext_show_leaf(inode, path); 3382 out: 3383 return err ? err : allocated; 3384 } 3385 3386 /* 3387 * This function is called by ext4_ext_map_blocks() if someone tries to write 3388 * to an unwritten extent. It may result in splitting the unwritten 3389 * extent into multiple extents (up to three - one initialized and two 3390 * unwritten). 3391 * There are three possibilities: 3392 * a> There is no split required: Entire extent should be initialized 3393 * b> Splits in two extents: Write is happening at either end of the extent 3394 * c> Splits in three extents: Somone is writing in middle of the extent 3395 * 3396 * Pre-conditions: 3397 * - The extent pointed to by 'path' is unwritten. 3398 * - The extent pointed to by 'path' contains a superset 3399 * of the logical span [map->m_lblk, map->m_lblk + map->m_len). 3400 * 3401 * Post-conditions on success: 3402 * - the returned value is the number of blocks beyond map->l_lblk 3403 * that are allocated and initialized. 3404 * It is guaranteed to be >= map->m_len. 3405 */ 3406 static int ext4_ext_convert_to_initialized(handle_t *handle, 3407 struct inode *inode, 3408 struct ext4_map_blocks *map, 3409 struct ext4_ext_path **ppath, 3410 int flags) 3411 { 3412 struct ext4_ext_path *path = *ppath; 3413 struct ext4_sb_info *sbi; 3414 struct ext4_extent_header *eh; 3415 struct ext4_map_blocks split_map; 3416 struct ext4_extent zero_ex; 3417 struct ext4_extent *ex, *abut_ex; 3418 ext4_lblk_t ee_block, eof_block; 3419 unsigned int ee_len, depth, map_len = map->m_len; 3420 int allocated = 0, max_zeroout = 0; 3421 int err = 0; 3422 int split_flag = 0; 3423 3424 ext_debug("ext4_ext_convert_to_initialized: inode %lu, logical" 3425 "block %llu, max_blocks %u\n", inode->i_ino, 3426 (unsigned long long)map->m_lblk, map_len); 3427 3428 sbi = EXT4_SB(inode->i_sb); 3429 eof_block = (inode->i_size + inode->i_sb->s_blocksize - 1) >> 3430 inode->i_sb->s_blocksize_bits; 3431 if (eof_block < map->m_lblk + map_len) 3432 eof_block = map->m_lblk + map_len; 3433 3434 depth = ext_depth(inode); 3435 eh = path[depth].p_hdr; 3436 ex = path[depth].p_ext; 3437 ee_block = le32_to_cpu(ex->ee_block); 3438 ee_len = ext4_ext_get_actual_len(ex); 3439 zero_ex.ee_len = 0; 3440 3441 trace_ext4_ext_convert_to_initialized_enter(inode, map, ex); 3442 3443 /* Pre-conditions */ 3444 BUG_ON(!ext4_ext_is_unwritten(ex)); 3445 BUG_ON(!in_range(map->m_lblk, ee_block, ee_len)); 3446 3447 /* 3448 * Attempt to transfer newly initialized blocks from the currently 3449 * unwritten extent to its neighbor. This is much cheaper 3450 * than an insertion followed by a merge as those involve costly 3451 * memmove() calls. Transferring to the left is the common case in 3452 * steady state for workloads doing fallocate(FALLOC_FL_KEEP_SIZE) 3453 * followed by append writes. 3454 * 3455 * Limitations of the current logic: 3456 * - L1: we do not deal with writes covering the whole extent. 3457 * This would require removing the extent if the transfer 3458 * is possible. 3459 * - L2: we only attempt to merge with an extent stored in the 3460 * same extent tree node. 3461 */ 3462 if ((map->m_lblk == ee_block) && 3463 /* See if we can merge left */ 3464 (map_len < ee_len) && /*L1*/ 3465 (ex > EXT_FIRST_EXTENT(eh))) { /*L2*/ 3466 ext4_lblk_t prev_lblk; 3467 ext4_fsblk_t prev_pblk, ee_pblk; 3468 unsigned int prev_len; 3469 3470 abut_ex = ex - 1; 3471 prev_lblk = le32_to_cpu(abut_ex->ee_block); 3472 prev_len = ext4_ext_get_actual_len(abut_ex); 3473 prev_pblk = ext4_ext_pblock(abut_ex); 3474 ee_pblk = ext4_ext_pblock(ex); 3475 3476 /* 3477 * A transfer of blocks from 'ex' to 'abut_ex' is allowed 3478 * upon those conditions: 3479 * - C1: abut_ex is initialized, 3480 * - C2: abut_ex is logically abutting ex, 3481 * - C3: abut_ex is physically abutting ex, 3482 * - C4: abut_ex can receive the additional blocks without 3483 * overflowing the (initialized) length limit. 3484 */ 3485 if ((!ext4_ext_is_unwritten(abut_ex)) && /*C1*/ 3486 ((prev_lblk + prev_len) == ee_block) && /*C2*/ 3487 ((prev_pblk + prev_len) == ee_pblk) && /*C3*/ 3488 (prev_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/ 3489 err = ext4_ext_get_access(handle, inode, path + depth); 3490 if (err) 3491 goto out; 3492 3493 trace_ext4_ext_convert_to_initialized_fastpath(inode, 3494 map, ex, abut_ex); 3495 3496 /* Shift the start of ex by 'map_len' blocks */ 3497 ex->ee_block = cpu_to_le32(ee_block + map_len); 3498 ext4_ext_store_pblock(ex, ee_pblk + map_len); 3499 ex->ee_len = cpu_to_le16(ee_len - map_len); 3500 ext4_ext_mark_unwritten(ex); /* Restore the flag */ 3501 3502 /* Extend abut_ex by 'map_len' blocks */ 3503 abut_ex->ee_len = cpu_to_le16(prev_len + map_len); 3504 3505 /* Result: number of initialized blocks past m_lblk */ 3506 allocated = map_len; 3507 } 3508 } else if (((map->m_lblk + map_len) == (ee_block + ee_len)) && 3509 (map_len < ee_len) && /*L1*/ 3510 ex < EXT_LAST_EXTENT(eh)) { /*L2*/ 3511 /* See if we can merge right */ 3512 ext4_lblk_t next_lblk; 3513 ext4_fsblk_t next_pblk, ee_pblk; 3514 unsigned int next_len; 3515 3516 abut_ex = ex + 1; 3517 next_lblk = le32_to_cpu(abut_ex->ee_block); 3518 next_len = ext4_ext_get_actual_len(abut_ex); 3519 next_pblk = ext4_ext_pblock(abut_ex); 3520 ee_pblk = ext4_ext_pblock(ex); 3521 3522 /* 3523 * A transfer of blocks from 'ex' to 'abut_ex' is allowed 3524 * upon those conditions: 3525 * - C1: abut_ex is initialized, 3526 * - C2: abut_ex is logically abutting ex, 3527 * - C3: abut_ex is physically abutting ex, 3528 * - C4: abut_ex can receive the additional blocks without 3529 * overflowing the (initialized) length limit. 3530 */ 3531 if ((!ext4_ext_is_unwritten(abut_ex)) && /*C1*/ 3532 ((map->m_lblk + map_len) == next_lblk) && /*C2*/ 3533 ((ee_pblk + ee_len) == next_pblk) && /*C3*/ 3534 (next_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/ 3535 err = ext4_ext_get_access(handle, inode, path + depth); 3536 if (err) 3537 goto out; 3538 3539 trace_ext4_ext_convert_to_initialized_fastpath(inode, 3540 map, ex, abut_ex); 3541 3542 /* Shift the start of abut_ex by 'map_len' blocks */ 3543 abut_ex->ee_block = cpu_to_le32(next_lblk - map_len); 3544 ext4_ext_store_pblock(abut_ex, next_pblk - map_len); 3545 ex->ee_len = cpu_to_le16(ee_len - map_len); 3546 ext4_ext_mark_unwritten(ex); /* Restore the flag */ 3547 3548 /* Extend abut_ex by 'map_len' blocks */ 3549 abut_ex->ee_len = cpu_to_le16(next_len + map_len); 3550 3551 /* Result: number of initialized blocks past m_lblk */ 3552 allocated = map_len; 3553 } 3554 } 3555 if (allocated) { 3556 /* Mark the block containing both extents as dirty */ 3557 ext4_ext_dirty(handle, inode, path + depth); 3558 3559 /* Update path to point to the right extent */ 3560 path[depth].p_ext = abut_ex; 3561 goto out; 3562 } else 3563 allocated = ee_len - (map->m_lblk - ee_block); 3564 3565 WARN_ON(map->m_lblk < ee_block); 3566 /* 3567 * It is safe to convert extent to initialized via explicit 3568 * zeroout only if extent is fully inside i_size or new_size. 3569 */ 3570 split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0; 3571 3572 if (EXT4_EXT_MAY_ZEROOUT & split_flag) 3573 max_zeroout = sbi->s_extent_max_zeroout_kb >> 3574 (inode->i_sb->s_blocksize_bits - 10); 3575 3576 if (ext4_encrypted_inode(inode)) 3577 max_zeroout = 0; 3578 3579 /* If extent is less than s_max_zeroout_kb, zeroout directly */ 3580 if (max_zeroout && (ee_len <= max_zeroout)) { 3581 err = ext4_ext_zeroout(inode, ex); 3582 if (err) 3583 goto out; 3584 zero_ex.ee_block = ex->ee_block; 3585 zero_ex.ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)); 3586 ext4_ext_store_pblock(&zero_ex, ext4_ext_pblock(ex)); 3587 3588 err = ext4_ext_get_access(handle, inode, path + depth); 3589 if (err) 3590 goto out; 3591 ext4_ext_mark_initialized(ex); 3592 ext4_ext_try_to_merge(handle, inode, path, ex); 3593 err = ext4_ext_dirty(handle, inode, path + path->p_depth); 3594 goto out; 3595 } 3596 3597 /* 3598 * four cases: 3599 * 1. split the extent into three extents. 3600 * 2. split the extent into two extents, zeroout the first half. 3601 * 3. split the extent into two extents, zeroout the second half. 3602 * 4. split the extent into two extents with out zeroout. 3603 */ 3604 split_map.m_lblk = map->m_lblk; 3605 split_map.m_len = map->m_len; 3606 3607 if (max_zeroout && (allocated > map->m_len)) { 3608 if (allocated <= max_zeroout) { 3609 /* case 3 */ 3610 zero_ex.ee_block = 3611 cpu_to_le32(map->m_lblk); 3612 zero_ex.ee_len = cpu_to_le16(allocated); 3613 ext4_ext_store_pblock(&zero_ex, 3614 ext4_ext_pblock(ex) + map->m_lblk - ee_block); 3615 err = ext4_ext_zeroout(inode, &zero_ex); 3616 if (err) 3617 goto out; 3618 split_map.m_lblk = map->m_lblk; 3619 split_map.m_len = allocated; 3620 } else if (map->m_lblk - ee_block + map->m_len < max_zeroout) { 3621 /* case 2 */ 3622 if (map->m_lblk != ee_block) { 3623 zero_ex.ee_block = ex->ee_block; 3624 zero_ex.ee_len = cpu_to_le16(map->m_lblk - 3625 ee_block); 3626 ext4_ext_store_pblock(&zero_ex, 3627 ext4_ext_pblock(ex)); 3628 err = ext4_ext_zeroout(inode, &zero_ex); 3629 if (err) 3630 goto out; 3631 } 3632 3633 split_map.m_lblk = ee_block; 3634 split_map.m_len = map->m_lblk - ee_block + map->m_len; 3635 allocated = map->m_len; 3636 } 3637 } 3638 3639 err = ext4_split_extent(handle, inode, ppath, &split_map, split_flag, 3640 flags); 3641 if (err > 0) 3642 err = 0; 3643 out: 3644 /* If we have gotten a failure, don't zero out status tree */ 3645 if (!err) 3646 err = ext4_zeroout_es(inode, &zero_ex); 3647 return err ? err : allocated; 3648 } 3649 3650 /* 3651 * This function is called by ext4_ext_map_blocks() from 3652 * ext4_get_blocks_dio_write() when DIO to write 3653 * to an unwritten extent. 3654 * 3655 * Writing to an unwritten extent may result in splitting the unwritten 3656 * extent into multiple initialized/unwritten extents (up to three) 3657 * There are three possibilities: 3658 * a> There is no split required: Entire extent should be unwritten 3659 * b> Splits in two extents: Write is happening at either end of the extent 3660 * c> Splits in three extents: Somone is writing in middle of the extent 3661 * 3662 * This works the same way in the case of initialized -> unwritten conversion. 3663 * 3664 * One of more index blocks maybe needed if the extent tree grow after 3665 * the unwritten extent split. To prevent ENOSPC occur at the IO 3666 * complete, we need to split the unwritten extent before DIO submit 3667 * the IO. The unwritten extent called at this time will be split 3668 * into three unwritten extent(at most). After IO complete, the part 3669 * being filled will be convert to initialized by the end_io callback function 3670 * via ext4_convert_unwritten_extents(). 3671 * 3672 * Returns the size of unwritten extent to be written on success. 3673 */ 3674 static int ext4_split_convert_extents(handle_t *handle, 3675 struct inode *inode, 3676 struct ext4_map_blocks *map, 3677 struct ext4_ext_path **ppath, 3678 int flags) 3679 { 3680 struct ext4_ext_path *path = *ppath; 3681 ext4_lblk_t eof_block; 3682 ext4_lblk_t ee_block; 3683 struct ext4_extent *ex; 3684 unsigned int ee_len; 3685 int split_flag = 0, depth; 3686 3687 ext_debug("%s: inode %lu, logical block %llu, max_blocks %u\n", 3688 __func__, inode->i_ino, 3689 (unsigned long long)map->m_lblk, map->m_len); 3690 3691 eof_block = (inode->i_size + inode->i_sb->s_blocksize - 1) >> 3692 inode->i_sb->s_blocksize_bits; 3693 if (eof_block < map->m_lblk + map->m_len) 3694 eof_block = map->m_lblk + map->m_len; 3695 /* 3696 * It is safe to convert extent to initialized via explicit 3697 * zeroout only if extent is fully insde i_size or new_size. 3698 */ 3699 depth = ext_depth(inode); 3700 ex = path[depth].p_ext; 3701 ee_block = le32_to_cpu(ex->ee_block); 3702 ee_len = ext4_ext_get_actual_len(ex); 3703 3704 /* Convert to unwritten */ 3705 if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) { 3706 split_flag |= EXT4_EXT_DATA_VALID1; 3707 /* Convert to initialized */ 3708 } else if (flags & EXT4_GET_BLOCKS_CONVERT) { 3709 split_flag |= ee_block + ee_len <= eof_block ? 3710 EXT4_EXT_MAY_ZEROOUT : 0; 3711 split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2); 3712 } 3713 flags |= EXT4_GET_BLOCKS_PRE_IO; 3714 return ext4_split_extent(handle, inode, ppath, map, split_flag, flags); 3715 } 3716 3717 static int ext4_convert_unwritten_extents_endio(handle_t *handle, 3718 struct inode *inode, 3719 struct ext4_map_blocks *map, 3720 struct ext4_ext_path **ppath) 3721 { 3722 struct ext4_ext_path *path = *ppath; 3723 struct ext4_extent *ex; 3724 ext4_lblk_t ee_block; 3725 unsigned int ee_len; 3726 int depth; 3727 int err = 0; 3728 3729 depth = ext_depth(inode); 3730 ex = path[depth].p_ext; 3731 ee_block = le32_to_cpu(ex->ee_block); 3732 ee_len = ext4_ext_get_actual_len(ex); 3733 3734 ext_debug("ext4_convert_unwritten_extents_endio: inode %lu, logical" 3735 "block %llu, max_blocks %u\n", inode->i_ino, 3736 (unsigned long long)ee_block, ee_len); 3737 3738 /* If extent is larger than requested it is a clear sign that we still 3739 * have some extent state machine issues left. So extent_split is still 3740 * required. 3741 * TODO: Once all related issues will be fixed this situation should be 3742 * illegal. 3743 */ 3744 if (ee_block != map->m_lblk || ee_len > map->m_len) { 3745 #ifdef EXT4_DEBUG 3746 ext4_warning("Inode (%ld) finished: extent logical block %llu," 3747 " len %u; IO logical block %llu, len %u", 3748 inode->i_ino, (unsigned long long)ee_block, ee_len, 3749 (unsigned long long)map->m_lblk, map->m_len); 3750 #endif 3751 err = ext4_split_convert_extents(handle, inode, map, ppath, 3752 EXT4_GET_BLOCKS_CONVERT); 3753 if (err < 0) 3754 return err; 3755 path = ext4_find_extent(inode, map->m_lblk, ppath, 0); 3756 if (IS_ERR(path)) 3757 return PTR_ERR(path); 3758 depth = ext_depth(inode); 3759 ex = path[depth].p_ext; 3760 } 3761 3762 err = ext4_ext_get_access(handle, inode, path + depth); 3763 if (err) 3764 goto out; 3765 /* first mark the extent as initialized */ 3766 ext4_ext_mark_initialized(ex); 3767 3768 /* note: ext4_ext_correct_indexes() isn't needed here because 3769 * borders are not changed 3770 */ 3771 ext4_ext_try_to_merge(handle, inode, path, ex); 3772 3773 /* Mark modified extent as dirty */ 3774 err = ext4_ext_dirty(handle, inode, path + path->p_depth); 3775 out: 3776 ext4_ext_show_leaf(inode, path); 3777 return err; 3778 } 3779 3780 static void unmap_underlying_metadata_blocks(struct block_device *bdev, 3781 sector_t block, int count) 3782 { 3783 int i; 3784 for (i = 0; i < count; i++) 3785 unmap_underlying_metadata(bdev, block + i); 3786 } 3787 3788 /* 3789 * Handle EOFBLOCKS_FL flag, clearing it if necessary 3790 */ 3791 static int check_eofblocks_fl(handle_t *handle, struct inode *inode, 3792 ext4_lblk_t lblk, 3793 struct ext4_ext_path *path, 3794 unsigned int len) 3795 { 3796 int i, depth; 3797 struct ext4_extent_header *eh; 3798 struct ext4_extent *last_ex; 3799 3800 if (!ext4_test_inode_flag(inode, EXT4_INODE_EOFBLOCKS)) 3801 return 0; 3802 3803 depth = ext_depth(inode); 3804 eh = path[depth].p_hdr; 3805 3806 /* 3807 * We're going to remove EOFBLOCKS_FL entirely in future so we 3808 * do not care for this case anymore. Simply remove the flag 3809 * if there are no extents. 3810 */ 3811 if (unlikely(!eh->eh_entries)) 3812 goto out; 3813 last_ex = EXT_LAST_EXTENT(eh); 3814 /* 3815 * We should clear the EOFBLOCKS_FL flag if we are writing the 3816 * last block in the last extent in the file. We test this by 3817 * first checking to see if the caller to 3818 * ext4_ext_get_blocks() was interested in the last block (or 3819 * a block beyond the last block) in the current extent. If 3820 * this turns out to be false, we can bail out from this 3821 * function immediately. 3822 */ 3823 if (lblk + len < le32_to_cpu(last_ex->ee_block) + 3824 ext4_ext_get_actual_len(last_ex)) 3825 return 0; 3826 /* 3827 * If the caller does appear to be planning to write at or 3828 * beyond the end of the current extent, we then test to see 3829 * if the current extent is the last extent in the file, by 3830 * checking to make sure it was reached via the rightmost node 3831 * at each level of the tree. 3832 */ 3833 for (i = depth-1; i >= 0; i--) 3834 if (path[i].p_idx != EXT_LAST_INDEX(path[i].p_hdr)) 3835 return 0; 3836 out: 3837 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS); 3838 return ext4_mark_inode_dirty(handle, inode); 3839 } 3840 3841 /** 3842 * ext4_find_delalloc_range: find delayed allocated block in the given range. 3843 * 3844 * Return 1 if there is a delalloc block in the range, otherwise 0. 3845 */ 3846 int ext4_find_delalloc_range(struct inode *inode, 3847 ext4_lblk_t lblk_start, 3848 ext4_lblk_t lblk_end) 3849 { 3850 struct extent_status es; 3851 3852 ext4_es_find_delayed_extent_range(inode, lblk_start, lblk_end, &es); 3853 if (es.es_len == 0) 3854 return 0; /* there is no delay extent in this tree */ 3855 else if (es.es_lblk <= lblk_start && 3856 lblk_start < es.es_lblk + es.es_len) 3857 return 1; 3858 else if (lblk_start <= es.es_lblk && es.es_lblk <= lblk_end) 3859 return 1; 3860 else 3861 return 0; 3862 } 3863 3864 int ext4_find_delalloc_cluster(struct inode *inode, ext4_lblk_t lblk) 3865 { 3866 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 3867 ext4_lblk_t lblk_start, lblk_end; 3868 lblk_start = EXT4_LBLK_CMASK(sbi, lblk); 3869 lblk_end = lblk_start + sbi->s_cluster_ratio - 1; 3870 3871 return ext4_find_delalloc_range(inode, lblk_start, lblk_end); 3872 } 3873 3874 /** 3875 * Determines how many complete clusters (out of those specified by the 'map') 3876 * are under delalloc and were reserved quota for. 3877 * This function is called when we are writing out the blocks that were 3878 * originally written with their allocation delayed, but then the space was 3879 * allocated using fallocate() before the delayed allocation could be resolved. 3880 * The cases to look for are: 3881 * ('=' indicated delayed allocated blocks 3882 * '-' indicates non-delayed allocated blocks) 3883 * (a) partial clusters towards beginning and/or end outside of allocated range 3884 * are not delalloc'ed. 3885 * Ex: 3886 * |----c---=|====c====|====c====|===-c----| 3887 * |++++++ allocated ++++++| 3888 * ==> 4 complete clusters in above example 3889 * 3890 * (b) partial cluster (outside of allocated range) towards either end is 3891 * marked for delayed allocation. In this case, we will exclude that 3892 * cluster. 3893 * Ex: 3894 * |----====c========|========c========| 3895 * |++++++ allocated ++++++| 3896 * ==> 1 complete clusters in above example 3897 * 3898 * Ex: 3899 * |================c================| 3900 * |++++++ allocated ++++++| 3901 * ==> 0 complete clusters in above example 3902 * 3903 * The ext4_da_update_reserve_space will be called only if we 3904 * determine here that there were some "entire" clusters that span 3905 * this 'allocated' range. 3906 * In the non-bigalloc case, this function will just end up returning num_blks 3907 * without ever calling ext4_find_delalloc_range. 3908 */ 3909 static unsigned int 3910 get_reserved_cluster_alloc(struct inode *inode, ext4_lblk_t lblk_start, 3911 unsigned int num_blks) 3912 { 3913 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 3914 ext4_lblk_t alloc_cluster_start, alloc_cluster_end; 3915 ext4_lblk_t lblk_from, lblk_to, c_offset; 3916 unsigned int allocated_clusters = 0; 3917 3918 alloc_cluster_start = EXT4_B2C(sbi, lblk_start); 3919 alloc_cluster_end = EXT4_B2C(sbi, lblk_start + num_blks - 1); 3920 3921 /* max possible clusters for this allocation */ 3922 allocated_clusters = alloc_cluster_end - alloc_cluster_start + 1; 3923 3924 trace_ext4_get_reserved_cluster_alloc(inode, lblk_start, num_blks); 3925 3926 /* Check towards left side */ 3927 c_offset = EXT4_LBLK_COFF(sbi, lblk_start); 3928 if (c_offset) { 3929 lblk_from = EXT4_LBLK_CMASK(sbi, lblk_start); 3930 lblk_to = lblk_from + c_offset - 1; 3931 3932 if (ext4_find_delalloc_range(inode, lblk_from, lblk_to)) 3933 allocated_clusters--; 3934 } 3935 3936 /* Now check towards right. */ 3937 c_offset = EXT4_LBLK_COFF(sbi, lblk_start + num_blks); 3938 if (allocated_clusters && c_offset) { 3939 lblk_from = lblk_start + num_blks; 3940 lblk_to = lblk_from + (sbi->s_cluster_ratio - c_offset) - 1; 3941 3942 if (ext4_find_delalloc_range(inode, lblk_from, lblk_to)) 3943 allocated_clusters--; 3944 } 3945 3946 return allocated_clusters; 3947 } 3948 3949 static int 3950 convert_initialized_extent(handle_t *handle, struct inode *inode, 3951 struct ext4_map_blocks *map, 3952 struct ext4_ext_path **ppath, 3953 unsigned int allocated) 3954 { 3955 struct ext4_ext_path *path = *ppath; 3956 struct ext4_extent *ex; 3957 ext4_lblk_t ee_block; 3958 unsigned int ee_len; 3959 int depth; 3960 int err = 0; 3961 3962 /* 3963 * Make sure that the extent is no bigger than we support with 3964 * unwritten extent 3965 */ 3966 if (map->m_len > EXT_UNWRITTEN_MAX_LEN) 3967 map->m_len = EXT_UNWRITTEN_MAX_LEN / 2; 3968 3969 depth = ext_depth(inode); 3970 ex = path[depth].p_ext; 3971 ee_block = le32_to_cpu(ex->ee_block); 3972 ee_len = ext4_ext_get_actual_len(ex); 3973 3974 ext_debug("%s: inode %lu, logical" 3975 "block %llu, max_blocks %u\n", __func__, inode->i_ino, 3976 (unsigned long long)ee_block, ee_len); 3977 3978 if (ee_block != map->m_lblk || ee_len > map->m_len) { 3979 err = ext4_split_convert_extents(handle, inode, map, ppath, 3980 EXT4_GET_BLOCKS_CONVERT_UNWRITTEN); 3981 if (err < 0) 3982 return err; 3983 path = ext4_find_extent(inode, map->m_lblk, ppath, 0); 3984 if (IS_ERR(path)) 3985 return PTR_ERR(path); 3986 depth = ext_depth(inode); 3987 ex = path[depth].p_ext; 3988 if (!ex) { 3989 EXT4_ERROR_INODE(inode, "unexpected hole at %lu", 3990 (unsigned long) map->m_lblk); 3991 return -EFSCORRUPTED; 3992 } 3993 } 3994 3995 err = ext4_ext_get_access(handle, inode, path + depth); 3996 if (err) 3997 return err; 3998 /* first mark the extent as unwritten */ 3999 ext4_ext_mark_unwritten(ex); 4000 4001 /* note: ext4_ext_correct_indexes() isn't needed here because 4002 * borders are not changed 4003 */ 4004 ext4_ext_try_to_merge(handle, inode, path, ex); 4005 4006 /* Mark modified extent as dirty */ 4007 err = ext4_ext_dirty(handle, inode, path + path->p_depth); 4008 if (err) 4009 return err; 4010 ext4_ext_show_leaf(inode, path); 4011 4012 ext4_update_inode_fsync_trans(handle, inode, 1); 4013 err = check_eofblocks_fl(handle, inode, map->m_lblk, path, map->m_len); 4014 if (err) 4015 return err; 4016 map->m_flags |= EXT4_MAP_UNWRITTEN; 4017 if (allocated > map->m_len) 4018 allocated = map->m_len; 4019 map->m_len = allocated; 4020 return allocated; 4021 } 4022 4023 static int 4024 ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode, 4025 struct ext4_map_blocks *map, 4026 struct ext4_ext_path **ppath, int flags, 4027 unsigned int allocated, ext4_fsblk_t newblock) 4028 { 4029 struct ext4_ext_path *path = *ppath; 4030 int ret = 0; 4031 int err = 0; 4032 4033 ext_debug("ext4_ext_handle_unwritten_extents: inode %lu, logical " 4034 "block %llu, max_blocks %u, flags %x, allocated %u\n", 4035 inode->i_ino, (unsigned long long)map->m_lblk, map->m_len, 4036 flags, allocated); 4037 ext4_ext_show_leaf(inode, path); 4038 4039 /* 4040 * When writing into unwritten space, we should not fail to 4041 * allocate metadata blocks for the new extent block if needed. 4042 */ 4043 flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL; 4044 4045 trace_ext4_ext_handle_unwritten_extents(inode, map, flags, 4046 allocated, newblock); 4047 4048 /* get_block() before submit the IO, split the extent */ 4049 if (flags & EXT4_GET_BLOCKS_PRE_IO) { 4050 ret = ext4_split_convert_extents(handle, inode, map, ppath, 4051 flags | EXT4_GET_BLOCKS_CONVERT); 4052 if (ret <= 0) 4053 goto out; 4054 map->m_flags |= EXT4_MAP_UNWRITTEN; 4055 goto out; 4056 } 4057 /* IO end_io complete, convert the filled extent to written */ 4058 if (flags & EXT4_GET_BLOCKS_CONVERT) { 4059 if (flags & EXT4_GET_BLOCKS_ZERO) { 4060 if (allocated > map->m_len) 4061 allocated = map->m_len; 4062 err = ext4_issue_zeroout(inode, map->m_lblk, newblock, 4063 allocated); 4064 if (err < 0) 4065 goto out2; 4066 } 4067 ret = ext4_convert_unwritten_extents_endio(handle, inode, map, 4068 ppath); 4069 if (ret >= 0) { 4070 ext4_update_inode_fsync_trans(handle, inode, 1); 4071 err = check_eofblocks_fl(handle, inode, map->m_lblk, 4072 path, map->m_len); 4073 } else 4074 err = ret; 4075 map->m_flags |= EXT4_MAP_MAPPED; 4076 map->m_pblk = newblock; 4077 if (allocated > map->m_len) 4078 allocated = map->m_len; 4079 map->m_len = allocated; 4080 goto out2; 4081 } 4082 /* buffered IO case */ 4083 /* 4084 * repeat fallocate creation request 4085 * we already have an unwritten extent 4086 */ 4087 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) { 4088 map->m_flags |= EXT4_MAP_UNWRITTEN; 4089 goto map_out; 4090 } 4091 4092 /* buffered READ or buffered write_begin() lookup */ 4093 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) { 4094 /* 4095 * We have blocks reserved already. We 4096 * return allocated blocks so that delalloc 4097 * won't do block reservation for us. But 4098 * the buffer head will be unmapped so that 4099 * a read from the block returns 0s. 4100 */ 4101 map->m_flags |= EXT4_MAP_UNWRITTEN; 4102 goto out1; 4103 } 4104 4105 /* buffered write, writepage time, convert*/ 4106 ret = ext4_ext_convert_to_initialized(handle, inode, map, ppath, flags); 4107 if (ret >= 0) 4108 ext4_update_inode_fsync_trans(handle, inode, 1); 4109 out: 4110 if (ret <= 0) { 4111 err = ret; 4112 goto out2; 4113 } else 4114 allocated = ret; 4115 map->m_flags |= EXT4_MAP_NEW; 4116 /* 4117 * if we allocated more blocks than requested 4118 * we need to make sure we unmap the extra block 4119 * allocated. The actual needed block will get 4120 * unmapped later when we find the buffer_head marked 4121 * new. 4122 */ 4123 if (allocated > map->m_len) { 4124 unmap_underlying_metadata_blocks(inode->i_sb->s_bdev, 4125 newblock + map->m_len, 4126 allocated - map->m_len); 4127 allocated = map->m_len; 4128 } 4129 map->m_len = allocated; 4130 4131 /* 4132 * If we have done fallocate with the offset that is already 4133 * delayed allocated, we would have block reservation 4134 * and quota reservation done in the delayed write path. 4135 * But fallocate would have already updated quota and block 4136 * count for this offset. So cancel these reservation 4137 */ 4138 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) { 4139 unsigned int reserved_clusters; 4140 reserved_clusters = get_reserved_cluster_alloc(inode, 4141 map->m_lblk, map->m_len); 4142 if (reserved_clusters) 4143 ext4_da_update_reserve_space(inode, 4144 reserved_clusters, 4145 0); 4146 } 4147 4148 map_out: 4149 map->m_flags |= EXT4_MAP_MAPPED; 4150 if ((flags & EXT4_GET_BLOCKS_KEEP_SIZE) == 0) { 4151 err = check_eofblocks_fl(handle, inode, map->m_lblk, path, 4152 map->m_len); 4153 if (err < 0) 4154 goto out2; 4155 } 4156 out1: 4157 if (allocated > map->m_len) 4158 allocated = map->m_len; 4159 ext4_ext_show_leaf(inode, path); 4160 map->m_pblk = newblock; 4161 map->m_len = allocated; 4162 out2: 4163 return err ? err : allocated; 4164 } 4165 4166 /* 4167 * get_implied_cluster_alloc - check to see if the requested 4168 * allocation (in the map structure) overlaps with a cluster already 4169 * allocated in an extent. 4170 * @sb The filesystem superblock structure 4171 * @map The requested lblk->pblk mapping 4172 * @ex The extent structure which might contain an implied 4173 * cluster allocation 4174 * 4175 * This function is called by ext4_ext_map_blocks() after we failed to 4176 * find blocks that were already in the inode's extent tree. Hence, 4177 * we know that the beginning of the requested region cannot overlap 4178 * the extent from the inode's extent tree. There are three cases we 4179 * want to catch. The first is this case: 4180 * 4181 * |--- cluster # N--| 4182 * |--- extent ---| |---- requested region ---| 4183 * |==========| 4184 * 4185 * The second case that we need to test for is this one: 4186 * 4187 * |--------- cluster # N ----------------| 4188 * |--- requested region --| |------- extent ----| 4189 * |=======================| 4190 * 4191 * The third case is when the requested region lies between two extents 4192 * within the same cluster: 4193 * |------------- cluster # N-------------| 4194 * |----- ex -----| |---- ex_right ----| 4195 * |------ requested region ------| 4196 * |================| 4197 * 4198 * In each of the above cases, we need to set the map->m_pblk and 4199 * map->m_len so it corresponds to the return the extent labelled as 4200 * "|====|" from cluster #N, since it is already in use for data in 4201 * cluster EXT4_B2C(sbi, map->m_lblk). We will then return 1 to 4202 * signal to ext4_ext_map_blocks() that map->m_pblk should be treated 4203 * as a new "allocated" block region. Otherwise, we will return 0 and 4204 * ext4_ext_map_blocks() will then allocate one or more new clusters 4205 * by calling ext4_mb_new_blocks(). 4206 */ 4207 static int get_implied_cluster_alloc(struct super_block *sb, 4208 struct ext4_map_blocks *map, 4209 struct ext4_extent *ex, 4210 struct ext4_ext_path *path) 4211 { 4212 struct ext4_sb_info *sbi = EXT4_SB(sb); 4213 ext4_lblk_t c_offset = EXT4_LBLK_COFF(sbi, map->m_lblk); 4214 ext4_lblk_t ex_cluster_start, ex_cluster_end; 4215 ext4_lblk_t rr_cluster_start; 4216 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block); 4217 ext4_fsblk_t ee_start = ext4_ext_pblock(ex); 4218 unsigned short ee_len = ext4_ext_get_actual_len(ex); 4219 4220 /* The extent passed in that we are trying to match */ 4221 ex_cluster_start = EXT4_B2C(sbi, ee_block); 4222 ex_cluster_end = EXT4_B2C(sbi, ee_block + ee_len - 1); 4223 4224 /* The requested region passed into ext4_map_blocks() */ 4225 rr_cluster_start = EXT4_B2C(sbi, map->m_lblk); 4226 4227 if ((rr_cluster_start == ex_cluster_end) || 4228 (rr_cluster_start == ex_cluster_start)) { 4229 if (rr_cluster_start == ex_cluster_end) 4230 ee_start += ee_len - 1; 4231 map->m_pblk = EXT4_PBLK_CMASK(sbi, ee_start) + c_offset; 4232 map->m_len = min(map->m_len, 4233 (unsigned) sbi->s_cluster_ratio - c_offset); 4234 /* 4235 * Check for and handle this case: 4236 * 4237 * |--------- cluster # N-------------| 4238 * |------- extent ----| 4239 * |--- requested region ---| 4240 * |===========| 4241 */ 4242 4243 if (map->m_lblk < ee_block) 4244 map->m_len = min(map->m_len, ee_block - map->m_lblk); 4245 4246 /* 4247 * Check for the case where there is already another allocated 4248 * block to the right of 'ex' but before the end of the cluster. 4249 * 4250 * |------------- cluster # N-------------| 4251 * |----- ex -----| |---- ex_right ----| 4252 * |------ requested region ------| 4253 * |================| 4254 */ 4255 if (map->m_lblk > ee_block) { 4256 ext4_lblk_t next = ext4_ext_next_allocated_block(path); 4257 map->m_len = min(map->m_len, next - map->m_lblk); 4258 } 4259 4260 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 1); 4261 return 1; 4262 } 4263 4264 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 0); 4265 return 0; 4266 } 4267 4268 4269 /* 4270 * Block allocation/map/preallocation routine for extents based files 4271 * 4272 * 4273 * Need to be called with 4274 * down_read(&EXT4_I(inode)->i_data_sem) if not allocating file system block 4275 * (ie, create is zero). Otherwise down_write(&EXT4_I(inode)->i_data_sem) 4276 * 4277 * return > 0, number of of blocks already mapped/allocated 4278 * if create == 0 and these are pre-allocated blocks 4279 * buffer head is unmapped 4280 * otherwise blocks are mapped 4281 * 4282 * return = 0, if plain look up failed (blocks have not been allocated) 4283 * buffer head is unmapped 4284 * 4285 * return < 0, error case. 4286 */ 4287 int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, 4288 struct ext4_map_blocks *map, int flags) 4289 { 4290 struct ext4_ext_path *path = NULL; 4291 struct ext4_extent newex, *ex, *ex2; 4292 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 4293 ext4_fsblk_t newblock = 0; 4294 int free_on_err = 0, err = 0, depth, ret; 4295 unsigned int allocated = 0, offset = 0; 4296 unsigned int allocated_clusters = 0; 4297 struct ext4_allocation_request ar; 4298 ext4_lblk_t cluster_offset; 4299 bool map_from_cluster = false; 4300 4301 ext_debug("blocks %u/%u requested for inode %lu\n", 4302 map->m_lblk, map->m_len, inode->i_ino); 4303 trace_ext4_ext_map_blocks_enter(inode, map->m_lblk, map->m_len, flags); 4304 4305 /* find extent for this block */ 4306 path = ext4_find_extent(inode, map->m_lblk, NULL, 0); 4307 if (IS_ERR(path)) { 4308 err = PTR_ERR(path); 4309 path = NULL; 4310 goto out2; 4311 } 4312 4313 depth = ext_depth(inode); 4314 4315 /* 4316 * consistent leaf must not be empty; 4317 * this situation is possible, though, _during_ tree modification; 4318 * this is why assert can't be put in ext4_find_extent() 4319 */ 4320 if (unlikely(path[depth].p_ext == NULL && depth != 0)) { 4321 EXT4_ERROR_INODE(inode, "bad extent address " 4322 "lblock: %lu, depth: %d pblock %lld", 4323 (unsigned long) map->m_lblk, depth, 4324 path[depth].p_block); 4325 err = -EFSCORRUPTED; 4326 goto out2; 4327 } 4328 4329 ex = path[depth].p_ext; 4330 if (ex) { 4331 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block); 4332 ext4_fsblk_t ee_start = ext4_ext_pblock(ex); 4333 unsigned short ee_len; 4334 4335 4336 /* 4337 * unwritten extents are treated as holes, except that 4338 * we split out initialized portions during a write. 4339 */ 4340 ee_len = ext4_ext_get_actual_len(ex); 4341 4342 trace_ext4_ext_show_extent(inode, ee_block, ee_start, ee_len); 4343 4344 /* if found extent covers block, simply return it */ 4345 if (in_range(map->m_lblk, ee_block, ee_len)) { 4346 newblock = map->m_lblk - ee_block + ee_start; 4347 /* number of remaining blocks in the extent */ 4348 allocated = ee_len - (map->m_lblk - ee_block); 4349 ext_debug("%u fit into %u:%d -> %llu\n", map->m_lblk, 4350 ee_block, ee_len, newblock); 4351 4352 /* 4353 * If the extent is initialized check whether the 4354 * caller wants to convert it to unwritten. 4355 */ 4356 if ((!ext4_ext_is_unwritten(ex)) && 4357 (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) { 4358 allocated = convert_initialized_extent( 4359 handle, inode, map, &path, 4360 allocated); 4361 goto out2; 4362 } else if (!ext4_ext_is_unwritten(ex)) 4363 goto out; 4364 4365 ret = ext4_ext_handle_unwritten_extents( 4366 handle, inode, map, &path, flags, 4367 allocated, newblock); 4368 if (ret < 0) 4369 err = ret; 4370 else 4371 allocated = ret; 4372 goto out2; 4373 } 4374 } 4375 4376 /* 4377 * requested block isn't allocated yet; 4378 * we couldn't try to create block if create flag is zero 4379 */ 4380 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) { 4381 ext4_lblk_t hole_start, hole_len; 4382 4383 hole_start = map->m_lblk; 4384 hole_len = ext4_ext_determine_hole(inode, path, &hole_start); 4385 /* 4386 * put just found gap into cache to speed up 4387 * subsequent requests 4388 */ 4389 ext4_ext_put_gap_in_cache(inode, hole_start, hole_len); 4390 4391 /* Update hole_len to reflect hole size after map->m_lblk */ 4392 if (hole_start != map->m_lblk) 4393 hole_len -= map->m_lblk - hole_start; 4394 map->m_pblk = 0; 4395 map->m_len = min_t(unsigned int, map->m_len, hole_len); 4396 4397 goto out2; 4398 } 4399 4400 /* 4401 * Okay, we need to do block allocation. 4402 */ 4403 newex.ee_block = cpu_to_le32(map->m_lblk); 4404 cluster_offset = EXT4_LBLK_COFF(sbi, map->m_lblk); 4405 4406 /* 4407 * If we are doing bigalloc, check to see if the extent returned 4408 * by ext4_find_extent() implies a cluster we can use. 4409 */ 4410 if (cluster_offset && ex && 4411 get_implied_cluster_alloc(inode->i_sb, map, ex, path)) { 4412 ar.len = allocated = map->m_len; 4413 newblock = map->m_pblk; 4414 map_from_cluster = true; 4415 goto got_allocated_blocks; 4416 } 4417 4418 /* find neighbour allocated blocks */ 4419 ar.lleft = map->m_lblk; 4420 err = ext4_ext_search_left(inode, path, &ar.lleft, &ar.pleft); 4421 if (err) 4422 goto out2; 4423 ar.lright = map->m_lblk; 4424 ex2 = NULL; 4425 err = ext4_ext_search_right(inode, path, &ar.lright, &ar.pright, &ex2); 4426 if (err) 4427 goto out2; 4428 4429 /* Check if the extent after searching to the right implies a 4430 * cluster we can use. */ 4431 if ((sbi->s_cluster_ratio > 1) && ex2 && 4432 get_implied_cluster_alloc(inode->i_sb, map, ex2, path)) { 4433 ar.len = allocated = map->m_len; 4434 newblock = map->m_pblk; 4435 map_from_cluster = true; 4436 goto got_allocated_blocks; 4437 } 4438 4439 /* 4440 * See if request is beyond maximum number of blocks we can have in 4441 * a single extent. For an initialized extent this limit is 4442 * EXT_INIT_MAX_LEN and for an unwritten extent this limit is 4443 * EXT_UNWRITTEN_MAX_LEN. 4444 */ 4445 if (map->m_len > EXT_INIT_MAX_LEN && 4446 !(flags & EXT4_GET_BLOCKS_UNWRIT_EXT)) 4447 map->m_len = EXT_INIT_MAX_LEN; 4448 else if (map->m_len > EXT_UNWRITTEN_MAX_LEN && 4449 (flags & EXT4_GET_BLOCKS_UNWRIT_EXT)) 4450 map->m_len = EXT_UNWRITTEN_MAX_LEN; 4451 4452 /* Check if we can really insert (m_lblk)::(m_lblk + m_len) extent */ 4453 newex.ee_len = cpu_to_le16(map->m_len); 4454 err = ext4_ext_check_overlap(sbi, inode, &newex, path); 4455 if (err) 4456 allocated = ext4_ext_get_actual_len(&newex); 4457 else 4458 allocated = map->m_len; 4459 4460 /* allocate new block */ 4461 ar.inode = inode; 4462 ar.goal = ext4_ext_find_goal(inode, path, map->m_lblk); 4463 ar.logical = map->m_lblk; 4464 /* 4465 * We calculate the offset from the beginning of the cluster 4466 * for the logical block number, since when we allocate a 4467 * physical cluster, the physical block should start at the 4468 * same offset from the beginning of the cluster. This is 4469 * needed so that future calls to get_implied_cluster_alloc() 4470 * work correctly. 4471 */ 4472 offset = EXT4_LBLK_COFF(sbi, map->m_lblk); 4473 ar.len = EXT4_NUM_B2C(sbi, offset+allocated); 4474 ar.goal -= offset; 4475 ar.logical -= offset; 4476 if (S_ISREG(inode->i_mode)) 4477 ar.flags = EXT4_MB_HINT_DATA; 4478 else 4479 /* disable in-core preallocation for non-regular files */ 4480 ar.flags = 0; 4481 if (flags & EXT4_GET_BLOCKS_NO_NORMALIZE) 4482 ar.flags |= EXT4_MB_HINT_NOPREALLOC; 4483 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) 4484 ar.flags |= EXT4_MB_DELALLOC_RESERVED; 4485 if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL) 4486 ar.flags |= EXT4_MB_USE_RESERVED; 4487 newblock = ext4_mb_new_blocks(handle, &ar, &err); 4488 if (!newblock) 4489 goto out2; 4490 ext_debug("allocate new block: goal %llu, found %llu/%u\n", 4491 ar.goal, newblock, allocated); 4492 free_on_err = 1; 4493 allocated_clusters = ar.len; 4494 ar.len = EXT4_C2B(sbi, ar.len) - offset; 4495 if (ar.len > allocated) 4496 ar.len = allocated; 4497 4498 got_allocated_blocks: 4499 /* try to insert new extent into found leaf and return */ 4500 ext4_ext_store_pblock(&newex, newblock + offset); 4501 newex.ee_len = cpu_to_le16(ar.len); 4502 /* Mark unwritten */ 4503 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT){ 4504 ext4_ext_mark_unwritten(&newex); 4505 map->m_flags |= EXT4_MAP_UNWRITTEN; 4506 } 4507 4508 err = 0; 4509 if ((flags & EXT4_GET_BLOCKS_KEEP_SIZE) == 0) 4510 err = check_eofblocks_fl(handle, inode, map->m_lblk, 4511 path, ar.len); 4512 if (!err) 4513 err = ext4_ext_insert_extent(handle, inode, &path, 4514 &newex, flags); 4515 4516 if (err && free_on_err) { 4517 int fb_flags = flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE ? 4518 EXT4_FREE_BLOCKS_NO_QUOT_UPDATE : 0; 4519 /* free data blocks we just allocated */ 4520 /* not a good idea to call discard here directly, 4521 * but otherwise we'd need to call it every free() */ 4522 ext4_discard_preallocations(inode); 4523 ext4_free_blocks(handle, inode, NULL, newblock, 4524 EXT4_C2B(sbi, allocated_clusters), fb_flags); 4525 goto out2; 4526 } 4527 4528 /* previous routine could use block we allocated */ 4529 newblock = ext4_ext_pblock(&newex); 4530 allocated = ext4_ext_get_actual_len(&newex); 4531 if (allocated > map->m_len) 4532 allocated = map->m_len; 4533 map->m_flags |= EXT4_MAP_NEW; 4534 4535 /* 4536 * Update reserved blocks/metadata blocks after successful 4537 * block allocation which had been deferred till now. 4538 */ 4539 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) { 4540 unsigned int reserved_clusters; 4541 /* 4542 * Check how many clusters we had reserved this allocated range 4543 */ 4544 reserved_clusters = get_reserved_cluster_alloc(inode, 4545 map->m_lblk, allocated); 4546 if (!map_from_cluster) { 4547 BUG_ON(allocated_clusters < reserved_clusters); 4548 if (reserved_clusters < allocated_clusters) { 4549 struct ext4_inode_info *ei = EXT4_I(inode); 4550 int reservation = allocated_clusters - 4551 reserved_clusters; 4552 /* 4553 * It seems we claimed few clusters outside of 4554 * the range of this allocation. We should give 4555 * it back to the reservation pool. This can 4556 * happen in the following case: 4557 * 4558 * * Suppose s_cluster_ratio is 4 (i.e., each 4559 * cluster has 4 blocks. Thus, the clusters 4560 * are [0-3],[4-7],[8-11]... 4561 * * First comes delayed allocation write for 4562 * logical blocks 10 & 11. Since there were no 4563 * previous delayed allocated blocks in the 4564 * range [8-11], we would reserve 1 cluster 4565 * for this write. 4566 * * Next comes write for logical blocks 3 to 8. 4567 * In this case, we will reserve 2 clusters 4568 * (for [0-3] and [4-7]; and not for [8-11] as 4569 * that range has a delayed allocated blocks. 4570 * Thus total reserved clusters now becomes 3. 4571 * * Now, during the delayed allocation writeout 4572 * time, we will first write blocks [3-8] and 4573 * allocate 3 clusters for writing these 4574 * blocks. Also, we would claim all these 4575 * three clusters above. 4576 * * Now when we come here to writeout the 4577 * blocks [10-11], we would expect to claim 4578 * the reservation of 1 cluster we had made 4579 * (and we would claim it since there are no 4580 * more delayed allocated blocks in the range 4581 * [8-11]. But our reserved cluster count had 4582 * already gone to 0. 4583 * 4584 * Thus, at the step 4 above when we determine 4585 * that there are still some unwritten delayed 4586 * allocated blocks outside of our current 4587 * block range, we should increment the 4588 * reserved clusters count so that when the 4589 * remaining blocks finally gets written, we 4590 * could claim them. 4591 */ 4592 dquot_reserve_block(inode, 4593 EXT4_C2B(sbi, reservation)); 4594 spin_lock(&ei->i_block_reservation_lock); 4595 ei->i_reserved_data_blocks += reservation; 4596 spin_unlock(&ei->i_block_reservation_lock); 4597 } 4598 /* 4599 * We will claim quota for all newly allocated blocks. 4600 * We're updating the reserved space *after* the 4601 * correction above so we do not accidentally free 4602 * all the metadata reservation because we might 4603 * actually need it later on. 4604 */ 4605 ext4_da_update_reserve_space(inode, allocated_clusters, 4606 1); 4607 } 4608 } 4609 4610 /* 4611 * Cache the extent and update transaction to commit on fdatasync only 4612 * when it is _not_ an unwritten extent. 4613 */ 4614 if ((flags & EXT4_GET_BLOCKS_UNWRIT_EXT) == 0) 4615 ext4_update_inode_fsync_trans(handle, inode, 1); 4616 else 4617 ext4_update_inode_fsync_trans(handle, inode, 0); 4618 out: 4619 if (allocated > map->m_len) 4620 allocated = map->m_len; 4621 ext4_ext_show_leaf(inode, path); 4622 map->m_flags |= EXT4_MAP_MAPPED; 4623 map->m_pblk = newblock; 4624 map->m_len = allocated; 4625 out2: 4626 ext4_ext_drop_refs(path); 4627 kfree(path); 4628 4629 trace_ext4_ext_map_blocks_exit(inode, flags, map, 4630 err ? err : allocated); 4631 return err ? err : allocated; 4632 } 4633 4634 void ext4_ext_truncate(handle_t *handle, struct inode *inode) 4635 { 4636 struct super_block *sb = inode->i_sb; 4637 ext4_lblk_t last_block; 4638 int err = 0; 4639 4640 /* 4641 * TODO: optimization is possible here. 4642 * Probably we need not scan at all, 4643 * because page truncation is enough. 4644 */ 4645 4646 /* we have to know where to truncate from in crash case */ 4647 EXT4_I(inode)->i_disksize = inode->i_size; 4648 ext4_mark_inode_dirty(handle, inode); 4649 4650 last_block = (inode->i_size + sb->s_blocksize - 1) 4651 >> EXT4_BLOCK_SIZE_BITS(sb); 4652 retry: 4653 err = ext4_es_remove_extent(inode, last_block, 4654 EXT_MAX_BLOCKS - last_block); 4655 if (err == -ENOMEM) { 4656 cond_resched(); 4657 congestion_wait(BLK_RW_ASYNC, HZ/50); 4658 goto retry; 4659 } 4660 if (err) { 4661 ext4_std_error(inode->i_sb, err); 4662 return; 4663 } 4664 err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1); 4665 ext4_std_error(inode->i_sb, err); 4666 } 4667 4668 static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset, 4669 ext4_lblk_t len, loff_t new_size, 4670 int flags, int mode) 4671 { 4672 struct inode *inode = file_inode(file); 4673 handle_t *handle; 4674 int ret = 0; 4675 int ret2 = 0; 4676 int retries = 0; 4677 int depth = 0; 4678 struct ext4_map_blocks map; 4679 unsigned int credits; 4680 loff_t epos; 4681 4682 BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)); 4683 map.m_lblk = offset; 4684 map.m_len = len; 4685 /* 4686 * Don't normalize the request if it can fit in one extent so 4687 * that it doesn't get unnecessarily split into multiple 4688 * extents. 4689 */ 4690 if (len <= EXT_UNWRITTEN_MAX_LEN) 4691 flags |= EXT4_GET_BLOCKS_NO_NORMALIZE; 4692 4693 /* 4694 * credits to insert 1 extent into extent tree 4695 */ 4696 credits = ext4_chunk_trans_blocks(inode, len); 4697 depth = ext_depth(inode); 4698 4699 retry: 4700 while (ret >= 0 && len) { 4701 /* 4702 * Recalculate credits when extent tree depth changes. 4703 */ 4704 if (depth >= 0 && depth != ext_depth(inode)) { 4705 credits = ext4_chunk_trans_blocks(inode, len); 4706 depth = ext_depth(inode); 4707 } 4708 4709 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, 4710 credits); 4711 if (IS_ERR(handle)) { 4712 ret = PTR_ERR(handle); 4713 break; 4714 } 4715 ret = ext4_map_blocks(handle, inode, &map, flags); 4716 if (ret <= 0) { 4717 ext4_debug("inode #%lu: block %u: len %u: " 4718 "ext4_ext_map_blocks returned %d", 4719 inode->i_ino, map.m_lblk, 4720 map.m_len, ret); 4721 ext4_mark_inode_dirty(handle, inode); 4722 ret2 = ext4_journal_stop(handle); 4723 break; 4724 } 4725 map.m_lblk += ret; 4726 map.m_len = len = len - ret; 4727 epos = (loff_t)map.m_lblk << inode->i_blkbits; 4728 inode->i_ctime = ext4_current_time(inode); 4729 if (new_size) { 4730 if (epos > new_size) 4731 epos = new_size; 4732 if (ext4_update_inode_size(inode, epos) & 0x1) 4733 inode->i_mtime = inode->i_ctime; 4734 } else { 4735 if (epos > inode->i_size) 4736 ext4_set_inode_flag(inode, 4737 EXT4_INODE_EOFBLOCKS); 4738 } 4739 ext4_mark_inode_dirty(handle, inode); 4740 ret2 = ext4_journal_stop(handle); 4741 if (ret2) 4742 break; 4743 } 4744 if (ret == -ENOSPC && 4745 ext4_should_retry_alloc(inode->i_sb, &retries)) { 4746 ret = 0; 4747 goto retry; 4748 } 4749 4750 return ret > 0 ? ret2 : ret; 4751 } 4752 4753 static long ext4_zero_range(struct file *file, loff_t offset, 4754 loff_t len, int mode) 4755 { 4756 struct inode *inode = file_inode(file); 4757 handle_t *handle = NULL; 4758 unsigned int max_blocks; 4759 loff_t new_size = 0; 4760 int ret = 0; 4761 int flags; 4762 int credits; 4763 int partial_begin, partial_end; 4764 loff_t start, end; 4765 ext4_lblk_t lblk; 4766 unsigned int blkbits = inode->i_blkbits; 4767 4768 trace_ext4_zero_range(inode, offset, len, mode); 4769 4770 if (!S_ISREG(inode->i_mode)) 4771 return -EINVAL; 4772 4773 /* Call ext4_force_commit to flush all data in case of data=journal. */ 4774 if (ext4_should_journal_data(inode)) { 4775 ret = ext4_force_commit(inode->i_sb); 4776 if (ret) 4777 return ret; 4778 } 4779 4780 /* 4781 * Round up offset. This is not fallocate, we neet to zero out 4782 * blocks, so convert interior block aligned part of the range to 4783 * unwritten and possibly manually zero out unaligned parts of the 4784 * range. 4785 */ 4786 start = round_up(offset, 1 << blkbits); 4787 end = round_down((offset + len), 1 << blkbits); 4788 4789 if (start < offset || end > offset + len) 4790 return -EINVAL; 4791 partial_begin = offset & ((1 << blkbits) - 1); 4792 partial_end = (offset + len) & ((1 << blkbits) - 1); 4793 4794 lblk = start >> blkbits; 4795 max_blocks = (end >> blkbits); 4796 if (max_blocks < lblk) 4797 max_blocks = 0; 4798 else 4799 max_blocks -= lblk; 4800 4801 inode_lock(inode); 4802 4803 /* 4804 * Indirect files do not support unwritten extnets 4805 */ 4806 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 4807 ret = -EOPNOTSUPP; 4808 goto out_mutex; 4809 } 4810 4811 if (!(mode & FALLOC_FL_KEEP_SIZE) && 4812 offset + len > i_size_read(inode)) { 4813 new_size = offset + len; 4814 ret = inode_newsize_ok(inode, new_size); 4815 if (ret) 4816 goto out_mutex; 4817 } 4818 4819 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; 4820 if (mode & FALLOC_FL_KEEP_SIZE) 4821 flags |= EXT4_GET_BLOCKS_KEEP_SIZE; 4822 4823 /* Wait all existing dio workers, newcomers will block on i_mutex */ 4824 ext4_inode_block_unlocked_dio(inode); 4825 inode_dio_wait(inode); 4826 4827 /* Preallocate the range including the unaligned edges */ 4828 if (partial_begin || partial_end) { 4829 ret = ext4_alloc_file_blocks(file, 4830 round_down(offset, 1 << blkbits) >> blkbits, 4831 (round_up((offset + len), 1 << blkbits) - 4832 round_down(offset, 1 << blkbits)) >> blkbits, 4833 new_size, flags, mode); 4834 if (ret) 4835 goto out_dio; 4836 4837 } 4838 4839 /* Zero range excluding the unaligned edges */ 4840 if (max_blocks > 0) { 4841 flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN | 4842 EXT4_EX_NOCACHE); 4843 4844 /* 4845 * Prevent page faults from reinstantiating pages we have 4846 * released from page cache. 4847 */ 4848 down_write(&EXT4_I(inode)->i_mmap_sem); 4849 ret = ext4_update_disksize_before_punch(inode, offset, len); 4850 if (ret) { 4851 up_write(&EXT4_I(inode)->i_mmap_sem); 4852 goto out_dio; 4853 } 4854 /* Now release the pages and zero block aligned part of pages */ 4855 truncate_pagecache_range(inode, start, end - 1); 4856 inode->i_mtime = inode->i_ctime = ext4_current_time(inode); 4857 4858 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, 4859 flags, mode); 4860 up_write(&EXT4_I(inode)->i_mmap_sem); 4861 if (ret) 4862 goto out_dio; 4863 } 4864 if (!partial_begin && !partial_end) 4865 goto out_dio; 4866 4867 /* 4868 * In worst case we have to writeout two nonadjacent unwritten 4869 * blocks and update the inode 4870 */ 4871 credits = (2 * ext4_ext_index_trans_blocks(inode, 2)) + 1; 4872 if (ext4_should_journal_data(inode)) 4873 credits += 2; 4874 handle = ext4_journal_start(inode, EXT4_HT_MISC, credits); 4875 if (IS_ERR(handle)) { 4876 ret = PTR_ERR(handle); 4877 ext4_std_error(inode->i_sb, ret); 4878 goto out_dio; 4879 } 4880 4881 inode->i_mtime = inode->i_ctime = ext4_current_time(inode); 4882 if (new_size) { 4883 ext4_update_inode_size(inode, new_size); 4884 } else { 4885 /* 4886 * Mark that we allocate beyond EOF so the subsequent truncate 4887 * can proceed even if the new size is the same as i_size. 4888 */ 4889 if ((offset + len) > i_size_read(inode)) 4890 ext4_set_inode_flag(inode, EXT4_INODE_EOFBLOCKS); 4891 } 4892 ext4_mark_inode_dirty(handle, inode); 4893 4894 /* Zero out partial block at the edges of the range */ 4895 ret = ext4_zero_partial_blocks(handle, inode, offset, len); 4896 4897 if (file->f_flags & O_SYNC) 4898 ext4_handle_sync(handle); 4899 4900 ext4_journal_stop(handle); 4901 out_dio: 4902 ext4_inode_resume_unlocked_dio(inode); 4903 out_mutex: 4904 inode_unlock(inode); 4905 return ret; 4906 } 4907 4908 /* 4909 * preallocate space for a file. This implements ext4's fallocate file 4910 * operation, which gets called from sys_fallocate system call. 4911 * For block-mapped files, posix_fallocate should fall back to the method 4912 * of writing zeroes to the required new blocks (the same behavior which is 4913 * expected for file systems which do not support fallocate() system call). 4914 */ 4915 long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len) 4916 { 4917 struct inode *inode = file_inode(file); 4918 loff_t new_size = 0; 4919 unsigned int max_blocks; 4920 int ret = 0; 4921 int flags; 4922 ext4_lblk_t lblk; 4923 unsigned int blkbits = inode->i_blkbits; 4924 4925 /* 4926 * Encrypted inodes can't handle collapse range or insert 4927 * range since we would need to re-encrypt blocks with a 4928 * different IV or XTS tweak (which are based on the logical 4929 * block number). 4930 * 4931 * XXX It's not clear why zero range isn't working, but we'll 4932 * leave it disabled for encrypted inodes for now. This is a 4933 * bug we should fix.... 4934 */ 4935 if (ext4_encrypted_inode(inode) && 4936 (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE | 4937 FALLOC_FL_ZERO_RANGE))) 4938 return -EOPNOTSUPP; 4939 4940 /* Return error if mode is not supported */ 4941 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | 4942 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | 4943 FALLOC_FL_INSERT_RANGE)) 4944 return -EOPNOTSUPP; 4945 4946 if (mode & FALLOC_FL_PUNCH_HOLE) 4947 return ext4_punch_hole(inode, offset, len); 4948 4949 ret = ext4_convert_inline_data(inode); 4950 if (ret) 4951 return ret; 4952 4953 if (mode & FALLOC_FL_COLLAPSE_RANGE) 4954 return ext4_collapse_range(inode, offset, len); 4955 4956 if (mode & FALLOC_FL_INSERT_RANGE) 4957 return ext4_insert_range(inode, offset, len); 4958 4959 if (mode & FALLOC_FL_ZERO_RANGE) 4960 return ext4_zero_range(file, offset, len, mode); 4961 4962 trace_ext4_fallocate_enter(inode, offset, len, mode); 4963 lblk = offset >> blkbits; 4964 4965 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); 4966 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; 4967 if (mode & FALLOC_FL_KEEP_SIZE) 4968 flags |= EXT4_GET_BLOCKS_KEEP_SIZE; 4969 4970 inode_lock(inode); 4971 4972 /* 4973 * We only support preallocation for extent-based files only 4974 */ 4975 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 4976 ret = -EOPNOTSUPP; 4977 goto out; 4978 } 4979 4980 if (!(mode & FALLOC_FL_KEEP_SIZE) && 4981 offset + len > i_size_read(inode)) { 4982 new_size = offset + len; 4983 ret = inode_newsize_ok(inode, new_size); 4984 if (ret) 4985 goto out; 4986 } 4987 4988 /* Wait all existing dio workers, newcomers will block on i_mutex */ 4989 ext4_inode_block_unlocked_dio(inode); 4990 inode_dio_wait(inode); 4991 4992 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, 4993 flags, mode); 4994 ext4_inode_resume_unlocked_dio(inode); 4995 if (ret) 4996 goto out; 4997 4998 if (file->f_flags & O_SYNC && EXT4_SB(inode->i_sb)->s_journal) { 4999 ret = jbd2_complete_transaction(EXT4_SB(inode->i_sb)->s_journal, 5000 EXT4_I(inode)->i_sync_tid); 5001 } 5002 out: 5003 inode_unlock(inode); 5004 trace_ext4_fallocate_exit(inode, offset, max_blocks, ret); 5005 return ret; 5006 } 5007 5008 /* 5009 * This function convert a range of blocks to written extents 5010 * The caller of this function will pass the start offset and the size. 5011 * all unwritten extents within this range will be converted to 5012 * written extents. 5013 * 5014 * This function is called from the direct IO end io call back 5015 * function, to convert the fallocated extents after IO is completed. 5016 * Returns 0 on success. 5017 */ 5018 int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, 5019 loff_t offset, ssize_t len) 5020 { 5021 unsigned int max_blocks; 5022 int ret = 0; 5023 int ret2 = 0; 5024 struct ext4_map_blocks map; 5025 unsigned int credits, blkbits = inode->i_blkbits; 5026 5027 map.m_lblk = offset >> blkbits; 5028 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); 5029 5030 /* 5031 * This is somewhat ugly but the idea is clear: When transaction is 5032 * reserved, everything goes into it. Otherwise we rather start several 5033 * smaller transactions for conversion of each extent separately. 5034 */ 5035 if (handle) { 5036 handle = ext4_journal_start_reserved(handle, 5037 EXT4_HT_EXT_CONVERT); 5038 if (IS_ERR(handle)) 5039 return PTR_ERR(handle); 5040 credits = 0; 5041 } else { 5042 /* 5043 * credits to insert 1 extent into extent tree 5044 */ 5045 credits = ext4_chunk_trans_blocks(inode, max_blocks); 5046 } 5047 while (ret >= 0 && ret < max_blocks) { 5048 map.m_lblk += ret; 5049 map.m_len = (max_blocks -= ret); 5050 if (credits) { 5051 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, 5052 credits); 5053 if (IS_ERR(handle)) { 5054 ret = PTR_ERR(handle); 5055 break; 5056 } 5057 } 5058 ret = ext4_map_blocks(handle, inode, &map, 5059 EXT4_GET_BLOCKS_IO_CONVERT_EXT); 5060 if (ret <= 0) 5061 ext4_warning(inode->i_sb, 5062 "inode #%lu: block %u: len %u: " 5063 "ext4_ext_map_blocks returned %d", 5064 inode->i_ino, map.m_lblk, 5065 map.m_len, ret); 5066 ext4_mark_inode_dirty(handle, inode); 5067 if (credits) 5068 ret2 = ext4_journal_stop(handle); 5069 if (ret <= 0 || ret2) 5070 break; 5071 } 5072 if (!credits) 5073 ret2 = ext4_journal_stop(handle); 5074 return ret > 0 ? ret2 : ret; 5075 } 5076 5077 /* 5078 * If newes is not existing extent (newes->ec_pblk equals zero) find 5079 * delayed extent at start of newes and update newes accordingly and 5080 * return start of the next delayed extent. 5081 * 5082 * If newes is existing extent (newes->ec_pblk is not equal zero) 5083 * return start of next delayed extent or EXT_MAX_BLOCKS if no delayed 5084 * extent found. Leave newes unmodified. 5085 */ 5086 static int ext4_find_delayed_extent(struct inode *inode, 5087 struct extent_status *newes) 5088 { 5089 struct extent_status es; 5090 ext4_lblk_t block, next_del; 5091 5092 if (newes->es_pblk == 0) { 5093 ext4_es_find_delayed_extent_range(inode, newes->es_lblk, 5094 newes->es_lblk + newes->es_len - 1, &es); 5095 5096 /* 5097 * No extent in extent-tree contains block @newes->es_pblk, 5098 * then the block may stay in 1)a hole or 2)delayed-extent. 5099 */ 5100 if (es.es_len == 0) 5101 /* A hole found. */ 5102 return 0; 5103 5104 if (es.es_lblk > newes->es_lblk) { 5105 /* A hole found. */ 5106 newes->es_len = min(es.es_lblk - newes->es_lblk, 5107 newes->es_len); 5108 return 0; 5109 } 5110 5111 newes->es_len = es.es_lblk + es.es_len - newes->es_lblk; 5112 } 5113 5114 block = newes->es_lblk + newes->es_len; 5115 ext4_es_find_delayed_extent_range(inode, block, EXT_MAX_BLOCKS, &es); 5116 if (es.es_len == 0) 5117 next_del = EXT_MAX_BLOCKS; 5118 else 5119 next_del = es.es_lblk; 5120 5121 return next_del; 5122 } 5123 /* fiemap flags we can handle specified here */ 5124 #define EXT4_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC|FIEMAP_FLAG_XATTR) 5125 5126 static int ext4_xattr_fiemap(struct inode *inode, 5127 struct fiemap_extent_info *fieinfo) 5128 { 5129 __u64 physical = 0; 5130 __u64 length; 5131 __u32 flags = FIEMAP_EXTENT_LAST; 5132 int blockbits = inode->i_sb->s_blocksize_bits; 5133 int error = 0; 5134 5135 /* in-inode? */ 5136 if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) { 5137 struct ext4_iloc iloc; 5138 int offset; /* offset of xattr in inode */ 5139 5140 error = ext4_get_inode_loc(inode, &iloc); 5141 if (error) 5142 return error; 5143 physical = (__u64)iloc.bh->b_blocknr << blockbits; 5144 offset = EXT4_GOOD_OLD_INODE_SIZE + 5145 EXT4_I(inode)->i_extra_isize; 5146 physical += offset; 5147 length = EXT4_SB(inode->i_sb)->s_inode_size - offset; 5148 flags |= FIEMAP_EXTENT_DATA_INLINE; 5149 brelse(iloc.bh); 5150 } else { /* external block */ 5151 physical = (__u64)EXT4_I(inode)->i_file_acl << blockbits; 5152 length = inode->i_sb->s_blocksize; 5153 } 5154 5155 if (physical) 5156 error = fiemap_fill_next_extent(fieinfo, 0, physical, 5157 length, flags); 5158 return (error < 0 ? error : 0); 5159 } 5160 5161 int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 5162 __u64 start, __u64 len) 5163 { 5164 ext4_lblk_t start_blk; 5165 int error = 0; 5166 5167 if (ext4_has_inline_data(inode)) { 5168 int has_inline = 1; 5169 5170 error = ext4_inline_data_fiemap(inode, fieinfo, &has_inline, 5171 start, len); 5172 5173 if (has_inline) 5174 return error; 5175 } 5176 5177 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { 5178 error = ext4_ext_precache(inode); 5179 if (error) 5180 return error; 5181 } 5182 5183 /* fallback to generic here if not in extents fmt */ 5184 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 5185 return generic_block_fiemap(inode, fieinfo, start, len, 5186 ext4_get_block); 5187 5188 if (fiemap_check_flags(fieinfo, EXT4_FIEMAP_FLAGS)) 5189 return -EBADR; 5190 5191 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { 5192 error = ext4_xattr_fiemap(inode, fieinfo); 5193 } else { 5194 ext4_lblk_t len_blks; 5195 __u64 last_blk; 5196 5197 start_blk = start >> inode->i_sb->s_blocksize_bits; 5198 last_blk = (start + len - 1) >> inode->i_sb->s_blocksize_bits; 5199 if (last_blk >= EXT_MAX_BLOCKS) 5200 last_blk = EXT_MAX_BLOCKS-1; 5201 len_blks = ((ext4_lblk_t) last_blk) - start_blk + 1; 5202 5203 /* 5204 * Walk the extent tree gathering extent information 5205 * and pushing extents back to the user. 5206 */ 5207 error = ext4_fill_fiemap_extents(inode, start_blk, 5208 len_blks, fieinfo); 5209 } 5210 return error; 5211 } 5212 5213 /* 5214 * ext4_access_path: 5215 * Function to access the path buffer for marking it dirty. 5216 * It also checks if there are sufficient credits left in the journal handle 5217 * to update path. 5218 */ 5219 static int 5220 ext4_access_path(handle_t *handle, struct inode *inode, 5221 struct ext4_ext_path *path) 5222 { 5223 int credits, err; 5224 5225 if (!ext4_handle_valid(handle)) 5226 return 0; 5227 5228 /* 5229 * Check if need to extend journal credits 5230 * 3 for leaf, sb, and inode plus 2 (bmap and group 5231 * descriptor) for each block group; assume two block 5232 * groups 5233 */ 5234 if (handle->h_buffer_credits < 7) { 5235 credits = ext4_writepage_trans_blocks(inode); 5236 err = ext4_ext_truncate_extend_restart(handle, inode, credits); 5237 /* EAGAIN is success */ 5238 if (err && err != -EAGAIN) 5239 return err; 5240 } 5241 5242 err = ext4_ext_get_access(handle, inode, path); 5243 return err; 5244 } 5245 5246 /* 5247 * ext4_ext_shift_path_extents: 5248 * Shift the extents of a path structure lying between path[depth].p_ext 5249 * and EXT_LAST_EXTENT(path[depth].p_hdr), by @shift blocks. @SHIFT tells 5250 * if it is right shift or left shift operation. 5251 */ 5252 static int 5253 ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift, 5254 struct inode *inode, handle_t *handle, 5255 enum SHIFT_DIRECTION SHIFT) 5256 { 5257 int depth, err = 0; 5258 struct ext4_extent *ex_start, *ex_last; 5259 bool update = 0; 5260 depth = path->p_depth; 5261 5262 while (depth >= 0) { 5263 if (depth == path->p_depth) { 5264 ex_start = path[depth].p_ext; 5265 if (!ex_start) 5266 return -EFSCORRUPTED; 5267 5268 ex_last = EXT_LAST_EXTENT(path[depth].p_hdr); 5269 5270 err = ext4_access_path(handle, inode, path + depth); 5271 if (err) 5272 goto out; 5273 5274 if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr)) 5275 update = 1; 5276 5277 while (ex_start <= ex_last) { 5278 if (SHIFT == SHIFT_LEFT) { 5279 le32_add_cpu(&ex_start->ee_block, 5280 -shift); 5281 /* Try to merge to the left. */ 5282 if ((ex_start > 5283 EXT_FIRST_EXTENT(path[depth].p_hdr)) 5284 && 5285 ext4_ext_try_to_merge_right(inode, 5286 path, ex_start - 1)) 5287 ex_last--; 5288 else 5289 ex_start++; 5290 } else { 5291 le32_add_cpu(&ex_last->ee_block, shift); 5292 ext4_ext_try_to_merge_right(inode, path, 5293 ex_last); 5294 ex_last--; 5295 } 5296 } 5297 err = ext4_ext_dirty(handle, inode, path + depth); 5298 if (err) 5299 goto out; 5300 5301 if (--depth < 0 || !update) 5302 break; 5303 } 5304 5305 /* Update index too */ 5306 err = ext4_access_path(handle, inode, path + depth); 5307 if (err) 5308 goto out; 5309 5310 if (SHIFT == SHIFT_LEFT) 5311 le32_add_cpu(&path[depth].p_idx->ei_block, -shift); 5312 else 5313 le32_add_cpu(&path[depth].p_idx->ei_block, shift); 5314 err = ext4_ext_dirty(handle, inode, path + depth); 5315 if (err) 5316 goto out; 5317 5318 /* we are done if current index is not a starting index */ 5319 if (path[depth].p_idx != EXT_FIRST_INDEX(path[depth].p_hdr)) 5320 break; 5321 5322 depth--; 5323 } 5324 5325 out: 5326 return err; 5327 } 5328 5329 /* 5330 * ext4_ext_shift_extents: 5331 * All the extents which lies in the range from @start to the last allocated 5332 * block for the @inode are shifted either towards left or right (depending 5333 * upon @SHIFT) by @shift blocks. 5334 * On success, 0 is returned, error otherwise. 5335 */ 5336 static int 5337 ext4_ext_shift_extents(struct inode *inode, handle_t *handle, 5338 ext4_lblk_t start, ext4_lblk_t shift, 5339 enum SHIFT_DIRECTION SHIFT) 5340 { 5341 struct ext4_ext_path *path; 5342 int ret = 0, depth; 5343 struct ext4_extent *extent; 5344 ext4_lblk_t stop, *iterator, ex_start, ex_end; 5345 5346 /* Let path point to the last extent */ 5347 path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL, 0); 5348 if (IS_ERR(path)) 5349 return PTR_ERR(path); 5350 5351 depth = path->p_depth; 5352 extent = path[depth].p_ext; 5353 if (!extent) 5354 goto out; 5355 5356 stop = le32_to_cpu(extent->ee_block) + 5357 ext4_ext_get_actual_len(extent); 5358 5359 /* 5360 * In case of left shift, Don't start shifting extents until we make 5361 * sure the hole is big enough to accommodate the shift. 5362 */ 5363 if (SHIFT == SHIFT_LEFT) { 5364 path = ext4_find_extent(inode, start - 1, &path, 0); 5365 if (IS_ERR(path)) 5366 return PTR_ERR(path); 5367 depth = path->p_depth; 5368 extent = path[depth].p_ext; 5369 if (extent) { 5370 ex_start = le32_to_cpu(extent->ee_block); 5371 ex_end = le32_to_cpu(extent->ee_block) + 5372 ext4_ext_get_actual_len(extent); 5373 } else { 5374 ex_start = 0; 5375 ex_end = 0; 5376 } 5377 5378 if ((start == ex_start && shift > ex_start) || 5379 (shift > start - ex_end)) { 5380 ext4_ext_drop_refs(path); 5381 kfree(path); 5382 return -EINVAL; 5383 } 5384 } 5385 5386 /* 5387 * In case of left shift, iterator points to start and it is increased 5388 * till we reach stop. In case of right shift, iterator points to stop 5389 * and it is decreased till we reach start. 5390 */ 5391 if (SHIFT == SHIFT_LEFT) 5392 iterator = &start; 5393 else 5394 iterator = &stop; 5395 5396 /* Its safe to start updating extents */ 5397 while (start < stop) { 5398 path = ext4_find_extent(inode, *iterator, &path, 0); 5399 if (IS_ERR(path)) 5400 return PTR_ERR(path); 5401 depth = path->p_depth; 5402 extent = path[depth].p_ext; 5403 if (!extent) { 5404 EXT4_ERROR_INODE(inode, "unexpected hole at %lu", 5405 (unsigned long) *iterator); 5406 return -EFSCORRUPTED; 5407 } 5408 if (SHIFT == SHIFT_LEFT && *iterator > 5409 le32_to_cpu(extent->ee_block)) { 5410 /* Hole, move to the next extent */ 5411 if (extent < EXT_LAST_EXTENT(path[depth].p_hdr)) { 5412 path[depth].p_ext++; 5413 } else { 5414 *iterator = ext4_ext_next_allocated_block(path); 5415 continue; 5416 } 5417 } 5418 5419 if (SHIFT == SHIFT_LEFT) { 5420 extent = EXT_LAST_EXTENT(path[depth].p_hdr); 5421 *iterator = le32_to_cpu(extent->ee_block) + 5422 ext4_ext_get_actual_len(extent); 5423 } else { 5424 extent = EXT_FIRST_EXTENT(path[depth].p_hdr); 5425 *iterator = le32_to_cpu(extent->ee_block) > 0 ? 5426 le32_to_cpu(extent->ee_block) - 1 : 0; 5427 /* Update path extent in case we need to stop */ 5428 while (le32_to_cpu(extent->ee_block) < start) 5429 extent++; 5430 path[depth].p_ext = extent; 5431 } 5432 ret = ext4_ext_shift_path_extents(path, shift, inode, 5433 handle, SHIFT); 5434 if (ret) 5435 break; 5436 } 5437 out: 5438 ext4_ext_drop_refs(path); 5439 kfree(path); 5440 return ret; 5441 } 5442 5443 /* 5444 * ext4_collapse_range: 5445 * This implements the fallocate's collapse range functionality for ext4 5446 * Returns: 0 and non-zero on error. 5447 */ 5448 int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len) 5449 { 5450 struct super_block *sb = inode->i_sb; 5451 ext4_lblk_t punch_start, punch_stop; 5452 handle_t *handle; 5453 unsigned int credits; 5454 loff_t new_size, ioffset; 5455 int ret; 5456 5457 /* 5458 * We need to test this early because xfstests assumes that a 5459 * collapse range of (0, 1) will return EOPNOTSUPP if the file 5460 * system does not support collapse range. 5461 */ 5462 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 5463 return -EOPNOTSUPP; 5464 5465 /* Collapse range works only on fs block size aligned offsets. */ 5466 if (offset & (EXT4_CLUSTER_SIZE(sb) - 1) || 5467 len & (EXT4_CLUSTER_SIZE(sb) - 1)) 5468 return -EINVAL; 5469 5470 if (!S_ISREG(inode->i_mode)) 5471 return -EINVAL; 5472 5473 trace_ext4_collapse_range(inode, offset, len); 5474 5475 punch_start = offset >> EXT4_BLOCK_SIZE_BITS(sb); 5476 punch_stop = (offset + len) >> EXT4_BLOCK_SIZE_BITS(sb); 5477 5478 /* Call ext4_force_commit to flush all data in case of data=journal. */ 5479 if (ext4_should_journal_data(inode)) { 5480 ret = ext4_force_commit(inode->i_sb); 5481 if (ret) 5482 return ret; 5483 } 5484 5485 inode_lock(inode); 5486 /* 5487 * There is no need to overlap collapse range with EOF, in which case 5488 * it is effectively a truncate operation 5489 */ 5490 if (offset + len >= i_size_read(inode)) { 5491 ret = -EINVAL; 5492 goto out_mutex; 5493 } 5494 5495 /* Currently just for extent based files */ 5496 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 5497 ret = -EOPNOTSUPP; 5498 goto out_mutex; 5499 } 5500 5501 /* Wait for existing dio to complete */ 5502 ext4_inode_block_unlocked_dio(inode); 5503 inode_dio_wait(inode); 5504 5505 /* 5506 * Prevent page faults from reinstantiating pages we have released from 5507 * page cache. 5508 */ 5509 down_write(&EXT4_I(inode)->i_mmap_sem); 5510 /* 5511 * Need to round down offset to be aligned with page size boundary 5512 * for page size > block size. 5513 */ 5514 ioffset = round_down(offset, PAGE_SIZE); 5515 /* 5516 * Write tail of the last page before removed range since it will get 5517 * removed from the page cache below. 5518 */ 5519 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset, offset); 5520 if (ret) 5521 goto out_mmap; 5522 /* 5523 * Write data that will be shifted to preserve them when discarding 5524 * page cache below. We are also protected from pages becoming dirty 5525 * by i_mmap_sem. 5526 */ 5527 ret = filemap_write_and_wait_range(inode->i_mapping, offset + len, 5528 LLONG_MAX); 5529 if (ret) 5530 goto out_mmap; 5531 truncate_pagecache(inode, ioffset); 5532 5533 credits = ext4_writepage_trans_blocks(inode); 5534 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 5535 if (IS_ERR(handle)) { 5536 ret = PTR_ERR(handle); 5537 goto out_mmap; 5538 } 5539 5540 down_write(&EXT4_I(inode)->i_data_sem); 5541 ext4_discard_preallocations(inode); 5542 5543 ret = ext4_es_remove_extent(inode, punch_start, 5544 EXT_MAX_BLOCKS - punch_start); 5545 if (ret) { 5546 up_write(&EXT4_I(inode)->i_data_sem); 5547 goto out_stop; 5548 } 5549 5550 ret = ext4_ext_remove_space(inode, punch_start, punch_stop - 1); 5551 if (ret) { 5552 up_write(&EXT4_I(inode)->i_data_sem); 5553 goto out_stop; 5554 } 5555 ext4_discard_preallocations(inode); 5556 5557 ret = ext4_ext_shift_extents(inode, handle, punch_stop, 5558 punch_stop - punch_start, SHIFT_LEFT); 5559 if (ret) { 5560 up_write(&EXT4_I(inode)->i_data_sem); 5561 goto out_stop; 5562 } 5563 5564 new_size = i_size_read(inode) - len; 5565 i_size_write(inode, new_size); 5566 EXT4_I(inode)->i_disksize = new_size; 5567 5568 up_write(&EXT4_I(inode)->i_data_sem); 5569 if (IS_SYNC(inode)) 5570 ext4_handle_sync(handle); 5571 inode->i_mtime = inode->i_ctime = ext4_current_time(inode); 5572 ext4_mark_inode_dirty(handle, inode); 5573 5574 out_stop: 5575 ext4_journal_stop(handle); 5576 out_mmap: 5577 up_write(&EXT4_I(inode)->i_mmap_sem); 5578 ext4_inode_resume_unlocked_dio(inode); 5579 out_mutex: 5580 inode_unlock(inode); 5581 return ret; 5582 } 5583 5584 /* 5585 * ext4_insert_range: 5586 * This function implements the FALLOC_FL_INSERT_RANGE flag of fallocate. 5587 * The data blocks starting from @offset to the EOF are shifted by @len 5588 * towards right to create a hole in the @inode. Inode size is increased 5589 * by len bytes. 5590 * Returns 0 on success, error otherwise. 5591 */ 5592 int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len) 5593 { 5594 struct super_block *sb = inode->i_sb; 5595 handle_t *handle; 5596 struct ext4_ext_path *path; 5597 struct ext4_extent *extent; 5598 ext4_lblk_t offset_lblk, len_lblk, ee_start_lblk = 0; 5599 unsigned int credits, ee_len; 5600 int ret = 0, depth, split_flag = 0; 5601 loff_t ioffset; 5602 5603 /* 5604 * We need to test this early because xfstests assumes that an 5605 * insert range of (0, 1) will return EOPNOTSUPP if the file 5606 * system does not support insert range. 5607 */ 5608 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 5609 return -EOPNOTSUPP; 5610 5611 /* Insert range works only on fs block size aligned offsets. */ 5612 if (offset & (EXT4_CLUSTER_SIZE(sb) - 1) || 5613 len & (EXT4_CLUSTER_SIZE(sb) - 1)) 5614 return -EINVAL; 5615 5616 if (!S_ISREG(inode->i_mode)) 5617 return -EOPNOTSUPP; 5618 5619 trace_ext4_insert_range(inode, offset, len); 5620 5621 offset_lblk = offset >> EXT4_BLOCK_SIZE_BITS(sb); 5622 len_lblk = len >> EXT4_BLOCK_SIZE_BITS(sb); 5623 5624 /* Call ext4_force_commit to flush all data in case of data=journal */ 5625 if (ext4_should_journal_data(inode)) { 5626 ret = ext4_force_commit(inode->i_sb); 5627 if (ret) 5628 return ret; 5629 } 5630 5631 inode_lock(inode); 5632 /* Currently just for extent based files */ 5633 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 5634 ret = -EOPNOTSUPP; 5635 goto out_mutex; 5636 } 5637 5638 /* Check for wrap through zero */ 5639 if (inode->i_size + len > inode->i_sb->s_maxbytes) { 5640 ret = -EFBIG; 5641 goto out_mutex; 5642 } 5643 5644 /* Offset should be less than i_size */ 5645 if (offset >= i_size_read(inode)) { 5646 ret = -EINVAL; 5647 goto out_mutex; 5648 } 5649 5650 /* Wait for existing dio to complete */ 5651 ext4_inode_block_unlocked_dio(inode); 5652 inode_dio_wait(inode); 5653 5654 /* 5655 * Prevent page faults from reinstantiating pages we have released from 5656 * page cache. 5657 */ 5658 down_write(&EXT4_I(inode)->i_mmap_sem); 5659 /* 5660 * Need to round down to align start offset to page size boundary 5661 * for page size > block size. 5662 */ 5663 ioffset = round_down(offset, PAGE_SIZE); 5664 /* Write out all dirty pages */ 5665 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset, 5666 LLONG_MAX); 5667 if (ret) 5668 goto out_mmap; 5669 truncate_pagecache(inode, ioffset); 5670 5671 credits = ext4_writepage_trans_blocks(inode); 5672 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 5673 if (IS_ERR(handle)) { 5674 ret = PTR_ERR(handle); 5675 goto out_mmap; 5676 } 5677 5678 /* Expand file to avoid data loss if there is error while shifting */ 5679 inode->i_size += len; 5680 EXT4_I(inode)->i_disksize += len; 5681 inode->i_mtime = inode->i_ctime = ext4_current_time(inode); 5682 ret = ext4_mark_inode_dirty(handle, inode); 5683 if (ret) 5684 goto out_stop; 5685 5686 down_write(&EXT4_I(inode)->i_data_sem); 5687 ext4_discard_preallocations(inode); 5688 5689 path = ext4_find_extent(inode, offset_lblk, NULL, 0); 5690 if (IS_ERR(path)) { 5691 up_write(&EXT4_I(inode)->i_data_sem); 5692 goto out_stop; 5693 } 5694 5695 depth = ext_depth(inode); 5696 extent = path[depth].p_ext; 5697 if (extent) { 5698 ee_start_lblk = le32_to_cpu(extent->ee_block); 5699 ee_len = ext4_ext_get_actual_len(extent); 5700 5701 /* 5702 * If offset_lblk is not the starting block of extent, split 5703 * the extent @offset_lblk 5704 */ 5705 if ((offset_lblk > ee_start_lblk) && 5706 (offset_lblk < (ee_start_lblk + ee_len))) { 5707 if (ext4_ext_is_unwritten(extent)) 5708 split_flag = EXT4_EXT_MARK_UNWRIT1 | 5709 EXT4_EXT_MARK_UNWRIT2; 5710 ret = ext4_split_extent_at(handle, inode, &path, 5711 offset_lblk, split_flag, 5712 EXT4_EX_NOCACHE | 5713 EXT4_GET_BLOCKS_PRE_IO | 5714 EXT4_GET_BLOCKS_METADATA_NOFAIL); 5715 } 5716 5717 ext4_ext_drop_refs(path); 5718 kfree(path); 5719 if (ret < 0) { 5720 up_write(&EXT4_I(inode)->i_data_sem); 5721 goto out_stop; 5722 } 5723 } else { 5724 ext4_ext_drop_refs(path); 5725 kfree(path); 5726 } 5727 5728 ret = ext4_es_remove_extent(inode, offset_lblk, 5729 EXT_MAX_BLOCKS - offset_lblk); 5730 if (ret) { 5731 up_write(&EXT4_I(inode)->i_data_sem); 5732 goto out_stop; 5733 } 5734 5735 /* 5736 * if offset_lblk lies in a hole which is at start of file, use 5737 * ee_start_lblk to shift extents 5738 */ 5739 ret = ext4_ext_shift_extents(inode, handle, 5740 ee_start_lblk > offset_lblk ? ee_start_lblk : offset_lblk, 5741 len_lblk, SHIFT_RIGHT); 5742 5743 up_write(&EXT4_I(inode)->i_data_sem); 5744 if (IS_SYNC(inode)) 5745 ext4_handle_sync(handle); 5746 5747 out_stop: 5748 ext4_journal_stop(handle); 5749 out_mmap: 5750 up_write(&EXT4_I(inode)->i_mmap_sem); 5751 ext4_inode_resume_unlocked_dio(inode); 5752 out_mutex: 5753 inode_unlock(inode); 5754 return ret; 5755 } 5756 5757 /** 5758 * ext4_swap_extents - Swap extents between two inodes 5759 * 5760 * @inode1: First inode 5761 * @inode2: Second inode 5762 * @lblk1: Start block for first inode 5763 * @lblk2: Start block for second inode 5764 * @count: Number of blocks to swap 5765 * @mark_unwritten: Mark second inode's extents as unwritten after swap 5766 * @erp: Pointer to save error value 5767 * 5768 * This helper routine does exactly what is promise "swap extents". All other 5769 * stuff such as page-cache locking consistency, bh mapping consistency or 5770 * extent's data copying must be performed by caller. 5771 * Locking: 5772 * i_mutex is held for both inodes 5773 * i_data_sem is locked for write for both inodes 5774 * Assumptions: 5775 * All pages from requested range are locked for both inodes 5776 */ 5777 int 5778 ext4_swap_extents(handle_t *handle, struct inode *inode1, 5779 struct inode *inode2, ext4_lblk_t lblk1, ext4_lblk_t lblk2, 5780 ext4_lblk_t count, int unwritten, int *erp) 5781 { 5782 struct ext4_ext_path *path1 = NULL; 5783 struct ext4_ext_path *path2 = NULL; 5784 int replaced_count = 0; 5785 5786 BUG_ON(!rwsem_is_locked(&EXT4_I(inode1)->i_data_sem)); 5787 BUG_ON(!rwsem_is_locked(&EXT4_I(inode2)->i_data_sem)); 5788 BUG_ON(!inode_is_locked(inode1)); 5789 BUG_ON(!inode_is_locked(inode2)); 5790 5791 *erp = ext4_es_remove_extent(inode1, lblk1, count); 5792 if (unlikely(*erp)) 5793 return 0; 5794 *erp = ext4_es_remove_extent(inode2, lblk2, count); 5795 if (unlikely(*erp)) 5796 return 0; 5797 5798 while (count) { 5799 struct ext4_extent *ex1, *ex2, tmp_ex; 5800 ext4_lblk_t e1_blk, e2_blk; 5801 int e1_len, e2_len, len; 5802 int split = 0; 5803 5804 path1 = ext4_find_extent(inode1, lblk1, NULL, EXT4_EX_NOCACHE); 5805 if (IS_ERR(path1)) { 5806 *erp = PTR_ERR(path1); 5807 path1 = NULL; 5808 finish: 5809 count = 0; 5810 goto repeat; 5811 } 5812 path2 = ext4_find_extent(inode2, lblk2, NULL, EXT4_EX_NOCACHE); 5813 if (IS_ERR(path2)) { 5814 *erp = PTR_ERR(path2); 5815 path2 = NULL; 5816 goto finish; 5817 } 5818 ex1 = path1[path1->p_depth].p_ext; 5819 ex2 = path2[path2->p_depth].p_ext; 5820 /* Do we have somthing to swap ? */ 5821 if (unlikely(!ex2 || !ex1)) 5822 goto finish; 5823 5824 e1_blk = le32_to_cpu(ex1->ee_block); 5825 e2_blk = le32_to_cpu(ex2->ee_block); 5826 e1_len = ext4_ext_get_actual_len(ex1); 5827 e2_len = ext4_ext_get_actual_len(ex2); 5828 5829 /* Hole handling */ 5830 if (!in_range(lblk1, e1_blk, e1_len) || 5831 !in_range(lblk2, e2_blk, e2_len)) { 5832 ext4_lblk_t next1, next2; 5833 5834 /* if hole after extent, then go to next extent */ 5835 next1 = ext4_ext_next_allocated_block(path1); 5836 next2 = ext4_ext_next_allocated_block(path2); 5837 /* If hole before extent, then shift to that extent */ 5838 if (e1_blk > lblk1) 5839 next1 = e1_blk; 5840 if (e2_blk > lblk2) 5841 next2 = e1_blk; 5842 /* Do we have something to swap */ 5843 if (next1 == EXT_MAX_BLOCKS || next2 == EXT_MAX_BLOCKS) 5844 goto finish; 5845 /* Move to the rightest boundary */ 5846 len = next1 - lblk1; 5847 if (len < next2 - lblk2) 5848 len = next2 - lblk2; 5849 if (len > count) 5850 len = count; 5851 lblk1 += len; 5852 lblk2 += len; 5853 count -= len; 5854 goto repeat; 5855 } 5856 5857 /* Prepare left boundary */ 5858 if (e1_blk < lblk1) { 5859 split = 1; 5860 *erp = ext4_force_split_extent_at(handle, inode1, 5861 &path1, lblk1, 0); 5862 if (unlikely(*erp)) 5863 goto finish; 5864 } 5865 if (e2_blk < lblk2) { 5866 split = 1; 5867 *erp = ext4_force_split_extent_at(handle, inode2, 5868 &path2, lblk2, 0); 5869 if (unlikely(*erp)) 5870 goto finish; 5871 } 5872 /* ext4_split_extent_at() may result in leaf extent split, 5873 * path must to be revalidated. */ 5874 if (split) 5875 goto repeat; 5876 5877 /* Prepare right boundary */ 5878 len = count; 5879 if (len > e1_blk + e1_len - lblk1) 5880 len = e1_blk + e1_len - lblk1; 5881 if (len > e2_blk + e2_len - lblk2) 5882 len = e2_blk + e2_len - lblk2; 5883 5884 if (len != e1_len) { 5885 split = 1; 5886 *erp = ext4_force_split_extent_at(handle, inode1, 5887 &path1, lblk1 + len, 0); 5888 if (unlikely(*erp)) 5889 goto finish; 5890 } 5891 if (len != e2_len) { 5892 split = 1; 5893 *erp = ext4_force_split_extent_at(handle, inode2, 5894 &path2, lblk2 + len, 0); 5895 if (*erp) 5896 goto finish; 5897 } 5898 /* ext4_split_extent_at() may result in leaf extent split, 5899 * path must to be revalidated. */ 5900 if (split) 5901 goto repeat; 5902 5903 BUG_ON(e2_len != e1_len); 5904 *erp = ext4_ext_get_access(handle, inode1, path1 + path1->p_depth); 5905 if (unlikely(*erp)) 5906 goto finish; 5907 *erp = ext4_ext_get_access(handle, inode2, path2 + path2->p_depth); 5908 if (unlikely(*erp)) 5909 goto finish; 5910 5911 /* Both extents are fully inside boundaries. Swap it now */ 5912 tmp_ex = *ex1; 5913 ext4_ext_store_pblock(ex1, ext4_ext_pblock(ex2)); 5914 ext4_ext_store_pblock(ex2, ext4_ext_pblock(&tmp_ex)); 5915 ex1->ee_len = cpu_to_le16(e2_len); 5916 ex2->ee_len = cpu_to_le16(e1_len); 5917 if (unwritten) 5918 ext4_ext_mark_unwritten(ex2); 5919 if (ext4_ext_is_unwritten(&tmp_ex)) 5920 ext4_ext_mark_unwritten(ex1); 5921 5922 ext4_ext_try_to_merge(handle, inode2, path2, ex2); 5923 ext4_ext_try_to_merge(handle, inode1, path1, ex1); 5924 *erp = ext4_ext_dirty(handle, inode2, path2 + 5925 path2->p_depth); 5926 if (unlikely(*erp)) 5927 goto finish; 5928 *erp = ext4_ext_dirty(handle, inode1, path1 + 5929 path1->p_depth); 5930 /* 5931 * Looks scarry ah..? second inode already points to new blocks, 5932 * and it was successfully dirtied. But luckily error may happen 5933 * only due to journal error, so full transaction will be 5934 * aborted anyway. 5935 */ 5936 if (unlikely(*erp)) 5937 goto finish; 5938 lblk1 += len; 5939 lblk2 += len; 5940 replaced_count += len; 5941 count -= len; 5942 5943 repeat: 5944 ext4_ext_drop_refs(path1); 5945 kfree(path1); 5946 ext4_ext_drop_refs(path2); 5947 kfree(path2); 5948 path1 = path2 = NULL; 5949 } 5950 return replaced_count; 5951 } 5952