1 /* 2 * Copyright (C) 2007 Oracle. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public 6 * License v2 as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public 14 * License along with this program; if not, write to the 15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 16 * Boston, MA 021110-1307, USA. 17 */ 18 19 #include <linux/kernel.h> 20 #include <linux/bio.h> 21 #include <linux/buffer_head.h> 22 #include <linux/file.h> 23 #include <linux/fs.h> 24 #include <linux/pagemap.h> 25 #include <linux/highmem.h> 26 #include <linux/time.h> 27 #include <linux/init.h> 28 #include <linux/string.h> 29 #include <linux/backing-dev.h> 30 #include <linux/mpage.h> 31 #include <linux/swap.h> 32 #include <linux/writeback.h> 33 #include <linux/compat.h> 34 #include <linux/bit_spinlock.h> 35 #include <linux/xattr.h> 36 #include <linux/posix_acl.h> 37 #include <linux/falloc.h> 38 #include <linux/slab.h> 39 #include <linux/ratelimit.h> 40 #include <linux/mount.h> 41 #include <linux/btrfs.h> 42 #include <linux/blkdev.h> 43 #include <linux/posix_acl_xattr.h> 44 #include <linux/uio.h> 45 #include <linux/magic.h> 46 #include <linux/iversion.h> 47 #include "ctree.h" 48 #include "disk-io.h" 49 #include "transaction.h" 50 #include "btrfs_inode.h" 51 #include "print-tree.h" 52 #include "ordered-data.h" 53 #include "xattr.h" 54 #include "tree-log.h" 55 #include "volumes.h" 56 #include "compression.h" 57 #include "locking.h" 58 #include "free-space-cache.h" 59 #include "inode-map.h" 60 #include "backref.h" 61 #include "props.h" 62 #include "qgroup.h" 63 #include "dedupe.h" 64 65 struct btrfs_iget_args { 66 struct btrfs_key *location; 67 struct btrfs_root *root; 68 }; 69 70 struct btrfs_dio_data { 71 u64 reserve; 72 u64 unsubmitted_oe_range_start; 73 u64 unsubmitted_oe_range_end; 74 int overwrite; 75 }; 76 77 static const struct inode_operations btrfs_dir_inode_operations; 78 static const struct inode_operations btrfs_symlink_inode_operations; 79 static const struct inode_operations btrfs_dir_ro_inode_operations; 80 static const struct inode_operations btrfs_special_inode_operations; 81 static const struct inode_operations btrfs_file_inode_operations; 82 static const struct address_space_operations btrfs_aops; 83 static const struct address_space_operations btrfs_symlink_aops; 84 static const struct file_operations btrfs_dir_file_operations; 85 static const struct extent_io_ops btrfs_extent_io_ops; 86 87 static struct kmem_cache *btrfs_inode_cachep; 88 struct kmem_cache *btrfs_trans_handle_cachep; 89 struct kmem_cache *btrfs_path_cachep; 90 struct kmem_cache *btrfs_free_space_cachep; 91 92 #define S_SHIFT 12 93 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = { 94 [S_IFREG >> S_SHIFT] = BTRFS_FT_REG_FILE, 95 [S_IFDIR >> S_SHIFT] = BTRFS_FT_DIR, 96 [S_IFCHR >> S_SHIFT] = BTRFS_FT_CHRDEV, 97 [S_IFBLK >> S_SHIFT] = BTRFS_FT_BLKDEV, 98 [S_IFIFO >> S_SHIFT] = BTRFS_FT_FIFO, 99 [S_IFSOCK >> S_SHIFT] = BTRFS_FT_SOCK, 100 [S_IFLNK >> S_SHIFT] = BTRFS_FT_SYMLINK, 101 }; 102 103 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 104 static int btrfs_truncate(struct inode *inode, bool skip_writeback); 105 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); 106 static noinline int cow_file_range(struct inode *inode, 107 struct page *locked_page, 108 u64 start, u64 end, u64 delalloc_end, 109 int *page_started, unsigned long *nr_written, 110 int unlock, struct btrfs_dedupe_hash *hash); 111 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, 112 u64 orig_start, u64 block_start, 113 u64 block_len, u64 orig_block_len, 114 u64 ram_bytes, int compress_type, 115 int type); 116 117 static void __endio_write_update_ordered(struct inode *inode, 118 const u64 offset, const u64 bytes, 119 const bool uptodate); 120 121 /* 122 * Cleanup all submitted ordered extents in specified range to handle errors 123 * from the fill_dellaloc() callback. 124 * 125 * NOTE: caller must ensure that when an error happens, it can not call 126 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING 127 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata 128 * to be released, which we want to happen only when finishing the ordered 129 * extent (btrfs_finish_ordered_io()). Also note that the caller of the 130 * fill_delalloc() callback already does proper cleanup for the first page of 131 * the range, that is, it invokes the callback writepage_end_io_hook() for the 132 * range of the first page. 133 */ 134 static inline void btrfs_cleanup_ordered_extents(struct inode *inode, 135 const u64 offset, 136 const u64 bytes) 137 { 138 unsigned long index = offset >> PAGE_SHIFT; 139 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT; 140 struct page *page; 141 142 while (index <= end_index) { 143 page = find_get_page(inode->i_mapping, index); 144 index++; 145 if (!page) 146 continue; 147 ClearPagePrivate2(page); 148 put_page(page); 149 } 150 return __endio_write_update_ordered(inode, offset + PAGE_SIZE, 151 bytes - PAGE_SIZE, false); 152 } 153 154 static int btrfs_dirty_inode(struct inode *inode); 155 156 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 157 void btrfs_test_inode_set_ops(struct inode *inode) 158 { 159 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 160 } 161 #endif 162 163 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 164 struct inode *inode, struct inode *dir, 165 const struct qstr *qstr) 166 { 167 int err; 168 169 err = btrfs_init_acl(trans, inode, dir); 170 if (!err) 171 err = btrfs_xattr_security_init(trans, inode, dir, qstr); 172 return err; 173 } 174 175 /* 176 * this does all the hard work for inserting an inline extent into 177 * the btree. The caller should have done a btrfs_drop_extents so that 178 * no overlapping inline items exist in the btree 179 */ 180 static int insert_inline_extent(struct btrfs_trans_handle *trans, 181 struct btrfs_path *path, int extent_inserted, 182 struct btrfs_root *root, struct inode *inode, 183 u64 start, size_t size, size_t compressed_size, 184 int compress_type, 185 struct page **compressed_pages) 186 { 187 struct extent_buffer *leaf; 188 struct page *page = NULL; 189 char *kaddr; 190 unsigned long ptr; 191 struct btrfs_file_extent_item *ei; 192 int ret; 193 size_t cur_size = size; 194 unsigned long offset; 195 196 if (compressed_size && compressed_pages) 197 cur_size = compressed_size; 198 199 inode_add_bytes(inode, size); 200 201 if (!extent_inserted) { 202 struct btrfs_key key; 203 size_t datasize; 204 205 key.objectid = btrfs_ino(BTRFS_I(inode)); 206 key.offset = start; 207 key.type = BTRFS_EXTENT_DATA_KEY; 208 209 datasize = btrfs_file_extent_calc_inline_size(cur_size); 210 path->leave_spinning = 1; 211 ret = btrfs_insert_empty_item(trans, root, path, &key, 212 datasize); 213 if (ret) 214 goto fail; 215 } 216 leaf = path->nodes[0]; 217 ei = btrfs_item_ptr(leaf, path->slots[0], 218 struct btrfs_file_extent_item); 219 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 220 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 221 btrfs_set_file_extent_encryption(leaf, ei, 0); 222 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 223 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 224 ptr = btrfs_file_extent_inline_start(ei); 225 226 if (compress_type != BTRFS_COMPRESS_NONE) { 227 struct page *cpage; 228 int i = 0; 229 while (compressed_size > 0) { 230 cpage = compressed_pages[i]; 231 cur_size = min_t(unsigned long, compressed_size, 232 PAGE_SIZE); 233 234 kaddr = kmap_atomic(cpage); 235 write_extent_buffer(leaf, kaddr, ptr, cur_size); 236 kunmap_atomic(kaddr); 237 238 i++; 239 ptr += cur_size; 240 compressed_size -= cur_size; 241 } 242 btrfs_set_file_extent_compression(leaf, ei, 243 compress_type); 244 } else { 245 page = find_get_page(inode->i_mapping, 246 start >> PAGE_SHIFT); 247 btrfs_set_file_extent_compression(leaf, ei, 0); 248 kaddr = kmap_atomic(page); 249 offset = start & (PAGE_SIZE - 1); 250 write_extent_buffer(leaf, kaddr + offset, ptr, size); 251 kunmap_atomic(kaddr); 252 put_page(page); 253 } 254 btrfs_mark_buffer_dirty(leaf); 255 btrfs_release_path(path); 256 257 /* 258 * we're an inline extent, so nobody can 259 * extend the file past i_size without locking 260 * a page we already have locked. 261 * 262 * We must do any isize and inode updates 263 * before we unlock the pages. Otherwise we 264 * could end up racing with unlink. 265 */ 266 BTRFS_I(inode)->disk_i_size = inode->i_size; 267 ret = btrfs_update_inode(trans, root, inode); 268 269 fail: 270 return ret; 271 } 272 273 274 /* 275 * conditionally insert an inline extent into the file. This 276 * does the checks required to make sure the data is small enough 277 * to fit as an inline extent. 278 */ 279 static noinline int cow_file_range_inline(struct inode *inode, u64 start, 280 u64 end, size_t compressed_size, 281 int compress_type, 282 struct page **compressed_pages) 283 { 284 struct btrfs_root *root = BTRFS_I(inode)->root; 285 struct btrfs_fs_info *fs_info = root->fs_info; 286 struct btrfs_trans_handle *trans; 287 u64 isize = i_size_read(inode); 288 u64 actual_end = min(end + 1, isize); 289 u64 inline_len = actual_end - start; 290 u64 aligned_end = ALIGN(end, fs_info->sectorsize); 291 u64 data_len = inline_len; 292 int ret; 293 struct btrfs_path *path; 294 int extent_inserted = 0; 295 u32 extent_item_size; 296 297 if (compressed_size) 298 data_len = compressed_size; 299 300 if (start > 0 || 301 actual_end > fs_info->sectorsize || 302 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 303 (!compressed_size && 304 (actual_end & (fs_info->sectorsize - 1)) == 0) || 305 end + 1 < isize || 306 data_len > fs_info->max_inline) { 307 return 1; 308 } 309 310 path = btrfs_alloc_path(); 311 if (!path) 312 return -ENOMEM; 313 314 trans = btrfs_join_transaction(root); 315 if (IS_ERR(trans)) { 316 btrfs_free_path(path); 317 return PTR_ERR(trans); 318 } 319 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 320 321 if (compressed_size && compressed_pages) 322 extent_item_size = btrfs_file_extent_calc_inline_size( 323 compressed_size); 324 else 325 extent_item_size = btrfs_file_extent_calc_inline_size( 326 inline_len); 327 328 ret = __btrfs_drop_extents(trans, root, inode, path, 329 start, aligned_end, NULL, 330 1, 1, extent_item_size, &extent_inserted); 331 if (ret) { 332 btrfs_abort_transaction(trans, ret); 333 goto out; 334 } 335 336 if (isize > actual_end) 337 inline_len = min_t(u64, isize, actual_end); 338 ret = insert_inline_extent(trans, path, extent_inserted, 339 root, inode, start, 340 inline_len, compressed_size, 341 compress_type, compressed_pages); 342 if (ret && ret != -ENOSPC) { 343 btrfs_abort_transaction(trans, ret); 344 goto out; 345 } else if (ret == -ENOSPC) { 346 ret = 1; 347 goto out; 348 } 349 350 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 351 btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0); 352 out: 353 /* 354 * Don't forget to free the reserved space, as for inlined extent 355 * it won't count as data extent, free them directly here. 356 * And at reserve time, it's always aligned to page size, so 357 * just free one page here. 358 */ 359 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE); 360 btrfs_free_path(path); 361 btrfs_end_transaction(trans); 362 return ret; 363 } 364 365 struct async_extent { 366 u64 start; 367 u64 ram_size; 368 u64 compressed_size; 369 struct page **pages; 370 unsigned long nr_pages; 371 int compress_type; 372 struct list_head list; 373 }; 374 375 struct async_cow { 376 struct inode *inode; 377 struct btrfs_root *root; 378 struct page *locked_page; 379 u64 start; 380 u64 end; 381 unsigned int write_flags; 382 struct list_head extents; 383 struct btrfs_work work; 384 }; 385 386 static noinline int add_async_extent(struct async_cow *cow, 387 u64 start, u64 ram_size, 388 u64 compressed_size, 389 struct page **pages, 390 unsigned long nr_pages, 391 int compress_type) 392 { 393 struct async_extent *async_extent; 394 395 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 396 BUG_ON(!async_extent); /* -ENOMEM */ 397 async_extent->start = start; 398 async_extent->ram_size = ram_size; 399 async_extent->compressed_size = compressed_size; 400 async_extent->pages = pages; 401 async_extent->nr_pages = nr_pages; 402 async_extent->compress_type = compress_type; 403 list_add_tail(&async_extent->list, &cow->extents); 404 return 0; 405 } 406 407 static inline int inode_need_compress(struct inode *inode, u64 start, u64 end) 408 { 409 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 410 411 /* force compress */ 412 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 413 return 1; 414 /* defrag ioctl */ 415 if (BTRFS_I(inode)->defrag_compress) 416 return 1; 417 /* bad compression ratios */ 418 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS) 419 return 0; 420 if (btrfs_test_opt(fs_info, COMPRESS) || 421 BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS || 422 BTRFS_I(inode)->prop_compress) 423 return btrfs_compress_heuristic(inode, start, end); 424 return 0; 425 } 426 427 static inline void inode_should_defrag(struct btrfs_inode *inode, 428 u64 start, u64 end, u64 num_bytes, u64 small_write) 429 { 430 /* If this is a small write inside eof, kick off a defrag */ 431 if (num_bytes < small_write && 432 (start > 0 || end + 1 < inode->disk_i_size)) 433 btrfs_add_inode_defrag(NULL, inode); 434 } 435 436 /* 437 * we create compressed extents in two phases. The first 438 * phase compresses a range of pages that have already been 439 * locked (both pages and state bits are locked). 440 * 441 * This is done inside an ordered work queue, and the compression 442 * is spread across many cpus. The actual IO submission is step 443 * two, and the ordered work queue takes care of making sure that 444 * happens in the same order things were put onto the queue by 445 * writepages and friends. 446 * 447 * If this code finds it can't get good compression, it puts an 448 * entry onto the work queue to write the uncompressed bytes. This 449 * makes sure that both compressed inodes and uncompressed inodes 450 * are written in the same order that the flusher thread sent them 451 * down. 452 */ 453 static noinline void compress_file_range(struct inode *inode, 454 struct page *locked_page, 455 u64 start, u64 end, 456 struct async_cow *async_cow, 457 int *num_added) 458 { 459 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 460 u64 blocksize = fs_info->sectorsize; 461 u64 actual_end; 462 u64 isize = i_size_read(inode); 463 int ret = 0; 464 struct page **pages = NULL; 465 unsigned long nr_pages; 466 unsigned long total_compressed = 0; 467 unsigned long total_in = 0; 468 int i; 469 int will_compress; 470 int compress_type = fs_info->compress_type; 471 int redirty = 0; 472 473 inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1, 474 SZ_16K); 475 476 actual_end = min_t(u64, isize, end + 1); 477 again: 478 will_compress = 0; 479 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; 480 BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0); 481 nr_pages = min_t(unsigned long, nr_pages, 482 BTRFS_MAX_COMPRESSED / PAGE_SIZE); 483 484 /* 485 * we don't want to send crud past the end of i_size through 486 * compression, that's just a waste of CPU time. So, if the 487 * end of the file is before the start of our current 488 * requested range of bytes, we bail out to the uncompressed 489 * cleanup code that can deal with all of this. 490 * 491 * It isn't really the fastest way to fix things, but this is a 492 * very uncommon corner. 493 */ 494 if (actual_end <= start) 495 goto cleanup_and_bail_uncompressed; 496 497 total_compressed = actual_end - start; 498 499 /* 500 * skip compression for a small file range(<=blocksize) that 501 * isn't an inline extent, since it doesn't save disk space at all. 502 */ 503 if (total_compressed <= blocksize && 504 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 505 goto cleanup_and_bail_uncompressed; 506 507 total_compressed = min_t(unsigned long, total_compressed, 508 BTRFS_MAX_UNCOMPRESSED); 509 total_in = 0; 510 ret = 0; 511 512 /* 513 * we do compression for mount -o compress and when the 514 * inode has not been flagged as nocompress. This flag can 515 * change at any time if we discover bad compression ratios. 516 */ 517 if (inode_need_compress(inode, start, end)) { 518 WARN_ON(pages); 519 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 520 if (!pages) { 521 /* just bail out to the uncompressed code */ 522 goto cont; 523 } 524 525 if (BTRFS_I(inode)->defrag_compress) 526 compress_type = BTRFS_I(inode)->defrag_compress; 527 else if (BTRFS_I(inode)->prop_compress) 528 compress_type = BTRFS_I(inode)->prop_compress; 529 530 /* 531 * we need to call clear_page_dirty_for_io on each 532 * page in the range. Otherwise applications with the file 533 * mmap'd can wander in and change the page contents while 534 * we are compressing them. 535 * 536 * If the compression fails for any reason, we set the pages 537 * dirty again later on. 538 * 539 * Note that the remaining part is redirtied, the start pointer 540 * has moved, the end is the original one. 541 */ 542 if (!redirty) { 543 extent_range_clear_dirty_for_io(inode, start, end); 544 redirty = 1; 545 } 546 547 /* Compression level is applied here and only here */ 548 ret = btrfs_compress_pages( 549 compress_type | (fs_info->compress_level << 4), 550 inode->i_mapping, start, 551 pages, 552 &nr_pages, 553 &total_in, 554 &total_compressed); 555 556 if (!ret) { 557 unsigned long offset = total_compressed & 558 (PAGE_SIZE - 1); 559 struct page *page = pages[nr_pages - 1]; 560 char *kaddr; 561 562 /* zero the tail end of the last page, we might be 563 * sending it down to disk 564 */ 565 if (offset) { 566 kaddr = kmap_atomic(page); 567 memset(kaddr + offset, 0, 568 PAGE_SIZE - offset); 569 kunmap_atomic(kaddr); 570 } 571 will_compress = 1; 572 } 573 } 574 cont: 575 if (start == 0) { 576 /* lets try to make an inline extent */ 577 if (ret || total_in < actual_end) { 578 /* we didn't compress the entire range, try 579 * to make an uncompressed inline extent. 580 */ 581 ret = cow_file_range_inline(inode, start, end, 0, 582 BTRFS_COMPRESS_NONE, NULL); 583 } else { 584 /* try making a compressed inline extent */ 585 ret = cow_file_range_inline(inode, start, end, 586 total_compressed, 587 compress_type, pages); 588 } 589 if (ret <= 0) { 590 unsigned long clear_flags = EXTENT_DELALLOC | 591 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 592 EXTENT_DO_ACCOUNTING; 593 unsigned long page_error_op; 594 595 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 596 597 /* 598 * inline extent creation worked or returned error, 599 * we don't need to create any more async work items. 600 * Unlock and free up our temp pages. 601 * 602 * We use DO_ACCOUNTING here because we need the 603 * delalloc_release_metadata to be done _after_ we drop 604 * our outstanding extent for clearing delalloc for this 605 * range. 606 */ 607 extent_clear_unlock_delalloc(inode, start, end, end, 608 NULL, clear_flags, 609 PAGE_UNLOCK | 610 PAGE_CLEAR_DIRTY | 611 PAGE_SET_WRITEBACK | 612 page_error_op | 613 PAGE_END_WRITEBACK); 614 goto free_pages_out; 615 } 616 } 617 618 if (will_compress) { 619 /* 620 * we aren't doing an inline extent round the compressed size 621 * up to a block size boundary so the allocator does sane 622 * things 623 */ 624 total_compressed = ALIGN(total_compressed, blocksize); 625 626 /* 627 * one last check to make sure the compression is really a 628 * win, compare the page count read with the blocks on disk, 629 * compression must free at least one sector size 630 */ 631 total_in = ALIGN(total_in, PAGE_SIZE); 632 if (total_compressed + blocksize <= total_in) { 633 *num_added += 1; 634 635 /* 636 * The async work queues will take care of doing actual 637 * allocation on disk for these compressed pages, and 638 * will submit them to the elevator. 639 */ 640 add_async_extent(async_cow, start, total_in, 641 total_compressed, pages, nr_pages, 642 compress_type); 643 644 if (start + total_in < end) { 645 start += total_in; 646 pages = NULL; 647 cond_resched(); 648 goto again; 649 } 650 return; 651 } 652 } 653 if (pages) { 654 /* 655 * the compression code ran but failed to make things smaller, 656 * free any pages it allocated and our page pointer array 657 */ 658 for (i = 0; i < nr_pages; i++) { 659 WARN_ON(pages[i]->mapping); 660 put_page(pages[i]); 661 } 662 kfree(pages); 663 pages = NULL; 664 total_compressed = 0; 665 nr_pages = 0; 666 667 /* flag the file so we don't compress in the future */ 668 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && 669 !(BTRFS_I(inode)->prop_compress)) { 670 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 671 } 672 } 673 cleanup_and_bail_uncompressed: 674 /* 675 * No compression, but we still need to write the pages in the file 676 * we've been given so far. redirty the locked page if it corresponds 677 * to our extent and set things up for the async work queue to run 678 * cow_file_range to do the normal delalloc dance. 679 */ 680 if (page_offset(locked_page) >= start && 681 page_offset(locked_page) <= end) 682 __set_page_dirty_nobuffers(locked_page); 683 /* unlocked later on in the async handlers */ 684 685 if (redirty) 686 extent_range_redirty_for_io(inode, start, end); 687 add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0, 688 BTRFS_COMPRESS_NONE); 689 *num_added += 1; 690 691 return; 692 693 free_pages_out: 694 for (i = 0; i < nr_pages; i++) { 695 WARN_ON(pages[i]->mapping); 696 put_page(pages[i]); 697 } 698 kfree(pages); 699 } 700 701 static void free_async_extent_pages(struct async_extent *async_extent) 702 { 703 int i; 704 705 if (!async_extent->pages) 706 return; 707 708 for (i = 0; i < async_extent->nr_pages; i++) { 709 WARN_ON(async_extent->pages[i]->mapping); 710 put_page(async_extent->pages[i]); 711 } 712 kfree(async_extent->pages); 713 async_extent->nr_pages = 0; 714 async_extent->pages = NULL; 715 } 716 717 /* 718 * phase two of compressed writeback. This is the ordered portion 719 * of the code, which only gets called in the order the work was 720 * queued. We walk all the async extents created by compress_file_range 721 * and send them down to the disk. 722 */ 723 static noinline void submit_compressed_extents(struct inode *inode, 724 struct async_cow *async_cow) 725 { 726 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 727 struct async_extent *async_extent; 728 u64 alloc_hint = 0; 729 struct btrfs_key ins; 730 struct extent_map *em; 731 struct btrfs_root *root = BTRFS_I(inode)->root; 732 struct extent_io_tree *io_tree; 733 int ret = 0; 734 735 again: 736 while (!list_empty(&async_cow->extents)) { 737 async_extent = list_entry(async_cow->extents.next, 738 struct async_extent, list); 739 list_del(&async_extent->list); 740 741 io_tree = &BTRFS_I(inode)->io_tree; 742 743 retry: 744 /* did the compression code fall back to uncompressed IO? */ 745 if (!async_extent->pages) { 746 int page_started = 0; 747 unsigned long nr_written = 0; 748 749 lock_extent(io_tree, async_extent->start, 750 async_extent->start + 751 async_extent->ram_size - 1); 752 753 /* allocate blocks */ 754 ret = cow_file_range(inode, async_cow->locked_page, 755 async_extent->start, 756 async_extent->start + 757 async_extent->ram_size - 1, 758 async_extent->start + 759 async_extent->ram_size - 1, 760 &page_started, &nr_written, 0, 761 NULL); 762 763 /* JDM XXX */ 764 765 /* 766 * if page_started, cow_file_range inserted an 767 * inline extent and took care of all the unlocking 768 * and IO for us. Otherwise, we need to submit 769 * all those pages down to the drive. 770 */ 771 if (!page_started && !ret) 772 extent_write_locked_range(inode, 773 async_extent->start, 774 async_extent->start + 775 async_extent->ram_size - 1, 776 WB_SYNC_ALL); 777 else if (ret) 778 unlock_page(async_cow->locked_page); 779 kfree(async_extent); 780 cond_resched(); 781 continue; 782 } 783 784 lock_extent(io_tree, async_extent->start, 785 async_extent->start + async_extent->ram_size - 1); 786 787 ret = btrfs_reserve_extent(root, async_extent->ram_size, 788 async_extent->compressed_size, 789 async_extent->compressed_size, 790 0, alloc_hint, &ins, 1, 1); 791 if (ret) { 792 free_async_extent_pages(async_extent); 793 794 if (ret == -ENOSPC) { 795 unlock_extent(io_tree, async_extent->start, 796 async_extent->start + 797 async_extent->ram_size - 1); 798 799 /* 800 * we need to redirty the pages if we decide to 801 * fallback to uncompressed IO, otherwise we 802 * will not submit these pages down to lower 803 * layers. 804 */ 805 extent_range_redirty_for_io(inode, 806 async_extent->start, 807 async_extent->start + 808 async_extent->ram_size - 1); 809 810 goto retry; 811 } 812 goto out_free; 813 } 814 /* 815 * here we're doing allocation and writeback of the 816 * compressed pages 817 */ 818 em = create_io_em(inode, async_extent->start, 819 async_extent->ram_size, /* len */ 820 async_extent->start, /* orig_start */ 821 ins.objectid, /* block_start */ 822 ins.offset, /* block_len */ 823 ins.offset, /* orig_block_len */ 824 async_extent->ram_size, /* ram_bytes */ 825 async_extent->compress_type, 826 BTRFS_ORDERED_COMPRESSED); 827 if (IS_ERR(em)) 828 /* ret value is not necessary due to void function */ 829 goto out_free_reserve; 830 free_extent_map(em); 831 832 ret = btrfs_add_ordered_extent_compress(inode, 833 async_extent->start, 834 ins.objectid, 835 async_extent->ram_size, 836 ins.offset, 837 BTRFS_ORDERED_COMPRESSED, 838 async_extent->compress_type); 839 if (ret) { 840 btrfs_drop_extent_cache(BTRFS_I(inode), 841 async_extent->start, 842 async_extent->start + 843 async_extent->ram_size - 1, 0); 844 goto out_free_reserve; 845 } 846 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 847 848 /* 849 * clear dirty, set writeback and unlock the pages. 850 */ 851 extent_clear_unlock_delalloc(inode, async_extent->start, 852 async_extent->start + 853 async_extent->ram_size - 1, 854 async_extent->start + 855 async_extent->ram_size - 1, 856 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 857 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 858 PAGE_SET_WRITEBACK); 859 if (btrfs_submit_compressed_write(inode, 860 async_extent->start, 861 async_extent->ram_size, 862 ins.objectid, 863 ins.offset, async_extent->pages, 864 async_extent->nr_pages, 865 async_cow->write_flags)) { 866 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 867 struct page *p = async_extent->pages[0]; 868 const u64 start = async_extent->start; 869 const u64 end = start + async_extent->ram_size - 1; 870 871 p->mapping = inode->i_mapping; 872 tree->ops->writepage_end_io_hook(p, start, end, 873 NULL, 0); 874 p->mapping = NULL; 875 extent_clear_unlock_delalloc(inode, start, end, end, 876 NULL, 0, 877 PAGE_END_WRITEBACK | 878 PAGE_SET_ERROR); 879 free_async_extent_pages(async_extent); 880 } 881 alloc_hint = ins.objectid + ins.offset; 882 kfree(async_extent); 883 cond_resched(); 884 } 885 return; 886 out_free_reserve: 887 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 888 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 889 out_free: 890 extent_clear_unlock_delalloc(inode, async_extent->start, 891 async_extent->start + 892 async_extent->ram_size - 1, 893 async_extent->start + 894 async_extent->ram_size - 1, 895 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 896 EXTENT_DELALLOC_NEW | 897 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 898 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 899 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | 900 PAGE_SET_ERROR); 901 free_async_extent_pages(async_extent); 902 kfree(async_extent); 903 goto again; 904 } 905 906 static u64 get_extent_allocation_hint(struct inode *inode, u64 start, 907 u64 num_bytes) 908 { 909 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 910 struct extent_map *em; 911 u64 alloc_hint = 0; 912 913 read_lock(&em_tree->lock); 914 em = search_extent_mapping(em_tree, start, num_bytes); 915 if (em) { 916 /* 917 * if block start isn't an actual block number then find the 918 * first block in this inode and use that as a hint. If that 919 * block is also bogus then just don't worry about it. 920 */ 921 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 922 free_extent_map(em); 923 em = search_extent_mapping(em_tree, 0, 0); 924 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 925 alloc_hint = em->block_start; 926 if (em) 927 free_extent_map(em); 928 } else { 929 alloc_hint = em->block_start; 930 free_extent_map(em); 931 } 932 } 933 read_unlock(&em_tree->lock); 934 935 return alloc_hint; 936 } 937 938 /* 939 * when extent_io.c finds a delayed allocation range in the file, 940 * the call backs end up in this code. The basic idea is to 941 * allocate extents on disk for the range, and create ordered data structs 942 * in ram to track those extents. 943 * 944 * locked_page is the page that writepage had locked already. We use 945 * it to make sure we don't do extra locks or unlocks. 946 * 947 * *page_started is set to one if we unlock locked_page and do everything 948 * required to start IO on it. It may be clean and already done with 949 * IO when we return. 950 */ 951 static noinline int cow_file_range(struct inode *inode, 952 struct page *locked_page, 953 u64 start, u64 end, u64 delalloc_end, 954 int *page_started, unsigned long *nr_written, 955 int unlock, struct btrfs_dedupe_hash *hash) 956 { 957 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 958 struct btrfs_root *root = BTRFS_I(inode)->root; 959 u64 alloc_hint = 0; 960 u64 num_bytes; 961 unsigned long ram_size; 962 u64 cur_alloc_size = 0; 963 u64 blocksize = fs_info->sectorsize; 964 struct btrfs_key ins; 965 struct extent_map *em; 966 unsigned clear_bits; 967 unsigned long page_ops; 968 bool extent_reserved = false; 969 int ret = 0; 970 971 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 972 WARN_ON_ONCE(1); 973 ret = -EINVAL; 974 goto out_unlock; 975 } 976 977 num_bytes = ALIGN(end - start + 1, blocksize); 978 num_bytes = max(blocksize, num_bytes); 979 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); 980 981 inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K); 982 983 if (start == 0) { 984 /* lets try to make an inline extent */ 985 ret = cow_file_range_inline(inode, start, end, 0, 986 BTRFS_COMPRESS_NONE, NULL); 987 if (ret == 0) { 988 /* 989 * We use DO_ACCOUNTING here because we need the 990 * delalloc_release_metadata to be run _after_ we drop 991 * our outstanding extent for clearing delalloc for this 992 * range. 993 */ 994 extent_clear_unlock_delalloc(inode, start, end, 995 delalloc_end, NULL, 996 EXTENT_LOCKED | EXTENT_DELALLOC | 997 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 998 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 999 PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 1000 PAGE_END_WRITEBACK); 1001 *nr_written = *nr_written + 1002 (end - start + PAGE_SIZE) / PAGE_SIZE; 1003 *page_started = 1; 1004 goto out; 1005 } else if (ret < 0) { 1006 goto out_unlock; 1007 } 1008 } 1009 1010 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 1011 btrfs_drop_extent_cache(BTRFS_I(inode), start, 1012 start + num_bytes - 1, 0); 1013 1014 while (num_bytes > 0) { 1015 cur_alloc_size = num_bytes; 1016 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size, 1017 fs_info->sectorsize, 0, alloc_hint, 1018 &ins, 1, 1); 1019 if (ret < 0) 1020 goto out_unlock; 1021 cur_alloc_size = ins.offset; 1022 extent_reserved = true; 1023 1024 ram_size = ins.offset; 1025 em = create_io_em(inode, start, ins.offset, /* len */ 1026 start, /* orig_start */ 1027 ins.objectid, /* block_start */ 1028 ins.offset, /* block_len */ 1029 ins.offset, /* orig_block_len */ 1030 ram_size, /* ram_bytes */ 1031 BTRFS_COMPRESS_NONE, /* compress_type */ 1032 BTRFS_ORDERED_REGULAR /* type */); 1033 if (IS_ERR(em)) 1034 goto out_reserve; 1035 free_extent_map(em); 1036 1037 ret = btrfs_add_ordered_extent(inode, start, ins.objectid, 1038 ram_size, cur_alloc_size, 0); 1039 if (ret) 1040 goto out_drop_extent_cache; 1041 1042 if (root->root_key.objectid == 1043 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1044 ret = btrfs_reloc_clone_csums(inode, start, 1045 cur_alloc_size); 1046 /* 1047 * Only drop cache here, and process as normal. 1048 * 1049 * We must not allow extent_clear_unlock_delalloc() 1050 * at out_unlock label to free meta of this ordered 1051 * extent, as its meta should be freed by 1052 * btrfs_finish_ordered_io(). 1053 * 1054 * So we must continue until @start is increased to 1055 * skip current ordered extent. 1056 */ 1057 if (ret) 1058 btrfs_drop_extent_cache(BTRFS_I(inode), start, 1059 start + ram_size - 1, 0); 1060 } 1061 1062 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1063 1064 /* we're not doing compressed IO, don't unlock the first 1065 * page (which the caller expects to stay locked), don't 1066 * clear any dirty bits and don't set any writeback bits 1067 * 1068 * Do set the Private2 bit so we know this page was properly 1069 * setup for writepage 1070 */ 1071 page_ops = unlock ? PAGE_UNLOCK : 0; 1072 page_ops |= PAGE_SET_PRIVATE2; 1073 1074 extent_clear_unlock_delalloc(inode, start, 1075 start + ram_size - 1, 1076 delalloc_end, locked_page, 1077 EXTENT_LOCKED | EXTENT_DELALLOC, 1078 page_ops); 1079 if (num_bytes < cur_alloc_size) 1080 num_bytes = 0; 1081 else 1082 num_bytes -= cur_alloc_size; 1083 alloc_hint = ins.objectid + ins.offset; 1084 start += cur_alloc_size; 1085 extent_reserved = false; 1086 1087 /* 1088 * btrfs_reloc_clone_csums() error, since start is increased 1089 * extent_clear_unlock_delalloc() at out_unlock label won't 1090 * free metadata of current ordered extent, we're OK to exit. 1091 */ 1092 if (ret) 1093 goto out_unlock; 1094 } 1095 out: 1096 return ret; 1097 1098 out_drop_extent_cache: 1099 btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0); 1100 out_reserve: 1101 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1102 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1103 out_unlock: 1104 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 1105 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; 1106 page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 1107 PAGE_END_WRITEBACK; 1108 /* 1109 * If we reserved an extent for our delalloc range (or a subrange) and 1110 * failed to create the respective ordered extent, then it means that 1111 * when we reserved the extent we decremented the extent's size from 1112 * the data space_info's bytes_may_use counter and incremented the 1113 * space_info's bytes_reserved counter by the same amount. We must make 1114 * sure extent_clear_unlock_delalloc() does not try to decrement again 1115 * the data space_info's bytes_may_use counter, therefore we do not pass 1116 * it the flag EXTENT_CLEAR_DATA_RESV. 1117 */ 1118 if (extent_reserved) { 1119 extent_clear_unlock_delalloc(inode, start, 1120 start + cur_alloc_size, 1121 start + cur_alloc_size, 1122 locked_page, 1123 clear_bits, 1124 page_ops); 1125 start += cur_alloc_size; 1126 if (start >= end) 1127 goto out; 1128 } 1129 extent_clear_unlock_delalloc(inode, start, end, delalloc_end, 1130 locked_page, 1131 clear_bits | EXTENT_CLEAR_DATA_RESV, 1132 page_ops); 1133 goto out; 1134 } 1135 1136 /* 1137 * work queue call back to started compression on a file and pages 1138 */ 1139 static noinline void async_cow_start(struct btrfs_work *work) 1140 { 1141 struct async_cow *async_cow; 1142 int num_added = 0; 1143 async_cow = container_of(work, struct async_cow, work); 1144 1145 compress_file_range(async_cow->inode, async_cow->locked_page, 1146 async_cow->start, async_cow->end, async_cow, 1147 &num_added); 1148 if (num_added == 0) { 1149 btrfs_add_delayed_iput(async_cow->inode); 1150 async_cow->inode = NULL; 1151 } 1152 } 1153 1154 /* 1155 * work queue call back to submit previously compressed pages 1156 */ 1157 static noinline void async_cow_submit(struct btrfs_work *work) 1158 { 1159 struct btrfs_fs_info *fs_info; 1160 struct async_cow *async_cow; 1161 struct btrfs_root *root; 1162 unsigned long nr_pages; 1163 1164 async_cow = container_of(work, struct async_cow, work); 1165 1166 root = async_cow->root; 1167 fs_info = root->fs_info; 1168 nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >> 1169 PAGE_SHIFT; 1170 1171 /* 1172 * atomic_sub_return implies a barrier for waitqueue_active 1173 */ 1174 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1175 5 * SZ_1M && 1176 waitqueue_active(&fs_info->async_submit_wait)) 1177 wake_up(&fs_info->async_submit_wait); 1178 1179 if (async_cow->inode) 1180 submit_compressed_extents(async_cow->inode, async_cow); 1181 } 1182 1183 static noinline void async_cow_free(struct btrfs_work *work) 1184 { 1185 struct async_cow *async_cow; 1186 async_cow = container_of(work, struct async_cow, work); 1187 if (async_cow->inode) 1188 btrfs_add_delayed_iput(async_cow->inode); 1189 kfree(async_cow); 1190 } 1191 1192 static int cow_file_range_async(struct inode *inode, struct page *locked_page, 1193 u64 start, u64 end, int *page_started, 1194 unsigned long *nr_written, 1195 unsigned int write_flags) 1196 { 1197 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1198 struct async_cow *async_cow; 1199 struct btrfs_root *root = BTRFS_I(inode)->root; 1200 unsigned long nr_pages; 1201 u64 cur_end; 1202 1203 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED, 1204 1, 0, NULL); 1205 while (start < end) { 1206 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS); 1207 BUG_ON(!async_cow); /* -ENOMEM */ 1208 async_cow->inode = igrab(inode); 1209 async_cow->root = root; 1210 async_cow->locked_page = locked_page; 1211 async_cow->start = start; 1212 async_cow->write_flags = write_flags; 1213 1214 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS && 1215 !btrfs_test_opt(fs_info, FORCE_COMPRESS)) 1216 cur_end = end; 1217 else 1218 cur_end = min(end, start + SZ_512K - 1); 1219 1220 async_cow->end = cur_end; 1221 INIT_LIST_HEAD(&async_cow->extents); 1222 1223 btrfs_init_work(&async_cow->work, 1224 btrfs_delalloc_helper, 1225 async_cow_start, async_cow_submit, 1226 async_cow_free); 1227 1228 nr_pages = (cur_end - start + PAGE_SIZE) >> 1229 PAGE_SHIFT; 1230 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1231 1232 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work); 1233 1234 *nr_written += nr_pages; 1235 start = cur_end + 1; 1236 } 1237 *page_started = 1; 1238 return 0; 1239 } 1240 1241 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, 1242 u64 bytenr, u64 num_bytes) 1243 { 1244 int ret; 1245 struct btrfs_ordered_sum *sums; 1246 LIST_HEAD(list); 1247 1248 ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr, 1249 bytenr + num_bytes - 1, &list, 0); 1250 if (ret == 0 && list_empty(&list)) 1251 return 0; 1252 1253 while (!list_empty(&list)) { 1254 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1255 list_del(&sums->list); 1256 kfree(sums); 1257 } 1258 if (ret < 0) 1259 return ret; 1260 return 1; 1261 } 1262 1263 /* 1264 * when nowcow writeback call back. This checks for snapshots or COW copies 1265 * of the extents that exist in the file, and COWs the file as required. 1266 * 1267 * If no cow copies or snapshots exist, we write directly to the existing 1268 * blocks on disk 1269 */ 1270 static noinline int run_delalloc_nocow(struct inode *inode, 1271 struct page *locked_page, 1272 u64 start, u64 end, int *page_started, int force, 1273 unsigned long *nr_written) 1274 { 1275 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1276 struct btrfs_root *root = BTRFS_I(inode)->root; 1277 struct extent_buffer *leaf; 1278 struct btrfs_path *path; 1279 struct btrfs_file_extent_item *fi; 1280 struct btrfs_key found_key; 1281 struct extent_map *em; 1282 u64 cow_start; 1283 u64 cur_offset; 1284 u64 extent_end; 1285 u64 extent_offset; 1286 u64 disk_bytenr; 1287 u64 num_bytes; 1288 u64 disk_num_bytes; 1289 u64 ram_bytes; 1290 int extent_type; 1291 int ret, err; 1292 int type; 1293 int nocow; 1294 int check_prev = 1; 1295 bool nolock; 1296 u64 ino = btrfs_ino(BTRFS_I(inode)); 1297 1298 path = btrfs_alloc_path(); 1299 if (!path) { 1300 extent_clear_unlock_delalloc(inode, start, end, end, 1301 locked_page, 1302 EXTENT_LOCKED | EXTENT_DELALLOC | 1303 EXTENT_DO_ACCOUNTING | 1304 EXTENT_DEFRAG, PAGE_UNLOCK | 1305 PAGE_CLEAR_DIRTY | 1306 PAGE_SET_WRITEBACK | 1307 PAGE_END_WRITEBACK); 1308 return -ENOMEM; 1309 } 1310 1311 nolock = btrfs_is_free_space_inode(BTRFS_I(inode)); 1312 1313 cow_start = (u64)-1; 1314 cur_offset = start; 1315 while (1) { 1316 ret = btrfs_lookup_file_extent(NULL, root, path, ino, 1317 cur_offset, 0); 1318 if (ret < 0) 1319 goto error; 1320 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1321 leaf = path->nodes[0]; 1322 btrfs_item_key_to_cpu(leaf, &found_key, 1323 path->slots[0] - 1); 1324 if (found_key.objectid == ino && 1325 found_key.type == BTRFS_EXTENT_DATA_KEY) 1326 path->slots[0]--; 1327 } 1328 check_prev = 0; 1329 next_slot: 1330 leaf = path->nodes[0]; 1331 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1332 ret = btrfs_next_leaf(root, path); 1333 if (ret < 0) { 1334 if (cow_start != (u64)-1) 1335 cur_offset = cow_start; 1336 goto error; 1337 } 1338 if (ret > 0) 1339 break; 1340 leaf = path->nodes[0]; 1341 } 1342 1343 nocow = 0; 1344 disk_bytenr = 0; 1345 num_bytes = 0; 1346 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1347 1348 if (found_key.objectid > ino) 1349 break; 1350 if (WARN_ON_ONCE(found_key.objectid < ino) || 1351 found_key.type < BTRFS_EXTENT_DATA_KEY) { 1352 path->slots[0]++; 1353 goto next_slot; 1354 } 1355 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 1356 found_key.offset > end) 1357 break; 1358 1359 if (found_key.offset > cur_offset) { 1360 extent_end = found_key.offset; 1361 extent_type = 0; 1362 goto out_check; 1363 } 1364 1365 fi = btrfs_item_ptr(leaf, path->slots[0], 1366 struct btrfs_file_extent_item); 1367 extent_type = btrfs_file_extent_type(leaf, fi); 1368 1369 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1370 if (extent_type == BTRFS_FILE_EXTENT_REG || 1371 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1372 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1373 extent_offset = btrfs_file_extent_offset(leaf, fi); 1374 extent_end = found_key.offset + 1375 btrfs_file_extent_num_bytes(leaf, fi); 1376 disk_num_bytes = 1377 btrfs_file_extent_disk_num_bytes(leaf, fi); 1378 if (extent_end <= start) { 1379 path->slots[0]++; 1380 goto next_slot; 1381 } 1382 if (disk_bytenr == 0) 1383 goto out_check; 1384 if (btrfs_file_extent_compression(leaf, fi) || 1385 btrfs_file_extent_encryption(leaf, fi) || 1386 btrfs_file_extent_other_encoding(leaf, fi)) 1387 goto out_check; 1388 if (extent_type == BTRFS_FILE_EXTENT_REG && !force) 1389 goto out_check; 1390 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 1391 goto out_check; 1392 ret = btrfs_cross_ref_exist(root, ino, 1393 found_key.offset - 1394 extent_offset, disk_bytenr); 1395 if (ret) { 1396 /* 1397 * ret could be -EIO if the above fails to read 1398 * metadata. 1399 */ 1400 if (ret < 0) { 1401 if (cow_start != (u64)-1) 1402 cur_offset = cow_start; 1403 goto error; 1404 } 1405 1406 WARN_ON_ONCE(nolock); 1407 goto out_check; 1408 } 1409 disk_bytenr += extent_offset; 1410 disk_bytenr += cur_offset - found_key.offset; 1411 num_bytes = min(end + 1, extent_end) - cur_offset; 1412 /* 1413 * if there are pending snapshots for this root, 1414 * we fall into common COW way. 1415 */ 1416 if (!nolock) { 1417 err = btrfs_start_write_no_snapshotting(root); 1418 if (!err) 1419 goto out_check; 1420 } 1421 /* 1422 * force cow if csum exists in the range. 1423 * this ensure that csum for a given extent are 1424 * either valid or do not exist. 1425 */ 1426 ret = csum_exist_in_range(fs_info, disk_bytenr, 1427 num_bytes); 1428 if (ret) { 1429 if (!nolock) 1430 btrfs_end_write_no_snapshotting(root); 1431 1432 /* 1433 * ret could be -EIO if the above fails to read 1434 * metadata. 1435 */ 1436 if (ret < 0) { 1437 if (cow_start != (u64)-1) 1438 cur_offset = cow_start; 1439 goto error; 1440 } 1441 WARN_ON_ONCE(nolock); 1442 goto out_check; 1443 } 1444 if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) { 1445 if (!nolock) 1446 btrfs_end_write_no_snapshotting(root); 1447 goto out_check; 1448 } 1449 nocow = 1; 1450 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 1451 extent_end = found_key.offset + 1452 btrfs_file_extent_inline_len(leaf, 1453 path->slots[0], fi); 1454 extent_end = ALIGN(extent_end, 1455 fs_info->sectorsize); 1456 } else { 1457 BUG_ON(1); 1458 } 1459 out_check: 1460 if (extent_end <= start) { 1461 path->slots[0]++; 1462 if (!nolock && nocow) 1463 btrfs_end_write_no_snapshotting(root); 1464 if (nocow) 1465 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1466 goto next_slot; 1467 } 1468 if (!nocow) { 1469 if (cow_start == (u64)-1) 1470 cow_start = cur_offset; 1471 cur_offset = extent_end; 1472 if (cur_offset > end) 1473 break; 1474 path->slots[0]++; 1475 goto next_slot; 1476 } 1477 1478 btrfs_release_path(path); 1479 if (cow_start != (u64)-1) { 1480 ret = cow_file_range(inode, locked_page, 1481 cow_start, found_key.offset - 1, 1482 end, page_started, nr_written, 1, 1483 NULL); 1484 if (ret) { 1485 if (!nolock && nocow) 1486 btrfs_end_write_no_snapshotting(root); 1487 if (nocow) 1488 btrfs_dec_nocow_writers(fs_info, 1489 disk_bytenr); 1490 goto error; 1491 } 1492 cow_start = (u64)-1; 1493 } 1494 1495 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1496 u64 orig_start = found_key.offset - extent_offset; 1497 1498 em = create_io_em(inode, cur_offset, num_bytes, 1499 orig_start, 1500 disk_bytenr, /* block_start */ 1501 num_bytes, /* block_len */ 1502 disk_num_bytes, /* orig_block_len */ 1503 ram_bytes, BTRFS_COMPRESS_NONE, 1504 BTRFS_ORDERED_PREALLOC); 1505 if (IS_ERR(em)) { 1506 if (!nolock && nocow) 1507 btrfs_end_write_no_snapshotting(root); 1508 if (nocow) 1509 btrfs_dec_nocow_writers(fs_info, 1510 disk_bytenr); 1511 ret = PTR_ERR(em); 1512 goto error; 1513 } 1514 free_extent_map(em); 1515 } 1516 1517 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1518 type = BTRFS_ORDERED_PREALLOC; 1519 } else { 1520 type = BTRFS_ORDERED_NOCOW; 1521 } 1522 1523 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr, 1524 num_bytes, num_bytes, type); 1525 if (nocow) 1526 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1527 BUG_ON(ret); /* -ENOMEM */ 1528 1529 if (root->root_key.objectid == 1530 BTRFS_DATA_RELOC_TREE_OBJECTID) 1531 /* 1532 * Error handled later, as we must prevent 1533 * extent_clear_unlock_delalloc() in error handler 1534 * from freeing metadata of created ordered extent. 1535 */ 1536 ret = btrfs_reloc_clone_csums(inode, cur_offset, 1537 num_bytes); 1538 1539 extent_clear_unlock_delalloc(inode, cur_offset, 1540 cur_offset + num_bytes - 1, end, 1541 locked_page, EXTENT_LOCKED | 1542 EXTENT_DELALLOC | 1543 EXTENT_CLEAR_DATA_RESV, 1544 PAGE_UNLOCK | PAGE_SET_PRIVATE2); 1545 1546 if (!nolock && nocow) 1547 btrfs_end_write_no_snapshotting(root); 1548 cur_offset = extent_end; 1549 1550 /* 1551 * btrfs_reloc_clone_csums() error, now we're OK to call error 1552 * handler, as metadata for created ordered extent will only 1553 * be freed by btrfs_finish_ordered_io(). 1554 */ 1555 if (ret) 1556 goto error; 1557 if (cur_offset > end) 1558 break; 1559 } 1560 btrfs_release_path(path); 1561 1562 if (cur_offset <= end && cow_start == (u64)-1) { 1563 cow_start = cur_offset; 1564 cur_offset = end; 1565 } 1566 1567 if (cow_start != (u64)-1) { 1568 ret = cow_file_range(inode, locked_page, cow_start, end, end, 1569 page_started, nr_written, 1, NULL); 1570 if (ret) 1571 goto error; 1572 } 1573 1574 error: 1575 if (ret && cur_offset < end) 1576 extent_clear_unlock_delalloc(inode, cur_offset, end, end, 1577 locked_page, EXTENT_LOCKED | 1578 EXTENT_DELALLOC | EXTENT_DEFRAG | 1579 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1580 PAGE_CLEAR_DIRTY | 1581 PAGE_SET_WRITEBACK | 1582 PAGE_END_WRITEBACK); 1583 btrfs_free_path(path); 1584 return ret; 1585 } 1586 1587 static inline int need_force_cow(struct inode *inode, u64 start, u64 end) 1588 { 1589 1590 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 1591 !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC)) 1592 return 0; 1593 1594 /* 1595 * @defrag_bytes is a hint value, no spinlock held here, 1596 * if is not zero, it means the file is defragging. 1597 * Force cow if given extent needs to be defragged. 1598 */ 1599 if (BTRFS_I(inode)->defrag_bytes && 1600 test_range_bit(&BTRFS_I(inode)->io_tree, start, end, 1601 EXTENT_DEFRAG, 0, NULL)) 1602 return 1; 1603 1604 return 0; 1605 } 1606 1607 /* 1608 * extent_io.c call back to do delayed allocation processing 1609 */ 1610 static int run_delalloc_range(void *private_data, struct page *locked_page, 1611 u64 start, u64 end, int *page_started, 1612 unsigned long *nr_written, 1613 struct writeback_control *wbc) 1614 { 1615 struct inode *inode = private_data; 1616 int ret; 1617 int force_cow = need_force_cow(inode, start, end); 1618 unsigned int write_flags = wbc_to_write_flags(wbc); 1619 1620 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) { 1621 ret = run_delalloc_nocow(inode, locked_page, start, end, 1622 page_started, 1, nr_written); 1623 } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) { 1624 ret = run_delalloc_nocow(inode, locked_page, start, end, 1625 page_started, 0, nr_written); 1626 } else if (!inode_need_compress(inode, start, end)) { 1627 ret = cow_file_range(inode, locked_page, start, end, end, 1628 page_started, nr_written, 1, NULL); 1629 } else { 1630 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 1631 &BTRFS_I(inode)->runtime_flags); 1632 ret = cow_file_range_async(inode, locked_page, start, end, 1633 page_started, nr_written, 1634 write_flags); 1635 } 1636 if (ret) 1637 btrfs_cleanup_ordered_extents(inode, start, end - start + 1); 1638 return ret; 1639 } 1640 1641 static void btrfs_split_extent_hook(void *private_data, 1642 struct extent_state *orig, u64 split) 1643 { 1644 struct inode *inode = private_data; 1645 u64 size; 1646 1647 /* not delalloc, ignore it */ 1648 if (!(orig->state & EXTENT_DELALLOC)) 1649 return; 1650 1651 size = orig->end - orig->start + 1; 1652 if (size > BTRFS_MAX_EXTENT_SIZE) { 1653 u32 num_extents; 1654 u64 new_size; 1655 1656 /* 1657 * See the explanation in btrfs_merge_extent_hook, the same 1658 * applies here, just in reverse. 1659 */ 1660 new_size = orig->end - split + 1; 1661 num_extents = count_max_extents(new_size); 1662 new_size = split - orig->start; 1663 num_extents += count_max_extents(new_size); 1664 if (count_max_extents(size) >= num_extents) 1665 return; 1666 } 1667 1668 spin_lock(&BTRFS_I(inode)->lock); 1669 btrfs_mod_outstanding_extents(BTRFS_I(inode), 1); 1670 spin_unlock(&BTRFS_I(inode)->lock); 1671 } 1672 1673 /* 1674 * extent_io.c merge_extent_hook, used to track merged delayed allocation 1675 * extents so we can keep track of new extents that are just merged onto old 1676 * extents, such as when we are doing sequential writes, so we can properly 1677 * account for the metadata space we'll need. 1678 */ 1679 static void btrfs_merge_extent_hook(void *private_data, 1680 struct extent_state *new, 1681 struct extent_state *other) 1682 { 1683 struct inode *inode = private_data; 1684 u64 new_size, old_size; 1685 u32 num_extents; 1686 1687 /* not delalloc, ignore it */ 1688 if (!(other->state & EXTENT_DELALLOC)) 1689 return; 1690 1691 if (new->start > other->start) 1692 new_size = new->end - other->start + 1; 1693 else 1694 new_size = other->end - new->start + 1; 1695 1696 /* we're not bigger than the max, unreserve the space and go */ 1697 if (new_size <= BTRFS_MAX_EXTENT_SIZE) { 1698 spin_lock(&BTRFS_I(inode)->lock); 1699 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1700 spin_unlock(&BTRFS_I(inode)->lock); 1701 return; 1702 } 1703 1704 /* 1705 * We have to add up either side to figure out how many extents were 1706 * accounted for before we merged into one big extent. If the number of 1707 * extents we accounted for is <= the amount we need for the new range 1708 * then we can return, otherwise drop. Think of it like this 1709 * 1710 * [ 4k][MAX_SIZE] 1711 * 1712 * So we've grown the extent by a MAX_SIZE extent, this would mean we 1713 * need 2 outstanding extents, on one side we have 1 and the other side 1714 * we have 1 so they are == and we can return. But in this case 1715 * 1716 * [MAX_SIZE+4k][MAX_SIZE+4k] 1717 * 1718 * Each range on their own accounts for 2 extents, but merged together 1719 * they are only 3 extents worth of accounting, so we need to drop in 1720 * this case. 1721 */ 1722 old_size = other->end - other->start + 1; 1723 num_extents = count_max_extents(old_size); 1724 old_size = new->end - new->start + 1; 1725 num_extents += count_max_extents(old_size); 1726 if (count_max_extents(new_size) >= num_extents) 1727 return; 1728 1729 spin_lock(&BTRFS_I(inode)->lock); 1730 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1731 spin_unlock(&BTRFS_I(inode)->lock); 1732 } 1733 1734 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 1735 struct inode *inode) 1736 { 1737 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1738 1739 spin_lock(&root->delalloc_lock); 1740 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1741 list_add_tail(&BTRFS_I(inode)->delalloc_inodes, 1742 &root->delalloc_inodes); 1743 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1744 &BTRFS_I(inode)->runtime_flags); 1745 root->nr_delalloc_inodes++; 1746 if (root->nr_delalloc_inodes == 1) { 1747 spin_lock(&fs_info->delalloc_root_lock); 1748 BUG_ON(!list_empty(&root->delalloc_root)); 1749 list_add_tail(&root->delalloc_root, 1750 &fs_info->delalloc_roots); 1751 spin_unlock(&fs_info->delalloc_root_lock); 1752 } 1753 } 1754 spin_unlock(&root->delalloc_lock); 1755 } 1756 1757 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 1758 struct btrfs_inode *inode) 1759 { 1760 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 1761 1762 spin_lock(&root->delalloc_lock); 1763 if (!list_empty(&inode->delalloc_inodes)) { 1764 list_del_init(&inode->delalloc_inodes); 1765 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1766 &inode->runtime_flags); 1767 root->nr_delalloc_inodes--; 1768 if (!root->nr_delalloc_inodes) { 1769 spin_lock(&fs_info->delalloc_root_lock); 1770 BUG_ON(list_empty(&root->delalloc_root)); 1771 list_del_init(&root->delalloc_root); 1772 spin_unlock(&fs_info->delalloc_root_lock); 1773 } 1774 } 1775 spin_unlock(&root->delalloc_lock); 1776 } 1777 1778 /* 1779 * extent_io.c set_bit_hook, used to track delayed allocation 1780 * bytes in this file, and to maintain the list of inodes that 1781 * have pending delalloc work to be done. 1782 */ 1783 static void btrfs_set_bit_hook(void *private_data, 1784 struct extent_state *state, unsigned *bits) 1785 { 1786 struct inode *inode = private_data; 1787 1788 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1789 1790 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC)) 1791 WARN_ON(1); 1792 /* 1793 * set_bit and clear bit hooks normally require _irqsave/restore 1794 * but in this case, we are only testing for the DELALLOC 1795 * bit, which is only set or cleared with irqs on 1796 */ 1797 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1798 struct btrfs_root *root = BTRFS_I(inode)->root; 1799 u64 len = state->end + 1 - state->start; 1800 u32 num_extents = count_max_extents(len); 1801 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode)); 1802 1803 spin_lock(&BTRFS_I(inode)->lock); 1804 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents); 1805 spin_unlock(&BTRFS_I(inode)->lock); 1806 1807 /* For sanity tests */ 1808 if (btrfs_is_testing(fs_info)) 1809 return; 1810 1811 percpu_counter_add_batch(&fs_info->delalloc_bytes, len, 1812 fs_info->delalloc_batch); 1813 spin_lock(&BTRFS_I(inode)->lock); 1814 BTRFS_I(inode)->delalloc_bytes += len; 1815 if (*bits & EXTENT_DEFRAG) 1816 BTRFS_I(inode)->defrag_bytes += len; 1817 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1818 &BTRFS_I(inode)->runtime_flags)) 1819 btrfs_add_delalloc_inodes(root, inode); 1820 spin_unlock(&BTRFS_I(inode)->lock); 1821 } 1822 1823 if (!(state->state & EXTENT_DELALLOC_NEW) && 1824 (*bits & EXTENT_DELALLOC_NEW)) { 1825 spin_lock(&BTRFS_I(inode)->lock); 1826 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 - 1827 state->start; 1828 spin_unlock(&BTRFS_I(inode)->lock); 1829 } 1830 } 1831 1832 /* 1833 * extent_io.c clear_bit_hook, see set_bit_hook for why 1834 */ 1835 static void btrfs_clear_bit_hook(void *private_data, 1836 struct extent_state *state, 1837 unsigned *bits) 1838 { 1839 struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data); 1840 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 1841 u64 len = state->end + 1 - state->start; 1842 u32 num_extents = count_max_extents(len); 1843 1844 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) { 1845 spin_lock(&inode->lock); 1846 inode->defrag_bytes -= len; 1847 spin_unlock(&inode->lock); 1848 } 1849 1850 /* 1851 * set_bit and clear bit hooks normally require _irqsave/restore 1852 * but in this case, we are only testing for the DELALLOC 1853 * bit, which is only set or cleared with irqs on 1854 */ 1855 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1856 struct btrfs_root *root = inode->root; 1857 bool do_list = !btrfs_is_free_space_inode(inode); 1858 1859 spin_lock(&inode->lock); 1860 btrfs_mod_outstanding_extents(inode, -num_extents); 1861 spin_unlock(&inode->lock); 1862 1863 /* 1864 * We don't reserve metadata space for space cache inodes so we 1865 * don't need to call dellalloc_release_metadata if there is an 1866 * error. 1867 */ 1868 if (*bits & EXTENT_CLEAR_META_RESV && 1869 root != fs_info->tree_root) 1870 btrfs_delalloc_release_metadata(inode, len, false); 1871 1872 /* For sanity tests. */ 1873 if (btrfs_is_testing(fs_info)) 1874 return; 1875 1876 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID && 1877 do_list && !(state->state & EXTENT_NORESERVE) && 1878 (*bits & EXTENT_CLEAR_DATA_RESV)) 1879 btrfs_free_reserved_data_space_noquota( 1880 &inode->vfs_inode, 1881 state->start, len); 1882 1883 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, 1884 fs_info->delalloc_batch); 1885 spin_lock(&inode->lock); 1886 inode->delalloc_bytes -= len; 1887 if (do_list && inode->delalloc_bytes == 0 && 1888 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1889 &inode->runtime_flags)) 1890 btrfs_del_delalloc_inode(root, inode); 1891 spin_unlock(&inode->lock); 1892 } 1893 1894 if ((state->state & EXTENT_DELALLOC_NEW) && 1895 (*bits & EXTENT_DELALLOC_NEW)) { 1896 spin_lock(&inode->lock); 1897 ASSERT(inode->new_delalloc_bytes >= len); 1898 inode->new_delalloc_bytes -= len; 1899 spin_unlock(&inode->lock); 1900 } 1901 } 1902 1903 /* 1904 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure 1905 * we don't create bios that span stripes or chunks 1906 * 1907 * return 1 if page cannot be merged to bio 1908 * return 0 if page can be merged to bio 1909 * return error otherwise 1910 */ 1911 int btrfs_merge_bio_hook(struct page *page, unsigned long offset, 1912 size_t size, struct bio *bio, 1913 unsigned long bio_flags) 1914 { 1915 struct inode *inode = page->mapping->host; 1916 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1917 u64 logical = (u64)bio->bi_iter.bi_sector << 9; 1918 u64 length = 0; 1919 u64 map_length; 1920 int ret; 1921 1922 if (bio_flags & EXTENT_BIO_COMPRESSED) 1923 return 0; 1924 1925 length = bio->bi_iter.bi_size; 1926 map_length = length; 1927 ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, 1928 NULL, 0); 1929 if (ret < 0) 1930 return ret; 1931 if (map_length < length + size) 1932 return 1; 1933 return 0; 1934 } 1935 1936 /* 1937 * in order to insert checksums into the metadata in large chunks, 1938 * we wait until bio submission time. All the pages in the bio are 1939 * checksummed and sums are attached onto the ordered extent record. 1940 * 1941 * At IO completion time the cums attached on the ordered extent record 1942 * are inserted into the btree 1943 */ 1944 static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio, 1945 u64 bio_offset) 1946 { 1947 struct inode *inode = private_data; 1948 blk_status_t ret = 0; 1949 1950 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 1951 BUG_ON(ret); /* -ENOMEM */ 1952 return 0; 1953 } 1954 1955 /* 1956 * in order to insert checksums into the metadata in large chunks, 1957 * we wait until bio submission time. All the pages in the bio are 1958 * checksummed and sums are attached onto the ordered extent record. 1959 * 1960 * At IO completion time the cums attached on the ordered extent record 1961 * are inserted into the btree 1962 */ 1963 static blk_status_t btrfs_submit_bio_done(void *private_data, struct bio *bio, 1964 int mirror_num) 1965 { 1966 struct inode *inode = private_data; 1967 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1968 blk_status_t ret; 1969 1970 ret = btrfs_map_bio(fs_info, bio, mirror_num, 1); 1971 if (ret) { 1972 bio->bi_status = ret; 1973 bio_endio(bio); 1974 } 1975 return ret; 1976 } 1977 1978 /* 1979 * extent_io.c submission hook. This does the right thing for csum calculation 1980 * on write, or reading the csums from the tree before a read. 1981 * 1982 * Rules about async/sync submit, 1983 * a) read: sync submit 1984 * 1985 * b) write without checksum: sync submit 1986 * 1987 * c) write with checksum: 1988 * c-1) if bio is issued by fsync: sync submit 1989 * (sync_writers != 0) 1990 * 1991 * c-2) if root is reloc root: sync submit 1992 * (only in case of buffered IO) 1993 * 1994 * c-3) otherwise: async submit 1995 */ 1996 static blk_status_t btrfs_submit_bio_hook(void *private_data, struct bio *bio, 1997 int mirror_num, unsigned long bio_flags, 1998 u64 bio_offset) 1999 { 2000 struct inode *inode = private_data; 2001 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2002 struct btrfs_root *root = BTRFS_I(inode)->root; 2003 enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA; 2004 blk_status_t ret = 0; 2005 int skip_sum; 2006 int async = !atomic_read(&BTRFS_I(inode)->sync_writers); 2007 2008 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 2009 2010 if (btrfs_is_free_space_inode(BTRFS_I(inode))) 2011 metadata = BTRFS_WQ_ENDIO_FREE_SPACE; 2012 2013 if (bio_op(bio) != REQ_OP_WRITE) { 2014 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); 2015 if (ret) 2016 goto out; 2017 2018 if (bio_flags & EXTENT_BIO_COMPRESSED) { 2019 ret = btrfs_submit_compressed_read(inode, bio, 2020 mirror_num, 2021 bio_flags); 2022 goto out; 2023 } else if (!skip_sum) { 2024 ret = btrfs_lookup_bio_sums(inode, bio, NULL); 2025 if (ret) 2026 goto out; 2027 } 2028 goto mapit; 2029 } else if (async && !skip_sum) { 2030 /* csum items have already been cloned */ 2031 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 2032 goto mapit; 2033 /* we're doing a write, do the async checksumming */ 2034 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags, 2035 bio_offset, inode, 2036 btrfs_submit_bio_start, 2037 btrfs_submit_bio_done); 2038 goto out; 2039 } else if (!skip_sum) { 2040 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 2041 if (ret) 2042 goto out; 2043 } 2044 2045 mapit: 2046 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 2047 2048 out: 2049 if (ret) { 2050 bio->bi_status = ret; 2051 bio_endio(bio); 2052 } 2053 return ret; 2054 } 2055 2056 /* 2057 * given a list of ordered sums record them in the inode. This happens 2058 * at IO completion time based on sums calculated at bio submission time. 2059 */ 2060 static noinline int add_pending_csums(struct btrfs_trans_handle *trans, 2061 struct inode *inode, struct list_head *list) 2062 { 2063 struct btrfs_ordered_sum *sum; 2064 int ret; 2065 2066 list_for_each_entry(sum, list, list) { 2067 trans->adding_csums = true; 2068 ret = btrfs_csum_file_blocks(trans, 2069 BTRFS_I(inode)->root->fs_info->csum_root, sum); 2070 trans->adding_csums = false; 2071 if (ret) 2072 return ret; 2073 } 2074 return 0; 2075 } 2076 2077 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end, 2078 unsigned int extra_bits, 2079 struct extent_state **cached_state, int dedupe) 2080 { 2081 WARN_ON((end & (PAGE_SIZE - 1)) == 0); 2082 return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end, 2083 extra_bits, cached_state); 2084 } 2085 2086 /* see btrfs_writepage_start_hook for details on why this is required */ 2087 struct btrfs_writepage_fixup { 2088 struct page *page; 2089 struct btrfs_work work; 2090 }; 2091 2092 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2093 { 2094 struct btrfs_writepage_fixup *fixup; 2095 struct btrfs_ordered_extent *ordered; 2096 struct extent_state *cached_state = NULL; 2097 struct extent_changeset *data_reserved = NULL; 2098 struct page *page; 2099 struct inode *inode; 2100 u64 page_start; 2101 u64 page_end; 2102 int ret; 2103 2104 fixup = container_of(work, struct btrfs_writepage_fixup, work); 2105 page = fixup->page; 2106 again: 2107 lock_page(page); 2108 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 2109 ClearPageChecked(page); 2110 goto out_page; 2111 } 2112 2113 inode = page->mapping->host; 2114 page_start = page_offset(page); 2115 page_end = page_offset(page) + PAGE_SIZE - 1; 2116 2117 lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end, 2118 &cached_state); 2119 2120 /* already ordered? We're done */ 2121 if (PagePrivate2(page)) 2122 goto out; 2123 2124 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 2125 PAGE_SIZE); 2126 if (ordered) { 2127 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, 2128 page_end, &cached_state); 2129 unlock_page(page); 2130 btrfs_start_ordered_extent(inode, ordered, 1); 2131 btrfs_put_ordered_extent(ordered); 2132 goto again; 2133 } 2134 2135 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 2136 PAGE_SIZE); 2137 if (ret) { 2138 mapping_set_error(page->mapping, ret); 2139 end_extent_writepage(page, ret, page_start, page_end); 2140 ClearPageChecked(page); 2141 goto out; 2142 } 2143 2144 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, 2145 &cached_state, 0); 2146 if (ret) { 2147 mapping_set_error(page->mapping, ret); 2148 end_extent_writepage(page, ret, page_start, page_end); 2149 ClearPageChecked(page); 2150 goto out; 2151 } 2152 2153 ClearPageChecked(page); 2154 set_page_dirty(page); 2155 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, false); 2156 out: 2157 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end, 2158 &cached_state); 2159 out_page: 2160 unlock_page(page); 2161 put_page(page); 2162 kfree(fixup); 2163 extent_changeset_free(data_reserved); 2164 } 2165 2166 /* 2167 * There are a few paths in the higher layers of the kernel that directly 2168 * set the page dirty bit without asking the filesystem if it is a 2169 * good idea. This causes problems because we want to make sure COW 2170 * properly happens and the data=ordered rules are followed. 2171 * 2172 * In our case any range that doesn't have the ORDERED bit set 2173 * hasn't been properly setup for IO. We kick off an async process 2174 * to fix it up. The async helper will wait for ordered extents, set 2175 * the delalloc bit and make it safe to write the page. 2176 */ 2177 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end) 2178 { 2179 struct inode *inode = page->mapping->host; 2180 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2181 struct btrfs_writepage_fixup *fixup; 2182 2183 /* this page is properly in the ordered list */ 2184 if (TestClearPagePrivate2(page)) 2185 return 0; 2186 2187 if (PageChecked(page)) 2188 return -EAGAIN; 2189 2190 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2191 if (!fixup) 2192 return -EAGAIN; 2193 2194 SetPageChecked(page); 2195 get_page(page); 2196 btrfs_init_work(&fixup->work, btrfs_fixup_helper, 2197 btrfs_writepage_fixup_worker, NULL, NULL); 2198 fixup->page = page; 2199 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2200 return -EBUSY; 2201 } 2202 2203 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2204 struct inode *inode, u64 file_pos, 2205 u64 disk_bytenr, u64 disk_num_bytes, 2206 u64 num_bytes, u64 ram_bytes, 2207 u8 compression, u8 encryption, 2208 u16 other_encoding, int extent_type) 2209 { 2210 struct btrfs_root *root = BTRFS_I(inode)->root; 2211 struct btrfs_file_extent_item *fi; 2212 struct btrfs_path *path; 2213 struct extent_buffer *leaf; 2214 struct btrfs_key ins; 2215 u64 qg_released; 2216 int extent_inserted = 0; 2217 int ret; 2218 2219 path = btrfs_alloc_path(); 2220 if (!path) 2221 return -ENOMEM; 2222 2223 /* 2224 * we may be replacing one extent in the tree with another. 2225 * The new extent is pinned in the extent map, and we don't want 2226 * to drop it from the cache until it is completely in the btree. 2227 * 2228 * So, tell btrfs_drop_extents to leave this extent in the cache. 2229 * the caller is expected to unpin it and allow it to be merged 2230 * with the others. 2231 */ 2232 ret = __btrfs_drop_extents(trans, root, inode, path, file_pos, 2233 file_pos + num_bytes, NULL, 0, 2234 1, sizeof(*fi), &extent_inserted); 2235 if (ret) 2236 goto out; 2237 2238 if (!extent_inserted) { 2239 ins.objectid = btrfs_ino(BTRFS_I(inode)); 2240 ins.offset = file_pos; 2241 ins.type = BTRFS_EXTENT_DATA_KEY; 2242 2243 path->leave_spinning = 1; 2244 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2245 sizeof(*fi)); 2246 if (ret) 2247 goto out; 2248 } 2249 leaf = path->nodes[0]; 2250 fi = btrfs_item_ptr(leaf, path->slots[0], 2251 struct btrfs_file_extent_item); 2252 btrfs_set_file_extent_generation(leaf, fi, trans->transid); 2253 btrfs_set_file_extent_type(leaf, fi, extent_type); 2254 btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr); 2255 btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes); 2256 btrfs_set_file_extent_offset(leaf, fi, 0); 2257 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); 2258 btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes); 2259 btrfs_set_file_extent_compression(leaf, fi, compression); 2260 btrfs_set_file_extent_encryption(leaf, fi, encryption); 2261 btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding); 2262 2263 btrfs_mark_buffer_dirty(leaf); 2264 btrfs_release_path(path); 2265 2266 inode_add_bytes(inode, num_bytes); 2267 2268 ins.objectid = disk_bytenr; 2269 ins.offset = disk_num_bytes; 2270 ins.type = BTRFS_EXTENT_ITEM_KEY; 2271 2272 /* 2273 * Release the reserved range from inode dirty range map, as it is 2274 * already moved into delayed_ref_head 2275 */ 2276 ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes); 2277 if (ret < 0) 2278 goto out; 2279 qg_released = ret; 2280 ret = btrfs_alloc_reserved_file_extent(trans, root, 2281 btrfs_ino(BTRFS_I(inode)), 2282 file_pos, qg_released, &ins); 2283 out: 2284 btrfs_free_path(path); 2285 2286 return ret; 2287 } 2288 2289 /* snapshot-aware defrag */ 2290 struct sa_defrag_extent_backref { 2291 struct rb_node node; 2292 struct old_sa_defrag_extent *old; 2293 u64 root_id; 2294 u64 inum; 2295 u64 file_pos; 2296 u64 extent_offset; 2297 u64 num_bytes; 2298 u64 generation; 2299 }; 2300 2301 struct old_sa_defrag_extent { 2302 struct list_head list; 2303 struct new_sa_defrag_extent *new; 2304 2305 u64 extent_offset; 2306 u64 bytenr; 2307 u64 offset; 2308 u64 len; 2309 int count; 2310 }; 2311 2312 struct new_sa_defrag_extent { 2313 struct rb_root root; 2314 struct list_head head; 2315 struct btrfs_path *path; 2316 struct inode *inode; 2317 u64 file_pos; 2318 u64 len; 2319 u64 bytenr; 2320 u64 disk_len; 2321 u8 compress_type; 2322 }; 2323 2324 static int backref_comp(struct sa_defrag_extent_backref *b1, 2325 struct sa_defrag_extent_backref *b2) 2326 { 2327 if (b1->root_id < b2->root_id) 2328 return -1; 2329 else if (b1->root_id > b2->root_id) 2330 return 1; 2331 2332 if (b1->inum < b2->inum) 2333 return -1; 2334 else if (b1->inum > b2->inum) 2335 return 1; 2336 2337 if (b1->file_pos < b2->file_pos) 2338 return -1; 2339 else if (b1->file_pos > b2->file_pos) 2340 return 1; 2341 2342 /* 2343 * [------------------------------] ===> (a range of space) 2344 * |<--->| |<---->| =============> (fs/file tree A) 2345 * |<---------------------------->| ===> (fs/file tree B) 2346 * 2347 * A range of space can refer to two file extents in one tree while 2348 * refer to only one file extent in another tree. 2349 * 2350 * So we may process a disk offset more than one time(two extents in A) 2351 * and locate at the same extent(one extent in B), then insert two same 2352 * backrefs(both refer to the extent in B). 2353 */ 2354 return 0; 2355 } 2356 2357 static void backref_insert(struct rb_root *root, 2358 struct sa_defrag_extent_backref *backref) 2359 { 2360 struct rb_node **p = &root->rb_node; 2361 struct rb_node *parent = NULL; 2362 struct sa_defrag_extent_backref *entry; 2363 int ret; 2364 2365 while (*p) { 2366 parent = *p; 2367 entry = rb_entry(parent, struct sa_defrag_extent_backref, node); 2368 2369 ret = backref_comp(backref, entry); 2370 if (ret < 0) 2371 p = &(*p)->rb_left; 2372 else 2373 p = &(*p)->rb_right; 2374 } 2375 2376 rb_link_node(&backref->node, parent, p); 2377 rb_insert_color(&backref->node, root); 2378 } 2379 2380 /* 2381 * Note the backref might has changed, and in this case we just return 0. 2382 */ 2383 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id, 2384 void *ctx) 2385 { 2386 struct btrfs_file_extent_item *extent; 2387 struct old_sa_defrag_extent *old = ctx; 2388 struct new_sa_defrag_extent *new = old->new; 2389 struct btrfs_path *path = new->path; 2390 struct btrfs_key key; 2391 struct btrfs_root *root; 2392 struct sa_defrag_extent_backref *backref; 2393 struct extent_buffer *leaf; 2394 struct inode *inode = new->inode; 2395 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2396 int slot; 2397 int ret; 2398 u64 extent_offset; 2399 u64 num_bytes; 2400 2401 if (BTRFS_I(inode)->root->root_key.objectid == root_id && 2402 inum == btrfs_ino(BTRFS_I(inode))) 2403 return 0; 2404 2405 key.objectid = root_id; 2406 key.type = BTRFS_ROOT_ITEM_KEY; 2407 key.offset = (u64)-1; 2408 2409 root = btrfs_read_fs_root_no_name(fs_info, &key); 2410 if (IS_ERR(root)) { 2411 if (PTR_ERR(root) == -ENOENT) 2412 return 0; 2413 WARN_ON(1); 2414 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu", 2415 inum, offset, root_id); 2416 return PTR_ERR(root); 2417 } 2418 2419 key.objectid = inum; 2420 key.type = BTRFS_EXTENT_DATA_KEY; 2421 if (offset > (u64)-1 << 32) 2422 key.offset = 0; 2423 else 2424 key.offset = offset; 2425 2426 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2427 if (WARN_ON(ret < 0)) 2428 return ret; 2429 ret = 0; 2430 2431 while (1) { 2432 cond_resched(); 2433 2434 leaf = path->nodes[0]; 2435 slot = path->slots[0]; 2436 2437 if (slot >= btrfs_header_nritems(leaf)) { 2438 ret = btrfs_next_leaf(root, path); 2439 if (ret < 0) { 2440 goto out; 2441 } else if (ret > 0) { 2442 ret = 0; 2443 goto out; 2444 } 2445 continue; 2446 } 2447 2448 path->slots[0]++; 2449 2450 btrfs_item_key_to_cpu(leaf, &key, slot); 2451 2452 if (key.objectid > inum) 2453 goto out; 2454 2455 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY) 2456 continue; 2457 2458 extent = btrfs_item_ptr(leaf, slot, 2459 struct btrfs_file_extent_item); 2460 2461 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr) 2462 continue; 2463 2464 /* 2465 * 'offset' refers to the exact key.offset, 2466 * NOT the 'offset' field in btrfs_extent_data_ref, ie. 2467 * (key.offset - extent_offset). 2468 */ 2469 if (key.offset != offset) 2470 continue; 2471 2472 extent_offset = btrfs_file_extent_offset(leaf, extent); 2473 num_bytes = btrfs_file_extent_num_bytes(leaf, extent); 2474 2475 if (extent_offset >= old->extent_offset + old->offset + 2476 old->len || extent_offset + num_bytes <= 2477 old->extent_offset + old->offset) 2478 continue; 2479 break; 2480 } 2481 2482 backref = kmalloc(sizeof(*backref), GFP_NOFS); 2483 if (!backref) { 2484 ret = -ENOENT; 2485 goto out; 2486 } 2487 2488 backref->root_id = root_id; 2489 backref->inum = inum; 2490 backref->file_pos = offset; 2491 backref->num_bytes = num_bytes; 2492 backref->extent_offset = extent_offset; 2493 backref->generation = btrfs_file_extent_generation(leaf, extent); 2494 backref->old = old; 2495 backref_insert(&new->root, backref); 2496 old->count++; 2497 out: 2498 btrfs_release_path(path); 2499 WARN_ON(ret); 2500 return ret; 2501 } 2502 2503 static noinline bool record_extent_backrefs(struct btrfs_path *path, 2504 struct new_sa_defrag_extent *new) 2505 { 2506 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2507 struct old_sa_defrag_extent *old, *tmp; 2508 int ret; 2509 2510 new->path = path; 2511 2512 list_for_each_entry_safe(old, tmp, &new->head, list) { 2513 ret = iterate_inodes_from_logical(old->bytenr + 2514 old->extent_offset, fs_info, 2515 path, record_one_backref, 2516 old, false); 2517 if (ret < 0 && ret != -ENOENT) 2518 return false; 2519 2520 /* no backref to be processed for this extent */ 2521 if (!old->count) { 2522 list_del(&old->list); 2523 kfree(old); 2524 } 2525 } 2526 2527 if (list_empty(&new->head)) 2528 return false; 2529 2530 return true; 2531 } 2532 2533 static int relink_is_mergable(struct extent_buffer *leaf, 2534 struct btrfs_file_extent_item *fi, 2535 struct new_sa_defrag_extent *new) 2536 { 2537 if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr) 2538 return 0; 2539 2540 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG) 2541 return 0; 2542 2543 if (btrfs_file_extent_compression(leaf, fi) != new->compress_type) 2544 return 0; 2545 2546 if (btrfs_file_extent_encryption(leaf, fi) || 2547 btrfs_file_extent_other_encoding(leaf, fi)) 2548 return 0; 2549 2550 return 1; 2551 } 2552 2553 /* 2554 * Note the backref might has changed, and in this case we just return 0. 2555 */ 2556 static noinline int relink_extent_backref(struct btrfs_path *path, 2557 struct sa_defrag_extent_backref *prev, 2558 struct sa_defrag_extent_backref *backref) 2559 { 2560 struct btrfs_file_extent_item *extent; 2561 struct btrfs_file_extent_item *item; 2562 struct btrfs_ordered_extent *ordered; 2563 struct btrfs_trans_handle *trans; 2564 struct btrfs_root *root; 2565 struct btrfs_key key; 2566 struct extent_buffer *leaf; 2567 struct old_sa_defrag_extent *old = backref->old; 2568 struct new_sa_defrag_extent *new = old->new; 2569 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2570 struct inode *inode; 2571 struct extent_state *cached = NULL; 2572 int ret = 0; 2573 u64 start; 2574 u64 len; 2575 u64 lock_start; 2576 u64 lock_end; 2577 bool merge = false; 2578 int index; 2579 2580 if (prev && prev->root_id == backref->root_id && 2581 prev->inum == backref->inum && 2582 prev->file_pos + prev->num_bytes == backref->file_pos) 2583 merge = true; 2584 2585 /* step 1: get root */ 2586 key.objectid = backref->root_id; 2587 key.type = BTRFS_ROOT_ITEM_KEY; 2588 key.offset = (u64)-1; 2589 2590 index = srcu_read_lock(&fs_info->subvol_srcu); 2591 2592 root = btrfs_read_fs_root_no_name(fs_info, &key); 2593 if (IS_ERR(root)) { 2594 srcu_read_unlock(&fs_info->subvol_srcu, index); 2595 if (PTR_ERR(root) == -ENOENT) 2596 return 0; 2597 return PTR_ERR(root); 2598 } 2599 2600 if (btrfs_root_readonly(root)) { 2601 srcu_read_unlock(&fs_info->subvol_srcu, index); 2602 return 0; 2603 } 2604 2605 /* step 2: get inode */ 2606 key.objectid = backref->inum; 2607 key.type = BTRFS_INODE_ITEM_KEY; 2608 key.offset = 0; 2609 2610 inode = btrfs_iget(fs_info->sb, &key, root, NULL); 2611 if (IS_ERR(inode)) { 2612 srcu_read_unlock(&fs_info->subvol_srcu, index); 2613 return 0; 2614 } 2615 2616 srcu_read_unlock(&fs_info->subvol_srcu, index); 2617 2618 /* step 3: relink backref */ 2619 lock_start = backref->file_pos; 2620 lock_end = backref->file_pos + backref->num_bytes - 1; 2621 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2622 &cached); 2623 2624 ordered = btrfs_lookup_first_ordered_extent(inode, lock_end); 2625 if (ordered) { 2626 btrfs_put_ordered_extent(ordered); 2627 goto out_unlock; 2628 } 2629 2630 trans = btrfs_join_transaction(root); 2631 if (IS_ERR(trans)) { 2632 ret = PTR_ERR(trans); 2633 goto out_unlock; 2634 } 2635 2636 key.objectid = backref->inum; 2637 key.type = BTRFS_EXTENT_DATA_KEY; 2638 key.offset = backref->file_pos; 2639 2640 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2641 if (ret < 0) { 2642 goto out_free_path; 2643 } else if (ret > 0) { 2644 ret = 0; 2645 goto out_free_path; 2646 } 2647 2648 extent = btrfs_item_ptr(path->nodes[0], path->slots[0], 2649 struct btrfs_file_extent_item); 2650 2651 if (btrfs_file_extent_generation(path->nodes[0], extent) != 2652 backref->generation) 2653 goto out_free_path; 2654 2655 btrfs_release_path(path); 2656 2657 start = backref->file_pos; 2658 if (backref->extent_offset < old->extent_offset + old->offset) 2659 start += old->extent_offset + old->offset - 2660 backref->extent_offset; 2661 2662 len = min(backref->extent_offset + backref->num_bytes, 2663 old->extent_offset + old->offset + old->len); 2664 len -= max(backref->extent_offset, old->extent_offset + old->offset); 2665 2666 ret = btrfs_drop_extents(trans, root, inode, start, 2667 start + len, 1); 2668 if (ret) 2669 goto out_free_path; 2670 again: 2671 key.objectid = btrfs_ino(BTRFS_I(inode)); 2672 key.type = BTRFS_EXTENT_DATA_KEY; 2673 key.offset = start; 2674 2675 path->leave_spinning = 1; 2676 if (merge) { 2677 struct btrfs_file_extent_item *fi; 2678 u64 extent_len; 2679 struct btrfs_key found_key; 2680 2681 ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 2682 if (ret < 0) 2683 goto out_free_path; 2684 2685 path->slots[0]--; 2686 leaf = path->nodes[0]; 2687 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 2688 2689 fi = btrfs_item_ptr(leaf, path->slots[0], 2690 struct btrfs_file_extent_item); 2691 extent_len = btrfs_file_extent_num_bytes(leaf, fi); 2692 2693 if (extent_len + found_key.offset == start && 2694 relink_is_mergable(leaf, fi, new)) { 2695 btrfs_set_file_extent_num_bytes(leaf, fi, 2696 extent_len + len); 2697 btrfs_mark_buffer_dirty(leaf); 2698 inode_add_bytes(inode, len); 2699 2700 ret = 1; 2701 goto out_free_path; 2702 } else { 2703 merge = false; 2704 btrfs_release_path(path); 2705 goto again; 2706 } 2707 } 2708 2709 ret = btrfs_insert_empty_item(trans, root, path, &key, 2710 sizeof(*extent)); 2711 if (ret) { 2712 btrfs_abort_transaction(trans, ret); 2713 goto out_free_path; 2714 } 2715 2716 leaf = path->nodes[0]; 2717 item = btrfs_item_ptr(leaf, path->slots[0], 2718 struct btrfs_file_extent_item); 2719 btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr); 2720 btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len); 2721 btrfs_set_file_extent_offset(leaf, item, start - new->file_pos); 2722 btrfs_set_file_extent_num_bytes(leaf, item, len); 2723 btrfs_set_file_extent_ram_bytes(leaf, item, new->len); 2724 btrfs_set_file_extent_generation(leaf, item, trans->transid); 2725 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); 2726 btrfs_set_file_extent_compression(leaf, item, new->compress_type); 2727 btrfs_set_file_extent_encryption(leaf, item, 0); 2728 btrfs_set_file_extent_other_encoding(leaf, item, 0); 2729 2730 btrfs_mark_buffer_dirty(leaf); 2731 inode_add_bytes(inode, len); 2732 btrfs_release_path(path); 2733 2734 ret = btrfs_inc_extent_ref(trans, root, new->bytenr, 2735 new->disk_len, 0, 2736 backref->root_id, backref->inum, 2737 new->file_pos); /* start - extent_offset */ 2738 if (ret) { 2739 btrfs_abort_transaction(trans, ret); 2740 goto out_free_path; 2741 } 2742 2743 ret = 1; 2744 out_free_path: 2745 btrfs_release_path(path); 2746 path->leave_spinning = 0; 2747 btrfs_end_transaction(trans); 2748 out_unlock: 2749 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2750 &cached); 2751 iput(inode); 2752 return ret; 2753 } 2754 2755 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new) 2756 { 2757 struct old_sa_defrag_extent *old, *tmp; 2758 2759 if (!new) 2760 return; 2761 2762 list_for_each_entry_safe(old, tmp, &new->head, list) { 2763 kfree(old); 2764 } 2765 kfree(new); 2766 } 2767 2768 static void relink_file_extents(struct new_sa_defrag_extent *new) 2769 { 2770 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2771 struct btrfs_path *path; 2772 struct sa_defrag_extent_backref *backref; 2773 struct sa_defrag_extent_backref *prev = NULL; 2774 struct inode *inode; 2775 struct rb_node *node; 2776 int ret; 2777 2778 inode = new->inode; 2779 2780 path = btrfs_alloc_path(); 2781 if (!path) 2782 return; 2783 2784 if (!record_extent_backrefs(path, new)) { 2785 btrfs_free_path(path); 2786 goto out; 2787 } 2788 btrfs_release_path(path); 2789 2790 while (1) { 2791 node = rb_first(&new->root); 2792 if (!node) 2793 break; 2794 rb_erase(node, &new->root); 2795 2796 backref = rb_entry(node, struct sa_defrag_extent_backref, node); 2797 2798 ret = relink_extent_backref(path, prev, backref); 2799 WARN_ON(ret < 0); 2800 2801 kfree(prev); 2802 2803 if (ret == 1) 2804 prev = backref; 2805 else 2806 prev = NULL; 2807 cond_resched(); 2808 } 2809 kfree(prev); 2810 2811 btrfs_free_path(path); 2812 out: 2813 free_sa_defrag_extent(new); 2814 2815 atomic_dec(&fs_info->defrag_running); 2816 wake_up(&fs_info->transaction_wait); 2817 } 2818 2819 static struct new_sa_defrag_extent * 2820 record_old_file_extents(struct inode *inode, 2821 struct btrfs_ordered_extent *ordered) 2822 { 2823 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2824 struct btrfs_root *root = BTRFS_I(inode)->root; 2825 struct btrfs_path *path; 2826 struct btrfs_key key; 2827 struct old_sa_defrag_extent *old; 2828 struct new_sa_defrag_extent *new; 2829 int ret; 2830 2831 new = kmalloc(sizeof(*new), GFP_NOFS); 2832 if (!new) 2833 return NULL; 2834 2835 new->inode = inode; 2836 new->file_pos = ordered->file_offset; 2837 new->len = ordered->len; 2838 new->bytenr = ordered->start; 2839 new->disk_len = ordered->disk_len; 2840 new->compress_type = ordered->compress_type; 2841 new->root = RB_ROOT; 2842 INIT_LIST_HEAD(&new->head); 2843 2844 path = btrfs_alloc_path(); 2845 if (!path) 2846 goto out_kfree; 2847 2848 key.objectid = btrfs_ino(BTRFS_I(inode)); 2849 key.type = BTRFS_EXTENT_DATA_KEY; 2850 key.offset = new->file_pos; 2851 2852 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2853 if (ret < 0) 2854 goto out_free_path; 2855 if (ret > 0 && path->slots[0] > 0) 2856 path->slots[0]--; 2857 2858 /* find out all the old extents for the file range */ 2859 while (1) { 2860 struct btrfs_file_extent_item *extent; 2861 struct extent_buffer *l; 2862 int slot; 2863 u64 num_bytes; 2864 u64 offset; 2865 u64 end; 2866 u64 disk_bytenr; 2867 u64 extent_offset; 2868 2869 l = path->nodes[0]; 2870 slot = path->slots[0]; 2871 2872 if (slot >= btrfs_header_nritems(l)) { 2873 ret = btrfs_next_leaf(root, path); 2874 if (ret < 0) 2875 goto out_free_path; 2876 else if (ret > 0) 2877 break; 2878 continue; 2879 } 2880 2881 btrfs_item_key_to_cpu(l, &key, slot); 2882 2883 if (key.objectid != btrfs_ino(BTRFS_I(inode))) 2884 break; 2885 if (key.type != BTRFS_EXTENT_DATA_KEY) 2886 break; 2887 if (key.offset >= new->file_pos + new->len) 2888 break; 2889 2890 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item); 2891 2892 num_bytes = btrfs_file_extent_num_bytes(l, extent); 2893 if (key.offset + num_bytes < new->file_pos) 2894 goto next; 2895 2896 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent); 2897 if (!disk_bytenr) 2898 goto next; 2899 2900 extent_offset = btrfs_file_extent_offset(l, extent); 2901 2902 old = kmalloc(sizeof(*old), GFP_NOFS); 2903 if (!old) 2904 goto out_free_path; 2905 2906 offset = max(new->file_pos, key.offset); 2907 end = min(new->file_pos + new->len, key.offset + num_bytes); 2908 2909 old->bytenr = disk_bytenr; 2910 old->extent_offset = extent_offset; 2911 old->offset = offset - key.offset; 2912 old->len = end - offset; 2913 old->new = new; 2914 old->count = 0; 2915 list_add_tail(&old->list, &new->head); 2916 next: 2917 path->slots[0]++; 2918 cond_resched(); 2919 } 2920 2921 btrfs_free_path(path); 2922 atomic_inc(&fs_info->defrag_running); 2923 2924 return new; 2925 2926 out_free_path: 2927 btrfs_free_path(path); 2928 out_kfree: 2929 free_sa_defrag_extent(new); 2930 return NULL; 2931 } 2932 2933 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 2934 u64 start, u64 len) 2935 { 2936 struct btrfs_block_group_cache *cache; 2937 2938 cache = btrfs_lookup_block_group(fs_info, start); 2939 ASSERT(cache); 2940 2941 spin_lock(&cache->lock); 2942 cache->delalloc_bytes -= len; 2943 spin_unlock(&cache->lock); 2944 2945 btrfs_put_block_group(cache); 2946 } 2947 2948 /* as ordered data IO finishes, this gets called so we can finish 2949 * an ordered extent if the range of bytes in the file it covers are 2950 * fully written. 2951 */ 2952 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 2953 { 2954 struct inode *inode = ordered_extent->inode; 2955 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2956 struct btrfs_root *root = BTRFS_I(inode)->root; 2957 struct btrfs_trans_handle *trans = NULL; 2958 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2959 struct extent_state *cached_state = NULL; 2960 struct new_sa_defrag_extent *new = NULL; 2961 int compress_type = 0; 2962 int ret = 0; 2963 u64 logical_len = ordered_extent->len; 2964 bool nolock; 2965 bool truncated = false; 2966 bool range_locked = false; 2967 bool clear_new_delalloc_bytes = false; 2968 2969 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 2970 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && 2971 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags)) 2972 clear_new_delalloc_bytes = true; 2973 2974 nolock = btrfs_is_free_space_inode(BTRFS_I(inode)); 2975 2976 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 2977 ret = -EIO; 2978 goto out; 2979 } 2980 2981 btrfs_free_io_failure_record(BTRFS_I(inode), 2982 ordered_extent->file_offset, 2983 ordered_extent->file_offset + 2984 ordered_extent->len - 1); 2985 2986 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 2987 truncated = true; 2988 logical_len = ordered_extent->truncated_len; 2989 /* Truncated the entire extent, don't bother adding */ 2990 if (!logical_len) 2991 goto out; 2992 } 2993 2994 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 2995 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 2996 2997 /* 2998 * For mwrite(mmap + memset to write) case, we still reserve 2999 * space for NOCOW range. 3000 * As NOCOW won't cause a new delayed ref, just free the space 3001 */ 3002 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset, 3003 ordered_extent->len); 3004 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 3005 if (nolock) 3006 trans = btrfs_join_transaction_nolock(root); 3007 else 3008 trans = btrfs_join_transaction(root); 3009 if (IS_ERR(trans)) { 3010 ret = PTR_ERR(trans); 3011 trans = NULL; 3012 goto out; 3013 } 3014 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 3015 ret = btrfs_update_inode_fallback(trans, root, inode); 3016 if (ret) /* -ENOMEM or corruption */ 3017 btrfs_abort_transaction(trans, ret); 3018 goto out; 3019 } 3020 3021 range_locked = true; 3022 lock_extent_bits(io_tree, ordered_extent->file_offset, 3023 ordered_extent->file_offset + ordered_extent->len - 1, 3024 &cached_state); 3025 3026 ret = test_range_bit(io_tree, ordered_extent->file_offset, 3027 ordered_extent->file_offset + ordered_extent->len - 1, 3028 EXTENT_DEFRAG, 0, cached_state); 3029 if (ret) { 3030 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item); 3031 if (0 && last_snapshot >= BTRFS_I(inode)->generation) 3032 /* the inode is shared */ 3033 new = record_old_file_extents(inode, ordered_extent); 3034 3035 clear_extent_bit(io_tree, ordered_extent->file_offset, 3036 ordered_extent->file_offset + ordered_extent->len - 1, 3037 EXTENT_DEFRAG, 0, 0, &cached_state); 3038 } 3039 3040 if (nolock) 3041 trans = btrfs_join_transaction_nolock(root); 3042 else 3043 trans = btrfs_join_transaction(root); 3044 if (IS_ERR(trans)) { 3045 ret = PTR_ERR(trans); 3046 trans = NULL; 3047 goto out; 3048 } 3049 3050 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 3051 3052 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 3053 compress_type = ordered_extent->compress_type; 3054 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3055 BUG_ON(compress_type); 3056 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset, 3057 ordered_extent->len); 3058 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode), 3059 ordered_extent->file_offset, 3060 ordered_extent->file_offset + 3061 logical_len); 3062 } else { 3063 BUG_ON(root == fs_info->tree_root); 3064 ret = insert_reserved_file_extent(trans, inode, 3065 ordered_extent->file_offset, 3066 ordered_extent->start, 3067 ordered_extent->disk_len, 3068 logical_len, logical_len, 3069 compress_type, 0, 0, 3070 BTRFS_FILE_EXTENT_REG); 3071 if (!ret) 3072 btrfs_release_delalloc_bytes(fs_info, 3073 ordered_extent->start, 3074 ordered_extent->disk_len); 3075 } 3076 unpin_extent_cache(&BTRFS_I(inode)->extent_tree, 3077 ordered_extent->file_offset, ordered_extent->len, 3078 trans->transid); 3079 if (ret < 0) { 3080 btrfs_abort_transaction(trans, ret); 3081 goto out; 3082 } 3083 3084 ret = add_pending_csums(trans, inode, &ordered_extent->list); 3085 if (ret) { 3086 btrfs_abort_transaction(trans, ret); 3087 goto out; 3088 } 3089 3090 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 3091 ret = btrfs_update_inode_fallback(trans, root, inode); 3092 if (ret) { /* -ENOMEM or corruption */ 3093 btrfs_abort_transaction(trans, ret); 3094 goto out; 3095 } 3096 ret = 0; 3097 out: 3098 if (range_locked || clear_new_delalloc_bytes) { 3099 unsigned int clear_bits = 0; 3100 3101 if (range_locked) 3102 clear_bits |= EXTENT_LOCKED; 3103 if (clear_new_delalloc_bytes) 3104 clear_bits |= EXTENT_DELALLOC_NEW; 3105 clear_extent_bit(&BTRFS_I(inode)->io_tree, 3106 ordered_extent->file_offset, 3107 ordered_extent->file_offset + 3108 ordered_extent->len - 1, 3109 clear_bits, 3110 (clear_bits & EXTENT_LOCKED) ? 1 : 0, 3111 0, &cached_state); 3112 } 3113 3114 if (trans) 3115 btrfs_end_transaction(trans); 3116 3117 if (ret || truncated) { 3118 u64 start, end; 3119 3120 if (truncated) 3121 start = ordered_extent->file_offset + logical_len; 3122 else 3123 start = ordered_extent->file_offset; 3124 end = ordered_extent->file_offset + ordered_extent->len - 1; 3125 clear_extent_uptodate(io_tree, start, end, NULL); 3126 3127 /* Drop the cache for the part of the extent we didn't write. */ 3128 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0); 3129 3130 /* 3131 * If the ordered extent had an IOERR or something else went 3132 * wrong we need to return the space for this ordered extent 3133 * back to the allocator. We only free the extent in the 3134 * truncated case if we didn't write out the extent at all. 3135 */ 3136 if ((ret || !logical_len) && 3137 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3138 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) 3139 btrfs_free_reserved_extent(fs_info, 3140 ordered_extent->start, 3141 ordered_extent->disk_len, 1); 3142 } 3143 3144 3145 /* 3146 * This needs to be done to make sure anybody waiting knows we are done 3147 * updating everything for this ordered extent. 3148 */ 3149 btrfs_remove_ordered_extent(inode, ordered_extent); 3150 3151 /* for snapshot-aware defrag */ 3152 if (new) { 3153 if (ret) { 3154 free_sa_defrag_extent(new); 3155 atomic_dec(&fs_info->defrag_running); 3156 } else { 3157 relink_file_extents(new); 3158 } 3159 } 3160 3161 /* once for us */ 3162 btrfs_put_ordered_extent(ordered_extent); 3163 /* once for the tree */ 3164 btrfs_put_ordered_extent(ordered_extent); 3165 3166 return ret; 3167 } 3168 3169 static void finish_ordered_fn(struct btrfs_work *work) 3170 { 3171 struct btrfs_ordered_extent *ordered_extent; 3172 ordered_extent = container_of(work, struct btrfs_ordered_extent, work); 3173 btrfs_finish_ordered_io(ordered_extent); 3174 } 3175 3176 static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end, 3177 struct extent_state *state, int uptodate) 3178 { 3179 struct inode *inode = page->mapping->host; 3180 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3181 struct btrfs_ordered_extent *ordered_extent = NULL; 3182 struct btrfs_workqueue *wq; 3183 btrfs_work_func_t func; 3184 3185 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); 3186 3187 ClearPagePrivate2(page); 3188 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, 3189 end - start + 1, uptodate)) 3190 return; 3191 3192 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 3193 wq = fs_info->endio_freespace_worker; 3194 func = btrfs_freespace_write_helper; 3195 } else { 3196 wq = fs_info->endio_write_workers; 3197 func = btrfs_endio_write_helper; 3198 } 3199 3200 btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL, 3201 NULL); 3202 btrfs_queue_work(wq, &ordered_extent->work); 3203 } 3204 3205 static int __readpage_endio_check(struct inode *inode, 3206 struct btrfs_io_bio *io_bio, 3207 int icsum, struct page *page, 3208 int pgoff, u64 start, size_t len) 3209 { 3210 char *kaddr; 3211 u32 csum_expected; 3212 u32 csum = ~(u32)0; 3213 3214 csum_expected = *(((u32 *)io_bio->csum) + icsum); 3215 3216 kaddr = kmap_atomic(page); 3217 csum = btrfs_csum_data(kaddr + pgoff, csum, len); 3218 btrfs_csum_final(csum, (u8 *)&csum); 3219 if (csum != csum_expected) 3220 goto zeroit; 3221 3222 kunmap_atomic(kaddr); 3223 return 0; 3224 zeroit: 3225 btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected, 3226 io_bio->mirror_num); 3227 memset(kaddr + pgoff, 1, len); 3228 flush_dcache_page(page); 3229 kunmap_atomic(kaddr); 3230 return -EIO; 3231 } 3232 3233 /* 3234 * when reads are done, we need to check csums to verify the data is correct 3235 * if there's a match, we allow the bio to finish. If not, the code in 3236 * extent_io.c will try to find good copies for us. 3237 */ 3238 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, 3239 u64 phy_offset, struct page *page, 3240 u64 start, u64 end, int mirror) 3241 { 3242 size_t offset = start - page_offset(page); 3243 struct inode *inode = page->mapping->host; 3244 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 3245 struct btrfs_root *root = BTRFS_I(inode)->root; 3246 3247 if (PageChecked(page)) { 3248 ClearPageChecked(page); 3249 return 0; 3250 } 3251 3252 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 3253 return 0; 3254 3255 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID && 3256 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) { 3257 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM); 3258 return 0; 3259 } 3260 3261 phy_offset >>= inode->i_sb->s_blocksize_bits; 3262 return __readpage_endio_check(inode, io_bio, phy_offset, page, offset, 3263 start, (size_t)(end - start + 1)); 3264 } 3265 3266 /* 3267 * btrfs_add_delayed_iput - perform a delayed iput on @inode 3268 * 3269 * @inode: The inode we want to perform iput on 3270 * 3271 * This function uses the generic vfs_inode::i_count to track whether we should 3272 * just decrement it (in case it's > 1) or if this is the last iput then link 3273 * the inode to the delayed iput machinery. Delayed iputs are processed at 3274 * transaction commit time/superblock commit/cleaner kthread. 3275 */ 3276 void btrfs_add_delayed_iput(struct inode *inode) 3277 { 3278 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3279 struct btrfs_inode *binode = BTRFS_I(inode); 3280 3281 if (atomic_add_unless(&inode->i_count, -1, 1)) 3282 return; 3283 3284 spin_lock(&fs_info->delayed_iput_lock); 3285 ASSERT(list_empty(&binode->delayed_iput)); 3286 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); 3287 spin_unlock(&fs_info->delayed_iput_lock); 3288 } 3289 3290 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 3291 { 3292 3293 spin_lock(&fs_info->delayed_iput_lock); 3294 while (!list_empty(&fs_info->delayed_iputs)) { 3295 struct btrfs_inode *inode; 3296 3297 inode = list_first_entry(&fs_info->delayed_iputs, 3298 struct btrfs_inode, delayed_iput); 3299 list_del_init(&inode->delayed_iput); 3300 spin_unlock(&fs_info->delayed_iput_lock); 3301 iput(&inode->vfs_inode); 3302 spin_lock(&fs_info->delayed_iput_lock); 3303 } 3304 spin_unlock(&fs_info->delayed_iput_lock); 3305 } 3306 3307 /* 3308 * This is called in transaction commit time. If there are no orphan 3309 * files in the subvolume, it removes orphan item and frees block_rsv 3310 * structure. 3311 */ 3312 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans, 3313 struct btrfs_root *root) 3314 { 3315 struct btrfs_fs_info *fs_info = root->fs_info; 3316 struct btrfs_block_rsv *block_rsv; 3317 int ret; 3318 3319 if (atomic_read(&root->orphan_inodes) || 3320 root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) 3321 return; 3322 3323 spin_lock(&root->orphan_lock); 3324 if (atomic_read(&root->orphan_inodes)) { 3325 spin_unlock(&root->orphan_lock); 3326 return; 3327 } 3328 3329 if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) { 3330 spin_unlock(&root->orphan_lock); 3331 return; 3332 } 3333 3334 block_rsv = root->orphan_block_rsv; 3335 root->orphan_block_rsv = NULL; 3336 spin_unlock(&root->orphan_lock); 3337 3338 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) && 3339 btrfs_root_refs(&root->root_item) > 0) { 3340 ret = btrfs_del_orphan_item(trans, fs_info->tree_root, 3341 root->root_key.objectid); 3342 if (ret) 3343 btrfs_abort_transaction(trans, ret); 3344 else 3345 clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, 3346 &root->state); 3347 } 3348 3349 if (block_rsv) { 3350 WARN_ON(block_rsv->size > 0); 3351 btrfs_free_block_rsv(fs_info, block_rsv); 3352 } 3353 } 3354 3355 /* 3356 * This creates an orphan entry for the given inode in case something goes 3357 * wrong in the middle of an unlink/truncate. 3358 * 3359 * NOTE: caller of this function should reserve 5 units of metadata for 3360 * this function. 3361 */ 3362 int btrfs_orphan_add(struct btrfs_trans_handle *trans, 3363 struct btrfs_inode *inode) 3364 { 3365 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 3366 struct btrfs_root *root = inode->root; 3367 struct btrfs_block_rsv *block_rsv = NULL; 3368 int reserve = 0; 3369 bool insert = false; 3370 int ret; 3371 3372 if (!root->orphan_block_rsv) { 3373 block_rsv = btrfs_alloc_block_rsv(fs_info, 3374 BTRFS_BLOCK_RSV_TEMP); 3375 if (!block_rsv) 3376 return -ENOMEM; 3377 } 3378 3379 if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3380 &inode->runtime_flags)) 3381 insert = true; 3382 3383 if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3384 &inode->runtime_flags)) 3385 reserve = 1; 3386 3387 spin_lock(&root->orphan_lock); 3388 /* If someone has created ->orphan_block_rsv, be happy to use it. */ 3389 if (!root->orphan_block_rsv) { 3390 root->orphan_block_rsv = block_rsv; 3391 } else if (block_rsv) { 3392 btrfs_free_block_rsv(fs_info, block_rsv); 3393 block_rsv = NULL; 3394 } 3395 3396 if (insert) 3397 atomic_inc(&root->orphan_inodes); 3398 spin_unlock(&root->orphan_lock); 3399 3400 /* grab metadata reservation from transaction handle */ 3401 if (reserve) { 3402 ret = btrfs_orphan_reserve_metadata(trans, inode); 3403 ASSERT(!ret); 3404 if (ret) { 3405 /* 3406 * dec doesn't need spin_lock as ->orphan_block_rsv 3407 * would be released only if ->orphan_inodes is 3408 * zero. 3409 */ 3410 atomic_dec(&root->orphan_inodes); 3411 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3412 &inode->runtime_flags); 3413 if (insert) 3414 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3415 &inode->runtime_flags); 3416 return ret; 3417 } 3418 } 3419 3420 /* insert an orphan item to track this unlinked/truncated file */ 3421 if (insert) { 3422 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode)); 3423 if (ret) { 3424 if (reserve) { 3425 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3426 &inode->runtime_flags); 3427 btrfs_orphan_release_metadata(inode); 3428 } 3429 /* 3430 * btrfs_orphan_commit_root may race with us and set 3431 * ->orphan_block_rsv to zero, in order to avoid that, 3432 * decrease ->orphan_inodes after everything is done. 3433 */ 3434 atomic_dec(&root->orphan_inodes); 3435 if (ret != -EEXIST) { 3436 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3437 &inode->runtime_flags); 3438 btrfs_abort_transaction(trans, ret); 3439 return ret; 3440 } 3441 } 3442 ret = 0; 3443 } 3444 3445 return 0; 3446 } 3447 3448 /* 3449 * We have done the truncate/delete so we can go ahead and remove the orphan 3450 * item for this particular inode. 3451 */ 3452 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3453 struct btrfs_inode *inode) 3454 { 3455 struct btrfs_root *root = inode->root; 3456 int delete_item = 0; 3457 int ret = 0; 3458 3459 if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3460 &inode->runtime_flags)) 3461 delete_item = 1; 3462 3463 if (delete_item && trans) 3464 ret = btrfs_del_orphan_item(trans, root, btrfs_ino(inode)); 3465 3466 if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3467 &inode->runtime_flags)) 3468 btrfs_orphan_release_metadata(inode); 3469 3470 /* 3471 * btrfs_orphan_commit_root may race with us and set ->orphan_block_rsv 3472 * to zero, in order to avoid that, decrease ->orphan_inodes after 3473 * everything is done. 3474 */ 3475 if (delete_item) 3476 atomic_dec(&root->orphan_inodes); 3477 3478 return ret; 3479 } 3480 3481 /* 3482 * this cleans up any orphans that may be left on the list from the last use 3483 * of this root. 3484 */ 3485 int btrfs_orphan_cleanup(struct btrfs_root *root) 3486 { 3487 struct btrfs_fs_info *fs_info = root->fs_info; 3488 struct btrfs_path *path; 3489 struct extent_buffer *leaf; 3490 struct btrfs_key key, found_key; 3491 struct btrfs_trans_handle *trans; 3492 struct inode *inode; 3493 u64 last_objectid = 0; 3494 int ret = 0, nr_unlink = 0, nr_truncate = 0; 3495 3496 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED)) 3497 return 0; 3498 3499 path = btrfs_alloc_path(); 3500 if (!path) { 3501 ret = -ENOMEM; 3502 goto out; 3503 } 3504 path->reada = READA_BACK; 3505 3506 key.objectid = BTRFS_ORPHAN_OBJECTID; 3507 key.type = BTRFS_ORPHAN_ITEM_KEY; 3508 key.offset = (u64)-1; 3509 3510 while (1) { 3511 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3512 if (ret < 0) 3513 goto out; 3514 3515 /* 3516 * if ret == 0 means we found what we were searching for, which 3517 * is weird, but possible, so only screw with path if we didn't 3518 * find the key and see if we have stuff that matches 3519 */ 3520 if (ret > 0) { 3521 ret = 0; 3522 if (path->slots[0] == 0) 3523 break; 3524 path->slots[0]--; 3525 } 3526 3527 /* pull out the item */ 3528 leaf = path->nodes[0]; 3529 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3530 3531 /* make sure the item matches what we want */ 3532 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3533 break; 3534 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3535 break; 3536 3537 /* release the path since we're done with it */ 3538 btrfs_release_path(path); 3539 3540 /* 3541 * this is where we are basically btrfs_lookup, without the 3542 * crossing root thing. we store the inode number in the 3543 * offset of the orphan item. 3544 */ 3545 3546 if (found_key.offset == last_objectid) { 3547 btrfs_err(fs_info, 3548 "Error removing orphan entry, stopping orphan cleanup"); 3549 ret = -EINVAL; 3550 goto out; 3551 } 3552 3553 last_objectid = found_key.offset; 3554 3555 found_key.objectid = found_key.offset; 3556 found_key.type = BTRFS_INODE_ITEM_KEY; 3557 found_key.offset = 0; 3558 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL); 3559 ret = PTR_ERR_OR_ZERO(inode); 3560 if (ret && ret != -ENOENT) 3561 goto out; 3562 3563 if (ret == -ENOENT && root == fs_info->tree_root) { 3564 struct btrfs_root *dead_root; 3565 struct btrfs_fs_info *fs_info = root->fs_info; 3566 int is_dead_root = 0; 3567 3568 /* 3569 * this is an orphan in the tree root. Currently these 3570 * could come from 2 sources: 3571 * a) a snapshot deletion in progress 3572 * b) a free space cache inode 3573 * We need to distinguish those two, as the snapshot 3574 * orphan must not get deleted. 3575 * find_dead_roots already ran before us, so if this 3576 * is a snapshot deletion, we should find the root 3577 * in the dead_roots list 3578 */ 3579 spin_lock(&fs_info->trans_lock); 3580 list_for_each_entry(dead_root, &fs_info->dead_roots, 3581 root_list) { 3582 if (dead_root->root_key.objectid == 3583 found_key.objectid) { 3584 is_dead_root = 1; 3585 break; 3586 } 3587 } 3588 spin_unlock(&fs_info->trans_lock); 3589 if (is_dead_root) { 3590 /* prevent this orphan from being found again */ 3591 key.offset = found_key.objectid - 1; 3592 continue; 3593 } 3594 } 3595 /* 3596 * Inode is already gone but the orphan item is still there, 3597 * kill the orphan item. 3598 */ 3599 if (ret == -ENOENT) { 3600 trans = btrfs_start_transaction(root, 1); 3601 if (IS_ERR(trans)) { 3602 ret = PTR_ERR(trans); 3603 goto out; 3604 } 3605 btrfs_debug(fs_info, "auto deleting %Lu", 3606 found_key.objectid); 3607 ret = btrfs_del_orphan_item(trans, root, 3608 found_key.objectid); 3609 btrfs_end_transaction(trans); 3610 if (ret) 3611 goto out; 3612 continue; 3613 } 3614 3615 /* 3616 * add this inode to the orphan list so btrfs_orphan_del does 3617 * the proper thing when we hit it 3618 */ 3619 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3620 &BTRFS_I(inode)->runtime_flags); 3621 atomic_inc(&root->orphan_inodes); 3622 3623 /* if we have links, this was a truncate, lets do that */ 3624 if (inode->i_nlink) { 3625 if (WARN_ON(!S_ISREG(inode->i_mode))) { 3626 iput(inode); 3627 continue; 3628 } 3629 nr_truncate++; 3630 3631 /* 1 for the orphan item deletion. */ 3632 trans = btrfs_start_transaction(root, 1); 3633 if (IS_ERR(trans)) { 3634 iput(inode); 3635 ret = PTR_ERR(trans); 3636 goto out; 3637 } 3638 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 3639 btrfs_end_transaction(trans); 3640 if (ret) { 3641 iput(inode); 3642 goto out; 3643 } 3644 3645 ret = btrfs_truncate(inode, false); 3646 if (ret) 3647 btrfs_orphan_del(NULL, BTRFS_I(inode)); 3648 } else { 3649 nr_unlink++; 3650 } 3651 3652 /* this will do delete_inode and everything for us */ 3653 iput(inode); 3654 if (ret) 3655 goto out; 3656 } 3657 /* release the path since we're done with it */ 3658 btrfs_release_path(path); 3659 3660 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE; 3661 3662 if (root->orphan_block_rsv) 3663 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv, 3664 (u64)-1); 3665 3666 if (root->orphan_block_rsv || 3667 test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3668 trans = btrfs_join_transaction(root); 3669 if (!IS_ERR(trans)) 3670 btrfs_end_transaction(trans); 3671 } 3672 3673 if (nr_unlink) 3674 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3675 if (nr_truncate) 3676 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate); 3677 3678 out: 3679 if (ret) 3680 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3681 btrfs_free_path(path); 3682 return ret; 3683 } 3684 3685 /* 3686 * very simple check to peek ahead in the leaf looking for xattrs. If we 3687 * don't find any xattrs, we know there can't be any acls. 3688 * 3689 * slot is the slot the inode is in, objectid is the objectid of the inode 3690 */ 3691 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3692 int slot, u64 objectid, 3693 int *first_xattr_slot) 3694 { 3695 u32 nritems = btrfs_header_nritems(leaf); 3696 struct btrfs_key found_key; 3697 static u64 xattr_access = 0; 3698 static u64 xattr_default = 0; 3699 int scanned = 0; 3700 3701 if (!xattr_access) { 3702 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3703 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3704 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3705 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3706 } 3707 3708 slot++; 3709 *first_xattr_slot = -1; 3710 while (slot < nritems) { 3711 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3712 3713 /* we found a different objectid, there must not be acls */ 3714 if (found_key.objectid != objectid) 3715 return 0; 3716 3717 /* we found an xattr, assume we've got an acl */ 3718 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3719 if (*first_xattr_slot == -1) 3720 *first_xattr_slot = slot; 3721 if (found_key.offset == xattr_access || 3722 found_key.offset == xattr_default) 3723 return 1; 3724 } 3725 3726 /* 3727 * we found a key greater than an xattr key, there can't 3728 * be any acls later on 3729 */ 3730 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3731 return 0; 3732 3733 slot++; 3734 scanned++; 3735 3736 /* 3737 * it goes inode, inode backrefs, xattrs, extents, 3738 * so if there are a ton of hard links to an inode there can 3739 * be a lot of backrefs. Don't waste time searching too hard, 3740 * this is just an optimization 3741 */ 3742 if (scanned >= 8) 3743 break; 3744 } 3745 /* we hit the end of the leaf before we found an xattr or 3746 * something larger than an xattr. We have to assume the inode 3747 * has acls 3748 */ 3749 if (*first_xattr_slot == -1) 3750 *first_xattr_slot = slot; 3751 return 1; 3752 } 3753 3754 /* 3755 * read an inode from the btree into the in-memory inode 3756 */ 3757 static int btrfs_read_locked_inode(struct inode *inode) 3758 { 3759 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3760 struct btrfs_path *path; 3761 struct extent_buffer *leaf; 3762 struct btrfs_inode_item *inode_item; 3763 struct btrfs_root *root = BTRFS_I(inode)->root; 3764 struct btrfs_key location; 3765 unsigned long ptr; 3766 int maybe_acls; 3767 u32 rdev; 3768 int ret; 3769 bool filled = false; 3770 int first_xattr_slot; 3771 3772 ret = btrfs_fill_inode(inode, &rdev); 3773 if (!ret) 3774 filled = true; 3775 3776 path = btrfs_alloc_path(); 3777 if (!path) { 3778 ret = -ENOMEM; 3779 goto make_bad; 3780 } 3781 3782 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3783 3784 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3785 if (ret) { 3786 if (ret > 0) 3787 ret = -ENOENT; 3788 goto make_bad; 3789 } 3790 3791 leaf = path->nodes[0]; 3792 3793 if (filled) 3794 goto cache_index; 3795 3796 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3797 struct btrfs_inode_item); 3798 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3799 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3800 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3801 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3802 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item)); 3803 3804 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime); 3805 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime); 3806 3807 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime); 3808 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime); 3809 3810 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime); 3811 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime); 3812 3813 BTRFS_I(inode)->i_otime.tv_sec = 3814 btrfs_timespec_sec(leaf, &inode_item->otime); 3815 BTRFS_I(inode)->i_otime.tv_nsec = 3816 btrfs_timespec_nsec(leaf, &inode_item->otime); 3817 3818 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3819 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3820 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3821 3822 inode_set_iversion_queried(inode, 3823 btrfs_inode_sequence(leaf, inode_item)); 3824 inode->i_generation = BTRFS_I(inode)->generation; 3825 inode->i_rdev = 0; 3826 rdev = btrfs_inode_rdev(leaf, inode_item); 3827 3828 BTRFS_I(inode)->index_cnt = (u64)-1; 3829 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item); 3830 3831 cache_index: 3832 /* 3833 * If we were modified in the current generation and evicted from memory 3834 * and then re-read we need to do a full sync since we don't have any 3835 * idea about which extents were modified before we were evicted from 3836 * cache. 3837 * 3838 * This is required for both inode re-read from disk and delayed inode 3839 * in delayed_nodes_tree. 3840 */ 3841 if (BTRFS_I(inode)->last_trans == fs_info->generation) 3842 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3843 &BTRFS_I(inode)->runtime_flags); 3844 3845 /* 3846 * We don't persist the id of the transaction where an unlink operation 3847 * against the inode was last made. So here we assume the inode might 3848 * have been evicted, and therefore the exact value of last_unlink_trans 3849 * lost, and set it to last_trans to avoid metadata inconsistencies 3850 * between the inode and its parent if the inode is fsync'ed and the log 3851 * replayed. For example, in the scenario: 3852 * 3853 * touch mydir/foo 3854 * ln mydir/foo mydir/bar 3855 * sync 3856 * unlink mydir/bar 3857 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 3858 * xfs_io -c fsync mydir/foo 3859 * <power failure> 3860 * mount fs, triggers fsync log replay 3861 * 3862 * We must make sure that when we fsync our inode foo we also log its 3863 * parent inode, otherwise after log replay the parent still has the 3864 * dentry with the "bar" name but our inode foo has a link count of 1 3865 * and doesn't have an inode ref with the name "bar" anymore. 3866 * 3867 * Setting last_unlink_trans to last_trans is a pessimistic approach, 3868 * but it guarantees correctness at the expense of occasional full 3869 * transaction commits on fsync if our inode is a directory, or if our 3870 * inode is not a directory, logging its parent unnecessarily. 3871 */ 3872 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans; 3873 3874 path->slots[0]++; 3875 if (inode->i_nlink != 1 || 3876 path->slots[0] >= btrfs_header_nritems(leaf)) 3877 goto cache_acl; 3878 3879 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3880 if (location.objectid != btrfs_ino(BTRFS_I(inode))) 3881 goto cache_acl; 3882 3883 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3884 if (location.type == BTRFS_INODE_REF_KEY) { 3885 struct btrfs_inode_ref *ref; 3886 3887 ref = (struct btrfs_inode_ref *)ptr; 3888 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3889 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3890 struct btrfs_inode_extref *extref; 3891 3892 extref = (struct btrfs_inode_extref *)ptr; 3893 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3894 extref); 3895 } 3896 cache_acl: 3897 /* 3898 * try to precache a NULL acl entry for files that don't have 3899 * any xattrs or acls 3900 */ 3901 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3902 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot); 3903 if (first_xattr_slot != -1) { 3904 path->slots[0] = first_xattr_slot; 3905 ret = btrfs_load_inode_props(inode, path); 3906 if (ret) 3907 btrfs_err(fs_info, 3908 "error loading props for ino %llu (root %llu): %d", 3909 btrfs_ino(BTRFS_I(inode)), 3910 root->root_key.objectid, ret); 3911 } 3912 btrfs_free_path(path); 3913 3914 if (!maybe_acls) 3915 cache_no_acl(inode); 3916 3917 switch (inode->i_mode & S_IFMT) { 3918 case S_IFREG: 3919 inode->i_mapping->a_ops = &btrfs_aops; 3920 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 3921 inode->i_fop = &btrfs_file_operations; 3922 inode->i_op = &btrfs_file_inode_operations; 3923 break; 3924 case S_IFDIR: 3925 inode->i_fop = &btrfs_dir_file_operations; 3926 inode->i_op = &btrfs_dir_inode_operations; 3927 break; 3928 case S_IFLNK: 3929 inode->i_op = &btrfs_symlink_inode_operations; 3930 inode_nohighmem(inode); 3931 inode->i_mapping->a_ops = &btrfs_symlink_aops; 3932 break; 3933 default: 3934 inode->i_op = &btrfs_special_inode_operations; 3935 init_special_inode(inode, inode->i_mode, rdev); 3936 break; 3937 } 3938 3939 btrfs_update_iflags(inode); 3940 return 0; 3941 3942 make_bad: 3943 btrfs_free_path(path); 3944 make_bad_inode(inode); 3945 return ret; 3946 } 3947 3948 /* 3949 * given a leaf and an inode, copy the inode fields into the leaf 3950 */ 3951 static void fill_inode_item(struct btrfs_trans_handle *trans, 3952 struct extent_buffer *leaf, 3953 struct btrfs_inode_item *item, 3954 struct inode *inode) 3955 { 3956 struct btrfs_map_token token; 3957 3958 btrfs_init_map_token(&token); 3959 3960 btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token); 3961 btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token); 3962 btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size, 3963 &token); 3964 btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token); 3965 btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token); 3966 3967 btrfs_set_token_timespec_sec(leaf, &item->atime, 3968 inode->i_atime.tv_sec, &token); 3969 btrfs_set_token_timespec_nsec(leaf, &item->atime, 3970 inode->i_atime.tv_nsec, &token); 3971 3972 btrfs_set_token_timespec_sec(leaf, &item->mtime, 3973 inode->i_mtime.tv_sec, &token); 3974 btrfs_set_token_timespec_nsec(leaf, &item->mtime, 3975 inode->i_mtime.tv_nsec, &token); 3976 3977 btrfs_set_token_timespec_sec(leaf, &item->ctime, 3978 inode->i_ctime.tv_sec, &token); 3979 btrfs_set_token_timespec_nsec(leaf, &item->ctime, 3980 inode->i_ctime.tv_nsec, &token); 3981 3982 btrfs_set_token_timespec_sec(leaf, &item->otime, 3983 BTRFS_I(inode)->i_otime.tv_sec, &token); 3984 btrfs_set_token_timespec_nsec(leaf, &item->otime, 3985 BTRFS_I(inode)->i_otime.tv_nsec, &token); 3986 3987 btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode), 3988 &token); 3989 btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation, 3990 &token); 3991 btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode), 3992 &token); 3993 btrfs_set_token_inode_transid(leaf, item, trans->transid, &token); 3994 btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token); 3995 btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token); 3996 btrfs_set_token_inode_block_group(leaf, item, 0, &token); 3997 } 3998 3999 /* 4000 * copy everything in the in-memory inode into the btree. 4001 */ 4002 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 4003 struct btrfs_root *root, struct inode *inode) 4004 { 4005 struct btrfs_inode_item *inode_item; 4006 struct btrfs_path *path; 4007 struct extent_buffer *leaf; 4008 int ret; 4009 4010 path = btrfs_alloc_path(); 4011 if (!path) 4012 return -ENOMEM; 4013 4014 path->leave_spinning = 1; 4015 ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location, 4016 1); 4017 if (ret) { 4018 if (ret > 0) 4019 ret = -ENOENT; 4020 goto failed; 4021 } 4022 4023 leaf = path->nodes[0]; 4024 inode_item = btrfs_item_ptr(leaf, path->slots[0], 4025 struct btrfs_inode_item); 4026 4027 fill_inode_item(trans, leaf, inode_item, inode); 4028 btrfs_mark_buffer_dirty(leaf); 4029 btrfs_set_inode_last_trans(trans, inode); 4030 ret = 0; 4031 failed: 4032 btrfs_free_path(path); 4033 return ret; 4034 } 4035 4036 /* 4037 * copy everything in the in-memory inode into the btree. 4038 */ 4039 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 4040 struct btrfs_root *root, struct inode *inode) 4041 { 4042 struct btrfs_fs_info *fs_info = root->fs_info; 4043 int ret; 4044 4045 /* 4046 * If the inode is a free space inode, we can deadlock during commit 4047 * if we put it into the delayed code. 4048 * 4049 * The data relocation inode should also be directly updated 4050 * without delay 4051 */ 4052 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) 4053 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 4054 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 4055 btrfs_update_root_times(trans, root); 4056 4057 ret = btrfs_delayed_update_inode(trans, root, inode); 4058 if (!ret) 4059 btrfs_set_inode_last_trans(trans, inode); 4060 return ret; 4061 } 4062 4063 return btrfs_update_inode_item(trans, root, inode); 4064 } 4065 4066 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 4067 struct btrfs_root *root, 4068 struct inode *inode) 4069 { 4070 int ret; 4071 4072 ret = btrfs_update_inode(trans, root, inode); 4073 if (ret == -ENOSPC) 4074 return btrfs_update_inode_item(trans, root, inode); 4075 return ret; 4076 } 4077 4078 /* 4079 * unlink helper that gets used here in inode.c and in the tree logging 4080 * recovery code. It remove a link in a directory with a given name, and 4081 * also drops the back refs in the inode to the directory 4082 */ 4083 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4084 struct btrfs_root *root, 4085 struct btrfs_inode *dir, 4086 struct btrfs_inode *inode, 4087 const char *name, int name_len) 4088 { 4089 struct btrfs_fs_info *fs_info = root->fs_info; 4090 struct btrfs_path *path; 4091 int ret = 0; 4092 struct extent_buffer *leaf; 4093 struct btrfs_dir_item *di; 4094 struct btrfs_key key; 4095 u64 index; 4096 u64 ino = btrfs_ino(inode); 4097 u64 dir_ino = btrfs_ino(dir); 4098 4099 path = btrfs_alloc_path(); 4100 if (!path) { 4101 ret = -ENOMEM; 4102 goto out; 4103 } 4104 4105 path->leave_spinning = 1; 4106 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4107 name, name_len, -1); 4108 if (IS_ERR(di)) { 4109 ret = PTR_ERR(di); 4110 goto err; 4111 } 4112 if (!di) { 4113 ret = -ENOENT; 4114 goto err; 4115 } 4116 leaf = path->nodes[0]; 4117 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4118 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4119 if (ret) 4120 goto err; 4121 btrfs_release_path(path); 4122 4123 /* 4124 * If we don't have dir index, we have to get it by looking up 4125 * the inode ref, since we get the inode ref, remove it directly, 4126 * it is unnecessary to do delayed deletion. 4127 * 4128 * But if we have dir index, needn't search inode ref to get it. 4129 * Since the inode ref is close to the inode item, it is better 4130 * that we delay to delete it, and just do this deletion when 4131 * we update the inode item. 4132 */ 4133 if (inode->dir_index) { 4134 ret = btrfs_delayed_delete_inode_ref(inode); 4135 if (!ret) { 4136 index = inode->dir_index; 4137 goto skip_backref; 4138 } 4139 } 4140 4141 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino, 4142 dir_ino, &index); 4143 if (ret) { 4144 btrfs_info(fs_info, 4145 "failed to delete reference to %.*s, inode %llu parent %llu", 4146 name_len, name, ino, dir_ino); 4147 btrfs_abort_transaction(trans, ret); 4148 goto err; 4149 } 4150 skip_backref: 4151 ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index); 4152 if (ret) { 4153 btrfs_abort_transaction(trans, ret); 4154 goto err; 4155 } 4156 4157 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode, 4158 dir_ino); 4159 if (ret != 0 && ret != -ENOENT) { 4160 btrfs_abort_transaction(trans, ret); 4161 goto err; 4162 } 4163 4164 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir, 4165 index); 4166 if (ret == -ENOENT) 4167 ret = 0; 4168 else if (ret) 4169 btrfs_abort_transaction(trans, ret); 4170 err: 4171 btrfs_free_path(path); 4172 if (ret) 4173 goto out; 4174 4175 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2); 4176 inode_inc_iversion(&inode->vfs_inode); 4177 inode_inc_iversion(&dir->vfs_inode); 4178 inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime = 4179 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode); 4180 ret = btrfs_update_inode(trans, root, &dir->vfs_inode); 4181 out: 4182 return ret; 4183 } 4184 4185 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4186 struct btrfs_root *root, 4187 struct btrfs_inode *dir, struct btrfs_inode *inode, 4188 const char *name, int name_len) 4189 { 4190 int ret; 4191 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len); 4192 if (!ret) { 4193 drop_nlink(&inode->vfs_inode); 4194 ret = btrfs_update_inode(trans, root, &inode->vfs_inode); 4195 } 4196 return ret; 4197 } 4198 4199 /* 4200 * helper to start transaction for unlink and rmdir. 4201 * 4202 * unlink and rmdir are special in btrfs, they do not always free space, so 4203 * if we cannot make our reservations the normal way try and see if there is 4204 * plenty of slack room in the global reserve to migrate, otherwise we cannot 4205 * allow the unlink to occur. 4206 */ 4207 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) 4208 { 4209 struct btrfs_root *root = BTRFS_I(dir)->root; 4210 4211 /* 4212 * 1 for the possible orphan item 4213 * 1 for the dir item 4214 * 1 for the dir index 4215 * 1 for the inode ref 4216 * 1 for the inode 4217 */ 4218 return btrfs_start_transaction_fallback_global_rsv(root, 5, 5); 4219 } 4220 4221 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 4222 { 4223 struct btrfs_root *root = BTRFS_I(dir)->root; 4224 struct btrfs_trans_handle *trans; 4225 struct inode *inode = d_inode(dentry); 4226 int ret; 4227 4228 trans = __unlink_start_trans(dir); 4229 if (IS_ERR(trans)) 4230 return PTR_ERR(trans); 4231 4232 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4233 0); 4234 4235 ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4236 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4237 dentry->d_name.len); 4238 if (ret) 4239 goto out; 4240 4241 if (inode->i_nlink == 0) { 4242 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 4243 if (ret) 4244 goto out; 4245 } 4246 4247 out: 4248 btrfs_end_transaction(trans); 4249 btrfs_btree_balance_dirty(root->fs_info); 4250 return ret; 4251 } 4252 4253 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 4254 struct btrfs_root *root, 4255 struct inode *dir, u64 objectid, 4256 const char *name, int name_len) 4257 { 4258 struct btrfs_fs_info *fs_info = root->fs_info; 4259 struct btrfs_path *path; 4260 struct extent_buffer *leaf; 4261 struct btrfs_dir_item *di; 4262 struct btrfs_key key; 4263 u64 index; 4264 int ret; 4265 u64 dir_ino = btrfs_ino(BTRFS_I(dir)); 4266 4267 path = btrfs_alloc_path(); 4268 if (!path) 4269 return -ENOMEM; 4270 4271 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4272 name, name_len, -1); 4273 if (IS_ERR_OR_NULL(di)) { 4274 if (!di) 4275 ret = -ENOENT; 4276 else 4277 ret = PTR_ERR(di); 4278 goto out; 4279 } 4280 4281 leaf = path->nodes[0]; 4282 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4283 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 4284 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4285 if (ret) { 4286 btrfs_abort_transaction(trans, ret); 4287 goto out; 4288 } 4289 btrfs_release_path(path); 4290 4291 ret = btrfs_del_root_ref(trans, fs_info, objectid, 4292 root->root_key.objectid, dir_ino, 4293 &index, name, name_len); 4294 if (ret < 0) { 4295 if (ret != -ENOENT) { 4296 btrfs_abort_transaction(trans, ret); 4297 goto out; 4298 } 4299 di = btrfs_search_dir_index_item(root, path, dir_ino, 4300 name, name_len); 4301 if (IS_ERR_OR_NULL(di)) { 4302 if (!di) 4303 ret = -ENOENT; 4304 else 4305 ret = PTR_ERR(di); 4306 btrfs_abort_transaction(trans, ret); 4307 goto out; 4308 } 4309 4310 leaf = path->nodes[0]; 4311 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4312 btrfs_release_path(path); 4313 index = key.offset; 4314 } 4315 btrfs_release_path(path); 4316 4317 ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index); 4318 if (ret) { 4319 btrfs_abort_transaction(trans, ret); 4320 goto out; 4321 } 4322 4323 btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2); 4324 inode_inc_iversion(dir); 4325 dir->i_mtime = dir->i_ctime = current_time(dir); 4326 ret = btrfs_update_inode_fallback(trans, root, dir); 4327 if (ret) 4328 btrfs_abort_transaction(trans, ret); 4329 out: 4330 btrfs_free_path(path); 4331 return ret; 4332 } 4333 4334 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4335 { 4336 struct inode *inode = d_inode(dentry); 4337 int err = 0; 4338 struct btrfs_root *root = BTRFS_I(dir)->root; 4339 struct btrfs_trans_handle *trans; 4340 u64 last_unlink_trans; 4341 4342 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4343 return -ENOTEMPTY; 4344 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) 4345 return -EPERM; 4346 4347 trans = __unlink_start_trans(dir); 4348 if (IS_ERR(trans)) 4349 return PTR_ERR(trans); 4350 4351 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4352 err = btrfs_unlink_subvol(trans, root, dir, 4353 BTRFS_I(inode)->location.objectid, 4354 dentry->d_name.name, 4355 dentry->d_name.len); 4356 goto out; 4357 } 4358 4359 err = btrfs_orphan_add(trans, BTRFS_I(inode)); 4360 if (err) 4361 goto out; 4362 4363 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; 4364 4365 /* now the directory is empty */ 4366 err = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4367 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4368 dentry->d_name.len); 4369 if (!err) { 4370 btrfs_i_size_write(BTRFS_I(inode), 0); 4371 /* 4372 * Propagate the last_unlink_trans value of the deleted dir to 4373 * its parent directory. This is to prevent an unrecoverable 4374 * log tree in the case we do something like this: 4375 * 1) create dir foo 4376 * 2) create snapshot under dir foo 4377 * 3) delete the snapshot 4378 * 4) rmdir foo 4379 * 5) mkdir foo 4380 * 6) fsync foo or some file inside foo 4381 */ 4382 if (last_unlink_trans >= trans->transid) 4383 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; 4384 } 4385 out: 4386 btrfs_end_transaction(trans); 4387 btrfs_btree_balance_dirty(root->fs_info); 4388 4389 return err; 4390 } 4391 4392 static int truncate_space_check(struct btrfs_trans_handle *trans, 4393 struct btrfs_root *root, 4394 u64 bytes_deleted) 4395 { 4396 struct btrfs_fs_info *fs_info = root->fs_info; 4397 int ret; 4398 4399 /* 4400 * This is only used to apply pressure to the enospc system, we don't 4401 * intend to use this reservation at all. 4402 */ 4403 bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted); 4404 bytes_deleted *= fs_info->nodesize; 4405 ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv, 4406 bytes_deleted, BTRFS_RESERVE_NO_FLUSH); 4407 if (!ret) { 4408 trace_btrfs_space_reservation(fs_info, "transaction", 4409 trans->transid, 4410 bytes_deleted, 1); 4411 trans->bytes_reserved += bytes_deleted; 4412 } 4413 return ret; 4414 4415 } 4416 4417 /* 4418 * Return this if we need to call truncate_block for the last bit of the 4419 * truncate. 4420 */ 4421 #define NEED_TRUNCATE_BLOCK 1 4422 4423 /* 4424 * this can truncate away extent items, csum items and directory items. 4425 * It starts at a high offset and removes keys until it can't find 4426 * any higher than new_size 4427 * 4428 * csum items that cross the new i_size are truncated to the new size 4429 * as well. 4430 * 4431 * min_type is the minimum key type to truncate down to. If set to 0, this 4432 * will kill all the items on this inode, including the INODE_ITEM_KEY. 4433 */ 4434 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, 4435 struct btrfs_root *root, 4436 struct inode *inode, 4437 u64 new_size, u32 min_type) 4438 { 4439 struct btrfs_fs_info *fs_info = root->fs_info; 4440 struct btrfs_path *path; 4441 struct extent_buffer *leaf; 4442 struct btrfs_file_extent_item *fi; 4443 struct btrfs_key key; 4444 struct btrfs_key found_key; 4445 u64 extent_start = 0; 4446 u64 extent_num_bytes = 0; 4447 u64 extent_offset = 0; 4448 u64 item_end = 0; 4449 u64 last_size = new_size; 4450 u32 found_type = (u8)-1; 4451 int found_extent; 4452 int del_item; 4453 int pending_del_nr = 0; 4454 int pending_del_slot = 0; 4455 int extent_type = -1; 4456 int ret; 4457 int err = 0; 4458 u64 ino = btrfs_ino(BTRFS_I(inode)); 4459 u64 bytes_deleted = 0; 4460 bool be_nice = false; 4461 bool should_throttle = false; 4462 bool should_end = false; 4463 4464 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); 4465 4466 /* 4467 * for non-free space inodes and ref cows, we want to back off from 4468 * time to time 4469 */ 4470 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) && 4471 test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4472 be_nice = true; 4473 4474 path = btrfs_alloc_path(); 4475 if (!path) 4476 return -ENOMEM; 4477 path->reada = READA_BACK; 4478 4479 /* 4480 * We want to drop from the next block forward in case this new size is 4481 * not block aligned since we will be keeping the last block of the 4482 * extent just the way it is. 4483 */ 4484 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4485 root == fs_info->tree_root) 4486 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size, 4487 fs_info->sectorsize), 4488 (u64)-1, 0); 4489 4490 /* 4491 * This function is also used to drop the items in the log tree before 4492 * we relog the inode, so if root != BTRFS_I(inode)->root, it means 4493 * it is used to drop the loged items. So we shouldn't kill the delayed 4494 * items. 4495 */ 4496 if (min_type == 0 && root == BTRFS_I(inode)->root) 4497 btrfs_kill_delayed_inode_items(BTRFS_I(inode)); 4498 4499 key.objectid = ino; 4500 key.offset = (u64)-1; 4501 key.type = (u8)-1; 4502 4503 search_again: 4504 /* 4505 * with a 16K leaf size and 128MB extents, you can actually queue 4506 * up a huge file in a single leaf. Most of the time that 4507 * bytes_deleted is > 0, it will be huge by the time we get here 4508 */ 4509 if (be_nice && bytes_deleted > SZ_32M) { 4510 if (btrfs_should_end_transaction(trans)) { 4511 err = -EAGAIN; 4512 goto error; 4513 } 4514 } 4515 4516 4517 path->leave_spinning = 1; 4518 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 4519 if (ret < 0) { 4520 err = ret; 4521 goto out; 4522 } 4523 4524 if (ret > 0) { 4525 /* there are no items in the tree for us to truncate, we're 4526 * done 4527 */ 4528 if (path->slots[0] == 0) 4529 goto out; 4530 path->slots[0]--; 4531 } 4532 4533 while (1) { 4534 fi = NULL; 4535 leaf = path->nodes[0]; 4536 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4537 found_type = found_key.type; 4538 4539 if (found_key.objectid != ino) 4540 break; 4541 4542 if (found_type < min_type) 4543 break; 4544 4545 item_end = found_key.offset; 4546 if (found_type == BTRFS_EXTENT_DATA_KEY) { 4547 fi = btrfs_item_ptr(leaf, path->slots[0], 4548 struct btrfs_file_extent_item); 4549 extent_type = btrfs_file_extent_type(leaf, fi); 4550 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4551 item_end += 4552 btrfs_file_extent_num_bytes(leaf, fi); 4553 4554 trace_btrfs_truncate_show_fi_regular( 4555 BTRFS_I(inode), leaf, fi, 4556 found_key.offset); 4557 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4558 item_end += btrfs_file_extent_inline_len(leaf, 4559 path->slots[0], fi); 4560 4561 trace_btrfs_truncate_show_fi_inline( 4562 BTRFS_I(inode), leaf, fi, path->slots[0], 4563 found_key.offset); 4564 } 4565 item_end--; 4566 } 4567 if (found_type > min_type) { 4568 del_item = 1; 4569 } else { 4570 if (item_end < new_size) 4571 break; 4572 if (found_key.offset >= new_size) 4573 del_item = 1; 4574 else 4575 del_item = 0; 4576 } 4577 found_extent = 0; 4578 /* FIXME, shrink the extent if the ref count is only 1 */ 4579 if (found_type != BTRFS_EXTENT_DATA_KEY) 4580 goto delete; 4581 4582 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4583 u64 num_dec; 4584 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); 4585 if (!del_item) { 4586 u64 orig_num_bytes = 4587 btrfs_file_extent_num_bytes(leaf, fi); 4588 extent_num_bytes = ALIGN(new_size - 4589 found_key.offset, 4590 fs_info->sectorsize); 4591 btrfs_set_file_extent_num_bytes(leaf, fi, 4592 extent_num_bytes); 4593 num_dec = (orig_num_bytes - 4594 extent_num_bytes); 4595 if (test_bit(BTRFS_ROOT_REF_COWS, 4596 &root->state) && 4597 extent_start != 0) 4598 inode_sub_bytes(inode, num_dec); 4599 btrfs_mark_buffer_dirty(leaf); 4600 } else { 4601 extent_num_bytes = 4602 btrfs_file_extent_disk_num_bytes(leaf, 4603 fi); 4604 extent_offset = found_key.offset - 4605 btrfs_file_extent_offset(leaf, fi); 4606 4607 /* FIXME blocksize != 4096 */ 4608 num_dec = btrfs_file_extent_num_bytes(leaf, fi); 4609 if (extent_start != 0) { 4610 found_extent = 1; 4611 if (test_bit(BTRFS_ROOT_REF_COWS, 4612 &root->state)) 4613 inode_sub_bytes(inode, num_dec); 4614 } 4615 } 4616 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4617 /* 4618 * we can't truncate inline items that have had 4619 * special encodings 4620 */ 4621 if (!del_item && 4622 btrfs_file_extent_encryption(leaf, fi) == 0 && 4623 btrfs_file_extent_other_encoding(leaf, fi) == 0 && 4624 btrfs_file_extent_compression(leaf, fi) == 0) { 4625 u32 size = (u32)(new_size - found_key.offset); 4626 4627 btrfs_set_file_extent_ram_bytes(leaf, fi, size); 4628 size = btrfs_file_extent_calc_inline_size(size); 4629 btrfs_truncate_item(root->fs_info, path, size, 1); 4630 } else if (!del_item) { 4631 /* 4632 * We have to bail so the last_size is set to 4633 * just before this extent. 4634 */ 4635 err = NEED_TRUNCATE_BLOCK; 4636 break; 4637 } 4638 4639 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4640 inode_sub_bytes(inode, item_end + 1 - new_size); 4641 } 4642 delete: 4643 if (del_item) 4644 last_size = found_key.offset; 4645 else 4646 last_size = new_size; 4647 if (del_item) { 4648 if (!pending_del_nr) { 4649 /* no pending yet, add ourselves */ 4650 pending_del_slot = path->slots[0]; 4651 pending_del_nr = 1; 4652 } else if (pending_del_nr && 4653 path->slots[0] + 1 == pending_del_slot) { 4654 /* hop on the pending chunk */ 4655 pending_del_nr++; 4656 pending_del_slot = path->slots[0]; 4657 } else { 4658 BUG(); 4659 } 4660 } else { 4661 break; 4662 } 4663 should_throttle = false; 4664 4665 if (found_extent && 4666 (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4667 root == fs_info->tree_root)) { 4668 btrfs_set_path_blocking(path); 4669 bytes_deleted += extent_num_bytes; 4670 ret = btrfs_free_extent(trans, root, extent_start, 4671 extent_num_bytes, 0, 4672 btrfs_header_owner(leaf), 4673 ino, extent_offset); 4674 BUG_ON(ret); 4675 if (btrfs_should_throttle_delayed_refs(trans, fs_info)) 4676 btrfs_async_run_delayed_refs(fs_info, 4677 trans->delayed_ref_updates * 2, 4678 trans->transid, 0); 4679 if (be_nice) { 4680 if (truncate_space_check(trans, root, 4681 extent_num_bytes)) { 4682 should_end = true; 4683 } 4684 if (btrfs_should_throttle_delayed_refs(trans, 4685 fs_info)) 4686 should_throttle = true; 4687 } 4688 } 4689 4690 if (found_type == BTRFS_INODE_ITEM_KEY) 4691 break; 4692 4693 if (path->slots[0] == 0 || 4694 path->slots[0] != pending_del_slot || 4695 should_throttle || should_end) { 4696 if (pending_del_nr) { 4697 ret = btrfs_del_items(trans, root, path, 4698 pending_del_slot, 4699 pending_del_nr); 4700 if (ret) { 4701 btrfs_abort_transaction(trans, ret); 4702 goto error; 4703 } 4704 pending_del_nr = 0; 4705 } 4706 btrfs_release_path(path); 4707 if (should_throttle) { 4708 unsigned long updates = trans->delayed_ref_updates; 4709 if (updates) { 4710 trans->delayed_ref_updates = 0; 4711 ret = btrfs_run_delayed_refs(trans, 4712 updates * 2); 4713 if (ret && !err) 4714 err = ret; 4715 } 4716 } 4717 /* 4718 * if we failed to refill our space rsv, bail out 4719 * and let the transaction restart 4720 */ 4721 if (should_end) { 4722 err = -EAGAIN; 4723 goto error; 4724 } 4725 goto search_again; 4726 } else { 4727 path->slots[0]--; 4728 } 4729 } 4730 out: 4731 if (pending_del_nr) { 4732 ret = btrfs_del_items(trans, root, path, pending_del_slot, 4733 pending_del_nr); 4734 if (ret) 4735 btrfs_abort_transaction(trans, ret); 4736 } 4737 error: 4738 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4739 ASSERT(last_size >= new_size); 4740 if (!err && last_size > new_size) 4741 last_size = new_size; 4742 btrfs_ordered_update_i_size(inode, last_size, NULL); 4743 } 4744 4745 btrfs_free_path(path); 4746 4747 if (be_nice && bytes_deleted > SZ_32M) { 4748 unsigned long updates = trans->delayed_ref_updates; 4749 if (updates) { 4750 trans->delayed_ref_updates = 0; 4751 ret = btrfs_run_delayed_refs(trans, updates * 2); 4752 if (ret && !err) 4753 err = ret; 4754 } 4755 } 4756 return err; 4757 } 4758 4759 /* 4760 * btrfs_truncate_block - read, zero a chunk and write a block 4761 * @inode - inode that we're zeroing 4762 * @from - the offset to start zeroing 4763 * @len - the length to zero, 0 to zero the entire range respective to the 4764 * offset 4765 * @front - zero up to the offset instead of from the offset on 4766 * 4767 * This will find the block for the "from" offset and cow the block and zero the 4768 * part we want to zero. This is used with truncate and hole punching. 4769 */ 4770 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len, 4771 int front) 4772 { 4773 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4774 struct address_space *mapping = inode->i_mapping; 4775 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4776 struct btrfs_ordered_extent *ordered; 4777 struct extent_state *cached_state = NULL; 4778 struct extent_changeset *data_reserved = NULL; 4779 char *kaddr; 4780 u32 blocksize = fs_info->sectorsize; 4781 pgoff_t index = from >> PAGE_SHIFT; 4782 unsigned offset = from & (blocksize - 1); 4783 struct page *page; 4784 gfp_t mask = btrfs_alloc_write_mask(mapping); 4785 int ret = 0; 4786 u64 block_start; 4787 u64 block_end; 4788 4789 if (IS_ALIGNED(offset, blocksize) && 4790 (!len || IS_ALIGNED(len, blocksize))) 4791 goto out; 4792 4793 block_start = round_down(from, blocksize); 4794 block_end = block_start + blocksize - 1; 4795 4796 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, 4797 block_start, blocksize); 4798 if (ret) 4799 goto out; 4800 4801 again: 4802 page = find_or_create_page(mapping, index, mask); 4803 if (!page) { 4804 btrfs_delalloc_release_space(inode, data_reserved, 4805 block_start, blocksize, true); 4806 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, true); 4807 ret = -ENOMEM; 4808 goto out; 4809 } 4810 4811 if (!PageUptodate(page)) { 4812 ret = btrfs_readpage(NULL, page); 4813 lock_page(page); 4814 if (page->mapping != mapping) { 4815 unlock_page(page); 4816 put_page(page); 4817 goto again; 4818 } 4819 if (!PageUptodate(page)) { 4820 ret = -EIO; 4821 goto out_unlock; 4822 } 4823 } 4824 wait_on_page_writeback(page); 4825 4826 lock_extent_bits(io_tree, block_start, block_end, &cached_state); 4827 set_page_extent_mapped(page); 4828 4829 ordered = btrfs_lookup_ordered_extent(inode, block_start); 4830 if (ordered) { 4831 unlock_extent_cached(io_tree, block_start, block_end, 4832 &cached_state); 4833 unlock_page(page); 4834 put_page(page); 4835 btrfs_start_ordered_extent(inode, ordered, 1); 4836 btrfs_put_ordered_extent(ordered); 4837 goto again; 4838 } 4839 4840 clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end, 4841 EXTENT_DIRTY | EXTENT_DELALLOC | 4842 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4843 0, 0, &cached_state); 4844 4845 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, 4846 &cached_state, 0); 4847 if (ret) { 4848 unlock_extent_cached(io_tree, block_start, block_end, 4849 &cached_state); 4850 goto out_unlock; 4851 } 4852 4853 if (offset != blocksize) { 4854 if (!len) 4855 len = blocksize - offset; 4856 kaddr = kmap(page); 4857 if (front) 4858 memset(kaddr + (block_start - page_offset(page)), 4859 0, offset); 4860 else 4861 memset(kaddr + (block_start - page_offset(page)) + offset, 4862 0, len); 4863 flush_dcache_page(page); 4864 kunmap(page); 4865 } 4866 ClearPageChecked(page); 4867 set_page_dirty(page); 4868 unlock_extent_cached(io_tree, block_start, block_end, &cached_state); 4869 4870 out_unlock: 4871 if (ret) 4872 btrfs_delalloc_release_space(inode, data_reserved, block_start, 4873 blocksize, true); 4874 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, (ret != 0)); 4875 unlock_page(page); 4876 put_page(page); 4877 out: 4878 extent_changeset_free(data_reserved); 4879 return ret; 4880 } 4881 4882 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode, 4883 u64 offset, u64 len) 4884 { 4885 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4886 struct btrfs_trans_handle *trans; 4887 int ret; 4888 4889 /* 4890 * Still need to make sure the inode looks like it's been updated so 4891 * that any holes get logged if we fsync. 4892 */ 4893 if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 4894 BTRFS_I(inode)->last_trans = fs_info->generation; 4895 BTRFS_I(inode)->last_sub_trans = root->log_transid; 4896 BTRFS_I(inode)->last_log_commit = root->last_log_commit; 4897 return 0; 4898 } 4899 4900 /* 4901 * 1 - for the one we're dropping 4902 * 1 - for the one we're adding 4903 * 1 - for updating the inode. 4904 */ 4905 trans = btrfs_start_transaction(root, 3); 4906 if (IS_ERR(trans)) 4907 return PTR_ERR(trans); 4908 4909 ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1); 4910 if (ret) { 4911 btrfs_abort_transaction(trans, ret); 4912 btrfs_end_transaction(trans); 4913 return ret; 4914 } 4915 4916 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)), 4917 offset, 0, 0, len, 0, len, 0, 0, 0); 4918 if (ret) 4919 btrfs_abort_transaction(trans, ret); 4920 else 4921 btrfs_update_inode(trans, root, inode); 4922 btrfs_end_transaction(trans); 4923 return ret; 4924 } 4925 4926 /* 4927 * This function puts in dummy file extents for the area we're creating a hole 4928 * for. So if we are truncating this file to a larger size we need to insert 4929 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 4930 * the range between oldsize and size 4931 */ 4932 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size) 4933 { 4934 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4935 struct btrfs_root *root = BTRFS_I(inode)->root; 4936 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4937 struct extent_map *em = NULL; 4938 struct extent_state *cached_state = NULL; 4939 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 4940 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 4941 u64 block_end = ALIGN(size, fs_info->sectorsize); 4942 u64 last_byte; 4943 u64 cur_offset; 4944 u64 hole_size; 4945 int err = 0; 4946 4947 /* 4948 * If our size started in the middle of a block we need to zero out the 4949 * rest of the block before we expand the i_size, otherwise we could 4950 * expose stale data. 4951 */ 4952 err = btrfs_truncate_block(inode, oldsize, 0, 0); 4953 if (err) 4954 return err; 4955 4956 if (size <= hole_start) 4957 return 0; 4958 4959 while (1) { 4960 struct btrfs_ordered_extent *ordered; 4961 4962 lock_extent_bits(io_tree, hole_start, block_end - 1, 4963 &cached_state); 4964 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start, 4965 block_end - hole_start); 4966 if (!ordered) 4967 break; 4968 unlock_extent_cached(io_tree, hole_start, block_end - 1, 4969 &cached_state); 4970 btrfs_start_ordered_extent(inode, ordered, 1); 4971 btrfs_put_ordered_extent(ordered); 4972 } 4973 4974 cur_offset = hole_start; 4975 while (1) { 4976 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset, 4977 block_end - cur_offset, 0); 4978 if (IS_ERR(em)) { 4979 err = PTR_ERR(em); 4980 em = NULL; 4981 break; 4982 } 4983 last_byte = min(extent_map_end(em), block_end); 4984 last_byte = ALIGN(last_byte, fs_info->sectorsize); 4985 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 4986 struct extent_map *hole_em; 4987 hole_size = last_byte - cur_offset; 4988 4989 err = maybe_insert_hole(root, inode, cur_offset, 4990 hole_size); 4991 if (err) 4992 break; 4993 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 4994 cur_offset + hole_size - 1, 0); 4995 hole_em = alloc_extent_map(); 4996 if (!hole_em) { 4997 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 4998 &BTRFS_I(inode)->runtime_flags); 4999 goto next; 5000 } 5001 hole_em->start = cur_offset; 5002 hole_em->len = hole_size; 5003 hole_em->orig_start = cur_offset; 5004 5005 hole_em->block_start = EXTENT_MAP_HOLE; 5006 hole_em->block_len = 0; 5007 hole_em->orig_block_len = 0; 5008 hole_em->ram_bytes = hole_size; 5009 hole_em->bdev = fs_info->fs_devices->latest_bdev; 5010 hole_em->compress_type = BTRFS_COMPRESS_NONE; 5011 hole_em->generation = fs_info->generation; 5012 5013 while (1) { 5014 write_lock(&em_tree->lock); 5015 err = add_extent_mapping(em_tree, hole_em, 1); 5016 write_unlock(&em_tree->lock); 5017 if (err != -EEXIST) 5018 break; 5019 btrfs_drop_extent_cache(BTRFS_I(inode), 5020 cur_offset, 5021 cur_offset + 5022 hole_size - 1, 0); 5023 } 5024 free_extent_map(hole_em); 5025 } 5026 next: 5027 free_extent_map(em); 5028 em = NULL; 5029 cur_offset = last_byte; 5030 if (cur_offset >= block_end) 5031 break; 5032 } 5033 free_extent_map(em); 5034 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state); 5035 return err; 5036 } 5037 5038 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 5039 { 5040 struct btrfs_root *root = BTRFS_I(inode)->root; 5041 struct btrfs_trans_handle *trans; 5042 loff_t oldsize = i_size_read(inode); 5043 loff_t newsize = attr->ia_size; 5044 int mask = attr->ia_valid; 5045 int ret; 5046 5047 /* 5048 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 5049 * special case where we need to update the times despite not having 5050 * these flags set. For all other operations the VFS set these flags 5051 * explicitly if it wants a timestamp update. 5052 */ 5053 if (newsize != oldsize) { 5054 inode_inc_iversion(inode); 5055 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) 5056 inode->i_ctime = inode->i_mtime = 5057 current_time(inode); 5058 } 5059 5060 if (newsize > oldsize) { 5061 /* 5062 * Don't do an expanding truncate while snapshotting is ongoing. 5063 * This is to ensure the snapshot captures a fully consistent 5064 * state of this file - if the snapshot captures this expanding 5065 * truncation, it must capture all writes that happened before 5066 * this truncation. 5067 */ 5068 btrfs_wait_for_snapshot_creation(root); 5069 ret = btrfs_cont_expand(inode, oldsize, newsize); 5070 if (ret) { 5071 btrfs_end_write_no_snapshotting(root); 5072 return ret; 5073 } 5074 5075 trans = btrfs_start_transaction(root, 1); 5076 if (IS_ERR(trans)) { 5077 btrfs_end_write_no_snapshotting(root); 5078 return PTR_ERR(trans); 5079 } 5080 5081 i_size_write(inode, newsize); 5082 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL); 5083 pagecache_isize_extended(inode, oldsize, newsize); 5084 ret = btrfs_update_inode(trans, root, inode); 5085 btrfs_end_write_no_snapshotting(root); 5086 btrfs_end_transaction(trans); 5087 } else { 5088 5089 /* 5090 * We're truncating a file that used to have good data down to 5091 * zero. Make sure it gets into the ordered flush list so that 5092 * any new writes get down to disk quickly. 5093 */ 5094 if (newsize == 0) 5095 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE, 5096 &BTRFS_I(inode)->runtime_flags); 5097 5098 /* 5099 * 1 for the orphan item we're going to add 5100 * 1 for the orphan item deletion. 5101 */ 5102 trans = btrfs_start_transaction(root, 2); 5103 if (IS_ERR(trans)) 5104 return PTR_ERR(trans); 5105 5106 /* 5107 * We need to do this in case we fail at _any_ point during the 5108 * actual truncate. Once we do the truncate_setsize we could 5109 * invalidate pages which forces any outstanding ordered io to 5110 * be instantly completed which will give us extents that need 5111 * to be truncated. If we fail to get an orphan inode down we 5112 * could have left over extents that were never meant to live, 5113 * so we need to guarantee from this point on that everything 5114 * will be consistent. 5115 */ 5116 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 5117 btrfs_end_transaction(trans); 5118 if (ret) 5119 return ret; 5120 5121 /* we don't support swapfiles, so vmtruncate shouldn't fail */ 5122 truncate_setsize(inode, newsize); 5123 5124 /* Disable nonlocked read DIO to avoid the end less truncate */ 5125 btrfs_inode_block_unlocked_dio(BTRFS_I(inode)); 5126 inode_dio_wait(inode); 5127 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode)); 5128 5129 ret = btrfs_truncate(inode, newsize == oldsize); 5130 if (ret && inode->i_nlink) { 5131 int err; 5132 5133 /* To get a stable disk_i_size */ 5134 err = btrfs_wait_ordered_range(inode, 0, (u64)-1); 5135 if (err) { 5136 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5137 return err; 5138 } 5139 5140 /* 5141 * failed to truncate, disk_i_size is only adjusted down 5142 * as we remove extents, so it should represent the true 5143 * size of the inode, so reset the in memory size and 5144 * delete our orphan entry. 5145 */ 5146 trans = btrfs_join_transaction(root); 5147 if (IS_ERR(trans)) { 5148 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5149 return ret; 5150 } 5151 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 5152 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 5153 if (err) 5154 btrfs_abort_transaction(trans, err); 5155 btrfs_end_transaction(trans); 5156 } 5157 } 5158 5159 return ret; 5160 } 5161 5162 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) 5163 { 5164 struct inode *inode = d_inode(dentry); 5165 struct btrfs_root *root = BTRFS_I(inode)->root; 5166 int err; 5167 5168 if (btrfs_root_readonly(root)) 5169 return -EROFS; 5170 5171 err = setattr_prepare(dentry, attr); 5172 if (err) 5173 return err; 5174 5175 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 5176 err = btrfs_setsize(inode, attr); 5177 if (err) 5178 return err; 5179 } 5180 5181 if (attr->ia_valid) { 5182 setattr_copy(inode, attr); 5183 inode_inc_iversion(inode); 5184 err = btrfs_dirty_inode(inode); 5185 5186 if (!err && attr->ia_valid & ATTR_MODE) 5187 err = posix_acl_chmod(inode, inode->i_mode); 5188 } 5189 5190 return err; 5191 } 5192 5193 /* 5194 * While truncating the inode pages during eviction, we get the VFS calling 5195 * btrfs_invalidatepage() against each page of the inode. This is slow because 5196 * the calls to btrfs_invalidatepage() result in a huge amount of calls to 5197 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting 5198 * extent_state structures over and over, wasting lots of time. 5199 * 5200 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all 5201 * those expensive operations on a per page basis and do only the ordered io 5202 * finishing, while we release here the extent_map and extent_state structures, 5203 * without the excessive merging and splitting. 5204 */ 5205 static void evict_inode_truncate_pages(struct inode *inode) 5206 { 5207 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 5208 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; 5209 struct rb_node *node; 5210 5211 ASSERT(inode->i_state & I_FREEING); 5212 truncate_inode_pages_final(&inode->i_data); 5213 5214 write_lock(&map_tree->lock); 5215 while (!RB_EMPTY_ROOT(&map_tree->map)) { 5216 struct extent_map *em; 5217 5218 node = rb_first(&map_tree->map); 5219 em = rb_entry(node, struct extent_map, rb_node); 5220 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 5221 clear_bit(EXTENT_FLAG_LOGGING, &em->flags); 5222 remove_extent_mapping(map_tree, em); 5223 free_extent_map(em); 5224 if (need_resched()) { 5225 write_unlock(&map_tree->lock); 5226 cond_resched(); 5227 write_lock(&map_tree->lock); 5228 } 5229 } 5230 write_unlock(&map_tree->lock); 5231 5232 /* 5233 * Keep looping until we have no more ranges in the io tree. 5234 * We can have ongoing bios started by readpages (called from readahead) 5235 * that have their endio callback (extent_io.c:end_bio_extent_readpage) 5236 * still in progress (unlocked the pages in the bio but did not yet 5237 * unlocked the ranges in the io tree). Therefore this means some 5238 * ranges can still be locked and eviction started because before 5239 * submitting those bios, which are executed by a separate task (work 5240 * queue kthread), inode references (inode->i_count) were not taken 5241 * (which would be dropped in the end io callback of each bio). 5242 * Therefore here we effectively end up waiting for those bios and 5243 * anyone else holding locked ranges without having bumped the inode's 5244 * reference count - if we don't do it, when they access the inode's 5245 * io_tree to unlock a range it may be too late, leading to an 5246 * use-after-free issue. 5247 */ 5248 spin_lock(&io_tree->lock); 5249 while (!RB_EMPTY_ROOT(&io_tree->state)) { 5250 struct extent_state *state; 5251 struct extent_state *cached_state = NULL; 5252 u64 start; 5253 u64 end; 5254 5255 node = rb_first(&io_tree->state); 5256 state = rb_entry(node, struct extent_state, rb_node); 5257 start = state->start; 5258 end = state->end; 5259 spin_unlock(&io_tree->lock); 5260 5261 lock_extent_bits(io_tree, start, end, &cached_state); 5262 5263 /* 5264 * If still has DELALLOC flag, the extent didn't reach disk, 5265 * and its reserved space won't be freed by delayed_ref. 5266 * So we need to free its reserved space here. 5267 * (Refer to comment in btrfs_invalidatepage, case 2) 5268 * 5269 * Note, end is the bytenr of last byte, so we need + 1 here. 5270 */ 5271 if (state->state & EXTENT_DELALLOC) 5272 btrfs_qgroup_free_data(inode, NULL, start, end - start + 1); 5273 5274 clear_extent_bit(io_tree, start, end, 5275 EXTENT_LOCKED | EXTENT_DIRTY | 5276 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 5277 EXTENT_DEFRAG, 1, 1, &cached_state); 5278 5279 cond_resched(); 5280 spin_lock(&io_tree->lock); 5281 } 5282 spin_unlock(&io_tree->lock); 5283 } 5284 5285 void btrfs_evict_inode(struct inode *inode) 5286 { 5287 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5288 struct btrfs_trans_handle *trans; 5289 struct btrfs_root *root = BTRFS_I(inode)->root; 5290 struct btrfs_block_rsv *rsv, *global_rsv; 5291 int steal_from_global = 0; 5292 u64 min_size; 5293 int ret; 5294 5295 trace_btrfs_inode_evict(inode); 5296 5297 if (!root) { 5298 clear_inode(inode); 5299 return; 5300 } 5301 5302 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 5303 5304 evict_inode_truncate_pages(inode); 5305 5306 if (inode->i_nlink && 5307 ((btrfs_root_refs(&root->root_item) != 0 && 5308 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 5309 btrfs_is_free_space_inode(BTRFS_I(inode)))) 5310 goto no_delete; 5311 5312 if (is_bad_inode(inode)) { 5313 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5314 goto no_delete; 5315 } 5316 /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */ 5317 if (!special_file(inode->i_mode)) 5318 btrfs_wait_ordered_range(inode, 0, (u64)-1); 5319 5320 btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1); 5321 5322 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 5323 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 5324 &BTRFS_I(inode)->runtime_flags)); 5325 goto no_delete; 5326 } 5327 5328 if (inode->i_nlink > 0) { 5329 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5330 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 5331 goto no_delete; 5332 } 5333 5334 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); 5335 if (ret) { 5336 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5337 goto no_delete; 5338 } 5339 5340 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 5341 if (!rsv) { 5342 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5343 goto no_delete; 5344 } 5345 rsv->size = min_size; 5346 rsv->failfast = 1; 5347 global_rsv = &fs_info->global_block_rsv; 5348 5349 btrfs_i_size_write(BTRFS_I(inode), 0); 5350 5351 /* 5352 * This is a bit simpler than btrfs_truncate since we've already 5353 * reserved our space for our orphan item in the unlink, so we just 5354 * need to reserve some slack space in case we add bytes and update 5355 * inode item when doing the truncate. 5356 */ 5357 while (1) { 5358 ret = btrfs_block_rsv_refill(root, rsv, min_size, 5359 BTRFS_RESERVE_FLUSH_LIMIT); 5360 5361 /* 5362 * Try and steal from the global reserve since we will 5363 * likely not use this space anyway, we want to try as 5364 * hard as possible to get this to work. 5365 */ 5366 if (ret) 5367 steal_from_global++; 5368 else 5369 steal_from_global = 0; 5370 ret = 0; 5371 5372 /* 5373 * steal_from_global == 0: we reserved stuff, hooray! 5374 * steal_from_global == 1: we didn't reserve stuff, boo! 5375 * steal_from_global == 2: we've committed, still not a lot of 5376 * room but maybe we'll have room in the global reserve this 5377 * time. 5378 * steal_from_global == 3: abandon all hope! 5379 */ 5380 if (steal_from_global > 2) { 5381 btrfs_warn(fs_info, 5382 "Could not get space for a delete, will truncate on mount %d", 5383 ret); 5384 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5385 btrfs_free_block_rsv(fs_info, rsv); 5386 goto no_delete; 5387 } 5388 5389 trans = btrfs_join_transaction(root); 5390 if (IS_ERR(trans)) { 5391 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5392 btrfs_free_block_rsv(fs_info, rsv); 5393 goto no_delete; 5394 } 5395 5396 /* 5397 * We can't just steal from the global reserve, we need to make 5398 * sure there is room to do it, if not we need to commit and try 5399 * again. 5400 */ 5401 if (steal_from_global) { 5402 if (!btrfs_check_space_for_delayed_refs(trans, fs_info)) 5403 ret = btrfs_block_rsv_migrate(global_rsv, rsv, 5404 min_size, 0); 5405 else 5406 ret = -ENOSPC; 5407 } 5408 5409 /* 5410 * Couldn't steal from the global reserve, we have too much 5411 * pending stuff built up, commit the transaction and try it 5412 * again. 5413 */ 5414 if (ret) { 5415 ret = btrfs_commit_transaction(trans); 5416 if (ret) { 5417 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5418 btrfs_free_block_rsv(fs_info, rsv); 5419 goto no_delete; 5420 } 5421 continue; 5422 } else { 5423 steal_from_global = 0; 5424 } 5425 5426 trans->block_rsv = rsv; 5427 5428 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0); 5429 if (ret != -ENOSPC && ret != -EAGAIN) 5430 break; 5431 5432 trans->block_rsv = &fs_info->trans_block_rsv; 5433 btrfs_end_transaction(trans); 5434 trans = NULL; 5435 btrfs_btree_balance_dirty(fs_info); 5436 } 5437 5438 btrfs_free_block_rsv(fs_info, rsv); 5439 5440 /* 5441 * Errors here aren't a big deal, it just means we leave orphan items 5442 * in the tree. They will be cleaned up on the next mount. 5443 */ 5444 if (ret == 0) { 5445 trans->block_rsv = root->orphan_block_rsv; 5446 btrfs_orphan_del(trans, BTRFS_I(inode)); 5447 } else { 5448 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5449 } 5450 5451 trans->block_rsv = &fs_info->trans_block_rsv; 5452 if (!(root == fs_info->tree_root || 5453 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)) 5454 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode))); 5455 5456 btrfs_end_transaction(trans); 5457 btrfs_btree_balance_dirty(fs_info); 5458 no_delete: 5459 btrfs_remove_delayed_node(BTRFS_I(inode)); 5460 clear_inode(inode); 5461 } 5462 5463 /* 5464 * this returns the key found in the dir entry in the location pointer. 5465 * If no dir entries were found, returns -ENOENT. 5466 * If found a corrupted location in dir entry, returns -EUCLEAN. 5467 */ 5468 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry, 5469 struct btrfs_key *location) 5470 { 5471 const char *name = dentry->d_name.name; 5472 int namelen = dentry->d_name.len; 5473 struct btrfs_dir_item *di; 5474 struct btrfs_path *path; 5475 struct btrfs_root *root = BTRFS_I(dir)->root; 5476 int ret = 0; 5477 5478 path = btrfs_alloc_path(); 5479 if (!path) 5480 return -ENOMEM; 5481 5482 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)), 5483 name, namelen, 0); 5484 if (!di) { 5485 ret = -ENOENT; 5486 goto out; 5487 } 5488 if (IS_ERR(di)) { 5489 ret = PTR_ERR(di); 5490 goto out; 5491 } 5492 5493 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5494 if (location->type != BTRFS_INODE_ITEM_KEY && 5495 location->type != BTRFS_ROOT_ITEM_KEY) { 5496 ret = -EUCLEAN; 5497 btrfs_warn(root->fs_info, 5498 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))", 5499 __func__, name, btrfs_ino(BTRFS_I(dir)), 5500 location->objectid, location->type, location->offset); 5501 } 5502 out: 5503 btrfs_free_path(path); 5504 return ret; 5505 } 5506 5507 /* 5508 * when we hit a tree root in a directory, the btrfs part of the inode 5509 * needs to be changed to reflect the root directory of the tree root. This 5510 * is kind of like crossing a mount point. 5511 */ 5512 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5513 struct inode *dir, 5514 struct dentry *dentry, 5515 struct btrfs_key *location, 5516 struct btrfs_root **sub_root) 5517 { 5518 struct btrfs_path *path; 5519 struct btrfs_root *new_root; 5520 struct btrfs_root_ref *ref; 5521 struct extent_buffer *leaf; 5522 struct btrfs_key key; 5523 int ret; 5524 int err = 0; 5525 5526 path = btrfs_alloc_path(); 5527 if (!path) { 5528 err = -ENOMEM; 5529 goto out; 5530 } 5531 5532 err = -ENOENT; 5533 key.objectid = BTRFS_I(dir)->root->root_key.objectid; 5534 key.type = BTRFS_ROOT_REF_KEY; 5535 key.offset = location->objectid; 5536 5537 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5538 if (ret) { 5539 if (ret < 0) 5540 err = ret; 5541 goto out; 5542 } 5543 5544 leaf = path->nodes[0]; 5545 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5546 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) || 5547 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len) 5548 goto out; 5549 5550 ret = memcmp_extent_buffer(leaf, dentry->d_name.name, 5551 (unsigned long)(ref + 1), 5552 dentry->d_name.len); 5553 if (ret) 5554 goto out; 5555 5556 btrfs_release_path(path); 5557 5558 new_root = btrfs_read_fs_root_no_name(fs_info, location); 5559 if (IS_ERR(new_root)) { 5560 err = PTR_ERR(new_root); 5561 goto out; 5562 } 5563 5564 *sub_root = new_root; 5565 location->objectid = btrfs_root_dirid(&new_root->root_item); 5566 location->type = BTRFS_INODE_ITEM_KEY; 5567 location->offset = 0; 5568 err = 0; 5569 out: 5570 btrfs_free_path(path); 5571 return err; 5572 } 5573 5574 static void inode_tree_add(struct inode *inode) 5575 { 5576 struct btrfs_root *root = BTRFS_I(inode)->root; 5577 struct btrfs_inode *entry; 5578 struct rb_node **p; 5579 struct rb_node *parent; 5580 struct rb_node *new = &BTRFS_I(inode)->rb_node; 5581 u64 ino = btrfs_ino(BTRFS_I(inode)); 5582 5583 if (inode_unhashed(inode)) 5584 return; 5585 parent = NULL; 5586 spin_lock(&root->inode_lock); 5587 p = &root->inode_tree.rb_node; 5588 while (*p) { 5589 parent = *p; 5590 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5591 5592 if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5593 p = &parent->rb_left; 5594 else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5595 p = &parent->rb_right; 5596 else { 5597 WARN_ON(!(entry->vfs_inode.i_state & 5598 (I_WILL_FREE | I_FREEING))); 5599 rb_replace_node(parent, new, &root->inode_tree); 5600 RB_CLEAR_NODE(parent); 5601 spin_unlock(&root->inode_lock); 5602 return; 5603 } 5604 } 5605 rb_link_node(new, parent, p); 5606 rb_insert_color(new, &root->inode_tree); 5607 spin_unlock(&root->inode_lock); 5608 } 5609 5610 static void inode_tree_del(struct inode *inode) 5611 { 5612 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5613 struct btrfs_root *root = BTRFS_I(inode)->root; 5614 int empty = 0; 5615 5616 spin_lock(&root->inode_lock); 5617 if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) { 5618 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree); 5619 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node); 5620 empty = RB_EMPTY_ROOT(&root->inode_tree); 5621 } 5622 spin_unlock(&root->inode_lock); 5623 5624 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5625 synchronize_srcu(&fs_info->subvol_srcu); 5626 spin_lock(&root->inode_lock); 5627 empty = RB_EMPTY_ROOT(&root->inode_tree); 5628 spin_unlock(&root->inode_lock); 5629 if (empty) 5630 btrfs_add_dead_root(root); 5631 } 5632 } 5633 5634 void btrfs_invalidate_inodes(struct btrfs_root *root) 5635 { 5636 struct btrfs_fs_info *fs_info = root->fs_info; 5637 struct rb_node *node; 5638 struct rb_node *prev; 5639 struct btrfs_inode *entry; 5640 struct inode *inode; 5641 u64 objectid = 0; 5642 5643 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 5644 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 5645 5646 spin_lock(&root->inode_lock); 5647 again: 5648 node = root->inode_tree.rb_node; 5649 prev = NULL; 5650 while (node) { 5651 prev = node; 5652 entry = rb_entry(node, struct btrfs_inode, rb_node); 5653 5654 if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5655 node = node->rb_left; 5656 else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5657 node = node->rb_right; 5658 else 5659 break; 5660 } 5661 if (!node) { 5662 while (prev) { 5663 entry = rb_entry(prev, struct btrfs_inode, rb_node); 5664 if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) { 5665 node = prev; 5666 break; 5667 } 5668 prev = rb_next(prev); 5669 } 5670 } 5671 while (node) { 5672 entry = rb_entry(node, struct btrfs_inode, rb_node); 5673 objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1; 5674 inode = igrab(&entry->vfs_inode); 5675 if (inode) { 5676 spin_unlock(&root->inode_lock); 5677 if (atomic_read(&inode->i_count) > 1) 5678 d_prune_aliases(inode); 5679 /* 5680 * btrfs_drop_inode will have it removed from 5681 * the inode cache when its usage count 5682 * hits zero. 5683 */ 5684 iput(inode); 5685 cond_resched(); 5686 spin_lock(&root->inode_lock); 5687 goto again; 5688 } 5689 5690 if (cond_resched_lock(&root->inode_lock)) 5691 goto again; 5692 5693 node = rb_next(node); 5694 } 5695 spin_unlock(&root->inode_lock); 5696 } 5697 5698 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5699 { 5700 struct btrfs_iget_args *args = p; 5701 inode->i_ino = args->location->objectid; 5702 memcpy(&BTRFS_I(inode)->location, args->location, 5703 sizeof(*args->location)); 5704 BTRFS_I(inode)->root = args->root; 5705 return 0; 5706 } 5707 5708 static int btrfs_find_actor(struct inode *inode, void *opaque) 5709 { 5710 struct btrfs_iget_args *args = opaque; 5711 return args->location->objectid == BTRFS_I(inode)->location.objectid && 5712 args->root == BTRFS_I(inode)->root; 5713 } 5714 5715 static struct inode *btrfs_iget_locked(struct super_block *s, 5716 struct btrfs_key *location, 5717 struct btrfs_root *root) 5718 { 5719 struct inode *inode; 5720 struct btrfs_iget_args args; 5721 unsigned long hashval = btrfs_inode_hash(location->objectid, root); 5722 5723 args.location = location; 5724 args.root = root; 5725 5726 inode = iget5_locked(s, hashval, btrfs_find_actor, 5727 btrfs_init_locked_inode, 5728 (void *)&args); 5729 return inode; 5730 } 5731 5732 /* Get an inode object given its location and corresponding root. 5733 * Returns in *is_new if the inode was read from disk 5734 */ 5735 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location, 5736 struct btrfs_root *root, int *new) 5737 { 5738 struct inode *inode; 5739 5740 inode = btrfs_iget_locked(s, location, root); 5741 if (!inode) 5742 return ERR_PTR(-ENOMEM); 5743 5744 if (inode->i_state & I_NEW) { 5745 int ret; 5746 5747 ret = btrfs_read_locked_inode(inode); 5748 if (!is_bad_inode(inode)) { 5749 inode_tree_add(inode); 5750 unlock_new_inode(inode); 5751 if (new) 5752 *new = 1; 5753 } else { 5754 unlock_new_inode(inode); 5755 iput(inode); 5756 ASSERT(ret < 0); 5757 inode = ERR_PTR(ret < 0 ? ret : -ESTALE); 5758 } 5759 } 5760 5761 return inode; 5762 } 5763 5764 static struct inode *new_simple_dir(struct super_block *s, 5765 struct btrfs_key *key, 5766 struct btrfs_root *root) 5767 { 5768 struct inode *inode = new_inode(s); 5769 5770 if (!inode) 5771 return ERR_PTR(-ENOMEM); 5772 5773 BTRFS_I(inode)->root = root; 5774 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5775 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5776 5777 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5778 inode->i_op = &btrfs_dir_ro_inode_operations; 5779 inode->i_opflags &= ~IOP_XATTR; 5780 inode->i_fop = &simple_dir_operations; 5781 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5782 inode->i_mtime = current_time(inode); 5783 inode->i_atime = inode->i_mtime; 5784 inode->i_ctime = inode->i_mtime; 5785 BTRFS_I(inode)->i_otime = inode->i_mtime; 5786 5787 return inode; 5788 } 5789 5790 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5791 { 5792 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 5793 struct inode *inode; 5794 struct btrfs_root *root = BTRFS_I(dir)->root; 5795 struct btrfs_root *sub_root = root; 5796 struct btrfs_key location; 5797 int index; 5798 int ret = 0; 5799 5800 if (dentry->d_name.len > BTRFS_NAME_LEN) 5801 return ERR_PTR(-ENAMETOOLONG); 5802 5803 ret = btrfs_inode_by_name(dir, dentry, &location); 5804 if (ret < 0) 5805 return ERR_PTR(ret); 5806 5807 if (location.type == BTRFS_INODE_ITEM_KEY) { 5808 inode = btrfs_iget(dir->i_sb, &location, root, NULL); 5809 return inode; 5810 } 5811 5812 index = srcu_read_lock(&fs_info->subvol_srcu); 5813 ret = fixup_tree_root_location(fs_info, dir, dentry, 5814 &location, &sub_root); 5815 if (ret < 0) { 5816 if (ret != -ENOENT) 5817 inode = ERR_PTR(ret); 5818 else 5819 inode = new_simple_dir(dir->i_sb, &location, sub_root); 5820 } else { 5821 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL); 5822 } 5823 srcu_read_unlock(&fs_info->subvol_srcu, index); 5824 5825 if (!IS_ERR(inode) && root != sub_root) { 5826 down_read(&fs_info->cleanup_work_sem); 5827 if (!sb_rdonly(inode->i_sb)) 5828 ret = btrfs_orphan_cleanup(sub_root); 5829 up_read(&fs_info->cleanup_work_sem); 5830 if (ret) { 5831 iput(inode); 5832 inode = ERR_PTR(ret); 5833 } 5834 } 5835 5836 return inode; 5837 } 5838 5839 static int btrfs_dentry_delete(const struct dentry *dentry) 5840 { 5841 struct btrfs_root *root; 5842 struct inode *inode = d_inode(dentry); 5843 5844 if (!inode && !IS_ROOT(dentry)) 5845 inode = d_inode(dentry->d_parent); 5846 5847 if (inode) { 5848 root = BTRFS_I(inode)->root; 5849 if (btrfs_root_refs(&root->root_item) == 0) 5850 return 1; 5851 5852 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5853 return 1; 5854 } 5855 return 0; 5856 } 5857 5858 static void btrfs_dentry_release(struct dentry *dentry) 5859 { 5860 kfree(dentry->d_fsdata); 5861 } 5862 5863 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5864 unsigned int flags) 5865 { 5866 struct inode *inode; 5867 5868 inode = btrfs_lookup_dentry(dir, dentry); 5869 if (IS_ERR(inode)) { 5870 if (PTR_ERR(inode) == -ENOENT) 5871 inode = NULL; 5872 else 5873 return ERR_CAST(inode); 5874 } 5875 5876 return d_splice_alias(inode, dentry); 5877 } 5878 5879 unsigned char btrfs_filetype_table[] = { 5880 DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK 5881 }; 5882 5883 /* 5884 * All this infrastructure exists because dir_emit can fault, and we are holding 5885 * the tree lock when doing readdir. For now just allocate a buffer and copy 5886 * our information into that, and then dir_emit from the buffer. This is 5887 * similar to what NFS does, only we don't keep the buffer around in pagecache 5888 * because I'm afraid I'll mess that up. Long term we need to make filldir do 5889 * copy_to_user_inatomic so we don't have to worry about page faulting under the 5890 * tree lock. 5891 */ 5892 static int btrfs_opendir(struct inode *inode, struct file *file) 5893 { 5894 struct btrfs_file_private *private; 5895 5896 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL); 5897 if (!private) 5898 return -ENOMEM; 5899 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 5900 if (!private->filldir_buf) { 5901 kfree(private); 5902 return -ENOMEM; 5903 } 5904 file->private_data = private; 5905 return 0; 5906 } 5907 5908 struct dir_entry { 5909 u64 ino; 5910 u64 offset; 5911 unsigned type; 5912 int name_len; 5913 }; 5914 5915 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) 5916 { 5917 while (entries--) { 5918 struct dir_entry *entry = addr; 5919 char *name = (char *)(entry + 1); 5920 5921 ctx->pos = entry->offset; 5922 if (!dir_emit(ctx, name, entry->name_len, entry->ino, 5923 entry->type)) 5924 return 1; 5925 addr += sizeof(struct dir_entry) + entry->name_len; 5926 ctx->pos++; 5927 } 5928 return 0; 5929 } 5930 5931 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5932 { 5933 struct inode *inode = file_inode(file); 5934 struct btrfs_root *root = BTRFS_I(inode)->root; 5935 struct btrfs_file_private *private = file->private_data; 5936 struct btrfs_dir_item *di; 5937 struct btrfs_key key; 5938 struct btrfs_key found_key; 5939 struct btrfs_path *path; 5940 void *addr; 5941 struct list_head ins_list; 5942 struct list_head del_list; 5943 int ret; 5944 struct extent_buffer *leaf; 5945 int slot; 5946 char *name_ptr; 5947 int name_len; 5948 int entries = 0; 5949 int total_len = 0; 5950 bool put = false; 5951 struct btrfs_key location; 5952 5953 if (!dir_emit_dots(file, ctx)) 5954 return 0; 5955 5956 path = btrfs_alloc_path(); 5957 if (!path) 5958 return -ENOMEM; 5959 5960 addr = private->filldir_buf; 5961 path->reada = READA_FORWARD; 5962 5963 INIT_LIST_HEAD(&ins_list); 5964 INIT_LIST_HEAD(&del_list); 5965 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list); 5966 5967 again: 5968 key.type = BTRFS_DIR_INDEX_KEY; 5969 key.offset = ctx->pos; 5970 key.objectid = btrfs_ino(BTRFS_I(inode)); 5971 5972 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5973 if (ret < 0) 5974 goto err; 5975 5976 while (1) { 5977 struct dir_entry *entry; 5978 5979 leaf = path->nodes[0]; 5980 slot = path->slots[0]; 5981 if (slot >= btrfs_header_nritems(leaf)) { 5982 ret = btrfs_next_leaf(root, path); 5983 if (ret < 0) 5984 goto err; 5985 else if (ret > 0) 5986 break; 5987 continue; 5988 } 5989 5990 btrfs_item_key_to_cpu(leaf, &found_key, slot); 5991 5992 if (found_key.objectid != key.objectid) 5993 break; 5994 if (found_key.type != BTRFS_DIR_INDEX_KEY) 5995 break; 5996 if (found_key.offset < ctx->pos) 5997 goto next; 5998 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 5999 goto next; 6000 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); 6001 name_len = btrfs_dir_name_len(leaf, di); 6002 if ((total_len + sizeof(struct dir_entry) + name_len) >= 6003 PAGE_SIZE) { 6004 btrfs_release_path(path); 6005 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6006 if (ret) 6007 goto nopos; 6008 addr = private->filldir_buf; 6009 entries = 0; 6010 total_len = 0; 6011 goto again; 6012 } 6013 6014 entry = addr; 6015 entry->name_len = name_len; 6016 name_ptr = (char *)(entry + 1); 6017 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1), 6018 name_len); 6019 entry->type = btrfs_filetype_table[btrfs_dir_type(leaf, di)]; 6020 btrfs_dir_item_key_to_cpu(leaf, di, &location); 6021 entry->ino = location.objectid; 6022 entry->offset = found_key.offset; 6023 entries++; 6024 addr += sizeof(struct dir_entry) + name_len; 6025 total_len += sizeof(struct dir_entry) + name_len; 6026 next: 6027 path->slots[0]++; 6028 } 6029 btrfs_release_path(path); 6030 6031 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6032 if (ret) 6033 goto nopos; 6034 6035 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 6036 if (ret) 6037 goto nopos; 6038 6039 /* 6040 * Stop new entries from being returned after we return the last 6041 * entry. 6042 * 6043 * New directory entries are assigned a strictly increasing 6044 * offset. This means that new entries created during readdir 6045 * are *guaranteed* to be seen in the future by that readdir. 6046 * This has broken buggy programs which operate on names as 6047 * they're returned by readdir. Until we re-use freed offsets 6048 * we have this hack to stop new entries from being returned 6049 * under the assumption that they'll never reach this huge 6050 * offset. 6051 * 6052 * This is being careful not to overflow 32bit loff_t unless the 6053 * last entry requires it because doing so has broken 32bit apps 6054 * in the past. 6055 */ 6056 if (ctx->pos >= INT_MAX) 6057 ctx->pos = LLONG_MAX; 6058 else 6059 ctx->pos = INT_MAX; 6060 nopos: 6061 ret = 0; 6062 err: 6063 if (put) 6064 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list); 6065 btrfs_free_path(path); 6066 return ret; 6067 } 6068 6069 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc) 6070 { 6071 struct btrfs_root *root = BTRFS_I(inode)->root; 6072 struct btrfs_trans_handle *trans; 6073 int ret = 0; 6074 bool nolock = false; 6075 6076 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 6077 return 0; 6078 6079 if (btrfs_fs_closing(root->fs_info) && 6080 btrfs_is_free_space_inode(BTRFS_I(inode))) 6081 nolock = true; 6082 6083 if (wbc->sync_mode == WB_SYNC_ALL) { 6084 if (nolock) 6085 trans = btrfs_join_transaction_nolock(root); 6086 else 6087 trans = btrfs_join_transaction(root); 6088 if (IS_ERR(trans)) 6089 return PTR_ERR(trans); 6090 ret = btrfs_commit_transaction(trans); 6091 } 6092 return ret; 6093 } 6094 6095 /* 6096 * This is somewhat expensive, updating the tree every time the 6097 * inode changes. But, it is most likely to find the inode in cache. 6098 * FIXME, needs more benchmarking...there are no reasons other than performance 6099 * to keep or drop this code. 6100 */ 6101 static int btrfs_dirty_inode(struct inode *inode) 6102 { 6103 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6104 struct btrfs_root *root = BTRFS_I(inode)->root; 6105 struct btrfs_trans_handle *trans; 6106 int ret; 6107 6108 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 6109 return 0; 6110 6111 trans = btrfs_join_transaction(root); 6112 if (IS_ERR(trans)) 6113 return PTR_ERR(trans); 6114 6115 ret = btrfs_update_inode(trans, root, inode); 6116 if (ret && ret == -ENOSPC) { 6117 /* whoops, lets try again with the full transaction */ 6118 btrfs_end_transaction(trans); 6119 trans = btrfs_start_transaction(root, 1); 6120 if (IS_ERR(trans)) 6121 return PTR_ERR(trans); 6122 6123 ret = btrfs_update_inode(trans, root, inode); 6124 } 6125 btrfs_end_transaction(trans); 6126 if (BTRFS_I(inode)->delayed_node) 6127 btrfs_balance_delayed_items(fs_info); 6128 6129 return ret; 6130 } 6131 6132 /* 6133 * This is a copy of file_update_time. We need this so we can return error on 6134 * ENOSPC for updating the inode in the case of file write and mmap writes. 6135 */ 6136 static int btrfs_update_time(struct inode *inode, struct timespec *now, 6137 int flags) 6138 { 6139 struct btrfs_root *root = BTRFS_I(inode)->root; 6140 bool dirty = flags & ~S_VERSION; 6141 6142 if (btrfs_root_readonly(root)) 6143 return -EROFS; 6144 6145 if (flags & S_VERSION) 6146 dirty |= inode_maybe_inc_iversion(inode, dirty); 6147 if (flags & S_CTIME) 6148 inode->i_ctime = *now; 6149 if (flags & S_MTIME) 6150 inode->i_mtime = *now; 6151 if (flags & S_ATIME) 6152 inode->i_atime = *now; 6153 return dirty ? btrfs_dirty_inode(inode) : 0; 6154 } 6155 6156 /* 6157 * find the highest existing sequence number in a directory 6158 * and then set the in-memory index_cnt variable to reflect 6159 * free sequence numbers 6160 */ 6161 static int btrfs_set_inode_index_count(struct btrfs_inode *inode) 6162 { 6163 struct btrfs_root *root = inode->root; 6164 struct btrfs_key key, found_key; 6165 struct btrfs_path *path; 6166 struct extent_buffer *leaf; 6167 int ret; 6168 6169 key.objectid = btrfs_ino(inode); 6170 key.type = BTRFS_DIR_INDEX_KEY; 6171 key.offset = (u64)-1; 6172 6173 path = btrfs_alloc_path(); 6174 if (!path) 6175 return -ENOMEM; 6176 6177 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 6178 if (ret < 0) 6179 goto out; 6180 /* FIXME: we should be able to handle this */ 6181 if (ret == 0) 6182 goto out; 6183 ret = 0; 6184 6185 /* 6186 * MAGIC NUMBER EXPLANATION: 6187 * since we search a directory based on f_pos we have to start at 2 6188 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody 6189 * else has to start at 2 6190 */ 6191 if (path->slots[0] == 0) { 6192 inode->index_cnt = 2; 6193 goto out; 6194 } 6195 6196 path->slots[0]--; 6197 6198 leaf = path->nodes[0]; 6199 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6200 6201 if (found_key.objectid != btrfs_ino(inode) || 6202 found_key.type != BTRFS_DIR_INDEX_KEY) { 6203 inode->index_cnt = 2; 6204 goto out; 6205 } 6206 6207 inode->index_cnt = found_key.offset + 1; 6208 out: 6209 btrfs_free_path(path); 6210 return ret; 6211 } 6212 6213 /* 6214 * helper to find a free sequence number in a given directory. This current 6215 * code is very simple, later versions will do smarter things in the btree 6216 */ 6217 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) 6218 { 6219 int ret = 0; 6220 6221 if (dir->index_cnt == (u64)-1) { 6222 ret = btrfs_inode_delayed_dir_index_count(dir); 6223 if (ret) { 6224 ret = btrfs_set_inode_index_count(dir); 6225 if (ret) 6226 return ret; 6227 } 6228 } 6229 6230 *index = dir->index_cnt; 6231 dir->index_cnt++; 6232 6233 return ret; 6234 } 6235 6236 static int btrfs_insert_inode_locked(struct inode *inode) 6237 { 6238 struct btrfs_iget_args args; 6239 args.location = &BTRFS_I(inode)->location; 6240 args.root = BTRFS_I(inode)->root; 6241 6242 return insert_inode_locked4(inode, 6243 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 6244 btrfs_find_actor, &args); 6245 } 6246 6247 /* 6248 * Inherit flags from the parent inode. 6249 * 6250 * Currently only the compression flags and the cow flags are inherited. 6251 */ 6252 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir) 6253 { 6254 unsigned int flags; 6255 6256 if (!dir) 6257 return; 6258 6259 flags = BTRFS_I(dir)->flags; 6260 6261 if (flags & BTRFS_INODE_NOCOMPRESS) { 6262 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS; 6263 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 6264 } else if (flags & BTRFS_INODE_COMPRESS) { 6265 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS; 6266 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS; 6267 } 6268 6269 if (flags & BTRFS_INODE_NODATACOW) { 6270 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW; 6271 if (S_ISREG(inode->i_mode)) 6272 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6273 } 6274 6275 btrfs_update_iflags(inode); 6276 } 6277 6278 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, 6279 struct btrfs_root *root, 6280 struct inode *dir, 6281 const char *name, int name_len, 6282 u64 ref_objectid, u64 objectid, 6283 umode_t mode, u64 *index) 6284 { 6285 struct btrfs_fs_info *fs_info = root->fs_info; 6286 struct inode *inode; 6287 struct btrfs_inode_item *inode_item; 6288 struct btrfs_key *location; 6289 struct btrfs_path *path; 6290 struct btrfs_inode_ref *ref; 6291 struct btrfs_key key[2]; 6292 u32 sizes[2]; 6293 int nitems = name ? 2 : 1; 6294 unsigned long ptr; 6295 int ret; 6296 6297 path = btrfs_alloc_path(); 6298 if (!path) 6299 return ERR_PTR(-ENOMEM); 6300 6301 inode = new_inode(fs_info->sb); 6302 if (!inode) { 6303 btrfs_free_path(path); 6304 return ERR_PTR(-ENOMEM); 6305 } 6306 6307 /* 6308 * O_TMPFILE, set link count to 0, so that after this point, 6309 * we fill in an inode item with the correct link count. 6310 */ 6311 if (!name) 6312 set_nlink(inode, 0); 6313 6314 /* 6315 * we have to initialize this early, so we can reclaim the inode 6316 * number if we fail afterwards in this function. 6317 */ 6318 inode->i_ino = objectid; 6319 6320 if (dir && name) { 6321 trace_btrfs_inode_request(dir); 6322 6323 ret = btrfs_set_inode_index(BTRFS_I(dir), index); 6324 if (ret) { 6325 btrfs_free_path(path); 6326 iput(inode); 6327 return ERR_PTR(ret); 6328 } 6329 } else if (dir) { 6330 *index = 0; 6331 } 6332 /* 6333 * index_cnt is ignored for everything but a dir, 6334 * btrfs_set_inode_index_count has an explanation for the magic 6335 * number 6336 */ 6337 BTRFS_I(inode)->index_cnt = 2; 6338 BTRFS_I(inode)->dir_index = *index; 6339 BTRFS_I(inode)->root = root; 6340 BTRFS_I(inode)->generation = trans->transid; 6341 inode->i_generation = BTRFS_I(inode)->generation; 6342 6343 /* 6344 * We could have gotten an inode number from somebody who was fsynced 6345 * and then removed in this same transaction, so let's just set full 6346 * sync since it will be a full sync anyway and this will blow away the 6347 * old info in the log. 6348 */ 6349 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 6350 6351 key[0].objectid = objectid; 6352 key[0].type = BTRFS_INODE_ITEM_KEY; 6353 key[0].offset = 0; 6354 6355 sizes[0] = sizeof(struct btrfs_inode_item); 6356 6357 if (name) { 6358 /* 6359 * Start new inodes with an inode_ref. This is slightly more 6360 * efficient for small numbers of hard links since they will 6361 * be packed into one item. Extended refs will kick in if we 6362 * add more hard links than can fit in the ref item. 6363 */ 6364 key[1].objectid = objectid; 6365 key[1].type = BTRFS_INODE_REF_KEY; 6366 key[1].offset = ref_objectid; 6367 6368 sizes[1] = name_len + sizeof(*ref); 6369 } 6370 6371 location = &BTRFS_I(inode)->location; 6372 location->objectid = objectid; 6373 location->offset = 0; 6374 location->type = BTRFS_INODE_ITEM_KEY; 6375 6376 ret = btrfs_insert_inode_locked(inode); 6377 if (ret < 0) 6378 goto fail; 6379 6380 path->leave_spinning = 1; 6381 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems); 6382 if (ret != 0) 6383 goto fail_unlock; 6384 6385 inode_init_owner(inode, dir, mode); 6386 inode_set_bytes(inode, 0); 6387 6388 inode->i_mtime = current_time(inode); 6389 inode->i_atime = inode->i_mtime; 6390 inode->i_ctime = inode->i_mtime; 6391 BTRFS_I(inode)->i_otime = inode->i_mtime; 6392 6393 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6394 struct btrfs_inode_item); 6395 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6396 sizeof(*inode_item)); 6397 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6398 6399 if (name) { 6400 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6401 struct btrfs_inode_ref); 6402 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len); 6403 btrfs_set_inode_ref_index(path->nodes[0], ref, *index); 6404 ptr = (unsigned long)(ref + 1); 6405 write_extent_buffer(path->nodes[0], name, ptr, name_len); 6406 } 6407 6408 btrfs_mark_buffer_dirty(path->nodes[0]); 6409 btrfs_free_path(path); 6410 6411 btrfs_inherit_iflags(inode, dir); 6412 6413 if (S_ISREG(mode)) { 6414 if (btrfs_test_opt(fs_info, NODATASUM)) 6415 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6416 if (btrfs_test_opt(fs_info, NODATACOW)) 6417 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6418 BTRFS_INODE_NODATASUM; 6419 } 6420 6421 inode_tree_add(inode); 6422 6423 trace_btrfs_inode_new(inode); 6424 btrfs_set_inode_last_trans(trans, inode); 6425 6426 btrfs_update_root_times(trans, root); 6427 6428 ret = btrfs_inode_inherit_props(trans, inode, dir); 6429 if (ret) 6430 btrfs_err(fs_info, 6431 "error inheriting props for ino %llu (root %llu): %d", 6432 btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret); 6433 6434 return inode; 6435 6436 fail_unlock: 6437 unlock_new_inode(inode); 6438 fail: 6439 if (dir && name) 6440 BTRFS_I(dir)->index_cnt--; 6441 btrfs_free_path(path); 6442 iput(inode); 6443 return ERR_PTR(ret); 6444 } 6445 6446 static inline u8 btrfs_inode_type(struct inode *inode) 6447 { 6448 return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT]; 6449 } 6450 6451 /* 6452 * utility function to add 'inode' into 'parent_inode' with 6453 * a give name and a given sequence number. 6454 * if 'add_backref' is true, also insert a backref from the 6455 * inode to the parent directory. 6456 */ 6457 int btrfs_add_link(struct btrfs_trans_handle *trans, 6458 struct btrfs_inode *parent_inode, struct btrfs_inode *inode, 6459 const char *name, int name_len, int add_backref, u64 index) 6460 { 6461 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 6462 int ret = 0; 6463 struct btrfs_key key; 6464 struct btrfs_root *root = parent_inode->root; 6465 u64 ino = btrfs_ino(inode); 6466 u64 parent_ino = btrfs_ino(parent_inode); 6467 6468 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6469 memcpy(&key, &inode->root->root_key, sizeof(key)); 6470 } else { 6471 key.objectid = ino; 6472 key.type = BTRFS_INODE_ITEM_KEY; 6473 key.offset = 0; 6474 } 6475 6476 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6477 ret = btrfs_add_root_ref(trans, fs_info, key.objectid, 6478 root->root_key.objectid, parent_ino, 6479 index, name, name_len); 6480 } else if (add_backref) { 6481 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino, 6482 parent_ino, index); 6483 } 6484 6485 /* Nothing to clean up yet */ 6486 if (ret) 6487 return ret; 6488 6489 ret = btrfs_insert_dir_item(trans, root, name, name_len, 6490 parent_inode, &key, 6491 btrfs_inode_type(&inode->vfs_inode), index); 6492 if (ret == -EEXIST || ret == -EOVERFLOW) 6493 goto fail_dir_item; 6494 else if (ret) { 6495 btrfs_abort_transaction(trans, ret); 6496 return ret; 6497 } 6498 6499 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + 6500 name_len * 2); 6501 inode_inc_iversion(&parent_inode->vfs_inode); 6502 parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime = 6503 current_time(&parent_inode->vfs_inode); 6504 ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode); 6505 if (ret) 6506 btrfs_abort_transaction(trans, ret); 6507 return ret; 6508 6509 fail_dir_item: 6510 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6511 u64 local_index; 6512 int err; 6513 err = btrfs_del_root_ref(trans, fs_info, key.objectid, 6514 root->root_key.objectid, parent_ino, 6515 &local_index, name, name_len); 6516 6517 } else if (add_backref) { 6518 u64 local_index; 6519 int err; 6520 6521 err = btrfs_del_inode_ref(trans, root, name, name_len, 6522 ino, parent_ino, &local_index); 6523 } 6524 return ret; 6525 } 6526 6527 static int btrfs_add_nondir(struct btrfs_trans_handle *trans, 6528 struct btrfs_inode *dir, struct dentry *dentry, 6529 struct btrfs_inode *inode, int backref, u64 index) 6530 { 6531 int err = btrfs_add_link(trans, dir, inode, 6532 dentry->d_name.name, dentry->d_name.len, 6533 backref, index); 6534 if (err > 0) 6535 err = -EEXIST; 6536 return err; 6537 } 6538 6539 static int btrfs_mknod(struct inode *dir, struct dentry *dentry, 6540 umode_t mode, dev_t rdev) 6541 { 6542 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6543 struct btrfs_trans_handle *trans; 6544 struct btrfs_root *root = BTRFS_I(dir)->root; 6545 struct inode *inode = NULL; 6546 int err; 6547 int drop_inode = 0; 6548 u64 objectid; 6549 u64 index = 0; 6550 6551 /* 6552 * 2 for inode item and ref 6553 * 2 for dir items 6554 * 1 for xattr if selinux is on 6555 */ 6556 trans = btrfs_start_transaction(root, 5); 6557 if (IS_ERR(trans)) 6558 return PTR_ERR(trans); 6559 6560 err = btrfs_find_free_ino(root, &objectid); 6561 if (err) 6562 goto out_unlock; 6563 6564 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6565 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6566 mode, &index); 6567 if (IS_ERR(inode)) { 6568 err = PTR_ERR(inode); 6569 goto out_unlock; 6570 } 6571 6572 /* 6573 * If the active LSM wants to access the inode during 6574 * d_instantiate it needs these. Smack checks to see 6575 * if the filesystem supports xattrs by looking at the 6576 * ops vector. 6577 */ 6578 inode->i_op = &btrfs_special_inode_operations; 6579 init_special_inode(inode, inode->i_mode, rdev); 6580 6581 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6582 if (err) 6583 goto out_unlock_inode; 6584 6585 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6586 0, index); 6587 if (err) { 6588 goto out_unlock_inode; 6589 } else { 6590 btrfs_update_inode(trans, root, inode); 6591 unlock_new_inode(inode); 6592 d_instantiate(dentry, inode); 6593 } 6594 6595 out_unlock: 6596 btrfs_end_transaction(trans); 6597 btrfs_btree_balance_dirty(fs_info); 6598 if (drop_inode) { 6599 inode_dec_link_count(inode); 6600 iput(inode); 6601 } 6602 return err; 6603 6604 out_unlock_inode: 6605 drop_inode = 1; 6606 unlock_new_inode(inode); 6607 goto out_unlock; 6608 6609 } 6610 6611 static int btrfs_create(struct inode *dir, struct dentry *dentry, 6612 umode_t mode, bool excl) 6613 { 6614 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6615 struct btrfs_trans_handle *trans; 6616 struct btrfs_root *root = BTRFS_I(dir)->root; 6617 struct inode *inode = NULL; 6618 int drop_inode_on_err = 0; 6619 int err; 6620 u64 objectid; 6621 u64 index = 0; 6622 6623 /* 6624 * 2 for inode item and ref 6625 * 2 for dir items 6626 * 1 for xattr if selinux is on 6627 */ 6628 trans = btrfs_start_transaction(root, 5); 6629 if (IS_ERR(trans)) 6630 return PTR_ERR(trans); 6631 6632 err = btrfs_find_free_ino(root, &objectid); 6633 if (err) 6634 goto out_unlock; 6635 6636 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6637 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6638 mode, &index); 6639 if (IS_ERR(inode)) { 6640 err = PTR_ERR(inode); 6641 goto out_unlock; 6642 } 6643 drop_inode_on_err = 1; 6644 /* 6645 * If the active LSM wants to access the inode during 6646 * d_instantiate it needs these. Smack checks to see 6647 * if the filesystem supports xattrs by looking at the 6648 * ops vector. 6649 */ 6650 inode->i_fop = &btrfs_file_operations; 6651 inode->i_op = &btrfs_file_inode_operations; 6652 inode->i_mapping->a_ops = &btrfs_aops; 6653 6654 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6655 if (err) 6656 goto out_unlock_inode; 6657 6658 err = btrfs_update_inode(trans, root, inode); 6659 if (err) 6660 goto out_unlock_inode; 6661 6662 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6663 0, index); 6664 if (err) 6665 goto out_unlock_inode; 6666 6667 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 6668 unlock_new_inode(inode); 6669 d_instantiate(dentry, inode); 6670 6671 out_unlock: 6672 btrfs_end_transaction(trans); 6673 if (err && drop_inode_on_err) { 6674 inode_dec_link_count(inode); 6675 iput(inode); 6676 } 6677 btrfs_btree_balance_dirty(fs_info); 6678 return err; 6679 6680 out_unlock_inode: 6681 unlock_new_inode(inode); 6682 goto out_unlock; 6683 6684 } 6685 6686 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6687 struct dentry *dentry) 6688 { 6689 struct btrfs_trans_handle *trans = NULL; 6690 struct btrfs_root *root = BTRFS_I(dir)->root; 6691 struct inode *inode = d_inode(old_dentry); 6692 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6693 u64 index; 6694 int err; 6695 int drop_inode = 0; 6696 6697 /* do not allow sys_link's with other subvols of the same device */ 6698 if (root->objectid != BTRFS_I(inode)->root->objectid) 6699 return -EXDEV; 6700 6701 if (inode->i_nlink >= BTRFS_LINK_MAX) 6702 return -EMLINK; 6703 6704 err = btrfs_set_inode_index(BTRFS_I(dir), &index); 6705 if (err) 6706 goto fail; 6707 6708 /* 6709 * 2 items for inode and inode ref 6710 * 2 items for dir items 6711 * 1 item for parent inode 6712 */ 6713 trans = btrfs_start_transaction(root, 5); 6714 if (IS_ERR(trans)) { 6715 err = PTR_ERR(trans); 6716 trans = NULL; 6717 goto fail; 6718 } 6719 6720 /* There are several dir indexes for this inode, clear the cache. */ 6721 BTRFS_I(inode)->dir_index = 0ULL; 6722 inc_nlink(inode); 6723 inode_inc_iversion(inode); 6724 inode->i_ctime = current_time(inode); 6725 ihold(inode); 6726 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6727 6728 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6729 1, index); 6730 6731 if (err) { 6732 drop_inode = 1; 6733 } else { 6734 struct dentry *parent = dentry->d_parent; 6735 err = btrfs_update_inode(trans, root, inode); 6736 if (err) 6737 goto fail; 6738 if (inode->i_nlink == 1) { 6739 /* 6740 * If new hard link count is 1, it's a file created 6741 * with open(2) O_TMPFILE flag. 6742 */ 6743 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 6744 if (err) 6745 goto fail; 6746 } 6747 d_instantiate(dentry, inode); 6748 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent); 6749 } 6750 6751 fail: 6752 if (trans) 6753 btrfs_end_transaction(trans); 6754 if (drop_inode) { 6755 inode_dec_link_count(inode); 6756 iput(inode); 6757 } 6758 btrfs_btree_balance_dirty(fs_info); 6759 return err; 6760 } 6761 6762 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 6763 { 6764 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6765 struct inode *inode = NULL; 6766 struct btrfs_trans_handle *trans; 6767 struct btrfs_root *root = BTRFS_I(dir)->root; 6768 int err = 0; 6769 int drop_on_err = 0; 6770 u64 objectid = 0; 6771 u64 index = 0; 6772 6773 /* 6774 * 2 items for inode and ref 6775 * 2 items for dir items 6776 * 1 for xattr if selinux is on 6777 */ 6778 trans = btrfs_start_transaction(root, 5); 6779 if (IS_ERR(trans)) 6780 return PTR_ERR(trans); 6781 6782 err = btrfs_find_free_ino(root, &objectid); 6783 if (err) 6784 goto out_fail; 6785 6786 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6787 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6788 S_IFDIR | mode, &index); 6789 if (IS_ERR(inode)) { 6790 err = PTR_ERR(inode); 6791 goto out_fail; 6792 } 6793 6794 drop_on_err = 1; 6795 /* these must be set before we unlock the inode */ 6796 inode->i_op = &btrfs_dir_inode_operations; 6797 inode->i_fop = &btrfs_dir_file_operations; 6798 6799 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6800 if (err) 6801 goto out_fail_inode; 6802 6803 btrfs_i_size_write(BTRFS_I(inode), 0); 6804 err = btrfs_update_inode(trans, root, inode); 6805 if (err) 6806 goto out_fail_inode; 6807 6808 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), 6809 dentry->d_name.name, 6810 dentry->d_name.len, 0, index); 6811 if (err) 6812 goto out_fail_inode; 6813 6814 d_instantiate(dentry, inode); 6815 /* 6816 * mkdir is special. We're unlocking after we call d_instantiate 6817 * to avoid a race with nfsd calling d_instantiate. 6818 */ 6819 unlock_new_inode(inode); 6820 drop_on_err = 0; 6821 6822 out_fail: 6823 btrfs_end_transaction(trans); 6824 if (drop_on_err) { 6825 inode_dec_link_count(inode); 6826 iput(inode); 6827 } 6828 btrfs_btree_balance_dirty(fs_info); 6829 return err; 6830 6831 out_fail_inode: 6832 unlock_new_inode(inode); 6833 goto out_fail; 6834 } 6835 6836 static noinline int uncompress_inline(struct btrfs_path *path, 6837 struct page *page, 6838 size_t pg_offset, u64 extent_offset, 6839 struct btrfs_file_extent_item *item) 6840 { 6841 int ret; 6842 struct extent_buffer *leaf = path->nodes[0]; 6843 char *tmp; 6844 size_t max_size; 6845 unsigned long inline_size; 6846 unsigned long ptr; 6847 int compress_type; 6848 6849 WARN_ON(pg_offset != 0); 6850 compress_type = btrfs_file_extent_compression(leaf, item); 6851 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6852 inline_size = btrfs_file_extent_inline_item_len(leaf, 6853 btrfs_item_nr(path->slots[0])); 6854 tmp = kmalloc(inline_size, GFP_NOFS); 6855 if (!tmp) 6856 return -ENOMEM; 6857 ptr = btrfs_file_extent_inline_start(item); 6858 6859 read_extent_buffer(leaf, tmp, ptr, inline_size); 6860 6861 max_size = min_t(unsigned long, PAGE_SIZE, max_size); 6862 ret = btrfs_decompress(compress_type, tmp, page, 6863 extent_offset, inline_size, max_size); 6864 6865 /* 6866 * decompression code contains a memset to fill in any space between the end 6867 * of the uncompressed data and the end of max_size in case the decompressed 6868 * data ends up shorter than ram_bytes. That doesn't cover the hole between 6869 * the end of an inline extent and the beginning of the next block, so we 6870 * cover that region here. 6871 */ 6872 6873 if (max_size + pg_offset < PAGE_SIZE) { 6874 char *map = kmap(page); 6875 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset); 6876 kunmap(page); 6877 } 6878 kfree(tmp); 6879 return ret; 6880 } 6881 6882 /* 6883 * a bit scary, this does extent mapping from logical file offset to the disk. 6884 * the ugly parts come from merging extents from the disk with the in-ram 6885 * representation. This gets more complex because of the data=ordered code, 6886 * where the in-ram extents might be locked pending data=ordered completion. 6887 * 6888 * This also copies inline extents directly into the page. 6889 */ 6890 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, 6891 struct page *page, 6892 size_t pg_offset, u64 start, u64 len, 6893 int create) 6894 { 6895 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 6896 int ret; 6897 int err = 0; 6898 u64 extent_start = 0; 6899 u64 extent_end = 0; 6900 u64 objectid = btrfs_ino(inode); 6901 u32 found_type; 6902 struct btrfs_path *path = NULL; 6903 struct btrfs_root *root = inode->root; 6904 struct btrfs_file_extent_item *item; 6905 struct extent_buffer *leaf; 6906 struct btrfs_key found_key; 6907 struct extent_map *em = NULL; 6908 struct extent_map_tree *em_tree = &inode->extent_tree; 6909 struct extent_io_tree *io_tree = &inode->io_tree; 6910 const bool new_inline = !page || create; 6911 6912 read_lock(&em_tree->lock); 6913 em = lookup_extent_mapping(em_tree, start, len); 6914 if (em) 6915 em->bdev = fs_info->fs_devices->latest_bdev; 6916 read_unlock(&em_tree->lock); 6917 6918 if (em) { 6919 if (em->start > start || em->start + em->len <= start) 6920 free_extent_map(em); 6921 else if (em->block_start == EXTENT_MAP_INLINE && page) 6922 free_extent_map(em); 6923 else 6924 goto out; 6925 } 6926 em = alloc_extent_map(); 6927 if (!em) { 6928 err = -ENOMEM; 6929 goto out; 6930 } 6931 em->bdev = fs_info->fs_devices->latest_bdev; 6932 em->start = EXTENT_MAP_HOLE; 6933 em->orig_start = EXTENT_MAP_HOLE; 6934 em->len = (u64)-1; 6935 em->block_len = (u64)-1; 6936 6937 if (!path) { 6938 path = btrfs_alloc_path(); 6939 if (!path) { 6940 err = -ENOMEM; 6941 goto out; 6942 } 6943 /* 6944 * Chances are we'll be called again, so go ahead and do 6945 * readahead 6946 */ 6947 path->reada = READA_FORWARD; 6948 } 6949 6950 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); 6951 if (ret < 0) { 6952 err = ret; 6953 goto out; 6954 } 6955 6956 if (ret != 0) { 6957 if (path->slots[0] == 0) 6958 goto not_found; 6959 path->slots[0]--; 6960 } 6961 6962 leaf = path->nodes[0]; 6963 item = btrfs_item_ptr(leaf, path->slots[0], 6964 struct btrfs_file_extent_item); 6965 /* are we inside the extent that was found? */ 6966 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6967 found_type = found_key.type; 6968 if (found_key.objectid != objectid || 6969 found_type != BTRFS_EXTENT_DATA_KEY) { 6970 /* 6971 * If we backup past the first extent we want to move forward 6972 * and see if there is an extent in front of us, otherwise we'll 6973 * say there is a hole for our whole search range which can 6974 * cause problems. 6975 */ 6976 extent_end = start; 6977 goto next; 6978 } 6979 6980 found_type = btrfs_file_extent_type(leaf, item); 6981 extent_start = found_key.offset; 6982 if (found_type == BTRFS_FILE_EXTENT_REG || 6983 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6984 extent_end = extent_start + 6985 btrfs_file_extent_num_bytes(leaf, item); 6986 6987 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, 6988 extent_start); 6989 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 6990 size_t size; 6991 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 6992 extent_end = ALIGN(extent_start + size, 6993 fs_info->sectorsize); 6994 6995 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, 6996 path->slots[0], 6997 extent_start); 6998 } 6999 next: 7000 if (start >= extent_end) { 7001 path->slots[0]++; 7002 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 7003 ret = btrfs_next_leaf(root, path); 7004 if (ret < 0) { 7005 err = ret; 7006 goto out; 7007 } 7008 if (ret > 0) 7009 goto not_found; 7010 leaf = path->nodes[0]; 7011 } 7012 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 7013 if (found_key.objectid != objectid || 7014 found_key.type != BTRFS_EXTENT_DATA_KEY) 7015 goto not_found; 7016 if (start + len <= found_key.offset) 7017 goto not_found; 7018 if (start > found_key.offset) 7019 goto next; 7020 em->start = start; 7021 em->orig_start = start; 7022 em->len = found_key.offset - start; 7023 goto not_found_em; 7024 } 7025 7026 btrfs_extent_item_to_extent_map(inode, path, item, 7027 new_inline, em); 7028 7029 if (found_type == BTRFS_FILE_EXTENT_REG || 7030 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7031 goto insert; 7032 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 7033 unsigned long ptr; 7034 char *map; 7035 size_t size; 7036 size_t extent_offset; 7037 size_t copy_size; 7038 7039 if (new_inline) 7040 goto out; 7041 7042 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 7043 extent_offset = page_offset(page) + pg_offset - extent_start; 7044 copy_size = min_t(u64, PAGE_SIZE - pg_offset, 7045 size - extent_offset); 7046 em->start = extent_start + extent_offset; 7047 em->len = ALIGN(copy_size, fs_info->sectorsize); 7048 em->orig_block_len = em->len; 7049 em->orig_start = em->start; 7050 ptr = btrfs_file_extent_inline_start(item) + extent_offset; 7051 if (!PageUptodate(page)) { 7052 if (btrfs_file_extent_compression(leaf, item) != 7053 BTRFS_COMPRESS_NONE) { 7054 ret = uncompress_inline(path, page, pg_offset, 7055 extent_offset, item); 7056 if (ret) { 7057 err = ret; 7058 goto out; 7059 } 7060 } else { 7061 map = kmap(page); 7062 read_extent_buffer(leaf, map + pg_offset, ptr, 7063 copy_size); 7064 if (pg_offset + copy_size < PAGE_SIZE) { 7065 memset(map + pg_offset + copy_size, 0, 7066 PAGE_SIZE - pg_offset - 7067 copy_size); 7068 } 7069 kunmap(page); 7070 } 7071 flush_dcache_page(page); 7072 } 7073 set_extent_uptodate(io_tree, em->start, 7074 extent_map_end(em) - 1, NULL, GFP_NOFS); 7075 goto insert; 7076 } 7077 not_found: 7078 em->start = start; 7079 em->orig_start = start; 7080 em->len = len; 7081 not_found_em: 7082 em->block_start = EXTENT_MAP_HOLE; 7083 insert: 7084 btrfs_release_path(path); 7085 if (em->start > start || extent_map_end(em) <= start) { 7086 btrfs_err(fs_info, 7087 "bad extent! em: [%llu %llu] passed [%llu %llu]", 7088 em->start, em->len, start, len); 7089 err = -EIO; 7090 goto out; 7091 } 7092 7093 err = 0; 7094 write_lock(&em_tree->lock); 7095 err = btrfs_add_extent_mapping(em_tree, &em, start, len); 7096 write_unlock(&em_tree->lock); 7097 out: 7098 7099 trace_btrfs_get_extent(root, inode, em); 7100 7101 btrfs_free_path(path); 7102 if (err) { 7103 free_extent_map(em); 7104 return ERR_PTR(err); 7105 } 7106 BUG_ON(!em); /* Error is always set */ 7107 return em; 7108 } 7109 7110 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, 7111 struct page *page, 7112 size_t pg_offset, u64 start, u64 len, 7113 int create) 7114 { 7115 struct extent_map *em; 7116 struct extent_map *hole_em = NULL; 7117 u64 range_start = start; 7118 u64 end; 7119 u64 found; 7120 u64 found_end; 7121 int err = 0; 7122 7123 em = btrfs_get_extent(inode, page, pg_offset, start, len, create); 7124 if (IS_ERR(em)) 7125 return em; 7126 /* 7127 * If our em maps to: 7128 * - a hole or 7129 * - a pre-alloc extent, 7130 * there might actually be delalloc bytes behind it. 7131 */ 7132 if (em->block_start != EXTENT_MAP_HOLE && 7133 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7134 return em; 7135 else 7136 hole_em = em; 7137 7138 /* check to see if we've wrapped (len == -1 or similar) */ 7139 end = start + len; 7140 if (end < start) 7141 end = (u64)-1; 7142 else 7143 end -= 1; 7144 7145 em = NULL; 7146 7147 /* ok, we didn't find anything, lets look for delalloc */ 7148 found = count_range_bits(&inode->io_tree, &range_start, 7149 end, len, EXTENT_DELALLOC, 1); 7150 found_end = range_start + found; 7151 if (found_end < range_start) 7152 found_end = (u64)-1; 7153 7154 /* 7155 * we didn't find anything useful, return 7156 * the original results from get_extent() 7157 */ 7158 if (range_start > end || found_end <= start) { 7159 em = hole_em; 7160 hole_em = NULL; 7161 goto out; 7162 } 7163 7164 /* adjust the range_start to make sure it doesn't 7165 * go backwards from the start they passed in 7166 */ 7167 range_start = max(start, range_start); 7168 found = found_end - range_start; 7169 7170 if (found > 0) { 7171 u64 hole_start = start; 7172 u64 hole_len = len; 7173 7174 em = alloc_extent_map(); 7175 if (!em) { 7176 err = -ENOMEM; 7177 goto out; 7178 } 7179 /* 7180 * when btrfs_get_extent can't find anything it 7181 * returns one huge hole 7182 * 7183 * make sure what it found really fits our range, and 7184 * adjust to make sure it is based on the start from 7185 * the caller 7186 */ 7187 if (hole_em) { 7188 u64 calc_end = extent_map_end(hole_em); 7189 7190 if (calc_end <= start || (hole_em->start > end)) { 7191 free_extent_map(hole_em); 7192 hole_em = NULL; 7193 } else { 7194 hole_start = max(hole_em->start, start); 7195 hole_len = calc_end - hole_start; 7196 } 7197 } 7198 em->bdev = NULL; 7199 if (hole_em && range_start > hole_start) { 7200 /* our hole starts before our delalloc, so we 7201 * have to return just the parts of the hole 7202 * that go until the delalloc starts 7203 */ 7204 em->len = min(hole_len, 7205 range_start - hole_start); 7206 em->start = hole_start; 7207 em->orig_start = hole_start; 7208 /* 7209 * don't adjust block start at all, 7210 * it is fixed at EXTENT_MAP_HOLE 7211 */ 7212 em->block_start = hole_em->block_start; 7213 em->block_len = hole_len; 7214 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) 7215 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 7216 } else { 7217 em->start = range_start; 7218 em->len = found; 7219 em->orig_start = range_start; 7220 em->block_start = EXTENT_MAP_DELALLOC; 7221 em->block_len = found; 7222 } 7223 } else { 7224 return hole_em; 7225 } 7226 out: 7227 7228 free_extent_map(hole_em); 7229 if (err) { 7230 free_extent_map(em); 7231 return ERR_PTR(err); 7232 } 7233 return em; 7234 } 7235 7236 static struct extent_map *btrfs_create_dio_extent(struct inode *inode, 7237 const u64 start, 7238 const u64 len, 7239 const u64 orig_start, 7240 const u64 block_start, 7241 const u64 block_len, 7242 const u64 orig_block_len, 7243 const u64 ram_bytes, 7244 const int type) 7245 { 7246 struct extent_map *em = NULL; 7247 int ret; 7248 7249 if (type != BTRFS_ORDERED_NOCOW) { 7250 em = create_io_em(inode, start, len, orig_start, 7251 block_start, block_len, orig_block_len, 7252 ram_bytes, 7253 BTRFS_COMPRESS_NONE, /* compress_type */ 7254 type); 7255 if (IS_ERR(em)) 7256 goto out; 7257 } 7258 ret = btrfs_add_ordered_extent_dio(inode, start, block_start, 7259 len, block_len, type); 7260 if (ret) { 7261 if (em) { 7262 free_extent_map(em); 7263 btrfs_drop_extent_cache(BTRFS_I(inode), start, 7264 start + len - 1, 0); 7265 } 7266 em = ERR_PTR(ret); 7267 } 7268 out: 7269 7270 return em; 7271 } 7272 7273 static struct extent_map *btrfs_new_extent_direct(struct inode *inode, 7274 u64 start, u64 len) 7275 { 7276 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7277 struct btrfs_root *root = BTRFS_I(inode)->root; 7278 struct extent_map *em; 7279 struct btrfs_key ins; 7280 u64 alloc_hint; 7281 int ret; 7282 7283 alloc_hint = get_extent_allocation_hint(inode, start, len); 7284 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, 7285 0, alloc_hint, &ins, 1, 1); 7286 if (ret) 7287 return ERR_PTR(ret); 7288 7289 em = btrfs_create_dio_extent(inode, start, ins.offset, start, 7290 ins.objectid, ins.offset, ins.offset, 7291 ins.offset, BTRFS_ORDERED_REGULAR); 7292 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 7293 if (IS_ERR(em)) 7294 btrfs_free_reserved_extent(fs_info, ins.objectid, 7295 ins.offset, 1); 7296 7297 return em; 7298 } 7299 7300 /* 7301 * returns 1 when the nocow is safe, < 1 on error, 0 if the 7302 * block must be cow'd 7303 */ 7304 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 7305 u64 *orig_start, u64 *orig_block_len, 7306 u64 *ram_bytes) 7307 { 7308 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7309 struct btrfs_path *path; 7310 int ret; 7311 struct extent_buffer *leaf; 7312 struct btrfs_root *root = BTRFS_I(inode)->root; 7313 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7314 struct btrfs_file_extent_item *fi; 7315 struct btrfs_key key; 7316 u64 disk_bytenr; 7317 u64 backref_offset; 7318 u64 extent_end; 7319 u64 num_bytes; 7320 int slot; 7321 int found_type; 7322 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW); 7323 7324 path = btrfs_alloc_path(); 7325 if (!path) 7326 return -ENOMEM; 7327 7328 ret = btrfs_lookup_file_extent(NULL, root, path, 7329 btrfs_ino(BTRFS_I(inode)), offset, 0); 7330 if (ret < 0) 7331 goto out; 7332 7333 slot = path->slots[0]; 7334 if (ret == 1) { 7335 if (slot == 0) { 7336 /* can't find the item, must cow */ 7337 ret = 0; 7338 goto out; 7339 } 7340 slot--; 7341 } 7342 ret = 0; 7343 leaf = path->nodes[0]; 7344 btrfs_item_key_to_cpu(leaf, &key, slot); 7345 if (key.objectid != btrfs_ino(BTRFS_I(inode)) || 7346 key.type != BTRFS_EXTENT_DATA_KEY) { 7347 /* not our file or wrong item type, must cow */ 7348 goto out; 7349 } 7350 7351 if (key.offset > offset) { 7352 /* Wrong offset, must cow */ 7353 goto out; 7354 } 7355 7356 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 7357 found_type = btrfs_file_extent_type(leaf, fi); 7358 if (found_type != BTRFS_FILE_EXTENT_REG && 7359 found_type != BTRFS_FILE_EXTENT_PREALLOC) { 7360 /* not a regular extent, must cow */ 7361 goto out; 7362 } 7363 7364 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG) 7365 goto out; 7366 7367 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 7368 if (extent_end <= offset) 7369 goto out; 7370 7371 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 7372 if (disk_bytenr == 0) 7373 goto out; 7374 7375 if (btrfs_file_extent_compression(leaf, fi) || 7376 btrfs_file_extent_encryption(leaf, fi) || 7377 btrfs_file_extent_other_encoding(leaf, fi)) 7378 goto out; 7379 7380 backref_offset = btrfs_file_extent_offset(leaf, fi); 7381 7382 if (orig_start) { 7383 *orig_start = key.offset - backref_offset; 7384 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 7385 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 7386 } 7387 7388 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 7389 goto out; 7390 7391 num_bytes = min(offset + *len, extent_end) - offset; 7392 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7393 u64 range_end; 7394 7395 range_end = round_up(offset + num_bytes, 7396 root->fs_info->sectorsize) - 1; 7397 ret = test_range_bit(io_tree, offset, range_end, 7398 EXTENT_DELALLOC, 0, NULL); 7399 if (ret) { 7400 ret = -EAGAIN; 7401 goto out; 7402 } 7403 } 7404 7405 btrfs_release_path(path); 7406 7407 /* 7408 * look for other files referencing this extent, if we 7409 * find any we must cow 7410 */ 7411 7412 ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)), 7413 key.offset - backref_offset, disk_bytenr); 7414 if (ret) { 7415 ret = 0; 7416 goto out; 7417 } 7418 7419 /* 7420 * adjust disk_bytenr and num_bytes to cover just the bytes 7421 * in this extent we are about to write. If there 7422 * are any csums in that range we have to cow in order 7423 * to keep the csums correct 7424 */ 7425 disk_bytenr += backref_offset; 7426 disk_bytenr += offset - key.offset; 7427 if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes)) 7428 goto out; 7429 /* 7430 * all of the above have passed, it is safe to overwrite this extent 7431 * without cow 7432 */ 7433 *len = num_bytes; 7434 ret = 1; 7435 out: 7436 btrfs_free_path(path); 7437 return ret; 7438 } 7439 7440 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7441 struct extent_state **cached_state, int writing) 7442 { 7443 struct btrfs_ordered_extent *ordered; 7444 int ret = 0; 7445 7446 while (1) { 7447 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7448 cached_state); 7449 /* 7450 * We're concerned with the entire range that we're going to be 7451 * doing DIO to, so we need to make sure there's no ordered 7452 * extents in this range. 7453 */ 7454 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart, 7455 lockend - lockstart + 1); 7456 7457 /* 7458 * We need to make sure there are no buffered pages in this 7459 * range either, we could have raced between the invalidate in 7460 * generic_file_direct_write and locking the extent. The 7461 * invalidate needs to happen so that reads after a write do not 7462 * get stale data. 7463 */ 7464 if (!ordered && 7465 (!writing || !filemap_range_has_page(inode->i_mapping, 7466 lockstart, lockend))) 7467 break; 7468 7469 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7470 cached_state); 7471 7472 if (ordered) { 7473 /* 7474 * If we are doing a DIO read and the ordered extent we 7475 * found is for a buffered write, we can not wait for it 7476 * to complete and retry, because if we do so we can 7477 * deadlock with concurrent buffered writes on page 7478 * locks. This happens only if our DIO read covers more 7479 * than one extent map, if at this point has already 7480 * created an ordered extent for a previous extent map 7481 * and locked its range in the inode's io tree, and a 7482 * concurrent write against that previous extent map's 7483 * range and this range started (we unlock the ranges 7484 * in the io tree only when the bios complete and 7485 * buffered writes always lock pages before attempting 7486 * to lock range in the io tree). 7487 */ 7488 if (writing || 7489 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) 7490 btrfs_start_ordered_extent(inode, ordered, 1); 7491 else 7492 ret = -ENOTBLK; 7493 btrfs_put_ordered_extent(ordered); 7494 } else { 7495 /* 7496 * We could trigger writeback for this range (and wait 7497 * for it to complete) and then invalidate the pages for 7498 * this range (through invalidate_inode_pages2_range()), 7499 * but that can lead us to a deadlock with a concurrent 7500 * call to readpages() (a buffered read or a defrag call 7501 * triggered a readahead) on a page lock due to an 7502 * ordered dio extent we created before but did not have 7503 * yet a corresponding bio submitted (whence it can not 7504 * complete), which makes readpages() wait for that 7505 * ordered extent to complete while holding a lock on 7506 * that page. 7507 */ 7508 ret = -ENOTBLK; 7509 } 7510 7511 if (ret) 7512 break; 7513 7514 cond_resched(); 7515 } 7516 7517 return ret; 7518 } 7519 7520 /* The callers of this must take lock_extent() */ 7521 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, 7522 u64 orig_start, u64 block_start, 7523 u64 block_len, u64 orig_block_len, 7524 u64 ram_bytes, int compress_type, 7525 int type) 7526 { 7527 struct extent_map_tree *em_tree; 7528 struct extent_map *em; 7529 struct btrfs_root *root = BTRFS_I(inode)->root; 7530 int ret; 7531 7532 ASSERT(type == BTRFS_ORDERED_PREALLOC || 7533 type == BTRFS_ORDERED_COMPRESSED || 7534 type == BTRFS_ORDERED_NOCOW || 7535 type == BTRFS_ORDERED_REGULAR); 7536 7537 em_tree = &BTRFS_I(inode)->extent_tree; 7538 em = alloc_extent_map(); 7539 if (!em) 7540 return ERR_PTR(-ENOMEM); 7541 7542 em->start = start; 7543 em->orig_start = orig_start; 7544 em->len = len; 7545 em->block_len = block_len; 7546 em->block_start = block_start; 7547 em->bdev = root->fs_info->fs_devices->latest_bdev; 7548 em->orig_block_len = orig_block_len; 7549 em->ram_bytes = ram_bytes; 7550 em->generation = -1; 7551 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7552 if (type == BTRFS_ORDERED_PREALLOC) { 7553 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7554 } else if (type == BTRFS_ORDERED_COMPRESSED) { 7555 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 7556 em->compress_type = compress_type; 7557 } 7558 7559 do { 7560 btrfs_drop_extent_cache(BTRFS_I(inode), em->start, 7561 em->start + em->len - 1, 0); 7562 write_lock(&em_tree->lock); 7563 ret = add_extent_mapping(em_tree, em, 1); 7564 write_unlock(&em_tree->lock); 7565 /* 7566 * The caller has taken lock_extent(), who could race with us 7567 * to add em? 7568 */ 7569 } while (ret == -EEXIST); 7570 7571 if (ret) { 7572 free_extent_map(em); 7573 return ERR_PTR(ret); 7574 } 7575 7576 /* em got 2 refs now, callers needs to do free_extent_map once. */ 7577 return em; 7578 } 7579 7580 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, 7581 struct buffer_head *bh_result, int create) 7582 { 7583 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7584 struct extent_map *em; 7585 struct extent_state *cached_state = NULL; 7586 struct btrfs_dio_data *dio_data = NULL; 7587 u64 start = iblock << inode->i_blkbits; 7588 u64 lockstart, lockend; 7589 u64 len = bh_result->b_size; 7590 int unlock_bits = EXTENT_LOCKED; 7591 int ret = 0; 7592 7593 if (create) 7594 unlock_bits |= EXTENT_DIRTY; 7595 else 7596 len = min_t(u64, len, fs_info->sectorsize); 7597 7598 lockstart = start; 7599 lockend = start + len - 1; 7600 7601 if (current->journal_info) { 7602 /* 7603 * Need to pull our outstanding extents and set journal_info to NULL so 7604 * that anything that needs to check if there's a transaction doesn't get 7605 * confused. 7606 */ 7607 dio_data = current->journal_info; 7608 current->journal_info = NULL; 7609 } 7610 7611 /* 7612 * If this errors out it's because we couldn't invalidate pagecache for 7613 * this range and we need to fallback to buffered. 7614 */ 7615 if (lock_extent_direct(inode, lockstart, lockend, &cached_state, 7616 create)) { 7617 ret = -ENOTBLK; 7618 goto err; 7619 } 7620 7621 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0); 7622 if (IS_ERR(em)) { 7623 ret = PTR_ERR(em); 7624 goto unlock_err; 7625 } 7626 7627 /* 7628 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7629 * io. INLINE is special, and we could probably kludge it in here, but 7630 * it's still buffered so for safety lets just fall back to the generic 7631 * buffered path. 7632 * 7633 * For COMPRESSED we _have_ to read the entire extent in so we can 7634 * decompress it, so there will be buffering required no matter what we 7635 * do, so go ahead and fallback to buffered. 7636 * 7637 * We return -ENOTBLK because that's what makes DIO go ahead and go back 7638 * to buffered IO. Don't blame me, this is the price we pay for using 7639 * the generic code. 7640 */ 7641 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7642 em->block_start == EXTENT_MAP_INLINE) { 7643 free_extent_map(em); 7644 ret = -ENOTBLK; 7645 goto unlock_err; 7646 } 7647 7648 /* Just a good old fashioned hole, return */ 7649 if (!create && (em->block_start == EXTENT_MAP_HOLE || 7650 test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) { 7651 free_extent_map(em); 7652 goto unlock_err; 7653 } 7654 7655 /* 7656 * We don't allocate a new extent in the following cases 7657 * 7658 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7659 * existing extent. 7660 * 2) The extent is marked as PREALLOC. We're good to go here and can 7661 * just use the extent. 7662 * 7663 */ 7664 if (!create) { 7665 len = min(len, em->len - (start - em->start)); 7666 lockstart = start + len; 7667 goto unlock; 7668 } 7669 7670 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7671 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7672 em->block_start != EXTENT_MAP_HOLE)) { 7673 int type; 7674 u64 block_start, orig_start, orig_block_len, ram_bytes; 7675 7676 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7677 type = BTRFS_ORDERED_PREALLOC; 7678 else 7679 type = BTRFS_ORDERED_NOCOW; 7680 len = min(len, em->len - (start - em->start)); 7681 block_start = em->block_start + (start - em->start); 7682 7683 if (can_nocow_extent(inode, start, &len, &orig_start, 7684 &orig_block_len, &ram_bytes) == 1 && 7685 btrfs_inc_nocow_writers(fs_info, block_start)) { 7686 struct extent_map *em2; 7687 7688 em2 = btrfs_create_dio_extent(inode, start, len, 7689 orig_start, block_start, 7690 len, orig_block_len, 7691 ram_bytes, type); 7692 btrfs_dec_nocow_writers(fs_info, block_start); 7693 if (type == BTRFS_ORDERED_PREALLOC) { 7694 free_extent_map(em); 7695 em = em2; 7696 } 7697 if (em2 && IS_ERR(em2)) { 7698 ret = PTR_ERR(em2); 7699 goto unlock_err; 7700 } 7701 /* 7702 * For inode marked NODATACOW or extent marked PREALLOC, 7703 * use the existing or preallocated extent, so does not 7704 * need to adjust btrfs_space_info's bytes_may_use. 7705 */ 7706 btrfs_free_reserved_data_space_noquota(inode, 7707 start, len); 7708 goto unlock; 7709 } 7710 } 7711 7712 /* 7713 * this will cow the extent, reset the len in case we changed 7714 * it above 7715 */ 7716 len = bh_result->b_size; 7717 free_extent_map(em); 7718 em = btrfs_new_extent_direct(inode, start, len); 7719 if (IS_ERR(em)) { 7720 ret = PTR_ERR(em); 7721 goto unlock_err; 7722 } 7723 len = min(len, em->len - (start - em->start)); 7724 unlock: 7725 bh_result->b_blocknr = (em->block_start + (start - em->start)) >> 7726 inode->i_blkbits; 7727 bh_result->b_size = len; 7728 bh_result->b_bdev = em->bdev; 7729 set_buffer_mapped(bh_result); 7730 if (create) { 7731 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7732 set_buffer_new(bh_result); 7733 7734 /* 7735 * Need to update the i_size under the extent lock so buffered 7736 * readers will get the updated i_size when we unlock. 7737 */ 7738 if (!dio_data->overwrite && start + len > i_size_read(inode)) 7739 i_size_write(inode, start + len); 7740 7741 WARN_ON(dio_data->reserve < len); 7742 dio_data->reserve -= len; 7743 dio_data->unsubmitted_oe_range_end = start + len; 7744 current->journal_info = dio_data; 7745 } 7746 7747 /* 7748 * In the case of write we need to clear and unlock the entire range, 7749 * in the case of read we need to unlock only the end area that we 7750 * aren't using if there is any left over space. 7751 */ 7752 if (lockstart < lockend) { 7753 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, 7754 lockend, unlock_bits, 1, 0, 7755 &cached_state); 7756 } else { 7757 free_extent_state(cached_state); 7758 } 7759 7760 free_extent_map(em); 7761 7762 return 0; 7763 7764 unlock_err: 7765 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7766 unlock_bits, 1, 0, &cached_state); 7767 err: 7768 if (dio_data) 7769 current->journal_info = dio_data; 7770 return ret; 7771 } 7772 7773 static inline blk_status_t submit_dio_repair_bio(struct inode *inode, 7774 struct bio *bio, 7775 int mirror_num) 7776 { 7777 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7778 blk_status_t ret; 7779 7780 BUG_ON(bio_op(bio) == REQ_OP_WRITE); 7781 7782 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR); 7783 if (ret) 7784 return ret; 7785 7786 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 7787 7788 return ret; 7789 } 7790 7791 static int btrfs_check_dio_repairable(struct inode *inode, 7792 struct bio *failed_bio, 7793 struct io_failure_record *failrec, 7794 int failed_mirror) 7795 { 7796 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7797 int num_copies; 7798 7799 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len); 7800 if (num_copies == 1) { 7801 /* 7802 * we only have a single copy of the data, so don't bother with 7803 * all the retry and error correction code that follows. no 7804 * matter what the error is, it is very likely to persist. 7805 */ 7806 btrfs_debug(fs_info, 7807 "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d", 7808 num_copies, failrec->this_mirror, failed_mirror); 7809 return 0; 7810 } 7811 7812 failrec->failed_mirror = failed_mirror; 7813 failrec->this_mirror++; 7814 if (failrec->this_mirror == failed_mirror) 7815 failrec->this_mirror++; 7816 7817 if (failrec->this_mirror > num_copies) { 7818 btrfs_debug(fs_info, 7819 "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d", 7820 num_copies, failrec->this_mirror, failed_mirror); 7821 return 0; 7822 } 7823 7824 return 1; 7825 } 7826 7827 static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio, 7828 struct page *page, unsigned int pgoff, 7829 u64 start, u64 end, int failed_mirror, 7830 bio_end_io_t *repair_endio, void *repair_arg) 7831 { 7832 struct io_failure_record *failrec; 7833 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7834 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 7835 struct bio *bio; 7836 int isector; 7837 unsigned int read_mode = 0; 7838 int segs; 7839 int ret; 7840 blk_status_t status; 7841 struct bio_vec bvec; 7842 7843 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); 7844 7845 ret = btrfs_get_io_failure_record(inode, start, end, &failrec); 7846 if (ret) 7847 return errno_to_blk_status(ret); 7848 7849 ret = btrfs_check_dio_repairable(inode, failed_bio, failrec, 7850 failed_mirror); 7851 if (!ret) { 7852 free_io_failure(failure_tree, io_tree, failrec); 7853 return BLK_STS_IOERR; 7854 } 7855 7856 segs = bio_segments(failed_bio); 7857 bio_get_first_bvec(failed_bio, &bvec); 7858 if (segs > 1 || 7859 (bvec.bv_len > btrfs_inode_sectorsize(inode))) 7860 read_mode |= REQ_FAILFAST_DEV; 7861 7862 isector = start - btrfs_io_bio(failed_bio)->logical; 7863 isector >>= inode->i_sb->s_blocksize_bits; 7864 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, 7865 pgoff, isector, repair_endio, repair_arg); 7866 bio_set_op_attrs(bio, REQ_OP_READ, read_mode); 7867 7868 btrfs_debug(BTRFS_I(inode)->root->fs_info, 7869 "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d", 7870 read_mode, failrec->this_mirror, failrec->in_validation); 7871 7872 status = submit_dio_repair_bio(inode, bio, failrec->this_mirror); 7873 if (status) { 7874 free_io_failure(failure_tree, io_tree, failrec); 7875 bio_put(bio); 7876 } 7877 7878 return status; 7879 } 7880 7881 struct btrfs_retry_complete { 7882 struct completion done; 7883 struct inode *inode; 7884 u64 start; 7885 int uptodate; 7886 }; 7887 7888 static void btrfs_retry_endio_nocsum(struct bio *bio) 7889 { 7890 struct btrfs_retry_complete *done = bio->bi_private; 7891 struct inode *inode = done->inode; 7892 struct bio_vec *bvec; 7893 struct extent_io_tree *io_tree, *failure_tree; 7894 int i; 7895 7896 if (bio->bi_status) 7897 goto end; 7898 7899 ASSERT(bio->bi_vcnt == 1); 7900 io_tree = &BTRFS_I(inode)->io_tree; 7901 failure_tree = &BTRFS_I(inode)->io_failure_tree; 7902 ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(inode)); 7903 7904 done->uptodate = 1; 7905 ASSERT(!bio_flagged(bio, BIO_CLONED)); 7906 bio_for_each_segment_all(bvec, bio, i) 7907 clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, 7908 io_tree, done->start, bvec->bv_page, 7909 btrfs_ino(BTRFS_I(inode)), 0); 7910 end: 7911 complete(&done->done); 7912 bio_put(bio); 7913 } 7914 7915 static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode, 7916 struct btrfs_io_bio *io_bio) 7917 { 7918 struct btrfs_fs_info *fs_info; 7919 struct bio_vec bvec; 7920 struct bvec_iter iter; 7921 struct btrfs_retry_complete done; 7922 u64 start; 7923 unsigned int pgoff; 7924 u32 sectorsize; 7925 int nr_sectors; 7926 blk_status_t ret; 7927 blk_status_t err = BLK_STS_OK; 7928 7929 fs_info = BTRFS_I(inode)->root->fs_info; 7930 sectorsize = fs_info->sectorsize; 7931 7932 start = io_bio->logical; 7933 done.inode = inode; 7934 io_bio->bio.bi_iter = io_bio->iter; 7935 7936 bio_for_each_segment(bvec, &io_bio->bio, iter) { 7937 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 7938 pgoff = bvec.bv_offset; 7939 7940 next_block_or_try_again: 7941 done.uptodate = 0; 7942 done.start = start; 7943 init_completion(&done.done); 7944 7945 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page, 7946 pgoff, start, start + sectorsize - 1, 7947 io_bio->mirror_num, 7948 btrfs_retry_endio_nocsum, &done); 7949 if (ret) { 7950 err = ret; 7951 goto next; 7952 } 7953 7954 wait_for_completion_io(&done.done); 7955 7956 if (!done.uptodate) { 7957 /* We might have another mirror, so try again */ 7958 goto next_block_or_try_again; 7959 } 7960 7961 next: 7962 start += sectorsize; 7963 7964 nr_sectors--; 7965 if (nr_sectors) { 7966 pgoff += sectorsize; 7967 ASSERT(pgoff < PAGE_SIZE); 7968 goto next_block_or_try_again; 7969 } 7970 } 7971 7972 return err; 7973 } 7974 7975 static void btrfs_retry_endio(struct bio *bio) 7976 { 7977 struct btrfs_retry_complete *done = bio->bi_private; 7978 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7979 struct extent_io_tree *io_tree, *failure_tree; 7980 struct inode *inode = done->inode; 7981 struct bio_vec *bvec; 7982 int uptodate; 7983 int ret; 7984 int i; 7985 7986 if (bio->bi_status) 7987 goto end; 7988 7989 uptodate = 1; 7990 7991 ASSERT(bio->bi_vcnt == 1); 7992 ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(done->inode)); 7993 7994 io_tree = &BTRFS_I(inode)->io_tree; 7995 failure_tree = &BTRFS_I(inode)->io_failure_tree; 7996 7997 ASSERT(!bio_flagged(bio, BIO_CLONED)); 7998 bio_for_each_segment_all(bvec, bio, i) { 7999 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page, 8000 bvec->bv_offset, done->start, 8001 bvec->bv_len); 8002 if (!ret) 8003 clean_io_failure(BTRFS_I(inode)->root->fs_info, 8004 failure_tree, io_tree, done->start, 8005 bvec->bv_page, 8006 btrfs_ino(BTRFS_I(inode)), 8007 bvec->bv_offset); 8008 else 8009 uptodate = 0; 8010 } 8011 8012 done->uptodate = uptodate; 8013 end: 8014 complete(&done->done); 8015 bio_put(bio); 8016 } 8017 8018 static blk_status_t __btrfs_subio_endio_read(struct inode *inode, 8019 struct btrfs_io_bio *io_bio, blk_status_t err) 8020 { 8021 struct btrfs_fs_info *fs_info; 8022 struct bio_vec bvec; 8023 struct bvec_iter iter; 8024 struct btrfs_retry_complete done; 8025 u64 start; 8026 u64 offset = 0; 8027 u32 sectorsize; 8028 int nr_sectors; 8029 unsigned int pgoff; 8030 int csum_pos; 8031 bool uptodate = (err == 0); 8032 int ret; 8033 blk_status_t status; 8034 8035 fs_info = BTRFS_I(inode)->root->fs_info; 8036 sectorsize = fs_info->sectorsize; 8037 8038 err = BLK_STS_OK; 8039 start = io_bio->logical; 8040 done.inode = inode; 8041 io_bio->bio.bi_iter = io_bio->iter; 8042 8043 bio_for_each_segment(bvec, &io_bio->bio, iter) { 8044 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 8045 8046 pgoff = bvec.bv_offset; 8047 next_block: 8048 if (uptodate) { 8049 csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset); 8050 ret = __readpage_endio_check(inode, io_bio, csum_pos, 8051 bvec.bv_page, pgoff, start, sectorsize); 8052 if (likely(!ret)) 8053 goto next; 8054 } 8055 try_again: 8056 done.uptodate = 0; 8057 done.start = start; 8058 init_completion(&done.done); 8059 8060 status = dio_read_error(inode, &io_bio->bio, bvec.bv_page, 8061 pgoff, start, start + sectorsize - 1, 8062 io_bio->mirror_num, btrfs_retry_endio, 8063 &done); 8064 if (status) { 8065 err = status; 8066 goto next; 8067 } 8068 8069 wait_for_completion_io(&done.done); 8070 8071 if (!done.uptodate) { 8072 /* We might have another mirror, so try again */ 8073 goto try_again; 8074 } 8075 next: 8076 offset += sectorsize; 8077 start += sectorsize; 8078 8079 ASSERT(nr_sectors); 8080 8081 nr_sectors--; 8082 if (nr_sectors) { 8083 pgoff += sectorsize; 8084 ASSERT(pgoff < PAGE_SIZE); 8085 goto next_block; 8086 } 8087 } 8088 8089 return err; 8090 } 8091 8092 static blk_status_t btrfs_subio_endio_read(struct inode *inode, 8093 struct btrfs_io_bio *io_bio, blk_status_t err) 8094 { 8095 bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 8096 8097 if (skip_csum) { 8098 if (unlikely(err)) 8099 return __btrfs_correct_data_nocsum(inode, io_bio); 8100 else 8101 return BLK_STS_OK; 8102 } else { 8103 return __btrfs_subio_endio_read(inode, io_bio, err); 8104 } 8105 } 8106 8107 static void btrfs_endio_direct_read(struct bio *bio) 8108 { 8109 struct btrfs_dio_private *dip = bio->bi_private; 8110 struct inode *inode = dip->inode; 8111 struct bio *dio_bio; 8112 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8113 blk_status_t err = bio->bi_status; 8114 8115 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) 8116 err = btrfs_subio_endio_read(inode, io_bio, err); 8117 8118 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset, 8119 dip->logical_offset + dip->bytes - 1); 8120 dio_bio = dip->dio_bio; 8121 8122 kfree(dip); 8123 8124 dio_bio->bi_status = err; 8125 dio_end_io(dio_bio); 8126 8127 if (io_bio->end_io) 8128 io_bio->end_io(io_bio, blk_status_to_errno(err)); 8129 bio_put(bio); 8130 } 8131 8132 static void __endio_write_update_ordered(struct inode *inode, 8133 const u64 offset, const u64 bytes, 8134 const bool uptodate) 8135 { 8136 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8137 struct btrfs_ordered_extent *ordered = NULL; 8138 struct btrfs_workqueue *wq; 8139 btrfs_work_func_t func; 8140 u64 ordered_offset = offset; 8141 u64 ordered_bytes = bytes; 8142 u64 last_offset; 8143 int ret; 8144 8145 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 8146 wq = fs_info->endio_freespace_worker; 8147 func = btrfs_freespace_write_helper; 8148 } else { 8149 wq = fs_info->endio_write_workers; 8150 func = btrfs_endio_write_helper; 8151 } 8152 8153 again: 8154 last_offset = ordered_offset; 8155 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, 8156 &ordered_offset, 8157 ordered_bytes, 8158 uptodate); 8159 if (!ret) 8160 goto out_test; 8161 8162 btrfs_init_work(&ordered->work, func, finish_ordered_fn, NULL, NULL); 8163 btrfs_queue_work(wq, &ordered->work); 8164 out_test: 8165 /* 8166 * If btrfs_dec_test_ordered_pending does not find any ordered extent 8167 * in the range, we can exit. 8168 */ 8169 if (ordered_offset == last_offset) 8170 return; 8171 /* 8172 * our bio might span multiple ordered extents. If we haven't 8173 * completed the accounting for the whole dio, go back and try again 8174 */ 8175 if (ordered_offset < offset + bytes) { 8176 ordered_bytes = offset + bytes - ordered_offset; 8177 ordered = NULL; 8178 goto again; 8179 } 8180 } 8181 8182 static void btrfs_endio_direct_write(struct bio *bio) 8183 { 8184 struct btrfs_dio_private *dip = bio->bi_private; 8185 struct bio *dio_bio = dip->dio_bio; 8186 8187 __endio_write_update_ordered(dip->inode, dip->logical_offset, 8188 dip->bytes, !bio->bi_status); 8189 8190 kfree(dip); 8191 8192 dio_bio->bi_status = bio->bi_status; 8193 dio_end_io(dio_bio); 8194 bio_put(bio); 8195 } 8196 8197 static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data, 8198 struct bio *bio, u64 offset) 8199 { 8200 struct inode *inode = private_data; 8201 blk_status_t ret; 8202 ret = btrfs_csum_one_bio(inode, bio, offset, 1); 8203 BUG_ON(ret); /* -ENOMEM */ 8204 return 0; 8205 } 8206 8207 static void btrfs_end_dio_bio(struct bio *bio) 8208 { 8209 struct btrfs_dio_private *dip = bio->bi_private; 8210 blk_status_t err = bio->bi_status; 8211 8212 if (err) 8213 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 8214 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", 8215 btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio), 8216 bio->bi_opf, 8217 (unsigned long long)bio->bi_iter.bi_sector, 8218 bio->bi_iter.bi_size, err); 8219 8220 if (dip->subio_endio) 8221 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err); 8222 8223 if (err) { 8224 /* 8225 * We want to perceive the errors flag being set before 8226 * decrementing the reference count. We don't need a barrier 8227 * since atomic operations with a return value are fully 8228 * ordered as per atomic_t.txt 8229 */ 8230 dip->errors = 1; 8231 } 8232 8233 /* if there are more bios still pending for this dio, just exit */ 8234 if (!atomic_dec_and_test(&dip->pending_bios)) 8235 goto out; 8236 8237 if (dip->errors) { 8238 bio_io_error(dip->orig_bio); 8239 } else { 8240 dip->dio_bio->bi_status = BLK_STS_OK; 8241 bio_endio(dip->orig_bio); 8242 } 8243 out: 8244 bio_put(bio); 8245 } 8246 8247 static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode, 8248 struct btrfs_dio_private *dip, 8249 struct bio *bio, 8250 u64 file_offset) 8251 { 8252 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8253 struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio); 8254 blk_status_t ret; 8255 8256 /* 8257 * We load all the csum data we need when we submit 8258 * the first bio to reduce the csum tree search and 8259 * contention. 8260 */ 8261 if (dip->logical_offset == file_offset) { 8262 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio, 8263 file_offset); 8264 if (ret) 8265 return ret; 8266 } 8267 8268 if (bio == dip->orig_bio) 8269 return 0; 8270 8271 file_offset -= dip->logical_offset; 8272 file_offset >>= inode->i_sb->s_blocksize_bits; 8273 io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset); 8274 8275 return 0; 8276 } 8277 8278 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, 8279 struct inode *inode, u64 file_offset, int async_submit) 8280 { 8281 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8282 struct btrfs_dio_private *dip = bio->bi_private; 8283 bool write = bio_op(bio) == REQ_OP_WRITE; 8284 blk_status_t ret; 8285 8286 /* Check btrfs_submit_bio_hook() for rules about async submit. */ 8287 if (async_submit) 8288 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers); 8289 8290 if (!write) { 8291 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 8292 if (ret) 8293 goto err; 8294 } 8295 8296 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 8297 goto map; 8298 8299 if (write && async_submit) { 8300 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0, 8301 file_offset, inode, 8302 btrfs_submit_bio_start_direct_io, 8303 btrfs_submit_bio_done); 8304 goto err; 8305 } else if (write) { 8306 /* 8307 * If we aren't doing async submit, calculate the csum of the 8308 * bio now. 8309 */ 8310 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1); 8311 if (ret) 8312 goto err; 8313 } else { 8314 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio, 8315 file_offset); 8316 if (ret) 8317 goto err; 8318 } 8319 map: 8320 ret = btrfs_map_bio(fs_info, bio, 0, 0); 8321 err: 8322 return ret; 8323 } 8324 8325 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) 8326 { 8327 struct inode *inode = dip->inode; 8328 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8329 struct bio *bio; 8330 struct bio *orig_bio = dip->orig_bio; 8331 u64 start_sector = orig_bio->bi_iter.bi_sector; 8332 u64 file_offset = dip->logical_offset; 8333 u64 map_length; 8334 int async_submit = 0; 8335 u64 submit_len; 8336 int clone_offset = 0; 8337 int clone_len; 8338 int ret; 8339 blk_status_t status; 8340 8341 map_length = orig_bio->bi_iter.bi_size; 8342 submit_len = map_length; 8343 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9, 8344 &map_length, NULL, 0); 8345 if (ret) 8346 return -EIO; 8347 8348 if (map_length >= submit_len) { 8349 bio = orig_bio; 8350 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED; 8351 goto submit; 8352 } 8353 8354 /* async crcs make it difficult to collect full stripe writes. */ 8355 if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK) 8356 async_submit = 0; 8357 else 8358 async_submit = 1; 8359 8360 /* bio split */ 8361 ASSERT(map_length <= INT_MAX); 8362 atomic_inc(&dip->pending_bios); 8363 do { 8364 clone_len = min_t(int, submit_len, map_length); 8365 8366 /* 8367 * This will never fail as it's passing GPF_NOFS and 8368 * the allocation is backed by btrfs_bioset. 8369 */ 8370 bio = btrfs_bio_clone_partial(orig_bio, clone_offset, 8371 clone_len); 8372 bio->bi_private = dip; 8373 bio->bi_end_io = btrfs_end_dio_bio; 8374 btrfs_io_bio(bio)->logical = file_offset; 8375 8376 ASSERT(submit_len >= clone_len); 8377 submit_len -= clone_len; 8378 if (submit_len == 0) 8379 break; 8380 8381 /* 8382 * Increase the count before we submit the bio so we know 8383 * the end IO handler won't happen before we increase the 8384 * count. Otherwise, the dip might get freed before we're 8385 * done setting it up. 8386 */ 8387 atomic_inc(&dip->pending_bios); 8388 8389 status = btrfs_submit_dio_bio(bio, inode, file_offset, 8390 async_submit); 8391 if (status) { 8392 bio_put(bio); 8393 atomic_dec(&dip->pending_bios); 8394 goto out_err; 8395 } 8396 8397 clone_offset += clone_len; 8398 start_sector += clone_len >> 9; 8399 file_offset += clone_len; 8400 8401 map_length = submit_len; 8402 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), 8403 start_sector << 9, &map_length, NULL, 0); 8404 if (ret) 8405 goto out_err; 8406 } while (submit_len > 0); 8407 8408 submit: 8409 status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit); 8410 if (!status) 8411 return 0; 8412 8413 bio_put(bio); 8414 out_err: 8415 dip->errors = 1; 8416 /* 8417 * Before atomic variable goto zero, we must make sure dip->errors is 8418 * perceived to be set. This ordering is ensured by the fact that an 8419 * atomic operations with a return value are fully ordered as per 8420 * atomic_t.txt 8421 */ 8422 if (atomic_dec_and_test(&dip->pending_bios)) 8423 bio_io_error(dip->orig_bio); 8424 8425 /* bio_end_io() will handle error, so we needn't return it */ 8426 return 0; 8427 } 8428 8429 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, 8430 loff_t file_offset) 8431 { 8432 struct btrfs_dio_private *dip = NULL; 8433 struct bio *bio = NULL; 8434 struct btrfs_io_bio *io_bio; 8435 bool write = (bio_op(dio_bio) == REQ_OP_WRITE); 8436 int ret = 0; 8437 8438 bio = btrfs_bio_clone(dio_bio); 8439 8440 dip = kzalloc(sizeof(*dip), GFP_NOFS); 8441 if (!dip) { 8442 ret = -ENOMEM; 8443 goto free_ordered; 8444 } 8445 8446 dip->private = dio_bio->bi_private; 8447 dip->inode = inode; 8448 dip->logical_offset = file_offset; 8449 dip->bytes = dio_bio->bi_iter.bi_size; 8450 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; 8451 bio->bi_private = dip; 8452 dip->orig_bio = bio; 8453 dip->dio_bio = dio_bio; 8454 atomic_set(&dip->pending_bios, 0); 8455 io_bio = btrfs_io_bio(bio); 8456 io_bio->logical = file_offset; 8457 8458 if (write) { 8459 bio->bi_end_io = btrfs_endio_direct_write; 8460 } else { 8461 bio->bi_end_io = btrfs_endio_direct_read; 8462 dip->subio_endio = btrfs_subio_endio_read; 8463 } 8464 8465 /* 8466 * Reset the range for unsubmitted ordered extents (to a 0 length range) 8467 * even if we fail to submit a bio, because in such case we do the 8468 * corresponding error handling below and it must not be done a second 8469 * time by btrfs_direct_IO(). 8470 */ 8471 if (write) { 8472 struct btrfs_dio_data *dio_data = current->journal_info; 8473 8474 dio_data->unsubmitted_oe_range_end = dip->logical_offset + 8475 dip->bytes; 8476 dio_data->unsubmitted_oe_range_start = 8477 dio_data->unsubmitted_oe_range_end; 8478 } 8479 8480 ret = btrfs_submit_direct_hook(dip); 8481 if (!ret) 8482 return; 8483 8484 if (io_bio->end_io) 8485 io_bio->end_io(io_bio, ret); 8486 8487 free_ordered: 8488 /* 8489 * If we arrived here it means either we failed to submit the dip 8490 * or we either failed to clone the dio_bio or failed to allocate the 8491 * dip. If we cloned the dio_bio and allocated the dip, we can just 8492 * call bio_endio against our io_bio so that we get proper resource 8493 * cleanup if we fail to submit the dip, otherwise, we must do the 8494 * same as btrfs_endio_direct_[write|read] because we can't call these 8495 * callbacks - they require an allocated dip and a clone of dio_bio. 8496 */ 8497 if (bio && dip) { 8498 bio_io_error(bio); 8499 /* 8500 * The end io callbacks free our dip, do the final put on bio 8501 * and all the cleanup and final put for dio_bio (through 8502 * dio_end_io()). 8503 */ 8504 dip = NULL; 8505 bio = NULL; 8506 } else { 8507 if (write) 8508 __endio_write_update_ordered(inode, 8509 file_offset, 8510 dio_bio->bi_iter.bi_size, 8511 false); 8512 else 8513 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, 8514 file_offset + dio_bio->bi_iter.bi_size - 1); 8515 8516 dio_bio->bi_status = BLK_STS_IOERR; 8517 /* 8518 * Releases and cleans up our dio_bio, no need to bio_put() 8519 * nor bio_endio()/bio_io_error() against dio_bio. 8520 */ 8521 dio_end_io(dio_bio); 8522 } 8523 if (bio) 8524 bio_put(bio); 8525 kfree(dip); 8526 } 8527 8528 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, 8529 const struct iov_iter *iter, loff_t offset) 8530 { 8531 int seg; 8532 int i; 8533 unsigned int blocksize_mask = fs_info->sectorsize - 1; 8534 ssize_t retval = -EINVAL; 8535 8536 if (offset & blocksize_mask) 8537 goto out; 8538 8539 if (iov_iter_alignment(iter) & blocksize_mask) 8540 goto out; 8541 8542 /* If this is a write we don't need to check anymore */ 8543 if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter)) 8544 return 0; 8545 /* 8546 * Check to make sure we don't have duplicate iov_base's in this 8547 * iovec, if so return EINVAL, otherwise we'll get csum errors 8548 * when reading back. 8549 */ 8550 for (seg = 0; seg < iter->nr_segs; seg++) { 8551 for (i = seg + 1; i < iter->nr_segs; i++) { 8552 if (iter->iov[seg].iov_base == iter->iov[i].iov_base) 8553 goto out; 8554 } 8555 } 8556 retval = 0; 8557 out: 8558 return retval; 8559 } 8560 8561 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 8562 { 8563 struct file *file = iocb->ki_filp; 8564 struct inode *inode = file->f_mapping->host; 8565 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8566 struct btrfs_dio_data dio_data = { 0 }; 8567 struct extent_changeset *data_reserved = NULL; 8568 loff_t offset = iocb->ki_pos; 8569 size_t count = 0; 8570 int flags = 0; 8571 bool wakeup = true; 8572 bool relock = false; 8573 ssize_t ret; 8574 8575 if (check_direct_IO(fs_info, iter, offset)) 8576 return 0; 8577 8578 inode_dio_begin(inode); 8579 8580 /* 8581 * The generic stuff only does filemap_write_and_wait_range, which 8582 * isn't enough if we've written compressed pages to this area, so 8583 * we need to flush the dirty pages again to make absolutely sure 8584 * that any outstanding dirty pages are on disk. 8585 */ 8586 count = iov_iter_count(iter); 8587 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 8588 &BTRFS_I(inode)->runtime_flags)) 8589 filemap_fdatawrite_range(inode->i_mapping, offset, 8590 offset + count - 1); 8591 8592 if (iov_iter_rw(iter) == WRITE) { 8593 /* 8594 * If the write DIO is beyond the EOF, we need update 8595 * the isize, but it is protected by i_mutex. So we can 8596 * not unlock the i_mutex at this case. 8597 */ 8598 if (offset + count <= inode->i_size) { 8599 dio_data.overwrite = 1; 8600 inode_unlock(inode); 8601 relock = true; 8602 } else if (iocb->ki_flags & IOCB_NOWAIT) { 8603 ret = -EAGAIN; 8604 goto out; 8605 } 8606 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, 8607 offset, count); 8608 if (ret) 8609 goto out; 8610 8611 /* 8612 * We need to know how many extents we reserved so that we can 8613 * do the accounting properly if we go over the number we 8614 * originally calculated. Abuse current->journal_info for this. 8615 */ 8616 dio_data.reserve = round_up(count, 8617 fs_info->sectorsize); 8618 dio_data.unsubmitted_oe_range_start = (u64)offset; 8619 dio_data.unsubmitted_oe_range_end = (u64)offset; 8620 current->journal_info = &dio_data; 8621 down_read(&BTRFS_I(inode)->dio_sem); 8622 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK, 8623 &BTRFS_I(inode)->runtime_flags)) { 8624 inode_dio_end(inode); 8625 flags = DIO_LOCKING | DIO_SKIP_HOLES; 8626 wakeup = false; 8627 } 8628 8629 ret = __blockdev_direct_IO(iocb, inode, 8630 fs_info->fs_devices->latest_bdev, 8631 iter, btrfs_get_blocks_direct, NULL, 8632 btrfs_submit_direct, flags); 8633 if (iov_iter_rw(iter) == WRITE) { 8634 up_read(&BTRFS_I(inode)->dio_sem); 8635 current->journal_info = NULL; 8636 if (ret < 0 && ret != -EIOCBQUEUED) { 8637 if (dio_data.reserve) 8638 btrfs_delalloc_release_space(inode, data_reserved, 8639 offset, dio_data.reserve, true); 8640 /* 8641 * On error we might have left some ordered extents 8642 * without submitting corresponding bios for them, so 8643 * cleanup them up to avoid other tasks getting them 8644 * and waiting for them to complete forever. 8645 */ 8646 if (dio_data.unsubmitted_oe_range_start < 8647 dio_data.unsubmitted_oe_range_end) 8648 __endio_write_update_ordered(inode, 8649 dio_data.unsubmitted_oe_range_start, 8650 dio_data.unsubmitted_oe_range_end - 8651 dio_data.unsubmitted_oe_range_start, 8652 false); 8653 } else if (ret >= 0 && (size_t)ret < count) 8654 btrfs_delalloc_release_space(inode, data_reserved, 8655 offset, count - (size_t)ret, true); 8656 btrfs_delalloc_release_extents(BTRFS_I(inode), count, false); 8657 } 8658 out: 8659 if (wakeup) 8660 inode_dio_end(inode); 8661 if (relock) 8662 inode_lock(inode); 8663 8664 extent_changeset_free(data_reserved); 8665 return ret; 8666 } 8667 8668 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC) 8669 8670 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 8671 __u64 start, __u64 len) 8672 { 8673 int ret; 8674 8675 ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS); 8676 if (ret) 8677 return ret; 8678 8679 return extent_fiemap(inode, fieinfo, start, len); 8680 } 8681 8682 int btrfs_readpage(struct file *file, struct page *page) 8683 { 8684 struct extent_io_tree *tree; 8685 tree = &BTRFS_I(page->mapping->host)->io_tree; 8686 return extent_read_full_page(tree, page, btrfs_get_extent, 0); 8687 } 8688 8689 static int btrfs_writepage(struct page *page, struct writeback_control *wbc) 8690 { 8691 struct inode *inode = page->mapping->host; 8692 int ret; 8693 8694 if (current->flags & PF_MEMALLOC) { 8695 redirty_page_for_writepage(wbc, page); 8696 unlock_page(page); 8697 return 0; 8698 } 8699 8700 /* 8701 * If we are under memory pressure we will call this directly from the 8702 * VM, we need to make sure we have the inode referenced for the ordered 8703 * extent. If not just return like we didn't do anything. 8704 */ 8705 if (!igrab(inode)) { 8706 redirty_page_for_writepage(wbc, page); 8707 return AOP_WRITEPAGE_ACTIVATE; 8708 } 8709 ret = extent_write_full_page(page, wbc); 8710 btrfs_add_delayed_iput(inode); 8711 return ret; 8712 } 8713 8714 static int btrfs_writepages(struct address_space *mapping, 8715 struct writeback_control *wbc) 8716 { 8717 struct extent_io_tree *tree; 8718 8719 tree = &BTRFS_I(mapping->host)->io_tree; 8720 return extent_writepages(tree, mapping, wbc); 8721 } 8722 8723 static int 8724 btrfs_readpages(struct file *file, struct address_space *mapping, 8725 struct list_head *pages, unsigned nr_pages) 8726 { 8727 struct extent_io_tree *tree; 8728 tree = &BTRFS_I(mapping->host)->io_tree; 8729 return extent_readpages(tree, mapping, pages, nr_pages); 8730 } 8731 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8732 { 8733 struct extent_io_tree *tree; 8734 struct extent_map_tree *map; 8735 int ret; 8736 8737 tree = &BTRFS_I(page->mapping->host)->io_tree; 8738 map = &BTRFS_I(page->mapping->host)->extent_tree; 8739 ret = try_release_extent_mapping(map, tree, page, gfp_flags); 8740 if (ret == 1) { 8741 ClearPagePrivate(page); 8742 set_page_private(page, 0); 8743 put_page(page); 8744 } 8745 return ret; 8746 } 8747 8748 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8749 { 8750 if (PageWriteback(page) || PageDirty(page)) 8751 return 0; 8752 return __btrfs_releasepage(page, gfp_flags); 8753 } 8754 8755 static void btrfs_invalidatepage(struct page *page, unsigned int offset, 8756 unsigned int length) 8757 { 8758 struct inode *inode = page->mapping->host; 8759 struct extent_io_tree *tree; 8760 struct btrfs_ordered_extent *ordered; 8761 struct extent_state *cached_state = NULL; 8762 u64 page_start = page_offset(page); 8763 u64 page_end = page_start + PAGE_SIZE - 1; 8764 u64 start; 8765 u64 end; 8766 int inode_evicting = inode->i_state & I_FREEING; 8767 8768 /* 8769 * we have the page locked, so new writeback can't start, 8770 * and the dirty bit won't be cleared while we are here. 8771 * 8772 * Wait for IO on this page so that we can safely clear 8773 * the PagePrivate2 bit and do ordered accounting 8774 */ 8775 wait_on_page_writeback(page); 8776 8777 tree = &BTRFS_I(inode)->io_tree; 8778 if (offset) { 8779 btrfs_releasepage(page, GFP_NOFS); 8780 return; 8781 } 8782 8783 if (!inode_evicting) 8784 lock_extent_bits(tree, page_start, page_end, &cached_state); 8785 again: 8786 start = page_start; 8787 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start, 8788 page_end - start + 1); 8789 if (ordered) { 8790 end = min(page_end, ordered->file_offset + ordered->len - 1); 8791 /* 8792 * IO on this page will never be started, so we need 8793 * to account for any ordered extents now 8794 */ 8795 if (!inode_evicting) 8796 clear_extent_bit(tree, start, end, 8797 EXTENT_DIRTY | EXTENT_DELALLOC | 8798 EXTENT_DELALLOC_NEW | 8799 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 8800 EXTENT_DEFRAG, 1, 0, &cached_state); 8801 /* 8802 * whoever cleared the private bit is responsible 8803 * for the finish_ordered_io 8804 */ 8805 if (TestClearPagePrivate2(page)) { 8806 struct btrfs_ordered_inode_tree *tree; 8807 u64 new_len; 8808 8809 tree = &BTRFS_I(inode)->ordered_tree; 8810 8811 spin_lock_irq(&tree->lock); 8812 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 8813 new_len = start - ordered->file_offset; 8814 if (new_len < ordered->truncated_len) 8815 ordered->truncated_len = new_len; 8816 spin_unlock_irq(&tree->lock); 8817 8818 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8819 start, 8820 end - start + 1, 1)) 8821 btrfs_finish_ordered_io(ordered); 8822 } 8823 btrfs_put_ordered_extent(ordered); 8824 if (!inode_evicting) { 8825 cached_state = NULL; 8826 lock_extent_bits(tree, start, end, 8827 &cached_state); 8828 } 8829 8830 start = end + 1; 8831 if (start < page_end) 8832 goto again; 8833 } 8834 8835 /* 8836 * Qgroup reserved space handler 8837 * Page here will be either 8838 * 1) Already written to disk 8839 * In this case, its reserved space is released from data rsv map 8840 * and will be freed by delayed_ref handler finally. 8841 * So even we call qgroup_free_data(), it won't decrease reserved 8842 * space. 8843 * 2) Not written to disk 8844 * This means the reserved space should be freed here. However, 8845 * if a truncate invalidates the page (by clearing PageDirty) 8846 * and the page is accounted for while allocating extent 8847 * in btrfs_check_data_free_space() we let delayed_ref to 8848 * free the entire extent. 8849 */ 8850 if (PageDirty(page)) 8851 btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE); 8852 if (!inode_evicting) { 8853 clear_extent_bit(tree, page_start, page_end, 8854 EXTENT_LOCKED | EXTENT_DIRTY | 8855 EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 8856 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1, 8857 &cached_state); 8858 8859 __btrfs_releasepage(page, GFP_NOFS); 8860 } 8861 8862 ClearPageChecked(page); 8863 if (PagePrivate(page)) { 8864 ClearPagePrivate(page); 8865 set_page_private(page, 0); 8866 put_page(page); 8867 } 8868 } 8869 8870 /* 8871 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8872 * called from a page fault handler when a page is first dirtied. Hence we must 8873 * be careful to check for EOF conditions here. We set the page up correctly 8874 * for a written page which means we get ENOSPC checking when writing into 8875 * holes and correct delalloc and unwritten extent mapping on filesystems that 8876 * support these features. 8877 * 8878 * We are not allowed to take the i_mutex here so we have to play games to 8879 * protect against truncate races as the page could now be beyond EOF. Because 8880 * vmtruncate() writes the inode size before removing pages, once we have the 8881 * page lock we can determine safely if the page is beyond EOF. If it is not 8882 * beyond EOF, then the page is guaranteed safe against truncation until we 8883 * unlock the page. 8884 */ 8885 int btrfs_page_mkwrite(struct vm_fault *vmf) 8886 { 8887 struct page *page = vmf->page; 8888 struct inode *inode = file_inode(vmf->vma->vm_file); 8889 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8890 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8891 struct btrfs_ordered_extent *ordered; 8892 struct extent_state *cached_state = NULL; 8893 struct extent_changeset *data_reserved = NULL; 8894 char *kaddr; 8895 unsigned long zero_start; 8896 loff_t size; 8897 int ret; 8898 int reserved = 0; 8899 u64 reserved_space; 8900 u64 page_start; 8901 u64 page_end; 8902 u64 end; 8903 8904 reserved_space = PAGE_SIZE; 8905 8906 sb_start_pagefault(inode->i_sb); 8907 page_start = page_offset(page); 8908 page_end = page_start + PAGE_SIZE - 1; 8909 end = page_end; 8910 8911 /* 8912 * Reserving delalloc space after obtaining the page lock can lead to 8913 * deadlock. For example, if a dirty page is locked by this function 8914 * and the call to btrfs_delalloc_reserve_space() ends up triggering 8915 * dirty page write out, then the btrfs_writepage() function could 8916 * end up waiting indefinitely to get a lock on the page currently 8917 * being processed by btrfs_page_mkwrite() function. 8918 */ 8919 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 8920 reserved_space); 8921 if (!ret) { 8922 ret = file_update_time(vmf->vma->vm_file); 8923 reserved = 1; 8924 } 8925 if (ret) { 8926 if (ret == -ENOMEM) 8927 ret = VM_FAULT_OOM; 8928 else /* -ENOSPC, -EIO, etc */ 8929 ret = VM_FAULT_SIGBUS; 8930 if (reserved) 8931 goto out; 8932 goto out_noreserve; 8933 } 8934 8935 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 8936 again: 8937 lock_page(page); 8938 size = i_size_read(inode); 8939 8940 if ((page->mapping != inode->i_mapping) || 8941 (page_start >= size)) { 8942 /* page got truncated out from underneath us */ 8943 goto out_unlock; 8944 } 8945 wait_on_page_writeback(page); 8946 8947 lock_extent_bits(io_tree, page_start, page_end, &cached_state); 8948 set_page_extent_mapped(page); 8949 8950 /* 8951 * we can't set the delalloc bits if there are pending ordered 8952 * extents. Drop our locks and wait for them to finish 8953 */ 8954 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 8955 PAGE_SIZE); 8956 if (ordered) { 8957 unlock_extent_cached(io_tree, page_start, page_end, 8958 &cached_state); 8959 unlock_page(page); 8960 btrfs_start_ordered_extent(inode, ordered, 1); 8961 btrfs_put_ordered_extent(ordered); 8962 goto again; 8963 } 8964 8965 if (page->index == ((size - 1) >> PAGE_SHIFT)) { 8966 reserved_space = round_up(size - page_start, 8967 fs_info->sectorsize); 8968 if (reserved_space < PAGE_SIZE) { 8969 end = page_start + reserved_space - 1; 8970 btrfs_delalloc_release_space(inode, data_reserved, 8971 page_start, PAGE_SIZE - reserved_space, 8972 true); 8973 } 8974 } 8975 8976 /* 8977 * page_mkwrite gets called when the page is firstly dirtied after it's 8978 * faulted in, but write(2) could also dirty a page and set delalloc 8979 * bits, thus in this case for space account reason, we still need to 8980 * clear any delalloc bits within this page range since we have to 8981 * reserve data&meta space before lock_page() (see above comments). 8982 */ 8983 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, 8984 EXTENT_DIRTY | EXTENT_DELALLOC | 8985 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 8986 0, 0, &cached_state); 8987 8988 ret = btrfs_set_extent_delalloc(inode, page_start, end, 0, 8989 &cached_state, 0); 8990 if (ret) { 8991 unlock_extent_cached(io_tree, page_start, page_end, 8992 &cached_state); 8993 ret = VM_FAULT_SIGBUS; 8994 goto out_unlock; 8995 } 8996 ret = 0; 8997 8998 /* page is wholly or partially inside EOF */ 8999 if (page_start + PAGE_SIZE > size) 9000 zero_start = size & ~PAGE_MASK; 9001 else 9002 zero_start = PAGE_SIZE; 9003 9004 if (zero_start != PAGE_SIZE) { 9005 kaddr = kmap(page); 9006 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start); 9007 flush_dcache_page(page); 9008 kunmap(page); 9009 } 9010 ClearPageChecked(page); 9011 set_page_dirty(page); 9012 SetPageUptodate(page); 9013 9014 BTRFS_I(inode)->last_trans = fs_info->generation; 9015 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; 9016 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; 9017 9018 unlock_extent_cached(io_tree, page_start, page_end, &cached_state); 9019 9020 out_unlock: 9021 if (!ret) { 9022 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, true); 9023 sb_end_pagefault(inode->i_sb); 9024 extent_changeset_free(data_reserved); 9025 return VM_FAULT_LOCKED; 9026 } 9027 unlock_page(page); 9028 out: 9029 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, (ret != 0)); 9030 btrfs_delalloc_release_space(inode, data_reserved, page_start, 9031 reserved_space, (ret != 0)); 9032 out_noreserve: 9033 sb_end_pagefault(inode->i_sb); 9034 extent_changeset_free(data_reserved); 9035 return ret; 9036 } 9037 9038 static int btrfs_truncate(struct inode *inode, bool skip_writeback) 9039 { 9040 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9041 struct btrfs_root *root = BTRFS_I(inode)->root; 9042 struct btrfs_block_rsv *rsv; 9043 int ret = 0; 9044 int err = 0; 9045 struct btrfs_trans_handle *trans; 9046 u64 mask = fs_info->sectorsize - 1; 9047 u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 9048 9049 if (!skip_writeback) { 9050 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), 9051 (u64)-1); 9052 if (ret) 9053 return ret; 9054 } 9055 9056 /* 9057 * Yes ladies and gentlemen, this is indeed ugly. The fact is we have 9058 * 3 things going on here 9059 * 9060 * 1) We need to reserve space for our orphan item and the space to 9061 * delete our orphan item. Lord knows we don't want to have a dangling 9062 * orphan item because we didn't reserve space to remove it. 9063 * 9064 * 2) We need to reserve space to update our inode. 9065 * 9066 * 3) We need to have something to cache all the space that is going to 9067 * be free'd up by the truncate operation, but also have some slack 9068 * space reserved in case it uses space during the truncate (thank you 9069 * very much snapshotting). 9070 * 9071 * And we need these to all be separate. The fact is we can use a lot of 9072 * space doing the truncate, and we have no earthly idea how much space 9073 * we will use, so we need the truncate reservation to be separate so it 9074 * doesn't end up using space reserved for updating the inode or 9075 * removing the orphan item. We also need to be able to stop the 9076 * transaction and start a new one, which means we need to be able to 9077 * update the inode several times, and we have no idea of knowing how 9078 * many times that will be, so we can't just reserve 1 item for the 9079 * entirety of the operation, so that has to be done separately as well. 9080 * Then there is the orphan item, which does indeed need to be held on 9081 * to for the whole operation, and we need nobody to touch this reserved 9082 * space except the orphan code. 9083 * 9084 * So that leaves us with 9085 * 9086 * 1) root->orphan_block_rsv - for the orphan deletion. 9087 * 2) rsv - for the truncate reservation, which we will steal from the 9088 * transaction reservation. 9089 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for 9090 * updating the inode. 9091 */ 9092 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 9093 if (!rsv) 9094 return -ENOMEM; 9095 rsv->size = min_size; 9096 rsv->failfast = 1; 9097 9098 /* 9099 * 1 for the truncate slack space 9100 * 1 for updating the inode. 9101 */ 9102 trans = btrfs_start_transaction(root, 2); 9103 if (IS_ERR(trans)) { 9104 err = PTR_ERR(trans); 9105 goto out; 9106 } 9107 9108 /* Migrate the slack space for the truncate to our reserve */ 9109 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv, 9110 min_size, 0); 9111 BUG_ON(ret); 9112 9113 /* 9114 * So if we truncate and then write and fsync we normally would just 9115 * write the extents that changed, which is a problem if we need to 9116 * first truncate that entire inode. So set this flag so we write out 9117 * all of the extents in the inode to the sync log so we're completely 9118 * safe. 9119 */ 9120 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 9121 trans->block_rsv = rsv; 9122 9123 while (1) { 9124 ret = btrfs_truncate_inode_items(trans, root, inode, 9125 inode->i_size, 9126 BTRFS_EXTENT_DATA_KEY); 9127 trans->block_rsv = &fs_info->trans_block_rsv; 9128 if (ret != -ENOSPC && ret != -EAGAIN) { 9129 err = ret; 9130 break; 9131 } 9132 9133 ret = btrfs_update_inode(trans, root, inode); 9134 if (ret) { 9135 err = ret; 9136 break; 9137 } 9138 9139 btrfs_end_transaction(trans); 9140 btrfs_btree_balance_dirty(fs_info); 9141 9142 trans = btrfs_start_transaction(root, 2); 9143 if (IS_ERR(trans)) { 9144 ret = err = PTR_ERR(trans); 9145 trans = NULL; 9146 break; 9147 } 9148 9149 btrfs_block_rsv_release(fs_info, rsv, -1); 9150 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 9151 rsv, min_size, 0); 9152 BUG_ON(ret); /* shouldn't happen */ 9153 trans->block_rsv = rsv; 9154 } 9155 9156 /* 9157 * We can't call btrfs_truncate_block inside a trans handle as we could 9158 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know 9159 * we've truncated everything except the last little bit, and can do 9160 * btrfs_truncate_block and then update the disk_i_size. 9161 */ 9162 if (ret == NEED_TRUNCATE_BLOCK) { 9163 btrfs_end_transaction(trans); 9164 btrfs_btree_balance_dirty(fs_info); 9165 9166 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0); 9167 if (ret) 9168 goto out; 9169 trans = btrfs_start_transaction(root, 1); 9170 if (IS_ERR(trans)) { 9171 ret = PTR_ERR(trans); 9172 goto out; 9173 } 9174 btrfs_ordered_update_i_size(inode, inode->i_size, NULL); 9175 } 9176 9177 if (ret == 0 && inode->i_nlink > 0) { 9178 trans->block_rsv = root->orphan_block_rsv; 9179 ret = btrfs_orphan_del(trans, BTRFS_I(inode)); 9180 if (ret) 9181 err = ret; 9182 } 9183 9184 if (trans) { 9185 trans->block_rsv = &fs_info->trans_block_rsv; 9186 ret = btrfs_update_inode(trans, root, inode); 9187 if (ret && !err) 9188 err = ret; 9189 9190 ret = btrfs_end_transaction(trans); 9191 btrfs_btree_balance_dirty(fs_info); 9192 } 9193 out: 9194 btrfs_free_block_rsv(fs_info, rsv); 9195 9196 if (ret && !err) 9197 err = ret; 9198 9199 return err; 9200 } 9201 9202 /* 9203 * create a new subvolume directory/inode (helper for the ioctl). 9204 */ 9205 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, 9206 struct btrfs_root *new_root, 9207 struct btrfs_root *parent_root, 9208 u64 new_dirid) 9209 { 9210 struct inode *inode; 9211 int err; 9212 u64 index = 0; 9213 9214 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, 9215 new_dirid, new_dirid, 9216 S_IFDIR | (~current_umask() & S_IRWXUGO), 9217 &index); 9218 if (IS_ERR(inode)) 9219 return PTR_ERR(inode); 9220 inode->i_op = &btrfs_dir_inode_operations; 9221 inode->i_fop = &btrfs_dir_file_operations; 9222 9223 set_nlink(inode, 1); 9224 btrfs_i_size_write(BTRFS_I(inode), 0); 9225 unlock_new_inode(inode); 9226 9227 err = btrfs_subvol_inherit_props(trans, new_root, parent_root); 9228 if (err) 9229 btrfs_err(new_root->fs_info, 9230 "error inheriting subvolume %llu properties: %d", 9231 new_root->root_key.objectid, err); 9232 9233 err = btrfs_update_inode(trans, new_root, inode); 9234 9235 iput(inode); 9236 return err; 9237 } 9238 9239 struct inode *btrfs_alloc_inode(struct super_block *sb) 9240 { 9241 struct btrfs_fs_info *fs_info = btrfs_sb(sb); 9242 struct btrfs_inode *ei; 9243 struct inode *inode; 9244 9245 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL); 9246 if (!ei) 9247 return NULL; 9248 9249 ei->root = NULL; 9250 ei->generation = 0; 9251 ei->last_trans = 0; 9252 ei->last_sub_trans = 0; 9253 ei->logged_trans = 0; 9254 ei->delalloc_bytes = 0; 9255 ei->new_delalloc_bytes = 0; 9256 ei->defrag_bytes = 0; 9257 ei->disk_i_size = 0; 9258 ei->flags = 0; 9259 ei->csum_bytes = 0; 9260 ei->index_cnt = (u64)-1; 9261 ei->dir_index = 0; 9262 ei->last_unlink_trans = 0; 9263 ei->last_log_commit = 0; 9264 9265 spin_lock_init(&ei->lock); 9266 ei->outstanding_extents = 0; 9267 if (sb->s_magic != BTRFS_TEST_MAGIC) 9268 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, 9269 BTRFS_BLOCK_RSV_DELALLOC); 9270 ei->runtime_flags = 0; 9271 ei->prop_compress = BTRFS_COMPRESS_NONE; 9272 ei->defrag_compress = BTRFS_COMPRESS_NONE; 9273 9274 ei->delayed_node = NULL; 9275 9276 ei->i_otime.tv_sec = 0; 9277 ei->i_otime.tv_nsec = 0; 9278 9279 inode = &ei->vfs_inode; 9280 extent_map_tree_init(&ei->extent_tree); 9281 extent_io_tree_init(&ei->io_tree, inode); 9282 extent_io_tree_init(&ei->io_failure_tree, inode); 9283 ei->io_tree.track_uptodate = 1; 9284 ei->io_failure_tree.track_uptodate = 1; 9285 atomic_set(&ei->sync_writers, 0); 9286 mutex_init(&ei->log_mutex); 9287 mutex_init(&ei->delalloc_mutex); 9288 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 9289 INIT_LIST_HEAD(&ei->delalloc_inodes); 9290 INIT_LIST_HEAD(&ei->delayed_iput); 9291 RB_CLEAR_NODE(&ei->rb_node); 9292 init_rwsem(&ei->dio_sem); 9293 9294 return inode; 9295 } 9296 9297 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 9298 void btrfs_test_destroy_inode(struct inode *inode) 9299 { 9300 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 9301 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9302 } 9303 #endif 9304 9305 static void btrfs_i_callback(struct rcu_head *head) 9306 { 9307 struct inode *inode = container_of(head, struct inode, i_rcu); 9308 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9309 } 9310 9311 void btrfs_destroy_inode(struct inode *inode) 9312 { 9313 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9314 struct btrfs_ordered_extent *ordered; 9315 struct btrfs_root *root = BTRFS_I(inode)->root; 9316 9317 WARN_ON(!hlist_empty(&inode->i_dentry)); 9318 WARN_ON(inode->i_data.nrpages); 9319 WARN_ON(BTRFS_I(inode)->block_rsv.reserved); 9320 WARN_ON(BTRFS_I(inode)->block_rsv.size); 9321 WARN_ON(BTRFS_I(inode)->outstanding_extents); 9322 WARN_ON(BTRFS_I(inode)->delalloc_bytes); 9323 WARN_ON(BTRFS_I(inode)->new_delalloc_bytes); 9324 WARN_ON(BTRFS_I(inode)->csum_bytes); 9325 WARN_ON(BTRFS_I(inode)->defrag_bytes); 9326 9327 /* 9328 * This can happen where we create an inode, but somebody else also 9329 * created the same inode and we need to destroy the one we already 9330 * created. 9331 */ 9332 if (!root) 9333 goto free; 9334 9335 if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 9336 &BTRFS_I(inode)->runtime_flags)) { 9337 btrfs_info(fs_info, "inode %llu still on the orphan list", 9338 btrfs_ino(BTRFS_I(inode))); 9339 atomic_dec(&root->orphan_inodes); 9340 } 9341 9342 while (1) { 9343 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 9344 if (!ordered) 9345 break; 9346 else { 9347 btrfs_err(fs_info, 9348 "found ordered extent %llu %llu on inode cleanup", 9349 ordered->file_offset, ordered->len); 9350 btrfs_remove_ordered_extent(inode, ordered); 9351 btrfs_put_ordered_extent(ordered); 9352 btrfs_put_ordered_extent(ordered); 9353 } 9354 } 9355 btrfs_qgroup_check_reserved_leak(inode); 9356 inode_tree_del(inode); 9357 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 9358 free: 9359 call_rcu(&inode->i_rcu, btrfs_i_callback); 9360 } 9361 9362 int btrfs_drop_inode(struct inode *inode) 9363 { 9364 struct btrfs_root *root = BTRFS_I(inode)->root; 9365 9366 if (root == NULL) 9367 return 1; 9368 9369 /* the snap/subvol tree is on deleting */ 9370 if (btrfs_root_refs(&root->root_item) == 0) 9371 return 1; 9372 else 9373 return generic_drop_inode(inode); 9374 } 9375 9376 static void init_once(void *foo) 9377 { 9378 struct btrfs_inode *ei = (struct btrfs_inode *) foo; 9379 9380 inode_init_once(&ei->vfs_inode); 9381 } 9382 9383 void __cold btrfs_destroy_cachep(void) 9384 { 9385 /* 9386 * Make sure all delayed rcu free inodes are flushed before we 9387 * destroy cache. 9388 */ 9389 rcu_barrier(); 9390 kmem_cache_destroy(btrfs_inode_cachep); 9391 kmem_cache_destroy(btrfs_trans_handle_cachep); 9392 kmem_cache_destroy(btrfs_path_cachep); 9393 kmem_cache_destroy(btrfs_free_space_cachep); 9394 } 9395 9396 int __init btrfs_init_cachep(void) 9397 { 9398 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 9399 sizeof(struct btrfs_inode), 0, 9400 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, 9401 init_once); 9402 if (!btrfs_inode_cachep) 9403 goto fail; 9404 9405 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", 9406 sizeof(struct btrfs_trans_handle), 0, 9407 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); 9408 if (!btrfs_trans_handle_cachep) 9409 goto fail; 9410 9411 btrfs_path_cachep = kmem_cache_create("btrfs_path", 9412 sizeof(struct btrfs_path), 0, 9413 SLAB_MEM_SPREAD, NULL); 9414 if (!btrfs_path_cachep) 9415 goto fail; 9416 9417 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space", 9418 sizeof(struct btrfs_free_space), 0, 9419 SLAB_MEM_SPREAD, NULL); 9420 if (!btrfs_free_space_cachep) 9421 goto fail; 9422 9423 return 0; 9424 fail: 9425 btrfs_destroy_cachep(); 9426 return -ENOMEM; 9427 } 9428 9429 static int btrfs_getattr(const struct path *path, struct kstat *stat, 9430 u32 request_mask, unsigned int flags) 9431 { 9432 u64 delalloc_bytes; 9433 struct inode *inode = d_inode(path->dentry); 9434 u32 blocksize = inode->i_sb->s_blocksize; 9435 u32 bi_flags = BTRFS_I(inode)->flags; 9436 9437 stat->result_mask |= STATX_BTIME; 9438 stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec; 9439 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec; 9440 if (bi_flags & BTRFS_INODE_APPEND) 9441 stat->attributes |= STATX_ATTR_APPEND; 9442 if (bi_flags & BTRFS_INODE_COMPRESS) 9443 stat->attributes |= STATX_ATTR_COMPRESSED; 9444 if (bi_flags & BTRFS_INODE_IMMUTABLE) 9445 stat->attributes |= STATX_ATTR_IMMUTABLE; 9446 if (bi_flags & BTRFS_INODE_NODUMP) 9447 stat->attributes |= STATX_ATTR_NODUMP; 9448 9449 stat->attributes_mask |= (STATX_ATTR_APPEND | 9450 STATX_ATTR_COMPRESSED | 9451 STATX_ATTR_IMMUTABLE | 9452 STATX_ATTR_NODUMP); 9453 9454 generic_fillattr(inode, stat); 9455 stat->dev = BTRFS_I(inode)->root->anon_dev; 9456 9457 spin_lock(&BTRFS_I(inode)->lock); 9458 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; 9459 spin_unlock(&BTRFS_I(inode)->lock); 9460 stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) + 9461 ALIGN(delalloc_bytes, blocksize)) >> 9; 9462 return 0; 9463 } 9464 9465 static int btrfs_rename_exchange(struct inode *old_dir, 9466 struct dentry *old_dentry, 9467 struct inode *new_dir, 9468 struct dentry *new_dentry) 9469 { 9470 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9471 struct btrfs_trans_handle *trans; 9472 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9473 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9474 struct inode *new_inode = new_dentry->d_inode; 9475 struct inode *old_inode = old_dentry->d_inode; 9476 struct timespec ctime = current_time(old_inode); 9477 struct dentry *parent; 9478 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9479 u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); 9480 u64 old_idx = 0; 9481 u64 new_idx = 0; 9482 u64 root_objectid; 9483 int ret; 9484 bool root_log_pinned = false; 9485 bool dest_log_pinned = false; 9486 9487 /* we only allow rename subvolume link between subvolumes */ 9488 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9489 return -EXDEV; 9490 9491 /* close the race window with snapshot create/destroy ioctl */ 9492 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9493 down_read(&fs_info->subvol_sem); 9494 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9495 down_read(&fs_info->subvol_sem); 9496 9497 /* 9498 * We want to reserve the absolute worst case amount of items. So if 9499 * both inodes are subvols and we need to unlink them then that would 9500 * require 4 item modifications, but if they are both normal inodes it 9501 * would require 5 item modifications, so we'll assume their normal 9502 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items 9503 * should cover the worst case number of items we'll modify. 9504 */ 9505 trans = btrfs_start_transaction(root, 12); 9506 if (IS_ERR(trans)) { 9507 ret = PTR_ERR(trans); 9508 goto out_notrans; 9509 } 9510 9511 /* 9512 * We need to find a free sequence number both in the source and 9513 * in the destination directory for the exchange. 9514 */ 9515 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); 9516 if (ret) 9517 goto out_fail; 9518 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); 9519 if (ret) 9520 goto out_fail; 9521 9522 BTRFS_I(old_inode)->dir_index = 0ULL; 9523 BTRFS_I(new_inode)->dir_index = 0ULL; 9524 9525 /* Reference for the source. */ 9526 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9527 /* force full log commit if subvolume involved. */ 9528 btrfs_set_log_full_commit(fs_info, trans); 9529 } else { 9530 btrfs_pin_log_trans(root); 9531 root_log_pinned = true; 9532 ret = btrfs_insert_inode_ref(trans, dest, 9533 new_dentry->d_name.name, 9534 new_dentry->d_name.len, 9535 old_ino, 9536 btrfs_ino(BTRFS_I(new_dir)), 9537 old_idx); 9538 if (ret) 9539 goto out_fail; 9540 } 9541 9542 /* And now for the dest. */ 9543 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9544 /* force full log commit if subvolume involved. */ 9545 btrfs_set_log_full_commit(fs_info, trans); 9546 } else { 9547 btrfs_pin_log_trans(dest); 9548 dest_log_pinned = true; 9549 ret = btrfs_insert_inode_ref(trans, root, 9550 old_dentry->d_name.name, 9551 old_dentry->d_name.len, 9552 new_ino, 9553 btrfs_ino(BTRFS_I(old_dir)), 9554 new_idx); 9555 if (ret) 9556 goto out_fail; 9557 } 9558 9559 /* Update inode version and ctime/mtime. */ 9560 inode_inc_iversion(old_dir); 9561 inode_inc_iversion(new_dir); 9562 inode_inc_iversion(old_inode); 9563 inode_inc_iversion(new_inode); 9564 old_dir->i_ctime = old_dir->i_mtime = ctime; 9565 new_dir->i_ctime = new_dir->i_mtime = ctime; 9566 old_inode->i_ctime = ctime; 9567 new_inode->i_ctime = ctime; 9568 9569 if (old_dentry->d_parent != new_dentry->d_parent) { 9570 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9571 BTRFS_I(old_inode), 1); 9572 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), 9573 BTRFS_I(new_inode), 1); 9574 } 9575 9576 /* src is a subvolume */ 9577 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9578 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9579 ret = btrfs_unlink_subvol(trans, root, old_dir, 9580 root_objectid, 9581 old_dentry->d_name.name, 9582 old_dentry->d_name.len); 9583 } else { /* src is an inode */ 9584 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9585 BTRFS_I(old_dentry->d_inode), 9586 old_dentry->d_name.name, 9587 old_dentry->d_name.len); 9588 if (!ret) 9589 ret = btrfs_update_inode(trans, root, old_inode); 9590 } 9591 if (ret) { 9592 btrfs_abort_transaction(trans, ret); 9593 goto out_fail; 9594 } 9595 9596 /* dest is a subvolume */ 9597 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9598 root_objectid = BTRFS_I(new_inode)->root->root_key.objectid; 9599 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9600 root_objectid, 9601 new_dentry->d_name.name, 9602 new_dentry->d_name.len); 9603 } else { /* dest is an inode */ 9604 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9605 BTRFS_I(new_dentry->d_inode), 9606 new_dentry->d_name.name, 9607 new_dentry->d_name.len); 9608 if (!ret) 9609 ret = btrfs_update_inode(trans, dest, new_inode); 9610 } 9611 if (ret) { 9612 btrfs_abort_transaction(trans, ret); 9613 goto out_fail; 9614 } 9615 9616 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9617 new_dentry->d_name.name, 9618 new_dentry->d_name.len, 0, old_idx); 9619 if (ret) { 9620 btrfs_abort_transaction(trans, ret); 9621 goto out_fail; 9622 } 9623 9624 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), 9625 old_dentry->d_name.name, 9626 old_dentry->d_name.len, 0, new_idx); 9627 if (ret) { 9628 btrfs_abort_transaction(trans, ret); 9629 goto out_fail; 9630 } 9631 9632 if (old_inode->i_nlink == 1) 9633 BTRFS_I(old_inode)->dir_index = old_idx; 9634 if (new_inode->i_nlink == 1) 9635 BTRFS_I(new_inode)->dir_index = new_idx; 9636 9637 if (root_log_pinned) { 9638 parent = new_dentry->d_parent; 9639 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9640 parent); 9641 btrfs_end_log_trans(root); 9642 root_log_pinned = false; 9643 } 9644 if (dest_log_pinned) { 9645 parent = old_dentry->d_parent; 9646 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir), 9647 parent); 9648 btrfs_end_log_trans(dest); 9649 dest_log_pinned = false; 9650 } 9651 out_fail: 9652 /* 9653 * If we have pinned a log and an error happened, we unpin tasks 9654 * trying to sync the log and force them to fallback to a transaction 9655 * commit if the log currently contains any of the inodes involved in 9656 * this rename operation (to ensure we do not persist a log with an 9657 * inconsistent state for any of these inodes or leading to any 9658 * inconsistencies when replayed). If the transaction was aborted, the 9659 * abortion reason is propagated to userspace when attempting to commit 9660 * the transaction. If the log does not contain any of these inodes, we 9661 * allow the tasks to sync it. 9662 */ 9663 if (ret && (root_log_pinned || dest_log_pinned)) { 9664 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9665 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9666 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9667 (new_inode && 9668 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9669 btrfs_set_log_full_commit(fs_info, trans); 9670 9671 if (root_log_pinned) { 9672 btrfs_end_log_trans(root); 9673 root_log_pinned = false; 9674 } 9675 if (dest_log_pinned) { 9676 btrfs_end_log_trans(dest); 9677 dest_log_pinned = false; 9678 } 9679 } 9680 ret = btrfs_end_transaction(trans); 9681 out_notrans: 9682 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9683 up_read(&fs_info->subvol_sem); 9684 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9685 up_read(&fs_info->subvol_sem); 9686 9687 return ret; 9688 } 9689 9690 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans, 9691 struct btrfs_root *root, 9692 struct inode *dir, 9693 struct dentry *dentry) 9694 { 9695 int ret; 9696 struct inode *inode; 9697 u64 objectid; 9698 u64 index; 9699 9700 ret = btrfs_find_free_ino(root, &objectid); 9701 if (ret) 9702 return ret; 9703 9704 inode = btrfs_new_inode(trans, root, dir, 9705 dentry->d_name.name, 9706 dentry->d_name.len, 9707 btrfs_ino(BTRFS_I(dir)), 9708 objectid, 9709 S_IFCHR | WHITEOUT_MODE, 9710 &index); 9711 9712 if (IS_ERR(inode)) { 9713 ret = PTR_ERR(inode); 9714 return ret; 9715 } 9716 9717 inode->i_op = &btrfs_special_inode_operations; 9718 init_special_inode(inode, inode->i_mode, 9719 WHITEOUT_DEV); 9720 9721 ret = btrfs_init_inode_security(trans, inode, dir, 9722 &dentry->d_name); 9723 if (ret) 9724 goto out; 9725 9726 ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 9727 BTRFS_I(inode), 0, index); 9728 if (ret) 9729 goto out; 9730 9731 ret = btrfs_update_inode(trans, root, inode); 9732 out: 9733 unlock_new_inode(inode); 9734 if (ret) 9735 inode_dec_link_count(inode); 9736 iput(inode); 9737 9738 return ret; 9739 } 9740 9741 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry, 9742 struct inode *new_dir, struct dentry *new_dentry, 9743 unsigned int flags) 9744 { 9745 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9746 struct btrfs_trans_handle *trans; 9747 unsigned int trans_num_items; 9748 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9749 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9750 struct inode *new_inode = d_inode(new_dentry); 9751 struct inode *old_inode = d_inode(old_dentry); 9752 u64 index = 0; 9753 u64 root_objectid; 9754 int ret; 9755 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9756 bool log_pinned = false; 9757 9758 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 9759 return -EPERM; 9760 9761 /* we only allow rename subvolume link between subvolumes */ 9762 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9763 return -EXDEV; 9764 9765 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 9766 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) 9767 return -ENOTEMPTY; 9768 9769 if (S_ISDIR(old_inode->i_mode) && new_inode && 9770 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 9771 return -ENOTEMPTY; 9772 9773 9774 /* check for collisions, even if the name isn't there */ 9775 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, 9776 new_dentry->d_name.name, 9777 new_dentry->d_name.len); 9778 9779 if (ret) { 9780 if (ret == -EEXIST) { 9781 /* we shouldn't get 9782 * eexist without a new_inode */ 9783 if (WARN_ON(!new_inode)) { 9784 return ret; 9785 } 9786 } else { 9787 /* maybe -EOVERFLOW */ 9788 return ret; 9789 } 9790 } 9791 ret = 0; 9792 9793 /* 9794 * we're using rename to replace one file with another. Start IO on it 9795 * now so we don't add too much work to the end of the transaction 9796 */ 9797 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 9798 filemap_flush(old_inode->i_mapping); 9799 9800 /* close the racy window with snapshot create/destroy ioctl */ 9801 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9802 down_read(&fs_info->subvol_sem); 9803 /* 9804 * We want to reserve the absolute worst case amount of items. So if 9805 * both inodes are subvols and we need to unlink them then that would 9806 * require 4 item modifications, but if they are both normal inodes it 9807 * would require 5 item modifications, so we'll assume they are normal 9808 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items 9809 * should cover the worst case number of items we'll modify. 9810 * If our rename has the whiteout flag, we need more 5 units for the 9811 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item 9812 * when selinux is enabled). 9813 */ 9814 trans_num_items = 11; 9815 if (flags & RENAME_WHITEOUT) 9816 trans_num_items += 5; 9817 trans = btrfs_start_transaction(root, trans_num_items); 9818 if (IS_ERR(trans)) { 9819 ret = PTR_ERR(trans); 9820 goto out_notrans; 9821 } 9822 9823 if (dest != root) 9824 btrfs_record_root_in_trans(trans, dest); 9825 9826 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); 9827 if (ret) 9828 goto out_fail; 9829 9830 BTRFS_I(old_inode)->dir_index = 0ULL; 9831 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9832 /* force full log commit if subvolume involved. */ 9833 btrfs_set_log_full_commit(fs_info, trans); 9834 } else { 9835 btrfs_pin_log_trans(root); 9836 log_pinned = true; 9837 ret = btrfs_insert_inode_ref(trans, dest, 9838 new_dentry->d_name.name, 9839 new_dentry->d_name.len, 9840 old_ino, 9841 btrfs_ino(BTRFS_I(new_dir)), index); 9842 if (ret) 9843 goto out_fail; 9844 } 9845 9846 inode_inc_iversion(old_dir); 9847 inode_inc_iversion(new_dir); 9848 inode_inc_iversion(old_inode); 9849 old_dir->i_ctime = old_dir->i_mtime = 9850 new_dir->i_ctime = new_dir->i_mtime = 9851 old_inode->i_ctime = current_time(old_dir); 9852 9853 if (old_dentry->d_parent != new_dentry->d_parent) 9854 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9855 BTRFS_I(old_inode), 1); 9856 9857 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9858 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9859 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid, 9860 old_dentry->d_name.name, 9861 old_dentry->d_name.len); 9862 } else { 9863 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9864 BTRFS_I(d_inode(old_dentry)), 9865 old_dentry->d_name.name, 9866 old_dentry->d_name.len); 9867 if (!ret) 9868 ret = btrfs_update_inode(trans, root, old_inode); 9869 } 9870 if (ret) { 9871 btrfs_abort_transaction(trans, ret); 9872 goto out_fail; 9873 } 9874 9875 if (new_inode) { 9876 inode_inc_iversion(new_inode); 9877 new_inode->i_ctime = current_time(new_inode); 9878 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == 9879 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 9880 root_objectid = BTRFS_I(new_inode)->location.objectid; 9881 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9882 root_objectid, 9883 new_dentry->d_name.name, 9884 new_dentry->d_name.len); 9885 BUG_ON(new_inode->i_nlink == 0); 9886 } else { 9887 ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9888 BTRFS_I(d_inode(new_dentry)), 9889 new_dentry->d_name.name, 9890 new_dentry->d_name.len); 9891 } 9892 if (!ret && new_inode->i_nlink == 0) 9893 ret = btrfs_orphan_add(trans, 9894 BTRFS_I(d_inode(new_dentry))); 9895 if (ret) { 9896 btrfs_abort_transaction(trans, ret); 9897 goto out_fail; 9898 } 9899 } 9900 9901 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9902 new_dentry->d_name.name, 9903 new_dentry->d_name.len, 0, index); 9904 if (ret) { 9905 btrfs_abort_transaction(trans, ret); 9906 goto out_fail; 9907 } 9908 9909 if (old_inode->i_nlink == 1) 9910 BTRFS_I(old_inode)->dir_index = index; 9911 9912 if (log_pinned) { 9913 struct dentry *parent = new_dentry->d_parent; 9914 9915 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9916 parent); 9917 btrfs_end_log_trans(root); 9918 log_pinned = false; 9919 } 9920 9921 if (flags & RENAME_WHITEOUT) { 9922 ret = btrfs_whiteout_for_rename(trans, root, old_dir, 9923 old_dentry); 9924 9925 if (ret) { 9926 btrfs_abort_transaction(trans, ret); 9927 goto out_fail; 9928 } 9929 } 9930 out_fail: 9931 /* 9932 * If we have pinned the log and an error happened, we unpin tasks 9933 * trying to sync the log and force them to fallback to a transaction 9934 * commit if the log currently contains any of the inodes involved in 9935 * this rename operation (to ensure we do not persist a log with an 9936 * inconsistent state for any of these inodes or leading to any 9937 * inconsistencies when replayed). If the transaction was aborted, the 9938 * abortion reason is propagated to userspace when attempting to commit 9939 * the transaction. If the log does not contain any of these inodes, we 9940 * allow the tasks to sync it. 9941 */ 9942 if (ret && log_pinned) { 9943 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9944 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9945 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9946 (new_inode && 9947 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9948 btrfs_set_log_full_commit(fs_info, trans); 9949 9950 btrfs_end_log_trans(root); 9951 log_pinned = false; 9952 } 9953 btrfs_end_transaction(trans); 9954 out_notrans: 9955 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9956 up_read(&fs_info->subvol_sem); 9957 9958 return ret; 9959 } 9960 9961 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, 9962 struct inode *new_dir, struct dentry *new_dentry, 9963 unsigned int flags) 9964 { 9965 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 9966 return -EINVAL; 9967 9968 if (flags & RENAME_EXCHANGE) 9969 return btrfs_rename_exchange(old_dir, old_dentry, new_dir, 9970 new_dentry); 9971 9972 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags); 9973 } 9974 9975 static void btrfs_run_delalloc_work(struct btrfs_work *work) 9976 { 9977 struct btrfs_delalloc_work *delalloc_work; 9978 struct inode *inode; 9979 9980 delalloc_work = container_of(work, struct btrfs_delalloc_work, 9981 work); 9982 inode = delalloc_work->inode; 9983 filemap_flush(inode->i_mapping); 9984 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9985 &BTRFS_I(inode)->runtime_flags)) 9986 filemap_flush(inode->i_mapping); 9987 9988 if (delalloc_work->delay_iput) 9989 btrfs_add_delayed_iput(inode); 9990 else 9991 iput(inode); 9992 complete(&delalloc_work->completion); 9993 } 9994 9995 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode, 9996 int delay_iput) 9997 { 9998 struct btrfs_delalloc_work *work; 9999 10000 work = kmalloc(sizeof(*work), GFP_NOFS); 10001 if (!work) 10002 return NULL; 10003 10004 init_completion(&work->completion); 10005 INIT_LIST_HEAD(&work->list); 10006 work->inode = inode; 10007 work->delay_iput = delay_iput; 10008 WARN_ON_ONCE(!inode); 10009 btrfs_init_work(&work->work, btrfs_flush_delalloc_helper, 10010 btrfs_run_delalloc_work, NULL, NULL); 10011 10012 return work; 10013 } 10014 10015 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work) 10016 { 10017 wait_for_completion(&work->completion); 10018 kfree(work); 10019 } 10020 10021 /* 10022 * some fairly slow code that needs optimization. This walks the list 10023 * of all the inodes with pending delalloc and forces them to disk. 10024 */ 10025 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput, 10026 int nr) 10027 { 10028 struct btrfs_inode *binode; 10029 struct inode *inode; 10030 struct btrfs_delalloc_work *work, *next; 10031 struct list_head works; 10032 struct list_head splice; 10033 int ret = 0; 10034 10035 INIT_LIST_HEAD(&works); 10036 INIT_LIST_HEAD(&splice); 10037 10038 mutex_lock(&root->delalloc_mutex); 10039 spin_lock(&root->delalloc_lock); 10040 list_splice_init(&root->delalloc_inodes, &splice); 10041 while (!list_empty(&splice)) { 10042 binode = list_entry(splice.next, struct btrfs_inode, 10043 delalloc_inodes); 10044 10045 list_move_tail(&binode->delalloc_inodes, 10046 &root->delalloc_inodes); 10047 inode = igrab(&binode->vfs_inode); 10048 if (!inode) { 10049 cond_resched_lock(&root->delalloc_lock); 10050 continue; 10051 } 10052 spin_unlock(&root->delalloc_lock); 10053 10054 work = btrfs_alloc_delalloc_work(inode, delay_iput); 10055 if (!work) { 10056 if (delay_iput) 10057 btrfs_add_delayed_iput(inode); 10058 else 10059 iput(inode); 10060 ret = -ENOMEM; 10061 goto out; 10062 } 10063 list_add_tail(&work->list, &works); 10064 btrfs_queue_work(root->fs_info->flush_workers, 10065 &work->work); 10066 ret++; 10067 if (nr != -1 && ret >= nr) 10068 goto out; 10069 cond_resched(); 10070 spin_lock(&root->delalloc_lock); 10071 } 10072 spin_unlock(&root->delalloc_lock); 10073 10074 out: 10075 list_for_each_entry_safe(work, next, &works, list) { 10076 list_del_init(&work->list); 10077 btrfs_wait_and_free_delalloc_work(work); 10078 } 10079 10080 if (!list_empty_careful(&splice)) { 10081 spin_lock(&root->delalloc_lock); 10082 list_splice_tail(&splice, &root->delalloc_inodes); 10083 spin_unlock(&root->delalloc_lock); 10084 } 10085 mutex_unlock(&root->delalloc_mutex); 10086 return ret; 10087 } 10088 10089 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput) 10090 { 10091 struct btrfs_fs_info *fs_info = root->fs_info; 10092 int ret; 10093 10094 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10095 return -EROFS; 10096 10097 ret = __start_delalloc_inodes(root, delay_iput, -1); 10098 if (ret > 0) 10099 ret = 0; 10100 return ret; 10101 } 10102 10103 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput, 10104 int nr) 10105 { 10106 struct btrfs_root *root; 10107 struct list_head splice; 10108 int ret; 10109 10110 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10111 return -EROFS; 10112 10113 INIT_LIST_HEAD(&splice); 10114 10115 mutex_lock(&fs_info->delalloc_root_mutex); 10116 spin_lock(&fs_info->delalloc_root_lock); 10117 list_splice_init(&fs_info->delalloc_roots, &splice); 10118 while (!list_empty(&splice) && nr) { 10119 root = list_first_entry(&splice, struct btrfs_root, 10120 delalloc_root); 10121 root = btrfs_grab_fs_root(root); 10122 BUG_ON(!root); 10123 list_move_tail(&root->delalloc_root, 10124 &fs_info->delalloc_roots); 10125 spin_unlock(&fs_info->delalloc_root_lock); 10126 10127 ret = __start_delalloc_inodes(root, delay_iput, nr); 10128 btrfs_put_fs_root(root); 10129 if (ret < 0) 10130 goto out; 10131 10132 if (nr != -1) { 10133 nr -= ret; 10134 WARN_ON(nr < 0); 10135 } 10136 spin_lock(&fs_info->delalloc_root_lock); 10137 } 10138 spin_unlock(&fs_info->delalloc_root_lock); 10139 10140 ret = 0; 10141 out: 10142 if (!list_empty_careful(&splice)) { 10143 spin_lock(&fs_info->delalloc_root_lock); 10144 list_splice_tail(&splice, &fs_info->delalloc_roots); 10145 spin_unlock(&fs_info->delalloc_root_lock); 10146 } 10147 mutex_unlock(&fs_info->delalloc_root_mutex); 10148 return ret; 10149 } 10150 10151 static int btrfs_symlink(struct inode *dir, struct dentry *dentry, 10152 const char *symname) 10153 { 10154 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10155 struct btrfs_trans_handle *trans; 10156 struct btrfs_root *root = BTRFS_I(dir)->root; 10157 struct btrfs_path *path; 10158 struct btrfs_key key; 10159 struct inode *inode = NULL; 10160 int err; 10161 int drop_inode = 0; 10162 u64 objectid; 10163 u64 index = 0; 10164 int name_len; 10165 int datasize; 10166 unsigned long ptr; 10167 struct btrfs_file_extent_item *ei; 10168 struct extent_buffer *leaf; 10169 10170 name_len = strlen(symname); 10171 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 10172 return -ENAMETOOLONG; 10173 10174 /* 10175 * 2 items for inode item and ref 10176 * 2 items for dir items 10177 * 1 item for updating parent inode item 10178 * 1 item for the inline extent item 10179 * 1 item for xattr if selinux is on 10180 */ 10181 trans = btrfs_start_transaction(root, 7); 10182 if (IS_ERR(trans)) 10183 return PTR_ERR(trans); 10184 10185 err = btrfs_find_free_ino(root, &objectid); 10186 if (err) 10187 goto out_unlock; 10188 10189 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 10190 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), 10191 objectid, S_IFLNK|S_IRWXUGO, &index); 10192 if (IS_ERR(inode)) { 10193 err = PTR_ERR(inode); 10194 goto out_unlock; 10195 } 10196 10197 /* 10198 * If the active LSM wants to access the inode during 10199 * d_instantiate it needs these. Smack checks to see 10200 * if the filesystem supports xattrs by looking at the 10201 * ops vector. 10202 */ 10203 inode->i_fop = &btrfs_file_operations; 10204 inode->i_op = &btrfs_file_inode_operations; 10205 inode->i_mapping->a_ops = &btrfs_aops; 10206 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10207 10208 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 10209 if (err) 10210 goto out_unlock_inode; 10211 10212 path = btrfs_alloc_path(); 10213 if (!path) { 10214 err = -ENOMEM; 10215 goto out_unlock_inode; 10216 } 10217 key.objectid = btrfs_ino(BTRFS_I(inode)); 10218 key.offset = 0; 10219 key.type = BTRFS_EXTENT_DATA_KEY; 10220 datasize = btrfs_file_extent_calc_inline_size(name_len); 10221 err = btrfs_insert_empty_item(trans, root, path, &key, 10222 datasize); 10223 if (err) { 10224 btrfs_free_path(path); 10225 goto out_unlock_inode; 10226 } 10227 leaf = path->nodes[0]; 10228 ei = btrfs_item_ptr(leaf, path->slots[0], 10229 struct btrfs_file_extent_item); 10230 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 10231 btrfs_set_file_extent_type(leaf, ei, 10232 BTRFS_FILE_EXTENT_INLINE); 10233 btrfs_set_file_extent_encryption(leaf, ei, 0); 10234 btrfs_set_file_extent_compression(leaf, ei, 0); 10235 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 10236 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 10237 10238 ptr = btrfs_file_extent_inline_start(ei); 10239 write_extent_buffer(leaf, symname, ptr, name_len); 10240 btrfs_mark_buffer_dirty(leaf); 10241 btrfs_free_path(path); 10242 10243 inode->i_op = &btrfs_symlink_inode_operations; 10244 inode_nohighmem(inode); 10245 inode->i_mapping->a_ops = &btrfs_symlink_aops; 10246 inode_set_bytes(inode, name_len); 10247 btrfs_i_size_write(BTRFS_I(inode), name_len); 10248 err = btrfs_update_inode(trans, root, inode); 10249 /* 10250 * Last step, add directory indexes for our symlink inode. This is the 10251 * last step to avoid extra cleanup of these indexes if an error happens 10252 * elsewhere above. 10253 */ 10254 if (!err) 10255 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 10256 BTRFS_I(inode), 0, index); 10257 if (err) { 10258 drop_inode = 1; 10259 goto out_unlock_inode; 10260 } 10261 10262 unlock_new_inode(inode); 10263 d_instantiate(dentry, inode); 10264 10265 out_unlock: 10266 btrfs_end_transaction(trans); 10267 if (drop_inode) { 10268 inode_dec_link_count(inode); 10269 iput(inode); 10270 } 10271 btrfs_btree_balance_dirty(fs_info); 10272 return err; 10273 10274 out_unlock_inode: 10275 drop_inode = 1; 10276 unlock_new_inode(inode); 10277 goto out_unlock; 10278 } 10279 10280 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 10281 u64 start, u64 num_bytes, u64 min_size, 10282 loff_t actual_len, u64 *alloc_hint, 10283 struct btrfs_trans_handle *trans) 10284 { 10285 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 10286 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 10287 struct extent_map *em; 10288 struct btrfs_root *root = BTRFS_I(inode)->root; 10289 struct btrfs_key ins; 10290 u64 cur_offset = start; 10291 u64 i_size; 10292 u64 cur_bytes; 10293 u64 last_alloc = (u64)-1; 10294 int ret = 0; 10295 bool own_trans = true; 10296 u64 end = start + num_bytes - 1; 10297 10298 if (trans) 10299 own_trans = false; 10300 while (num_bytes > 0) { 10301 if (own_trans) { 10302 trans = btrfs_start_transaction(root, 3); 10303 if (IS_ERR(trans)) { 10304 ret = PTR_ERR(trans); 10305 break; 10306 } 10307 } 10308 10309 cur_bytes = min_t(u64, num_bytes, SZ_256M); 10310 cur_bytes = max(cur_bytes, min_size); 10311 /* 10312 * If we are severely fragmented we could end up with really 10313 * small allocations, so if the allocator is returning small 10314 * chunks lets make its job easier by only searching for those 10315 * sized chunks. 10316 */ 10317 cur_bytes = min(cur_bytes, last_alloc); 10318 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 10319 min_size, 0, *alloc_hint, &ins, 1, 0); 10320 if (ret) { 10321 if (own_trans) 10322 btrfs_end_transaction(trans); 10323 break; 10324 } 10325 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10326 10327 last_alloc = ins.offset; 10328 ret = insert_reserved_file_extent(trans, inode, 10329 cur_offset, ins.objectid, 10330 ins.offset, ins.offset, 10331 ins.offset, 0, 0, 0, 10332 BTRFS_FILE_EXTENT_PREALLOC); 10333 if (ret) { 10334 btrfs_free_reserved_extent(fs_info, ins.objectid, 10335 ins.offset, 0); 10336 btrfs_abort_transaction(trans, ret); 10337 if (own_trans) 10338 btrfs_end_transaction(trans); 10339 break; 10340 } 10341 10342 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 10343 cur_offset + ins.offset -1, 0); 10344 10345 em = alloc_extent_map(); 10346 if (!em) { 10347 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 10348 &BTRFS_I(inode)->runtime_flags); 10349 goto next; 10350 } 10351 10352 em->start = cur_offset; 10353 em->orig_start = cur_offset; 10354 em->len = ins.offset; 10355 em->block_start = ins.objectid; 10356 em->block_len = ins.offset; 10357 em->orig_block_len = ins.offset; 10358 em->ram_bytes = ins.offset; 10359 em->bdev = fs_info->fs_devices->latest_bdev; 10360 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 10361 em->generation = trans->transid; 10362 10363 while (1) { 10364 write_lock(&em_tree->lock); 10365 ret = add_extent_mapping(em_tree, em, 1); 10366 write_unlock(&em_tree->lock); 10367 if (ret != -EEXIST) 10368 break; 10369 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 10370 cur_offset + ins.offset - 1, 10371 0); 10372 } 10373 free_extent_map(em); 10374 next: 10375 num_bytes -= ins.offset; 10376 cur_offset += ins.offset; 10377 *alloc_hint = ins.objectid + ins.offset; 10378 10379 inode_inc_iversion(inode); 10380 inode->i_ctime = current_time(inode); 10381 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 10382 if (!(mode & FALLOC_FL_KEEP_SIZE) && 10383 (actual_len > inode->i_size) && 10384 (cur_offset > inode->i_size)) { 10385 if (cur_offset > actual_len) 10386 i_size = actual_len; 10387 else 10388 i_size = cur_offset; 10389 i_size_write(inode, i_size); 10390 btrfs_ordered_update_i_size(inode, i_size, NULL); 10391 } 10392 10393 ret = btrfs_update_inode(trans, root, inode); 10394 10395 if (ret) { 10396 btrfs_abort_transaction(trans, ret); 10397 if (own_trans) 10398 btrfs_end_transaction(trans); 10399 break; 10400 } 10401 10402 if (own_trans) 10403 btrfs_end_transaction(trans); 10404 } 10405 if (cur_offset < end) 10406 btrfs_free_reserved_data_space(inode, NULL, cur_offset, 10407 end - cur_offset + 1); 10408 return ret; 10409 } 10410 10411 int btrfs_prealloc_file_range(struct inode *inode, int mode, 10412 u64 start, u64 num_bytes, u64 min_size, 10413 loff_t actual_len, u64 *alloc_hint) 10414 { 10415 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10416 min_size, actual_len, alloc_hint, 10417 NULL); 10418 } 10419 10420 int btrfs_prealloc_file_range_trans(struct inode *inode, 10421 struct btrfs_trans_handle *trans, int mode, 10422 u64 start, u64 num_bytes, u64 min_size, 10423 loff_t actual_len, u64 *alloc_hint) 10424 { 10425 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10426 min_size, actual_len, alloc_hint, trans); 10427 } 10428 10429 static int btrfs_set_page_dirty(struct page *page) 10430 { 10431 return __set_page_dirty_nobuffers(page); 10432 } 10433 10434 static int btrfs_permission(struct inode *inode, int mask) 10435 { 10436 struct btrfs_root *root = BTRFS_I(inode)->root; 10437 umode_t mode = inode->i_mode; 10438 10439 if (mask & MAY_WRITE && 10440 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 10441 if (btrfs_root_readonly(root)) 10442 return -EROFS; 10443 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 10444 return -EACCES; 10445 } 10446 return generic_permission(inode, mask); 10447 } 10448 10449 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 10450 { 10451 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10452 struct btrfs_trans_handle *trans; 10453 struct btrfs_root *root = BTRFS_I(dir)->root; 10454 struct inode *inode = NULL; 10455 u64 objectid; 10456 u64 index; 10457 int ret = 0; 10458 10459 /* 10460 * 5 units required for adding orphan entry 10461 */ 10462 trans = btrfs_start_transaction(root, 5); 10463 if (IS_ERR(trans)) 10464 return PTR_ERR(trans); 10465 10466 ret = btrfs_find_free_ino(root, &objectid); 10467 if (ret) 10468 goto out; 10469 10470 inode = btrfs_new_inode(trans, root, dir, NULL, 0, 10471 btrfs_ino(BTRFS_I(dir)), objectid, mode, &index); 10472 if (IS_ERR(inode)) { 10473 ret = PTR_ERR(inode); 10474 inode = NULL; 10475 goto out; 10476 } 10477 10478 inode->i_fop = &btrfs_file_operations; 10479 inode->i_op = &btrfs_file_inode_operations; 10480 10481 inode->i_mapping->a_ops = &btrfs_aops; 10482 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10483 10484 ret = btrfs_init_inode_security(trans, inode, dir, NULL); 10485 if (ret) 10486 goto out_inode; 10487 10488 ret = btrfs_update_inode(trans, root, inode); 10489 if (ret) 10490 goto out_inode; 10491 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 10492 if (ret) 10493 goto out_inode; 10494 10495 /* 10496 * We set number of links to 0 in btrfs_new_inode(), and here we set 10497 * it to 1 because d_tmpfile() will issue a warning if the count is 0, 10498 * through: 10499 * 10500 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 10501 */ 10502 set_nlink(inode, 1); 10503 unlock_new_inode(inode); 10504 d_tmpfile(dentry, inode); 10505 mark_inode_dirty(inode); 10506 10507 out: 10508 btrfs_end_transaction(trans); 10509 if (ret) 10510 iput(inode); 10511 btrfs_btree_balance_dirty(fs_info); 10512 return ret; 10513 10514 out_inode: 10515 unlock_new_inode(inode); 10516 goto out; 10517 10518 } 10519 10520 __attribute__((const)) 10521 static int btrfs_readpage_io_failed_hook(struct page *page, int failed_mirror) 10522 { 10523 return -EAGAIN; 10524 } 10525 10526 static struct btrfs_fs_info *iotree_fs_info(void *private_data) 10527 { 10528 struct inode *inode = private_data; 10529 return btrfs_sb(inode->i_sb); 10530 } 10531 10532 static void btrfs_check_extent_io_range(void *private_data, const char *caller, 10533 u64 start, u64 end) 10534 { 10535 struct inode *inode = private_data; 10536 u64 isize; 10537 10538 isize = i_size_read(inode); 10539 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { 10540 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info, 10541 "%s: ino %llu isize %llu odd range [%llu,%llu]", 10542 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end); 10543 } 10544 } 10545 10546 void btrfs_set_range_writeback(void *private_data, u64 start, u64 end) 10547 { 10548 struct inode *inode = private_data; 10549 unsigned long index = start >> PAGE_SHIFT; 10550 unsigned long end_index = end >> PAGE_SHIFT; 10551 struct page *page; 10552 10553 while (index <= end_index) { 10554 page = find_get_page(inode->i_mapping, index); 10555 ASSERT(page); /* Pages should be in the extent_io_tree */ 10556 set_page_writeback(page); 10557 put_page(page); 10558 index++; 10559 } 10560 } 10561 10562 static const struct inode_operations btrfs_dir_inode_operations = { 10563 .getattr = btrfs_getattr, 10564 .lookup = btrfs_lookup, 10565 .create = btrfs_create, 10566 .unlink = btrfs_unlink, 10567 .link = btrfs_link, 10568 .mkdir = btrfs_mkdir, 10569 .rmdir = btrfs_rmdir, 10570 .rename = btrfs_rename2, 10571 .symlink = btrfs_symlink, 10572 .setattr = btrfs_setattr, 10573 .mknod = btrfs_mknod, 10574 .listxattr = btrfs_listxattr, 10575 .permission = btrfs_permission, 10576 .get_acl = btrfs_get_acl, 10577 .set_acl = btrfs_set_acl, 10578 .update_time = btrfs_update_time, 10579 .tmpfile = btrfs_tmpfile, 10580 }; 10581 static const struct inode_operations btrfs_dir_ro_inode_operations = { 10582 .lookup = btrfs_lookup, 10583 .permission = btrfs_permission, 10584 .update_time = btrfs_update_time, 10585 }; 10586 10587 static const struct file_operations btrfs_dir_file_operations = { 10588 .llseek = generic_file_llseek, 10589 .read = generic_read_dir, 10590 .iterate_shared = btrfs_real_readdir, 10591 .open = btrfs_opendir, 10592 .unlocked_ioctl = btrfs_ioctl, 10593 #ifdef CONFIG_COMPAT 10594 .compat_ioctl = btrfs_compat_ioctl, 10595 #endif 10596 .release = btrfs_release_file, 10597 .fsync = btrfs_sync_file, 10598 }; 10599 10600 static const struct extent_io_ops btrfs_extent_io_ops = { 10601 /* mandatory callbacks */ 10602 .submit_bio_hook = btrfs_submit_bio_hook, 10603 .readpage_end_io_hook = btrfs_readpage_end_io_hook, 10604 .merge_bio_hook = btrfs_merge_bio_hook, 10605 .readpage_io_failed_hook = btrfs_readpage_io_failed_hook, 10606 .tree_fs_info = iotree_fs_info, 10607 .set_range_writeback = btrfs_set_range_writeback, 10608 10609 /* optional callbacks */ 10610 .fill_delalloc = run_delalloc_range, 10611 .writepage_end_io_hook = btrfs_writepage_end_io_hook, 10612 .writepage_start_hook = btrfs_writepage_start_hook, 10613 .set_bit_hook = btrfs_set_bit_hook, 10614 .clear_bit_hook = btrfs_clear_bit_hook, 10615 .merge_extent_hook = btrfs_merge_extent_hook, 10616 .split_extent_hook = btrfs_split_extent_hook, 10617 .check_extent_io_range = btrfs_check_extent_io_range, 10618 }; 10619 10620 /* 10621 * btrfs doesn't support the bmap operation because swapfiles 10622 * use bmap to make a mapping of extents in the file. They assume 10623 * these extents won't change over the life of the file and they 10624 * use the bmap result to do IO directly to the drive. 10625 * 10626 * the btrfs bmap call would return logical addresses that aren't 10627 * suitable for IO and they also will change frequently as COW 10628 * operations happen. So, swapfile + btrfs == corruption. 10629 * 10630 * For now we're avoiding this by dropping bmap. 10631 */ 10632 static const struct address_space_operations btrfs_aops = { 10633 .readpage = btrfs_readpage, 10634 .writepage = btrfs_writepage, 10635 .writepages = btrfs_writepages, 10636 .readpages = btrfs_readpages, 10637 .direct_IO = btrfs_direct_IO, 10638 .invalidatepage = btrfs_invalidatepage, 10639 .releasepage = btrfs_releasepage, 10640 .set_page_dirty = btrfs_set_page_dirty, 10641 .error_remove_page = generic_error_remove_page, 10642 }; 10643 10644 static const struct address_space_operations btrfs_symlink_aops = { 10645 .readpage = btrfs_readpage, 10646 .writepage = btrfs_writepage, 10647 .invalidatepage = btrfs_invalidatepage, 10648 .releasepage = btrfs_releasepage, 10649 }; 10650 10651 static const struct inode_operations btrfs_file_inode_operations = { 10652 .getattr = btrfs_getattr, 10653 .setattr = btrfs_setattr, 10654 .listxattr = btrfs_listxattr, 10655 .permission = btrfs_permission, 10656 .fiemap = btrfs_fiemap, 10657 .get_acl = btrfs_get_acl, 10658 .set_acl = btrfs_set_acl, 10659 .update_time = btrfs_update_time, 10660 }; 10661 static const struct inode_operations btrfs_special_inode_operations = { 10662 .getattr = btrfs_getattr, 10663 .setattr = btrfs_setattr, 10664 .permission = btrfs_permission, 10665 .listxattr = btrfs_listxattr, 10666 .get_acl = btrfs_get_acl, 10667 .set_acl = btrfs_set_acl, 10668 .update_time = btrfs_update_time, 10669 }; 10670 static const struct inode_operations btrfs_symlink_inode_operations = { 10671 .get_link = page_get_link, 10672 .getattr = btrfs_getattr, 10673 .setattr = btrfs_setattr, 10674 .permission = btrfs_permission, 10675 .listxattr = btrfs_listxattr, 10676 .update_time = btrfs_update_time, 10677 }; 10678 10679 const struct dentry_operations btrfs_dentry_operations = { 10680 .d_delete = btrfs_dentry_delete, 10681 .d_release = btrfs_dentry_release, 10682 }; 10683