1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/bio.h> 8 #include <linux/buffer_head.h> 9 #include <linux/file.h> 10 #include <linux/fs.h> 11 #include <linux/pagemap.h> 12 #include <linux/highmem.h> 13 #include <linux/time.h> 14 #include <linux/init.h> 15 #include <linux/string.h> 16 #include <linux/backing-dev.h> 17 #include <linux/mpage.h> 18 #include <linux/swap.h> 19 #include <linux/writeback.h> 20 #include <linux/compat.h> 21 #include <linux/bit_spinlock.h> 22 #include <linux/xattr.h> 23 #include <linux/posix_acl.h> 24 #include <linux/falloc.h> 25 #include <linux/slab.h> 26 #include <linux/ratelimit.h> 27 #include <linux/mount.h> 28 #include <linux/btrfs.h> 29 #include <linux/blkdev.h> 30 #include <linux/posix_acl_xattr.h> 31 #include <linux/uio.h> 32 #include <linux/magic.h> 33 #include <linux/iversion.h> 34 #include <asm/unaligned.h> 35 #include "ctree.h" 36 #include "disk-io.h" 37 #include "transaction.h" 38 #include "btrfs_inode.h" 39 #include "print-tree.h" 40 #include "ordered-data.h" 41 #include "xattr.h" 42 #include "tree-log.h" 43 #include "volumes.h" 44 #include "compression.h" 45 #include "locking.h" 46 #include "free-space-cache.h" 47 #include "inode-map.h" 48 #include "backref.h" 49 #include "props.h" 50 #include "qgroup.h" 51 #include "dedupe.h" 52 53 struct btrfs_iget_args { 54 struct btrfs_key *location; 55 struct btrfs_root *root; 56 }; 57 58 struct btrfs_dio_data { 59 u64 reserve; 60 u64 unsubmitted_oe_range_start; 61 u64 unsubmitted_oe_range_end; 62 int overwrite; 63 }; 64 65 static const struct inode_operations btrfs_dir_inode_operations; 66 static const struct inode_operations btrfs_symlink_inode_operations; 67 static const struct inode_operations btrfs_dir_ro_inode_operations; 68 static const struct inode_operations btrfs_special_inode_operations; 69 static const struct inode_operations btrfs_file_inode_operations; 70 static const struct address_space_operations btrfs_aops; 71 static const struct address_space_operations btrfs_symlink_aops; 72 static const struct file_operations btrfs_dir_file_operations; 73 static const struct extent_io_ops btrfs_extent_io_ops; 74 75 static struct kmem_cache *btrfs_inode_cachep; 76 struct kmem_cache *btrfs_trans_handle_cachep; 77 struct kmem_cache *btrfs_path_cachep; 78 struct kmem_cache *btrfs_free_space_cachep; 79 80 #define S_SHIFT 12 81 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = { 82 [S_IFREG >> S_SHIFT] = BTRFS_FT_REG_FILE, 83 [S_IFDIR >> S_SHIFT] = BTRFS_FT_DIR, 84 [S_IFCHR >> S_SHIFT] = BTRFS_FT_CHRDEV, 85 [S_IFBLK >> S_SHIFT] = BTRFS_FT_BLKDEV, 86 [S_IFIFO >> S_SHIFT] = BTRFS_FT_FIFO, 87 [S_IFSOCK >> S_SHIFT] = BTRFS_FT_SOCK, 88 [S_IFLNK >> S_SHIFT] = BTRFS_FT_SYMLINK, 89 }; 90 91 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 92 static int btrfs_truncate(struct inode *inode, bool skip_writeback); 93 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); 94 static noinline int cow_file_range(struct inode *inode, 95 struct page *locked_page, 96 u64 start, u64 end, u64 delalloc_end, 97 int *page_started, unsigned long *nr_written, 98 int unlock, struct btrfs_dedupe_hash *hash); 99 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, 100 u64 orig_start, u64 block_start, 101 u64 block_len, u64 orig_block_len, 102 u64 ram_bytes, int compress_type, 103 int type); 104 105 static void __endio_write_update_ordered(struct inode *inode, 106 const u64 offset, const u64 bytes, 107 const bool uptodate); 108 109 /* 110 * Cleanup all submitted ordered extents in specified range to handle errors 111 * from the fill_dellaloc() callback. 112 * 113 * NOTE: caller must ensure that when an error happens, it can not call 114 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING 115 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata 116 * to be released, which we want to happen only when finishing the ordered 117 * extent (btrfs_finish_ordered_io()). Also note that the caller of the 118 * fill_delalloc() callback already does proper cleanup for the first page of 119 * the range, that is, it invokes the callback writepage_end_io_hook() for the 120 * range of the first page. 121 */ 122 static inline void btrfs_cleanup_ordered_extents(struct inode *inode, 123 const u64 offset, 124 const u64 bytes) 125 { 126 unsigned long index = offset >> PAGE_SHIFT; 127 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT; 128 struct page *page; 129 130 while (index <= end_index) { 131 page = find_get_page(inode->i_mapping, index); 132 index++; 133 if (!page) 134 continue; 135 ClearPagePrivate2(page); 136 put_page(page); 137 } 138 return __endio_write_update_ordered(inode, offset + PAGE_SIZE, 139 bytes - PAGE_SIZE, false); 140 } 141 142 static int btrfs_dirty_inode(struct inode *inode); 143 144 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 145 void btrfs_test_inode_set_ops(struct inode *inode) 146 { 147 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 148 } 149 #endif 150 151 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 152 struct inode *inode, struct inode *dir, 153 const struct qstr *qstr) 154 { 155 int err; 156 157 err = btrfs_init_acl(trans, inode, dir); 158 if (!err) 159 err = btrfs_xattr_security_init(trans, inode, dir, qstr); 160 return err; 161 } 162 163 /* 164 * this does all the hard work for inserting an inline extent into 165 * the btree. The caller should have done a btrfs_drop_extents so that 166 * no overlapping inline items exist in the btree 167 */ 168 static int insert_inline_extent(struct btrfs_trans_handle *trans, 169 struct btrfs_path *path, int extent_inserted, 170 struct btrfs_root *root, struct inode *inode, 171 u64 start, size_t size, size_t compressed_size, 172 int compress_type, 173 struct page **compressed_pages) 174 { 175 struct extent_buffer *leaf; 176 struct page *page = NULL; 177 char *kaddr; 178 unsigned long ptr; 179 struct btrfs_file_extent_item *ei; 180 int ret; 181 size_t cur_size = size; 182 unsigned long offset; 183 184 if (compressed_size && compressed_pages) 185 cur_size = compressed_size; 186 187 inode_add_bytes(inode, size); 188 189 if (!extent_inserted) { 190 struct btrfs_key key; 191 size_t datasize; 192 193 key.objectid = btrfs_ino(BTRFS_I(inode)); 194 key.offset = start; 195 key.type = BTRFS_EXTENT_DATA_KEY; 196 197 datasize = btrfs_file_extent_calc_inline_size(cur_size); 198 path->leave_spinning = 1; 199 ret = btrfs_insert_empty_item(trans, root, path, &key, 200 datasize); 201 if (ret) 202 goto fail; 203 } 204 leaf = path->nodes[0]; 205 ei = btrfs_item_ptr(leaf, path->slots[0], 206 struct btrfs_file_extent_item); 207 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 208 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 209 btrfs_set_file_extent_encryption(leaf, ei, 0); 210 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 211 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 212 ptr = btrfs_file_extent_inline_start(ei); 213 214 if (compress_type != BTRFS_COMPRESS_NONE) { 215 struct page *cpage; 216 int i = 0; 217 while (compressed_size > 0) { 218 cpage = compressed_pages[i]; 219 cur_size = min_t(unsigned long, compressed_size, 220 PAGE_SIZE); 221 222 kaddr = kmap_atomic(cpage); 223 write_extent_buffer(leaf, kaddr, ptr, cur_size); 224 kunmap_atomic(kaddr); 225 226 i++; 227 ptr += cur_size; 228 compressed_size -= cur_size; 229 } 230 btrfs_set_file_extent_compression(leaf, ei, 231 compress_type); 232 } else { 233 page = find_get_page(inode->i_mapping, 234 start >> PAGE_SHIFT); 235 btrfs_set_file_extent_compression(leaf, ei, 0); 236 kaddr = kmap_atomic(page); 237 offset = start & (PAGE_SIZE - 1); 238 write_extent_buffer(leaf, kaddr + offset, ptr, size); 239 kunmap_atomic(kaddr); 240 put_page(page); 241 } 242 btrfs_mark_buffer_dirty(leaf); 243 btrfs_release_path(path); 244 245 /* 246 * we're an inline extent, so nobody can 247 * extend the file past i_size without locking 248 * a page we already have locked. 249 * 250 * We must do any isize and inode updates 251 * before we unlock the pages. Otherwise we 252 * could end up racing with unlink. 253 */ 254 BTRFS_I(inode)->disk_i_size = inode->i_size; 255 ret = btrfs_update_inode(trans, root, inode); 256 257 fail: 258 return ret; 259 } 260 261 262 /* 263 * conditionally insert an inline extent into the file. This 264 * does the checks required to make sure the data is small enough 265 * to fit as an inline extent. 266 */ 267 static noinline int cow_file_range_inline(struct inode *inode, u64 start, 268 u64 end, size_t compressed_size, 269 int compress_type, 270 struct page **compressed_pages) 271 { 272 struct btrfs_root *root = BTRFS_I(inode)->root; 273 struct btrfs_fs_info *fs_info = root->fs_info; 274 struct btrfs_trans_handle *trans; 275 u64 isize = i_size_read(inode); 276 u64 actual_end = min(end + 1, isize); 277 u64 inline_len = actual_end - start; 278 u64 aligned_end = ALIGN(end, fs_info->sectorsize); 279 u64 data_len = inline_len; 280 int ret; 281 struct btrfs_path *path; 282 int extent_inserted = 0; 283 u32 extent_item_size; 284 285 if (compressed_size) 286 data_len = compressed_size; 287 288 if (start > 0 || 289 actual_end > fs_info->sectorsize || 290 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 291 (!compressed_size && 292 (actual_end & (fs_info->sectorsize - 1)) == 0) || 293 end + 1 < isize || 294 data_len > fs_info->max_inline) { 295 return 1; 296 } 297 298 path = btrfs_alloc_path(); 299 if (!path) 300 return -ENOMEM; 301 302 trans = btrfs_join_transaction(root); 303 if (IS_ERR(trans)) { 304 btrfs_free_path(path); 305 return PTR_ERR(trans); 306 } 307 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 308 309 if (compressed_size && compressed_pages) 310 extent_item_size = btrfs_file_extent_calc_inline_size( 311 compressed_size); 312 else 313 extent_item_size = btrfs_file_extent_calc_inline_size( 314 inline_len); 315 316 ret = __btrfs_drop_extents(trans, root, inode, path, 317 start, aligned_end, NULL, 318 1, 1, extent_item_size, &extent_inserted); 319 if (ret) { 320 btrfs_abort_transaction(trans, ret); 321 goto out; 322 } 323 324 if (isize > actual_end) 325 inline_len = min_t(u64, isize, actual_end); 326 ret = insert_inline_extent(trans, path, extent_inserted, 327 root, inode, start, 328 inline_len, compressed_size, 329 compress_type, compressed_pages); 330 if (ret && ret != -ENOSPC) { 331 btrfs_abort_transaction(trans, ret); 332 goto out; 333 } else if (ret == -ENOSPC) { 334 ret = 1; 335 goto out; 336 } 337 338 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 339 btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0); 340 out: 341 /* 342 * Don't forget to free the reserved space, as for inlined extent 343 * it won't count as data extent, free them directly here. 344 * And at reserve time, it's always aligned to page size, so 345 * just free one page here. 346 */ 347 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE); 348 btrfs_free_path(path); 349 btrfs_end_transaction(trans); 350 return ret; 351 } 352 353 struct async_extent { 354 u64 start; 355 u64 ram_size; 356 u64 compressed_size; 357 struct page **pages; 358 unsigned long nr_pages; 359 int compress_type; 360 struct list_head list; 361 }; 362 363 struct async_cow { 364 struct inode *inode; 365 struct btrfs_root *root; 366 struct page *locked_page; 367 u64 start; 368 u64 end; 369 unsigned int write_flags; 370 struct list_head extents; 371 struct btrfs_work work; 372 }; 373 374 static noinline int add_async_extent(struct async_cow *cow, 375 u64 start, u64 ram_size, 376 u64 compressed_size, 377 struct page **pages, 378 unsigned long nr_pages, 379 int compress_type) 380 { 381 struct async_extent *async_extent; 382 383 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 384 BUG_ON(!async_extent); /* -ENOMEM */ 385 async_extent->start = start; 386 async_extent->ram_size = ram_size; 387 async_extent->compressed_size = compressed_size; 388 async_extent->pages = pages; 389 async_extent->nr_pages = nr_pages; 390 async_extent->compress_type = compress_type; 391 list_add_tail(&async_extent->list, &cow->extents); 392 return 0; 393 } 394 395 static inline int inode_need_compress(struct inode *inode, u64 start, u64 end) 396 { 397 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 398 399 /* force compress */ 400 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 401 return 1; 402 /* defrag ioctl */ 403 if (BTRFS_I(inode)->defrag_compress) 404 return 1; 405 /* bad compression ratios */ 406 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS) 407 return 0; 408 if (btrfs_test_opt(fs_info, COMPRESS) || 409 BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS || 410 BTRFS_I(inode)->prop_compress) 411 return btrfs_compress_heuristic(inode, start, end); 412 return 0; 413 } 414 415 static inline void inode_should_defrag(struct btrfs_inode *inode, 416 u64 start, u64 end, u64 num_bytes, u64 small_write) 417 { 418 /* If this is a small write inside eof, kick off a defrag */ 419 if (num_bytes < small_write && 420 (start > 0 || end + 1 < inode->disk_i_size)) 421 btrfs_add_inode_defrag(NULL, inode); 422 } 423 424 /* 425 * we create compressed extents in two phases. The first 426 * phase compresses a range of pages that have already been 427 * locked (both pages and state bits are locked). 428 * 429 * This is done inside an ordered work queue, and the compression 430 * is spread across many cpus. The actual IO submission is step 431 * two, and the ordered work queue takes care of making sure that 432 * happens in the same order things were put onto the queue by 433 * writepages and friends. 434 * 435 * If this code finds it can't get good compression, it puts an 436 * entry onto the work queue to write the uncompressed bytes. This 437 * makes sure that both compressed inodes and uncompressed inodes 438 * are written in the same order that the flusher thread sent them 439 * down. 440 */ 441 static noinline void compress_file_range(struct inode *inode, 442 struct page *locked_page, 443 u64 start, u64 end, 444 struct async_cow *async_cow, 445 int *num_added) 446 { 447 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 448 u64 blocksize = fs_info->sectorsize; 449 u64 actual_end; 450 u64 isize = i_size_read(inode); 451 int ret = 0; 452 struct page **pages = NULL; 453 unsigned long nr_pages; 454 unsigned long total_compressed = 0; 455 unsigned long total_in = 0; 456 int i; 457 int will_compress; 458 int compress_type = fs_info->compress_type; 459 int redirty = 0; 460 461 inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1, 462 SZ_16K); 463 464 actual_end = min_t(u64, isize, end + 1); 465 again: 466 will_compress = 0; 467 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; 468 BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0); 469 nr_pages = min_t(unsigned long, nr_pages, 470 BTRFS_MAX_COMPRESSED / PAGE_SIZE); 471 472 /* 473 * we don't want to send crud past the end of i_size through 474 * compression, that's just a waste of CPU time. So, if the 475 * end of the file is before the start of our current 476 * requested range of bytes, we bail out to the uncompressed 477 * cleanup code that can deal with all of this. 478 * 479 * It isn't really the fastest way to fix things, but this is a 480 * very uncommon corner. 481 */ 482 if (actual_end <= start) 483 goto cleanup_and_bail_uncompressed; 484 485 total_compressed = actual_end - start; 486 487 /* 488 * skip compression for a small file range(<=blocksize) that 489 * isn't an inline extent, since it doesn't save disk space at all. 490 */ 491 if (total_compressed <= blocksize && 492 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 493 goto cleanup_and_bail_uncompressed; 494 495 total_compressed = min_t(unsigned long, total_compressed, 496 BTRFS_MAX_UNCOMPRESSED); 497 total_in = 0; 498 ret = 0; 499 500 /* 501 * we do compression for mount -o compress and when the 502 * inode has not been flagged as nocompress. This flag can 503 * change at any time if we discover bad compression ratios. 504 */ 505 if (inode_need_compress(inode, start, end)) { 506 WARN_ON(pages); 507 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 508 if (!pages) { 509 /* just bail out to the uncompressed code */ 510 goto cont; 511 } 512 513 if (BTRFS_I(inode)->defrag_compress) 514 compress_type = BTRFS_I(inode)->defrag_compress; 515 else if (BTRFS_I(inode)->prop_compress) 516 compress_type = BTRFS_I(inode)->prop_compress; 517 518 /* 519 * we need to call clear_page_dirty_for_io on each 520 * page in the range. Otherwise applications with the file 521 * mmap'd can wander in and change the page contents while 522 * we are compressing them. 523 * 524 * If the compression fails for any reason, we set the pages 525 * dirty again later on. 526 * 527 * Note that the remaining part is redirtied, the start pointer 528 * has moved, the end is the original one. 529 */ 530 if (!redirty) { 531 extent_range_clear_dirty_for_io(inode, start, end); 532 redirty = 1; 533 } 534 535 /* Compression level is applied here and only here */ 536 ret = btrfs_compress_pages( 537 compress_type | (fs_info->compress_level << 4), 538 inode->i_mapping, start, 539 pages, 540 &nr_pages, 541 &total_in, 542 &total_compressed); 543 544 if (!ret) { 545 unsigned long offset = total_compressed & 546 (PAGE_SIZE - 1); 547 struct page *page = pages[nr_pages - 1]; 548 char *kaddr; 549 550 /* zero the tail end of the last page, we might be 551 * sending it down to disk 552 */ 553 if (offset) { 554 kaddr = kmap_atomic(page); 555 memset(kaddr + offset, 0, 556 PAGE_SIZE - offset); 557 kunmap_atomic(kaddr); 558 } 559 will_compress = 1; 560 } 561 } 562 cont: 563 if (start == 0) { 564 /* lets try to make an inline extent */ 565 if (ret || total_in < actual_end) { 566 /* we didn't compress the entire range, try 567 * to make an uncompressed inline extent. 568 */ 569 ret = cow_file_range_inline(inode, start, end, 0, 570 BTRFS_COMPRESS_NONE, NULL); 571 } else { 572 /* try making a compressed inline extent */ 573 ret = cow_file_range_inline(inode, start, end, 574 total_compressed, 575 compress_type, pages); 576 } 577 if (ret <= 0) { 578 unsigned long clear_flags = EXTENT_DELALLOC | 579 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 580 EXTENT_DO_ACCOUNTING; 581 unsigned long page_error_op; 582 583 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 584 585 /* 586 * inline extent creation worked or returned error, 587 * we don't need to create any more async work items. 588 * Unlock and free up our temp pages. 589 * 590 * We use DO_ACCOUNTING here because we need the 591 * delalloc_release_metadata to be done _after_ we drop 592 * our outstanding extent for clearing delalloc for this 593 * range. 594 */ 595 extent_clear_unlock_delalloc(inode, start, end, end, 596 NULL, clear_flags, 597 PAGE_UNLOCK | 598 PAGE_CLEAR_DIRTY | 599 PAGE_SET_WRITEBACK | 600 page_error_op | 601 PAGE_END_WRITEBACK); 602 goto free_pages_out; 603 } 604 } 605 606 if (will_compress) { 607 /* 608 * we aren't doing an inline extent round the compressed size 609 * up to a block size boundary so the allocator does sane 610 * things 611 */ 612 total_compressed = ALIGN(total_compressed, blocksize); 613 614 /* 615 * one last check to make sure the compression is really a 616 * win, compare the page count read with the blocks on disk, 617 * compression must free at least one sector size 618 */ 619 total_in = ALIGN(total_in, PAGE_SIZE); 620 if (total_compressed + blocksize <= total_in) { 621 *num_added += 1; 622 623 /* 624 * The async work queues will take care of doing actual 625 * allocation on disk for these compressed pages, and 626 * will submit them to the elevator. 627 */ 628 add_async_extent(async_cow, start, total_in, 629 total_compressed, pages, nr_pages, 630 compress_type); 631 632 if (start + total_in < end) { 633 start += total_in; 634 pages = NULL; 635 cond_resched(); 636 goto again; 637 } 638 return; 639 } 640 } 641 if (pages) { 642 /* 643 * the compression code ran but failed to make things smaller, 644 * free any pages it allocated and our page pointer array 645 */ 646 for (i = 0; i < nr_pages; i++) { 647 WARN_ON(pages[i]->mapping); 648 put_page(pages[i]); 649 } 650 kfree(pages); 651 pages = NULL; 652 total_compressed = 0; 653 nr_pages = 0; 654 655 /* flag the file so we don't compress in the future */ 656 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && 657 !(BTRFS_I(inode)->prop_compress)) { 658 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 659 } 660 } 661 cleanup_and_bail_uncompressed: 662 /* 663 * No compression, but we still need to write the pages in the file 664 * we've been given so far. redirty the locked page if it corresponds 665 * to our extent and set things up for the async work queue to run 666 * cow_file_range to do the normal delalloc dance. 667 */ 668 if (page_offset(locked_page) >= start && 669 page_offset(locked_page) <= end) 670 __set_page_dirty_nobuffers(locked_page); 671 /* unlocked later on in the async handlers */ 672 673 if (redirty) 674 extent_range_redirty_for_io(inode, start, end); 675 add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0, 676 BTRFS_COMPRESS_NONE); 677 *num_added += 1; 678 679 return; 680 681 free_pages_out: 682 for (i = 0; i < nr_pages; i++) { 683 WARN_ON(pages[i]->mapping); 684 put_page(pages[i]); 685 } 686 kfree(pages); 687 } 688 689 static void free_async_extent_pages(struct async_extent *async_extent) 690 { 691 int i; 692 693 if (!async_extent->pages) 694 return; 695 696 for (i = 0; i < async_extent->nr_pages; i++) { 697 WARN_ON(async_extent->pages[i]->mapping); 698 put_page(async_extent->pages[i]); 699 } 700 kfree(async_extent->pages); 701 async_extent->nr_pages = 0; 702 async_extent->pages = NULL; 703 } 704 705 /* 706 * phase two of compressed writeback. This is the ordered portion 707 * of the code, which only gets called in the order the work was 708 * queued. We walk all the async extents created by compress_file_range 709 * and send them down to the disk. 710 */ 711 static noinline void submit_compressed_extents(struct inode *inode, 712 struct async_cow *async_cow) 713 { 714 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 715 struct async_extent *async_extent; 716 u64 alloc_hint = 0; 717 struct btrfs_key ins; 718 struct extent_map *em; 719 struct btrfs_root *root = BTRFS_I(inode)->root; 720 struct extent_io_tree *io_tree; 721 int ret = 0; 722 723 again: 724 while (!list_empty(&async_cow->extents)) { 725 async_extent = list_entry(async_cow->extents.next, 726 struct async_extent, list); 727 list_del(&async_extent->list); 728 729 io_tree = &BTRFS_I(inode)->io_tree; 730 731 retry: 732 /* did the compression code fall back to uncompressed IO? */ 733 if (!async_extent->pages) { 734 int page_started = 0; 735 unsigned long nr_written = 0; 736 737 lock_extent(io_tree, async_extent->start, 738 async_extent->start + 739 async_extent->ram_size - 1); 740 741 /* allocate blocks */ 742 ret = cow_file_range(inode, async_cow->locked_page, 743 async_extent->start, 744 async_extent->start + 745 async_extent->ram_size - 1, 746 async_extent->start + 747 async_extent->ram_size - 1, 748 &page_started, &nr_written, 0, 749 NULL); 750 751 /* JDM XXX */ 752 753 /* 754 * if page_started, cow_file_range inserted an 755 * inline extent and took care of all the unlocking 756 * and IO for us. Otherwise, we need to submit 757 * all those pages down to the drive. 758 */ 759 if (!page_started && !ret) 760 extent_write_locked_range(inode, 761 async_extent->start, 762 async_extent->start + 763 async_extent->ram_size - 1, 764 WB_SYNC_ALL); 765 else if (ret) 766 unlock_page(async_cow->locked_page); 767 kfree(async_extent); 768 cond_resched(); 769 continue; 770 } 771 772 lock_extent(io_tree, async_extent->start, 773 async_extent->start + async_extent->ram_size - 1); 774 775 ret = btrfs_reserve_extent(root, async_extent->ram_size, 776 async_extent->compressed_size, 777 async_extent->compressed_size, 778 0, alloc_hint, &ins, 1, 1); 779 if (ret) { 780 free_async_extent_pages(async_extent); 781 782 if (ret == -ENOSPC) { 783 unlock_extent(io_tree, async_extent->start, 784 async_extent->start + 785 async_extent->ram_size - 1); 786 787 /* 788 * we need to redirty the pages if we decide to 789 * fallback to uncompressed IO, otherwise we 790 * will not submit these pages down to lower 791 * layers. 792 */ 793 extent_range_redirty_for_io(inode, 794 async_extent->start, 795 async_extent->start + 796 async_extent->ram_size - 1); 797 798 goto retry; 799 } 800 goto out_free; 801 } 802 /* 803 * here we're doing allocation and writeback of the 804 * compressed pages 805 */ 806 em = create_io_em(inode, async_extent->start, 807 async_extent->ram_size, /* len */ 808 async_extent->start, /* orig_start */ 809 ins.objectid, /* block_start */ 810 ins.offset, /* block_len */ 811 ins.offset, /* orig_block_len */ 812 async_extent->ram_size, /* ram_bytes */ 813 async_extent->compress_type, 814 BTRFS_ORDERED_COMPRESSED); 815 if (IS_ERR(em)) 816 /* ret value is not necessary due to void function */ 817 goto out_free_reserve; 818 free_extent_map(em); 819 820 ret = btrfs_add_ordered_extent_compress(inode, 821 async_extent->start, 822 ins.objectid, 823 async_extent->ram_size, 824 ins.offset, 825 BTRFS_ORDERED_COMPRESSED, 826 async_extent->compress_type); 827 if (ret) { 828 btrfs_drop_extent_cache(BTRFS_I(inode), 829 async_extent->start, 830 async_extent->start + 831 async_extent->ram_size - 1, 0); 832 goto out_free_reserve; 833 } 834 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 835 836 /* 837 * clear dirty, set writeback and unlock the pages. 838 */ 839 extent_clear_unlock_delalloc(inode, async_extent->start, 840 async_extent->start + 841 async_extent->ram_size - 1, 842 async_extent->start + 843 async_extent->ram_size - 1, 844 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 845 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 846 PAGE_SET_WRITEBACK); 847 if (btrfs_submit_compressed_write(inode, 848 async_extent->start, 849 async_extent->ram_size, 850 ins.objectid, 851 ins.offset, async_extent->pages, 852 async_extent->nr_pages, 853 async_cow->write_flags)) { 854 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 855 struct page *p = async_extent->pages[0]; 856 const u64 start = async_extent->start; 857 const u64 end = start + async_extent->ram_size - 1; 858 859 p->mapping = inode->i_mapping; 860 tree->ops->writepage_end_io_hook(p, start, end, 861 NULL, 0); 862 p->mapping = NULL; 863 extent_clear_unlock_delalloc(inode, start, end, end, 864 NULL, 0, 865 PAGE_END_WRITEBACK | 866 PAGE_SET_ERROR); 867 free_async_extent_pages(async_extent); 868 } 869 alloc_hint = ins.objectid + ins.offset; 870 kfree(async_extent); 871 cond_resched(); 872 } 873 return; 874 out_free_reserve: 875 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 876 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 877 out_free: 878 extent_clear_unlock_delalloc(inode, async_extent->start, 879 async_extent->start + 880 async_extent->ram_size - 1, 881 async_extent->start + 882 async_extent->ram_size - 1, 883 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 884 EXTENT_DELALLOC_NEW | 885 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 886 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 887 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | 888 PAGE_SET_ERROR); 889 free_async_extent_pages(async_extent); 890 kfree(async_extent); 891 goto again; 892 } 893 894 static u64 get_extent_allocation_hint(struct inode *inode, u64 start, 895 u64 num_bytes) 896 { 897 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 898 struct extent_map *em; 899 u64 alloc_hint = 0; 900 901 read_lock(&em_tree->lock); 902 em = search_extent_mapping(em_tree, start, num_bytes); 903 if (em) { 904 /* 905 * if block start isn't an actual block number then find the 906 * first block in this inode and use that as a hint. If that 907 * block is also bogus then just don't worry about it. 908 */ 909 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 910 free_extent_map(em); 911 em = search_extent_mapping(em_tree, 0, 0); 912 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 913 alloc_hint = em->block_start; 914 if (em) 915 free_extent_map(em); 916 } else { 917 alloc_hint = em->block_start; 918 free_extent_map(em); 919 } 920 } 921 read_unlock(&em_tree->lock); 922 923 return alloc_hint; 924 } 925 926 /* 927 * when extent_io.c finds a delayed allocation range in the file, 928 * the call backs end up in this code. The basic idea is to 929 * allocate extents on disk for the range, and create ordered data structs 930 * in ram to track those extents. 931 * 932 * locked_page is the page that writepage had locked already. We use 933 * it to make sure we don't do extra locks or unlocks. 934 * 935 * *page_started is set to one if we unlock locked_page and do everything 936 * required to start IO on it. It may be clean and already done with 937 * IO when we return. 938 */ 939 static noinline int cow_file_range(struct inode *inode, 940 struct page *locked_page, 941 u64 start, u64 end, u64 delalloc_end, 942 int *page_started, unsigned long *nr_written, 943 int unlock, struct btrfs_dedupe_hash *hash) 944 { 945 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 946 struct btrfs_root *root = BTRFS_I(inode)->root; 947 u64 alloc_hint = 0; 948 u64 num_bytes; 949 unsigned long ram_size; 950 u64 cur_alloc_size = 0; 951 u64 blocksize = fs_info->sectorsize; 952 struct btrfs_key ins; 953 struct extent_map *em; 954 unsigned clear_bits; 955 unsigned long page_ops; 956 bool extent_reserved = false; 957 int ret = 0; 958 959 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 960 WARN_ON_ONCE(1); 961 ret = -EINVAL; 962 goto out_unlock; 963 } 964 965 num_bytes = ALIGN(end - start + 1, blocksize); 966 num_bytes = max(blocksize, num_bytes); 967 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); 968 969 inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K); 970 971 if (start == 0) { 972 /* lets try to make an inline extent */ 973 ret = cow_file_range_inline(inode, start, end, 0, 974 BTRFS_COMPRESS_NONE, NULL); 975 if (ret == 0) { 976 /* 977 * We use DO_ACCOUNTING here because we need the 978 * delalloc_release_metadata to be run _after_ we drop 979 * our outstanding extent for clearing delalloc for this 980 * range. 981 */ 982 extent_clear_unlock_delalloc(inode, start, end, 983 delalloc_end, NULL, 984 EXTENT_LOCKED | EXTENT_DELALLOC | 985 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 986 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 987 PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 988 PAGE_END_WRITEBACK); 989 *nr_written = *nr_written + 990 (end - start + PAGE_SIZE) / PAGE_SIZE; 991 *page_started = 1; 992 goto out; 993 } else if (ret < 0) { 994 goto out_unlock; 995 } 996 } 997 998 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 999 btrfs_drop_extent_cache(BTRFS_I(inode), start, 1000 start + num_bytes - 1, 0); 1001 1002 while (num_bytes > 0) { 1003 cur_alloc_size = num_bytes; 1004 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size, 1005 fs_info->sectorsize, 0, alloc_hint, 1006 &ins, 1, 1); 1007 if (ret < 0) 1008 goto out_unlock; 1009 cur_alloc_size = ins.offset; 1010 extent_reserved = true; 1011 1012 ram_size = ins.offset; 1013 em = create_io_em(inode, start, ins.offset, /* len */ 1014 start, /* orig_start */ 1015 ins.objectid, /* block_start */ 1016 ins.offset, /* block_len */ 1017 ins.offset, /* orig_block_len */ 1018 ram_size, /* ram_bytes */ 1019 BTRFS_COMPRESS_NONE, /* compress_type */ 1020 BTRFS_ORDERED_REGULAR /* type */); 1021 if (IS_ERR(em)) 1022 goto out_reserve; 1023 free_extent_map(em); 1024 1025 ret = btrfs_add_ordered_extent(inode, start, ins.objectid, 1026 ram_size, cur_alloc_size, 0); 1027 if (ret) 1028 goto out_drop_extent_cache; 1029 1030 if (root->root_key.objectid == 1031 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1032 ret = btrfs_reloc_clone_csums(inode, start, 1033 cur_alloc_size); 1034 /* 1035 * Only drop cache here, and process as normal. 1036 * 1037 * We must not allow extent_clear_unlock_delalloc() 1038 * at out_unlock label to free meta of this ordered 1039 * extent, as its meta should be freed by 1040 * btrfs_finish_ordered_io(). 1041 * 1042 * So we must continue until @start is increased to 1043 * skip current ordered extent. 1044 */ 1045 if (ret) 1046 btrfs_drop_extent_cache(BTRFS_I(inode), start, 1047 start + ram_size - 1, 0); 1048 } 1049 1050 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1051 1052 /* we're not doing compressed IO, don't unlock the first 1053 * page (which the caller expects to stay locked), don't 1054 * clear any dirty bits and don't set any writeback bits 1055 * 1056 * Do set the Private2 bit so we know this page was properly 1057 * setup for writepage 1058 */ 1059 page_ops = unlock ? PAGE_UNLOCK : 0; 1060 page_ops |= PAGE_SET_PRIVATE2; 1061 1062 extent_clear_unlock_delalloc(inode, start, 1063 start + ram_size - 1, 1064 delalloc_end, locked_page, 1065 EXTENT_LOCKED | EXTENT_DELALLOC, 1066 page_ops); 1067 if (num_bytes < cur_alloc_size) 1068 num_bytes = 0; 1069 else 1070 num_bytes -= cur_alloc_size; 1071 alloc_hint = ins.objectid + ins.offset; 1072 start += cur_alloc_size; 1073 extent_reserved = false; 1074 1075 /* 1076 * btrfs_reloc_clone_csums() error, since start is increased 1077 * extent_clear_unlock_delalloc() at out_unlock label won't 1078 * free metadata of current ordered extent, we're OK to exit. 1079 */ 1080 if (ret) 1081 goto out_unlock; 1082 } 1083 out: 1084 return ret; 1085 1086 out_drop_extent_cache: 1087 btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0); 1088 out_reserve: 1089 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1090 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1091 out_unlock: 1092 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 1093 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; 1094 page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 1095 PAGE_END_WRITEBACK; 1096 /* 1097 * If we reserved an extent for our delalloc range (or a subrange) and 1098 * failed to create the respective ordered extent, then it means that 1099 * when we reserved the extent we decremented the extent's size from 1100 * the data space_info's bytes_may_use counter and incremented the 1101 * space_info's bytes_reserved counter by the same amount. We must make 1102 * sure extent_clear_unlock_delalloc() does not try to decrement again 1103 * the data space_info's bytes_may_use counter, therefore we do not pass 1104 * it the flag EXTENT_CLEAR_DATA_RESV. 1105 */ 1106 if (extent_reserved) { 1107 extent_clear_unlock_delalloc(inode, start, 1108 start + cur_alloc_size, 1109 start + cur_alloc_size, 1110 locked_page, 1111 clear_bits, 1112 page_ops); 1113 start += cur_alloc_size; 1114 if (start >= end) 1115 goto out; 1116 } 1117 extent_clear_unlock_delalloc(inode, start, end, delalloc_end, 1118 locked_page, 1119 clear_bits | EXTENT_CLEAR_DATA_RESV, 1120 page_ops); 1121 goto out; 1122 } 1123 1124 /* 1125 * work queue call back to started compression on a file and pages 1126 */ 1127 static noinline void async_cow_start(struct btrfs_work *work) 1128 { 1129 struct async_cow *async_cow; 1130 int num_added = 0; 1131 async_cow = container_of(work, struct async_cow, work); 1132 1133 compress_file_range(async_cow->inode, async_cow->locked_page, 1134 async_cow->start, async_cow->end, async_cow, 1135 &num_added); 1136 if (num_added == 0) { 1137 btrfs_add_delayed_iput(async_cow->inode); 1138 async_cow->inode = NULL; 1139 } 1140 } 1141 1142 /* 1143 * work queue call back to submit previously compressed pages 1144 */ 1145 static noinline void async_cow_submit(struct btrfs_work *work) 1146 { 1147 struct btrfs_fs_info *fs_info; 1148 struct async_cow *async_cow; 1149 struct btrfs_root *root; 1150 unsigned long nr_pages; 1151 1152 async_cow = container_of(work, struct async_cow, work); 1153 1154 root = async_cow->root; 1155 fs_info = root->fs_info; 1156 nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >> 1157 PAGE_SHIFT; 1158 1159 /* 1160 * atomic_sub_return implies a barrier for waitqueue_active 1161 */ 1162 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1163 5 * SZ_1M && 1164 waitqueue_active(&fs_info->async_submit_wait)) 1165 wake_up(&fs_info->async_submit_wait); 1166 1167 if (async_cow->inode) 1168 submit_compressed_extents(async_cow->inode, async_cow); 1169 } 1170 1171 static noinline void async_cow_free(struct btrfs_work *work) 1172 { 1173 struct async_cow *async_cow; 1174 async_cow = container_of(work, struct async_cow, work); 1175 if (async_cow->inode) 1176 btrfs_add_delayed_iput(async_cow->inode); 1177 kfree(async_cow); 1178 } 1179 1180 static int cow_file_range_async(struct inode *inode, struct page *locked_page, 1181 u64 start, u64 end, int *page_started, 1182 unsigned long *nr_written, 1183 unsigned int write_flags) 1184 { 1185 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1186 struct async_cow *async_cow; 1187 struct btrfs_root *root = BTRFS_I(inode)->root; 1188 unsigned long nr_pages; 1189 u64 cur_end; 1190 1191 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED, 1192 1, 0, NULL); 1193 while (start < end) { 1194 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS); 1195 BUG_ON(!async_cow); /* -ENOMEM */ 1196 async_cow->inode = igrab(inode); 1197 async_cow->root = root; 1198 async_cow->locked_page = locked_page; 1199 async_cow->start = start; 1200 async_cow->write_flags = write_flags; 1201 1202 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS && 1203 !btrfs_test_opt(fs_info, FORCE_COMPRESS)) 1204 cur_end = end; 1205 else 1206 cur_end = min(end, start + SZ_512K - 1); 1207 1208 async_cow->end = cur_end; 1209 INIT_LIST_HEAD(&async_cow->extents); 1210 1211 btrfs_init_work(&async_cow->work, 1212 btrfs_delalloc_helper, 1213 async_cow_start, async_cow_submit, 1214 async_cow_free); 1215 1216 nr_pages = (cur_end - start + PAGE_SIZE) >> 1217 PAGE_SHIFT; 1218 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1219 1220 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work); 1221 1222 *nr_written += nr_pages; 1223 start = cur_end + 1; 1224 } 1225 *page_started = 1; 1226 return 0; 1227 } 1228 1229 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, 1230 u64 bytenr, u64 num_bytes) 1231 { 1232 int ret; 1233 struct btrfs_ordered_sum *sums; 1234 LIST_HEAD(list); 1235 1236 ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr, 1237 bytenr + num_bytes - 1, &list, 0); 1238 if (ret == 0 && list_empty(&list)) 1239 return 0; 1240 1241 while (!list_empty(&list)) { 1242 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1243 list_del(&sums->list); 1244 kfree(sums); 1245 } 1246 if (ret < 0) 1247 return ret; 1248 return 1; 1249 } 1250 1251 /* 1252 * when nowcow writeback call back. This checks for snapshots or COW copies 1253 * of the extents that exist in the file, and COWs the file as required. 1254 * 1255 * If no cow copies or snapshots exist, we write directly to the existing 1256 * blocks on disk 1257 */ 1258 static noinline int run_delalloc_nocow(struct inode *inode, 1259 struct page *locked_page, 1260 u64 start, u64 end, int *page_started, int force, 1261 unsigned long *nr_written) 1262 { 1263 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1264 struct btrfs_root *root = BTRFS_I(inode)->root; 1265 struct extent_buffer *leaf; 1266 struct btrfs_path *path; 1267 struct btrfs_file_extent_item *fi; 1268 struct btrfs_key found_key; 1269 struct extent_map *em; 1270 u64 cow_start; 1271 u64 cur_offset; 1272 u64 extent_end; 1273 u64 extent_offset; 1274 u64 disk_bytenr; 1275 u64 num_bytes; 1276 u64 disk_num_bytes; 1277 u64 ram_bytes; 1278 int extent_type; 1279 int ret, err; 1280 int type; 1281 int nocow; 1282 int check_prev = 1; 1283 bool nolock; 1284 u64 ino = btrfs_ino(BTRFS_I(inode)); 1285 1286 path = btrfs_alloc_path(); 1287 if (!path) { 1288 extent_clear_unlock_delalloc(inode, start, end, end, 1289 locked_page, 1290 EXTENT_LOCKED | EXTENT_DELALLOC | 1291 EXTENT_DO_ACCOUNTING | 1292 EXTENT_DEFRAG, PAGE_UNLOCK | 1293 PAGE_CLEAR_DIRTY | 1294 PAGE_SET_WRITEBACK | 1295 PAGE_END_WRITEBACK); 1296 return -ENOMEM; 1297 } 1298 1299 nolock = btrfs_is_free_space_inode(BTRFS_I(inode)); 1300 1301 cow_start = (u64)-1; 1302 cur_offset = start; 1303 while (1) { 1304 ret = btrfs_lookup_file_extent(NULL, root, path, ino, 1305 cur_offset, 0); 1306 if (ret < 0) 1307 goto error; 1308 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1309 leaf = path->nodes[0]; 1310 btrfs_item_key_to_cpu(leaf, &found_key, 1311 path->slots[0] - 1); 1312 if (found_key.objectid == ino && 1313 found_key.type == BTRFS_EXTENT_DATA_KEY) 1314 path->slots[0]--; 1315 } 1316 check_prev = 0; 1317 next_slot: 1318 leaf = path->nodes[0]; 1319 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1320 ret = btrfs_next_leaf(root, path); 1321 if (ret < 0) { 1322 if (cow_start != (u64)-1) 1323 cur_offset = cow_start; 1324 goto error; 1325 } 1326 if (ret > 0) 1327 break; 1328 leaf = path->nodes[0]; 1329 } 1330 1331 nocow = 0; 1332 disk_bytenr = 0; 1333 num_bytes = 0; 1334 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1335 1336 if (found_key.objectid > ino) 1337 break; 1338 if (WARN_ON_ONCE(found_key.objectid < ino) || 1339 found_key.type < BTRFS_EXTENT_DATA_KEY) { 1340 path->slots[0]++; 1341 goto next_slot; 1342 } 1343 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 1344 found_key.offset > end) 1345 break; 1346 1347 if (found_key.offset > cur_offset) { 1348 extent_end = found_key.offset; 1349 extent_type = 0; 1350 goto out_check; 1351 } 1352 1353 fi = btrfs_item_ptr(leaf, path->slots[0], 1354 struct btrfs_file_extent_item); 1355 extent_type = btrfs_file_extent_type(leaf, fi); 1356 1357 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1358 if (extent_type == BTRFS_FILE_EXTENT_REG || 1359 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1360 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1361 extent_offset = btrfs_file_extent_offset(leaf, fi); 1362 extent_end = found_key.offset + 1363 btrfs_file_extent_num_bytes(leaf, fi); 1364 disk_num_bytes = 1365 btrfs_file_extent_disk_num_bytes(leaf, fi); 1366 if (extent_end <= start) { 1367 path->slots[0]++; 1368 goto next_slot; 1369 } 1370 if (disk_bytenr == 0) 1371 goto out_check; 1372 if (btrfs_file_extent_compression(leaf, fi) || 1373 btrfs_file_extent_encryption(leaf, fi) || 1374 btrfs_file_extent_other_encoding(leaf, fi)) 1375 goto out_check; 1376 if (extent_type == BTRFS_FILE_EXTENT_REG && !force) 1377 goto out_check; 1378 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 1379 goto out_check; 1380 ret = btrfs_cross_ref_exist(root, ino, 1381 found_key.offset - 1382 extent_offset, disk_bytenr); 1383 if (ret) { 1384 /* 1385 * ret could be -EIO if the above fails to read 1386 * metadata. 1387 */ 1388 if (ret < 0) { 1389 if (cow_start != (u64)-1) 1390 cur_offset = cow_start; 1391 goto error; 1392 } 1393 1394 WARN_ON_ONCE(nolock); 1395 goto out_check; 1396 } 1397 disk_bytenr += extent_offset; 1398 disk_bytenr += cur_offset - found_key.offset; 1399 num_bytes = min(end + 1, extent_end) - cur_offset; 1400 /* 1401 * if there are pending snapshots for this root, 1402 * we fall into common COW way. 1403 */ 1404 if (!nolock) { 1405 err = btrfs_start_write_no_snapshotting(root); 1406 if (!err) 1407 goto out_check; 1408 } 1409 /* 1410 * force cow if csum exists in the range. 1411 * this ensure that csum for a given extent are 1412 * either valid or do not exist. 1413 */ 1414 ret = csum_exist_in_range(fs_info, disk_bytenr, 1415 num_bytes); 1416 if (ret) { 1417 if (!nolock) 1418 btrfs_end_write_no_snapshotting(root); 1419 1420 /* 1421 * ret could be -EIO if the above fails to read 1422 * metadata. 1423 */ 1424 if (ret < 0) { 1425 if (cow_start != (u64)-1) 1426 cur_offset = cow_start; 1427 goto error; 1428 } 1429 WARN_ON_ONCE(nolock); 1430 goto out_check; 1431 } 1432 if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) { 1433 if (!nolock) 1434 btrfs_end_write_no_snapshotting(root); 1435 goto out_check; 1436 } 1437 nocow = 1; 1438 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 1439 extent_end = found_key.offset + 1440 btrfs_file_extent_inline_len(leaf, 1441 path->slots[0], fi); 1442 extent_end = ALIGN(extent_end, 1443 fs_info->sectorsize); 1444 } else { 1445 BUG_ON(1); 1446 } 1447 out_check: 1448 if (extent_end <= start) { 1449 path->slots[0]++; 1450 if (!nolock && nocow) 1451 btrfs_end_write_no_snapshotting(root); 1452 if (nocow) 1453 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1454 goto next_slot; 1455 } 1456 if (!nocow) { 1457 if (cow_start == (u64)-1) 1458 cow_start = cur_offset; 1459 cur_offset = extent_end; 1460 if (cur_offset > end) 1461 break; 1462 path->slots[0]++; 1463 goto next_slot; 1464 } 1465 1466 btrfs_release_path(path); 1467 if (cow_start != (u64)-1) { 1468 ret = cow_file_range(inode, locked_page, 1469 cow_start, found_key.offset - 1, 1470 end, page_started, nr_written, 1, 1471 NULL); 1472 if (ret) { 1473 if (!nolock && nocow) 1474 btrfs_end_write_no_snapshotting(root); 1475 if (nocow) 1476 btrfs_dec_nocow_writers(fs_info, 1477 disk_bytenr); 1478 goto error; 1479 } 1480 cow_start = (u64)-1; 1481 } 1482 1483 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1484 u64 orig_start = found_key.offset - extent_offset; 1485 1486 em = create_io_em(inode, cur_offset, num_bytes, 1487 orig_start, 1488 disk_bytenr, /* block_start */ 1489 num_bytes, /* block_len */ 1490 disk_num_bytes, /* orig_block_len */ 1491 ram_bytes, BTRFS_COMPRESS_NONE, 1492 BTRFS_ORDERED_PREALLOC); 1493 if (IS_ERR(em)) { 1494 if (!nolock && nocow) 1495 btrfs_end_write_no_snapshotting(root); 1496 if (nocow) 1497 btrfs_dec_nocow_writers(fs_info, 1498 disk_bytenr); 1499 ret = PTR_ERR(em); 1500 goto error; 1501 } 1502 free_extent_map(em); 1503 } 1504 1505 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1506 type = BTRFS_ORDERED_PREALLOC; 1507 } else { 1508 type = BTRFS_ORDERED_NOCOW; 1509 } 1510 1511 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr, 1512 num_bytes, num_bytes, type); 1513 if (nocow) 1514 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1515 BUG_ON(ret); /* -ENOMEM */ 1516 1517 if (root->root_key.objectid == 1518 BTRFS_DATA_RELOC_TREE_OBJECTID) 1519 /* 1520 * Error handled later, as we must prevent 1521 * extent_clear_unlock_delalloc() in error handler 1522 * from freeing metadata of created ordered extent. 1523 */ 1524 ret = btrfs_reloc_clone_csums(inode, cur_offset, 1525 num_bytes); 1526 1527 extent_clear_unlock_delalloc(inode, cur_offset, 1528 cur_offset + num_bytes - 1, end, 1529 locked_page, EXTENT_LOCKED | 1530 EXTENT_DELALLOC | 1531 EXTENT_CLEAR_DATA_RESV, 1532 PAGE_UNLOCK | PAGE_SET_PRIVATE2); 1533 1534 if (!nolock && nocow) 1535 btrfs_end_write_no_snapshotting(root); 1536 cur_offset = extent_end; 1537 1538 /* 1539 * btrfs_reloc_clone_csums() error, now we're OK to call error 1540 * handler, as metadata for created ordered extent will only 1541 * be freed by btrfs_finish_ordered_io(). 1542 */ 1543 if (ret) 1544 goto error; 1545 if (cur_offset > end) 1546 break; 1547 } 1548 btrfs_release_path(path); 1549 1550 if (cur_offset <= end && cow_start == (u64)-1) { 1551 cow_start = cur_offset; 1552 cur_offset = end; 1553 } 1554 1555 if (cow_start != (u64)-1) { 1556 ret = cow_file_range(inode, locked_page, cow_start, end, end, 1557 page_started, nr_written, 1, NULL); 1558 if (ret) 1559 goto error; 1560 } 1561 1562 error: 1563 if (ret && cur_offset < end) 1564 extent_clear_unlock_delalloc(inode, cur_offset, end, end, 1565 locked_page, EXTENT_LOCKED | 1566 EXTENT_DELALLOC | EXTENT_DEFRAG | 1567 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1568 PAGE_CLEAR_DIRTY | 1569 PAGE_SET_WRITEBACK | 1570 PAGE_END_WRITEBACK); 1571 btrfs_free_path(path); 1572 return ret; 1573 } 1574 1575 static inline int need_force_cow(struct inode *inode, u64 start, u64 end) 1576 { 1577 1578 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 1579 !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC)) 1580 return 0; 1581 1582 /* 1583 * @defrag_bytes is a hint value, no spinlock held here, 1584 * if is not zero, it means the file is defragging. 1585 * Force cow if given extent needs to be defragged. 1586 */ 1587 if (BTRFS_I(inode)->defrag_bytes && 1588 test_range_bit(&BTRFS_I(inode)->io_tree, start, end, 1589 EXTENT_DEFRAG, 0, NULL)) 1590 return 1; 1591 1592 return 0; 1593 } 1594 1595 /* 1596 * extent_io.c call back to do delayed allocation processing 1597 */ 1598 static int run_delalloc_range(void *private_data, struct page *locked_page, 1599 u64 start, u64 end, int *page_started, 1600 unsigned long *nr_written, 1601 struct writeback_control *wbc) 1602 { 1603 struct inode *inode = private_data; 1604 int ret; 1605 int force_cow = need_force_cow(inode, start, end); 1606 unsigned int write_flags = wbc_to_write_flags(wbc); 1607 1608 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) { 1609 ret = run_delalloc_nocow(inode, locked_page, start, end, 1610 page_started, 1, nr_written); 1611 } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) { 1612 ret = run_delalloc_nocow(inode, locked_page, start, end, 1613 page_started, 0, nr_written); 1614 } else if (!inode_need_compress(inode, start, end)) { 1615 ret = cow_file_range(inode, locked_page, start, end, end, 1616 page_started, nr_written, 1, NULL); 1617 } else { 1618 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 1619 &BTRFS_I(inode)->runtime_flags); 1620 ret = cow_file_range_async(inode, locked_page, start, end, 1621 page_started, nr_written, 1622 write_flags); 1623 } 1624 if (ret) 1625 btrfs_cleanup_ordered_extents(inode, start, end - start + 1); 1626 return ret; 1627 } 1628 1629 static void btrfs_split_extent_hook(void *private_data, 1630 struct extent_state *orig, u64 split) 1631 { 1632 struct inode *inode = private_data; 1633 u64 size; 1634 1635 /* not delalloc, ignore it */ 1636 if (!(orig->state & EXTENT_DELALLOC)) 1637 return; 1638 1639 size = orig->end - orig->start + 1; 1640 if (size > BTRFS_MAX_EXTENT_SIZE) { 1641 u32 num_extents; 1642 u64 new_size; 1643 1644 /* 1645 * See the explanation in btrfs_merge_extent_hook, the same 1646 * applies here, just in reverse. 1647 */ 1648 new_size = orig->end - split + 1; 1649 num_extents = count_max_extents(new_size); 1650 new_size = split - orig->start; 1651 num_extents += count_max_extents(new_size); 1652 if (count_max_extents(size) >= num_extents) 1653 return; 1654 } 1655 1656 spin_lock(&BTRFS_I(inode)->lock); 1657 btrfs_mod_outstanding_extents(BTRFS_I(inode), 1); 1658 spin_unlock(&BTRFS_I(inode)->lock); 1659 } 1660 1661 /* 1662 * extent_io.c merge_extent_hook, used to track merged delayed allocation 1663 * extents so we can keep track of new extents that are just merged onto old 1664 * extents, such as when we are doing sequential writes, so we can properly 1665 * account for the metadata space we'll need. 1666 */ 1667 static void btrfs_merge_extent_hook(void *private_data, 1668 struct extent_state *new, 1669 struct extent_state *other) 1670 { 1671 struct inode *inode = private_data; 1672 u64 new_size, old_size; 1673 u32 num_extents; 1674 1675 /* not delalloc, ignore it */ 1676 if (!(other->state & EXTENT_DELALLOC)) 1677 return; 1678 1679 if (new->start > other->start) 1680 new_size = new->end - other->start + 1; 1681 else 1682 new_size = other->end - new->start + 1; 1683 1684 /* we're not bigger than the max, unreserve the space and go */ 1685 if (new_size <= BTRFS_MAX_EXTENT_SIZE) { 1686 spin_lock(&BTRFS_I(inode)->lock); 1687 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1688 spin_unlock(&BTRFS_I(inode)->lock); 1689 return; 1690 } 1691 1692 /* 1693 * We have to add up either side to figure out how many extents were 1694 * accounted for before we merged into one big extent. If the number of 1695 * extents we accounted for is <= the amount we need for the new range 1696 * then we can return, otherwise drop. Think of it like this 1697 * 1698 * [ 4k][MAX_SIZE] 1699 * 1700 * So we've grown the extent by a MAX_SIZE extent, this would mean we 1701 * need 2 outstanding extents, on one side we have 1 and the other side 1702 * we have 1 so they are == and we can return. But in this case 1703 * 1704 * [MAX_SIZE+4k][MAX_SIZE+4k] 1705 * 1706 * Each range on their own accounts for 2 extents, but merged together 1707 * they are only 3 extents worth of accounting, so we need to drop in 1708 * this case. 1709 */ 1710 old_size = other->end - other->start + 1; 1711 num_extents = count_max_extents(old_size); 1712 old_size = new->end - new->start + 1; 1713 num_extents += count_max_extents(old_size); 1714 if (count_max_extents(new_size) >= num_extents) 1715 return; 1716 1717 spin_lock(&BTRFS_I(inode)->lock); 1718 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1719 spin_unlock(&BTRFS_I(inode)->lock); 1720 } 1721 1722 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 1723 struct inode *inode) 1724 { 1725 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1726 1727 spin_lock(&root->delalloc_lock); 1728 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1729 list_add_tail(&BTRFS_I(inode)->delalloc_inodes, 1730 &root->delalloc_inodes); 1731 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1732 &BTRFS_I(inode)->runtime_flags); 1733 root->nr_delalloc_inodes++; 1734 if (root->nr_delalloc_inodes == 1) { 1735 spin_lock(&fs_info->delalloc_root_lock); 1736 BUG_ON(!list_empty(&root->delalloc_root)); 1737 list_add_tail(&root->delalloc_root, 1738 &fs_info->delalloc_roots); 1739 spin_unlock(&fs_info->delalloc_root_lock); 1740 } 1741 } 1742 spin_unlock(&root->delalloc_lock); 1743 } 1744 1745 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 1746 struct btrfs_inode *inode) 1747 { 1748 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 1749 1750 spin_lock(&root->delalloc_lock); 1751 if (!list_empty(&inode->delalloc_inodes)) { 1752 list_del_init(&inode->delalloc_inodes); 1753 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1754 &inode->runtime_flags); 1755 root->nr_delalloc_inodes--; 1756 if (!root->nr_delalloc_inodes) { 1757 spin_lock(&fs_info->delalloc_root_lock); 1758 BUG_ON(list_empty(&root->delalloc_root)); 1759 list_del_init(&root->delalloc_root); 1760 spin_unlock(&fs_info->delalloc_root_lock); 1761 } 1762 } 1763 spin_unlock(&root->delalloc_lock); 1764 } 1765 1766 /* 1767 * extent_io.c set_bit_hook, used to track delayed allocation 1768 * bytes in this file, and to maintain the list of inodes that 1769 * have pending delalloc work to be done. 1770 */ 1771 static void btrfs_set_bit_hook(void *private_data, 1772 struct extent_state *state, unsigned *bits) 1773 { 1774 struct inode *inode = private_data; 1775 1776 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1777 1778 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC)) 1779 WARN_ON(1); 1780 /* 1781 * set_bit and clear bit hooks normally require _irqsave/restore 1782 * but in this case, we are only testing for the DELALLOC 1783 * bit, which is only set or cleared with irqs on 1784 */ 1785 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1786 struct btrfs_root *root = BTRFS_I(inode)->root; 1787 u64 len = state->end + 1 - state->start; 1788 u32 num_extents = count_max_extents(len); 1789 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode)); 1790 1791 spin_lock(&BTRFS_I(inode)->lock); 1792 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents); 1793 spin_unlock(&BTRFS_I(inode)->lock); 1794 1795 /* For sanity tests */ 1796 if (btrfs_is_testing(fs_info)) 1797 return; 1798 1799 percpu_counter_add_batch(&fs_info->delalloc_bytes, len, 1800 fs_info->delalloc_batch); 1801 spin_lock(&BTRFS_I(inode)->lock); 1802 BTRFS_I(inode)->delalloc_bytes += len; 1803 if (*bits & EXTENT_DEFRAG) 1804 BTRFS_I(inode)->defrag_bytes += len; 1805 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1806 &BTRFS_I(inode)->runtime_flags)) 1807 btrfs_add_delalloc_inodes(root, inode); 1808 spin_unlock(&BTRFS_I(inode)->lock); 1809 } 1810 1811 if (!(state->state & EXTENT_DELALLOC_NEW) && 1812 (*bits & EXTENT_DELALLOC_NEW)) { 1813 spin_lock(&BTRFS_I(inode)->lock); 1814 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 - 1815 state->start; 1816 spin_unlock(&BTRFS_I(inode)->lock); 1817 } 1818 } 1819 1820 /* 1821 * extent_io.c clear_bit_hook, see set_bit_hook for why 1822 */ 1823 static void btrfs_clear_bit_hook(void *private_data, 1824 struct extent_state *state, 1825 unsigned *bits) 1826 { 1827 struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data); 1828 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 1829 u64 len = state->end + 1 - state->start; 1830 u32 num_extents = count_max_extents(len); 1831 1832 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) { 1833 spin_lock(&inode->lock); 1834 inode->defrag_bytes -= len; 1835 spin_unlock(&inode->lock); 1836 } 1837 1838 /* 1839 * set_bit and clear bit hooks normally require _irqsave/restore 1840 * but in this case, we are only testing for the DELALLOC 1841 * bit, which is only set or cleared with irqs on 1842 */ 1843 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1844 struct btrfs_root *root = inode->root; 1845 bool do_list = !btrfs_is_free_space_inode(inode); 1846 1847 spin_lock(&inode->lock); 1848 btrfs_mod_outstanding_extents(inode, -num_extents); 1849 spin_unlock(&inode->lock); 1850 1851 /* 1852 * We don't reserve metadata space for space cache inodes so we 1853 * don't need to call dellalloc_release_metadata if there is an 1854 * error. 1855 */ 1856 if (*bits & EXTENT_CLEAR_META_RESV && 1857 root != fs_info->tree_root) 1858 btrfs_delalloc_release_metadata(inode, len, false); 1859 1860 /* For sanity tests. */ 1861 if (btrfs_is_testing(fs_info)) 1862 return; 1863 1864 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID && 1865 do_list && !(state->state & EXTENT_NORESERVE) && 1866 (*bits & EXTENT_CLEAR_DATA_RESV)) 1867 btrfs_free_reserved_data_space_noquota( 1868 &inode->vfs_inode, 1869 state->start, len); 1870 1871 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, 1872 fs_info->delalloc_batch); 1873 spin_lock(&inode->lock); 1874 inode->delalloc_bytes -= len; 1875 if (do_list && inode->delalloc_bytes == 0 && 1876 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1877 &inode->runtime_flags)) 1878 btrfs_del_delalloc_inode(root, inode); 1879 spin_unlock(&inode->lock); 1880 } 1881 1882 if ((state->state & EXTENT_DELALLOC_NEW) && 1883 (*bits & EXTENT_DELALLOC_NEW)) { 1884 spin_lock(&inode->lock); 1885 ASSERT(inode->new_delalloc_bytes >= len); 1886 inode->new_delalloc_bytes -= len; 1887 spin_unlock(&inode->lock); 1888 } 1889 } 1890 1891 /* 1892 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure 1893 * we don't create bios that span stripes or chunks 1894 * 1895 * return 1 if page cannot be merged to bio 1896 * return 0 if page can be merged to bio 1897 * return error otherwise 1898 */ 1899 int btrfs_merge_bio_hook(struct page *page, unsigned long offset, 1900 size_t size, struct bio *bio, 1901 unsigned long bio_flags) 1902 { 1903 struct inode *inode = page->mapping->host; 1904 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1905 u64 logical = (u64)bio->bi_iter.bi_sector << 9; 1906 u64 length = 0; 1907 u64 map_length; 1908 int ret; 1909 1910 if (bio_flags & EXTENT_BIO_COMPRESSED) 1911 return 0; 1912 1913 length = bio->bi_iter.bi_size; 1914 map_length = length; 1915 ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, 1916 NULL, 0); 1917 if (ret < 0) 1918 return ret; 1919 if (map_length < length + size) 1920 return 1; 1921 return 0; 1922 } 1923 1924 /* 1925 * in order to insert checksums into the metadata in large chunks, 1926 * we wait until bio submission time. All the pages in the bio are 1927 * checksummed and sums are attached onto the ordered extent record. 1928 * 1929 * At IO completion time the cums attached on the ordered extent record 1930 * are inserted into the btree 1931 */ 1932 static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio, 1933 u64 bio_offset) 1934 { 1935 struct inode *inode = private_data; 1936 blk_status_t ret = 0; 1937 1938 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 1939 BUG_ON(ret); /* -ENOMEM */ 1940 return 0; 1941 } 1942 1943 /* 1944 * in order to insert checksums into the metadata in large chunks, 1945 * we wait until bio submission time. All the pages in the bio are 1946 * checksummed and sums are attached onto the ordered extent record. 1947 * 1948 * At IO completion time the cums attached on the ordered extent record 1949 * are inserted into the btree 1950 */ 1951 static blk_status_t btrfs_submit_bio_done(void *private_data, struct bio *bio, 1952 int mirror_num) 1953 { 1954 struct inode *inode = private_data; 1955 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1956 blk_status_t ret; 1957 1958 ret = btrfs_map_bio(fs_info, bio, mirror_num, 1); 1959 if (ret) { 1960 bio->bi_status = ret; 1961 bio_endio(bio); 1962 } 1963 return ret; 1964 } 1965 1966 /* 1967 * extent_io.c submission hook. This does the right thing for csum calculation 1968 * on write, or reading the csums from the tree before a read. 1969 * 1970 * Rules about async/sync submit, 1971 * a) read: sync submit 1972 * 1973 * b) write without checksum: sync submit 1974 * 1975 * c) write with checksum: 1976 * c-1) if bio is issued by fsync: sync submit 1977 * (sync_writers != 0) 1978 * 1979 * c-2) if root is reloc root: sync submit 1980 * (only in case of buffered IO) 1981 * 1982 * c-3) otherwise: async submit 1983 */ 1984 static blk_status_t btrfs_submit_bio_hook(void *private_data, struct bio *bio, 1985 int mirror_num, unsigned long bio_flags, 1986 u64 bio_offset) 1987 { 1988 struct inode *inode = private_data; 1989 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1990 struct btrfs_root *root = BTRFS_I(inode)->root; 1991 enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA; 1992 blk_status_t ret = 0; 1993 int skip_sum; 1994 int async = !atomic_read(&BTRFS_I(inode)->sync_writers); 1995 1996 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 1997 1998 if (btrfs_is_free_space_inode(BTRFS_I(inode))) 1999 metadata = BTRFS_WQ_ENDIO_FREE_SPACE; 2000 2001 if (bio_op(bio) != REQ_OP_WRITE) { 2002 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); 2003 if (ret) 2004 goto out; 2005 2006 if (bio_flags & EXTENT_BIO_COMPRESSED) { 2007 ret = btrfs_submit_compressed_read(inode, bio, 2008 mirror_num, 2009 bio_flags); 2010 goto out; 2011 } else if (!skip_sum) { 2012 ret = btrfs_lookup_bio_sums(inode, bio, NULL); 2013 if (ret) 2014 goto out; 2015 } 2016 goto mapit; 2017 } else if (async && !skip_sum) { 2018 /* csum items have already been cloned */ 2019 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 2020 goto mapit; 2021 /* we're doing a write, do the async checksumming */ 2022 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags, 2023 bio_offset, inode, 2024 btrfs_submit_bio_start, 2025 btrfs_submit_bio_done); 2026 goto out; 2027 } else if (!skip_sum) { 2028 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 2029 if (ret) 2030 goto out; 2031 } 2032 2033 mapit: 2034 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 2035 2036 out: 2037 if (ret) { 2038 bio->bi_status = ret; 2039 bio_endio(bio); 2040 } 2041 return ret; 2042 } 2043 2044 /* 2045 * given a list of ordered sums record them in the inode. This happens 2046 * at IO completion time based on sums calculated at bio submission time. 2047 */ 2048 static noinline int add_pending_csums(struct btrfs_trans_handle *trans, 2049 struct inode *inode, struct list_head *list) 2050 { 2051 struct btrfs_ordered_sum *sum; 2052 int ret; 2053 2054 list_for_each_entry(sum, list, list) { 2055 trans->adding_csums = true; 2056 ret = btrfs_csum_file_blocks(trans, 2057 BTRFS_I(inode)->root->fs_info->csum_root, sum); 2058 trans->adding_csums = false; 2059 if (ret) 2060 return ret; 2061 } 2062 return 0; 2063 } 2064 2065 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end, 2066 unsigned int extra_bits, 2067 struct extent_state **cached_state, int dedupe) 2068 { 2069 WARN_ON((end & (PAGE_SIZE - 1)) == 0); 2070 return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end, 2071 extra_bits, cached_state); 2072 } 2073 2074 /* see btrfs_writepage_start_hook for details on why this is required */ 2075 struct btrfs_writepage_fixup { 2076 struct page *page; 2077 struct btrfs_work work; 2078 }; 2079 2080 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2081 { 2082 struct btrfs_writepage_fixup *fixup; 2083 struct btrfs_ordered_extent *ordered; 2084 struct extent_state *cached_state = NULL; 2085 struct extent_changeset *data_reserved = NULL; 2086 struct page *page; 2087 struct inode *inode; 2088 u64 page_start; 2089 u64 page_end; 2090 int ret; 2091 2092 fixup = container_of(work, struct btrfs_writepage_fixup, work); 2093 page = fixup->page; 2094 again: 2095 lock_page(page); 2096 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 2097 ClearPageChecked(page); 2098 goto out_page; 2099 } 2100 2101 inode = page->mapping->host; 2102 page_start = page_offset(page); 2103 page_end = page_offset(page) + PAGE_SIZE - 1; 2104 2105 lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end, 2106 &cached_state); 2107 2108 /* already ordered? We're done */ 2109 if (PagePrivate2(page)) 2110 goto out; 2111 2112 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 2113 PAGE_SIZE); 2114 if (ordered) { 2115 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, 2116 page_end, &cached_state); 2117 unlock_page(page); 2118 btrfs_start_ordered_extent(inode, ordered, 1); 2119 btrfs_put_ordered_extent(ordered); 2120 goto again; 2121 } 2122 2123 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 2124 PAGE_SIZE); 2125 if (ret) { 2126 mapping_set_error(page->mapping, ret); 2127 end_extent_writepage(page, ret, page_start, page_end); 2128 ClearPageChecked(page); 2129 goto out; 2130 } 2131 2132 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, 2133 &cached_state, 0); 2134 if (ret) { 2135 mapping_set_error(page->mapping, ret); 2136 end_extent_writepage(page, ret, page_start, page_end); 2137 ClearPageChecked(page); 2138 goto out; 2139 } 2140 2141 ClearPageChecked(page); 2142 set_page_dirty(page); 2143 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, false); 2144 out: 2145 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end, 2146 &cached_state); 2147 out_page: 2148 unlock_page(page); 2149 put_page(page); 2150 kfree(fixup); 2151 extent_changeset_free(data_reserved); 2152 } 2153 2154 /* 2155 * There are a few paths in the higher layers of the kernel that directly 2156 * set the page dirty bit without asking the filesystem if it is a 2157 * good idea. This causes problems because we want to make sure COW 2158 * properly happens and the data=ordered rules are followed. 2159 * 2160 * In our case any range that doesn't have the ORDERED bit set 2161 * hasn't been properly setup for IO. We kick off an async process 2162 * to fix it up. The async helper will wait for ordered extents, set 2163 * the delalloc bit and make it safe to write the page. 2164 */ 2165 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end) 2166 { 2167 struct inode *inode = page->mapping->host; 2168 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2169 struct btrfs_writepage_fixup *fixup; 2170 2171 /* this page is properly in the ordered list */ 2172 if (TestClearPagePrivate2(page)) 2173 return 0; 2174 2175 if (PageChecked(page)) 2176 return -EAGAIN; 2177 2178 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2179 if (!fixup) 2180 return -EAGAIN; 2181 2182 SetPageChecked(page); 2183 get_page(page); 2184 btrfs_init_work(&fixup->work, btrfs_fixup_helper, 2185 btrfs_writepage_fixup_worker, NULL, NULL); 2186 fixup->page = page; 2187 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2188 return -EBUSY; 2189 } 2190 2191 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2192 struct inode *inode, u64 file_pos, 2193 u64 disk_bytenr, u64 disk_num_bytes, 2194 u64 num_bytes, u64 ram_bytes, 2195 u8 compression, u8 encryption, 2196 u16 other_encoding, int extent_type) 2197 { 2198 struct btrfs_root *root = BTRFS_I(inode)->root; 2199 struct btrfs_file_extent_item *fi; 2200 struct btrfs_path *path; 2201 struct extent_buffer *leaf; 2202 struct btrfs_key ins; 2203 u64 qg_released; 2204 int extent_inserted = 0; 2205 int ret; 2206 2207 path = btrfs_alloc_path(); 2208 if (!path) 2209 return -ENOMEM; 2210 2211 /* 2212 * we may be replacing one extent in the tree with another. 2213 * The new extent is pinned in the extent map, and we don't want 2214 * to drop it from the cache until it is completely in the btree. 2215 * 2216 * So, tell btrfs_drop_extents to leave this extent in the cache. 2217 * the caller is expected to unpin it and allow it to be merged 2218 * with the others. 2219 */ 2220 ret = __btrfs_drop_extents(trans, root, inode, path, file_pos, 2221 file_pos + num_bytes, NULL, 0, 2222 1, sizeof(*fi), &extent_inserted); 2223 if (ret) 2224 goto out; 2225 2226 if (!extent_inserted) { 2227 ins.objectid = btrfs_ino(BTRFS_I(inode)); 2228 ins.offset = file_pos; 2229 ins.type = BTRFS_EXTENT_DATA_KEY; 2230 2231 path->leave_spinning = 1; 2232 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2233 sizeof(*fi)); 2234 if (ret) 2235 goto out; 2236 } 2237 leaf = path->nodes[0]; 2238 fi = btrfs_item_ptr(leaf, path->slots[0], 2239 struct btrfs_file_extent_item); 2240 btrfs_set_file_extent_generation(leaf, fi, trans->transid); 2241 btrfs_set_file_extent_type(leaf, fi, extent_type); 2242 btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr); 2243 btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes); 2244 btrfs_set_file_extent_offset(leaf, fi, 0); 2245 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); 2246 btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes); 2247 btrfs_set_file_extent_compression(leaf, fi, compression); 2248 btrfs_set_file_extent_encryption(leaf, fi, encryption); 2249 btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding); 2250 2251 btrfs_mark_buffer_dirty(leaf); 2252 btrfs_release_path(path); 2253 2254 inode_add_bytes(inode, num_bytes); 2255 2256 ins.objectid = disk_bytenr; 2257 ins.offset = disk_num_bytes; 2258 ins.type = BTRFS_EXTENT_ITEM_KEY; 2259 2260 /* 2261 * Release the reserved range from inode dirty range map, as it is 2262 * already moved into delayed_ref_head 2263 */ 2264 ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes); 2265 if (ret < 0) 2266 goto out; 2267 qg_released = ret; 2268 ret = btrfs_alloc_reserved_file_extent(trans, root, 2269 btrfs_ino(BTRFS_I(inode)), 2270 file_pos, qg_released, &ins); 2271 out: 2272 btrfs_free_path(path); 2273 2274 return ret; 2275 } 2276 2277 /* snapshot-aware defrag */ 2278 struct sa_defrag_extent_backref { 2279 struct rb_node node; 2280 struct old_sa_defrag_extent *old; 2281 u64 root_id; 2282 u64 inum; 2283 u64 file_pos; 2284 u64 extent_offset; 2285 u64 num_bytes; 2286 u64 generation; 2287 }; 2288 2289 struct old_sa_defrag_extent { 2290 struct list_head list; 2291 struct new_sa_defrag_extent *new; 2292 2293 u64 extent_offset; 2294 u64 bytenr; 2295 u64 offset; 2296 u64 len; 2297 int count; 2298 }; 2299 2300 struct new_sa_defrag_extent { 2301 struct rb_root root; 2302 struct list_head head; 2303 struct btrfs_path *path; 2304 struct inode *inode; 2305 u64 file_pos; 2306 u64 len; 2307 u64 bytenr; 2308 u64 disk_len; 2309 u8 compress_type; 2310 }; 2311 2312 static int backref_comp(struct sa_defrag_extent_backref *b1, 2313 struct sa_defrag_extent_backref *b2) 2314 { 2315 if (b1->root_id < b2->root_id) 2316 return -1; 2317 else if (b1->root_id > b2->root_id) 2318 return 1; 2319 2320 if (b1->inum < b2->inum) 2321 return -1; 2322 else if (b1->inum > b2->inum) 2323 return 1; 2324 2325 if (b1->file_pos < b2->file_pos) 2326 return -1; 2327 else if (b1->file_pos > b2->file_pos) 2328 return 1; 2329 2330 /* 2331 * [------------------------------] ===> (a range of space) 2332 * |<--->| |<---->| =============> (fs/file tree A) 2333 * |<---------------------------->| ===> (fs/file tree B) 2334 * 2335 * A range of space can refer to two file extents in one tree while 2336 * refer to only one file extent in another tree. 2337 * 2338 * So we may process a disk offset more than one time(two extents in A) 2339 * and locate at the same extent(one extent in B), then insert two same 2340 * backrefs(both refer to the extent in B). 2341 */ 2342 return 0; 2343 } 2344 2345 static void backref_insert(struct rb_root *root, 2346 struct sa_defrag_extent_backref *backref) 2347 { 2348 struct rb_node **p = &root->rb_node; 2349 struct rb_node *parent = NULL; 2350 struct sa_defrag_extent_backref *entry; 2351 int ret; 2352 2353 while (*p) { 2354 parent = *p; 2355 entry = rb_entry(parent, struct sa_defrag_extent_backref, node); 2356 2357 ret = backref_comp(backref, entry); 2358 if (ret < 0) 2359 p = &(*p)->rb_left; 2360 else 2361 p = &(*p)->rb_right; 2362 } 2363 2364 rb_link_node(&backref->node, parent, p); 2365 rb_insert_color(&backref->node, root); 2366 } 2367 2368 /* 2369 * Note the backref might has changed, and in this case we just return 0. 2370 */ 2371 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id, 2372 void *ctx) 2373 { 2374 struct btrfs_file_extent_item *extent; 2375 struct old_sa_defrag_extent *old = ctx; 2376 struct new_sa_defrag_extent *new = old->new; 2377 struct btrfs_path *path = new->path; 2378 struct btrfs_key key; 2379 struct btrfs_root *root; 2380 struct sa_defrag_extent_backref *backref; 2381 struct extent_buffer *leaf; 2382 struct inode *inode = new->inode; 2383 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2384 int slot; 2385 int ret; 2386 u64 extent_offset; 2387 u64 num_bytes; 2388 2389 if (BTRFS_I(inode)->root->root_key.objectid == root_id && 2390 inum == btrfs_ino(BTRFS_I(inode))) 2391 return 0; 2392 2393 key.objectid = root_id; 2394 key.type = BTRFS_ROOT_ITEM_KEY; 2395 key.offset = (u64)-1; 2396 2397 root = btrfs_read_fs_root_no_name(fs_info, &key); 2398 if (IS_ERR(root)) { 2399 if (PTR_ERR(root) == -ENOENT) 2400 return 0; 2401 WARN_ON(1); 2402 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu", 2403 inum, offset, root_id); 2404 return PTR_ERR(root); 2405 } 2406 2407 key.objectid = inum; 2408 key.type = BTRFS_EXTENT_DATA_KEY; 2409 if (offset > (u64)-1 << 32) 2410 key.offset = 0; 2411 else 2412 key.offset = offset; 2413 2414 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2415 if (WARN_ON(ret < 0)) 2416 return ret; 2417 ret = 0; 2418 2419 while (1) { 2420 cond_resched(); 2421 2422 leaf = path->nodes[0]; 2423 slot = path->slots[0]; 2424 2425 if (slot >= btrfs_header_nritems(leaf)) { 2426 ret = btrfs_next_leaf(root, path); 2427 if (ret < 0) { 2428 goto out; 2429 } else if (ret > 0) { 2430 ret = 0; 2431 goto out; 2432 } 2433 continue; 2434 } 2435 2436 path->slots[0]++; 2437 2438 btrfs_item_key_to_cpu(leaf, &key, slot); 2439 2440 if (key.objectid > inum) 2441 goto out; 2442 2443 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY) 2444 continue; 2445 2446 extent = btrfs_item_ptr(leaf, slot, 2447 struct btrfs_file_extent_item); 2448 2449 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr) 2450 continue; 2451 2452 /* 2453 * 'offset' refers to the exact key.offset, 2454 * NOT the 'offset' field in btrfs_extent_data_ref, ie. 2455 * (key.offset - extent_offset). 2456 */ 2457 if (key.offset != offset) 2458 continue; 2459 2460 extent_offset = btrfs_file_extent_offset(leaf, extent); 2461 num_bytes = btrfs_file_extent_num_bytes(leaf, extent); 2462 2463 if (extent_offset >= old->extent_offset + old->offset + 2464 old->len || extent_offset + num_bytes <= 2465 old->extent_offset + old->offset) 2466 continue; 2467 break; 2468 } 2469 2470 backref = kmalloc(sizeof(*backref), GFP_NOFS); 2471 if (!backref) { 2472 ret = -ENOENT; 2473 goto out; 2474 } 2475 2476 backref->root_id = root_id; 2477 backref->inum = inum; 2478 backref->file_pos = offset; 2479 backref->num_bytes = num_bytes; 2480 backref->extent_offset = extent_offset; 2481 backref->generation = btrfs_file_extent_generation(leaf, extent); 2482 backref->old = old; 2483 backref_insert(&new->root, backref); 2484 old->count++; 2485 out: 2486 btrfs_release_path(path); 2487 WARN_ON(ret); 2488 return ret; 2489 } 2490 2491 static noinline bool record_extent_backrefs(struct btrfs_path *path, 2492 struct new_sa_defrag_extent *new) 2493 { 2494 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2495 struct old_sa_defrag_extent *old, *tmp; 2496 int ret; 2497 2498 new->path = path; 2499 2500 list_for_each_entry_safe(old, tmp, &new->head, list) { 2501 ret = iterate_inodes_from_logical(old->bytenr + 2502 old->extent_offset, fs_info, 2503 path, record_one_backref, 2504 old, false); 2505 if (ret < 0 && ret != -ENOENT) 2506 return false; 2507 2508 /* no backref to be processed for this extent */ 2509 if (!old->count) { 2510 list_del(&old->list); 2511 kfree(old); 2512 } 2513 } 2514 2515 if (list_empty(&new->head)) 2516 return false; 2517 2518 return true; 2519 } 2520 2521 static int relink_is_mergable(struct extent_buffer *leaf, 2522 struct btrfs_file_extent_item *fi, 2523 struct new_sa_defrag_extent *new) 2524 { 2525 if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr) 2526 return 0; 2527 2528 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG) 2529 return 0; 2530 2531 if (btrfs_file_extent_compression(leaf, fi) != new->compress_type) 2532 return 0; 2533 2534 if (btrfs_file_extent_encryption(leaf, fi) || 2535 btrfs_file_extent_other_encoding(leaf, fi)) 2536 return 0; 2537 2538 return 1; 2539 } 2540 2541 /* 2542 * Note the backref might has changed, and in this case we just return 0. 2543 */ 2544 static noinline int relink_extent_backref(struct btrfs_path *path, 2545 struct sa_defrag_extent_backref *prev, 2546 struct sa_defrag_extent_backref *backref) 2547 { 2548 struct btrfs_file_extent_item *extent; 2549 struct btrfs_file_extent_item *item; 2550 struct btrfs_ordered_extent *ordered; 2551 struct btrfs_trans_handle *trans; 2552 struct btrfs_root *root; 2553 struct btrfs_key key; 2554 struct extent_buffer *leaf; 2555 struct old_sa_defrag_extent *old = backref->old; 2556 struct new_sa_defrag_extent *new = old->new; 2557 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2558 struct inode *inode; 2559 struct extent_state *cached = NULL; 2560 int ret = 0; 2561 u64 start; 2562 u64 len; 2563 u64 lock_start; 2564 u64 lock_end; 2565 bool merge = false; 2566 int index; 2567 2568 if (prev && prev->root_id == backref->root_id && 2569 prev->inum == backref->inum && 2570 prev->file_pos + prev->num_bytes == backref->file_pos) 2571 merge = true; 2572 2573 /* step 1: get root */ 2574 key.objectid = backref->root_id; 2575 key.type = BTRFS_ROOT_ITEM_KEY; 2576 key.offset = (u64)-1; 2577 2578 index = srcu_read_lock(&fs_info->subvol_srcu); 2579 2580 root = btrfs_read_fs_root_no_name(fs_info, &key); 2581 if (IS_ERR(root)) { 2582 srcu_read_unlock(&fs_info->subvol_srcu, index); 2583 if (PTR_ERR(root) == -ENOENT) 2584 return 0; 2585 return PTR_ERR(root); 2586 } 2587 2588 if (btrfs_root_readonly(root)) { 2589 srcu_read_unlock(&fs_info->subvol_srcu, index); 2590 return 0; 2591 } 2592 2593 /* step 2: get inode */ 2594 key.objectid = backref->inum; 2595 key.type = BTRFS_INODE_ITEM_KEY; 2596 key.offset = 0; 2597 2598 inode = btrfs_iget(fs_info->sb, &key, root, NULL); 2599 if (IS_ERR(inode)) { 2600 srcu_read_unlock(&fs_info->subvol_srcu, index); 2601 return 0; 2602 } 2603 2604 srcu_read_unlock(&fs_info->subvol_srcu, index); 2605 2606 /* step 3: relink backref */ 2607 lock_start = backref->file_pos; 2608 lock_end = backref->file_pos + backref->num_bytes - 1; 2609 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2610 &cached); 2611 2612 ordered = btrfs_lookup_first_ordered_extent(inode, lock_end); 2613 if (ordered) { 2614 btrfs_put_ordered_extent(ordered); 2615 goto out_unlock; 2616 } 2617 2618 trans = btrfs_join_transaction(root); 2619 if (IS_ERR(trans)) { 2620 ret = PTR_ERR(trans); 2621 goto out_unlock; 2622 } 2623 2624 key.objectid = backref->inum; 2625 key.type = BTRFS_EXTENT_DATA_KEY; 2626 key.offset = backref->file_pos; 2627 2628 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2629 if (ret < 0) { 2630 goto out_free_path; 2631 } else if (ret > 0) { 2632 ret = 0; 2633 goto out_free_path; 2634 } 2635 2636 extent = btrfs_item_ptr(path->nodes[0], path->slots[0], 2637 struct btrfs_file_extent_item); 2638 2639 if (btrfs_file_extent_generation(path->nodes[0], extent) != 2640 backref->generation) 2641 goto out_free_path; 2642 2643 btrfs_release_path(path); 2644 2645 start = backref->file_pos; 2646 if (backref->extent_offset < old->extent_offset + old->offset) 2647 start += old->extent_offset + old->offset - 2648 backref->extent_offset; 2649 2650 len = min(backref->extent_offset + backref->num_bytes, 2651 old->extent_offset + old->offset + old->len); 2652 len -= max(backref->extent_offset, old->extent_offset + old->offset); 2653 2654 ret = btrfs_drop_extents(trans, root, inode, start, 2655 start + len, 1); 2656 if (ret) 2657 goto out_free_path; 2658 again: 2659 key.objectid = btrfs_ino(BTRFS_I(inode)); 2660 key.type = BTRFS_EXTENT_DATA_KEY; 2661 key.offset = start; 2662 2663 path->leave_spinning = 1; 2664 if (merge) { 2665 struct btrfs_file_extent_item *fi; 2666 u64 extent_len; 2667 struct btrfs_key found_key; 2668 2669 ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 2670 if (ret < 0) 2671 goto out_free_path; 2672 2673 path->slots[0]--; 2674 leaf = path->nodes[0]; 2675 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 2676 2677 fi = btrfs_item_ptr(leaf, path->slots[0], 2678 struct btrfs_file_extent_item); 2679 extent_len = btrfs_file_extent_num_bytes(leaf, fi); 2680 2681 if (extent_len + found_key.offset == start && 2682 relink_is_mergable(leaf, fi, new)) { 2683 btrfs_set_file_extent_num_bytes(leaf, fi, 2684 extent_len + len); 2685 btrfs_mark_buffer_dirty(leaf); 2686 inode_add_bytes(inode, len); 2687 2688 ret = 1; 2689 goto out_free_path; 2690 } else { 2691 merge = false; 2692 btrfs_release_path(path); 2693 goto again; 2694 } 2695 } 2696 2697 ret = btrfs_insert_empty_item(trans, root, path, &key, 2698 sizeof(*extent)); 2699 if (ret) { 2700 btrfs_abort_transaction(trans, ret); 2701 goto out_free_path; 2702 } 2703 2704 leaf = path->nodes[0]; 2705 item = btrfs_item_ptr(leaf, path->slots[0], 2706 struct btrfs_file_extent_item); 2707 btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr); 2708 btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len); 2709 btrfs_set_file_extent_offset(leaf, item, start - new->file_pos); 2710 btrfs_set_file_extent_num_bytes(leaf, item, len); 2711 btrfs_set_file_extent_ram_bytes(leaf, item, new->len); 2712 btrfs_set_file_extent_generation(leaf, item, trans->transid); 2713 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); 2714 btrfs_set_file_extent_compression(leaf, item, new->compress_type); 2715 btrfs_set_file_extent_encryption(leaf, item, 0); 2716 btrfs_set_file_extent_other_encoding(leaf, item, 0); 2717 2718 btrfs_mark_buffer_dirty(leaf); 2719 inode_add_bytes(inode, len); 2720 btrfs_release_path(path); 2721 2722 ret = btrfs_inc_extent_ref(trans, root, new->bytenr, 2723 new->disk_len, 0, 2724 backref->root_id, backref->inum, 2725 new->file_pos); /* start - extent_offset */ 2726 if (ret) { 2727 btrfs_abort_transaction(trans, ret); 2728 goto out_free_path; 2729 } 2730 2731 ret = 1; 2732 out_free_path: 2733 btrfs_release_path(path); 2734 path->leave_spinning = 0; 2735 btrfs_end_transaction(trans); 2736 out_unlock: 2737 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2738 &cached); 2739 iput(inode); 2740 return ret; 2741 } 2742 2743 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new) 2744 { 2745 struct old_sa_defrag_extent *old, *tmp; 2746 2747 if (!new) 2748 return; 2749 2750 list_for_each_entry_safe(old, tmp, &new->head, list) { 2751 kfree(old); 2752 } 2753 kfree(new); 2754 } 2755 2756 static void relink_file_extents(struct new_sa_defrag_extent *new) 2757 { 2758 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2759 struct btrfs_path *path; 2760 struct sa_defrag_extent_backref *backref; 2761 struct sa_defrag_extent_backref *prev = NULL; 2762 struct inode *inode; 2763 struct rb_node *node; 2764 int ret; 2765 2766 inode = new->inode; 2767 2768 path = btrfs_alloc_path(); 2769 if (!path) 2770 return; 2771 2772 if (!record_extent_backrefs(path, new)) { 2773 btrfs_free_path(path); 2774 goto out; 2775 } 2776 btrfs_release_path(path); 2777 2778 while (1) { 2779 node = rb_first(&new->root); 2780 if (!node) 2781 break; 2782 rb_erase(node, &new->root); 2783 2784 backref = rb_entry(node, struct sa_defrag_extent_backref, node); 2785 2786 ret = relink_extent_backref(path, prev, backref); 2787 WARN_ON(ret < 0); 2788 2789 kfree(prev); 2790 2791 if (ret == 1) 2792 prev = backref; 2793 else 2794 prev = NULL; 2795 cond_resched(); 2796 } 2797 kfree(prev); 2798 2799 btrfs_free_path(path); 2800 out: 2801 free_sa_defrag_extent(new); 2802 2803 atomic_dec(&fs_info->defrag_running); 2804 wake_up(&fs_info->transaction_wait); 2805 } 2806 2807 static struct new_sa_defrag_extent * 2808 record_old_file_extents(struct inode *inode, 2809 struct btrfs_ordered_extent *ordered) 2810 { 2811 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2812 struct btrfs_root *root = BTRFS_I(inode)->root; 2813 struct btrfs_path *path; 2814 struct btrfs_key key; 2815 struct old_sa_defrag_extent *old; 2816 struct new_sa_defrag_extent *new; 2817 int ret; 2818 2819 new = kmalloc(sizeof(*new), GFP_NOFS); 2820 if (!new) 2821 return NULL; 2822 2823 new->inode = inode; 2824 new->file_pos = ordered->file_offset; 2825 new->len = ordered->len; 2826 new->bytenr = ordered->start; 2827 new->disk_len = ordered->disk_len; 2828 new->compress_type = ordered->compress_type; 2829 new->root = RB_ROOT; 2830 INIT_LIST_HEAD(&new->head); 2831 2832 path = btrfs_alloc_path(); 2833 if (!path) 2834 goto out_kfree; 2835 2836 key.objectid = btrfs_ino(BTRFS_I(inode)); 2837 key.type = BTRFS_EXTENT_DATA_KEY; 2838 key.offset = new->file_pos; 2839 2840 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2841 if (ret < 0) 2842 goto out_free_path; 2843 if (ret > 0 && path->slots[0] > 0) 2844 path->slots[0]--; 2845 2846 /* find out all the old extents for the file range */ 2847 while (1) { 2848 struct btrfs_file_extent_item *extent; 2849 struct extent_buffer *l; 2850 int slot; 2851 u64 num_bytes; 2852 u64 offset; 2853 u64 end; 2854 u64 disk_bytenr; 2855 u64 extent_offset; 2856 2857 l = path->nodes[0]; 2858 slot = path->slots[0]; 2859 2860 if (slot >= btrfs_header_nritems(l)) { 2861 ret = btrfs_next_leaf(root, path); 2862 if (ret < 0) 2863 goto out_free_path; 2864 else if (ret > 0) 2865 break; 2866 continue; 2867 } 2868 2869 btrfs_item_key_to_cpu(l, &key, slot); 2870 2871 if (key.objectid != btrfs_ino(BTRFS_I(inode))) 2872 break; 2873 if (key.type != BTRFS_EXTENT_DATA_KEY) 2874 break; 2875 if (key.offset >= new->file_pos + new->len) 2876 break; 2877 2878 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item); 2879 2880 num_bytes = btrfs_file_extent_num_bytes(l, extent); 2881 if (key.offset + num_bytes < new->file_pos) 2882 goto next; 2883 2884 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent); 2885 if (!disk_bytenr) 2886 goto next; 2887 2888 extent_offset = btrfs_file_extent_offset(l, extent); 2889 2890 old = kmalloc(sizeof(*old), GFP_NOFS); 2891 if (!old) 2892 goto out_free_path; 2893 2894 offset = max(new->file_pos, key.offset); 2895 end = min(new->file_pos + new->len, key.offset + num_bytes); 2896 2897 old->bytenr = disk_bytenr; 2898 old->extent_offset = extent_offset; 2899 old->offset = offset - key.offset; 2900 old->len = end - offset; 2901 old->new = new; 2902 old->count = 0; 2903 list_add_tail(&old->list, &new->head); 2904 next: 2905 path->slots[0]++; 2906 cond_resched(); 2907 } 2908 2909 btrfs_free_path(path); 2910 atomic_inc(&fs_info->defrag_running); 2911 2912 return new; 2913 2914 out_free_path: 2915 btrfs_free_path(path); 2916 out_kfree: 2917 free_sa_defrag_extent(new); 2918 return NULL; 2919 } 2920 2921 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 2922 u64 start, u64 len) 2923 { 2924 struct btrfs_block_group_cache *cache; 2925 2926 cache = btrfs_lookup_block_group(fs_info, start); 2927 ASSERT(cache); 2928 2929 spin_lock(&cache->lock); 2930 cache->delalloc_bytes -= len; 2931 spin_unlock(&cache->lock); 2932 2933 btrfs_put_block_group(cache); 2934 } 2935 2936 /* as ordered data IO finishes, this gets called so we can finish 2937 * an ordered extent if the range of bytes in the file it covers are 2938 * fully written. 2939 */ 2940 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 2941 { 2942 struct inode *inode = ordered_extent->inode; 2943 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2944 struct btrfs_root *root = BTRFS_I(inode)->root; 2945 struct btrfs_trans_handle *trans = NULL; 2946 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2947 struct extent_state *cached_state = NULL; 2948 struct new_sa_defrag_extent *new = NULL; 2949 int compress_type = 0; 2950 int ret = 0; 2951 u64 logical_len = ordered_extent->len; 2952 bool nolock; 2953 bool truncated = false; 2954 bool range_locked = false; 2955 bool clear_new_delalloc_bytes = false; 2956 2957 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 2958 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && 2959 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags)) 2960 clear_new_delalloc_bytes = true; 2961 2962 nolock = btrfs_is_free_space_inode(BTRFS_I(inode)); 2963 2964 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 2965 ret = -EIO; 2966 goto out; 2967 } 2968 2969 btrfs_free_io_failure_record(BTRFS_I(inode), 2970 ordered_extent->file_offset, 2971 ordered_extent->file_offset + 2972 ordered_extent->len - 1); 2973 2974 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 2975 truncated = true; 2976 logical_len = ordered_extent->truncated_len; 2977 /* Truncated the entire extent, don't bother adding */ 2978 if (!logical_len) 2979 goto out; 2980 } 2981 2982 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 2983 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 2984 2985 /* 2986 * For mwrite(mmap + memset to write) case, we still reserve 2987 * space for NOCOW range. 2988 * As NOCOW won't cause a new delayed ref, just free the space 2989 */ 2990 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset, 2991 ordered_extent->len); 2992 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 2993 if (nolock) 2994 trans = btrfs_join_transaction_nolock(root); 2995 else 2996 trans = btrfs_join_transaction(root); 2997 if (IS_ERR(trans)) { 2998 ret = PTR_ERR(trans); 2999 trans = NULL; 3000 goto out; 3001 } 3002 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 3003 ret = btrfs_update_inode_fallback(trans, root, inode); 3004 if (ret) /* -ENOMEM or corruption */ 3005 btrfs_abort_transaction(trans, ret); 3006 goto out; 3007 } 3008 3009 range_locked = true; 3010 lock_extent_bits(io_tree, ordered_extent->file_offset, 3011 ordered_extent->file_offset + ordered_extent->len - 1, 3012 &cached_state); 3013 3014 ret = test_range_bit(io_tree, ordered_extent->file_offset, 3015 ordered_extent->file_offset + ordered_extent->len - 1, 3016 EXTENT_DEFRAG, 0, cached_state); 3017 if (ret) { 3018 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item); 3019 if (0 && last_snapshot >= BTRFS_I(inode)->generation) 3020 /* the inode is shared */ 3021 new = record_old_file_extents(inode, ordered_extent); 3022 3023 clear_extent_bit(io_tree, ordered_extent->file_offset, 3024 ordered_extent->file_offset + ordered_extent->len - 1, 3025 EXTENT_DEFRAG, 0, 0, &cached_state); 3026 } 3027 3028 if (nolock) 3029 trans = btrfs_join_transaction_nolock(root); 3030 else 3031 trans = btrfs_join_transaction(root); 3032 if (IS_ERR(trans)) { 3033 ret = PTR_ERR(trans); 3034 trans = NULL; 3035 goto out; 3036 } 3037 3038 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 3039 3040 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 3041 compress_type = ordered_extent->compress_type; 3042 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3043 BUG_ON(compress_type); 3044 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset, 3045 ordered_extent->len); 3046 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode), 3047 ordered_extent->file_offset, 3048 ordered_extent->file_offset + 3049 logical_len); 3050 } else { 3051 BUG_ON(root == fs_info->tree_root); 3052 ret = insert_reserved_file_extent(trans, inode, 3053 ordered_extent->file_offset, 3054 ordered_extent->start, 3055 ordered_extent->disk_len, 3056 logical_len, logical_len, 3057 compress_type, 0, 0, 3058 BTRFS_FILE_EXTENT_REG); 3059 if (!ret) 3060 btrfs_release_delalloc_bytes(fs_info, 3061 ordered_extent->start, 3062 ordered_extent->disk_len); 3063 } 3064 unpin_extent_cache(&BTRFS_I(inode)->extent_tree, 3065 ordered_extent->file_offset, ordered_extent->len, 3066 trans->transid); 3067 if (ret < 0) { 3068 btrfs_abort_transaction(trans, ret); 3069 goto out; 3070 } 3071 3072 ret = add_pending_csums(trans, inode, &ordered_extent->list); 3073 if (ret) { 3074 btrfs_abort_transaction(trans, ret); 3075 goto out; 3076 } 3077 3078 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 3079 ret = btrfs_update_inode_fallback(trans, root, inode); 3080 if (ret) { /* -ENOMEM or corruption */ 3081 btrfs_abort_transaction(trans, ret); 3082 goto out; 3083 } 3084 ret = 0; 3085 out: 3086 if (range_locked || clear_new_delalloc_bytes) { 3087 unsigned int clear_bits = 0; 3088 3089 if (range_locked) 3090 clear_bits |= EXTENT_LOCKED; 3091 if (clear_new_delalloc_bytes) 3092 clear_bits |= EXTENT_DELALLOC_NEW; 3093 clear_extent_bit(&BTRFS_I(inode)->io_tree, 3094 ordered_extent->file_offset, 3095 ordered_extent->file_offset + 3096 ordered_extent->len - 1, 3097 clear_bits, 3098 (clear_bits & EXTENT_LOCKED) ? 1 : 0, 3099 0, &cached_state); 3100 } 3101 3102 if (trans) 3103 btrfs_end_transaction(trans); 3104 3105 if (ret || truncated) { 3106 u64 start, end; 3107 3108 if (truncated) 3109 start = ordered_extent->file_offset + logical_len; 3110 else 3111 start = ordered_extent->file_offset; 3112 end = ordered_extent->file_offset + ordered_extent->len - 1; 3113 clear_extent_uptodate(io_tree, start, end, NULL); 3114 3115 /* Drop the cache for the part of the extent we didn't write. */ 3116 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0); 3117 3118 /* 3119 * If the ordered extent had an IOERR or something else went 3120 * wrong we need to return the space for this ordered extent 3121 * back to the allocator. We only free the extent in the 3122 * truncated case if we didn't write out the extent at all. 3123 */ 3124 if ((ret || !logical_len) && 3125 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3126 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) 3127 btrfs_free_reserved_extent(fs_info, 3128 ordered_extent->start, 3129 ordered_extent->disk_len, 1); 3130 } 3131 3132 3133 /* 3134 * This needs to be done to make sure anybody waiting knows we are done 3135 * updating everything for this ordered extent. 3136 */ 3137 btrfs_remove_ordered_extent(inode, ordered_extent); 3138 3139 /* for snapshot-aware defrag */ 3140 if (new) { 3141 if (ret) { 3142 free_sa_defrag_extent(new); 3143 atomic_dec(&fs_info->defrag_running); 3144 } else { 3145 relink_file_extents(new); 3146 } 3147 } 3148 3149 /* once for us */ 3150 btrfs_put_ordered_extent(ordered_extent); 3151 /* once for the tree */ 3152 btrfs_put_ordered_extent(ordered_extent); 3153 3154 return ret; 3155 } 3156 3157 static void finish_ordered_fn(struct btrfs_work *work) 3158 { 3159 struct btrfs_ordered_extent *ordered_extent; 3160 ordered_extent = container_of(work, struct btrfs_ordered_extent, work); 3161 btrfs_finish_ordered_io(ordered_extent); 3162 } 3163 3164 static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end, 3165 struct extent_state *state, int uptodate) 3166 { 3167 struct inode *inode = page->mapping->host; 3168 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3169 struct btrfs_ordered_extent *ordered_extent = NULL; 3170 struct btrfs_workqueue *wq; 3171 btrfs_work_func_t func; 3172 3173 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); 3174 3175 ClearPagePrivate2(page); 3176 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, 3177 end - start + 1, uptodate)) 3178 return; 3179 3180 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 3181 wq = fs_info->endio_freespace_worker; 3182 func = btrfs_freespace_write_helper; 3183 } else { 3184 wq = fs_info->endio_write_workers; 3185 func = btrfs_endio_write_helper; 3186 } 3187 3188 btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL, 3189 NULL); 3190 btrfs_queue_work(wq, &ordered_extent->work); 3191 } 3192 3193 static int __readpage_endio_check(struct inode *inode, 3194 struct btrfs_io_bio *io_bio, 3195 int icsum, struct page *page, 3196 int pgoff, u64 start, size_t len) 3197 { 3198 char *kaddr; 3199 u32 csum_expected; 3200 u32 csum = ~(u32)0; 3201 3202 csum_expected = *(((u32 *)io_bio->csum) + icsum); 3203 3204 kaddr = kmap_atomic(page); 3205 csum = btrfs_csum_data(kaddr + pgoff, csum, len); 3206 btrfs_csum_final(csum, (u8 *)&csum); 3207 if (csum != csum_expected) 3208 goto zeroit; 3209 3210 kunmap_atomic(kaddr); 3211 return 0; 3212 zeroit: 3213 btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected, 3214 io_bio->mirror_num); 3215 memset(kaddr + pgoff, 1, len); 3216 flush_dcache_page(page); 3217 kunmap_atomic(kaddr); 3218 return -EIO; 3219 } 3220 3221 /* 3222 * when reads are done, we need to check csums to verify the data is correct 3223 * if there's a match, we allow the bio to finish. If not, the code in 3224 * extent_io.c will try to find good copies for us. 3225 */ 3226 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, 3227 u64 phy_offset, struct page *page, 3228 u64 start, u64 end, int mirror) 3229 { 3230 size_t offset = start - page_offset(page); 3231 struct inode *inode = page->mapping->host; 3232 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 3233 struct btrfs_root *root = BTRFS_I(inode)->root; 3234 3235 if (PageChecked(page)) { 3236 ClearPageChecked(page); 3237 return 0; 3238 } 3239 3240 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 3241 return 0; 3242 3243 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID && 3244 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) { 3245 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM); 3246 return 0; 3247 } 3248 3249 phy_offset >>= inode->i_sb->s_blocksize_bits; 3250 return __readpage_endio_check(inode, io_bio, phy_offset, page, offset, 3251 start, (size_t)(end - start + 1)); 3252 } 3253 3254 /* 3255 * btrfs_add_delayed_iput - perform a delayed iput on @inode 3256 * 3257 * @inode: The inode we want to perform iput on 3258 * 3259 * This function uses the generic vfs_inode::i_count to track whether we should 3260 * just decrement it (in case it's > 1) or if this is the last iput then link 3261 * the inode to the delayed iput machinery. Delayed iputs are processed at 3262 * transaction commit time/superblock commit/cleaner kthread. 3263 */ 3264 void btrfs_add_delayed_iput(struct inode *inode) 3265 { 3266 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3267 struct btrfs_inode *binode = BTRFS_I(inode); 3268 3269 if (atomic_add_unless(&inode->i_count, -1, 1)) 3270 return; 3271 3272 spin_lock(&fs_info->delayed_iput_lock); 3273 ASSERT(list_empty(&binode->delayed_iput)); 3274 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); 3275 spin_unlock(&fs_info->delayed_iput_lock); 3276 } 3277 3278 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 3279 { 3280 3281 spin_lock(&fs_info->delayed_iput_lock); 3282 while (!list_empty(&fs_info->delayed_iputs)) { 3283 struct btrfs_inode *inode; 3284 3285 inode = list_first_entry(&fs_info->delayed_iputs, 3286 struct btrfs_inode, delayed_iput); 3287 list_del_init(&inode->delayed_iput); 3288 spin_unlock(&fs_info->delayed_iput_lock); 3289 iput(&inode->vfs_inode); 3290 spin_lock(&fs_info->delayed_iput_lock); 3291 } 3292 spin_unlock(&fs_info->delayed_iput_lock); 3293 } 3294 3295 /* 3296 * This is called in transaction commit time. If there are no orphan 3297 * files in the subvolume, it removes orphan item and frees block_rsv 3298 * structure. 3299 */ 3300 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans, 3301 struct btrfs_root *root) 3302 { 3303 struct btrfs_fs_info *fs_info = root->fs_info; 3304 struct btrfs_block_rsv *block_rsv; 3305 int ret; 3306 3307 if (atomic_read(&root->orphan_inodes) || 3308 root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) 3309 return; 3310 3311 spin_lock(&root->orphan_lock); 3312 if (atomic_read(&root->orphan_inodes)) { 3313 spin_unlock(&root->orphan_lock); 3314 return; 3315 } 3316 3317 if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) { 3318 spin_unlock(&root->orphan_lock); 3319 return; 3320 } 3321 3322 block_rsv = root->orphan_block_rsv; 3323 root->orphan_block_rsv = NULL; 3324 spin_unlock(&root->orphan_lock); 3325 3326 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) && 3327 btrfs_root_refs(&root->root_item) > 0) { 3328 ret = btrfs_del_orphan_item(trans, fs_info->tree_root, 3329 root->root_key.objectid); 3330 if (ret) 3331 btrfs_abort_transaction(trans, ret); 3332 else 3333 clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, 3334 &root->state); 3335 } 3336 3337 if (block_rsv) { 3338 WARN_ON(block_rsv->size > 0); 3339 btrfs_free_block_rsv(fs_info, block_rsv); 3340 } 3341 } 3342 3343 /* 3344 * This creates an orphan entry for the given inode in case something goes 3345 * wrong in the middle of an unlink/truncate. 3346 * 3347 * NOTE: caller of this function should reserve 5 units of metadata for 3348 * this function. 3349 */ 3350 int btrfs_orphan_add(struct btrfs_trans_handle *trans, 3351 struct btrfs_inode *inode) 3352 { 3353 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 3354 struct btrfs_root *root = inode->root; 3355 struct btrfs_block_rsv *block_rsv = NULL; 3356 int reserve = 0; 3357 bool insert = false; 3358 int ret; 3359 3360 if (!root->orphan_block_rsv) { 3361 block_rsv = btrfs_alloc_block_rsv(fs_info, 3362 BTRFS_BLOCK_RSV_TEMP); 3363 if (!block_rsv) 3364 return -ENOMEM; 3365 } 3366 3367 if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3368 &inode->runtime_flags)) 3369 insert = true; 3370 3371 if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3372 &inode->runtime_flags)) 3373 reserve = 1; 3374 3375 spin_lock(&root->orphan_lock); 3376 /* If someone has created ->orphan_block_rsv, be happy to use it. */ 3377 if (!root->orphan_block_rsv) { 3378 root->orphan_block_rsv = block_rsv; 3379 } else if (block_rsv) { 3380 btrfs_free_block_rsv(fs_info, block_rsv); 3381 block_rsv = NULL; 3382 } 3383 3384 if (insert) 3385 atomic_inc(&root->orphan_inodes); 3386 spin_unlock(&root->orphan_lock); 3387 3388 /* grab metadata reservation from transaction handle */ 3389 if (reserve) { 3390 ret = btrfs_orphan_reserve_metadata(trans, inode); 3391 ASSERT(!ret); 3392 if (ret) { 3393 /* 3394 * dec doesn't need spin_lock as ->orphan_block_rsv 3395 * would be released only if ->orphan_inodes is 3396 * zero. 3397 */ 3398 atomic_dec(&root->orphan_inodes); 3399 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3400 &inode->runtime_flags); 3401 if (insert) 3402 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3403 &inode->runtime_flags); 3404 return ret; 3405 } 3406 } 3407 3408 /* insert an orphan item to track this unlinked/truncated file */ 3409 if (insert) { 3410 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode)); 3411 if (ret) { 3412 if (reserve) { 3413 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3414 &inode->runtime_flags); 3415 btrfs_orphan_release_metadata(inode); 3416 } 3417 /* 3418 * btrfs_orphan_commit_root may race with us and set 3419 * ->orphan_block_rsv to zero, in order to avoid that, 3420 * decrease ->orphan_inodes after everything is done. 3421 */ 3422 atomic_dec(&root->orphan_inodes); 3423 if (ret != -EEXIST) { 3424 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3425 &inode->runtime_flags); 3426 btrfs_abort_transaction(trans, ret); 3427 return ret; 3428 } 3429 } 3430 ret = 0; 3431 } 3432 3433 return 0; 3434 } 3435 3436 /* 3437 * We have done the truncate/delete so we can go ahead and remove the orphan 3438 * item for this particular inode. 3439 */ 3440 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3441 struct btrfs_inode *inode) 3442 { 3443 struct btrfs_root *root = inode->root; 3444 int delete_item = 0; 3445 int ret = 0; 3446 3447 if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3448 &inode->runtime_flags)) 3449 delete_item = 1; 3450 3451 if (delete_item && trans) 3452 ret = btrfs_del_orphan_item(trans, root, btrfs_ino(inode)); 3453 3454 if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3455 &inode->runtime_flags)) 3456 btrfs_orphan_release_metadata(inode); 3457 3458 /* 3459 * btrfs_orphan_commit_root may race with us and set ->orphan_block_rsv 3460 * to zero, in order to avoid that, decrease ->orphan_inodes after 3461 * everything is done. 3462 */ 3463 if (delete_item) 3464 atomic_dec(&root->orphan_inodes); 3465 3466 return ret; 3467 } 3468 3469 /* 3470 * this cleans up any orphans that may be left on the list from the last use 3471 * of this root. 3472 */ 3473 int btrfs_orphan_cleanup(struct btrfs_root *root) 3474 { 3475 struct btrfs_fs_info *fs_info = root->fs_info; 3476 struct btrfs_path *path; 3477 struct extent_buffer *leaf; 3478 struct btrfs_key key, found_key; 3479 struct btrfs_trans_handle *trans; 3480 struct inode *inode; 3481 u64 last_objectid = 0; 3482 int ret = 0, nr_unlink = 0, nr_truncate = 0; 3483 3484 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED)) 3485 return 0; 3486 3487 path = btrfs_alloc_path(); 3488 if (!path) { 3489 ret = -ENOMEM; 3490 goto out; 3491 } 3492 path->reada = READA_BACK; 3493 3494 key.objectid = BTRFS_ORPHAN_OBJECTID; 3495 key.type = BTRFS_ORPHAN_ITEM_KEY; 3496 key.offset = (u64)-1; 3497 3498 while (1) { 3499 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3500 if (ret < 0) 3501 goto out; 3502 3503 /* 3504 * if ret == 0 means we found what we were searching for, which 3505 * is weird, but possible, so only screw with path if we didn't 3506 * find the key and see if we have stuff that matches 3507 */ 3508 if (ret > 0) { 3509 ret = 0; 3510 if (path->slots[0] == 0) 3511 break; 3512 path->slots[0]--; 3513 } 3514 3515 /* pull out the item */ 3516 leaf = path->nodes[0]; 3517 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3518 3519 /* make sure the item matches what we want */ 3520 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3521 break; 3522 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3523 break; 3524 3525 /* release the path since we're done with it */ 3526 btrfs_release_path(path); 3527 3528 /* 3529 * this is where we are basically btrfs_lookup, without the 3530 * crossing root thing. we store the inode number in the 3531 * offset of the orphan item. 3532 */ 3533 3534 if (found_key.offset == last_objectid) { 3535 btrfs_err(fs_info, 3536 "Error removing orphan entry, stopping orphan cleanup"); 3537 ret = -EINVAL; 3538 goto out; 3539 } 3540 3541 last_objectid = found_key.offset; 3542 3543 found_key.objectid = found_key.offset; 3544 found_key.type = BTRFS_INODE_ITEM_KEY; 3545 found_key.offset = 0; 3546 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL); 3547 ret = PTR_ERR_OR_ZERO(inode); 3548 if (ret && ret != -ENOENT) 3549 goto out; 3550 3551 if (ret == -ENOENT && root == fs_info->tree_root) { 3552 struct btrfs_root *dead_root; 3553 struct btrfs_fs_info *fs_info = root->fs_info; 3554 int is_dead_root = 0; 3555 3556 /* 3557 * this is an orphan in the tree root. Currently these 3558 * could come from 2 sources: 3559 * a) a snapshot deletion in progress 3560 * b) a free space cache inode 3561 * We need to distinguish those two, as the snapshot 3562 * orphan must not get deleted. 3563 * find_dead_roots already ran before us, so if this 3564 * is a snapshot deletion, we should find the root 3565 * in the dead_roots list 3566 */ 3567 spin_lock(&fs_info->trans_lock); 3568 list_for_each_entry(dead_root, &fs_info->dead_roots, 3569 root_list) { 3570 if (dead_root->root_key.objectid == 3571 found_key.objectid) { 3572 is_dead_root = 1; 3573 break; 3574 } 3575 } 3576 spin_unlock(&fs_info->trans_lock); 3577 if (is_dead_root) { 3578 /* prevent this orphan from being found again */ 3579 key.offset = found_key.objectid - 1; 3580 continue; 3581 } 3582 } 3583 /* 3584 * Inode is already gone but the orphan item is still there, 3585 * kill the orphan item. 3586 */ 3587 if (ret == -ENOENT) { 3588 trans = btrfs_start_transaction(root, 1); 3589 if (IS_ERR(trans)) { 3590 ret = PTR_ERR(trans); 3591 goto out; 3592 } 3593 btrfs_debug(fs_info, "auto deleting %Lu", 3594 found_key.objectid); 3595 ret = btrfs_del_orphan_item(trans, root, 3596 found_key.objectid); 3597 btrfs_end_transaction(trans); 3598 if (ret) 3599 goto out; 3600 continue; 3601 } 3602 3603 /* 3604 * add this inode to the orphan list so btrfs_orphan_del does 3605 * the proper thing when we hit it 3606 */ 3607 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3608 &BTRFS_I(inode)->runtime_flags); 3609 atomic_inc(&root->orphan_inodes); 3610 3611 /* if we have links, this was a truncate, lets do that */ 3612 if (inode->i_nlink) { 3613 if (WARN_ON(!S_ISREG(inode->i_mode))) { 3614 iput(inode); 3615 continue; 3616 } 3617 nr_truncate++; 3618 3619 /* 1 for the orphan item deletion. */ 3620 trans = btrfs_start_transaction(root, 1); 3621 if (IS_ERR(trans)) { 3622 iput(inode); 3623 ret = PTR_ERR(trans); 3624 goto out; 3625 } 3626 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 3627 btrfs_end_transaction(trans); 3628 if (ret) { 3629 iput(inode); 3630 goto out; 3631 } 3632 3633 ret = btrfs_truncate(inode, false); 3634 if (ret) 3635 btrfs_orphan_del(NULL, BTRFS_I(inode)); 3636 } else { 3637 nr_unlink++; 3638 } 3639 3640 /* this will do delete_inode and everything for us */ 3641 iput(inode); 3642 if (ret) 3643 goto out; 3644 } 3645 /* release the path since we're done with it */ 3646 btrfs_release_path(path); 3647 3648 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE; 3649 3650 if (root->orphan_block_rsv) 3651 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv, 3652 (u64)-1); 3653 3654 if (root->orphan_block_rsv || 3655 test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3656 trans = btrfs_join_transaction(root); 3657 if (!IS_ERR(trans)) 3658 btrfs_end_transaction(trans); 3659 } 3660 3661 if (nr_unlink) 3662 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3663 if (nr_truncate) 3664 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate); 3665 3666 out: 3667 if (ret) 3668 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3669 btrfs_free_path(path); 3670 return ret; 3671 } 3672 3673 /* 3674 * very simple check to peek ahead in the leaf looking for xattrs. If we 3675 * don't find any xattrs, we know there can't be any acls. 3676 * 3677 * slot is the slot the inode is in, objectid is the objectid of the inode 3678 */ 3679 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3680 int slot, u64 objectid, 3681 int *first_xattr_slot) 3682 { 3683 u32 nritems = btrfs_header_nritems(leaf); 3684 struct btrfs_key found_key; 3685 static u64 xattr_access = 0; 3686 static u64 xattr_default = 0; 3687 int scanned = 0; 3688 3689 if (!xattr_access) { 3690 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3691 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3692 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3693 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3694 } 3695 3696 slot++; 3697 *first_xattr_slot = -1; 3698 while (slot < nritems) { 3699 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3700 3701 /* we found a different objectid, there must not be acls */ 3702 if (found_key.objectid != objectid) 3703 return 0; 3704 3705 /* we found an xattr, assume we've got an acl */ 3706 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3707 if (*first_xattr_slot == -1) 3708 *first_xattr_slot = slot; 3709 if (found_key.offset == xattr_access || 3710 found_key.offset == xattr_default) 3711 return 1; 3712 } 3713 3714 /* 3715 * we found a key greater than an xattr key, there can't 3716 * be any acls later on 3717 */ 3718 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3719 return 0; 3720 3721 slot++; 3722 scanned++; 3723 3724 /* 3725 * it goes inode, inode backrefs, xattrs, extents, 3726 * so if there are a ton of hard links to an inode there can 3727 * be a lot of backrefs. Don't waste time searching too hard, 3728 * this is just an optimization 3729 */ 3730 if (scanned >= 8) 3731 break; 3732 } 3733 /* we hit the end of the leaf before we found an xattr or 3734 * something larger than an xattr. We have to assume the inode 3735 * has acls 3736 */ 3737 if (*first_xattr_slot == -1) 3738 *first_xattr_slot = slot; 3739 return 1; 3740 } 3741 3742 /* 3743 * read an inode from the btree into the in-memory inode 3744 */ 3745 static int btrfs_read_locked_inode(struct inode *inode) 3746 { 3747 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3748 struct btrfs_path *path; 3749 struct extent_buffer *leaf; 3750 struct btrfs_inode_item *inode_item; 3751 struct btrfs_root *root = BTRFS_I(inode)->root; 3752 struct btrfs_key location; 3753 unsigned long ptr; 3754 int maybe_acls; 3755 u32 rdev; 3756 int ret; 3757 bool filled = false; 3758 int first_xattr_slot; 3759 3760 ret = btrfs_fill_inode(inode, &rdev); 3761 if (!ret) 3762 filled = true; 3763 3764 path = btrfs_alloc_path(); 3765 if (!path) { 3766 ret = -ENOMEM; 3767 goto make_bad; 3768 } 3769 3770 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3771 3772 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3773 if (ret) { 3774 if (ret > 0) 3775 ret = -ENOENT; 3776 goto make_bad; 3777 } 3778 3779 leaf = path->nodes[0]; 3780 3781 if (filled) 3782 goto cache_index; 3783 3784 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3785 struct btrfs_inode_item); 3786 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3787 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3788 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3789 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3790 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item)); 3791 3792 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime); 3793 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime); 3794 3795 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime); 3796 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime); 3797 3798 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime); 3799 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime); 3800 3801 BTRFS_I(inode)->i_otime.tv_sec = 3802 btrfs_timespec_sec(leaf, &inode_item->otime); 3803 BTRFS_I(inode)->i_otime.tv_nsec = 3804 btrfs_timespec_nsec(leaf, &inode_item->otime); 3805 3806 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3807 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3808 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3809 3810 inode_set_iversion_queried(inode, 3811 btrfs_inode_sequence(leaf, inode_item)); 3812 inode->i_generation = BTRFS_I(inode)->generation; 3813 inode->i_rdev = 0; 3814 rdev = btrfs_inode_rdev(leaf, inode_item); 3815 3816 BTRFS_I(inode)->index_cnt = (u64)-1; 3817 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item); 3818 3819 cache_index: 3820 /* 3821 * If we were modified in the current generation and evicted from memory 3822 * and then re-read we need to do a full sync since we don't have any 3823 * idea about which extents were modified before we were evicted from 3824 * cache. 3825 * 3826 * This is required for both inode re-read from disk and delayed inode 3827 * in delayed_nodes_tree. 3828 */ 3829 if (BTRFS_I(inode)->last_trans == fs_info->generation) 3830 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3831 &BTRFS_I(inode)->runtime_flags); 3832 3833 /* 3834 * We don't persist the id of the transaction where an unlink operation 3835 * against the inode was last made. So here we assume the inode might 3836 * have been evicted, and therefore the exact value of last_unlink_trans 3837 * lost, and set it to last_trans to avoid metadata inconsistencies 3838 * between the inode and its parent if the inode is fsync'ed and the log 3839 * replayed. For example, in the scenario: 3840 * 3841 * touch mydir/foo 3842 * ln mydir/foo mydir/bar 3843 * sync 3844 * unlink mydir/bar 3845 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 3846 * xfs_io -c fsync mydir/foo 3847 * <power failure> 3848 * mount fs, triggers fsync log replay 3849 * 3850 * We must make sure that when we fsync our inode foo we also log its 3851 * parent inode, otherwise after log replay the parent still has the 3852 * dentry with the "bar" name but our inode foo has a link count of 1 3853 * and doesn't have an inode ref with the name "bar" anymore. 3854 * 3855 * Setting last_unlink_trans to last_trans is a pessimistic approach, 3856 * but it guarantees correctness at the expense of occasional full 3857 * transaction commits on fsync if our inode is a directory, or if our 3858 * inode is not a directory, logging its parent unnecessarily. 3859 */ 3860 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans; 3861 3862 path->slots[0]++; 3863 if (inode->i_nlink != 1 || 3864 path->slots[0] >= btrfs_header_nritems(leaf)) 3865 goto cache_acl; 3866 3867 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3868 if (location.objectid != btrfs_ino(BTRFS_I(inode))) 3869 goto cache_acl; 3870 3871 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3872 if (location.type == BTRFS_INODE_REF_KEY) { 3873 struct btrfs_inode_ref *ref; 3874 3875 ref = (struct btrfs_inode_ref *)ptr; 3876 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3877 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3878 struct btrfs_inode_extref *extref; 3879 3880 extref = (struct btrfs_inode_extref *)ptr; 3881 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3882 extref); 3883 } 3884 cache_acl: 3885 /* 3886 * try to precache a NULL acl entry for files that don't have 3887 * any xattrs or acls 3888 */ 3889 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3890 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot); 3891 if (first_xattr_slot != -1) { 3892 path->slots[0] = first_xattr_slot; 3893 ret = btrfs_load_inode_props(inode, path); 3894 if (ret) 3895 btrfs_err(fs_info, 3896 "error loading props for ino %llu (root %llu): %d", 3897 btrfs_ino(BTRFS_I(inode)), 3898 root->root_key.objectid, ret); 3899 } 3900 btrfs_free_path(path); 3901 3902 if (!maybe_acls) 3903 cache_no_acl(inode); 3904 3905 switch (inode->i_mode & S_IFMT) { 3906 case S_IFREG: 3907 inode->i_mapping->a_ops = &btrfs_aops; 3908 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 3909 inode->i_fop = &btrfs_file_operations; 3910 inode->i_op = &btrfs_file_inode_operations; 3911 break; 3912 case S_IFDIR: 3913 inode->i_fop = &btrfs_dir_file_operations; 3914 inode->i_op = &btrfs_dir_inode_operations; 3915 break; 3916 case S_IFLNK: 3917 inode->i_op = &btrfs_symlink_inode_operations; 3918 inode_nohighmem(inode); 3919 inode->i_mapping->a_ops = &btrfs_symlink_aops; 3920 break; 3921 default: 3922 inode->i_op = &btrfs_special_inode_operations; 3923 init_special_inode(inode, inode->i_mode, rdev); 3924 break; 3925 } 3926 3927 btrfs_update_iflags(inode); 3928 return 0; 3929 3930 make_bad: 3931 btrfs_free_path(path); 3932 make_bad_inode(inode); 3933 return ret; 3934 } 3935 3936 /* 3937 * given a leaf and an inode, copy the inode fields into the leaf 3938 */ 3939 static void fill_inode_item(struct btrfs_trans_handle *trans, 3940 struct extent_buffer *leaf, 3941 struct btrfs_inode_item *item, 3942 struct inode *inode) 3943 { 3944 struct btrfs_map_token token; 3945 3946 btrfs_init_map_token(&token); 3947 3948 btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token); 3949 btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token); 3950 btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size, 3951 &token); 3952 btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token); 3953 btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token); 3954 3955 btrfs_set_token_timespec_sec(leaf, &item->atime, 3956 inode->i_atime.tv_sec, &token); 3957 btrfs_set_token_timespec_nsec(leaf, &item->atime, 3958 inode->i_atime.tv_nsec, &token); 3959 3960 btrfs_set_token_timespec_sec(leaf, &item->mtime, 3961 inode->i_mtime.tv_sec, &token); 3962 btrfs_set_token_timespec_nsec(leaf, &item->mtime, 3963 inode->i_mtime.tv_nsec, &token); 3964 3965 btrfs_set_token_timespec_sec(leaf, &item->ctime, 3966 inode->i_ctime.tv_sec, &token); 3967 btrfs_set_token_timespec_nsec(leaf, &item->ctime, 3968 inode->i_ctime.tv_nsec, &token); 3969 3970 btrfs_set_token_timespec_sec(leaf, &item->otime, 3971 BTRFS_I(inode)->i_otime.tv_sec, &token); 3972 btrfs_set_token_timespec_nsec(leaf, &item->otime, 3973 BTRFS_I(inode)->i_otime.tv_nsec, &token); 3974 3975 btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode), 3976 &token); 3977 btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation, 3978 &token); 3979 btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode), 3980 &token); 3981 btrfs_set_token_inode_transid(leaf, item, trans->transid, &token); 3982 btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token); 3983 btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token); 3984 btrfs_set_token_inode_block_group(leaf, item, 0, &token); 3985 } 3986 3987 /* 3988 * copy everything in the in-memory inode into the btree. 3989 */ 3990 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 3991 struct btrfs_root *root, struct inode *inode) 3992 { 3993 struct btrfs_inode_item *inode_item; 3994 struct btrfs_path *path; 3995 struct extent_buffer *leaf; 3996 int ret; 3997 3998 path = btrfs_alloc_path(); 3999 if (!path) 4000 return -ENOMEM; 4001 4002 path->leave_spinning = 1; 4003 ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location, 4004 1); 4005 if (ret) { 4006 if (ret > 0) 4007 ret = -ENOENT; 4008 goto failed; 4009 } 4010 4011 leaf = path->nodes[0]; 4012 inode_item = btrfs_item_ptr(leaf, path->slots[0], 4013 struct btrfs_inode_item); 4014 4015 fill_inode_item(trans, leaf, inode_item, inode); 4016 btrfs_mark_buffer_dirty(leaf); 4017 btrfs_set_inode_last_trans(trans, inode); 4018 ret = 0; 4019 failed: 4020 btrfs_free_path(path); 4021 return ret; 4022 } 4023 4024 /* 4025 * copy everything in the in-memory inode into the btree. 4026 */ 4027 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 4028 struct btrfs_root *root, struct inode *inode) 4029 { 4030 struct btrfs_fs_info *fs_info = root->fs_info; 4031 int ret; 4032 4033 /* 4034 * If the inode is a free space inode, we can deadlock during commit 4035 * if we put it into the delayed code. 4036 * 4037 * The data relocation inode should also be directly updated 4038 * without delay 4039 */ 4040 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) 4041 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 4042 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 4043 btrfs_update_root_times(trans, root); 4044 4045 ret = btrfs_delayed_update_inode(trans, root, inode); 4046 if (!ret) 4047 btrfs_set_inode_last_trans(trans, inode); 4048 return ret; 4049 } 4050 4051 return btrfs_update_inode_item(trans, root, inode); 4052 } 4053 4054 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 4055 struct btrfs_root *root, 4056 struct inode *inode) 4057 { 4058 int ret; 4059 4060 ret = btrfs_update_inode(trans, root, inode); 4061 if (ret == -ENOSPC) 4062 return btrfs_update_inode_item(trans, root, inode); 4063 return ret; 4064 } 4065 4066 /* 4067 * unlink helper that gets used here in inode.c and in the tree logging 4068 * recovery code. It remove a link in a directory with a given name, and 4069 * also drops the back refs in the inode to the directory 4070 */ 4071 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4072 struct btrfs_root *root, 4073 struct btrfs_inode *dir, 4074 struct btrfs_inode *inode, 4075 const char *name, int name_len) 4076 { 4077 struct btrfs_fs_info *fs_info = root->fs_info; 4078 struct btrfs_path *path; 4079 int ret = 0; 4080 struct extent_buffer *leaf; 4081 struct btrfs_dir_item *di; 4082 struct btrfs_key key; 4083 u64 index; 4084 u64 ino = btrfs_ino(inode); 4085 u64 dir_ino = btrfs_ino(dir); 4086 4087 path = btrfs_alloc_path(); 4088 if (!path) { 4089 ret = -ENOMEM; 4090 goto out; 4091 } 4092 4093 path->leave_spinning = 1; 4094 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4095 name, name_len, -1); 4096 if (IS_ERR(di)) { 4097 ret = PTR_ERR(di); 4098 goto err; 4099 } 4100 if (!di) { 4101 ret = -ENOENT; 4102 goto err; 4103 } 4104 leaf = path->nodes[0]; 4105 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4106 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4107 if (ret) 4108 goto err; 4109 btrfs_release_path(path); 4110 4111 /* 4112 * If we don't have dir index, we have to get it by looking up 4113 * the inode ref, since we get the inode ref, remove it directly, 4114 * it is unnecessary to do delayed deletion. 4115 * 4116 * But if we have dir index, needn't search inode ref to get it. 4117 * Since the inode ref is close to the inode item, it is better 4118 * that we delay to delete it, and just do this deletion when 4119 * we update the inode item. 4120 */ 4121 if (inode->dir_index) { 4122 ret = btrfs_delayed_delete_inode_ref(inode); 4123 if (!ret) { 4124 index = inode->dir_index; 4125 goto skip_backref; 4126 } 4127 } 4128 4129 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino, 4130 dir_ino, &index); 4131 if (ret) { 4132 btrfs_info(fs_info, 4133 "failed to delete reference to %.*s, inode %llu parent %llu", 4134 name_len, name, ino, dir_ino); 4135 btrfs_abort_transaction(trans, ret); 4136 goto err; 4137 } 4138 skip_backref: 4139 ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index); 4140 if (ret) { 4141 btrfs_abort_transaction(trans, ret); 4142 goto err; 4143 } 4144 4145 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode, 4146 dir_ino); 4147 if (ret != 0 && ret != -ENOENT) { 4148 btrfs_abort_transaction(trans, ret); 4149 goto err; 4150 } 4151 4152 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir, 4153 index); 4154 if (ret == -ENOENT) 4155 ret = 0; 4156 else if (ret) 4157 btrfs_abort_transaction(trans, ret); 4158 err: 4159 btrfs_free_path(path); 4160 if (ret) 4161 goto out; 4162 4163 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2); 4164 inode_inc_iversion(&inode->vfs_inode); 4165 inode_inc_iversion(&dir->vfs_inode); 4166 inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime = 4167 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode); 4168 ret = btrfs_update_inode(trans, root, &dir->vfs_inode); 4169 out: 4170 return ret; 4171 } 4172 4173 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4174 struct btrfs_root *root, 4175 struct btrfs_inode *dir, struct btrfs_inode *inode, 4176 const char *name, int name_len) 4177 { 4178 int ret; 4179 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len); 4180 if (!ret) { 4181 drop_nlink(&inode->vfs_inode); 4182 ret = btrfs_update_inode(trans, root, &inode->vfs_inode); 4183 } 4184 return ret; 4185 } 4186 4187 /* 4188 * helper to start transaction for unlink and rmdir. 4189 * 4190 * unlink and rmdir are special in btrfs, they do not always free space, so 4191 * if we cannot make our reservations the normal way try and see if there is 4192 * plenty of slack room in the global reserve to migrate, otherwise we cannot 4193 * allow the unlink to occur. 4194 */ 4195 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) 4196 { 4197 struct btrfs_root *root = BTRFS_I(dir)->root; 4198 4199 /* 4200 * 1 for the possible orphan item 4201 * 1 for the dir item 4202 * 1 for the dir index 4203 * 1 for the inode ref 4204 * 1 for the inode 4205 */ 4206 return btrfs_start_transaction_fallback_global_rsv(root, 5, 5); 4207 } 4208 4209 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 4210 { 4211 struct btrfs_root *root = BTRFS_I(dir)->root; 4212 struct btrfs_trans_handle *trans; 4213 struct inode *inode = d_inode(dentry); 4214 int ret; 4215 4216 trans = __unlink_start_trans(dir); 4217 if (IS_ERR(trans)) 4218 return PTR_ERR(trans); 4219 4220 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4221 0); 4222 4223 ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4224 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4225 dentry->d_name.len); 4226 if (ret) 4227 goto out; 4228 4229 if (inode->i_nlink == 0) { 4230 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 4231 if (ret) 4232 goto out; 4233 } 4234 4235 out: 4236 btrfs_end_transaction(trans); 4237 btrfs_btree_balance_dirty(root->fs_info); 4238 return ret; 4239 } 4240 4241 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 4242 struct btrfs_root *root, 4243 struct inode *dir, u64 objectid, 4244 const char *name, int name_len) 4245 { 4246 struct btrfs_fs_info *fs_info = root->fs_info; 4247 struct btrfs_path *path; 4248 struct extent_buffer *leaf; 4249 struct btrfs_dir_item *di; 4250 struct btrfs_key key; 4251 u64 index; 4252 int ret; 4253 u64 dir_ino = btrfs_ino(BTRFS_I(dir)); 4254 4255 path = btrfs_alloc_path(); 4256 if (!path) 4257 return -ENOMEM; 4258 4259 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4260 name, name_len, -1); 4261 if (IS_ERR_OR_NULL(di)) { 4262 if (!di) 4263 ret = -ENOENT; 4264 else 4265 ret = PTR_ERR(di); 4266 goto out; 4267 } 4268 4269 leaf = path->nodes[0]; 4270 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4271 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 4272 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4273 if (ret) { 4274 btrfs_abort_transaction(trans, ret); 4275 goto out; 4276 } 4277 btrfs_release_path(path); 4278 4279 ret = btrfs_del_root_ref(trans, fs_info, objectid, 4280 root->root_key.objectid, dir_ino, 4281 &index, name, name_len); 4282 if (ret < 0) { 4283 if (ret != -ENOENT) { 4284 btrfs_abort_transaction(trans, ret); 4285 goto out; 4286 } 4287 di = btrfs_search_dir_index_item(root, path, dir_ino, 4288 name, name_len); 4289 if (IS_ERR_OR_NULL(di)) { 4290 if (!di) 4291 ret = -ENOENT; 4292 else 4293 ret = PTR_ERR(di); 4294 btrfs_abort_transaction(trans, ret); 4295 goto out; 4296 } 4297 4298 leaf = path->nodes[0]; 4299 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4300 btrfs_release_path(path); 4301 index = key.offset; 4302 } 4303 btrfs_release_path(path); 4304 4305 ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index); 4306 if (ret) { 4307 btrfs_abort_transaction(trans, ret); 4308 goto out; 4309 } 4310 4311 btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2); 4312 inode_inc_iversion(dir); 4313 dir->i_mtime = dir->i_ctime = current_time(dir); 4314 ret = btrfs_update_inode_fallback(trans, root, dir); 4315 if (ret) 4316 btrfs_abort_transaction(trans, ret); 4317 out: 4318 btrfs_free_path(path); 4319 return ret; 4320 } 4321 4322 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4323 { 4324 struct inode *inode = d_inode(dentry); 4325 int err = 0; 4326 struct btrfs_root *root = BTRFS_I(dir)->root; 4327 struct btrfs_trans_handle *trans; 4328 u64 last_unlink_trans; 4329 4330 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4331 return -ENOTEMPTY; 4332 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) 4333 return -EPERM; 4334 4335 trans = __unlink_start_trans(dir); 4336 if (IS_ERR(trans)) 4337 return PTR_ERR(trans); 4338 4339 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4340 err = btrfs_unlink_subvol(trans, root, dir, 4341 BTRFS_I(inode)->location.objectid, 4342 dentry->d_name.name, 4343 dentry->d_name.len); 4344 goto out; 4345 } 4346 4347 err = btrfs_orphan_add(trans, BTRFS_I(inode)); 4348 if (err) 4349 goto out; 4350 4351 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; 4352 4353 /* now the directory is empty */ 4354 err = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4355 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4356 dentry->d_name.len); 4357 if (!err) { 4358 btrfs_i_size_write(BTRFS_I(inode), 0); 4359 /* 4360 * Propagate the last_unlink_trans value of the deleted dir to 4361 * its parent directory. This is to prevent an unrecoverable 4362 * log tree in the case we do something like this: 4363 * 1) create dir foo 4364 * 2) create snapshot under dir foo 4365 * 3) delete the snapshot 4366 * 4) rmdir foo 4367 * 5) mkdir foo 4368 * 6) fsync foo or some file inside foo 4369 */ 4370 if (last_unlink_trans >= trans->transid) 4371 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; 4372 } 4373 out: 4374 btrfs_end_transaction(trans); 4375 btrfs_btree_balance_dirty(root->fs_info); 4376 4377 return err; 4378 } 4379 4380 static int truncate_space_check(struct btrfs_trans_handle *trans, 4381 struct btrfs_root *root, 4382 u64 bytes_deleted) 4383 { 4384 struct btrfs_fs_info *fs_info = root->fs_info; 4385 int ret; 4386 4387 /* 4388 * This is only used to apply pressure to the enospc system, we don't 4389 * intend to use this reservation at all. 4390 */ 4391 bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted); 4392 bytes_deleted *= fs_info->nodesize; 4393 ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv, 4394 bytes_deleted, BTRFS_RESERVE_NO_FLUSH); 4395 if (!ret) { 4396 trace_btrfs_space_reservation(fs_info, "transaction", 4397 trans->transid, 4398 bytes_deleted, 1); 4399 trans->bytes_reserved += bytes_deleted; 4400 } 4401 return ret; 4402 4403 } 4404 4405 /* 4406 * Return this if we need to call truncate_block for the last bit of the 4407 * truncate. 4408 */ 4409 #define NEED_TRUNCATE_BLOCK 1 4410 4411 /* 4412 * this can truncate away extent items, csum items and directory items. 4413 * It starts at a high offset and removes keys until it can't find 4414 * any higher than new_size 4415 * 4416 * csum items that cross the new i_size are truncated to the new size 4417 * as well. 4418 * 4419 * min_type is the minimum key type to truncate down to. If set to 0, this 4420 * will kill all the items on this inode, including the INODE_ITEM_KEY. 4421 */ 4422 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, 4423 struct btrfs_root *root, 4424 struct inode *inode, 4425 u64 new_size, u32 min_type) 4426 { 4427 struct btrfs_fs_info *fs_info = root->fs_info; 4428 struct btrfs_path *path; 4429 struct extent_buffer *leaf; 4430 struct btrfs_file_extent_item *fi; 4431 struct btrfs_key key; 4432 struct btrfs_key found_key; 4433 u64 extent_start = 0; 4434 u64 extent_num_bytes = 0; 4435 u64 extent_offset = 0; 4436 u64 item_end = 0; 4437 u64 last_size = new_size; 4438 u32 found_type = (u8)-1; 4439 int found_extent; 4440 int del_item; 4441 int pending_del_nr = 0; 4442 int pending_del_slot = 0; 4443 int extent_type = -1; 4444 int ret; 4445 int err = 0; 4446 u64 ino = btrfs_ino(BTRFS_I(inode)); 4447 u64 bytes_deleted = 0; 4448 bool be_nice = false; 4449 bool should_throttle = false; 4450 bool should_end = false; 4451 4452 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); 4453 4454 /* 4455 * for non-free space inodes and ref cows, we want to back off from 4456 * time to time 4457 */ 4458 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) && 4459 test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4460 be_nice = true; 4461 4462 path = btrfs_alloc_path(); 4463 if (!path) 4464 return -ENOMEM; 4465 path->reada = READA_BACK; 4466 4467 /* 4468 * We want to drop from the next block forward in case this new size is 4469 * not block aligned since we will be keeping the last block of the 4470 * extent just the way it is. 4471 */ 4472 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4473 root == fs_info->tree_root) 4474 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size, 4475 fs_info->sectorsize), 4476 (u64)-1, 0); 4477 4478 /* 4479 * This function is also used to drop the items in the log tree before 4480 * we relog the inode, so if root != BTRFS_I(inode)->root, it means 4481 * it is used to drop the loged items. So we shouldn't kill the delayed 4482 * items. 4483 */ 4484 if (min_type == 0 && root == BTRFS_I(inode)->root) 4485 btrfs_kill_delayed_inode_items(BTRFS_I(inode)); 4486 4487 key.objectid = ino; 4488 key.offset = (u64)-1; 4489 key.type = (u8)-1; 4490 4491 search_again: 4492 /* 4493 * with a 16K leaf size and 128MB extents, you can actually queue 4494 * up a huge file in a single leaf. Most of the time that 4495 * bytes_deleted is > 0, it will be huge by the time we get here 4496 */ 4497 if (be_nice && bytes_deleted > SZ_32M) { 4498 if (btrfs_should_end_transaction(trans)) { 4499 err = -EAGAIN; 4500 goto error; 4501 } 4502 } 4503 4504 4505 path->leave_spinning = 1; 4506 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 4507 if (ret < 0) { 4508 err = ret; 4509 goto out; 4510 } 4511 4512 if (ret > 0) { 4513 /* there are no items in the tree for us to truncate, we're 4514 * done 4515 */ 4516 if (path->slots[0] == 0) 4517 goto out; 4518 path->slots[0]--; 4519 } 4520 4521 while (1) { 4522 fi = NULL; 4523 leaf = path->nodes[0]; 4524 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4525 found_type = found_key.type; 4526 4527 if (found_key.objectid != ino) 4528 break; 4529 4530 if (found_type < min_type) 4531 break; 4532 4533 item_end = found_key.offset; 4534 if (found_type == BTRFS_EXTENT_DATA_KEY) { 4535 fi = btrfs_item_ptr(leaf, path->slots[0], 4536 struct btrfs_file_extent_item); 4537 extent_type = btrfs_file_extent_type(leaf, fi); 4538 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4539 item_end += 4540 btrfs_file_extent_num_bytes(leaf, fi); 4541 4542 trace_btrfs_truncate_show_fi_regular( 4543 BTRFS_I(inode), leaf, fi, 4544 found_key.offset); 4545 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4546 item_end += btrfs_file_extent_inline_len(leaf, 4547 path->slots[0], fi); 4548 4549 trace_btrfs_truncate_show_fi_inline( 4550 BTRFS_I(inode), leaf, fi, path->slots[0], 4551 found_key.offset); 4552 } 4553 item_end--; 4554 } 4555 if (found_type > min_type) { 4556 del_item = 1; 4557 } else { 4558 if (item_end < new_size) 4559 break; 4560 if (found_key.offset >= new_size) 4561 del_item = 1; 4562 else 4563 del_item = 0; 4564 } 4565 found_extent = 0; 4566 /* FIXME, shrink the extent if the ref count is only 1 */ 4567 if (found_type != BTRFS_EXTENT_DATA_KEY) 4568 goto delete; 4569 4570 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4571 u64 num_dec; 4572 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); 4573 if (!del_item) { 4574 u64 orig_num_bytes = 4575 btrfs_file_extent_num_bytes(leaf, fi); 4576 extent_num_bytes = ALIGN(new_size - 4577 found_key.offset, 4578 fs_info->sectorsize); 4579 btrfs_set_file_extent_num_bytes(leaf, fi, 4580 extent_num_bytes); 4581 num_dec = (orig_num_bytes - 4582 extent_num_bytes); 4583 if (test_bit(BTRFS_ROOT_REF_COWS, 4584 &root->state) && 4585 extent_start != 0) 4586 inode_sub_bytes(inode, num_dec); 4587 btrfs_mark_buffer_dirty(leaf); 4588 } else { 4589 extent_num_bytes = 4590 btrfs_file_extent_disk_num_bytes(leaf, 4591 fi); 4592 extent_offset = found_key.offset - 4593 btrfs_file_extent_offset(leaf, fi); 4594 4595 /* FIXME blocksize != 4096 */ 4596 num_dec = btrfs_file_extent_num_bytes(leaf, fi); 4597 if (extent_start != 0) { 4598 found_extent = 1; 4599 if (test_bit(BTRFS_ROOT_REF_COWS, 4600 &root->state)) 4601 inode_sub_bytes(inode, num_dec); 4602 } 4603 } 4604 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4605 /* 4606 * we can't truncate inline items that have had 4607 * special encodings 4608 */ 4609 if (!del_item && 4610 btrfs_file_extent_encryption(leaf, fi) == 0 && 4611 btrfs_file_extent_other_encoding(leaf, fi) == 0 && 4612 btrfs_file_extent_compression(leaf, fi) == 0) { 4613 u32 size = (u32)(new_size - found_key.offset); 4614 4615 btrfs_set_file_extent_ram_bytes(leaf, fi, size); 4616 size = btrfs_file_extent_calc_inline_size(size); 4617 btrfs_truncate_item(root->fs_info, path, size, 1); 4618 } else if (!del_item) { 4619 /* 4620 * We have to bail so the last_size is set to 4621 * just before this extent. 4622 */ 4623 err = NEED_TRUNCATE_BLOCK; 4624 break; 4625 } 4626 4627 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4628 inode_sub_bytes(inode, item_end + 1 - new_size); 4629 } 4630 delete: 4631 if (del_item) 4632 last_size = found_key.offset; 4633 else 4634 last_size = new_size; 4635 if (del_item) { 4636 if (!pending_del_nr) { 4637 /* no pending yet, add ourselves */ 4638 pending_del_slot = path->slots[0]; 4639 pending_del_nr = 1; 4640 } else if (pending_del_nr && 4641 path->slots[0] + 1 == pending_del_slot) { 4642 /* hop on the pending chunk */ 4643 pending_del_nr++; 4644 pending_del_slot = path->slots[0]; 4645 } else { 4646 BUG(); 4647 } 4648 } else { 4649 break; 4650 } 4651 should_throttle = false; 4652 4653 if (found_extent && 4654 (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4655 root == fs_info->tree_root)) { 4656 btrfs_set_path_blocking(path); 4657 bytes_deleted += extent_num_bytes; 4658 ret = btrfs_free_extent(trans, root, extent_start, 4659 extent_num_bytes, 0, 4660 btrfs_header_owner(leaf), 4661 ino, extent_offset); 4662 BUG_ON(ret); 4663 if (btrfs_should_throttle_delayed_refs(trans, fs_info)) 4664 btrfs_async_run_delayed_refs(fs_info, 4665 trans->delayed_ref_updates * 2, 4666 trans->transid, 0); 4667 if (be_nice) { 4668 if (truncate_space_check(trans, root, 4669 extent_num_bytes)) { 4670 should_end = true; 4671 } 4672 if (btrfs_should_throttle_delayed_refs(trans, 4673 fs_info)) 4674 should_throttle = true; 4675 } 4676 } 4677 4678 if (found_type == BTRFS_INODE_ITEM_KEY) 4679 break; 4680 4681 if (path->slots[0] == 0 || 4682 path->slots[0] != pending_del_slot || 4683 should_throttle || should_end) { 4684 if (pending_del_nr) { 4685 ret = btrfs_del_items(trans, root, path, 4686 pending_del_slot, 4687 pending_del_nr); 4688 if (ret) { 4689 btrfs_abort_transaction(trans, ret); 4690 goto error; 4691 } 4692 pending_del_nr = 0; 4693 } 4694 btrfs_release_path(path); 4695 if (should_throttle) { 4696 unsigned long updates = trans->delayed_ref_updates; 4697 if (updates) { 4698 trans->delayed_ref_updates = 0; 4699 ret = btrfs_run_delayed_refs(trans, 4700 updates * 2); 4701 if (ret && !err) 4702 err = ret; 4703 } 4704 } 4705 /* 4706 * if we failed to refill our space rsv, bail out 4707 * and let the transaction restart 4708 */ 4709 if (should_end) { 4710 err = -EAGAIN; 4711 goto error; 4712 } 4713 goto search_again; 4714 } else { 4715 path->slots[0]--; 4716 } 4717 } 4718 out: 4719 if (pending_del_nr) { 4720 ret = btrfs_del_items(trans, root, path, pending_del_slot, 4721 pending_del_nr); 4722 if (ret) 4723 btrfs_abort_transaction(trans, ret); 4724 } 4725 error: 4726 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4727 ASSERT(last_size >= new_size); 4728 if (!err && last_size > new_size) 4729 last_size = new_size; 4730 btrfs_ordered_update_i_size(inode, last_size, NULL); 4731 } 4732 4733 btrfs_free_path(path); 4734 4735 if (be_nice && bytes_deleted > SZ_32M) { 4736 unsigned long updates = trans->delayed_ref_updates; 4737 if (updates) { 4738 trans->delayed_ref_updates = 0; 4739 ret = btrfs_run_delayed_refs(trans, updates * 2); 4740 if (ret && !err) 4741 err = ret; 4742 } 4743 } 4744 return err; 4745 } 4746 4747 /* 4748 * btrfs_truncate_block - read, zero a chunk and write a block 4749 * @inode - inode that we're zeroing 4750 * @from - the offset to start zeroing 4751 * @len - the length to zero, 0 to zero the entire range respective to the 4752 * offset 4753 * @front - zero up to the offset instead of from the offset on 4754 * 4755 * This will find the block for the "from" offset and cow the block and zero the 4756 * part we want to zero. This is used with truncate and hole punching. 4757 */ 4758 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len, 4759 int front) 4760 { 4761 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4762 struct address_space *mapping = inode->i_mapping; 4763 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4764 struct btrfs_ordered_extent *ordered; 4765 struct extent_state *cached_state = NULL; 4766 struct extent_changeset *data_reserved = NULL; 4767 char *kaddr; 4768 u32 blocksize = fs_info->sectorsize; 4769 pgoff_t index = from >> PAGE_SHIFT; 4770 unsigned offset = from & (blocksize - 1); 4771 struct page *page; 4772 gfp_t mask = btrfs_alloc_write_mask(mapping); 4773 int ret = 0; 4774 u64 block_start; 4775 u64 block_end; 4776 4777 if (IS_ALIGNED(offset, blocksize) && 4778 (!len || IS_ALIGNED(len, blocksize))) 4779 goto out; 4780 4781 block_start = round_down(from, blocksize); 4782 block_end = block_start + blocksize - 1; 4783 4784 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, 4785 block_start, blocksize); 4786 if (ret) 4787 goto out; 4788 4789 again: 4790 page = find_or_create_page(mapping, index, mask); 4791 if (!page) { 4792 btrfs_delalloc_release_space(inode, data_reserved, 4793 block_start, blocksize, true); 4794 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, true); 4795 ret = -ENOMEM; 4796 goto out; 4797 } 4798 4799 if (!PageUptodate(page)) { 4800 ret = btrfs_readpage(NULL, page); 4801 lock_page(page); 4802 if (page->mapping != mapping) { 4803 unlock_page(page); 4804 put_page(page); 4805 goto again; 4806 } 4807 if (!PageUptodate(page)) { 4808 ret = -EIO; 4809 goto out_unlock; 4810 } 4811 } 4812 wait_on_page_writeback(page); 4813 4814 lock_extent_bits(io_tree, block_start, block_end, &cached_state); 4815 set_page_extent_mapped(page); 4816 4817 ordered = btrfs_lookup_ordered_extent(inode, block_start); 4818 if (ordered) { 4819 unlock_extent_cached(io_tree, block_start, block_end, 4820 &cached_state); 4821 unlock_page(page); 4822 put_page(page); 4823 btrfs_start_ordered_extent(inode, ordered, 1); 4824 btrfs_put_ordered_extent(ordered); 4825 goto again; 4826 } 4827 4828 clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end, 4829 EXTENT_DIRTY | EXTENT_DELALLOC | 4830 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4831 0, 0, &cached_state); 4832 4833 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, 4834 &cached_state, 0); 4835 if (ret) { 4836 unlock_extent_cached(io_tree, block_start, block_end, 4837 &cached_state); 4838 goto out_unlock; 4839 } 4840 4841 if (offset != blocksize) { 4842 if (!len) 4843 len = blocksize - offset; 4844 kaddr = kmap(page); 4845 if (front) 4846 memset(kaddr + (block_start - page_offset(page)), 4847 0, offset); 4848 else 4849 memset(kaddr + (block_start - page_offset(page)) + offset, 4850 0, len); 4851 flush_dcache_page(page); 4852 kunmap(page); 4853 } 4854 ClearPageChecked(page); 4855 set_page_dirty(page); 4856 unlock_extent_cached(io_tree, block_start, block_end, &cached_state); 4857 4858 out_unlock: 4859 if (ret) 4860 btrfs_delalloc_release_space(inode, data_reserved, block_start, 4861 blocksize, true); 4862 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, (ret != 0)); 4863 unlock_page(page); 4864 put_page(page); 4865 out: 4866 extent_changeset_free(data_reserved); 4867 return ret; 4868 } 4869 4870 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode, 4871 u64 offset, u64 len) 4872 { 4873 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4874 struct btrfs_trans_handle *trans; 4875 int ret; 4876 4877 /* 4878 * Still need to make sure the inode looks like it's been updated so 4879 * that any holes get logged if we fsync. 4880 */ 4881 if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 4882 BTRFS_I(inode)->last_trans = fs_info->generation; 4883 BTRFS_I(inode)->last_sub_trans = root->log_transid; 4884 BTRFS_I(inode)->last_log_commit = root->last_log_commit; 4885 return 0; 4886 } 4887 4888 /* 4889 * 1 - for the one we're dropping 4890 * 1 - for the one we're adding 4891 * 1 - for updating the inode. 4892 */ 4893 trans = btrfs_start_transaction(root, 3); 4894 if (IS_ERR(trans)) 4895 return PTR_ERR(trans); 4896 4897 ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1); 4898 if (ret) { 4899 btrfs_abort_transaction(trans, ret); 4900 btrfs_end_transaction(trans); 4901 return ret; 4902 } 4903 4904 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)), 4905 offset, 0, 0, len, 0, len, 0, 0, 0); 4906 if (ret) 4907 btrfs_abort_transaction(trans, ret); 4908 else 4909 btrfs_update_inode(trans, root, inode); 4910 btrfs_end_transaction(trans); 4911 return ret; 4912 } 4913 4914 /* 4915 * This function puts in dummy file extents for the area we're creating a hole 4916 * for. So if we are truncating this file to a larger size we need to insert 4917 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 4918 * the range between oldsize and size 4919 */ 4920 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size) 4921 { 4922 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4923 struct btrfs_root *root = BTRFS_I(inode)->root; 4924 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4925 struct extent_map *em = NULL; 4926 struct extent_state *cached_state = NULL; 4927 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 4928 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 4929 u64 block_end = ALIGN(size, fs_info->sectorsize); 4930 u64 last_byte; 4931 u64 cur_offset; 4932 u64 hole_size; 4933 int err = 0; 4934 4935 /* 4936 * If our size started in the middle of a block we need to zero out the 4937 * rest of the block before we expand the i_size, otherwise we could 4938 * expose stale data. 4939 */ 4940 err = btrfs_truncate_block(inode, oldsize, 0, 0); 4941 if (err) 4942 return err; 4943 4944 if (size <= hole_start) 4945 return 0; 4946 4947 while (1) { 4948 struct btrfs_ordered_extent *ordered; 4949 4950 lock_extent_bits(io_tree, hole_start, block_end - 1, 4951 &cached_state); 4952 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start, 4953 block_end - hole_start); 4954 if (!ordered) 4955 break; 4956 unlock_extent_cached(io_tree, hole_start, block_end - 1, 4957 &cached_state); 4958 btrfs_start_ordered_extent(inode, ordered, 1); 4959 btrfs_put_ordered_extent(ordered); 4960 } 4961 4962 cur_offset = hole_start; 4963 while (1) { 4964 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset, 4965 block_end - cur_offset, 0); 4966 if (IS_ERR(em)) { 4967 err = PTR_ERR(em); 4968 em = NULL; 4969 break; 4970 } 4971 last_byte = min(extent_map_end(em), block_end); 4972 last_byte = ALIGN(last_byte, fs_info->sectorsize); 4973 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 4974 struct extent_map *hole_em; 4975 hole_size = last_byte - cur_offset; 4976 4977 err = maybe_insert_hole(root, inode, cur_offset, 4978 hole_size); 4979 if (err) 4980 break; 4981 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 4982 cur_offset + hole_size - 1, 0); 4983 hole_em = alloc_extent_map(); 4984 if (!hole_em) { 4985 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 4986 &BTRFS_I(inode)->runtime_flags); 4987 goto next; 4988 } 4989 hole_em->start = cur_offset; 4990 hole_em->len = hole_size; 4991 hole_em->orig_start = cur_offset; 4992 4993 hole_em->block_start = EXTENT_MAP_HOLE; 4994 hole_em->block_len = 0; 4995 hole_em->orig_block_len = 0; 4996 hole_em->ram_bytes = hole_size; 4997 hole_em->bdev = fs_info->fs_devices->latest_bdev; 4998 hole_em->compress_type = BTRFS_COMPRESS_NONE; 4999 hole_em->generation = fs_info->generation; 5000 5001 while (1) { 5002 write_lock(&em_tree->lock); 5003 err = add_extent_mapping(em_tree, hole_em, 1); 5004 write_unlock(&em_tree->lock); 5005 if (err != -EEXIST) 5006 break; 5007 btrfs_drop_extent_cache(BTRFS_I(inode), 5008 cur_offset, 5009 cur_offset + 5010 hole_size - 1, 0); 5011 } 5012 free_extent_map(hole_em); 5013 } 5014 next: 5015 free_extent_map(em); 5016 em = NULL; 5017 cur_offset = last_byte; 5018 if (cur_offset >= block_end) 5019 break; 5020 } 5021 free_extent_map(em); 5022 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state); 5023 return err; 5024 } 5025 5026 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 5027 { 5028 struct btrfs_root *root = BTRFS_I(inode)->root; 5029 struct btrfs_trans_handle *trans; 5030 loff_t oldsize = i_size_read(inode); 5031 loff_t newsize = attr->ia_size; 5032 int mask = attr->ia_valid; 5033 int ret; 5034 5035 /* 5036 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 5037 * special case where we need to update the times despite not having 5038 * these flags set. For all other operations the VFS set these flags 5039 * explicitly if it wants a timestamp update. 5040 */ 5041 if (newsize != oldsize) { 5042 inode_inc_iversion(inode); 5043 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) 5044 inode->i_ctime = inode->i_mtime = 5045 current_time(inode); 5046 } 5047 5048 if (newsize > oldsize) { 5049 /* 5050 * Don't do an expanding truncate while snapshotting is ongoing. 5051 * This is to ensure the snapshot captures a fully consistent 5052 * state of this file - if the snapshot captures this expanding 5053 * truncation, it must capture all writes that happened before 5054 * this truncation. 5055 */ 5056 btrfs_wait_for_snapshot_creation(root); 5057 ret = btrfs_cont_expand(inode, oldsize, newsize); 5058 if (ret) { 5059 btrfs_end_write_no_snapshotting(root); 5060 return ret; 5061 } 5062 5063 trans = btrfs_start_transaction(root, 1); 5064 if (IS_ERR(trans)) { 5065 btrfs_end_write_no_snapshotting(root); 5066 return PTR_ERR(trans); 5067 } 5068 5069 i_size_write(inode, newsize); 5070 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL); 5071 pagecache_isize_extended(inode, oldsize, newsize); 5072 ret = btrfs_update_inode(trans, root, inode); 5073 btrfs_end_write_no_snapshotting(root); 5074 btrfs_end_transaction(trans); 5075 } else { 5076 5077 /* 5078 * We're truncating a file that used to have good data down to 5079 * zero. Make sure it gets into the ordered flush list so that 5080 * any new writes get down to disk quickly. 5081 */ 5082 if (newsize == 0) 5083 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE, 5084 &BTRFS_I(inode)->runtime_flags); 5085 5086 /* 5087 * 1 for the orphan item we're going to add 5088 * 1 for the orphan item deletion. 5089 */ 5090 trans = btrfs_start_transaction(root, 2); 5091 if (IS_ERR(trans)) 5092 return PTR_ERR(trans); 5093 5094 /* 5095 * We need to do this in case we fail at _any_ point during the 5096 * actual truncate. Once we do the truncate_setsize we could 5097 * invalidate pages which forces any outstanding ordered io to 5098 * be instantly completed which will give us extents that need 5099 * to be truncated. If we fail to get an orphan inode down we 5100 * could have left over extents that were never meant to live, 5101 * so we need to guarantee from this point on that everything 5102 * will be consistent. 5103 */ 5104 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 5105 btrfs_end_transaction(trans); 5106 if (ret) 5107 return ret; 5108 5109 /* we don't support swapfiles, so vmtruncate shouldn't fail */ 5110 truncate_setsize(inode, newsize); 5111 5112 /* Disable nonlocked read DIO to avoid the end less truncate */ 5113 btrfs_inode_block_unlocked_dio(BTRFS_I(inode)); 5114 inode_dio_wait(inode); 5115 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode)); 5116 5117 ret = btrfs_truncate(inode, newsize == oldsize); 5118 if (ret && inode->i_nlink) { 5119 int err; 5120 5121 /* To get a stable disk_i_size */ 5122 err = btrfs_wait_ordered_range(inode, 0, (u64)-1); 5123 if (err) { 5124 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5125 return err; 5126 } 5127 5128 /* 5129 * failed to truncate, disk_i_size is only adjusted down 5130 * as we remove extents, so it should represent the true 5131 * size of the inode, so reset the in memory size and 5132 * delete our orphan entry. 5133 */ 5134 trans = btrfs_join_transaction(root); 5135 if (IS_ERR(trans)) { 5136 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5137 return ret; 5138 } 5139 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 5140 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 5141 if (err) 5142 btrfs_abort_transaction(trans, err); 5143 btrfs_end_transaction(trans); 5144 } 5145 } 5146 5147 return ret; 5148 } 5149 5150 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) 5151 { 5152 struct inode *inode = d_inode(dentry); 5153 struct btrfs_root *root = BTRFS_I(inode)->root; 5154 int err; 5155 5156 if (btrfs_root_readonly(root)) 5157 return -EROFS; 5158 5159 err = setattr_prepare(dentry, attr); 5160 if (err) 5161 return err; 5162 5163 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 5164 err = btrfs_setsize(inode, attr); 5165 if (err) 5166 return err; 5167 } 5168 5169 if (attr->ia_valid) { 5170 setattr_copy(inode, attr); 5171 inode_inc_iversion(inode); 5172 err = btrfs_dirty_inode(inode); 5173 5174 if (!err && attr->ia_valid & ATTR_MODE) 5175 err = posix_acl_chmod(inode, inode->i_mode); 5176 } 5177 5178 return err; 5179 } 5180 5181 /* 5182 * While truncating the inode pages during eviction, we get the VFS calling 5183 * btrfs_invalidatepage() against each page of the inode. This is slow because 5184 * the calls to btrfs_invalidatepage() result in a huge amount of calls to 5185 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting 5186 * extent_state structures over and over, wasting lots of time. 5187 * 5188 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all 5189 * those expensive operations on a per page basis and do only the ordered io 5190 * finishing, while we release here the extent_map and extent_state structures, 5191 * without the excessive merging and splitting. 5192 */ 5193 static void evict_inode_truncate_pages(struct inode *inode) 5194 { 5195 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 5196 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; 5197 struct rb_node *node; 5198 5199 ASSERT(inode->i_state & I_FREEING); 5200 truncate_inode_pages_final(&inode->i_data); 5201 5202 write_lock(&map_tree->lock); 5203 while (!RB_EMPTY_ROOT(&map_tree->map)) { 5204 struct extent_map *em; 5205 5206 node = rb_first(&map_tree->map); 5207 em = rb_entry(node, struct extent_map, rb_node); 5208 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 5209 clear_bit(EXTENT_FLAG_LOGGING, &em->flags); 5210 remove_extent_mapping(map_tree, em); 5211 free_extent_map(em); 5212 if (need_resched()) { 5213 write_unlock(&map_tree->lock); 5214 cond_resched(); 5215 write_lock(&map_tree->lock); 5216 } 5217 } 5218 write_unlock(&map_tree->lock); 5219 5220 /* 5221 * Keep looping until we have no more ranges in the io tree. 5222 * We can have ongoing bios started by readpages (called from readahead) 5223 * that have their endio callback (extent_io.c:end_bio_extent_readpage) 5224 * still in progress (unlocked the pages in the bio but did not yet 5225 * unlocked the ranges in the io tree). Therefore this means some 5226 * ranges can still be locked and eviction started because before 5227 * submitting those bios, which are executed by a separate task (work 5228 * queue kthread), inode references (inode->i_count) were not taken 5229 * (which would be dropped in the end io callback of each bio). 5230 * Therefore here we effectively end up waiting for those bios and 5231 * anyone else holding locked ranges without having bumped the inode's 5232 * reference count - if we don't do it, when they access the inode's 5233 * io_tree to unlock a range it may be too late, leading to an 5234 * use-after-free issue. 5235 */ 5236 spin_lock(&io_tree->lock); 5237 while (!RB_EMPTY_ROOT(&io_tree->state)) { 5238 struct extent_state *state; 5239 struct extent_state *cached_state = NULL; 5240 u64 start; 5241 u64 end; 5242 5243 node = rb_first(&io_tree->state); 5244 state = rb_entry(node, struct extent_state, rb_node); 5245 start = state->start; 5246 end = state->end; 5247 spin_unlock(&io_tree->lock); 5248 5249 lock_extent_bits(io_tree, start, end, &cached_state); 5250 5251 /* 5252 * If still has DELALLOC flag, the extent didn't reach disk, 5253 * and its reserved space won't be freed by delayed_ref. 5254 * So we need to free its reserved space here. 5255 * (Refer to comment in btrfs_invalidatepage, case 2) 5256 * 5257 * Note, end is the bytenr of last byte, so we need + 1 here. 5258 */ 5259 if (state->state & EXTENT_DELALLOC) 5260 btrfs_qgroup_free_data(inode, NULL, start, end - start + 1); 5261 5262 clear_extent_bit(io_tree, start, end, 5263 EXTENT_LOCKED | EXTENT_DIRTY | 5264 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 5265 EXTENT_DEFRAG, 1, 1, &cached_state); 5266 5267 cond_resched(); 5268 spin_lock(&io_tree->lock); 5269 } 5270 spin_unlock(&io_tree->lock); 5271 } 5272 5273 void btrfs_evict_inode(struct inode *inode) 5274 { 5275 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5276 struct btrfs_trans_handle *trans; 5277 struct btrfs_root *root = BTRFS_I(inode)->root; 5278 struct btrfs_block_rsv *rsv, *global_rsv; 5279 int steal_from_global = 0; 5280 u64 min_size; 5281 int ret; 5282 5283 trace_btrfs_inode_evict(inode); 5284 5285 if (!root) { 5286 clear_inode(inode); 5287 return; 5288 } 5289 5290 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 5291 5292 evict_inode_truncate_pages(inode); 5293 5294 if (inode->i_nlink && 5295 ((btrfs_root_refs(&root->root_item) != 0 && 5296 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 5297 btrfs_is_free_space_inode(BTRFS_I(inode)))) 5298 goto no_delete; 5299 5300 if (is_bad_inode(inode)) { 5301 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5302 goto no_delete; 5303 } 5304 /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */ 5305 if (!special_file(inode->i_mode)) 5306 btrfs_wait_ordered_range(inode, 0, (u64)-1); 5307 5308 btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1); 5309 5310 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 5311 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 5312 &BTRFS_I(inode)->runtime_flags)); 5313 goto no_delete; 5314 } 5315 5316 if (inode->i_nlink > 0) { 5317 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5318 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 5319 goto no_delete; 5320 } 5321 5322 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); 5323 if (ret) { 5324 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5325 goto no_delete; 5326 } 5327 5328 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 5329 if (!rsv) { 5330 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5331 goto no_delete; 5332 } 5333 rsv->size = min_size; 5334 rsv->failfast = 1; 5335 global_rsv = &fs_info->global_block_rsv; 5336 5337 btrfs_i_size_write(BTRFS_I(inode), 0); 5338 5339 /* 5340 * This is a bit simpler than btrfs_truncate since we've already 5341 * reserved our space for our orphan item in the unlink, so we just 5342 * need to reserve some slack space in case we add bytes and update 5343 * inode item when doing the truncate. 5344 */ 5345 while (1) { 5346 ret = btrfs_block_rsv_refill(root, rsv, min_size, 5347 BTRFS_RESERVE_FLUSH_LIMIT); 5348 5349 /* 5350 * Try and steal from the global reserve since we will 5351 * likely not use this space anyway, we want to try as 5352 * hard as possible to get this to work. 5353 */ 5354 if (ret) 5355 steal_from_global++; 5356 else 5357 steal_from_global = 0; 5358 ret = 0; 5359 5360 /* 5361 * steal_from_global == 0: we reserved stuff, hooray! 5362 * steal_from_global == 1: we didn't reserve stuff, boo! 5363 * steal_from_global == 2: we've committed, still not a lot of 5364 * room but maybe we'll have room in the global reserve this 5365 * time. 5366 * steal_from_global == 3: abandon all hope! 5367 */ 5368 if (steal_from_global > 2) { 5369 btrfs_warn(fs_info, 5370 "Could not get space for a delete, will truncate on mount %d", 5371 ret); 5372 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5373 btrfs_free_block_rsv(fs_info, rsv); 5374 goto no_delete; 5375 } 5376 5377 trans = btrfs_join_transaction(root); 5378 if (IS_ERR(trans)) { 5379 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5380 btrfs_free_block_rsv(fs_info, rsv); 5381 goto no_delete; 5382 } 5383 5384 /* 5385 * We can't just steal from the global reserve, we need to make 5386 * sure there is room to do it, if not we need to commit and try 5387 * again. 5388 */ 5389 if (steal_from_global) { 5390 if (!btrfs_check_space_for_delayed_refs(trans, fs_info)) 5391 ret = btrfs_block_rsv_migrate(global_rsv, rsv, 5392 min_size, 0); 5393 else 5394 ret = -ENOSPC; 5395 } 5396 5397 /* 5398 * Couldn't steal from the global reserve, we have too much 5399 * pending stuff built up, commit the transaction and try it 5400 * again. 5401 */ 5402 if (ret) { 5403 ret = btrfs_commit_transaction(trans); 5404 if (ret) { 5405 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5406 btrfs_free_block_rsv(fs_info, rsv); 5407 goto no_delete; 5408 } 5409 continue; 5410 } else { 5411 steal_from_global = 0; 5412 } 5413 5414 trans->block_rsv = rsv; 5415 5416 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0); 5417 if (ret != -ENOSPC && ret != -EAGAIN) 5418 break; 5419 5420 trans->block_rsv = &fs_info->trans_block_rsv; 5421 btrfs_end_transaction(trans); 5422 trans = NULL; 5423 btrfs_btree_balance_dirty(fs_info); 5424 } 5425 5426 btrfs_free_block_rsv(fs_info, rsv); 5427 5428 /* 5429 * Errors here aren't a big deal, it just means we leave orphan items 5430 * in the tree. They will be cleaned up on the next mount. 5431 */ 5432 if (ret == 0) { 5433 trans->block_rsv = root->orphan_block_rsv; 5434 btrfs_orphan_del(trans, BTRFS_I(inode)); 5435 } else { 5436 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5437 } 5438 5439 trans->block_rsv = &fs_info->trans_block_rsv; 5440 if (!(root == fs_info->tree_root || 5441 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)) 5442 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode))); 5443 5444 btrfs_end_transaction(trans); 5445 btrfs_btree_balance_dirty(fs_info); 5446 no_delete: 5447 btrfs_remove_delayed_node(BTRFS_I(inode)); 5448 clear_inode(inode); 5449 } 5450 5451 /* 5452 * this returns the key found in the dir entry in the location pointer. 5453 * If no dir entries were found, returns -ENOENT. 5454 * If found a corrupted location in dir entry, returns -EUCLEAN. 5455 */ 5456 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry, 5457 struct btrfs_key *location) 5458 { 5459 const char *name = dentry->d_name.name; 5460 int namelen = dentry->d_name.len; 5461 struct btrfs_dir_item *di; 5462 struct btrfs_path *path; 5463 struct btrfs_root *root = BTRFS_I(dir)->root; 5464 int ret = 0; 5465 5466 path = btrfs_alloc_path(); 5467 if (!path) 5468 return -ENOMEM; 5469 5470 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)), 5471 name, namelen, 0); 5472 if (!di) { 5473 ret = -ENOENT; 5474 goto out; 5475 } 5476 if (IS_ERR(di)) { 5477 ret = PTR_ERR(di); 5478 goto out; 5479 } 5480 5481 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5482 if (location->type != BTRFS_INODE_ITEM_KEY && 5483 location->type != BTRFS_ROOT_ITEM_KEY) { 5484 ret = -EUCLEAN; 5485 btrfs_warn(root->fs_info, 5486 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))", 5487 __func__, name, btrfs_ino(BTRFS_I(dir)), 5488 location->objectid, location->type, location->offset); 5489 } 5490 out: 5491 btrfs_free_path(path); 5492 return ret; 5493 } 5494 5495 /* 5496 * when we hit a tree root in a directory, the btrfs part of the inode 5497 * needs to be changed to reflect the root directory of the tree root. This 5498 * is kind of like crossing a mount point. 5499 */ 5500 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5501 struct inode *dir, 5502 struct dentry *dentry, 5503 struct btrfs_key *location, 5504 struct btrfs_root **sub_root) 5505 { 5506 struct btrfs_path *path; 5507 struct btrfs_root *new_root; 5508 struct btrfs_root_ref *ref; 5509 struct extent_buffer *leaf; 5510 struct btrfs_key key; 5511 int ret; 5512 int err = 0; 5513 5514 path = btrfs_alloc_path(); 5515 if (!path) { 5516 err = -ENOMEM; 5517 goto out; 5518 } 5519 5520 err = -ENOENT; 5521 key.objectid = BTRFS_I(dir)->root->root_key.objectid; 5522 key.type = BTRFS_ROOT_REF_KEY; 5523 key.offset = location->objectid; 5524 5525 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5526 if (ret) { 5527 if (ret < 0) 5528 err = ret; 5529 goto out; 5530 } 5531 5532 leaf = path->nodes[0]; 5533 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5534 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) || 5535 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len) 5536 goto out; 5537 5538 ret = memcmp_extent_buffer(leaf, dentry->d_name.name, 5539 (unsigned long)(ref + 1), 5540 dentry->d_name.len); 5541 if (ret) 5542 goto out; 5543 5544 btrfs_release_path(path); 5545 5546 new_root = btrfs_read_fs_root_no_name(fs_info, location); 5547 if (IS_ERR(new_root)) { 5548 err = PTR_ERR(new_root); 5549 goto out; 5550 } 5551 5552 *sub_root = new_root; 5553 location->objectid = btrfs_root_dirid(&new_root->root_item); 5554 location->type = BTRFS_INODE_ITEM_KEY; 5555 location->offset = 0; 5556 err = 0; 5557 out: 5558 btrfs_free_path(path); 5559 return err; 5560 } 5561 5562 static void inode_tree_add(struct inode *inode) 5563 { 5564 struct btrfs_root *root = BTRFS_I(inode)->root; 5565 struct btrfs_inode *entry; 5566 struct rb_node **p; 5567 struct rb_node *parent; 5568 struct rb_node *new = &BTRFS_I(inode)->rb_node; 5569 u64 ino = btrfs_ino(BTRFS_I(inode)); 5570 5571 if (inode_unhashed(inode)) 5572 return; 5573 parent = NULL; 5574 spin_lock(&root->inode_lock); 5575 p = &root->inode_tree.rb_node; 5576 while (*p) { 5577 parent = *p; 5578 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5579 5580 if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5581 p = &parent->rb_left; 5582 else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5583 p = &parent->rb_right; 5584 else { 5585 WARN_ON(!(entry->vfs_inode.i_state & 5586 (I_WILL_FREE | I_FREEING))); 5587 rb_replace_node(parent, new, &root->inode_tree); 5588 RB_CLEAR_NODE(parent); 5589 spin_unlock(&root->inode_lock); 5590 return; 5591 } 5592 } 5593 rb_link_node(new, parent, p); 5594 rb_insert_color(new, &root->inode_tree); 5595 spin_unlock(&root->inode_lock); 5596 } 5597 5598 static void inode_tree_del(struct inode *inode) 5599 { 5600 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5601 struct btrfs_root *root = BTRFS_I(inode)->root; 5602 int empty = 0; 5603 5604 spin_lock(&root->inode_lock); 5605 if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) { 5606 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree); 5607 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node); 5608 empty = RB_EMPTY_ROOT(&root->inode_tree); 5609 } 5610 spin_unlock(&root->inode_lock); 5611 5612 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5613 synchronize_srcu(&fs_info->subvol_srcu); 5614 spin_lock(&root->inode_lock); 5615 empty = RB_EMPTY_ROOT(&root->inode_tree); 5616 spin_unlock(&root->inode_lock); 5617 if (empty) 5618 btrfs_add_dead_root(root); 5619 } 5620 } 5621 5622 void btrfs_invalidate_inodes(struct btrfs_root *root) 5623 { 5624 struct btrfs_fs_info *fs_info = root->fs_info; 5625 struct rb_node *node; 5626 struct rb_node *prev; 5627 struct btrfs_inode *entry; 5628 struct inode *inode; 5629 u64 objectid = 0; 5630 5631 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 5632 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 5633 5634 spin_lock(&root->inode_lock); 5635 again: 5636 node = root->inode_tree.rb_node; 5637 prev = NULL; 5638 while (node) { 5639 prev = node; 5640 entry = rb_entry(node, struct btrfs_inode, rb_node); 5641 5642 if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5643 node = node->rb_left; 5644 else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5645 node = node->rb_right; 5646 else 5647 break; 5648 } 5649 if (!node) { 5650 while (prev) { 5651 entry = rb_entry(prev, struct btrfs_inode, rb_node); 5652 if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) { 5653 node = prev; 5654 break; 5655 } 5656 prev = rb_next(prev); 5657 } 5658 } 5659 while (node) { 5660 entry = rb_entry(node, struct btrfs_inode, rb_node); 5661 objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1; 5662 inode = igrab(&entry->vfs_inode); 5663 if (inode) { 5664 spin_unlock(&root->inode_lock); 5665 if (atomic_read(&inode->i_count) > 1) 5666 d_prune_aliases(inode); 5667 /* 5668 * btrfs_drop_inode will have it removed from 5669 * the inode cache when its usage count 5670 * hits zero. 5671 */ 5672 iput(inode); 5673 cond_resched(); 5674 spin_lock(&root->inode_lock); 5675 goto again; 5676 } 5677 5678 if (cond_resched_lock(&root->inode_lock)) 5679 goto again; 5680 5681 node = rb_next(node); 5682 } 5683 spin_unlock(&root->inode_lock); 5684 } 5685 5686 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5687 { 5688 struct btrfs_iget_args *args = p; 5689 inode->i_ino = args->location->objectid; 5690 memcpy(&BTRFS_I(inode)->location, args->location, 5691 sizeof(*args->location)); 5692 BTRFS_I(inode)->root = args->root; 5693 return 0; 5694 } 5695 5696 static int btrfs_find_actor(struct inode *inode, void *opaque) 5697 { 5698 struct btrfs_iget_args *args = opaque; 5699 return args->location->objectid == BTRFS_I(inode)->location.objectid && 5700 args->root == BTRFS_I(inode)->root; 5701 } 5702 5703 static struct inode *btrfs_iget_locked(struct super_block *s, 5704 struct btrfs_key *location, 5705 struct btrfs_root *root) 5706 { 5707 struct inode *inode; 5708 struct btrfs_iget_args args; 5709 unsigned long hashval = btrfs_inode_hash(location->objectid, root); 5710 5711 args.location = location; 5712 args.root = root; 5713 5714 inode = iget5_locked(s, hashval, btrfs_find_actor, 5715 btrfs_init_locked_inode, 5716 (void *)&args); 5717 return inode; 5718 } 5719 5720 /* Get an inode object given its location and corresponding root. 5721 * Returns in *is_new if the inode was read from disk 5722 */ 5723 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location, 5724 struct btrfs_root *root, int *new) 5725 { 5726 struct inode *inode; 5727 5728 inode = btrfs_iget_locked(s, location, root); 5729 if (!inode) 5730 return ERR_PTR(-ENOMEM); 5731 5732 if (inode->i_state & I_NEW) { 5733 int ret; 5734 5735 ret = btrfs_read_locked_inode(inode); 5736 if (!is_bad_inode(inode)) { 5737 inode_tree_add(inode); 5738 unlock_new_inode(inode); 5739 if (new) 5740 *new = 1; 5741 } else { 5742 unlock_new_inode(inode); 5743 iput(inode); 5744 ASSERT(ret < 0); 5745 inode = ERR_PTR(ret < 0 ? ret : -ESTALE); 5746 } 5747 } 5748 5749 return inode; 5750 } 5751 5752 static struct inode *new_simple_dir(struct super_block *s, 5753 struct btrfs_key *key, 5754 struct btrfs_root *root) 5755 { 5756 struct inode *inode = new_inode(s); 5757 5758 if (!inode) 5759 return ERR_PTR(-ENOMEM); 5760 5761 BTRFS_I(inode)->root = root; 5762 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5763 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5764 5765 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5766 inode->i_op = &btrfs_dir_ro_inode_operations; 5767 inode->i_opflags &= ~IOP_XATTR; 5768 inode->i_fop = &simple_dir_operations; 5769 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5770 inode->i_mtime = current_time(inode); 5771 inode->i_atime = inode->i_mtime; 5772 inode->i_ctime = inode->i_mtime; 5773 BTRFS_I(inode)->i_otime = inode->i_mtime; 5774 5775 return inode; 5776 } 5777 5778 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5779 { 5780 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 5781 struct inode *inode; 5782 struct btrfs_root *root = BTRFS_I(dir)->root; 5783 struct btrfs_root *sub_root = root; 5784 struct btrfs_key location; 5785 int index; 5786 int ret = 0; 5787 5788 if (dentry->d_name.len > BTRFS_NAME_LEN) 5789 return ERR_PTR(-ENAMETOOLONG); 5790 5791 ret = btrfs_inode_by_name(dir, dentry, &location); 5792 if (ret < 0) 5793 return ERR_PTR(ret); 5794 5795 if (location.type == BTRFS_INODE_ITEM_KEY) { 5796 inode = btrfs_iget(dir->i_sb, &location, root, NULL); 5797 return inode; 5798 } 5799 5800 index = srcu_read_lock(&fs_info->subvol_srcu); 5801 ret = fixup_tree_root_location(fs_info, dir, dentry, 5802 &location, &sub_root); 5803 if (ret < 0) { 5804 if (ret != -ENOENT) 5805 inode = ERR_PTR(ret); 5806 else 5807 inode = new_simple_dir(dir->i_sb, &location, sub_root); 5808 } else { 5809 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL); 5810 } 5811 srcu_read_unlock(&fs_info->subvol_srcu, index); 5812 5813 if (!IS_ERR(inode) && root != sub_root) { 5814 down_read(&fs_info->cleanup_work_sem); 5815 if (!sb_rdonly(inode->i_sb)) 5816 ret = btrfs_orphan_cleanup(sub_root); 5817 up_read(&fs_info->cleanup_work_sem); 5818 if (ret) { 5819 iput(inode); 5820 inode = ERR_PTR(ret); 5821 } 5822 } 5823 5824 return inode; 5825 } 5826 5827 static int btrfs_dentry_delete(const struct dentry *dentry) 5828 { 5829 struct btrfs_root *root; 5830 struct inode *inode = d_inode(dentry); 5831 5832 if (!inode && !IS_ROOT(dentry)) 5833 inode = d_inode(dentry->d_parent); 5834 5835 if (inode) { 5836 root = BTRFS_I(inode)->root; 5837 if (btrfs_root_refs(&root->root_item) == 0) 5838 return 1; 5839 5840 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5841 return 1; 5842 } 5843 return 0; 5844 } 5845 5846 static void btrfs_dentry_release(struct dentry *dentry) 5847 { 5848 kfree(dentry->d_fsdata); 5849 } 5850 5851 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5852 unsigned int flags) 5853 { 5854 struct inode *inode; 5855 5856 inode = btrfs_lookup_dentry(dir, dentry); 5857 if (IS_ERR(inode)) { 5858 if (PTR_ERR(inode) == -ENOENT) 5859 inode = NULL; 5860 else 5861 return ERR_CAST(inode); 5862 } 5863 5864 return d_splice_alias(inode, dentry); 5865 } 5866 5867 unsigned char btrfs_filetype_table[] = { 5868 DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK 5869 }; 5870 5871 /* 5872 * All this infrastructure exists because dir_emit can fault, and we are holding 5873 * the tree lock when doing readdir. For now just allocate a buffer and copy 5874 * our information into that, and then dir_emit from the buffer. This is 5875 * similar to what NFS does, only we don't keep the buffer around in pagecache 5876 * because I'm afraid I'll mess that up. Long term we need to make filldir do 5877 * copy_to_user_inatomic so we don't have to worry about page faulting under the 5878 * tree lock. 5879 */ 5880 static int btrfs_opendir(struct inode *inode, struct file *file) 5881 { 5882 struct btrfs_file_private *private; 5883 5884 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL); 5885 if (!private) 5886 return -ENOMEM; 5887 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 5888 if (!private->filldir_buf) { 5889 kfree(private); 5890 return -ENOMEM; 5891 } 5892 file->private_data = private; 5893 return 0; 5894 } 5895 5896 struct dir_entry { 5897 u64 ino; 5898 u64 offset; 5899 unsigned type; 5900 int name_len; 5901 }; 5902 5903 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) 5904 { 5905 while (entries--) { 5906 struct dir_entry *entry = addr; 5907 char *name = (char *)(entry + 1); 5908 5909 ctx->pos = get_unaligned(&entry->offset); 5910 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len), 5911 get_unaligned(&entry->ino), 5912 get_unaligned(&entry->type))) 5913 return 1; 5914 addr += sizeof(struct dir_entry) + 5915 get_unaligned(&entry->name_len); 5916 ctx->pos++; 5917 } 5918 return 0; 5919 } 5920 5921 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5922 { 5923 struct inode *inode = file_inode(file); 5924 struct btrfs_root *root = BTRFS_I(inode)->root; 5925 struct btrfs_file_private *private = file->private_data; 5926 struct btrfs_dir_item *di; 5927 struct btrfs_key key; 5928 struct btrfs_key found_key; 5929 struct btrfs_path *path; 5930 void *addr; 5931 struct list_head ins_list; 5932 struct list_head del_list; 5933 int ret; 5934 struct extent_buffer *leaf; 5935 int slot; 5936 char *name_ptr; 5937 int name_len; 5938 int entries = 0; 5939 int total_len = 0; 5940 bool put = false; 5941 struct btrfs_key location; 5942 5943 if (!dir_emit_dots(file, ctx)) 5944 return 0; 5945 5946 path = btrfs_alloc_path(); 5947 if (!path) 5948 return -ENOMEM; 5949 5950 addr = private->filldir_buf; 5951 path->reada = READA_FORWARD; 5952 5953 INIT_LIST_HEAD(&ins_list); 5954 INIT_LIST_HEAD(&del_list); 5955 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list); 5956 5957 again: 5958 key.type = BTRFS_DIR_INDEX_KEY; 5959 key.offset = ctx->pos; 5960 key.objectid = btrfs_ino(BTRFS_I(inode)); 5961 5962 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5963 if (ret < 0) 5964 goto err; 5965 5966 while (1) { 5967 struct dir_entry *entry; 5968 5969 leaf = path->nodes[0]; 5970 slot = path->slots[0]; 5971 if (slot >= btrfs_header_nritems(leaf)) { 5972 ret = btrfs_next_leaf(root, path); 5973 if (ret < 0) 5974 goto err; 5975 else if (ret > 0) 5976 break; 5977 continue; 5978 } 5979 5980 btrfs_item_key_to_cpu(leaf, &found_key, slot); 5981 5982 if (found_key.objectid != key.objectid) 5983 break; 5984 if (found_key.type != BTRFS_DIR_INDEX_KEY) 5985 break; 5986 if (found_key.offset < ctx->pos) 5987 goto next; 5988 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 5989 goto next; 5990 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); 5991 name_len = btrfs_dir_name_len(leaf, di); 5992 if ((total_len + sizeof(struct dir_entry) + name_len) >= 5993 PAGE_SIZE) { 5994 btrfs_release_path(path); 5995 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 5996 if (ret) 5997 goto nopos; 5998 addr = private->filldir_buf; 5999 entries = 0; 6000 total_len = 0; 6001 goto again; 6002 } 6003 6004 entry = addr; 6005 put_unaligned(name_len, &entry->name_len); 6006 name_ptr = (char *)(entry + 1); 6007 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1), 6008 name_len); 6009 put_unaligned(btrfs_filetype_table[btrfs_dir_type(leaf, di)], 6010 &entry->type); 6011 btrfs_dir_item_key_to_cpu(leaf, di, &location); 6012 put_unaligned(location.objectid, &entry->ino); 6013 put_unaligned(found_key.offset, &entry->offset); 6014 entries++; 6015 addr += sizeof(struct dir_entry) + name_len; 6016 total_len += sizeof(struct dir_entry) + name_len; 6017 next: 6018 path->slots[0]++; 6019 } 6020 btrfs_release_path(path); 6021 6022 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6023 if (ret) 6024 goto nopos; 6025 6026 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 6027 if (ret) 6028 goto nopos; 6029 6030 /* 6031 * Stop new entries from being returned after we return the last 6032 * entry. 6033 * 6034 * New directory entries are assigned a strictly increasing 6035 * offset. This means that new entries created during readdir 6036 * are *guaranteed* to be seen in the future by that readdir. 6037 * This has broken buggy programs which operate on names as 6038 * they're returned by readdir. Until we re-use freed offsets 6039 * we have this hack to stop new entries from being returned 6040 * under the assumption that they'll never reach this huge 6041 * offset. 6042 * 6043 * This is being careful not to overflow 32bit loff_t unless the 6044 * last entry requires it because doing so has broken 32bit apps 6045 * in the past. 6046 */ 6047 if (ctx->pos >= INT_MAX) 6048 ctx->pos = LLONG_MAX; 6049 else 6050 ctx->pos = INT_MAX; 6051 nopos: 6052 ret = 0; 6053 err: 6054 if (put) 6055 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list); 6056 btrfs_free_path(path); 6057 return ret; 6058 } 6059 6060 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc) 6061 { 6062 struct btrfs_root *root = BTRFS_I(inode)->root; 6063 struct btrfs_trans_handle *trans; 6064 int ret = 0; 6065 bool nolock = false; 6066 6067 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 6068 return 0; 6069 6070 if (btrfs_fs_closing(root->fs_info) && 6071 btrfs_is_free_space_inode(BTRFS_I(inode))) 6072 nolock = true; 6073 6074 if (wbc->sync_mode == WB_SYNC_ALL) { 6075 if (nolock) 6076 trans = btrfs_join_transaction_nolock(root); 6077 else 6078 trans = btrfs_join_transaction(root); 6079 if (IS_ERR(trans)) 6080 return PTR_ERR(trans); 6081 ret = btrfs_commit_transaction(trans); 6082 } 6083 return ret; 6084 } 6085 6086 /* 6087 * This is somewhat expensive, updating the tree every time the 6088 * inode changes. But, it is most likely to find the inode in cache. 6089 * FIXME, needs more benchmarking...there are no reasons other than performance 6090 * to keep or drop this code. 6091 */ 6092 static int btrfs_dirty_inode(struct inode *inode) 6093 { 6094 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6095 struct btrfs_root *root = BTRFS_I(inode)->root; 6096 struct btrfs_trans_handle *trans; 6097 int ret; 6098 6099 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 6100 return 0; 6101 6102 trans = btrfs_join_transaction(root); 6103 if (IS_ERR(trans)) 6104 return PTR_ERR(trans); 6105 6106 ret = btrfs_update_inode(trans, root, inode); 6107 if (ret && ret == -ENOSPC) { 6108 /* whoops, lets try again with the full transaction */ 6109 btrfs_end_transaction(trans); 6110 trans = btrfs_start_transaction(root, 1); 6111 if (IS_ERR(trans)) 6112 return PTR_ERR(trans); 6113 6114 ret = btrfs_update_inode(trans, root, inode); 6115 } 6116 btrfs_end_transaction(trans); 6117 if (BTRFS_I(inode)->delayed_node) 6118 btrfs_balance_delayed_items(fs_info); 6119 6120 return ret; 6121 } 6122 6123 /* 6124 * This is a copy of file_update_time. We need this so we can return error on 6125 * ENOSPC for updating the inode in the case of file write and mmap writes. 6126 */ 6127 static int btrfs_update_time(struct inode *inode, struct timespec *now, 6128 int flags) 6129 { 6130 struct btrfs_root *root = BTRFS_I(inode)->root; 6131 bool dirty = flags & ~S_VERSION; 6132 6133 if (btrfs_root_readonly(root)) 6134 return -EROFS; 6135 6136 if (flags & S_VERSION) 6137 dirty |= inode_maybe_inc_iversion(inode, dirty); 6138 if (flags & S_CTIME) 6139 inode->i_ctime = *now; 6140 if (flags & S_MTIME) 6141 inode->i_mtime = *now; 6142 if (flags & S_ATIME) 6143 inode->i_atime = *now; 6144 return dirty ? btrfs_dirty_inode(inode) : 0; 6145 } 6146 6147 /* 6148 * find the highest existing sequence number in a directory 6149 * and then set the in-memory index_cnt variable to reflect 6150 * free sequence numbers 6151 */ 6152 static int btrfs_set_inode_index_count(struct btrfs_inode *inode) 6153 { 6154 struct btrfs_root *root = inode->root; 6155 struct btrfs_key key, found_key; 6156 struct btrfs_path *path; 6157 struct extent_buffer *leaf; 6158 int ret; 6159 6160 key.objectid = btrfs_ino(inode); 6161 key.type = BTRFS_DIR_INDEX_KEY; 6162 key.offset = (u64)-1; 6163 6164 path = btrfs_alloc_path(); 6165 if (!path) 6166 return -ENOMEM; 6167 6168 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 6169 if (ret < 0) 6170 goto out; 6171 /* FIXME: we should be able to handle this */ 6172 if (ret == 0) 6173 goto out; 6174 ret = 0; 6175 6176 /* 6177 * MAGIC NUMBER EXPLANATION: 6178 * since we search a directory based on f_pos we have to start at 2 6179 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody 6180 * else has to start at 2 6181 */ 6182 if (path->slots[0] == 0) { 6183 inode->index_cnt = 2; 6184 goto out; 6185 } 6186 6187 path->slots[0]--; 6188 6189 leaf = path->nodes[0]; 6190 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6191 6192 if (found_key.objectid != btrfs_ino(inode) || 6193 found_key.type != BTRFS_DIR_INDEX_KEY) { 6194 inode->index_cnt = 2; 6195 goto out; 6196 } 6197 6198 inode->index_cnt = found_key.offset + 1; 6199 out: 6200 btrfs_free_path(path); 6201 return ret; 6202 } 6203 6204 /* 6205 * helper to find a free sequence number in a given directory. This current 6206 * code is very simple, later versions will do smarter things in the btree 6207 */ 6208 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) 6209 { 6210 int ret = 0; 6211 6212 if (dir->index_cnt == (u64)-1) { 6213 ret = btrfs_inode_delayed_dir_index_count(dir); 6214 if (ret) { 6215 ret = btrfs_set_inode_index_count(dir); 6216 if (ret) 6217 return ret; 6218 } 6219 } 6220 6221 *index = dir->index_cnt; 6222 dir->index_cnt++; 6223 6224 return ret; 6225 } 6226 6227 static int btrfs_insert_inode_locked(struct inode *inode) 6228 { 6229 struct btrfs_iget_args args; 6230 args.location = &BTRFS_I(inode)->location; 6231 args.root = BTRFS_I(inode)->root; 6232 6233 return insert_inode_locked4(inode, 6234 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 6235 btrfs_find_actor, &args); 6236 } 6237 6238 /* 6239 * Inherit flags from the parent inode. 6240 * 6241 * Currently only the compression flags and the cow flags are inherited. 6242 */ 6243 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir) 6244 { 6245 unsigned int flags; 6246 6247 if (!dir) 6248 return; 6249 6250 flags = BTRFS_I(dir)->flags; 6251 6252 if (flags & BTRFS_INODE_NOCOMPRESS) { 6253 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS; 6254 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 6255 } else if (flags & BTRFS_INODE_COMPRESS) { 6256 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS; 6257 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS; 6258 } 6259 6260 if (flags & BTRFS_INODE_NODATACOW) { 6261 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW; 6262 if (S_ISREG(inode->i_mode)) 6263 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6264 } 6265 6266 btrfs_update_iflags(inode); 6267 } 6268 6269 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, 6270 struct btrfs_root *root, 6271 struct inode *dir, 6272 const char *name, int name_len, 6273 u64 ref_objectid, u64 objectid, 6274 umode_t mode, u64 *index) 6275 { 6276 struct btrfs_fs_info *fs_info = root->fs_info; 6277 struct inode *inode; 6278 struct btrfs_inode_item *inode_item; 6279 struct btrfs_key *location; 6280 struct btrfs_path *path; 6281 struct btrfs_inode_ref *ref; 6282 struct btrfs_key key[2]; 6283 u32 sizes[2]; 6284 int nitems = name ? 2 : 1; 6285 unsigned long ptr; 6286 int ret; 6287 6288 path = btrfs_alloc_path(); 6289 if (!path) 6290 return ERR_PTR(-ENOMEM); 6291 6292 inode = new_inode(fs_info->sb); 6293 if (!inode) { 6294 btrfs_free_path(path); 6295 return ERR_PTR(-ENOMEM); 6296 } 6297 6298 /* 6299 * O_TMPFILE, set link count to 0, so that after this point, 6300 * we fill in an inode item with the correct link count. 6301 */ 6302 if (!name) 6303 set_nlink(inode, 0); 6304 6305 /* 6306 * we have to initialize this early, so we can reclaim the inode 6307 * number if we fail afterwards in this function. 6308 */ 6309 inode->i_ino = objectid; 6310 6311 if (dir && name) { 6312 trace_btrfs_inode_request(dir); 6313 6314 ret = btrfs_set_inode_index(BTRFS_I(dir), index); 6315 if (ret) { 6316 btrfs_free_path(path); 6317 iput(inode); 6318 return ERR_PTR(ret); 6319 } 6320 } else if (dir) { 6321 *index = 0; 6322 } 6323 /* 6324 * index_cnt is ignored for everything but a dir, 6325 * btrfs_set_inode_index_count has an explanation for the magic 6326 * number 6327 */ 6328 BTRFS_I(inode)->index_cnt = 2; 6329 BTRFS_I(inode)->dir_index = *index; 6330 BTRFS_I(inode)->root = root; 6331 BTRFS_I(inode)->generation = trans->transid; 6332 inode->i_generation = BTRFS_I(inode)->generation; 6333 6334 /* 6335 * We could have gotten an inode number from somebody who was fsynced 6336 * and then removed in this same transaction, so let's just set full 6337 * sync since it will be a full sync anyway and this will blow away the 6338 * old info in the log. 6339 */ 6340 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 6341 6342 key[0].objectid = objectid; 6343 key[0].type = BTRFS_INODE_ITEM_KEY; 6344 key[0].offset = 0; 6345 6346 sizes[0] = sizeof(struct btrfs_inode_item); 6347 6348 if (name) { 6349 /* 6350 * Start new inodes with an inode_ref. This is slightly more 6351 * efficient for small numbers of hard links since they will 6352 * be packed into one item. Extended refs will kick in if we 6353 * add more hard links than can fit in the ref item. 6354 */ 6355 key[1].objectid = objectid; 6356 key[1].type = BTRFS_INODE_REF_KEY; 6357 key[1].offset = ref_objectid; 6358 6359 sizes[1] = name_len + sizeof(*ref); 6360 } 6361 6362 location = &BTRFS_I(inode)->location; 6363 location->objectid = objectid; 6364 location->offset = 0; 6365 location->type = BTRFS_INODE_ITEM_KEY; 6366 6367 ret = btrfs_insert_inode_locked(inode); 6368 if (ret < 0) 6369 goto fail; 6370 6371 path->leave_spinning = 1; 6372 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems); 6373 if (ret != 0) 6374 goto fail_unlock; 6375 6376 inode_init_owner(inode, dir, mode); 6377 inode_set_bytes(inode, 0); 6378 6379 inode->i_mtime = current_time(inode); 6380 inode->i_atime = inode->i_mtime; 6381 inode->i_ctime = inode->i_mtime; 6382 BTRFS_I(inode)->i_otime = inode->i_mtime; 6383 6384 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6385 struct btrfs_inode_item); 6386 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6387 sizeof(*inode_item)); 6388 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6389 6390 if (name) { 6391 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6392 struct btrfs_inode_ref); 6393 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len); 6394 btrfs_set_inode_ref_index(path->nodes[0], ref, *index); 6395 ptr = (unsigned long)(ref + 1); 6396 write_extent_buffer(path->nodes[0], name, ptr, name_len); 6397 } 6398 6399 btrfs_mark_buffer_dirty(path->nodes[0]); 6400 btrfs_free_path(path); 6401 6402 btrfs_inherit_iflags(inode, dir); 6403 6404 if (S_ISREG(mode)) { 6405 if (btrfs_test_opt(fs_info, NODATASUM)) 6406 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6407 if (btrfs_test_opt(fs_info, NODATACOW)) 6408 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6409 BTRFS_INODE_NODATASUM; 6410 } 6411 6412 inode_tree_add(inode); 6413 6414 trace_btrfs_inode_new(inode); 6415 btrfs_set_inode_last_trans(trans, inode); 6416 6417 btrfs_update_root_times(trans, root); 6418 6419 ret = btrfs_inode_inherit_props(trans, inode, dir); 6420 if (ret) 6421 btrfs_err(fs_info, 6422 "error inheriting props for ino %llu (root %llu): %d", 6423 btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret); 6424 6425 return inode; 6426 6427 fail_unlock: 6428 unlock_new_inode(inode); 6429 fail: 6430 if (dir && name) 6431 BTRFS_I(dir)->index_cnt--; 6432 btrfs_free_path(path); 6433 iput(inode); 6434 return ERR_PTR(ret); 6435 } 6436 6437 static inline u8 btrfs_inode_type(struct inode *inode) 6438 { 6439 return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT]; 6440 } 6441 6442 /* 6443 * utility function to add 'inode' into 'parent_inode' with 6444 * a give name and a given sequence number. 6445 * if 'add_backref' is true, also insert a backref from the 6446 * inode to the parent directory. 6447 */ 6448 int btrfs_add_link(struct btrfs_trans_handle *trans, 6449 struct btrfs_inode *parent_inode, struct btrfs_inode *inode, 6450 const char *name, int name_len, int add_backref, u64 index) 6451 { 6452 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 6453 int ret = 0; 6454 struct btrfs_key key; 6455 struct btrfs_root *root = parent_inode->root; 6456 u64 ino = btrfs_ino(inode); 6457 u64 parent_ino = btrfs_ino(parent_inode); 6458 6459 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6460 memcpy(&key, &inode->root->root_key, sizeof(key)); 6461 } else { 6462 key.objectid = ino; 6463 key.type = BTRFS_INODE_ITEM_KEY; 6464 key.offset = 0; 6465 } 6466 6467 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6468 ret = btrfs_add_root_ref(trans, fs_info, key.objectid, 6469 root->root_key.objectid, parent_ino, 6470 index, name, name_len); 6471 } else if (add_backref) { 6472 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino, 6473 parent_ino, index); 6474 } 6475 6476 /* Nothing to clean up yet */ 6477 if (ret) 6478 return ret; 6479 6480 ret = btrfs_insert_dir_item(trans, root, name, name_len, 6481 parent_inode, &key, 6482 btrfs_inode_type(&inode->vfs_inode), index); 6483 if (ret == -EEXIST || ret == -EOVERFLOW) 6484 goto fail_dir_item; 6485 else if (ret) { 6486 btrfs_abort_transaction(trans, ret); 6487 return ret; 6488 } 6489 6490 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + 6491 name_len * 2); 6492 inode_inc_iversion(&parent_inode->vfs_inode); 6493 parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime = 6494 current_time(&parent_inode->vfs_inode); 6495 ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode); 6496 if (ret) 6497 btrfs_abort_transaction(trans, ret); 6498 return ret; 6499 6500 fail_dir_item: 6501 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6502 u64 local_index; 6503 int err; 6504 err = btrfs_del_root_ref(trans, fs_info, key.objectid, 6505 root->root_key.objectid, parent_ino, 6506 &local_index, name, name_len); 6507 6508 } else if (add_backref) { 6509 u64 local_index; 6510 int err; 6511 6512 err = btrfs_del_inode_ref(trans, root, name, name_len, 6513 ino, parent_ino, &local_index); 6514 } 6515 return ret; 6516 } 6517 6518 static int btrfs_add_nondir(struct btrfs_trans_handle *trans, 6519 struct btrfs_inode *dir, struct dentry *dentry, 6520 struct btrfs_inode *inode, int backref, u64 index) 6521 { 6522 int err = btrfs_add_link(trans, dir, inode, 6523 dentry->d_name.name, dentry->d_name.len, 6524 backref, index); 6525 if (err > 0) 6526 err = -EEXIST; 6527 return err; 6528 } 6529 6530 static int btrfs_mknod(struct inode *dir, struct dentry *dentry, 6531 umode_t mode, dev_t rdev) 6532 { 6533 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6534 struct btrfs_trans_handle *trans; 6535 struct btrfs_root *root = BTRFS_I(dir)->root; 6536 struct inode *inode = NULL; 6537 int err; 6538 int drop_inode = 0; 6539 u64 objectid; 6540 u64 index = 0; 6541 6542 /* 6543 * 2 for inode item and ref 6544 * 2 for dir items 6545 * 1 for xattr if selinux is on 6546 */ 6547 trans = btrfs_start_transaction(root, 5); 6548 if (IS_ERR(trans)) 6549 return PTR_ERR(trans); 6550 6551 err = btrfs_find_free_ino(root, &objectid); 6552 if (err) 6553 goto out_unlock; 6554 6555 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6556 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6557 mode, &index); 6558 if (IS_ERR(inode)) { 6559 err = PTR_ERR(inode); 6560 goto out_unlock; 6561 } 6562 6563 /* 6564 * If the active LSM wants to access the inode during 6565 * d_instantiate it needs these. Smack checks to see 6566 * if the filesystem supports xattrs by looking at the 6567 * ops vector. 6568 */ 6569 inode->i_op = &btrfs_special_inode_operations; 6570 init_special_inode(inode, inode->i_mode, rdev); 6571 6572 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6573 if (err) 6574 goto out_unlock_inode; 6575 6576 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6577 0, index); 6578 if (err) { 6579 goto out_unlock_inode; 6580 } else { 6581 btrfs_update_inode(trans, root, inode); 6582 unlock_new_inode(inode); 6583 d_instantiate(dentry, inode); 6584 } 6585 6586 out_unlock: 6587 btrfs_end_transaction(trans); 6588 btrfs_btree_balance_dirty(fs_info); 6589 if (drop_inode) { 6590 inode_dec_link_count(inode); 6591 iput(inode); 6592 } 6593 return err; 6594 6595 out_unlock_inode: 6596 drop_inode = 1; 6597 unlock_new_inode(inode); 6598 goto out_unlock; 6599 6600 } 6601 6602 static int btrfs_create(struct inode *dir, struct dentry *dentry, 6603 umode_t mode, bool excl) 6604 { 6605 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6606 struct btrfs_trans_handle *trans; 6607 struct btrfs_root *root = BTRFS_I(dir)->root; 6608 struct inode *inode = NULL; 6609 int drop_inode_on_err = 0; 6610 int err; 6611 u64 objectid; 6612 u64 index = 0; 6613 6614 /* 6615 * 2 for inode item and ref 6616 * 2 for dir items 6617 * 1 for xattr if selinux is on 6618 */ 6619 trans = btrfs_start_transaction(root, 5); 6620 if (IS_ERR(trans)) 6621 return PTR_ERR(trans); 6622 6623 err = btrfs_find_free_ino(root, &objectid); 6624 if (err) 6625 goto out_unlock; 6626 6627 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6628 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6629 mode, &index); 6630 if (IS_ERR(inode)) { 6631 err = PTR_ERR(inode); 6632 goto out_unlock; 6633 } 6634 drop_inode_on_err = 1; 6635 /* 6636 * If the active LSM wants to access the inode during 6637 * d_instantiate it needs these. Smack checks to see 6638 * if the filesystem supports xattrs by looking at the 6639 * ops vector. 6640 */ 6641 inode->i_fop = &btrfs_file_operations; 6642 inode->i_op = &btrfs_file_inode_operations; 6643 inode->i_mapping->a_ops = &btrfs_aops; 6644 6645 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6646 if (err) 6647 goto out_unlock_inode; 6648 6649 err = btrfs_update_inode(trans, root, inode); 6650 if (err) 6651 goto out_unlock_inode; 6652 6653 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6654 0, index); 6655 if (err) 6656 goto out_unlock_inode; 6657 6658 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 6659 unlock_new_inode(inode); 6660 d_instantiate(dentry, inode); 6661 6662 out_unlock: 6663 btrfs_end_transaction(trans); 6664 if (err && drop_inode_on_err) { 6665 inode_dec_link_count(inode); 6666 iput(inode); 6667 } 6668 btrfs_btree_balance_dirty(fs_info); 6669 return err; 6670 6671 out_unlock_inode: 6672 unlock_new_inode(inode); 6673 goto out_unlock; 6674 6675 } 6676 6677 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6678 struct dentry *dentry) 6679 { 6680 struct btrfs_trans_handle *trans = NULL; 6681 struct btrfs_root *root = BTRFS_I(dir)->root; 6682 struct inode *inode = d_inode(old_dentry); 6683 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6684 u64 index; 6685 int err; 6686 int drop_inode = 0; 6687 6688 /* do not allow sys_link's with other subvols of the same device */ 6689 if (root->objectid != BTRFS_I(inode)->root->objectid) 6690 return -EXDEV; 6691 6692 if (inode->i_nlink >= BTRFS_LINK_MAX) 6693 return -EMLINK; 6694 6695 err = btrfs_set_inode_index(BTRFS_I(dir), &index); 6696 if (err) 6697 goto fail; 6698 6699 /* 6700 * 2 items for inode and inode ref 6701 * 2 items for dir items 6702 * 1 item for parent inode 6703 */ 6704 trans = btrfs_start_transaction(root, 5); 6705 if (IS_ERR(trans)) { 6706 err = PTR_ERR(trans); 6707 trans = NULL; 6708 goto fail; 6709 } 6710 6711 /* There are several dir indexes for this inode, clear the cache. */ 6712 BTRFS_I(inode)->dir_index = 0ULL; 6713 inc_nlink(inode); 6714 inode_inc_iversion(inode); 6715 inode->i_ctime = current_time(inode); 6716 ihold(inode); 6717 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6718 6719 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6720 1, index); 6721 6722 if (err) { 6723 drop_inode = 1; 6724 } else { 6725 struct dentry *parent = dentry->d_parent; 6726 err = btrfs_update_inode(trans, root, inode); 6727 if (err) 6728 goto fail; 6729 if (inode->i_nlink == 1) { 6730 /* 6731 * If new hard link count is 1, it's a file created 6732 * with open(2) O_TMPFILE flag. 6733 */ 6734 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 6735 if (err) 6736 goto fail; 6737 } 6738 d_instantiate(dentry, inode); 6739 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent); 6740 } 6741 6742 fail: 6743 if (trans) 6744 btrfs_end_transaction(trans); 6745 if (drop_inode) { 6746 inode_dec_link_count(inode); 6747 iput(inode); 6748 } 6749 btrfs_btree_balance_dirty(fs_info); 6750 return err; 6751 } 6752 6753 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 6754 { 6755 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6756 struct inode *inode = NULL; 6757 struct btrfs_trans_handle *trans; 6758 struct btrfs_root *root = BTRFS_I(dir)->root; 6759 int err = 0; 6760 int drop_on_err = 0; 6761 u64 objectid = 0; 6762 u64 index = 0; 6763 6764 /* 6765 * 2 items for inode and ref 6766 * 2 items for dir items 6767 * 1 for xattr if selinux is on 6768 */ 6769 trans = btrfs_start_transaction(root, 5); 6770 if (IS_ERR(trans)) 6771 return PTR_ERR(trans); 6772 6773 err = btrfs_find_free_ino(root, &objectid); 6774 if (err) 6775 goto out_fail; 6776 6777 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6778 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6779 S_IFDIR | mode, &index); 6780 if (IS_ERR(inode)) { 6781 err = PTR_ERR(inode); 6782 goto out_fail; 6783 } 6784 6785 drop_on_err = 1; 6786 /* these must be set before we unlock the inode */ 6787 inode->i_op = &btrfs_dir_inode_operations; 6788 inode->i_fop = &btrfs_dir_file_operations; 6789 6790 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6791 if (err) 6792 goto out_fail_inode; 6793 6794 btrfs_i_size_write(BTRFS_I(inode), 0); 6795 err = btrfs_update_inode(trans, root, inode); 6796 if (err) 6797 goto out_fail_inode; 6798 6799 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), 6800 dentry->d_name.name, 6801 dentry->d_name.len, 0, index); 6802 if (err) 6803 goto out_fail_inode; 6804 6805 d_instantiate(dentry, inode); 6806 /* 6807 * mkdir is special. We're unlocking after we call d_instantiate 6808 * to avoid a race with nfsd calling d_instantiate. 6809 */ 6810 unlock_new_inode(inode); 6811 drop_on_err = 0; 6812 6813 out_fail: 6814 btrfs_end_transaction(trans); 6815 if (drop_on_err) { 6816 inode_dec_link_count(inode); 6817 iput(inode); 6818 } 6819 btrfs_btree_balance_dirty(fs_info); 6820 return err; 6821 6822 out_fail_inode: 6823 unlock_new_inode(inode); 6824 goto out_fail; 6825 } 6826 6827 static noinline int uncompress_inline(struct btrfs_path *path, 6828 struct page *page, 6829 size_t pg_offset, u64 extent_offset, 6830 struct btrfs_file_extent_item *item) 6831 { 6832 int ret; 6833 struct extent_buffer *leaf = path->nodes[0]; 6834 char *tmp; 6835 size_t max_size; 6836 unsigned long inline_size; 6837 unsigned long ptr; 6838 int compress_type; 6839 6840 WARN_ON(pg_offset != 0); 6841 compress_type = btrfs_file_extent_compression(leaf, item); 6842 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6843 inline_size = btrfs_file_extent_inline_item_len(leaf, 6844 btrfs_item_nr(path->slots[0])); 6845 tmp = kmalloc(inline_size, GFP_NOFS); 6846 if (!tmp) 6847 return -ENOMEM; 6848 ptr = btrfs_file_extent_inline_start(item); 6849 6850 read_extent_buffer(leaf, tmp, ptr, inline_size); 6851 6852 max_size = min_t(unsigned long, PAGE_SIZE, max_size); 6853 ret = btrfs_decompress(compress_type, tmp, page, 6854 extent_offset, inline_size, max_size); 6855 6856 /* 6857 * decompression code contains a memset to fill in any space between the end 6858 * of the uncompressed data and the end of max_size in case the decompressed 6859 * data ends up shorter than ram_bytes. That doesn't cover the hole between 6860 * the end of an inline extent and the beginning of the next block, so we 6861 * cover that region here. 6862 */ 6863 6864 if (max_size + pg_offset < PAGE_SIZE) { 6865 char *map = kmap(page); 6866 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset); 6867 kunmap(page); 6868 } 6869 kfree(tmp); 6870 return ret; 6871 } 6872 6873 /* 6874 * a bit scary, this does extent mapping from logical file offset to the disk. 6875 * the ugly parts come from merging extents from the disk with the in-ram 6876 * representation. This gets more complex because of the data=ordered code, 6877 * where the in-ram extents might be locked pending data=ordered completion. 6878 * 6879 * This also copies inline extents directly into the page. 6880 */ 6881 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, 6882 struct page *page, 6883 size_t pg_offset, u64 start, u64 len, 6884 int create) 6885 { 6886 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 6887 int ret; 6888 int err = 0; 6889 u64 extent_start = 0; 6890 u64 extent_end = 0; 6891 u64 objectid = btrfs_ino(inode); 6892 u32 found_type; 6893 struct btrfs_path *path = NULL; 6894 struct btrfs_root *root = inode->root; 6895 struct btrfs_file_extent_item *item; 6896 struct extent_buffer *leaf; 6897 struct btrfs_key found_key; 6898 struct extent_map *em = NULL; 6899 struct extent_map_tree *em_tree = &inode->extent_tree; 6900 struct extent_io_tree *io_tree = &inode->io_tree; 6901 const bool new_inline = !page || create; 6902 6903 read_lock(&em_tree->lock); 6904 em = lookup_extent_mapping(em_tree, start, len); 6905 if (em) 6906 em->bdev = fs_info->fs_devices->latest_bdev; 6907 read_unlock(&em_tree->lock); 6908 6909 if (em) { 6910 if (em->start > start || em->start + em->len <= start) 6911 free_extent_map(em); 6912 else if (em->block_start == EXTENT_MAP_INLINE && page) 6913 free_extent_map(em); 6914 else 6915 goto out; 6916 } 6917 em = alloc_extent_map(); 6918 if (!em) { 6919 err = -ENOMEM; 6920 goto out; 6921 } 6922 em->bdev = fs_info->fs_devices->latest_bdev; 6923 em->start = EXTENT_MAP_HOLE; 6924 em->orig_start = EXTENT_MAP_HOLE; 6925 em->len = (u64)-1; 6926 em->block_len = (u64)-1; 6927 6928 if (!path) { 6929 path = btrfs_alloc_path(); 6930 if (!path) { 6931 err = -ENOMEM; 6932 goto out; 6933 } 6934 /* 6935 * Chances are we'll be called again, so go ahead and do 6936 * readahead 6937 */ 6938 path->reada = READA_FORWARD; 6939 } 6940 6941 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); 6942 if (ret < 0) { 6943 err = ret; 6944 goto out; 6945 } 6946 6947 if (ret != 0) { 6948 if (path->slots[0] == 0) 6949 goto not_found; 6950 path->slots[0]--; 6951 } 6952 6953 leaf = path->nodes[0]; 6954 item = btrfs_item_ptr(leaf, path->slots[0], 6955 struct btrfs_file_extent_item); 6956 /* are we inside the extent that was found? */ 6957 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6958 found_type = found_key.type; 6959 if (found_key.objectid != objectid || 6960 found_type != BTRFS_EXTENT_DATA_KEY) { 6961 /* 6962 * If we backup past the first extent we want to move forward 6963 * and see if there is an extent in front of us, otherwise we'll 6964 * say there is a hole for our whole search range which can 6965 * cause problems. 6966 */ 6967 extent_end = start; 6968 goto next; 6969 } 6970 6971 found_type = btrfs_file_extent_type(leaf, item); 6972 extent_start = found_key.offset; 6973 if (found_type == BTRFS_FILE_EXTENT_REG || 6974 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6975 extent_end = extent_start + 6976 btrfs_file_extent_num_bytes(leaf, item); 6977 6978 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, 6979 extent_start); 6980 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 6981 size_t size; 6982 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 6983 extent_end = ALIGN(extent_start + size, 6984 fs_info->sectorsize); 6985 6986 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, 6987 path->slots[0], 6988 extent_start); 6989 } 6990 next: 6991 if (start >= extent_end) { 6992 path->slots[0]++; 6993 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 6994 ret = btrfs_next_leaf(root, path); 6995 if (ret < 0) { 6996 err = ret; 6997 goto out; 6998 } 6999 if (ret > 0) 7000 goto not_found; 7001 leaf = path->nodes[0]; 7002 } 7003 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 7004 if (found_key.objectid != objectid || 7005 found_key.type != BTRFS_EXTENT_DATA_KEY) 7006 goto not_found; 7007 if (start + len <= found_key.offset) 7008 goto not_found; 7009 if (start > found_key.offset) 7010 goto next; 7011 em->start = start; 7012 em->orig_start = start; 7013 em->len = found_key.offset - start; 7014 goto not_found_em; 7015 } 7016 7017 btrfs_extent_item_to_extent_map(inode, path, item, 7018 new_inline, em); 7019 7020 if (found_type == BTRFS_FILE_EXTENT_REG || 7021 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7022 goto insert; 7023 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 7024 unsigned long ptr; 7025 char *map; 7026 size_t size; 7027 size_t extent_offset; 7028 size_t copy_size; 7029 7030 if (new_inline) 7031 goto out; 7032 7033 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 7034 extent_offset = page_offset(page) + pg_offset - extent_start; 7035 copy_size = min_t(u64, PAGE_SIZE - pg_offset, 7036 size - extent_offset); 7037 em->start = extent_start + extent_offset; 7038 em->len = ALIGN(copy_size, fs_info->sectorsize); 7039 em->orig_block_len = em->len; 7040 em->orig_start = em->start; 7041 ptr = btrfs_file_extent_inline_start(item) + extent_offset; 7042 if (!PageUptodate(page)) { 7043 if (btrfs_file_extent_compression(leaf, item) != 7044 BTRFS_COMPRESS_NONE) { 7045 ret = uncompress_inline(path, page, pg_offset, 7046 extent_offset, item); 7047 if (ret) { 7048 err = ret; 7049 goto out; 7050 } 7051 } else { 7052 map = kmap(page); 7053 read_extent_buffer(leaf, map + pg_offset, ptr, 7054 copy_size); 7055 if (pg_offset + copy_size < PAGE_SIZE) { 7056 memset(map + pg_offset + copy_size, 0, 7057 PAGE_SIZE - pg_offset - 7058 copy_size); 7059 } 7060 kunmap(page); 7061 } 7062 flush_dcache_page(page); 7063 } 7064 set_extent_uptodate(io_tree, em->start, 7065 extent_map_end(em) - 1, NULL, GFP_NOFS); 7066 goto insert; 7067 } 7068 not_found: 7069 em->start = start; 7070 em->orig_start = start; 7071 em->len = len; 7072 not_found_em: 7073 em->block_start = EXTENT_MAP_HOLE; 7074 insert: 7075 btrfs_release_path(path); 7076 if (em->start > start || extent_map_end(em) <= start) { 7077 btrfs_err(fs_info, 7078 "bad extent! em: [%llu %llu] passed [%llu %llu]", 7079 em->start, em->len, start, len); 7080 err = -EIO; 7081 goto out; 7082 } 7083 7084 err = 0; 7085 write_lock(&em_tree->lock); 7086 err = btrfs_add_extent_mapping(em_tree, &em, start, len); 7087 write_unlock(&em_tree->lock); 7088 out: 7089 7090 trace_btrfs_get_extent(root, inode, em); 7091 7092 btrfs_free_path(path); 7093 if (err) { 7094 free_extent_map(em); 7095 return ERR_PTR(err); 7096 } 7097 BUG_ON(!em); /* Error is always set */ 7098 return em; 7099 } 7100 7101 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, 7102 struct page *page, 7103 size_t pg_offset, u64 start, u64 len, 7104 int create) 7105 { 7106 struct extent_map *em; 7107 struct extent_map *hole_em = NULL; 7108 u64 range_start = start; 7109 u64 end; 7110 u64 found; 7111 u64 found_end; 7112 int err = 0; 7113 7114 em = btrfs_get_extent(inode, page, pg_offset, start, len, create); 7115 if (IS_ERR(em)) 7116 return em; 7117 /* 7118 * If our em maps to: 7119 * - a hole or 7120 * - a pre-alloc extent, 7121 * there might actually be delalloc bytes behind it. 7122 */ 7123 if (em->block_start != EXTENT_MAP_HOLE && 7124 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7125 return em; 7126 else 7127 hole_em = em; 7128 7129 /* check to see if we've wrapped (len == -1 or similar) */ 7130 end = start + len; 7131 if (end < start) 7132 end = (u64)-1; 7133 else 7134 end -= 1; 7135 7136 em = NULL; 7137 7138 /* ok, we didn't find anything, lets look for delalloc */ 7139 found = count_range_bits(&inode->io_tree, &range_start, 7140 end, len, EXTENT_DELALLOC, 1); 7141 found_end = range_start + found; 7142 if (found_end < range_start) 7143 found_end = (u64)-1; 7144 7145 /* 7146 * we didn't find anything useful, return 7147 * the original results from get_extent() 7148 */ 7149 if (range_start > end || found_end <= start) { 7150 em = hole_em; 7151 hole_em = NULL; 7152 goto out; 7153 } 7154 7155 /* adjust the range_start to make sure it doesn't 7156 * go backwards from the start they passed in 7157 */ 7158 range_start = max(start, range_start); 7159 found = found_end - range_start; 7160 7161 if (found > 0) { 7162 u64 hole_start = start; 7163 u64 hole_len = len; 7164 7165 em = alloc_extent_map(); 7166 if (!em) { 7167 err = -ENOMEM; 7168 goto out; 7169 } 7170 /* 7171 * when btrfs_get_extent can't find anything it 7172 * returns one huge hole 7173 * 7174 * make sure what it found really fits our range, and 7175 * adjust to make sure it is based on the start from 7176 * the caller 7177 */ 7178 if (hole_em) { 7179 u64 calc_end = extent_map_end(hole_em); 7180 7181 if (calc_end <= start || (hole_em->start > end)) { 7182 free_extent_map(hole_em); 7183 hole_em = NULL; 7184 } else { 7185 hole_start = max(hole_em->start, start); 7186 hole_len = calc_end - hole_start; 7187 } 7188 } 7189 em->bdev = NULL; 7190 if (hole_em && range_start > hole_start) { 7191 /* our hole starts before our delalloc, so we 7192 * have to return just the parts of the hole 7193 * that go until the delalloc starts 7194 */ 7195 em->len = min(hole_len, 7196 range_start - hole_start); 7197 em->start = hole_start; 7198 em->orig_start = hole_start; 7199 /* 7200 * don't adjust block start at all, 7201 * it is fixed at EXTENT_MAP_HOLE 7202 */ 7203 em->block_start = hole_em->block_start; 7204 em->block_len = hole_len; 7205 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) 7206 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 7207 } else { 7208 em->start = range_start; 7209 em->len = found; 7210 em->orig_start = range_start; 7211 em->block_start = EXTENT_MAP_DELALLOC; 7212 em->block_len = found; 7213 } 7214 } else { 7215 return hole_em; 7216 } 7217 out: 7218 7219 free_extent_map(hole_em); 7220 if (err) { 7221 free_extent_map(em); 7222 return ERR_PTR(err); 7223 } 7224 return em; 7225 } 7226 7227 static struct extent_map *btrfs_create_dio_extent(struct inode *inode, 7228 const u64 start, 7229 const u64 len, 7230 const u64 orig_start, 7231 const u64 block_start, 7232 const u64 block_len, 7233 const u64 orig_block_len, 7234 const u64 ram_bytes, 7235 const int type) 7236 { 7237 struct extent_map *em = NULL; 7238 int ret; 7239 7240 if (type != BTRFS_ORDERED_NOCOW) { 7241 em = create_io_em(inode, start, len, orig_start, 7242 block_start, block_len, orig_block_len, 7243 ram_bytes, 7244 BTRFS_COMPRESS_NONE, /* compress_type */ 7245 type); 7246 if (IS_ERR(em)) 7247 goto out; 7248 } 7249 ret = btrfs_add_ordered_extent_dio(inode, start, block_start, 7250 len, block_len, type); 7251 if (ret) { 7252 if (em) { 7253 free_extent_map(em); 7254 btrfs_drop_extent_cache(BTRFS_I(inode), start, 7255 start + len - 1, 0); 7256 } 7257 em = ERR_PTR(ret); 7258 } 7259 out: 7260 7261 return em; 7262 } 7263 7264 static struct extent_map *btrfs_new_extent_direct(struct inode *inode, 7265 u64 start, u64 len) 7266 { 7267 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7268 struct btrfs_root *root = BTRFS_I(inode)->root; 7269 struct extent_map *em; 7270 struct btrfs_key ins; 7271 u64 alloc_hint; 7272 int ret; 7273 7274 alloc_hint = get_extent_allocation_hint(inode, start, len); 7275 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, 7276 0, alloc_hint, &ins, 1, 1); 7277 if (ret) 7278 return ERR_PTR(ret); 7279 7280 em = btrfs_create_dio_extent(inode, start, ins.offset, start, 7281 ins.objectid, ins.offset, ins.offset, 7282 ins.offset, BTRFS_ORDERED_REGULAR); 7283 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 7284 if (IS_ERR(em)) 7285 btrfs_free_reserved_extent(fs_info, ins.objectid, 7286 ins.offset, 1); 7287 7288 return em; 7289 } 7290 7291 /* 7292 * returns 1 when the nocow is safe, < 1 on error, 0 if the 7293 * block must be cow'd 7294 */ 7295 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 7296 u64 *orig_start, u64 *orig_block_len, 7297 u64 *ram_bytes) 7298 { 7299 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7300 struct btrfs_path *path; 7301 int ret; 7302 struct extent_buffer *leaf; 7303 struct btrfs_root *root = BTRFS_I(inode)->root; 7304 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7305 struct btrfs_file_extent_item *fi; 7306 struct btrfs_key key; 7307 u64 disk_bytenr; 7308 u64 backref_offset; 7309 u64 extent_end; 7310 u64 num_bytes; 7311 int slot; 7312 int found_type; 7313 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW); 7314 7315 path = btrfs_alloc_path(); 7316 if (!path) 7317 return -ENOMEM; 7318 7319 ret = btrfs_lookup_file_extent(NULL, root, path, 7320 btrfs_ino(BTRFS_I(inode)), offset, 0); 7321 if (ret < 0) 7322 goto out; 7323 7324 slot = path->slots[0]; 7325 if (ret == 1) { 7326 if (slot == 0) { 7327 /* can't find the item, must cow */ 7328 ret = 0; 7329 goto out; 7330 } 7331 slot--; 7332 } 7333 ret = 0; 7334 leaf = path->nodes[0]; 7335 btrfs_item_key_to_cpu(leaf, &key, slot); 7336 if (key.objectid != btrfs_ino(BTRFS_I(inode)) || 7337 key.type != BTRFS_EXTENT_DATA_KEY) { 7338 /* not our file or wrong item type, must cow */ 7339 goto out; 7340 } 7341 7342 if (key.offset > offset) { 7343 /* Wrong offset, must cow */ 7344 goto out; 7345 } 7346 7347 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 7348 found_type = btrfs_file_extent_type(leaf, fi); 7349 if (found_type != BTRFS_FILE_EXTENT_REG && 7350 found_type != BTRFS_FILE_EXTENT_PREALLOC) { 7351 /* not a regular extent, must cow */ 7352 goto out; 7353 } 7354 7355 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG) 7356 goto out; 7357 7358 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 7359 if (extent_end <= offset) 7360 goto out; 7361 7362 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 7363 if (disk_bytenr == 0) 7364 goto out; 7365 7366 if (btrfs_file_extent_compression(leaf, fi) || 7367 btrfs_file_extent_encryption(leaf, fi) || 7368 btrfs_file_extent_other_encoding(leaf, fi)) 7369 goto out; 7370 7371 backref_offset = btrfs_file_extent_offset(leaf, fi); 7372 7373 if (orig_start) { 7374 *orig_start = key.offset - backref_offset; 7375 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 7376 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 7377 } 7378 7379 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 7380 goto out; 7381 7382 num_bytes = min(offset + *len, extent_end) - offset; 7383 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7384 u64 range_end; 7385 7386 range_end = round_up(offset + num_bytes, 7387 root->fs_info->sectorsize) - 1; 7388 ret = test_range_bit(io_tree, offset, range_end, 7389 EXTENT_DELALLOC, 0, NULL); 7390 if (ret) { 7391 ret = -EAGAIN; 7392 goto out; 7393 } 7394 } 7395 7396 btrfs_release_path(path); 7397 7398 /* 7399 * look for other files referencing this extent, if we 7400 * find any we must cow 7401 */ 7402 7403 ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)), 7404 key.offset - backref_offset, disk_bytenr); 7405 if (ret) { 7406 ret = 0; 7407 goto out; 7408 } 7409 7410 /* 7411 * adjust disk_bytenr and num_bytes to cover just the bytes 7412 * in this extent we are about to write. If there 7413 * are any csums in that range we have to cow in order 7414 * to keep the csums correct 7415 */ 7416 disk_bytenr += backref_offset; 7417 disk_bytenr += offset - key.offset; 7418 if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes)) 7419 goto out; 7420 /* 7421 * all of the above have passed, it is safe to overwrite this extent 7422 * without cow 7423 */ 7424 *len = num_bytes; 7425 ret = 1; 7426 out: 7427 btrfs_free_path(path); 7428 return ret; 7429 } 7430 7431 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7432 struct extent_state **cached_state, int writing) 7433 { 7434 struct btrfs_ordered_extent *ordered; 7435 int ret = 0; 7436 7437 while (1) { 7438 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7439 cached_state); 7440 /* 7441 * We're concerned with the entire range that we're going to be 7442 * doing DIO to, so we need to make sure there's no ordered 7443 * extents in this range. 7444 */ 7445 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart, 7446 lockend - lockstart + 1); 7447 7448 /* 7449 * We need to make sure there are no buffered pages in this 7450 * range either, we could have raced between the invalidate in 7451 * generic_file_direct_write and locking the extent. The 7452 * invalidate needs to happen so that reads after a write do not 7453 * get stale data. 7454 */ 7455 if (!ordered && 7456 (!writing || !filemap_range_has_page(inode->i_mapping, 7457 lockstart, lockend))) 7458 break; 7459 7460 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7461 cached_state); 7462 7463 if (ordered) { 7464 /* 7465 * If we are doing a DIO read and the ordered extent we 7466 * found is for a buffered write, we can not wait for it 7467 * to complete and retry, because if we do so we can 7468 * deadlock with concurrent buffered writes on page 7469 * locks. This happens only if our DIO read covers more 7470 * than one extent map, if at this point has already 7471 * created an ordered extent for a previous extent map 7472 * and locked its range in the inode's io tree, and a 7473 * concurrent write against that previous extent map's 7474 * range and this range started (we unlock the ranges 7475 * in the io tree only when the bios complete and 7476 * buffered writes always lock pages before attempting 7477 * to lock range in the io tree). 7478 */ 7479 if (writing || 7480 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) 7481 btrfs_start_ordered_extent(inode, ordered, 1); 7482 else 7483 ret = -ENOTBLK; 7484 btrfs_put_ordered_extent(ordered); 7485 } else { 7486 /* 7487 * We could trigger writeback for this range (and wait 7488 * for it to complete) and then invalidate the pages for 7489 * this range (through invalidate_inode_pages2_range()), 7490 * but that can lead us to a deadlock with a concurrent 7491 * call to readpages() (a buffered read or a defrag call 7492 * triggered a readahead) on a page lock due to an 7493 * ordered dio extent we created before but did not have 7494 * yet a corresponding bio submitted (whence it can not 7495 * complete), which makes readpages() wait for that 7496 * ordered extent to complete while holding a lock on 7497 * that page. 7498 */ 7499 ret = -ENOTBLK; 7500 } 7501 7502 if (ret) 7503 break; 7504 7505 cond_resched(); 7506 } 7507 7508 return ret; 7509 } 7510 7511 /* The callers of this must take lock_extent() */ 7512 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, 7513 u64 orig_start, u64 block_start, 7514 u64 block_len, u64 orig_block_len, 7515 u64 ram_bytes, int compress_type, 7516 int type) 7517 { 7518 struct extent_map_tree *em_tree; 7519 struct extent_map *em; 7520 struct btrfs_root *root = BTRFS_I(inode)->root; 7521 int ret; 7522 7523 ASSERT(type == BTRFS_ORDERED_PREALLOC || 7524 type == BTRFS_ORDERED_COMPRESSED || 7525 type == BTRFS_ORDERED_NOCOW || 7526 type == BTRFS_ORDERED_REGULAR); 7527 7528 em_tree = &BTRFS_I(inode)->extent_tree; 7529 em = alloc_extent_map(); 7530 if (!em) 7531 return ERR_PTR(-ENOMEM); 7532 7533 em->start = start; 7534 em->orig_start = orig_start; 7535 em->len = len; 7536 em->block_len = block_len; 7537 em->block_start = block_start; 7538 em->bdev = root->fs_info->fs_devices->latest_bdev; 7539 em->orig_block_len = orig_block_len; 7540 em->ram_bytes = ram_bytes; 7541 em->generation = -1; 7542 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7543 if (type == BTRFS_ORDERED_PREALLOC) { 7544 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7545 } else if (type == BTRFS_ORDERED_COMPRESSED) { 7546 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 7547 em->compress_type = compress_type; 7548 } 7549 7550 do { 7551 btrfs_drop_extent_cache(BTRFS_I(inode), em->start, 7552 em->start + em->len - 1, 0); 7553 write_lock(&em_tree->lock); 7554 ret = add_extent_mapping(em_tree, em, 1); 7555 write_unlock(&em_tree->lock); 7556 /* 7557 * The caller has taken lock_extent(), who could race with us 7558 * to add em? 7559 */ 7560 } while (ret == -EEXIST); 7561 7562 if (ret) { 7563 free_extent_map(em); 7564 return ERR_PTR(ret); 7565 } 7566 7567 /* em got 2 refs now, callers needs to do free_extent_map once. */ 7568 return em; 7569 } 7570 7571 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, 7572 struct buffer_head *bh_result, int create) 7573 { 7574 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7575 struct extent_map *em; 7576 struct extent_state *cached_state = NULL; 7577 struct btrfs_dio_data *dio_data = NULL; 7578 u64 start = iblock << inode->i_blkbits; 7579 u64 lockstart, lockend; 7580 u64 len = bh_result->b_size; 7581 int unlock_bits = EXTENT_LOCKED; 7582 int ret = 0; 7583 7584 if (create) 7585 unlock_bits |= EXTENT_DIRTY; 7586 else 7587 len = min_t(u64, len, fs_info->sectorsize); 7588 7589 lockstart = start; 7590 lockend = start + len - 1; 7591 7592 if (current->journal_info) { 7593 /* 7594 * Need to pull our outstanding extents and set journal_info to NULL so 7595 * that anything that needs to check if there's a transaction doesn't get 7596 * confused. 7597 */ 7598 dio_data = current->journal_info; 7599 current->journal_info = NULL; 7600 } 7601 7602 /* 7603 * If this errors out it's because we couldn't invalidate pagecache for 7604 * this range and we need to fallback to buffered. 7605 */ 7606 if (lock_extent_direct(inode, lockstart, lockend, &cached_state, 7607 create)) { 7608 ret = -ENOTBLK; 7609 goto err; 7610 } 7611 7612 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0); 7613 if (IS_ERR(em)) { 7614 ret = PTR_ERR(em); 7615 goto unlock_err; 7616 } 7617 7618 /* 7619 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7620 * io. INLINE is special, and we could probably kludge it in here, but 7621 * it's still buffered so for safety lets just fall back to the generic 7622 * buffered path. 7623 * 7624 * For COMPRESSED we _have_ to read the entire extent in so we can 7625 * decompress it, so there will be buffering required no matter what we 7626 * do, so go ahead and fallback to buffered. 7627 * 7628 * We return -ENOTBLK because that's what makes DIO go ahead and go back 7629 * to buffered IO. Don't blame me, this is the price we pay for using 7630 * the generic code. 7631 */ 7632 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7633 em->block_start == EXTENT_MAP_INLINE) { 7634 free_extent_map(em); 7635 ret = -ENOTBLK; 7636 goto unlock_err; 7637 } 7638 7639 /* Just a good old fashioned hole, return */ 7640 if (!create && (em->block_start == EXTENT_MAP_HOLE || 7641 test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) { 7642 free_extent_map(em); 7643 goto unlock_err; 7644 } 7645 7646 /* 7647 * We don't allocate a new extent in the following cases 7648 * 7649 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7650 * existing extent. 7651 * 2) The extent is marked as PREALLOC. We're good to go here and can 7652 * just use the extent. 7653 * 7654 */ 7655 if (!create) { 7656 len = min(len, em->len - (start - em->start)); 7657 lockstart = start + len; 7658 goto unlock; 7659 } 7660 7661 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7662 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7663 em->block_start != EXTENT_MAP_HOLE)) { 7664 int type; 7665 u64 block_start, orig_start, orig_block_len, ram_bytes; 7666 7667 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7668 type = BTRFS_ORDERED_PREALLOC; 7669 else 7670 type = BTRFS_ORDERED_NOCOW; 7671 len = min(len, em->len - (start - em->start)); 7672 block_start = em->block_start + (start - em->start); 7673 7674 if (can_nocow_extent(inode, start, &len, &orig_start, 7675 &orig_block_len, &ram_bytes) == 1 && 7676 btrfs_inc_nocow_writers(fs_info, block_start)) { 7677 struct extent_map *em2; 7678 7679 em2 = btrfs_create_dio_extent(inode, start, len, 7680 orig_start, block_start, 7681 len, orig_block_len, 7682 ram_bytes, type); 7683 btrfs_dec_nocow_writers(fs_info, block_start); 7684 if (type == BTRFS_ORDERED_PREALLOC) { 7685 free_extent_map(em); 7686 em = em2; 7687 } 7688 if (em2 && IS_ERR(em2)) { 7689 ret = PTR_ERR(em2); 7690 goto unlock_err; 7691 } 7692 /* 7693 * For inode marked NODATACOW or extent marked PREALLOC, 7694 * use the existing or preallocated extent, so does not 7695 * need to adjust btrfs_space_info's bytes_may_use. 7696 */ 7697 btrfs_free_reserved_data_space_noquota(inode, 7698 start, len); 7699 goto unlock; 7700 } 7701 } 7702 7703 /* 7704 * this will cow the extent, reset the len in case we changed 7705 * it above 7706 */ 7707 len = bh_result->b_size; 7708 free_extent_map(em); 7709 em = btrfs_new_extent_direct(inode, start, len); 7710 if (IS_ERR(em)) { 7711 ret = PTR_ERR(em); 7712 goto unlock_err; 7713 } 7714 len = min(len, em->len - (start - em->start)); 7715 unlock: 7716 bh_result->b_blocknr = (em->block_start + (start - em->start)) >> 7717 inode->i_blkbits; 7718 bh_result->b_size = len; 7719 bh_result->b_bdev = em->bdev; 7720 set_buffer_mapped(bh_result); 7721 if (create) { 7722 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7723 set_buffer_new(bh_result); 7724 7725 /* 7726 * Need to update the i_size under the extent lock so buffered 7727 * readers will get the updated i_size when we unlock. 7728 */ 7729 if (!dio_data->overwrite && start + len > i_size_read(inode)) 7730 i_size_write(inode, start + len); 7731 7732 WARN_ON(dio_data->reserve < len); 7733 dio_data->reserve -= len; 7734 dio_data->unsubmitted_oe_range_end = start + len; 7735 current->journal_info = dio_data; 7736 } 7737 7738 /* 7739 * In the case of write we need to clear and unlock the entire range, 7740 * in the case of read we need to unlock only the end area that we 7741 * aren't using if there is any left over space. 7742 */ 7743 if (lockstart < lockend) { 7744 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, 7745 lockend, unlock_bits, 1, 0, 7746 &cached_state); 7747 } else { 7748 free_extent_state(cached_state); 7749 } 7750 7751 free_extent_map(em); 7752 7753 return 0; 7754 7755 unlock_err: 7756 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7757 unlock_bits, 1, 0, &cached_state); 7758 err: 7759 if (dio_data) 7760 current->journal_info = dio_data; 7761 return ret; 7762 } 7763 7764 static inline blk_status_t submit_dio_repair_bio(struct inode *inode, 7765 struct bio *bio, 7766 int mirror_num) 7767 { 7768 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7769 blk_status_t ret; 7770 7771 BUG_ON(bio_op(bio) == REQ_OP_WRITE); 7772 7773 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR); 7774 if (ret) 7775 return ret; 7776 7777 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 7778 7779 return ret; 7780 } 7781 7782 static int btrfs_check_dio_repairable(struct inode *inode, 7783 struct bio *failed_bio, 7784 struct io_failure_record *failrec, 7785 int failed_mirror) 7786 { 7787 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7788 int num_copies; 7789 7790 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len); 7791 if (num_copies == 1) { 7792 /* 7793 * we only have a single copy of the data, so don't bother with 7794 * all the retry and error correction code that follows. no 7795 * matter what the error is, it is very likely to persist. 7796 */ 7797 btrfs_debug(fs_info, 7798 "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d", 7799 num_copies, failrec->this_mirror, failed_mirror); 7800 return 0; 7801 } 7802 7803 failrec->failed_mirror = failed_mirror; 7804 failrec->this_mirror++; 7805 if (failrec->this_mirror == failed_mirror) 7806 failrec->this_mirror++; 7807 7808 if (failrec->this_mirror > num_copies) { 7809 btrfs_debug(fs_info, 7810 "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d", 7811 num_copies, failrec->this_mirror, failed_mirror); 7812 return 0; 7813 } 7814 7815 return 1; 7816 } 7817 7818 static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio, 7819 struct page *page, unsigned int pgoff, 7820 u64 start, u64 end, int failed_mirror, 7821 bio_end_io_t *repair_endio, void *repair_arg) 7822 { 7823 struct io_failure_record *failrec; 7824 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7825 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 7826 struct bio *bio; 7827 int isector; 7828 unsigned int read_mode = 0; 7829 int segs; 7830 int ret; 7831 blk_status_t status; 7832 struct bio_vec bvec; 7833 7834 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); 7835 7836 ret = btrfs_get_io_failure_record(inode, start, end, &failrec); 7837 if (ret) 7838 return errno_to_blk_status(ret); 7839 7840 ret = btrfs_check_dio_repairable(inode, failed_bio, failrec, 7841 failed_mirror); 7842 if (!ret) { 7843 free_io_failure(failure_tree, io_tree, failrec); 7844 return BLK_STS_IOERR; 7845 } 7846 7847 segs = bio_segments(failed_bio); 7848 bio_get_first_bvec(failed_bio, &bvec); 7849 if (segs > 1 || 7850 (bvec.bv_len > btrfs_inode_sectorsize(inode))) 7851 read_mode |= REQ_FAILFAST_DEV; 7852 7853 isector = start - btrfs_io_bio(failed_bio)->logical; 7854 isector >>= inode->i_sb->s_blocksize_bits; 7855 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, 7856 pgoff, isector, repair_endio, repair_arg); 7857 bio_set_op_attrs(bio, REQ_OP_READ, read_mode); 7858 7859 btrfs_debug(BTRFS_I(inode)->root->fs_info, 7860 "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d", 7861 read_mode, failrec->this_mirror, failrec->in_validation); 7862 7863 status = submit_dio_repair_bio(inode, bio, failrec->this_mirror); 7864 if (status) { 7865 free_io_failure(failure_tree, io_tree, failrec); 7866 bio_put(bio); 7867 } 7868 7869 return status; 7870 } 7871 7872 struct btrfs_retry_complete { 7873 struct completion done; 7874 struct inode *inode; 7875 u64 start; 7876 int uptodate; 7877 }; 7878 7879 static void btrfs_retry_endio_nocsum(struct bio *bio) 7880 { 7881 struct btrfs_retry_complete *done = bio->bi_private; 7882 struct inode *inode = done->inode; 7883 struct bio_vec *bvec; 7884 struct extent_io_tree *io_tree, *failure_tree; 7885 int i; 7886 7887 if (bio->bi_status) 7888 goto end; 7889 7890 ASSERT(bio->bi_vcnt == 1); 7891 io_tree = &BTRFS_I(inode)->io_tree; 7892 failure_tree = &BTRFS_I(inode)->io_failure_tree; 7893 ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(inode)); 7894 7895 done->uptodate = 1; 7896 ASSERT(!bio_flagged(bio, BIO_CLONED)); 7897 bio_for_each_segment_all(bvec, bio, i) 7898 clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, 7899 io_tree, done->start, bvec->bv_page, 7900 btrfs_ino(BTRFS_I(inode)), 0); 7901 end: 7902 complete(&done->done); 7903 bio_put(bio); 7904 } 7905 7906 static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode, 7907 struct btrfs_io_bio *io_bio) 7908 { 7909 struct btrfs_fs_info *fs_info; 7910 struct bio_vec bvec; 7911 struct bvec_iter iter; 7912 struct btrfs_retry_complete done; 7913 u64 start; 7914 unsigned int pgoff; 7915 u32 sectorsize; 7916 int nr_sectors; 7917 blk_status_t ret; 7918 blk_status_t err = BLK_STS_OK; 7919 7920 fs_info = BTRFS_I(inode)->root->fs_info; 7921 sectorsize = fs_info->sectorsize; 7922 7923 start = io_bio->logical; 7924 done.inode = inode; 7925 io_bio->bio.bi_iter = io_bio->iter; 7926 7927 bio_for_each_segment(bvec, &io_bio->bio, iter) { 7928 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 7929 pgoff = bvec.bv_offset; 7930 7931 next_block_or_try_again: 7932 done.uptodate = 0; 7933 done.start = start; 7934 init_completion(&done.done); 7935 7936 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page, 7937 pgoff, start, start + sectorsize - 1, 7938 io_bio->mirror_num, 7939 btrfs_retry_endio_nocsum, &done); 7940 if (ret) { 7941 err = ret; 7942 goto next; 7943 } 7944 7945 wait_for_completion_io(&done.done); 7946 7947 if (!done.uptodate) { 7948 /* We might have another mirror, so try again */ 7949 goto next_block_or_try_again; 7950 } 7951 7952 next: 7953 start += sectorsize; 7954 7955 nr_sectors--; 7956 if (nr_sectors) { 7957 pgoff += sectorsize; 7958 ASSERT(pgoff < PAGE_SIZE); 7959 goto next_block_or_try_again; 7960 } 7961 } 7962 7963 return err; 7964 } 7965 7966 static void btrfs_retry_endio(struct bio *bio) 7967 { 7968 struct btrfs_retry_complete *done = bio->bi_private; 7969 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7970 struct extent_io_tree *io_tree, *failure_tree; 7971 struct inode *inode = done->inode; 7972 struct bio_vec *bvec; 7973 int uptodate; 7974 int ret; 7975 int i; 7976 7977 if (bio->bi_status) 7978 goto end; 7979 7980 uptodate = 1; 7981 7982 ASSERT(bio->bi_vcnt == 1); 7983 ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(done->inode)); 7984 7985 io_tree = &BTRFS_I(inode)->io_tree; 7986 failure_tree = &BTRFS_I(inode)->io_failure_tree; 7987 7988 ASSERT(!bio_flagged(bio, BIO_CLONED)); 7989 bio_for_each_segment_all(bvec, bio, i) { 7990 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page, 7991 bvec->bv_offset, done->start, 7992 bvec->bv_len); 7993 if (!ret) 7994 clean_io_failure(BTRFS_I(inode)->root->fs_info, 7995 failure_tree, io_tree, done->start, 7996 bvec->bv_page, 7997 btrfs_ino(BTRFS_I(inode)), 7998 bvec->bv_offset); 7999 else 8000 uptodate = 0; 8001 } 8002 8003 done->uptodate = uptodate; 8004 end: 8005 complete(&done->done); 8006 bio_put(bio); 8007 } 8008 8009 static blk_status_t __btrfs_subio_endio_read(struct inode *inode, 8010 struct btrfs_io_bio *io_bio, blk_status_t err) 8011 { 8012 struct btrfs_fs_info *fs_info; 8013 struct bio_vec bvec; 8014 struct bvec_iter iter; 8015 struct btrfs_retry_complete done; 8016 u64 start; 8017 u64 offset = 0; 8018 u32 sectorsize; 8019 int nr_sectors; 8020 unsigned int pgoff; 8021 int csum_pos; 8022 bool uptodate = (err == 0); 8023 int ret; 8024 blk_status_t status; 8025 8026 fs_info = BTRFS_I(inode)->root->fs_info; 8027 sectorsize = fs_info->sectorsize; 8028 8029 err = BLK_STS_OK; 8030 start = io_bio->logical; 8031 done.inode = inode; 8032 io_bio->bio.bi_iter = io_bio->iter; 8033 8034 bio_for_each_segment(bvec, &io_bio->bio, iter) { 8035 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 8036 8037 pgoff = bvec.bv_offset; 8038 next_block: 8039 if (uptodate) { 8040 csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset); 8041 ret = __readpage_endio_check(inode, io_bio, csum_pos, 8042 bvec.bv_page, pgoff, start, sectorsize); 8043 if (likely(!ret)) 8044 goto next; 8045 } 8046 try_again: 8047 done.uptodate = 0; 8048 done.start = start; 8049 init_completion(&done.done); 8050 8051 status = dio_read_error(inode, &io_bio->bio, bvec.bv_page, 8052 pgoff, start, start + sectorsize - 1, 8053 io_bio->mirror_num, btrfs_retry_endio, 8054 &done); 8055 if (status) { 8056 err = status; 8057 goto next; 8058 } 8059 8060 wait_for_completion_io(&done.done); 8061 8062 if (!done.uptodate) { 8063 /* We might have another mirror, so try again */ 8064 goto try_again; 8065 } 8066 next: 8067 offset += sectorsize; 8068 start += sectorsize; 8069 8070 ASSERT(nr_sectors); 8071 8072 nr_sectors--; 8073 if (nr_sectors) { 8074 pgoff += sectorsize; 8075 ASSERT(pgoff < PAGE_SIZE); 8076 goto next_block; 8077 } 8078 } 8079 8080 return err; 8081 } 8082 8083 static blk_status_t btrfs_subio_endio_read(struct inode *inode, 8084 struct btrfs_io_bio *io_bio, blk_status_t err) 8085 { 8086 bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 8087 8088 if (skip_csum) { 8089 if (unlikely(err)) 8090 return __btrfs_correct_data_nocsum(inode, io_bio); 8091 else 8092 return BLK_STS_OK; 8093 } else { 8094 return __btrfs_subio_endio_read(inode, io_bio, err); 8095 } 8096 } 8097 8098 static void btrfs_endio_direct_read(struct bio *bio) 8099 { 8100 struct btrfs_dio_private *dip = bio->bi_private; 8101 struct inode *inode = dip->inode; 8102 struct bio *dio_bio; 8103 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8104 blk_status_t err = bio->bi_status; 8105 8106 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) 8107 err = btrfs_subio_endio_read(inode, io_bio, err); 8108 8109 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset, 8110 dip->logical_offset + dip->bytes - 1); 8111 dio_bio = dip->dio_bio; 8112 8113 kfree(dip); 8114 8115 dio_bio->bi_status = err; 8116 dio_end_io(dio_bio); 8117 8118 if (io_bio->end_io) 8119 io_bio->end_io(io_bio, blk_status_to_errno(err)); 8120 bio_put(bio); 8121 } 8122 8123 static void __endio_write_update_ordered(struct inode *inode, 8124 const u64 offset, const u64 bytes, 8125 const bool uptodate) 8126 { 8127 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8128 struct btrfs_ordered_extent *ordered = NULL; 8129 struct btrfs_workqueue *wq; 8130 btrfs_work_func_t func; 8131 u64 ordered_offset = offset; 8132 u64 ordered_bytes = bytes; 8133 u64 last_offset; 8134 int ret; 8135 8136 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 8137 wq = fs_info->endio_freespace_worker; 8138 func = btrfs_freespace_write_helper; 8139 } else { 8140 wq = fs_info->endio_write_workers; 8141 func = btrfs_endio_write_helper; 8142 } 8143 8144 again: 8145 last_offset = ordered_offset; 8146 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, 8147 &ordered_offset, 8148 ordered_bytes, 8149 uptodate); 8150 if (!ret) 8151 goto out_test; 8152 8153 btrfs_init_work(&ordered->work, func, finish_ordered_fn, NULL, NULL); 8154 btrfs_queue_work(wq, &ordered->work); 8155 out_test: 8156 /* 8157 * If btrfs_dec_test_ordered_pending does not find any ordered extent 8158 * in the range, we can exit. 8159 */ 8160 if (ordered_offset == last_offset) 8161 return; 8162 /* 8163 * our bio might span multiple ordered extents. If we haven't 8164 * completed the accounting for the whole dio, go back and try again 8165 */ 8166 if (ordered_offset < offset + bytes) { 8167 ordered_bytes = offset + bytes - ordered_offset; 8168 ordered = NULL; 8169 goto again; 8170 } 8171 } 8172 8173 static void btrfs_endio_direct_write(struct bio *bio) 8174 { 8175 struct btrfs_dio_private *dip = bio->bi_private; 8176 struct bio *dio_bio = dip->dio_bio; 8177 8178 __endio_write_update_ordered(dip->inode, dip->logical_offset, 8179 dip->bytes, !bio->bi_status); 8180 8181 kfree(dip); 8182 8183 dio_bio->bi_status = bio->bi_status; 8184 dio_end_io(dio_bio); 8185 bio_put(bio); 8186 } 8187 8188 static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data, 8189 struct bio *bio, u64 offset) 8190 { 8191 struct inode *inode = private_data; 8192 blk_status_t ret; 8193 ret = btrfs_csum_one_bio(inode, bio, offset, 1); 8194 BUG_ON(ret); /* -ENOMEM */ 8195 return 0; 8196 } 8197 8198 static void btrfs_end_dio_bio(struct bio *bio) 8199 { 8200 struct btrfs_dio_private *dip = bio->bi_private; 8201 blk_status_t err = bio->bi_status; 8202 8203 if (err) 8204 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 8205 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", 8206 btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio), 8207 bio->bi_opf, 8208 (unsigned long long)bio->bi_iter.bi_sector, 8209 bio->bi_iter.bi_size, err); 8210 8211 if (dip->subio_endio) 8212 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err); 8213 8214 if (err) { 8215 /* 8216 * We want to perceive the errors flag being set before 8217 * decrementing the reference count. We don't need a barrier 8218 * since atomic operations with a return value are fully 8219 * ordered as per atomic_t.txt 8220 */ 8221 dip->errors = 1; 8222 } 8223 8224 /* if there are more bios still pending for this dio, just exit */ 8225 if (!atomic_dec_and_test(&dip->pending_bios)) 8226 goto out; 8227 8228 if (dip->errors) { 8229 bio_io_error(dip->orig_bio); 8230 } else { 8231 dip->dio_bio->bi_status = BLK_STS_OK; 8232 bio_endio(dip->orig_bio); 8233 } 8234 out: 8235 bio_put(bio); 8236 } 8237 8238 static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode, 8239 struct btrfs_dio_private *dip, 8240 struct bio *bio, 8241 u64 file_offset) 8242 { 8243 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8244 struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio); 8245 blk_status_t ret; 8246 8247 /* 8248 * We load all the csum data we need when we submit 8249 * the first bio to reduce the csum tree search and 8250 * contention. 8251 */ 8252 if (dip->logical_offset == file_offset) { 8253 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio, 8254 file_offset); 8255 if (ret) 8256 return ret; 8257 } 8258 8259 if (bio == dip->orig_bio) 8260 return 0; 8261 8262 file_offset -= dip->logical_offset; 8263 file_offset >>= inode->i_sb->s_blocksize_bits; 8264 io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset); 8265 8266 return 0; 8267 } 8268 8269 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, 8270 struct inode *inode, u64 file_offset, int async_submit) 8271 { 8272 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8273 struct btrfs_dio_private *dip = bio->bi_private; 8274 bool write = bio_op(bio) == REQ_OP_WRITE; 8275 blk_status_t ret; 8276 8277 /* Check btrfs_submit_bio_hook() for rules about async submit. */ 8278 if (async_submit) 8279 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers); 8280 8281 if (!write) { 8282 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 8283 if (ret) 8284 goto err; 8285 } 8286 8287 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 8288 goto map; 8289 8290 if (write && async_submit) { 8291 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0, 8292 file_offset, inode, 8293 btrfs_submit_bio_start_direct_io, 8294 btrfs_submit_bio_done); 8295 goto err; 8296 } else if (write) { 8297 /* 8298 * If we aren't doing async submit, calculate the csum of the 8299 * bio now. 8300 */ 8301 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1); 8302 if (ret) 8303 goto err; 8304 } else { 8305 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio, 8306 file_offset); 8307 if (ret) 8308 goto err; 8309 } 8310 map: 8311 ret = btrfs_map_bio(fs_info, bio, 0, 0); 8312 err: 8313 return ret; 8314 } 8315 8316 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) 8317 { 8318 struct inode *inode = dip->inode; 8319 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8320 struct bio *bio; 8321 struct bio *orig_bio = dip->orig_bio; 8322 u64 start_sector = orig_bio->bi_iter.bi_sector; 8323 u64 file_offset = dip->logical_offset; 8324 u64 map_length; 8325 int async_submit = 0; 8326 u64 submit_len; 8327 int clone_offset = 0; 8328 int clone_len; 8329 int ret; 8330 blk_status_t status; 8331 8332 map_length = orig_bio->bi_iter.bi_size; 8333 submit_len = map_length; 8334 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9, 8335 &map_length, NULL, 0); 8336 if (ret) 8337 return -EIO; 8338 8339 if (map_length >= submit_len) { 8340 bio = orig_bio; 8341 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED; 8342 goto submit; 8343 } 8344 8345 /* async crcs make it difficult to collect full stripe writes. */ 8346 if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK) 8347 async_submit = 0; 8348 else 8349 async_submit = 1; 8350 8351 /* bio split */ 8352 ASSERT(map_length <= INT_MAX); 8353 atomic_inc(&dip->pending_bios); 8354 do { 8355 clone_len = min_t(int, submit_len, map_length); 8356 8357 /* 8358 * This will never fail as it's passing GPF_NOFS and 8359 * the allocation is backed by btrfs_bioset. 8360 */ 8361 bio = btrfs_bio_clone_partial(orig_bio, clone_offset, 8362 clone_len); 8363 bio->bi_private = dip; 8364 bio->bi_end_io = btrfs_end_dio_bio; 8365 btrfs_io_bio(bio)->logical = file_offset; 8366 8367 ASSERT(submit_len >= clone_len); 8368 submit_len -= clone_len; 8369 if (submit_len == 0) 8370 break; 8371 8372 /* 8373 * Increase the count before we submit the bio so we know 8374 * the end IO handler won't happen before we increase the 8375 * count. Otherwise, the dip might get freed before we're 8376 * done setting it up. 8377 */ 8378 atomic_inc(&dip->pending_bios); 8379 8380 status = btrfs_submit_dio_bio(bio, inode, file_offset, 8381 async_submit); 8382 if (status) { 8383 bio_put(bio); 8384 atomic_dec(&dip->pending_bios); 8385 goto out_err; 8386 } 8387 8388 clone_offset += clone_len; 8389 start_sector += clone_len >> 9; 8390 file_offset += clone_len; 8391 8392 map_length = submit_len; 8393 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), 8394 start_sector << 9, &map_length, NULL, 0); 8395 if (ret) 8396 goto out_err; 8397 } while (submit_len > 0); 8398 8399 submit: 8400 status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit); 8401 if (!status) 8402 return 0; 8403 8404 bio_put(bio); 8405 out_err: 8406 dip->errors = 1; 8407 /* 8408 * Before atomic variable goto zero, we must make sure dip->errors is 8409 * perceived to be set. This ordering is ensured by the fact that an 8410 * atomic operations with a return value are fully ordered as per 8411 * atomic_t.txt 8412 */ 8413 if (atomic_dec_and_test(&dip->pending_bios)) 8414 bio_io_error(dip->orig_bio); 8415 8416 /* bio_end_io() will handle error, so we needn't return it */ 8417 return 0; 8418 } 8419 8420 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, 8421 loff_t file_offset) 8422 { 8423 struct btrfs_dio_private *dip = NULL; 8424 struct bio *bio = NULL; 8425 struct btrfs_io_bio *io_bio; 8426 bool write = (bio_op(dio_bio) == REQ_OP_WRITE); 8427 int ret = 0; 8428 8429 bio = btrfs_bio_clone(dio_bio); 8430 8431 dip = kzalloc(sizeof(*dip), GFP_NOFS); 8432 if (!dip) { 8433 ret = -ENOMEM; 8434 goto free_ordered; 8435 } 8436 8437 dip->private = dio_bio->bi_private; 8438 dip->inode = inode; 8439 dip->logical_offset = file_offset; 8440 dip->bytes = dio_bio->bi_iter.bi_size; 8441 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; 8442 bio->bi_private = dip; 8443 dip->orig_bio = bio; 8444 dip->dio_bio = dio_bio; 8445 atomic_set(&dip->pending_bios, 0); 8446 io_bio = btrfs_io_bio(bio); 8447 io_bio->logical = file_offset; 8448 8449 if (write) { 8450 bio->bi_end_io = btrfs_endio_direct_write; 8451 } else { 8452 bio->bi_end_io = btrfs_endio_direct_read; 8453 dip->subio_endio = btrfs_subio_endio_read; 8454 } 8455 8456 /* 8457 * Reset the range for unsubmitted ordered extents (to a 0 length range) 8458 * even if we fail to submit a bio, because in such case we do the 8459 * corresponding error handling below and it must not be done a second 8460 * time by btrfs_direct_IO(). 8461 */ 8462 if (write) { 8463 struct btrfs_dio_data *dio_data = current->journal_info; 8464 8465 dio_data->unsubmitted_oe_range_end = dip->logical_offset + 8466 dip->bytes; 8467 dio_data->unsubmitted_oe_range_start = 8468 dio_data->unsubmitted_oe_range_end; 8469 } 8470 8471 ret = btrfs_submit_direct_hook(dip); 8472 if (!ret) 8473 return; 8474 8475 if (io_bio->end_io) 8476 io_bio->end_io(io_bio, ret); 8477 8478 free_ordered: 8479 /* 8480 * If we arrived here it means either we failed to submit the dip 8481 * or we either failed to clone the dio_bio or failed to allocate the 8482 * dip. If we cloned the dio_bio and allocated the dip, we can just 8483 * call bio_endio against our io_bio so that we get proper resource 8484 * cleanup if we fail to submit the dip, otherwise, we must do the 8485 * same as btrfs_endio_direct_[write|read] because we can't call these 8486 * callbacks - they require an allocated dip and a clone of dio_bio. 8487 */ 8488 if (bio && dip) { 8489 bio_io_error(bio); 8490 /* 8491 * The end io callbacks free our dip, do the final put on bio 8492 * and all the cleanup and final put for dio_bio (through 8493 * dio_end_io()). 8494 */ 8495 dip = NULL; 8496 bio = NULL; 8497 } else { 8498 if (write) 8499 __endio_write_update_ordered(inode, 8500 file_offset, 8501 dio_bio->bi_iter.bi_size, 8502 false); 8503 else 8504 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, 8505 file_offset + dio_bio->bi_iter.bi_size - 1); 8506 8507 dio_bio->bi_status = BLK_STS_IOERR; 8508 /* 8509 * Releases and cleans up our dio_bio, no need to bio_put() 8510 * nor bio_endio()/bio_io_error() against dio_bio. 8511 */ 8512 dio_end_io(dio_bio); 8513 } 8514 if (bio) 8515 bio_put(bio); 8516 kfree(dip); 8517 } 8518 8519 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, 8520 const struct iov_iter *iter, loff_t offset) 8521 { 8522 int seg; 8523 int i; 8524 unsigned int blocksize_mask = fs_info->sectorsize - 1; 8525 ssize_t retval = -EINVAL; 8526 8527 if (offset & blocksize_mask) 8528 goto out; 8529 8530 if (iov_iter_alignment(iter) & blocksize_mask) 8531 goto out; 8532 8533 /* If this is a write we don't need to check anymore */ 8534 if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter)) 8535 return 0; 8536 /* 8537 * Check to make sure we don't have duplicate iov_base's in this 8538 * iovec, if so return EINVAL, otherwise we'll get csum errors 8539 * when reading back. 8540 */ 8541 for (seg = 0; seg < iter->nr_segs; seg++) { 8542 for (i = seg + 1; i < iter->nr_segs; i++) { 8543 if (iter->iov[seg].iov_base == iter->iov[i].iov_base) 8544 goto out; 8545 } 8546 } 8547 retval = 0; 8548 out: 8549 return retval; 8550 } 8551 8552 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 8553 { 8554 struct file *file = iocb->ki_filp; 8555 struct inode *inode = file->f_mapping->host; 8556 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8557 struct btrfs_dio_data dio_data = { 0 }; 8558 struct extent_changeset *data_reserved = NULL; 8559 loff_t offset = iocb->ki_pos; 8560 size_t count = 0; 8561 int flags = 0; 8562 bool wakeup = true; 8563 bool relock = false; 8564 ssize_t ret; 8565 8566 if (check_direct_IO(fs_info, iter, offset)) 8567 return 0; 8568 8569 inode_dio_begin(inode); 8570 8571 /* 8572 * The generic stuff only does filemap_write_and_wait_range, which 8573 * isn't enough if we've written compressed pages to this area, so 8574 * we need to flush the dirty pages again to make absolutely sure 8575 * that any outstanding dirty pages are on disk. 8576 */ 8577 count = iov_iter_count(iter); 8578 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 8579 &BTRFS_I(inode)->runtime_flags)) 8580 filemap_fdatawrite_range(inode->i_mapping, offset, 8581 offset + count - 1); 8582 8583 if (iov_iter_rw(iter) == WRITE) { 8584 /* 8585 * If the write DIO is beyond the EOF, we need update 8586 * the isize, but it is protected by i_mutex. So we can 8587 * not unlock the i_mutex at this case. 8588 */ 8589 if (offset + count <= inode->i_size) { 8590 dio_data.overwrite = 1; 8591 inode_unlock(inode); 8592 relock = true; 8593 } else if (iocb->ki_flags & IOCB_NOWAIT) { 8594 ret = -EAGAIN; 8595 goto out; 8596 } 8597 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, 8598 offset, count); 8599 if (ret) 8600 goto out; 8601 8602 /* 8603 * We need to know how many extents we reserved so that we can 8604 * do the accounting properly if we go over the number we 8605 * originally calculated. Abuse current->journal_info for this. 8606 */ 8607 dio_data.reserve = round_up(count, 8608 fs_info->sectorsize); 8609 dio_data.unsubmitted_oe_range_start = (u64)offset; 8610 dio_data.unsubmitted_oe_range_end = (u64)offset; 8611 current->journal_info = &dio_data; 8612 down_read(&BTRFS_I(inode)->dio_sem); 8613 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK, 8614 &BTRFS_I(inode)->runtime_flags)) { 8615 inode_dio_end(inode); 8616 flags = DIO_LOCKING | DIO_SKIP_HOLES; 8617 wakeup = false; 8618 } 8619 8620 ret = __blockdev_direct_IO(iocb, inode, 8621 fs_info->fs_devices->latest_bdev, 8622 iter, btrfs_get_blocks_direct, NULL, 8623 btrfs_submit_direct, flags); 8624 if (iov_iter_rw(iter) == WRITE) { 8625 up_read(&BTRFS_I(inode)->dio_sem); 8626 current->journal_info = NULL; 8627 if (ret < 0 && ret != -EIOCBQUEUED) { 8628 if (dio_data.reserve) 8629 btrfs_delalloc_release_space(inode, data_reserved, 8630 offset, dio_data.reserve, true); 8631 /* 8632 * On error we might have left some ordered extents 8633 * without submitting corresponding bios for them, so 8634 * cleanup them up to avoid other tasks getting them 8635 * and waiting for them to complete forever. 8636 */ 8637 if (dio_data.unsubmitted_oe_range_start < 8638 dio_data.unsubmitted_oe_range_end) 8639 __endio_write_update_ordered(inode, 8640 dio_data.unsubmitted_oe_range_start, 8641 dio_data.unsubmitted_oe_range_end - 8642 dio_data.unsubmitted_oe_range_start, 8643 false); 8644 } else if (ret >= 0 && (size_t)ret < count) 8645 btrfs_delalloc_release_space(inode, data_reserved, 8646 offset, count - (size_t)ret, true); 8647 btrfs_delalloc_release_extents(BTRFS_I(inode), count, false); 8648 } 8649 out: 8650 if (wakeup) 8651 inode_dio_end(inode); 8652 if (relock) 8653 inode_lock(inode); 8654 8655 extent_changeset_free(data_reserved); 8656 return ret; 8657 } 8658 8659 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC) 8660 8661 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 8662 __u64 start, __u64 len) 8663 { 8664 int ret; 8665 8666 ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS); 8667 if (ret) 8668 return ret; 8669 8670 return extent_fiemap(inode, fieinfo, start, len); 8671 } 8672 8673 int btrfs_readpage(struct file *file, struct page *page) 8674 { 8675 struct extent_io_tree *tree; 8676 tree = &BTRFS_I(page->mapping->host)->io_tree; 8677 return extent_read_full_page(tree, page, btrfs_get_extent, 0); 8678 } 8679 8680 static int btrfs_writepage(struct page *page, struct writeback_control *wbc) 8681 { 8682 struct inode *inode = page->mapping->host; 8683 int ret; 8684 8685 if (current->flags & PF_MEMALLOC) { 8686 redirty_page_for_writepage(wbc, page); 8687 unlock_page(page); 8688 return 0; 8689 } 8690 8691 /* 8692 * If we are under memory pressure we will call this directly from the 8693 * VM, we need to make sure we have the inode referenced for the ordered 8694 * extent. If not just return like we didn't do anything. 8695 */ 8696 if (!igrab(inode)) { 8697 redirty_page_for_writepage(wbc, page); 8698 return AOP_WRITEPAGE_ACTIVATE; 8699 } 8700 ret = extent_write_full_page(page, wbc); 8701 btrfs_add_delayed_iput(inode); 8702 return ret; 8703 } 8704 8705 static int btrfs_writepages(struct address_space *mapping, 8706 struct writeback_control *wbc) 8707 { 8708 struct extent_io_tree *tree; 8709 8710 tree = &BTRFS_I(mapping->host)->io_tree; 8711 return extent_writepages(tree, mapping, wbc); 8712 } 8713 8714 static int 8715 btrfs_readpages(struct file *file, struct address_space *mapping, 8716 struct list_head *pages, unsigned nr_pages) 8717 { 8718 struct extent_io_tree *tree; 8719 tree = &BTRFS_I(mapping->host)->io_tree; 8720 return extent_readpages(tree, mapping, pages, nr_pages); 8721 } 8722 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8723 { 8724 struct extent_io_tree *tree; 8725 struct extent_map_tree *map; 8726 int ret; 8727 8728 tree = &BTRFS_I(page->mapping->host)->io_tree; 8729 map = &BTRFS_I(page->mapping->host)->extent_tree; 8730 ret = try_release_extent_mapping(map, tree, page, gfp_flags); 8731 if (ret == 1) { 8732 ClearPagePrivate(page); 8733 set_page_private(page, 0); 8734 put_page(page); 8735 } 8736 return ret; 8737 } 8738 8739 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8740 { 8741 if (PageWriteback(page) || PageDirty(page)) 8742 return 0; 8743 return __btrfs_releasepage(page, gfp_flags); 8744 } 8745 8746 static void btrfs_invalidatepage(struct page *page, unsigned int offset, 8747 unsigned int length) 8748 { 8749 struct inode *inode = page->mapping->host; 8750 struct extent_io_tree *tree; 8751 struct btrfs_ordered_extent *ordered; 8752 struct extent_state *cached_state = NULL; 8753 u64 page_start = page_offset(page); 8754 u64 page_end = page_start + PAGE_SIZE - 1; 8755 u64 start; 8756 u64 end; 8757 int inode_evicting = inode->i_state & I_FREEING; 8758 8759 /* 8760 * we have the page locked, so new writeback can't start, 8761 * and the dirty bit won't be cleared while we are here. 8762 * 8763 * Wait for IO on this page so that we can safely clear 8764 * the PagePrivate2 bit and do ordered accounting 8765 */ 8766 wait_on_page_writeback(page); 8767 8768 tree = &BTRFS_I(inode)->io_tree; 8769 if (offset) { 8770 btrfs_releasepage(page, GFP_NOFS); 8771 return; 8772 } 8773 8774 if (!inode_evicting) 8775 lock_extent_bits(tree, page_start, page_end, &cached_state); 8776 again: 8777 start = page_start; 8778 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start, 8779 page_end - start + 1); 8780 if (ordered) { 8781 end = min(page_end, ordered->file_offset + ordered->len - 1); 8782 /* 8783 * IO on this page will never be started, so we need 8784 * to account for any ordered extents now 8785 */ 8786 if (!inode_evicting) 8787 clear_extent_bit(tree, start, end, 8788 EXTENT_DIRTY | EXTENT_DELALLOC | 8789 EXTENT_DELALLOC_NEW | 8790 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 8791 EXTENT_DEFRAG, 1, 0, &cached_state); 8792 /* 8793 * whoever cleared the private bit is responsible 8794 * for the finish_ordered_io 8795 */ 8796 if (TestClearPagePrivate2(page)) { 8797 struct btrfs_ordered_inode_tree *tree; 8798 u64 new_len; 8799 8800 tree = &BTRFS_I(inode)->ordered_tree; 8801 8802 spin_lock_irq(&tree->lock); 8803 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 8804 new_len = start - ordered->file_offset; 8805 if (new_len < ordered->truncated_len) 8806 ordered->truncated_len = new_len; 8807 spin_unlock_irq(&tree->lock); 8808 8809 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8810 start, 8811 end - start + 1, 1)) 8812 btrfs_finish_ordered_io(ordered); 8813 } 8814 btrfs_put_ordered_extent(ordered); 8815 if (!inode_evicting) { 8816 cached_state = NULL; 8817 lock_extent_bits(tree, start, end, 8818 &cached_state); 8819 } 8820 8821 start = end + 1; 8822 if (start < page_end) 8823 goto again; 8824 } 8825 8826 /* 8827 * Qgroup reserved space handler 8828 * Page here will be either 8829 * 1) Already written to disk 8830 * In this case, its reserved space is released from data rsv map 8831 * and will be freed by delayed_ref handler finally. 8832 * So even we call qgroup_free_data(), it won't decrease reserved 8833 * space. 8834 * 2) Not written to disk 8835 * This means the reserved space should be freed here. However, 8836 * if a truncate invalidates the page (by clearing PageDirty) 8837 * and the page is accounted for while allocating extent 8838 * in btrfs_check_data_free_space() we let delayed_ref to 8839 * free the entire extent. 8840 */ 8841 if (PageDirty(page)) 8842 btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE); 8843 if (!inode_evicting) { 8844 clear_extent_bit(tree, page_start, page_end, 8845 EXTENT_LOCKED | EXTENT_DIRTY | 8846 EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 8847 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1, 8848 &cached_state); 8849 8850 __btrfs_releasepage(page, GFP_NOFS); 8851 } 8852 8853 ClearPageChecked(page); 8854 if (PagePrivate(page)) { 8855 ClearPagePrivate(page); 8856 set_page_private(page, 0); 8857 put_page(page); 8858 } 8859 } 8860 8861 /* 8862 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8863 * called from a page fault handler when a page is first dirtied. Hence we must 8864 * be careful to check for EOF conditions here. We set the page up correctly 8865 * for a written page which means we get ENOSPC checking when writing into 8866 * holes and correct delalloc and unwritten extent mapping on filesystems that 8867 * support these features. 8868 * 8869 * We are not allowed to take the i_mutex here so we have to play games to 8870 * protect against truncate races as the page could now be beyond EOF. Because 8871 * vmtruncate() writes the inode size before removing pages, once we have the 8872 * page lock we can determine safely if the page is beyond EOF. If it is not 8873 * beyond EOF, then the page is guaranteed safe against truncation until we 8874 * unlock the page. 8875 */ 8876 int btrfs_page_mkwrite(struct vm_fault *vmf) 8877 { 8878 struct page *page = vmf->page; 8879 struct inode *inode = file_inode(vmf->vma->vm_file); 8880 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8881 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8882 struct btrfs_ordered_extent *ordered; 8883 struct extent_state *cached_state = NULL; 8884 struct extent_changeset *data_reserved = NULL; 8885 char *kaddr; 8886 unsigned long zero_start; 8887 loff_t size; 8888 int ret; 8889 int reserved = 0; 8890 u64 reserved_space; 8891 u64 page_start; 8892 u64 page_end; 8893 u64 end; 8894 8895 reserved_space = PAGE_SIZE; 8896 8897 sb_start_pagefault(inode->i_sb); 8898 page_start = page_offset(page); 8899 page_end = page_start + PAGE_SIZE - 1; 8900 end = page_end; 8901 8902 /* 8903 * Reserving delalloc space after obtaining the page lock can lead to 8904 * deadlock. For example, if a dirty page is locked by this function 8905 * and the call to btrfs_delalloc_reserve_space() ends up triggering 8906 * dirty page write out, then the btrfs_writepage() function could 8907 * end up waiting indefinitely to get a lock on the page currently 8908 * being processed by btrfs_page_mkwrite() function. 8909 */ 8910 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 8911 reserved_space); 8912 if (!ret) { 8913 ret = file_update_time(vmf->vma->vm_file); 8914 reserved = 1; 8915 } 8916 if (ret) { 8917 if (ret == -ENOMEM) 8918 ret = VM_FAULT_OOM; 8919 else /* -ENOSPC, -EIO, etc */ 8920 ret = VM_FAULT_SIGBUS; 8921 if (reserved) 8922 goto out; 8923 goto out_noreserve; 8924 } 8925 8926 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 8927 again: 8928 lock_page(page); 8929 size = i_size_read(inode); 8930 8931 if ((page->mapping != inode->i_mapping) || 8932 (page_start >= size)) { 8933 /* page got truncated out from underneath us */ 8934 goto out_unlock; 8935 } 8936 wait_on_page_writeback(page); 8937 8938 lock_extent_bits(io_tree, page_start, page_end, &cached_state); 8939 set_page_extent_mapped(page); 8940 8941 /* 8942 * we can't set the delalloc bits if there are pending ordered 8943 * extents. Drop our locks and wait for them to finish 8944 */ 8945 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 8946 PAGE_SIZE); 8947 if (ordered) { 8948 unlock_extent_cached(io_tree, page_start, page_end, 8949 &cached_state); 8950 unlock_page(page); 8951 btrfs_start_ordered_extent(inode, ordered, 1); 8952 btrfs_put_ordered_extent(ordered); 8953 goto again; 8954 } 8955 8956 if (page->index == ((size - 1) >> PAGE_SHIFT)) { 8957 reserved_space = round_up(size - page_start, 8958 fs_info->sectorsize); 8959 if (reserved_space < PAGE_SIZE) { 8960 end = page_start + reserved_space - 1; 8961 btrfs_delalloc_release_space(inode, data_reserved, 8962 page_start, PAGE_SIZE - reserved_space, 8963 true); 8964 } 8965 } 8966 8967 /* 8968 * page_mkwrite gets called when the page is firstly dirtied after it's 8969 * faulted in, but write(2) could also dirty a page and set delalloc 8970 * bits, thus in this case for space account reason, we still need to 8971 * clear any delalloc bits within this page range since we have to 8972 * reserve data&meta space before lock_page() (see above comments). 8973 */ 8974 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, 8975 EXTENT_DIRTY | EXTENT_DELALLOC | 8976 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 8977 0, 0, &cached_state); 8978 8979 ret = btrfs_set_extent_delalloc(inode, page_start, end, 0, 8980 &cached_state, 0); 8981 if (ret) { 8982 unlock_extent_cached(io_tree, page_start, page_end, 8983 &cached_state); 8984 ret = VM_FAULT_SIGBUS; 8985 goto out_unlock; 8986 } 8987 ret = 0; 8988 8989 /* page is wholly or partially inside EOF */ 8990 if (page_start + PAGE_SIZE > size) 8991 zero_start = size & ~PAGE_MASK; 8992 else 8993 zero_start = PAGE_SIZE; 8994 8995 if (zero_start != PAGE_SIZE) { 8996 kaddr = kmap(page); 8997 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start); 8998 flush_dcache_page(page); 8999 kunmap(page); 9000 } 9001 ClearPageChecked(page); 9002 set_page_dirty(page); 9003 SetPageUptodate(page); 9004 9005 BTRFS_I(inode)->last_trans = fs_info->generation; 9006 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; 9007 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; 9008 9009 unlock_extent_cached(io_tree, page_start, page_end, &cached_state); 9010 9011 out_unlock: 9012 if (!ret) { 9013 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, true); 9014 sb_end_pagefault(inode->i_sb); 9015 extent_changeset_free(data_reserved); 9016 return VM_FAULT_LOCKED; 9017 } 9018 unlock_page(page); 9019 out: 9020 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, (ret != 0)); 9021 btrfs_delalloc_release_space(inode, data_reserved, page_start, 9022 reserved_space, (ret != 0)); 9023 out_noreserve: 9024 sb_end_pagefault(inode->i_sb); 9025 extent_changeset_free(data_reserved); 9026 return ret; 9027 } 9028 9029 static int btrfs_truncate(struct inode *inode, bool skip_writeback) 9030 { 9031 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9032 struct btrfs_root *root = BTRFS_I(inode)->root; 9033 struct btrfs_block_rsv *rsv; 9034 int ret = 0; 9035 int err = 0; 9036 struct btrfs_trans_handle *trans; 9037 u64 mask = fs_info->sectorsize - 1; 9038 u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 9039 9040 if (!skip_writeback) { 9041 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), 9042 (u64)-1); 9043 if (ret) 9044 return ret; 9045 } 9046 9047 /* 9048 * Yes ladies and gentlemen, this is indeed ugly. The fact is we have 9049 * 3 things going on here 9050 * 9051 * 1) We need to reserve space for our orphan item and the space to 9052 * delete our orphan item. Lord knows we don't want to have a dangling 9053 * orphan item because we didn't reserve space to remove it. 9054 * 9055 * 2) We need to reserve space to update our inode. 9056 * 9057 * 3) We need to have something to cache all the space that is going to 9058 * be free'd up by the truncate operation, but also have some slack 9059 * space reserved in case it uses space during the truncate (thank you 9060 * very much snapshotting). 9061 * 9062 * And we need these to all be separate. The fact is we can use a lot of 9063 * space doing the truncate, and we have no earthly idea how much space 9064 * we will use, so we need the truncate reservation to be separate so it 9065 * doesn't end up using space reserved for updating the inode or 9066 * removing the orphan item. We also need to be able to stop the 9067 * transaction and start a new one, which means we need to be able to 9068 * update the inode several times, and we have no idea of knowing how 9069 * many times that will be, so we can't just reserve 1 item for the 9070 * entirety of the operation, so that has to be done separately as well. 9071 * Then there is the orphan item, which does indeed need to be held on 9072 * to for the whole operation, and we need nobody to touch this reserved 9073 * space except the orphan code. 9074 * 9075 * So that leaves us with 9076 * 9077 * 1) root->orphan_block_rsv - for the orphan deletion. 9078 * 2) rsv - for the truncate reservation, which we will steal from the 9079 * transaction reservation. 9080 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for 9081 * updating the inode. 9082 */ 9083 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 9084 if (!rsv) 9085 return -ENOMEM; 9086 rsv->size = min_size; 9087 rsv->failfast = 1; 9088 9089 /* 9090 * 1 for the truncate slack space 9091 * 1 for updating the inode. 9092 */ 9093 trans = btrfs_start_transaction(root, 2); 9094 if (IS_ERR(trans)) { 9095 err = PTR_ERR(trans); 9096 goto out; 9097 } 9098 9099 /* Migrate the slack space for the truncate to our reserve */ 9100 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv, 9101 min_size, 0); 9102 BUG_ON(ret); 9103 9104 /* 9105 * So if we truncate and then write and fsync we normally would just 9106 * write the extents that changed, which is a problem if we need to 9107 * first truncate that entire inode. So set this flag so we write out 9108 * all of the extents in the inode to the sync log so we're completely 9109 * safe. 9110 */ 9111 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 9112 trans->block_rsv = rsv; 9113 9114 while (1) { 9115 ret = btrfs_truncate_inode_items(trans, root, inode, 9116 inode->i_size, 9117 BTRFS_EXTENT_DATA_KEY); 9118 trans->block_rsv = &fs_info->trans_block_rsv; 9119 if (ret != -ENOSPC && ret != -EAGAIN) { 9120 err = ret; 9121 break; 9122 } 9123 9124 ret = btrfs_update_inode(trans, root, inode); 9125 if (ret) { 9126 err = ret; 9127 break; 9128 } 9129 9130 btrfs_end_transaction(trans); 9131 btrfs_btree_balance_dirty(fs_info); 9132 9133 trans = btrfs_start_transaction(root, 2); 9134 if (IS_ERR(trans)) { 9135 ret = err = PTR_ERR(trans); 9136 trans = NULL; 9137 break; 9138 } 9139 9140 btrfs_block_rsv_release(fs_info, rsv, -1); 9141 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 9142 rsv, min_size, 0); 9143 BUG_ON(ret); /* shouldn't happen */ 9144 trans->block_rsv = rsv; 9145 } 9146 9147 /* 9148 * We can't call btrfs_truncate_block inside a trans handle as we could 9149 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know 9150 * we've truncated everything except the last little bit, and can do 9151 * btrfs_truncate_block and then update the disk_i_size. 9152 */ 9153 if (ret == NEED_TRUNCATE_BLOCK) { 9154 btrfs_end_transaction(trans); 9155 btrfs_btree_balance_dirty(fs_info); 9156 9157 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0); 9158 if (ret) 9159 goto out; 9160 trans = btrfs_start_transaction(root, 1); 9161 if (IS_ERR(trans)) { 9162 ret = PTR_ERR(trans); 9163 goto out; 9164 } 9165 btrfs_ordered_update_i_size(inode, inode->i_size, NULL); 9166 } 9167 9168 if (ret == 0 && inode->i_nlink > 0) { 9169 trans->block_rsv = root->orphan_block_rsv; 9170 ret = btrfs_orphan_del(trans, BTRFS_I(inode)); 9171 if (ret) 9172 err = ret; 9173 } 9174 9175 if (trans) { 9176 trans->block_rsv = &fs_info->trans_block_rsv; 9177 ret = btrfs_update_inode(trans, root, inode); 9178 if (ret && !err) 9179 err = ret; 9180 9181 ret = btrfs_end_transaction(trans); 9182 btrfs_btree_balance_dirty(fs_info); 9183 } 9184 out: 9185 btrfs_free_block_rsv(fs_info, rsv); 9186 9187 if (ret && !err) 9188 err = ret; 9189 9190 return err; 9191 } 9192 9193 /* 9194 * create a new subvolume directory/inode (helper for the ioctl). 9195 */ 9196 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, 9197 struct btrfs_root *new_root, 9198 struct btrfs_root *parent_root, 9199 u64 new_dirid) 9200 { 9201 struct inode *inode; 9202 int err; 9203 u64 index = 0; 9204 9205 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, 9206 new_dirid, new_dirid, 9207 S_IFDIR | (~current_umask() & S_IRWXUGO), 9208 &index); 9209 if (IS_ERR(inode)) 9210 return PTR_ERR(inode); 9211 inode->i_op = &btrfs_dir_inode_operations; 9212 inode->i_fop = &btrfs_dir_file_operations; 9213 9214 set_nlink(inode, 1); 9215 btrfs_i_size_write(BTRFS_I(inode), 0); 9216 unlock_new_inode(inode); 9217 9218 err = btrfs_subvol_inherit_props(trans, new_root, parent_root); 9219 if (err) 9220 btrfs_err(new_root->fs_info, 9221 "error inheriting subvolume %llu properties: %d", 9222 new_root->root_key.objectid, err); 9223 9224 err = btrfs_update_inode(trans, new_root, inode); 9225 9226 iput(inode); 9227 return err; 9228 } 9229 9230 struct inode *btrfs_alloc_inode(struct super_block *sb) 9231 { 9232 struct btrfs_fs_info *fs_info = btrfs_sb(sb); 9233 struct btrfs_inode *ei; 9234 struct inode *inode; 9235 9236 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL); 9237 if (!ei) 9238 return NULL; 9239 9240 ei->root = NULL; 9241 ei->generation = 0; 9242 ei->last_trans = 0; 9243 ei->last_sub_trans = 0; 9244 ei->logged_trans = 0; 9245 ei->delalloc_bytes = 0; 9246 ei->new_delalloc_bytes = 0; 9247 ei->defrag_bytes = 0; 9248 ei->disk_i_size = 0; 9249 ei->flags = 0; 9250 ei->csum_bytes = 0; 9251 ei->index_cnt = (u64)-1; 9252 ei->dir_index = 0; 9253 ei->last_unlink_trans = 0; 9254 ei->last_log_commit = 0; 9255 9256 spin_lock_init(&ei->lock); 9257 ei->outstanding_extents = 0; 9258 if (sb->s_magic != BTRFS_TEST_MAGIC) 9259 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, 9260 BTRFS_BLOCK_RSV_DELALLOC); 9261 ei->runtime_flags = 0; 9262 ei->prop_compress = BTRFS_COMPRESS_NONE; 9263 ei->defrag_compress = BTRFS_COMPRESS_NONE; 9264 9265 ei->delayed_node = NULL; 9266 9267 ei->i_otime.tv_sec = 0; 9268 ei->i_otime.tv_nsec = 0; 9269 9270 inode = &ei->vfs_inode; 9271 extent_map_tree_init(&ei->extent_tree); 9272 extent_io_tree_init(&ei->io_tree, inode); 9273 extent_io_tree_init(&ei->io_failure_tree, inode); 9274 ei->io_tree.track_uptodate = 1; 9275 ei->io_failure_tree.track_uptodate = 1; 9276 atomic_set(&ei->sync_writers, 0); 9277 mutex_init(&ei->log_mutex); 9278 mutex_init(&ei->delalloc_mutex); 9279 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 9280 INIT_LIST_HEAD(&ei->delalloc_inodes); 9281 INIT_LIST_HEAD(&ei->delayed_iput); 9282 RB_CLEAR_NODE(&ei->rb_node); 9283 init_rwsem(&ei->dio_sem); 9284 9285 return inode; 9286 } 9287 9288 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 9289 void btrfs_test_destroy_inode(struct inode *inode) 9290 { 9291 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 9292 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9293 } 9294 #endif 9295 9296 static void btrfs_i_callback(struct rcu_head *head) 9297 { 9298 struct inode *inode = container_of(head, struct inode, i_rcu); 9299 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9300 } 9301 9302 void btrfs_destroy_inode(struct inode *inode) 9303 { 9304 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9305 struct btrfs_ordered_extent *ordered; 9306 struct btrfs_root *root = BTRFS_I(inode)->root; 9307 9308 WARN_ON(!hlist_empty(&inode->i_dentry)); 9309 WARN_ON(inode->i_data.nrpages); 9310 WARN_ON(BTRFS_I(inode)->block_rsv.reserved); 9311 WARN_ON(BTRFS_I(inode)->block_rsv.size); 9312 WARN_ON(BTRFS_I(inode)->outstanding_extents); 9313 WARN_ON(BTRFS_I(inode)->delalloc_bytes); 9314 WARN_ON(BTRFS_I(inode)->new_delalloc_bytes); 9315 WARN_ON(BTRFS_I(inode)->csum_bytes); 9316 WARN_ON(BTRFS_I(inode)->defrag_bytes); 9317 9318 /* 9319 * This can happen where we create an inode, but somebody else also 9320 * created the same inode and we need to destroy the one we already 9321 * created. 9322 */ 9323 if (!root) 9324 goto free; 9325 9326 if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 9327 &BTRFS_I(inode)->runtime_flags)) { 9328 btrfs_info(fs_info, "inode %llu still on the orphan list", 9329 btrfs_ino(BTRFS_I(inode))); 9330 atomic_dec(&root->orphan_inodes); 9331 } 9332 9333 while (1) { 9334 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 9335 if (!ordered) 9336 break; 9337 else { 9338 btrfs_err(fs_info, 9339 "found ordered extent %llu %llu on inode cleanup", 9340 ordered->file_offset, ordered->len); 9341 btrfs_remove_ordered_extent(inode, ordered); 9342 btrfs_put_ordered_extent(ordered); 9343 btrfs_put_ordered_extent(ordered); 9344 } 9345 } 9346 btrfs_qgroup_check_reserved_leak(inode); 9347 inode_tree_del(inode); 9348 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 9349 free: 9350 call_rcu(&inode->i_rcu, btrfs_i_callback); 9351 } 9352 9353 int btrfs_drop_inode(struct inode *inode) 9354 { 9355 struct btrfs_root *root = BTRFS_I(inode)->root; 9356 9357 if (root == NULL) 9358 return 1; 9359 9360 /* the snap/subvol tree is on deleting */ 9361 if (btrfs_root_refs(&root->root_item) == 0) 9362 return 1; 9363 else 9364 return generic_drop_inode(inode); 9365 } 9366 9367 static void init_once(void *foo) 9368 { 9369 struct btrfs_inode *ei = (struct btrfs_inode *) foo; 9370 9371 inode_init_once(&ei->vfs_inode); 9372 } 9373 9374 void __cold btrfs_destroy_cachep(void) 9375 { 9376 /* 9377 * Make sure all delayed rcu free inodes are flushed before we 9378 * destroy cache. 9379 */ 9380 rcu_barrier(); 9381 kmem_cache_destroy(btrfs_inode_cachep); 9382 kmem_cache_destroy(btrfs_trans_handle_cachep); 9383 kmem_cache_destroy(btrfs_path_cachep); 9384 kmem_cache_destroy(btrfs_free_space_cachep); 9385 } 9386 9387 int __init btrfs_init_cachep(void) 9388 { 9389 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 9390 sizeof(struct btrfs_inode), 0, 9391 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, 9392 init_once); 9393 if (!btrfs_inode_cachep) 9394 goto fail; 9395 9396 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", 9397 sizeof(struct btrfs_trans_handle), 0, 9398 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); 9399 if (!btrfs_trans_handle_cachep) 9400 goto fail; 9401 9402 btrfs_path_cachep = kmem_cache_create("btrfs_path", 9403 sizeof(struct btrfs_path), 0, 9404 SLAB_MEM_SPREAD, NULL); 9405 if (!btrfs_path_cachep) 9406 goto fail; 9407 9408 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space", 9409 sizeof(struct btrfs_free_space), 0, 9410 SLAB_MEM_SPREAD, NULL); 9411 if (!btrfs_free_space_cachep) 9412 goto fail; 9413 9414 return 0; 9415 fail: 9416 btrfs_destroy_cachep(); 9417 return -ENOMEM; 9418 } 9419 9420 static int btrfs_getattr(const struct path *path, struct kstat *stat, 9421 u32 request_mask, unsigned int flags) 9422 { 9423 u64 delalloc_bytes; 9424 struct inode *inode = d_inode(path->dentry); 9425 u32 blocksize = inode->i_sb->s_blocksize; 9426 u32 bi_flags = BTRFS_I(inode)->flags; 9427 9428 stat->result_mask |= STATX_BTIME; 9429 stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec; 9430 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec; 9431 if (bi_flags & BTRFS_INODE_APPEND) 9432 stat->attributes |= STATX_ATTR_APPEND; 9433 if (bi_flags & BTRFS_INODE_COMPRESS) 9434 stat->attributes |= STATX_ATTR_COMPRESSED; 9435 if (bi_flags & BTRFS_INODE_IMMUTABLE) 9436 stat->attributes |= STATX_ATTR_IMMUTABLE; 9437 if (bi_flags & BTRFS_INODE_NODUMP) 9438 stat->attributes |= STATX_ATTR_NODUMP; 9439 9440 stat->attributes_mask |= (STATX_ATTR_APPEND | 9441 STATX_ATTR_COMPRESSED | 9442 STATX_ATTR_IMMUTABLE | 9443 STATX_ATTR_NODUMP); 9444 9445 generic_fillattr(inode, stat); 9446 stat->dev = BTRFS_I(inode)->root->anon_dev; 9447 9448 spin_lock(&BTRFS_I(inode)->lock); 9449 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; 9450 spin_unlock(&BTRFS_I(inode)->lock); 9451 stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) + 9452 ALIGN(delalloc_bytes, blocksize)) >> 9; 9453 return 0; 9454 } 9455 9456 static int btrfs_rename_exchange(struct inode *old_dir, 9457 struct dentry *old_dentry, 9458 struct inode *new_dir, 9459 struct dentry *new_dentry) 9460 { 9461 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9462 struct btrfs_trans_handle *trans; 9463 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9464 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9465 struct inode *new_inode = new_dentry->d_inode; 9466 struct inode *old_inode = old_dentry->d_inode; 9467 struct timespec ctime = current_time(old_inode); 9468 struct dentry *parent; 9469 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9470 u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); 9471 u64 old_idx = 0; 9472 u64 new_idx = 0; 9473 u64 root_objectid; 9474 int ret; 9475 bool root_log_pinned = false; 9476 bool dest_log_pinned = false; 9477 9478 /* we only allow rename subvolume link between subvolumes */ 9479 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9480 return -EXDEV; 9481 9482 /* close the race window with snapshot create/destroy ioctl */ 9483 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9484 down_read(&fs_info->subvol_sem); 9485 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9486 down_read(&fs_info->subvol_sem); 9487 9488 /* 9489 * We want to reserve the absolute worst case amount of items. So if 9490 * both inodes are subvols and we need to unlink them then that would 9491 * require 4 item modifications, but if they are both normal inodes it 9492 * would require 5 item modifications, so we'll assume their normal 9493 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items 9494 * should cover the worst case number of items we'll modify. 9495 */ 9496 trans = btrfs_start_transaction(root, 12); 9497 if (IS_ERR(trans)) { 9498 ret = PTR_ERR(trans); 9499 goto out_notrans; 9500 } 9501 9502 /* 9503 * We need to find a free sequence number both in the source and 9504 * in the destination directory for the exchange. 9505 */ 9506 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); 9507 if (ret) 9508 goto out_fail; 9509 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); 9510 if (ret) 9511 goto out_fail; 9512 9513 BTRFS_I(old_inode)->dir_index = 0ULL; 9514 BTRFS_I(new_inode)->dir_index = 0ULL; 9515 9516 /* Reference for the source. */ 9517 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9518 /* force full log commit if subvolume involved. */ 9519 btrfs_set_log_full_commit(fs_info, trans); 9520 } else { 9521 btrfs_pin_log_trans(root); 9522 root_log_pinned = true; 9523 ret = btrfs_insert_inode_ref(trans, dest, 9524 new_dentry->d_name.name, 9525 new_dentry->d_name.len, 9526 old_ino, 9527 btrfs_ino(BTRFS_I(new_dir)), 9528 old_idx); 9529 if (ret) 9530 goto out_fail; 9531 } 9532 9533 /* And now for the dest. */ 9534 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9535 /* force full log commit if subvolume involved. */ 9536 btrfs_set_log_full_commit(fs_info, trans); 9537 } else { 9538 btrfs_pin_log_trans(dest); 9539 dest_log_pinned = true; 9540 ret = btrfs_insert_inode_ref(trans, root, 9541 old_dentry->d_name.name, 9542 old_dentry->d_name.len, 9543 new_ino, 9544 btrfs_ino(BTRFS_I(old_dir)), 9545 new_idx); 9546 if (ret) 9547 goto out_fail; 9548 } 9549 9550 /* Update inode version and ctime/mtime. */ 9551 inode_inc_iversion(old_dir); 9552 inode_inc_iversion(new_dir); 9553 inode_inc_iversion(old_inode); 9554 inode_inc_iversion(new_inode); 9555 old_dir->i_ctime = old_dir->i_mtime = ctime; 9556 new_dir->i_ctime = new_dir->i_mtime = ctime; 9557 old_inode->i_ctime = ctime; 9558 new_inode->i_ctime = ctime; 9559 9560 if (old_dentry->d_parent != new_dentry->d_parent) { 9561 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9562 BTRFS_I(old_inode), 1); 9563 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), 9564 BTRFS_I(new_inode), 1); 9565 } 9566 9567 /* src is a subvolume */ 9568 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9569 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9570 ret = btrfs_unlink_subvol(trans, root, old_dir, 9571 root_objectid, 9572 old_dentry->d_name.name, 9573 old_dentry->d_name.len); 9574 } else { /* src is an inode */ 9575 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9576 BTRFS_I(old_dentry->d_inode), 9577 old_dentry->d_name.name, 9578 old_dentry->d_name.len); 9579 if (!ret) 9580 ret = btrfs_update_inode(trans, root, old_inode); 9581 } 9582 if (ret) { 9583 btrfs_abort_transaction(trans, ret); 9584 goto out_fail; 9585 } 9586 9587 /* dest is a subvolume */ 9588 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9589 root_objectid = BTRFS_I(new_inode)->root->root_key.objectid; 9590 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9591 root_objectid, 9592 new_dentry->d_name.name, 9593 new_dentry->d_name.len); 9594 } else { /* dest is an inode */ 9595 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9596 BTRFS_I(new_dentry->d_inode), 9597 new_dentry->d_name.name, 9598 new_dentry->d_name.len); 9599 if (!ret) 9600 ret = btrfs_update_inode(trans, dest, new_inode); 9601 } 9602 if (ret) { 9603 btrfs_abort_transaction(trans, ret); 9604 goto out_fail; 9605 } 9606 9607 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9608 new_dentry->d_name.name, 9609 new_dentry->d_name.len, 0, old_idx); 9610 if (ret) { 9611 btrfs_abort_transaction(trans, ret); 9612 goto out_fail; 9613 } 9614 9615 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), 9616 old_dentry->d_name.name, 9617 old_dentry->d_name.len, 0, new_idx); 9618 if (ret) { 9619 btrfs_abort_transaction(trans, ret); 9620 goto out_fail; 9621 } 9622 9623 if (old_inode->i_nlink == 1) 9624 BTRFS_I(old_inode)->dir_index = old_idx; 9625 if (new_inode->i_nlink == 1) 9626 BTRFS_I(new_inode)->dir_index = new_idx; 9627 9628 if (root_log_pinned) { 9629 parent = new_dentry->d_parent; 9630 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9631 parent); 9632 btrfs_end_log_trans(root); 9633 root_log_pinned = false; 9634 } 9635 if (dest_log_pinned) { 9636 parent = old_dentry->d_parent; 9637 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir), 9638 parent); 9639 btrfs_end_log_trans(dest); 9640 dest_log_pinned = false; 9641 } 9642 out_fail: 9643 /* 9644 * If we have pinned a log and an error happened, we unpin tasks 9645 * trying to sync the log and force them to fallback to a transaction 9646 * commit if the log currently contains any of the inodes involved in 9647 * this rename operation (to ensure we do not persist a log with an 9648 * inconsistent state for any of these inodes or leading to any 9649 * inconsistencies when replayed). If the transaction was aborted, the 9650 * abortion reason is propagated to userspace when attempting to commit 9651 * the transaction. If the log does not contain any of these inodes, we 9652 * allow the tasks to sync it. 9653 */ 9654 if (ret && (root_log_pinned || dest_log_pinned)) { 9655 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9656 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9657 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9658 (new_inode && 9659 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9660 btrfs_set_log_full_commit(fs_info, trans); 9661 9662 if (root_log_pinned) { 9663 btrfs_end_log_trans(root); 9664 root_log_pinned = false; 9665 } 9666 if (dest_log_pinned) { 9667 btrfs_end_log_trans(dest); 9668 dest_log_pinned = false; 9669 } 9670 } 9671 ret = btrfs_end_transaction(trans); 9672 out_notrans: 9673 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9674 up_read(&fs_info->subvol_sem); 9675 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9676 up_read(&fs_info->subvol_sem); 9677 9678 return ret; 9679 } 9680 9681 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans, 9682 struct btrfs_root *root, 9683 struct inode *dir, 9684 struct dentry *dentry) 9685 { 9686 int ret; 9687 struct inode *inode; 9688 u64 objectid; 9689 u64 index; 9690 9691 ret = btrfs_find_free_ino(root, &objectid); 9692 if (ret) 9693 return ret; 9694 9695 inode = btrfs_new_inode(trans, root, dir, 9696 dentry->d_name.name, 9697 dentry->d_name.len, 9698 btrfs_ino(BTRFS_I(dir)), 9699 objectid, 9700 S_IFCHR | WHITEOUT_MODE, 9701 &index); 9702 9703 if (IS_ERR(inode)) { 9704 ret = PTR_ERR(inode); 9705 return ret; 9706 } 9707 9708 inode->i_op = &btrfs_special_inode_operations; 9709 init_special_inode(inode, inode->i_mode, 9710 WHITEOUT_DEV); 9711 9712 ret = btrfs_init_inode_security(trans, inode, dir, 9713 &dentry->d_name); 9714 if (ret) 9715 goto out; 9716 9717 ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 9718 BTRFS_I(inode), 0, index); 9719 if (ret) 9720 goto out; 9721 9722 ret = btrfs_update_inode(trans, root, inode); 9723 out: 9724 unlock_new_inode(inode); 9725 if (ret) 9726 inode_dec_link_count(inode); 9727 iput(inode); 9728 9729 return ret; 9730 } 9731 9732 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry, 9733 struct inode *new_dir, struct dentry *new_dentry, 9734 unsigned int flags) 9735 { 9736 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9737 struct btrfs_trans_handle *trans; 9738 unsigned int trans_num_items; 9739 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9740 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9741 struct inode *new_inode = d_inode(new_dentry); 9742 struct inode *old_inode = d_inode(old_dentry); 9743 u64 index = 0; 9744 u64 root_objectid; 9745 int ret; 9746 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9747 bool log_pinned = false; 9748 9749 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 9750 return -EPERM; 9751 9752 /* we only allow rename subvolume link between subvolumes */ 9753 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9754 return -EXDEV; 9755 9756 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 9757 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) 9758 return -ENOTEMPTY; 9759 9760 if (S_ISDIR(old_inode->i_mode) && new_inode && 9761 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 9762 return -ENOTEMPTY; 9763 9764 9765 /* check for collisions, even if the name isn't there */ 9766 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, 9767 new_dentry->d_name.name, 9768 new_dentry->d_name.len); 9769 9770 if (ret) { 9771 if (ret == -EEXIST) { 9772 /* we shouldn't get 9773 * eexist without a new_inode */ 9774 if (WARN_ON(!new_inode)) { 9775 return ret; 9776 } 9777 } else { 9778 /* maybe -EOVERFLOW */ 9779 return ret; 9780 } 9781 } 9782 ret = 0; 9783 9784 /* 9785 * we're using rename to replace one file with another. Start IO on it 9786 * now so we don't add too much work to the end of the transaction 9787 */ 9788 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 9789 filemap_flush(old_inode->i_mapping); 9790 9791 /* close the racy window with snapshot create/destroy ioctl */ 9792 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9793 down_read(&fs_info->subvol_sem); 9794 /* 9795 * We want to reserve the absolute worst case amount of items. So if 9796 * both inodes are subvols and we need to unlink them then that would 9797 * require 4 item modifications, but if they are both normal inodes it 9798 * would require 5 item modifications, so we'll assume they are normal 9799 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items 9800 * should cover the worst case number of items we'll modify. 9801 * If our rename has the whiteout flag, we need more 5 units for the 9802 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item 9803 * when selinux is enabled). 9804 */ 9805 trans_num_items = 11; 9806 if (flags & RENAME_WHITEOUT) 9807 trans_num_items += 5; 9808 trans = btrfs_start_transaction(root, trans_num_items); 9809 if (IS_ERR(trans)) { 9810 ret = PTR_ERR(trans); 9811 goto out_notrans; 9812 } 9813 9814 if (dest != root) 9815 btrfs_record_root_in_trans(trans, dest); 9816 9817 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); 9818 if (ret) 9819 goto out_fail; 9820 9821 BTRFS_I(old_inode)->dir_index = 0ULL; 9822 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9823 /* force full log commit if subvolume involved. */ 9824 btrfs_set_log_full_commit(fs_info, trans); 9825 } else { 9826 btrfs_pin_log_trans(root); 9827 log_pinned = true; 9828 ret = btrfs_insert_inode_ref(trans, dest, 9829 new_dentry->d_name.name, 9830 new_dentry->d_name.len, 9831 old_ino, 9832 btrfs_ino(BTRFS_I(new_dir)), index); 9833 if (ret) 9834 goto out_fail; 9835 } 9836 9837 inode_inc_iversion(old_dir); 9838 inode_inc_iversion(new_dir); 9839 inode_inc_iversion(old_inode); 9840 old_dir->i_ctime = old_dir->i_mtime = 9841 new_dir->i_ctime = new_dir->i_mtime = 9842 old_inode->i_ctime = current_time(old_dir); 9843 9844 if (old_dentry->d_parent != new_dentry->d_parent) 9845 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9846 BTRFS_I(old_inode), 1); 9847 9848 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9849 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9850 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid, 9851 old_dentry->d_name.name, 9852 old_dentry->d_name.len); 9853 } else { 9854 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9855 BTRFS_I(d_inode(old_dentry)), 9856 old_dentry->d_name.name, 9857 old_dentry->d_name.len); 9858 if (!ret) 9859 ret = btrfs_update_inode(trans, root, old_inode); 9860 } 9861 if (ret) { 9862 btrfs_abort_transaction(trans, ret); 9863 goto out_fail; 9864 } 9865 9866 if (new_inode) { 9867 inode_inc_iversion(new_inode); 9868 new_inode->i_ctime = current_time(new_inode); 9869 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == 9870 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 9871 root_objectid = BTRFS_I(new_inode)->location.objectid; 9872 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9873 root_objectid, 9874 new_dentry->d_name.name, 9875 new_dentry->d_name.len); 9876 BUG_ON(new_inode->i_nlink == 0); 9877 } else { 9878 ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9879 BTRFS_I(d_inode(new_dentry)), 9880 new_dentry->d_name.name, 9881 new_dentry->d_name.len); 9882 } 9883 if (!ret && new_inode->i_nlink == 0) 9884 ret = btrfs_orphan_add(trans, 9885 BTRFS_I(d_inode(new_dentry))); 9886 if (ret) { 9887 btrfs_abort_transaction(trans, ret); 9888 goto out_fail; 9889 } 9890 } 9891 9892 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9893 new_dentry->d_name.name, 9894 new_dentry->d_name.len, 0, index); 9895 if (ret) { 9896 btrfs_abort_transaction(trans, ret); 9897 goto out_fail; 9898 } 9899 9900 if (old_inode->i_nlink == 1) 9901 BTRFS_I(old_inode)->dir_index = index; 9902 9903 if (log_pinned) { 9904 struct dentry *parent = new_dentry->d_parent; 9905 9906 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9907 parent); 9908 btrfs_end_log_trans(root); 9909 log_pinned = false; 9910 } 9911 9912 if (flags & RENAME_WHITEOUT) { 9913 ret = btrfs_whiteout_for_rename(trans, root, old_dir, 9914 old_dentry); 9915 9916 if (ret) { 9917 btrfs_abort_transaction(trans, ret); 9918 goto out_fail; 9919 } 9920 } 9921 out_fail: 9922 /* 9923 * If we have pinned the log and an error happened, we unpin tasks 9924 * trying to sync the log and force them to fallback to a transaction 9925 * commit if the log currently contains any of the inodes involved in 9926 * this rename operation (to ensure we do not persist a log with an 9927 * inconsistent state for any of these inodes or leading to any 9928 * inconsistencies when replayed). If the transaction was aborted, the 9929 * abortion reason is propagated to userspace when attempting to commit 9930 * the transaction. If the log does not contain any of these inodes, we 9931 * allow the tasks to sync it. 9932 */ 9933 if (ret && log_pinned) { 9934 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9935 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9936 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9937 (new_inode && 9938 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9939 btrfs_set_log_full_commit(fs_info, trans); 9940 9941 btrfs_end_log_trans(root); 9942 log_pinned = false; 9943 } 9944 btrfs_end_transaction(trans); 9945 out_notrans: 9946 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9947 up_read(&fs_info->subvol_sem); 9948 9949 return ret; 9950 } 9951 9952 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, 9953 struct inode *new_dir, struct dentry *new_dentry, 9954 unsigned int flags) 9955 { 9956 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 9957 return -EINVAL; 9958 9959 if (flags & RENAME_EXCHANGE) 9960 return btrfs_rename_exchange(old_dir, old_dentry, new_dir, 9961 new_dentry); 9962 9963 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags); 9964 } 9965 9966 static void btrfs_run_delalloc_work(struct btrfs_work *work) 9967 { 9968 struct btrfs_delalloc_work *delalloc_work; 9969 struct inode *inode; 9970 9971 delalloc_work = container_of(work, struct btrfs_delalloc_work, 9972 work); 9973 inode = delalloc_work->inode; 9974 filemap_flush(inode->i_mapping); 9975 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9976 &BTRFS_I(inode)->runtime_flags)) 9977 filemap_flush(inode->i_mapping); 9978 9979 if (delalloc_work->delay_iput) 9980 btrfs_add_delayed_iput(inode); 9981 else 9982 iput(inode); 9983 complete(&delalloc_work->completion); 9984 } 9985 9986 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode, 9987 int delay_iput) 9988 { 9989 struct btrfs_delalloc_work *work; 9990 9991 work = kmalloc(sizeof(*work), GFP_NOFS); 9992 if (!work) 9993 return NULL; 9994 9995 init_completion(&work->completion); 9996 INIT_LIST_HEAD(&work->list); 9997 work->inode = inode; 9998 work->delay_iput = delay_iput; 9999 WARN_ON_ONCE(!inode); 10000 btrfs_init_work(&work->work, btrfs_flush_delalloc_helper, 10001 btrfs_run_delalloc_work, NULL, NULL); 10002 10003 return work; 10004 } 10005 10006 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work) 10007 { 10008 wait_for_completion(&work->completion); 10009 kfree(work); 10010 } 10011 10012 /* 10013 * some fairly slow code that needs optimization. This walks the list 10014 * of all the inodes with pending delalloc and forces them to disk. 10015 */ 10016 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput, 10017 int nr) 10018 { 10019 struct btrfs_inode *binode; 10020 struct inode *inode; 10021 struct btrfs_delalloc_work *work, *next; 10022 struct list_head works; 10023 struct list_head splice; 10024 int ret = 0; 10025 10026 INIT_LIST_HEAD(&works); 10027 INIT_LIST_HEAD(&splice); 10028 10029 mutex_lock(&root->delalloc_mutex); 10030 spin_lock(&root->delalloc_lock); 10031 list_splice_init(&root->delalloc_inodes, &splice); 10032 while (!list_empty(&splice)) { 10033 binode = list_entry(splice.next, struct btrfs_inode, 10034 delalloc_inodes); 10035 10036 list_move_tail(&binode->delalloc_inodes, 10037 &root->delalloc_inodes); 10038 inode = igrab(&binode->vfs_inode); 10039 if (!inode) { 10040 cond_resched_lock(&root->delalloc_lock); 10041 continue; 10042 } 10043 spin_unlock(&root->delalloc_lock); 10044 10045 work = btrfs_alloc_delalloc_work(inode, delay_iput); 10046 if (!work) { 10047 if (delay_iput) 10048 btrfs_add_delayed_iput(inode); 10049 else 10050 iput(inode); 10051 ret = -ENOMEM; 10052 goto out; 10053 } 10054 list_add_tail(&work->list, &works); 10055 btrfs_queue_work(root->fs_info->flush_workers, 10056 &work->work); 10057 ret++; 10058 if (nr != -1 && ret >= nr) 10059 goto out; 10060 cond_resched(); 10061 spin_lock(&root->delalloc_lock); 10062 } 10063 spin_unlock(&root->delalloc_lock); 10064 10065 out: 10066 list_for_each_entry_safe(work, next, &works, list) { 10067 list_del_init(&work->list); 10068 btrfs_wait_and_free_delalloc_work(work); 10069 } 10070 10071 if (!list_empty_careful(&splice)) { 10072 spin_lock(&root->delalloc_lock); 10073 list_splice_tail(&splice, &root->delalloc_inodes); 10074 spin_unlock(&root->delalloc_lock); 10075 } 10076 mutex_unlock(&root->delalloc_mutex); 10077 return ret; 10078 } 10079 10080 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput) 10081 { 10082 struct btrfs_fs_info *fs_info = root->fs_info; 10083 int ret; 10084 10085 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10086 return -EROFS; 10087 10088 ret = __start_delalloc_inodes(root, delay_iput, -1); 10089 if (ret > 0) 10090 ret = 0; 10091 return ret; 10092 } 10093 10094 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput, 10095 int nr) 10096 { 10097 struct btrfs_root *root; 10098 struct list_head splice; 10099 int ret; 10100 10101 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10102 return -EROFS; 10103 10104 INIT_LIST_HEAD(&splice); 10105 10106 mutex_lock(&fs_info->delalloc_root_mutex); 10107 spin_lock(&fs_info->delalloc_root_lock); 10108 list_splice_init(&fs_info->delalloc_roots, &splice); 10109 while (!list_empty(&splice) && nr) { 10110 root = list_first_entry(&splice, struct btrfs_root, 10111 delalloc_root); 10112 root = btrfs_grab_fs_root(root); 10113 BUG_ON(!root); 10114 list_move_tail(&root->delalloc_root, 10115 &fs_info->delalloc_roots); 10116 spin_unlock(&fs_info->delalloc_root_lock); 10117 10118 ret = __start_delalloc_inodes(root, delay_iput, nr); 10119 btrfs_put_fs_root(root); 10120 if (ret < 0) 10121 goto out; 10122 10123 if (nr != -1) { 10124 nr -= ret; 10125 WARN_ON(nr < 0); 10126 } 10127 spin_lock(&fs_info->delalloc_root_lock); 10128 } 10129 spin_unlock(&fs_info->delalloc_root_lock); 10130 10131 ret = 0; 10132 out: 10133 if (!list_empty_careful(&splice)) { 10134 spin_lock(&fs_info->delalloc_root_lock); 10135 list_splice_tail(&splice, &fs_info->delalloc_roots); 10136 spin_unlock(&fs_info->delalloc_root_lock); 10137 } 10138 mutex_unlock(&fs_info->delalloc_root_mutex); 10139 return ret; 10140 } 10141 10142 static int btrfs_symlink(struct inode *dir, struct dentry *dentry, 10143 const char *symname) 10144 { 10145 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10146 struct btrfs_trans_handle *trans; 10147 struct btrfs_root *root = BTRFS_I(dir)->root; 10148 struct btrfs_path *path; 10149 struct btrfs_key key; 10150 struct inode *inode = NULL; 10151 int err; 10152 int drop_inode = 0; 10153 u64 objectid; 10154 u64 index = 0; 10155 int name_len; 10156 int datasize; 10157 unsigned long ptr; 10158 struct btrfs_file_extent_item *ei; 10159 struct extent_buffer *leaf; 10160 10161 name_len = strlen(symname); 10162 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 10163 return -ENAMETOOLONG; 10164 10165 /* 10166 * 2 items for inode item and ref 10167 * 2 items for dir items 10168 * 1 item for updating parent inode item 10169 * 1 item for the inline extent item 10170 * 1 item for xattr if selinux is on 10171 */ 10172 trans = btrfs_start_transaction(root, 7); 10173 if (IS_ERR(trans)) 10174 return PTR_ERR(trans); 10175 10176 err = btrfs_find_free_ino(root, &objectid); 10177 if (err) 10178 goto out_unlock; 10179 10180 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 10181 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), 10182 objectid, S_IFLNK|S_IRWXUGO, &index); 10183 if (IS_ERR(inode)) { 10184 err = PTR_ERR(inode); 10185 goto out_unlock; 10186 } 10187 10188 /* 10189 * If the active LSM wants to access the inode during 10190 * d_instantiate it needs these. Smack checks to see 10191 * if the filesystem supports xattrs by looking at the 10192 * ops vector. 10193 */ 10194 inode->i_fop = &btrfs_file_operations; 10195 inode->i_op = &btrfs_file_inode_operations; 10196 inode->i_mapping->a_ops = &btrfs_aops; 10197 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10198 10199 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 10200 if (err) 10201 goto out_unlock_inode; 10202 10203 path = btrfs_alloc_path(); 10204 if (!path) { 10205 err = -ENOMEM; 10206 goto out_unlock_inode; 10207 } 10208 key.objectid = btrfs_ino(BTRFS_I(inode)); 10209 key.offset = 0; 10210 key.type = BTRFS_EXTENT_DATA_KEY; 10211 datasize = btrfs_file_extent_calc_inline_size(name_len); 10212 err = btrfs_insert_empty_item(trans, root, path, &key, 10213 datasize); 10214 if (err) { 10215 btrfs_free_path(path); 10216 goto out_unlock_inode; 10217 } 10218 leaf = path->nodes[0]; 10219 ei = btrfs_item_ptr(leaf, path->slots[0], 10220 struct btrfs_file_extent_item); 10221 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 10222 btrfs_set_file_extent_type(leaf, ei, 10223 BTRFS_FILE_EXTENT_INLINE); 10224 btrfs_set_file_extent_encryption(leaf, ei, 0); 10225 btrfs_set_file_extent_compression(leaf, ei, 0); 10226 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 10227 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 10228 10229 ptr = btrfs_file_extent_inline_start(ei); 10230 write_extent_buffer(leaf, symname, ptr, name_len); 10231 btrfs_mark_buffer_dirty(leaf); 10232 btrfs_free_path(path); 10233 10234 inode->i_op = &btrfs_symlink_inode_operations; 10235 inode_nohighmem(inode); 10236 inode->i_mapping->a_ops = &btrfs_symlink_aops; 10237 inode_set_bytes(inode, name_len); 10238 btrfs_i_size_write(BTRFS_I(inode), name_len); 10239 err = btrfs_update_inode(trans, root, inode); 10240 /* 10241 * Last step, add directory indexes for our symlink inode. This is the 10242 * last step to avoid extra cleanup of these indexes if an error happens 10243 * elsewhere above. 10244 */ 10245 if (!err) 10246 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 10247 BTRFS_I(inode), 0, index); 10248 if (err) { 10249 drop_inode = 1; 10250 goto out_unlock_inode; 10251 } 10252 10253 unlock_new_inode(inode); 10254 d_instantiate(dentry, inode); 10255 10256 out_unlock: 10257 btrfs_end_transaction(trans); 10258 if (drop_inode) { 10259 inode_dec_link_count(inode); 10260 iput(inode); 10261 } 10262 btrfs_btree_balance_dirty(fs_info); 10263 return err; 10264 10265 out_unlock_inode: 10266 drop_inode = 1; 10267 unlock_new_inode(inode); 10268 goto out_unlock; 10269 } 10270 10271 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 10272 u64 start, u64 num_bytes, u64 min_size, 10273 loff_t actual_len, u64 *alloc_hint, 10274 struct btrfs_trans_handle *trans) 10275 { 10276 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 10277 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 10278 struct extent_map *em; 10279 struct btrfs_root *root = BTRFS_I(inode)->root; 10280 struct btrfs_key ins; 10281 u64 cur_offset = start; 10282 u64 i_size; 10283 u64 cur_bytes; 10284 u64 last_alloc = (u64)-1; 10285 int ret = 0; 10286 bool own_trans = true; 10287 u64 end = start + num_bytes - 1; 10288 10289 if (trans) 10290 own_trans = false; 10291 while (num_bytes > 0) { 10292 if (own_trans) { 10293 trans = btrfs_start_transaction(root, 3); 10294 if (IS_ERR(trans)) { 10295 ret = PTR_ERR(trans); 10296 break; 10297 } 10298 } 10299 10300 cur_bytes = min_t(u64, num_bytes, SZ_256M); 10301 cur_bytes = max(cur_bytes, min_size); 10302 /* 10303 * If we are severely fragmented we could end up with really 10304 * small allocations, so if the allocator is returning small 10305 * chunks lets make its job easier by only searching for those 10306 * sized chunks. 10307 */ 10308 cur_bytes = min(cur_bytes, last_alloc); 10309 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 10310 min_size, 0, *alloc_hint, &ins, 1, 0); 10311 if (ret) { 10312 if (own_trans) 10313 btrfs_end_transaction(trans); 10314 break; 10315 } 10316 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10317 10318 last_alloc = ins.offset; 10319 ret = insert_reserved_file_extent(trans, inode, 10320 cur_offset, ins.objectid, 10321 ins.offset, ins.offset, 10322 ins.offset, 0, 0, 0, 10323 BTRFS_FILE_EXTENT_PREALLOC); 10324 if (ret) { 10325 btrfs_free_reserved_extent(fs_info, ins.objectid, 10326 ins.offset, 0); 10327 btrfs_abort_transaction(trans, ret); 10328 if (own_trans) 10329 btrfs_end_transaction(trans); 10330 break; 10331 } 10332 10333 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 10334 cur_offset + ins.offset -1, 0); 10335 10336 em = alloc_extent_map(); 10337 if (!em) { 10338 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 10339 &BTRFS_I(inode)->runtime_flags); 10340 goto next; 10341 } 10342 10343 em->start = cur_offset; 10344 em->orig_start = cur_offset; 10345 em->len = ins.offset; 10346 em->block_start = ins.objectid; 10347 em->block_len = ins.offset; 10348 em->orig_block_len = ins.offset; 10349 em->ram_bytes = ins.offset; 10350 em->bdev = fs_info->fs_devices->latest_bdev; 10351 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 10352 em->generation = trans->transid; 10353 10354 while (1) { 10355 write_lock(&em_tree->lock); 10356 ret = add_extent_mapping(em_tree, em, 1); 10357 write_unlock(&em_tree->lock); 10358 if (ret != -EEXIST) 10359 break; 10360 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 10361 cur_offset + ins.offset - 1, 10362 0); 10363 } 10364 free_extent_map(em); 10365 next: 10366 num_bytes -= ins.offset; 10367 cur_offset += ins.offset; 10368 *alloc_hint = ins.objectid + ins.offset; 10369 10370 inode_inc_iversion(inode); 10371 inode->i_ctime = current_time(inode); 10372 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 10373 if (!(mode & FALLOC_FL_KEEP_SIZE) && 10374 (actual_len > inode->i_size) && 10375 (cur_offset > inode->i_size)) { 10376 if (cur_offset > actual_len) 10377 i_size = actual_len; 10378 else 10379 i_size = cur_offset; 10380 i_size_write(inode, i_size); 10381 btrfs_ordered_update_i_size(inode, i_size, NULL); 10382 } 10383 10384 ret = btrfs_update_inode(trans, root, inode); 10385 10386 if (ret) { 10387 btrfs_abort_transaction(trans, ret); 10388 if (own_trans) 10389 btrfs_end_transaction(trans); 10390 break; 10391 } 10392 10393 if (own_trans) 10394 btrfs_end_transaction(trans); 10395 } 10396 if (cur_offset < end) 10397 btrfs_free_reserved_data_space(inode, NULL, cur_offset, 10398 end - cur_offset + 1); 10399 return ret; 10400 } 10401 10402 int btrfs_prealloc_file_range(struct inode *inode, int mode, 10403 u64 start, u64 num_bytes, u64 min_size, 10404 loff_t actual_len, u64 *alloc_hint) 10405 { 10406 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10407 min_size, actual_len, alloc_hint, 10408 NULL); 10409 } 10410 10411 int btrfs_prealloc_file_range_trans(struct inode *inode, 10412 struct btrfs_trans_handle *trans, int mode, 10413 u64 start, u64 num_bytes, u64 min_size, 10414 loff_t actual_len, u64 *alloc_hint) 10415 { 10416 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10417 min_size, actual_len, alloc_hint, trans); 10418 } 10419 10420 static int btrfs_set_page_dirty(struct page *page) 10421 { 10422 return __set_page_dirty_nobuffers(page); 10423 } 10424 10425 static int btrfs_permission(struct inode *inode, int mask) 10426 { 10427 struct btrfs_root *root = BTRFS_I(inode)->root; 10428 umode_t mode = inode->i_mode; 10429 10430 if (mask & MAY_WRITE && 10431 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 10432 if (btrfs_root_readonly(root)) 10433 return -EROFS; 10434 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 10435 return -EACCES; 10436 } 10437 return generic_permission(inode, mask); 10438 } 10439 10440 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 10441 { 10442 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10443 struct btrfs_trans_handle *trans; 10444 struct btrfs_root *root = BTRFS_I(dir)->root; 10445 struct inode *inode = NULL; 10446 u64 objectid; 10447 u64 index; 10448 int ret = 0; 10449 10450 /* 10451 * 5 units required for adding orphan entry 10452 */ 10453 trans = btrfs_start_transaction(root, 5); 10454 if (IS_ERR(trans)) 10455 return PTR_ERR(trans); 10456 10457 ret = btrfs_find_free_ino(root, &objectid); 10458 if (ret) 10459 goto out; 10460 10461 inode = btrfs_new_inode(trans, root, dir, NULL, 0, 10462 btrfs_ino(BTRFS_I(dir)), objectid, mode, &index); 10463 if (IS_ERR(inode)) { 10464 ret = PTR_ERR(inode); 10465 inode = NULL; 10466 goto out; 10467 } 10468 10469 inode->i_fop = &btrfs_file_operations; 10470 inode->i_op = &btrfs_file_inode_operations; 10471 10472 inode->i_mapping->a_ops = &btrfs_aops; 10473 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10474 10475 ret = btrfs_init_inode_security(trans, inode, dir, NULL); 10476 if (ret) 10477 goto out_inode; 10478 10479 ret = btrfs_update_inode(trans, root, inode); 10480 if (ret) 10481 goto out_inode; 10482 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 10483 if (ret) 10484 goto out_inode; 10485 10486 /* 10487 * We set number of links to 0 in btrfs_new_inode(), and here we set 10488 * it to 1 because d_tmpfile() will issue a warning if the count is 0, 10489 * through: 10490 * 10491 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 10492 */ 10493 set_nlink(inode, 1); 10494 unlock_new_inode(inode); 10495 d_tmpfile(dentry, inode); 10496 mark_inode_dirty(inode); 10497 10498 out: 10499 btrfs_end_transaction(trans); 10500 if (ret) 10501 iput(inode); 10502 btrfs_btree_balance_dirty(fs_info); 10503 return ret; 10504 10505 out_inode: 10506 unlock_new_inode(inode); 10507 goto out; 10508 10509 } 10510 10511 __attribute__((const)) 10512 static int btrfs_readpage_io_failed_hook(struct page *page, int failed_mirror) 10513 { 10514 return -EAGAIN; 10515 } 10516 10517 static struct btrfs_fs_info *iotree_fs_info(void *private_data) 10518 { 10519 struct inode *inode = private_data; 10520 return btrfs_sb(inode->i_sb); 10521 } 10522 10523 static void btrfs_check_extent_io_range(void *private_data, const char *caller, 10524 u64 start, u64 end) 10525 { 10526 struct inode *inode = private_data; 10527 u64 isize; 10528 10529 isize = i_size_read(inode); 10530 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { 10531 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info, 10532 "%s: ino %llu isize %llu odd range [%llu,%llu]", 10533 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end); 10534 } 10535 } 10536 10537 void btrfs_set_range_writeback(void *private_data, u64 start, u64 end) 10538 { 10539 struct inode *inode = private_data; 10540 unsigned long index = start >> PAGE_SHIFT; 10541 unsigned long end_index = end >> PAGE_SHIFT; 10542 struct page *page; 10543 10544 while (index <= end_index) { 10545 page = find_get_page(inode->i_mapping, index); 10546 ASSERT(page); /* Pages should be in the extent_io_tree */ 10547 set_page_writeback(page); 10548 put_page(page); 10549 index++; 10550 } 10551 } 10552 10553 static const struct inode_operations btrfs_dir_inode_operations = { 10554 .getattr = btrfs_getattr, 10555 .lookup = btrfs_lookup, 10556 .create = btrfs_create, 10557 .unlink = btrfs_unlink, 10558 .link = btrfs_link, 10559 .mkdir = btrfs_mkdir, 10560 .rmdir = btrfs_rmdir, 10561 .rename = btrfs_rename2, 10562 .symlink = btrfs_symlink, 10563 .setattr = btrfs_setattr, 10564 .mknod = btrfs_mknod, 10565 .listxattr = btrfs_listxattr, 10566 .permission = btrfs_permission, 10567 .get_acl = btrfs_get_acl, 10568 .set_acl = btrfs_set_acl, 10569 .update_time = btrfs_update_time, 10570 .tmpfile = btrfs_tmpfile, 10571 }; 10572 static const struct inode_operations btrfs_dir_ro_inode_operations = { 10573 .lookup = btrfs_lookup, 10574 .permission = btrfs_permission, 10575 .update_time = btrfs_update_time, 10576 }; 10577 10578 static const struct file_operations btrfs_dir_file_operations = { 10579 .llseek = generic_file_llseek, 10580 .read = generic_read_dir, 10581 .iterate_shared = btrfs_real_readdir, 10582 .open = btrfs_opendir, 10583 .unlocked_ioctl = btrfs_ioctl, 10584 #ifdef CONFIG_COMPAT 10585 .compat_ioctl = btrfs_compat_ioctl, 10586 #endif 10587 .release = btrfs_release_file, 10588 .fsync = btrfs_sync_file, 10589 }; 10590 10591 static const struct extent_io_ops btrfs_extent_io_ops = { 10592 /* mandatory callbacks */ 10593 .submit_bio_hook = btrfs_submit_bio_hook, 10594 .readpage_end_io_hook = btrfs_readpage_end_io_hook, 10595 .merge_bio_hook = btrfs_merge_bio_hook, 10596 .readpage_io_failed_hook = btrfs_readpage_io_failed_hook, 10597 .tree_fs_info = iotree_fs_info, 10598 .set_range_writeback = btrfs_set_range_writeback, 10599 10600 /* optional callbacks */ 10601 .fill_delalloc = run_delalloc_range, 10602 .writepage_end_io_hook = btrfs_writepage_end_io_hook, 10603 .writepage_start_hook = btrfs_writepage_start_hook, 10604 .set_bit_hook = btrfs_set_bit_hook, 10605 .clear_bit_hook = btrfs_clear_bit_hook, 10606 .merge_extent_hook = btrfs_merge_extent_hook, 10607 .split_extent_hook = btrfs_split_extent_hook, 10608 .check_extent_io_range = btrfs_check_extent_io_range, 10609 }; 10610 10611 /* 10612 * btrfs doesn't support the bmap operation because swapfiles 10613 * use bmap to make a mapping of extents in the file. They assume 10614 * these extents won't change over the life of the file and they 10615 * use the bmap result to do IO directly to the drive. 10616 * 10617 * the btrfs bmap call would return logical addresses that aren't 10618 * suitable for IO and they also will change frequently as COW 10619 * operations happen. So, swapfile + btrfs == corruption. 10620 * 10621 * For now we're avoiding this by dropping bmap. 10622 */ 10623 static const struct address_space_operations btrfs_aops = { 10624 .readpage = btrfs_readpage, 10625 .writepage = btrfs_writepage, 10626 .writepages = btrfs_writepages, 10627 .readpages = btrfs_readpages, 10628 .direct_IO = btrfs_direct_IO, 10629 .invalidatepage = btrfs_invalidatepage, 10630 .releasepage = btrfs_releasepage, 10631 .set_page_dirty = btrfs_set_page_dirty, 10632 .error_remove_page = generic_error_remove_page, 10633 }; 10634 10635 static const struct address_space_operations btrfs_symlink_aops = { 10636 .readpage = btrfs_readpage, 10637 .writepage = btrfs_writepage, 10638 .invalidatepage = btrfs_invalidatepage, 10639 .releasepage = btrfs_releasepage, 10640 }; 10641 10642 static const struct inode_operations btrfs_file_inode_operations = { 10643 .getattr = btrfs_getattr, 10644 .setattr = btrfs_setattr, 10645 .listxattr = btrfs_listxattr, 10646 .permission = btrfs_permission, 10647 .fiemap = btrfs_fiemap, 10648 .get_acl = btrfs_get_acl, 10649 .set_acl = btrfs_set_acl, 10650 .update_time = btrfs_update_time, 10651 }; 10652 static const struct inode_operations btrfs_special_inode_operations = { 10653 .getattr = btrfs_getattr, 10654 .setattr = btrfs_setattr, 10655 .permission = btrfs_permission, 10656 .listxattr = btrfs_listxattr, 10657 .get_acl = btrfs_get_acl, 10658 .set_acl = btrfs_set_acl, 10659 .update_time = btrfs_update_time, 10660 }; 10661 static const struct inode_operations btrfs_symlink_inode_operations = { 10662 .get_link = page_get_link, 10663 .getattr = btrfs_getattr, 10664 .setattr = btrfs_setattr, 10665 .permission = btrfs_permission, 10666 .listxattr = btrfs_listxattr, 10667 .update_time = btrfs_update_time, 10668 }; 10669 10670 const struct dentry_operations btrfs_dentry_operations = { 10671 .d_delete = btrfs_dentry_delete, 10672 .d_release = btrfs_dentry_release, 10673 }; 10674