1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/bio.h> 8 #include <linux/buffer_head.h> 9 #include <linux/file.h> 10 #include <linux/fs.h> 11 #include <linux/pagemap.h> 12 #include <linux/highmem.h> 13 #include <linux/time.h> 14 #include <linux/init.h> 15 #include <linux/string.h> 16 #include <linux/backing-dev.h> 17 #include <linux/mpage.h> 18 #include <linux/swap.h> 19 #include <linux/writeback.h> 20 #include <linux/compat.h> 21 #include <linux/bit_spinlock.h> 22 #include <linux/xattr.h> 23 #include <linux/posix_acl.h> 24 #include <linux/falloc.h> 25 #include <linux/slab.h> 26 #include <linux/ratelimit.h> 27 #include <linux/mount.h> 28 #include <linux/btrfs.h> 29 #include <linux/blkdev.h> 30 #include <linux/posix_acl_xattr.h> 31 #include <linux/uio.h> 32 #include <linux/magic.h> 33 #include <linux/iversion.h> 34 #include "ctree.h" 35 #include "disk-io.h" 36 #include "transaction.h" 37 #include "btrfs_inode.h" 38 #include "print-tree.h" 39 #include "ordered-data.h" 40 #include "xattr.h" 41 #include "tree-log.h" 42 #include "volumes.h" 43 #include "compression.h" 44 #include "locking.h" 45 #include "free-space-cache.h" 46 #include "inode-map.h" 47 #include "backref.h" 48 #include "props.h" 49 #include "qgroup.h" 50 #include "dedupe.h" 51 52 struct btrfs_iget_args { 53 struct btrfs_key *location; 54 struct btrfs_root *root; 55 }; 56 57 struct btrfs_dio_data { 58 u64 reserve; 59 u64 unsubmitted_oe_range_start; 60 u64 unsubmitted_oe_range_end; 61 int overwrite; 62 }; 63 64 static const struct inode_operations btrfs_dir_inode_operations; 65 static const struct inode_operations btrfs_symlink_inode_operations; 66 static const struct inode_operations btrfs_dir_ro_inode_operations; 67 static const struct inode_operations btrfs_special_inode_operations; 68 static const struct inode_operations btrfs_file_inode_operations; 69 static const struct address_space_operations btrfs_aops; 70 static const struct address_space_operations btrfs_symlink_aops; 71 static const struct file_operations btrfs_dir_file_operations; 72 static const struct extent_io_ops btrfs_extent_io_ops; 73 74 static struct kmem_cache *btrfs_inode_cachep; 75 struct kmem_cache *btrfs_trans_handle_cachep; 76 struct kmem_cache *btrfs_path_cachep; 77 struct kmem_cache *btrfs_free_space_cachep; 78 79 #define S_SHIFT 12 80 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = { 81 [S_IFREG >> S_SHIFT] = BTRFS_FT_REG_FILE, 82 [S_IFDIR >> S_SHIFT] = BTRFS_FT_DIR, 83 [S_IFCHR >> S_SHIFT] = BTRFS_FT_CHRDEV, 84 [S_IFBLK >> S_SHIFT] = BTRFS_FT_BLKDEV, 85 [S_IFIFO >> S_SHIFT] = BTRFS_FT_FIFO, 86 [S_IFSOCK >> S_SHIFT] = BTRFS_FT_SOCK, 87 [S_IFLNK >> S_SHIFT] = BTRFS_FT_SYMLINK, 88 }; 89 90 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 91 static int btrfs_truncate(struct inode *inode, bool skip_writeback); 92 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); 93 static noinline int cow_file_range(struct inode *inode, 94 struct page *locked_page, 95 u64 start, u64 end, u64 delalloc_end, 96 int *page_started, unsigned long *nr_written, 97 int unlock, struct btrfs_dedupe_hash *hash); 98 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, 99 u64 orig_start, u64 block_start, 100 u64 block_len, u64 orig_block_len, 101 u64 ram_bytes, int compress_type, 102 int type); 103 104 static void __endio_write_update_ordered(struct inode *inode, 105 const u64 offset, const u64 bytes, 106 const bool uptodate); 107 108 /* 109 * Cleanup all submitted ordered extents in specified range to handle errors 110 * from the fill_dellaloc() callback. 111 * 112 * NOTE: caller must ensure that when an error happens, it can not call 113 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING 114 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata 115 * to be released, which we want to happen only when finishing the ordered 116 * extent (btrfs_finish_ordered_io()). Also note that the caller of the 117 * fill_delalloc() callback already does proper cleanup for the first page of 118 * the range, that is, it invokes the callback writepage_end_io_hook() for the 119 * range of the first page. 120 */ 121 static inline void btrfs_cleanup_ordered_extents(struct inode *inode, 122 const u64 offset, 123 const u64 bytes) 124 { 125 unsigned long index = offset >> PAGE_SHIFT; 126 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT; 127 struct page *page; 128 129 while (index <= end_index) { 130 page = find_get_page(inode->i_mapping, index); 131 index++; 132 if (!page) 133 continue; 134 ClearPagePrivate2(page); 135 put_page(page); 136 } 137 return __endio_write_update_ordered(inode, offset + PAGE_SIZE, 138 bytes - PAGE_SIZE, false); 139 } 140 141 static int btrfs_dirty_inode(struct inode *inode); 142 143 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 144 void btrfs_test_inode_set_ops(struct inode *inode) 145 { 146 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 147 } 148 #endif 149 150 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 151 struct inode *inode, struct inode *dir, 152 const struct qstr *qstr) 153 { 154 int err; 155 156 err = btrfs_init_acl(trans, inode, dir); 157 if (!err) 158 err = btrfs_xattr_security_init(trans, inode, dir, qstr); 159 return err; 160 } 161 162 /* 163 * this does all the hard work for inserting an inline extent into 164 * the btree. The caller should have done a btrfs_drop_extents so that 165 * no overlapping inline items exist in the btree 166 */ 167 static int insert_inline_extent(struct btrfs_trans_handle *trans, 168 struct btrfs_path *path, int extent_inserted, 169 struct btrfs_root *root, struct inode *inode, 170 u64 start, size_t size, size_t compressed_size, 171 int compress_type, 172 struct page **compressed_pages) 173 { 174 struct extent_buffer *leaf; 175 struct page *page = NULL; 176 char *kaddr; 177 unsigned long ptr; 178 struct btrfs_file_extent_item *ei; 179 int ret; 180 size_t cur_size = size; 181 unsigned long offset; 182 183 if (compressed_size && compressed_pages) 184 cur_size = compressed_size; 185 186 inode_add_bytes(inode, size); 187 188 if (!extent_inserted) { 189 struct btrfs_key key; 190 size_t datasize; 191 192 key.objectid = btrfs_ino(BTRFS_I(inode)); 193 key.offset = start; 194 key.type = BTRFS_EXTENT_DATA_KEY; 195 196 datasize = btrfs_file_extent_calc_inline_size(cur_size); 197 path->leave_spinning = 1; 198 ret = btrfs_insert_empty_item(trans, root, path, &key, 199 datasize); 200 if (ret) 201 goto fail; 202 } 203 leaf = path->nodes[0]; 204 ei = btrfs_item_ptr(leaf, path->slots[0], 205 struct btrfs_file_extent_item); 206 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 207 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 208 btrfs_set_file_extent_encryption(leaf, ei, 0); 209 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 210 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 211 ptr = btrfs_file_extent_inline_start(ei); 212 213 if (compress_type != BTRFS_COMPRESS_NONE) { 214 struct page *cpage; 215 int i = 0; 216 while (compressed_size > 0) { 217 cpage = compressed_pages[i]; 218 cur_size = min_t(unsigned long, compressed_size, 219 PAGE_SIZE); 220 221 kaddr = kmap_atomic(cpage); 222 write_extent_buffer(leaf, kaddr, ptr, cur_size); 223 kunmap_atomic(kaddr); 224 225 i++; 226 ptr += cur_size; 227 compressed_size -= cur_size; 228 } 229 btrfs_set_file_extent_compression(leaf, ei, 230 compress_type); 231 } else { 232 page = find_get_page(inode->i_mapping, 233 start >> PAGE_SHIFT); 234 btrfs_set_file_extent_compression(leaf, ei, 0); 235 kaddr = kmap_atomic(page); 236 offset = start & (PAGE_SIZE - 1); 237 write_extent_buffer(leaf, kaddr + offset, ptr, size); 238 kunmap_atomic(kaddr); 239 put_page(page); 240 } 241 btrfs_mark_buffer_dirty(leaf); 242 btrfs_release_path(path); 243 244 /* 245 * we're an inline extent, so nobody can 246 * extend the file past i_size without locking 247 * a page we already have locked. 248 * 249 * We must do any isize and inode updates 250 * before we unlock the pages. Otherwise we 251 * could end up racing with unlink. 252 */ 253 BTRFS_I(inode)->disk_i_size = inode->i_size; 254 ret = btrfs_update_inode(trans, root, inode); 255 256 fail: 257 return ret; 258 } 259 260 261 /* 262 * conditionally insert an inline extent into the file. This 263 * does the checks required to make sure the data is small enough 264 * to fit as an inline extent. 265 */ 266 static noinline int cow_file_range_inline(struct inode *inode, u64 start, 267 u64 end, size_t compressed_size, 268 int compress_type, 269 struct page **compressed_pages) 270 { 271 struct btrfs_root *root = BTRFS_I(inode)->root; 272 struct btrfs_fs_info *fs_info = root->fs_info; 273 struct btrfs_trans_handle *trans; 274 u64 isize = i_size_read(inode); 275 u64 actual_end = min(end + 1, isize); 276 u64 inline_len = actual_end - start; 277 u64 aligned_end = ALIGN(end, fs_info->sectorsize); 278 u64 data_len = inline_len; 279 int ret; 280 struct btrfs_path *path; 281 int extent_inserted = 0; 282 u32 extent_item_size; 283 284 if (compressed_size) 285 data_len = compressed_size; 286 287 if (start > 0 || 288 actual_end > fs_info->sectorsize || 289 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 290 (!compressed_size && 291 (actual_end & (fs_info->sectorsize - 1)) == 0) || 292 end + 1 < isize || 293 data_len > fs_info->max_inline) { 294 return 1; 295 } 296 297 path = btrfs_alloc_path(); 298 if (!path) 299 return -ENOMEM; 300 301 trans = btrfs_join_transaction(root); 302 if (IS_ERR(trans)) { 303 btrfs_free_path(path); 304 return PTR_ERR(trans); 305 } 306 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 307 308 if (compressed_size && compressed_pages) 309 extent_item_size = btrfs_file_extent_calc_inline_size( 310 compressed_size); 311 else 312 extent_item_size = btrfs_file_extent_calc_inline_size( 313 inline_len); 314 315 ret = __btrfs_drop_extents(trans, root, inode, path, 316 start, aligned_end, NULL, 317 1, 1, extent_item_size, &extent_inserted); 318 if (ret) { 319 btrfs_abort_transaction(trans, ret); 320 goto out; 321 } 322 323 if (isize > actual_end) 324 inline_len = min_t(u64, isize, actual_end); 325 ret = insert_inline_extent(trans, path, extent_inserted, 326 root, inode, start, 327 inline_len, compressed_size, 328 compress_type, compressed_pages); 329 if (ret && ret != -ENOSPC) { 330 btrfs_abort_transaction(trans, ret); 331 goto out; 332 } else if (ret == -ENOSPC) { 333 ret = 1; 334 goto out; 335 } 336 337 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 338 btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0); 339 out: 340 /* 341 * Don't forget to free the reserved space, as for inlined extent 342 * it won't count as data extent, free them directly here. 343 * And at reserve time, it's always aligned to page size, so 344 * just free one page here. 345 */ 346 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE); 347 btrfs_free_path(path); 348 btrfs_end_transaction(trans); 349 return ret; 350 } 351 352 struct async_extent { 353 u64 start; 354 u64 ram_size; 355 u64 compressed_size; 356 struct page **pages; 357 unsigned long nr_pages; 358 int compress_type; 359 struct list_head list; 360 }; 361 362 struct async_cow { 363 struct inode *inode; 364 struct btrfs_root *root; 365 struct page *locked_page; 366 u64 start; 367 u64 end; 368 unsigned int write_flags; 369 struct list_head extents; 370 struct btrfs_work work; 371 }; 372 373 static noinline int add_async_extent(struct async_cow *cow, 374 u64 start, u64 ram_size, 375 u64 compressed_size, 376 struct page **pages, 377 unsigned long nr_pages, 378 int compress_type) 379 { 380 struct async_extent *async_extent; 381 382 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 383 BUG_ON(!async_extent); /* -ENOMEM */ 384 async_extent->start = start; 385 async_extent->ram_size = ram_size; 386 async_extent->compressed_size = compressed_size; 387 async_extent->pages = pages; 388 async_extent->nr_pages = nr_pages; 389 async_extent->compress_type = compress_type; 390 list_add_tail(&async_extent->list, &cow->extents); 391 return 0; 392 } 393 394 static inline int inode_need_compress(struct inode *inode, u64 start, u64 end) 395 { 396 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 397 398 /* force compress */ 399 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 400 return 1; 401 /* defrag ioctl */ 402 if (BTRFS_I(inode)->defrag_compress) 403 return 1; 404 /* bad compression ratios */ 405 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS) 406 return 0; 407 if (btrfs_test_opt(fs_info, COMPRESS) || 408 BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS || 409 BTRFS_I(inode)->prop_compress) 410 return btrfs_compress_heuristic(inode, start, end); 411 return 0; 412 } 413 414 static inline void inode_should_defrag(struct btrfs_inode *inode, 415 u64 start, u64 end, u64 num_bytes, u64 small_write) 416 { 417 /* If this is a small write inside eof, kick off a defrag */ 418 if (num_bytes < small_write && 419 (start > 0 || end + 1 < inode->disk_i_size)) 420 btrfs_add_inode_defrag(NULL, inode); 421 } 422 423 /* 424 * we create compressed extents in two phases. The first 425 * phase compresses a range of pages that have already been 426 * locked (both pages and state bits are locked). 427 * 428 * This is done inside an ordered work queue, and the compression 429 * is spread across many cpus. The actual IO submission is step 430 * two, and the ordered work queue takes care of making sure that 431 * happens in the same order things were put onto the queue by 432 * writepages and friends. 433 * 434 * If this code finds it can't get good compression, it puts an 435 * entry onto the work queue to write the uncompressed bytes. This 436 * makes sure that both compressed inodes and uncompressed inodes 437 * are written in the same order that the flusher thread sent them 438 * down. 439 */ 440 static noinline void compress_file_range(struct inode *inode, 441 struct page *locked_page, 442 u64 start, u64 end, 443 struct async_cow *async_cow, 444 int *num_added) 445 { 446 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 447 u64 blocksize = fs_info->sectorsize; 448 u64 actual_end; 449 u64 isize = i_size_read(inode); 450 int ret = 0; 451 struct page **pages = NULL; 452 unsigned long nr_pages; 453 unsigned long total_compressed = 0; 454 unsigned long total_in = 0; 455 int i; 456 int will_compress; 457 int compress_type = fs_info->compress_type; 458 int redirty = 0; 459 460 inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1, 461 SZ_16K); 462 463 actual_end = min_t(u64, isize, end + 1); 464 again: 465 will_compress = 0; 466 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; 467 BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0); 468 nr_pages = min_t(unsigned long, nr_pages, 469 BTRFS_MAX_COMPRESSED / PAGE_SIZE); 470 471 /* 472 * we don't want to send crud past the end of i_size through 473 * compression, that's just a waste of CPU time. So, if the 474 * end of the file is before the start of our current 475 * requested range of bytes, we bail out to the uncompressed 476 * cleanup code that can deal with all of this. 477 * 478 * It isn't really the fastest way to fix things, but this is a 479 * very uncommon corner. 480 */ 481 if (actual_end <= start) 482 goto cleanup_and_bail_uncompressed; 483 484 total_compressed = actual_end - start; 485 486 /* 487 * skip compression for a small file range(<=blocksize) that 488 * isn't an inline extent, since it doesn't save disk space at all. 489 */ 490 if (total_compressed <= blocksize && 491 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 492 goto cleanup_and_bail_uncompressed; 493 494 total_compressed = min_t(unsigned long, total_compressed, 495 BTRFS_MAX_UNCOMPRESSED); 496 total_in = 0; 497 ret = 0; 498 499 /* 500 * we do compression for mount -o compress and when the 501 * inode has not been flagged as nocompress. This flag can 502 * change at any time if we discover bad compression ratios. 503 */ 504 if (inode_need_compress(inode, start, end)) { 505 WARN_ON(pages); 506 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 507 if (!pages) { 508 /* just bail out to the uncompressed code */ 509 goto cont; 510 } 511 512 if (BTRFS_I(inode)->defrag_compress) 513 compress_type = BTRFS_I(inode)->defrag_compress; 514 else if (BTRFS_I(inode)->prop_compress) 515 compress_type = BTRFS_I(inode)->prop_compress; 516 517 /* 518 * we need to call clear_page_dirty_for_io on each 519 * page in the range. Otherwise applications with the file 520 * mmap'd can wander in and change the page contents while 521 * we are compressing them. 522 * 523 * If the compression fails for any reason, we set the pages 524 * dirty again later on. 525 * 526 * Note that the remaining part is redirtied, the start pointer 527 * has moved, the end is the original one. 528 */ 529 if (!redirty) { 530 extent_range_clear_dirty_for_io(inode, start, end); 531 redirty = 1; 532 } 533 534 /* Compression level is applied here and only here */ 535 ret = btrfs_compress_pages( 536 compress_type | (fs_info->compress_level << 4), 537 inode->i_mapping, start, 538 pages, 539 &nr_pages, 540 &total_in, 541 &total_compressed); 542 543 if (!ret) { 544 unsigned long offset = total_compressed & 545 (PAGE_SIZE - 1); 546 struct page *page = pages[nr_pages - 1]; 547 char *kaddr; 548 549 /* zero the tail end of the last page, we might be 550 * sending it down to disk 551 */ 552 if (offset) { 553 kaddr = kmap_atomic(page); 554 memset(kaddr + offset, 0, 555 PAGE_SIZE - offset); 556 kunmap_atomic(kaddr); 557 } 558 will_compress = 1; 559 } 560 } 561 cont: 562 if (start == 0) { 563 /* lets try to make an inline extent */ 564 if (ret || total_in < actual_end) { 565 /* we didn't compress the entire range, try 566 * to make an uncompressed inline extent. 567 */ 568 ret = cow_file_range_inline(inode, start, end, 0, 569 BTRFS_COMPRESS_NONE, NULL); 570 } else { 571 /* try making a compressed inline extent */ 572 ret = cow_file_range_inline(inode, start, end, 573 total_compressed, 574 compress_type, pages); 575 } 576 if (ret <= 0) { 577 unsigned long clear_flags = EXTENT_DELALLOC | 578 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 579 EXTENT_DO_ACCOUNTING; 580 unsigned long page_error_op; 581 582 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 583 584 /* 585 * inline extent creation worked or returned error, 586 * we don't need to create any more async work items. 587 * Unlock and free up our temp pages. 588 * 589 * We use DO_ACCOUNTING here because we need the 590 * delalloc_release_metadata to be done _after_ we drop 591 * our outstanding extent for clearing delalloc for this 592 * range. 593 */ 594 extent_clear_unlock_delalloc(inode, start, end, end, 595 NULL, clear_flags, 596 PAGE_UNLOCK | 597 PAGE_CLEAR_DIRTY | 598 PAGE_SET_WRITEBACK | 599 page_error_op | 600 PAGE_END_WRITEBACK); 601 goto free_pages_out; 602 } 603 } 604 605 if (will_compress) { 606 /* 607 * we aren't doing an inline extent round the compressed size 608 * up to a block size boundary so the allocator does sane 609 * things 610 */ 611 total_compressed = ALIGN(total_compressed, blocksize); 612 613 /* 614 * one last check to make sure the compression is really a 615 * win, compare the page count read with the blocks on disk, 616 * compression must free at least one sector size 617 */ 618 total_in = ALIGN(total_in, PAGE_SIZE); 619 if (total_compressed + blocksize <= total_in) { 620 *num_added += 1; 621 622 /* 623 * The async work queues will take care of doing actual 624 * allocation on disk for these compressed pages, and 625 * will submit them to the elevator. 626 */ 627 add_async_extent(async_cow, start, total_in, 628 total_compressed, pages, nr_pages, 629 compress_type); 630 631 if (start + total_in < end) { 632 start += total_in; 633 pages = NULL; 634 cond_resched(); 635 goto again; 636 } 637 return; 638 } 639 } 640 if (pages) { 641 /* 642 * the compression code ran but failed to make things smaller, 643 * free any pages it allocated and our page pointer array 644 */ 645 for (i = 0; i < nr_pages; i++) { 646 WARN_ON(pages[i]->mapping); 647 put_page(pages[i]); 648 } 649 kfree(pages); 650 pages = NULL; 651 total_compressed = 0; 652 nr_pages = 0; 653 654 /* flag the file so we don't compress in the future */ 655 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && 656 !(BTRFS_I(inode)->prop_compress)) { 657 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 658 } 659 } 660 cleanup_and_bail_uncompressed: 661 /* 662 * No compression, but we still need to write the pages in the file 663 * we've been given so far. redirty the locked page if it corresponds 664 * to our extent and set things up for the async work queue to run 665 * cow_file_range to do the normal delalloc dance. 666 */ 667 if (page_offset(locked_page) >= start && 668 page_offset(locked_page) <= end) 669 __set_page_dirty_nobuffers(locked_page); 670 /* unlocked later on in the async handlers */ 671 672 if (redirty) 673 extent_range_redirty_for_io(inode, start, end); 674 add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0, 675 BTRFS_COMPRESS_NONE); 676 *num_added += 1; 677 678 return; 679 680 free_pages_out: 681 for (i = 0; i < nr_pages; i++) { 682 WARN_ON(pages[i]->mapping); 683 put_page(pages[i]); 684 } 685 kfree(pages); 686 } 687 688 static void free_async_extent_pages(struct async_extent *async_extent) 689 { 690 int i; 691 692 if (!async_extent->pages) 693 return; 694 695 for (i = 0; i < async_extent->nr_pages; i++) { 696 WARN_ON(async_extent->pages[i]->mapping); 697 put_page(async_extent->pages[i]); 698 } 699 kfree(async_extent->pages); 700 async_extent->nr_pages = 0; 701 async_extent->pages = NULL; 702 } 703 704 /* 705 * phase two of compressed writeback. This is the ordered portion 706 * of the code, which only gets called in the order the work was 707 * queued. We walk all the async extents created by compress_file_range 708 * and send them down to the disk. 709 */ 710 static noinline void submit_compressed_extents(struct inode *inode, 711 struct async_cow *async_cow) 712 { 713 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 714 struct async_extent *async_extent; 715 u64 alloc_hint = 0; 716 struct btrfs_key ins; 717 struct extent_map *em; 718 struct btrfs_root *root = BTRFS_I(inode)->root; 719 struct extent_io_tree *io_tree; 720 int ret = 0; 721 722 again: 723 while (!list_empty(&async_cow->extents)) { 724 async_extent = list_entry(async_cow->extents.next, 725 struct async_extent, list); 726 list_del(&async_extent->list); 727 728 io_tree = &BTRFS_I(inode)->io_tree; 729 730 retry: 731 /* did the compression code fall back to uncompressed IO? */ 732 if (!async_extent->pages) { 733 int page_started = 0; 734 unsigned long nr_written = 0; 735 736 lock_extent(io_tree, async_extent->start, 737 async_extent->start + 738 async_extent->ram_size - 1); 739 740 /* allocate blocks */ 741 ret = cow_file_range(inode, async_cow->locked_page, 742 async_extent->start, 743 async_extent->start + 744 async_extent->ram_size - 1, 745 async_extent->start + 746 async_extent->ram_size - 1, 747 &page_started, &nr_written, 0, 748 NULL); 749 750 /* JDM XXX */ 751 752 /* 753 * if page_started, cow_file_range inserted an 754 * inline extent and took care of all the unlocking 755 * and IO for us. Otherwise, we need to submit 756 * all those pages down to the drive. 757 */ 758 if (!page_started && !ret) 759 extent_write_locked_range(inode, 760 async_extent->start, 761 async_extent->start + 762 async_extent->ram_size - 1, 763 WB_SYNC_ALL); 764 else if (ret) 765 unlock_page(async_cow->locked_page); 766 kfree(async_extent); 767 cond_resched(); 768 continue; 769 } 770 771 lock_extent(io_tree, async_extent->start, 772 async_extent->start + async_extent->ram_size - 1); 773 774 ret = btrfs_reserve_extent(root, async_extent->ram_size, 775 async_extent->compressed_size, 776 async_extent->compressed_size, 777 0, alloc_hint, &ins, 1, 1); 778 if (ret) { 779 free_async_extent_pages(async_extent); 780 781 if (ret == -ENOSPC) { 782 unlock_extent(io_tree, async_extent->start, 783 async_extent->start + 784 async_extent->ram_size - 1); 785 786 /* 787 * we need to redirty the pages if we decide to 788 * fallback to uncompressed IO, otherwise we 789 * will not submit these pages down to lower 790 * layers. 791 */ 792 extent_range_redirty_for_io(inode, 793 async_extent->start, 794 async_extent->start + 795 async_extent->ram_size - 1); 796 797 goto retry; 798 } 799 goto out_free; 800 } 801 /* 802 * here we're doing allocation and writeback of the 803 * compressed pages 804 */ 805 em = create_io_em(inode, async_extent->start, 806 async_extent->ram_size, /* len */ 807 async_extent->start, /* orig_start */ 808 ins.objectid, /* block_start */ 809 ins.offset, /* block_len */ 810 ins.offset, /* orig_block_len */ 811 async_extent->ram_size, /* ram_bytes */ 812 async_extent->compress_type, 813 BTRFS_ORDERED_COMPRESSED); 814 if (IS_ERR(em)) 815 /* ret value is not necessary due to void function */ 816 goto out_free_reserve; 817 free_extent_map(em); 818 819 ret = btrfs_add_ordered_extent_compress(inode, 820 async_extent->start, 821 ins.objectid, 822 async_extent->ram_size, 823 ins.offset, 824 BTRFS_ORDERED_COMPRESSED, 825 async_extent->compress_type); 826 if (ret) { 827 btrfs_drop_extent_cache(BTRFS_I(inode), 828 async_extent->start, 829 async_extent->start + 830 async_extent->ram_size - 1, 0); 831 goto out_free_reserve; 832 } 833 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 834 835 /* 836 * clear dirty, set writeback and unlock the pages. 837 */ 838 extent_clear_unlock_delalloc(inode, async_extent->start, 839 async_extent->start + 840 async_extent->ram_size - 1, 841 async_extent->start + 842 async_extent->ram_size - 1, 843 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 844 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 845 PAGE_SET_WRITEBACK); 846 if (btrfs_submit_compressed_write(inode, 847 async_extent->start, 848 async_extent->ram_size, 849 ins.objectid, 850 ins.offset, async_extent->pages, 851 async_extent->nr_pages, 852 async_cow->write_flags)) { 853 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 854 struct page *p = async_extent->pages[0]; 855 const u64 start = async_extent->start; 856 const u64 end = start + async_extent->ram_size - 1; 857 858 p->mapping = inode->i_mapping; 859 tree->ops->writepage_end_io_hook(p, start, end, 860 NULL, 0); 861 p->mapping = NULL; 862 extent_clear_unlock_delalloc(inode, start, end, end, 863 NULL, 0, 864 PAGE_END_WRITEBACK | 865 PAGE_SET_ERROR); 866 free_async_extent_pages(async_extent); 867 } 868 alloc_hint = ins.objectid + ins.offset; 869 kfree(async_extent); 870 cond_resched(); 871 } 872 return; 873 out_free_reserve: 874 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 875 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 876 out_free: 877 extent_clear_unlock_delalloc(inode, async_extent->start, 878 async_extent->start + 879 async_extent->ram_size - 1, 880 async_extent->start + 881 async_extent->ram_size - 1, 882 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 883 EXTENT_DELALLOC_NEW | 884 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 885 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 886 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | 887 PAGE_SET_ERROR); 888 free_async_extent_pages(async_extent); 889 kfree(async_extent); 890 goto again; 891 } 892 893 static u64 get_extent_allocation_hint(struct inode *inode, u64 start, 894 u64 num_bytes) 895 { 896 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 897 struct extent_map *em; 898 u64 alloc_hint = 0; 899 900 read_lock(&em_tree->lock); 901 em = search_extent_mapping(em_tree, start, num_bytes); 902 if (em) { 903 /* 904 * if block start isn't an actual block number then find the 905 * first block in this inode and use that as a hint. If that 906 * block is also bogus then just don't worry about it. 907 */ 908 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 909 free_extent_map(em); 910 em = search_extent_mapping(em_tree, 0, 0); 911 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 912 alloc_hint = em->block_start; 913 if (em) 914 free_extent_map(em); 915 } else { 916 alloc_hint = em->block_start; 917 free_extent_map(em); 918 } 919 } 920 read_unlock(&em_tree->lock); 921 922 return alloc_hint; 923 } 924 925 /* 926 * when extent_io.c finds a delayed allocation range in the file, 927 * the call backs end up in this code. The basic idea is to 928 * allocate extents on disk for the range, and create ordered data structs 929 * in ram to track those extents. 930 * 931 * locked_page is the page that writepage had locked already. We use 932 * it to make sure we don't do extra locks or unlocks. 933 * 934 * *page_started is set to one if we unlock locked_page and do everything 935 * required to start IO on it. It may be clean and already done with 936 * IO when we return. 937 */ 938 static noinline int cow_file_range(struct inode *inode, 939 struct page *locked_page, 940 u64 start, u64 end, u64 delalloc_end, 941 int *page_started, unsigned long *nr_written, 942 int unlock, struct btrfs_dedupe_hash *hash) 943 { 944 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 945 struct btrfs_root *root = BTRFS_I(inode)->root; 946 u64 alloc_hint = 0; 947 u64 num_bytes; 948 unsigned long ram_size; 949 u64 cur_alloc_size = 0; 950 u64 blocksize = fs_info->sectorsize; 951 struct btrfs_key ins; 952 struct extent_map *em; 953 unsigned clear_bits; 954 unsigned long page_ops; 955 bool extent_reserved = false; 956 int ret = 0; 957 958 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 959 WARN_ON_ONCE(1); 960 ret = -EINVAL; 961 goto out_unlock; 962 } 963 964 num_bytes = ALIGN(end - start + 1, blocksize); 965 num_bytes = max(blocksize, num_bytes); 966 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); 967 968 inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K); 969 970 if (start == 0) { 971 /* lets try to make an inline extent */ 972 ret = cow_file_range_inline(inode, start, end, 0, 973 BTRFS_COMPRESS_NONE, NULL); 974 if (ret == 0) { 975 /* 976 * We use DO_ACCOUNTING here because we need the 977 * delalloc_release_metadata to be run _after_ we drop 978 * our outstanding extent for clearing delalloc for this 979 * range. 980 */ 981 extent_clear_unlock_delalloc(inode, start, end, 982 delalloc_end, NULL, 983 EXTENT_LOCKED | EXTENT_DELALLOC | 984 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 985 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 986 PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 987 PAGE_END_WRITEBACK); 988 *nr_written = *nr_written + 989 (end - start + PAGE_SIZE) / PAGE_SIZE; 990 *page_started = 1; 991 goto out; 992 } else if (ret < 0) { 993 goto out_unlock; 994 } 995 } 996 997 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 998 btrfs_drop_extent_cache(BTRFS_I(inode), start, 999 start + num_bytes - 1, 0); 1000 1001 while (num_bytes > 0) { 1002 cur_alloc_size = num_bytes; 1003 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size, 1004 fs_info->sectorsize, 0, alloc_hint, 1005 &ins, 1, 1); 1006 if (ret < 0) 1007 goto out_unlock; 1008 cur_alloc_size = ins.offset; 1009 extent_reserved = true; 1010 1011 ram_size = ins.offset; 1012 em = create_io_em(inode, start, ins.offset, /* len */ 1013 start, /* orig_start */ 1014 ins.objectid, /* block_start */ 1015 ins.offset, /* block_len */ 1016 ins.offset, /* orig_block_len */ 1017 ram_size, /* ram_bytes */ 1018 BTRFS_COMPRESS_NONE, /* compress_type */ 1019 BTRFS_ORDERED_REGULAR /* type */); 1020 if (IS_ERR(em)) 1021 goto out_reserve; 1022 free_extent_map(em); 1023 1024 ret = btrfs_add_ordered_extent(inode, start, ins.objectid, 1025 ram_size, cur_alloc_size, 0); 1026 if (ret) 1027 goto out_drop_extent_cache; 1028 1029 if (root->root_key.objectid == 1030 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1031 ret = btrfs_reloc_clone_csums(inode, start, 1032 cur_alloc_size); 1033 /* 1034 * Only drop cache here, and process as normal. 1035 * 1036 * We must not allow extent_clear_unlock_delalloc() 1037 * at out_unlock label to free meta of this ordered 1038 * extent, as its meta should be freed by 1039 * btrfs_finish_ordered_io(). 1040 * 1041 * So we must continue until @start is increased to 1042 * skip current ordered extent. 1043 */ 1044 if (ret) 1045 btrfs_drop_extent_cache(BTRFS_I(inode), start, 1046 start + ram_size - 1, 0); 1047 } 1048 1049 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1050 1051 /* we're not doing compressed IO, don't unlock the first 1052 * page (which the caller expects to stay locked), don't 1053 * clear any dirty bits and don't set any writeback bits 1054 * 1055 * Do set the Private2 bit so we know this page was properly 1056 * setup for writepage 1057 */ 1058 page_ops = unlock ? PAGE_UNLOCK : 0; 1059 page_ops |= PAGE_SET_PRIVATE2; 1060 1061 extent_clear_unlock_delalloc(inode, start, 1062 start + ram_size - 1, 1063 delalloc_end, locked_page, 1064 EXTENT_LOCKED | EXTENT_DELALLOC, 1065 page_ops); 1066 if (num_bytes < cur_alloc_size) 1067 num_bytes = 0; 1068 else 1069 num_bytes -= cur_alloc_size; 1070 alloc_hint = ins.objectid + ins.offset; 1071 start += cur_alloc_size; 1072 extent_reserved = false; 1073 1074 /* 1075 * btrfs_reloc_clone_csums() error, since start is increased 1076 * extent_clear_unlock_delalloc() at out_unlock label won't 1077 * free metadata of current ordered extent, we're OK to exit. 1078 */ 1079 if (ret) 1080 goto out_unlock; 1081 } 1082 out: 1083 return ret; 1084 1085 out_drop_extent_cache: 1086 btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0); 1087 out_reserve: 1088 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1089 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1090 out_unlock: 1091 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 1092 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; 1093 page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 1094 PAGE_END_WRITEBACK; 1095 /* 1096 * If we reserved an extent for our delalloc range (or a subrange) and 1097 * failed to create the respective ordered extent, then it means that 1098 * when we reserved the extent we decremented the extent's size from 1099 * the data space_info's bytes_may_use counter and incremented the 1100 * space_info's bytes_reserved counter by the same amount. We must make 1101 * sure extent_clear_unlock_delalloc() does not try to decrement again 1102 * the data space_info's bytes_may_use counter, therefore we do not pass 1103 * it the flag EXTENT_CLEAR_DATA_RESV. 1104 */ 1105 if (extent_reserved) { 1106 extent_clear_unlock_delalloc(inode, start, 1107 start + cur_alloc_size, 1108 start + cur_alloc_size, 1109 locked_page, 1110 clear_bits, 1111 page_ops); 1112 start += cur_alloc_size; 1113 if (start >= end) 1114 goto out; 1115 } 1116 extent_clear_unlock_delalloc(inode, start, end, delalloc_end, 1117 locked_page, 1118 clear_bits | EXTENT_CLEAR_DATA_RESV, 1119 page_ops); 1120 goto out; 1121 } 1122 1123 /* 1124 * work queue call back to started compression on a file and pages 1125 */ 1126 static noinline void async_cow_start(struct btrfs_work *work) 1127 { 1128 struct async_cow *async_cow; 1129 int num_added = 0; 1130 async_cow = container_of(work, struct async_cow, work); 1131 1132 compress_file_range(async_cow->inode, async_cow->locked_page, 1133 async_cow->start, async_cow->end, async_cow, 1134 &num_added); 1135 if (num_added == 0) { 1136 btrfs_add_delayed_iput(async_cow->inode); 1137 async_cow->inode = NULL; 1138 } 1139 } 1140 1141 /* 1142 * work queue call back to submit previously compressed pages 1143 */ 1144 static noinline void async_cow_submit(struct btrfs_work *work) 1145 { 1146 struct btrfs_fs_info *fs_info; 1147 struct async_cow *async_cow; 1148 struct btrfs_root *root; 1149 unsigned long nr_pages; 1150 1151 async_cow = container_of(work, struct async_cow, work); 1152 1153 root = async_cow->root; 1154 fs_info = root->fs_info; 1155 nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >> 1156 PAGE_SHIFT; 1157 1158 /* 1159 * atomic_sub_return implies a barrier for waitqueue_active 1160 */ 1161 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1162 5 * SZ_1M && 1163 waitqueue_active(&fs_info->async_submit_wait)) 1164 wake_up(&fs_info->async_submit_wait); 1165 1166 if (async_cow->inode) 1167 submit_compressed_extents(async_cow->inode, async_cow); 1168 } 1169 1170 static noinline void async_cow_free(struct btrfs_work *work) 1171 { 1172 struct async_cow *async_cow; 1173 async_cow = container_of(work, struct async_cow, work); 1174 if (async_cow->inode) 1175 btrfs_add_delayed_iput(async_cow->inode); 1176 kfree(async_cow); 1177 } 1178 1179 static int cow_file_range_async(struct inode *inode, struct page *locked_page, 1180 u64 start, u64 end, int *page_started, 1181 unsigned long *nr_written, 1182 unsigned int write_flags) 1183 { 1184 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1185 struct async_cow *async_cow; 1186 struct btrfs_root *root = BTRFS_I(inode)->root; 1187 unsigned long nr_pages; 1188 u64 cur_end; 1189 1190 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED, 1191 1, 0, NULL); 1192 while (start < end) { 1193 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS); 1194 BUG_ON(!async_cow); /* -ENOMEM */ 1195 async_cow->inode = igrab(inode); 1196 async_cow->root = root; 1197 async_cow->locked_page = locked_page; 1198 async_cow->start = start; 1199 async_cow->write_flags = write_flags; 1200 1201 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS && 1202 !btrfs_test_opt(fs_info, FORCE_COMPRESS)) 1203 cur_end = end; 1204 else 1205 cur_end = min(end, start + SZ_512K - 1); 1206 1207 async_cow->end = cur_end; 1208 INIT_LIST_HEAD(&async_cow->extents); 1209 1210 btrfs_init_work(&async_cow->work, 1211 btrfs_delalloc_helper, 1212 async_cow_start, async_cow_submit, 1213 async_cow_free); 1214 1215 nr_pages = (cur_end - start + PAGE_SIZE) >> 1216 PAGE_SHIFT; 1217 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1218 1219 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work); 1220 1221 *nr_written += nr_pages; 1222 start = cur_end + 1; 1223 } 1224 *page_started = 1; 1225 return 0; 1226 } 1227 1228 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, 1229 u64 bytenr, u64 num_bytes) 1230 { 1231 int ret; 1232 struct btrfs_ordered_sum *sums; 1233 LIST_HEAD(list); 1234 1235 ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr, 1236 bytenr + num_bytes - 1, &list, 0); 1237 if (ret == 0 && list_empty(&list)) 1238 return 0; 1239 1240 while (!list_empty(&list)) { 1241 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1242 list_del(&sums->list); 1243 kfree(sums); 1244 } 1245 if (ret < 0) 1246 return ret; 1247 return 1; 1248 } 1249 1250 /* 1251 * when nowcow writeback call back. This checks for snapshots or COW copies 1252 * of the extents that exist in the file, and COWs the file as required. 1253 * 1254 * If no cow copies or snapshots exist, we write directly to the existing 1255 * blocks on disk 1256 */ 1257 static noinline int run_delalloc_nocow(struct inode *inode, 1258 struct page *locked_page, 1259 u64 start, u64 end, int *page_started, int force, 1260 unsigned long *nr_written) 1261 { 1262 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1263 struct btrfs_root *root = BTRFS_I(inode)->root; 1264 struct extent_buffer *leaf; 1265 struct btrfs_path *path; 1266 struct btrfs_file_extent_item *fi; 1267 struct btrfs_key found_key; 1268 struct extent_map *em; 1269 u64 cow_start; 1270 u64 cur_offset; 1271 u64 extent_end; 1272 u64 extent_offset; 1273 u64 disk_bytenr; 1274 u64 num_bytes; 1275 u64 disk_num_bytes; 1276 u64 ram_bytes; 1277 int extent_type; 1278 int ret, err; 1279 int type; 1280 int nocow; 1281 int check_prev = 1; 1282 bool nolock; 1283 u64 ino = btrfs_ino(BTRFS_I(inode)); 1284 1285 path = btrfs_alloc_path(); 1286 if (!path) { 1287 extent_clear_unlock_delalloc(inode, start, end, end, 1288 locked_page, 1289 EXTENT_LOCKED | EXTENT_DELALLOC | 1290 EXTENT_DO_ACCOUNTING | 1291 EXTENT_DEFRAG, PAGE_UNLOCK | 1292 PAGE_CLEAR_DIRTY | 1293 PAGE_SET_WRITEBACK | 1294 PAGE_END_WRITEBACK); 1295 return -ENOMEM; 1296 } 1297 1298 nolock = btrfs_is_free_space_inode(BTRFS_I(inode)); 1299 1300 cow_start = (u64)-1; 1301 cur_offset = start; 1302 while (1) { 1303 ret = btrfs_lookup_file_extent(NULL, root, path, ino, 1304 cur_offset, 0); 1305 if (ret < 0) 1306 goto error; 1307 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1308 leaf = path->nodes[0]; 1309 btrfs_item_key_to_cpu(leaf, &found_key, 1310 path->slots[0] - 1); 1311 if (found_key.objectid == ino && 1312 found_key.type == BTRFS_EXTENT_DATA_KEY) 1313 path->slots[0]--; 1314 } 1315 check_prev = 0; 1316 next_slot: 1317 leaf = path->nodes[0]; 1318 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1319 ret = btrfs_next_leaf(root, path); 1320 if (ret < 0) { 1321 if (cow_start != (u64)-1) 1322 cur_offset = cow_start; 1323 goto error; 1324 } 1325 if (ret > 0) 1326 break; 1327 leaf = path->nodes[0]; 1328 } 1329 1330 nocow = 0; 1331 disk_bytenr = 0; 1332 num_bytes = 0; 1333 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1334 1335 if (found_key.objectid > ino) 1336 break; 1337 if (WARN_ON_ONCE(found_key.objectid < ino) || 1338 found_key.type < BTRFS_EXTENT_DATA_KEY) { 1339 path->slots[0]++; 1340 goto next_slot; 1341 } 1342 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 1343 found_key.offset > end) 1344 break; 1345 1346 if (found_key.offset > cur_offset) { 1347 extent_end = found_key.offset; 1348 extent_type = 0; 1349 goto out_check; 1350 } 1351 1352 fi = btrfs_item_ptr(leaf, path->slots[0], 1353 struct btrfs_file_extent_item); 1354 extent_type = btrfs_file_extent_type(leaf, fi); 1355 1356 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1357 if (extent_type == BTRFS_FILE_EXTENT_REG || 1358 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1359 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1360 extent_offset = btrfs_file_extent_offset(leaf, fi); 1361 extent_end = found_key.offset + 1362 btrfs_file_extent_num_bytes(leaf, fi); 1363 disk_num_bytes = 1364 btrfs_file_extent_disk_num_bytes(leaf, fi); 1365 if (extent_end <= start) { 1366 path->slots[0]++; 1367 goto next_slot; 1368 } 1369 if (disk_bytenr == 0) 1370 goto out_check; 1371 if (btrfs_file_extent_compression(leaf, fi) || 1372 btrfs_file_extent_encryption(leaf, fi) || 1373 btrfs_file_extent_other_encoding(leaf, fi)) 1374 goto out_check; 1375 if (extent_type == BTRFS_FILE_EXTENT_REG && !force) 1376 goto out_check; 1377 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 1378 goto out_check; 1379 ret = btrfs_cross_ref_exist(root, ino, 1380 found_key.offset - 1381 extent_offset, disk_bytenr); 1382 if (ret) { 1383 /* 1384 * ret could be -EIO if the above fails to read 1385 * metadata. 1386 */ 1387 if (ret < 0) { 1388 if (cow_start != (u64)-1) 1389 cur_offset = cow_start; 1390 goto error; 1391 } 1392 1393 WARN_ON_ONCE(nolock); 1394 goto out_check; 1395 } 1396 disk_bytenr += extent_offset; 1397 disk_bytenr += cur_offset - found_key.offset; 1398 num_bytes = min(end + 1, extent_end) - cur_offset; 1399 /* 1400 * if there are pending snapshots for this root, 1401 * we fall into common COW way. 1402 */ 1403 if (!nolock) { 1404 err = btrfs_start_write_no_snapshotting(root); 1405 if (!err) 1406 goto out_check; 1407 } 1408 /* 1409 * force cow if csum exists in the range. 1410 * this ensure that csum for a given extent are 1411 * either valid or do not exist. 1412 */ 1413 ret = csum_exist_in_range(fs_info, disk_bytenr, 1414 num_bytes); 1415 if (ret) { 1416 if (!nolock) 1417 btrfs_end_write_no_snapshotting(root); 1418 1419 /* 1420 * ret could be -EIO if the above fails to read 1421 * metadata. 1422 */ 1423 if (ret < 0) { 1424 if (cow_start != (u64)-1) 1425 cur_offset = cow_start; 1426 goto error; 1427 } 1428 WARN_ON_ONCE(nolock); 1429 goto out_check; 1430 } 1431 if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) { 1432 if (!nolock) 1433 btrfs_end_write_no_snapshotting(root); 1434 goto out_check; 1435 } 1436 nocow = 1; 1437 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 1438 extent_end = found_key.offset + 1439 btrfs_file_extent_inline_len(leaf, 1440 path->slots[0], fi); 1441 extent_end = ALIGN(extent_end, 1442 fs_info->sectorsize); 1443 } else { 1444 BUG_ON(1); 1445 } 1446 out_check: 1447 if (extent_end <= start) { 1448 path->slots[0]++; 1449 if (!nolock && nocow) 1450 btrfs_end_write_no_snapshotting(root); 1451 if (nocow) 1452 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1453 goto next_slot; 1454 } 1455 if (!nocow) { 1456 if (cow_start == (u64)-1) 1457 cow_start = cur_offset; 1458 cur_offset = extent_end; 1459 if (cur_offset > end) 1460 break; 1461 path->slots[0]++; 1462 goto next_slot; 1463 } 1464 1465 btrfs_release_path(path); 1466 if (cow_start != (u64)-1) { 1467 ret = cow_file_range(inode, locked_page, 1468 cow_start, found_key.offset - 1, 1469 end, page_started, nr_written, 1, 1470 NULL); 1471 if (ret) { 1472 if (!nolock && nocow) 1473 btrfs_end_write_no_snapshotting(root); 1474 if (nocow) 1475 btrfs_dec_nocow_writers(fs_info, 1476 disk_bytenr); 1477 goto error; 1478 } 1479 cow_start = (u64)-1; 1480 } 1481 1482 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1483 u64 orig_start = found_key.offset - extent_offset; 1484 1485 em = create_io_em(inode, cur_offset, num_bytes, 1486 orig_start, 1487 disk_bytenr, /* block_start */ 1488 num_bytes, /* block_len */ 1489 disk_num_bytes, /* orig_block_len */ 1490 ram_bytes, BTRFS_COMPRESS_NONE, 1491 BTRFS_ORDERED_PREALLOC); 1492 if (IS_ERR(em)) { 1493 if (!nolock && nocow) 1494 btrfs_end_write_no_snapshotting(root); 1495 if (nocow) 1496 btrfs_dec_nocow_writers(fs_info, 1497 disk_bytenr); 1498 ret = PTR_ERR(em); 1499 goto error; 1500 } 1501 free_extent_map(em); 1502 } 1503 1504 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1505 type = BTRFS_ORDERED_PREALLOC; 1506 } else { 1507 type = BTRFS_ORDERED_NOCOW; 1508 } 1509 1510 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr, 1511 num_bytes, num_bytes, type); 1512 if (nocow) 1513 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1514 BUG_ON(ret); /* -ENOMEM */ 1515 1516 if (root->root_key.objectid == 1517 BTRFS_DATA_RELOC_TREE_OBJECTID) 1518 /* 1519 * Error handled later, as we must prevent 1520 * extent_clear_unlock_delalloc() in error handler 1521 * from freeing metadata of created ordered extent. 1522 */ 1523 ret = btrfs_reloc_clone_csums(inode, cur_offset, 1524 num_bytes); 1525 1526 extent_clear_unlock_delalloc(inode, cur_offset, 1527 cur_offset + num_bytes - 1, end, 1528 locked_page, EXTENT_LOCKED | 1529 EXTENT_DELALLOC | 1530 EXTENT_CLEAR_DATA_RESV, 1531 PAGE_UNLOCK | PAGE_SET_PRIVATE2); 1532 1533 if (!nolock && nocow) 1534 btrfs_end_write_no_snapshotting(root); 1535 cur_offset = extent_end; 1536 1537 /* 1538 * btrfs_reloc_clone_csums() error, now we're OK to call error 1539 * handler, as metadata for created ordered extent will only 1540 * be freed by btrfs_finish_ordered_io(). 1541 */ 1542 if (ret) 1543 goto error; 1544 if (cur_offset > end) 1545 break; 1546 } 1547 btrfs_release_path(path); 1548 1549 if (cur_offset <= end && cow_start == (u64)-1) { 1550 cow_start = cur_offset; 1551 cur_offset = end; 1552 } 1553 1554 if (cow_start != (u64)-1) { 1555 ret = cow_file_range(inode, locked_page, cow_start, end, end, 1556 page_started, nr_written, 1, NULL); 1557 if (ret) 1558 goto error; 1559 } 1560 1561 error: 1562 if (ret && cur_offset < end) 1563 extent_clear_unlock_delalloc(inode, cur_offset, end, end, 1564 locked_page, EXTENT_LOCKED | 1565 EXTENT_DELALLOC | EXTENT_DEFRAG | 1566 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1567 PAGE_CLEAR_DIRTY | 1568 PAGE_SET_WRITEBACK | 1569 PAGE_END_WRITEBACK); 1570 btrfs_free_path(path); 1571 return ret; 1572 } 1573 1574 static inline int need_force_cow(struct inode *inode, u64 start, u64 end) 1575 { 1576 1577 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 1578 !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC)) 1579 return 0; 1580 1581 /* 1582 * @defrag_bytes is a hint value, no spinlock held here, 1583 * if is not zero, it means the file is defragging. 1584 * Force cow if given extent needs to be defragged. 1585 */ 1586 if (BTRFS_I(inode)->defrag_bytes && 1587 test_range_bit(&BTRFS_I(inode)->io_tree, start, end, 1588 EXTENT_DEFRAG, 0, NULL)) 1589 return 1; 1590 1591 return 0; 1592 } 1593 1594 /* 1595 * extent_io.c call back to do delayed allocation processing 1596 */ 1597 static int run_delalloc_range(void *private_data, struct page *locked_page, 1598 u64 start, u64 end, int *page_started, 1599 unsigned long *nr_written, 1600 struct writeback_control *wbc) 1601 { 1602 struct inode *inode = private_data; 1603 int ret; 1604 int force_cow = need_force_cow(inode, start, end); 1605 unsigned int write_flags = wbc_to_write_flags(wbc); 1606 1607 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) { 1608 ret = run_delalloc_nocow(inode, locked_page, start, end, 1609 page_started, 1, nr_written); 1610 } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) { 1611 ret = run_delalloc_nocow(inode, locked_page, start, end, 1612 page_started, 0, nr_written); 1613 } else if (!inode_need_compress(inode, start, end)) { 1614 ret = cow_file_range(inode, locked_page, start, end, end, 1615 page_started, nr_written, 1, NULL); 1616 } else { 1617 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 1618 &BTRFS_I(inode)->runtime_flags); 1619 ret = cow_file_range_async(inode, locked_page, start, end, 1620 page_started, nr_written, 1621 write_flags); 1622 } 1623 if (ret) 1624 btrfs_cleanup_ordered_extents(inode, start, end - start + 1); 1625 return ret; 1626 } 1627 1628 static void btrfs_split_extent_hook(void *private_data, 1629 struct extent_state *orig, u64 split) 1630 { 1631 struct inode *inode = private_data; 1632 u64 size; 1633 1634 /* not delalloc, ignore it */ 1635 if (!(orig->state & EXTENT_DELALLOC)) 1636 return; 1637 1638 size = orig->end - orig->start + 1; 1639 if (size > BTRFS_MAX_EXTENT_SIZE) { 1640 u32 num_extents; 1641 u64 new_size; 1642 1643 /* 1644 * See the explanation in btrfs_merge_extent_hook, the same 1645 * applies here, just in reverse. 1646 */ 1647 new_size = orig->end - split + 1; 1648 num_extents = count_max_extents(new_size); 1649 new_size = split - orig->start; 1650 num_extents += count_max_extents(new_size); 1651 if (count_max_extents(size) >= num_extents) 1652 return; 1653 } 1654 1655 spin_lock(&BTRFS_I(inode)->lock); 1656 btrfs_mod_outstanding_extents(BTRFS_I(inode), 1); 1657 spin_unlock(&BTRFS_I(inode)->lock); 1658 } 1659 1660 /* 1661 * extent_io.c merge_extent_hook, used to track merged delayed allocation 1662 * extents so we can keep track of new extents that are just merged onto old 1663 * extents, such as when we are doing sequential writes, so we can properly 1664 * account for the metadata space we'll need. 1665 */ 1666 static void btrfs_merge_extent_hook(void *private_data, 1667 struct extent_state *new, 1668 struct extent_state *other) 1669 { 1670 struct inode *inode = private_data; 1671 u64 new_size, old_size; 1672 u32 num_extents; 1673 1674 /* not delalloc, ignore it */ 1675 if (!(other->state & EXTENT_DELALLOC)) 1676 return; 1677 1678 if (new->start > other->start) 1679 new_size = new->end - other->start + 1; 1680 else 1681 new_size = other->end - new->start + 1; 1682 1683 /* we're not bigger than the max, unreserve the space and go */ 1684 if (new_size <= BTRFS_MAX_EXTENT_SIZE) { 1685 spin_lock(&BTRFS_I(inode)->lock); 1686 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1687 spin_unlock(&BTRFS_I(inode)->lock); 1688 return; 1689 } 1690 1691 /* 1692 * We have to add up either side to figure out how many extents were 1693 * accounted for before we merged into one big extent. If the number of 1694 * extents we accounted for is <= the amount we need for the new range 1695 * then we can return, otherwise drop. Think of it like this 1696 * 1697 * [ 4k][MAX_SIZE] 1698 * 1699 * So we've grown the extent by a MAX_SIZE extent, this would mean we 1700 * need 2 outstanding extents, on one side we have 1 and the other side 1701 * we have 1 so they are == and we can return. But in this case 1702 * 1703 * [MAX_SIZE+4k][MAX_SIZE+4k] 1704 * 1705 * Each range on their own accounts for 2 extents, but merged together 1706 * they are only 3 extents worth of accounting, so we need to drop in 1707 * this case. 1708 */ 1709 old_size = other->end - other->start + 1; 1710 num_extents = count_max_extents(old_size); 1711 old_size = new->end - new->start + 1; 1712 num_extents += count_max_extents(old_size); 1713 if (count_max_extents(new_size) >= num_extents) 1714 return; 1715 1716 spin_lock(&BTRFS_I(inode)->lock); 1717 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1718 spin_unlock(&BTRFS_I(inode)->lock); 1719 } 1720 1721 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 1722 struct inode *inode) 1723 { 1724 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1725 1726 spin_lock(&root->delalloc_lock); 1727 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1728 list_add_tail(&BTRFS_I(inode)->delalloc_inodes, 1729 &root->delalloc_inodes); 1730 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1731 &BTRFS_I(inode)->runtime_flags); 1732 root->nr_delalloc_inodes++; 1733 if (root->nr_delalloc_inodes == 1) { 1734 spin_lock(&fs_info->delalloc_root_lock); 1735 BUG_ON(!list_empty(&root->delalloc_root)); 1736 list_add_tail(&root->delalloc_root, 1737 &fs_info->delalloc_roots); 1738 spin_unlock(&fs_info->delalloc_root_lock); 1739 } 1740 } 1741 spin_unlock(&root->delalloc_lock); 1742 } 1743 1744 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 1745 struct btrfs_inode *inode) 1746 { 1747 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 1748 1749 spin_lock(&root->delalloc_lock); 1750 if (!list_empty(&inode->delalloc_inodes)) { 1751 list_del_init(&inode->delalloc_inodes); 1752 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1753 &inode->runtime_flags); 1754 root->nr_delalloc_inodes--; 1755 if (!root->nr_delalloc_inodes) { 1756 spin_lock(&fs_info->delalloc_root_lock); 1757 BUG_ON(list_empty(&root->delalloc_root)); 1758 list_del_init(&root->delalloc_root); 1759 spin_unlock(&fs_info->delalloc_root_lock); 1760 } 1761 } 1762 spin_unlock(&root->delalloc_lock); 1763 } 1764 1765 /* 1766 * extent_io.c set_bit_hook, used to track delayed allocation 1767 * bytes in this file, and to maintain the list of inodes that 1768 * have pending delalloc work to be done. 1769 */ 1770 static void btrfs_set_bit_hook(void *private_data, 1771 struct extent_state *state, unsigned *bits) 1772 { 1773 struct inode *inode = private_data; 1774 1775 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1776 1777 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC)) 1778 WARN_ON(1); 1779 /* 1780 * set_bit and clear bit hooks normally require _irqsave/restore 1781 * but in this case, we are only testing for the DELALLOC 1782 * bit, which is only set or cleared with irqs on 1783 */ 1784 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1785 struct btrfs_root *root = BTRFS_I(inode)->root; 1786 u64 len = state->end + 1 - state->start; 1787 u32 num_extents = count_max_extents(len); 1788 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode)); 1789 1790 spin_lock(&BTRFS_I(inode)->lock); 1791 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents); 1792 spin_unlock(&BTRFS_I(inode)->lock); 1793 1794 /* For sanity tests */ 1795 if (btrfs_is_testing(fs_info)) 1796 return; 1797 1798 percpu_counter_add_batch(&fs_info->delalloc_bytes, len, 1799 fs_info->delalloc_batch); 1800 spin_lock(&BTRFS_I(inode)->lock); 1801 BTRFS_I(inode)->delalloc_bytes += len; 1802 if (*bits & EXTENT_DEFRAG) 1803 BTRFS_I(inode)->defrag_bytes += len; 1804 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1805 &BTRFS_I(inode)->runtime_flags)) 1806 btrfs_add_delalloc_inodes(root, inode); 1807 spin_unlock(&BTRFS_I(inode)->lock); 1808 } 1809 1810 if (!(state->state & EXTENT_DELALLOC_NEW) && 1811 (*bits & EXTENT_DELALLOC_NEW)) { 1812 spin_lock(&BTRFS_I(inode)->lock); 1813 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 - 1814 state->start; 1815 spin_unlock(&BTRFS_I(inode)->lock); 1816 } 1817 } 1818 1819 /* 1820 * extent_io.c clear_bit_hook, see set_bit_hook for why 1821 */ 1822 static void btrfs_clear_bit_hook(void *private_data, 1823 struct extent_state *state, 1824 unsigned *bits) 1825 { 1826 struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data); 1827 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 1828 u64 len = state->end + 1 - state->start; 1829 u32 num_extents = count_max_extents(len); 1830 1831 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) { 1832 spin_lock(&inode->lock); 1833 inode->defrag_bytes -= len; 1834 spin_unlock(&inode->lock); 1835 } 1836 1837 /* 1838 * set_bit and clear bit hooks normally require _irqsave/restore 1839 * but in this case, we are only testing for the DELALLOC 1840 * bit, which is only set or cleared with irqs on 1841 */ 1842 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1843 struct btrfs_root *root = inode->root; 1844 bool do_list = !btrfs_is_free_space_inode(inode); 1845 1846 spin_lock(&inode->lock); 1847 btrfs_mod_outstanding_extents(inode, -num_extents); 1848 spin_unlock(&inode->lock); 1849 1850 /* 1851 * We don't reserve metadata space for space cache inodes so we 1852 * don't need to call dellalloc_release_metadata if there is an 1853 * error. 1854 */ 1855 if (*bits & EXTENT_CLEAR_META_RESV && 1856 root != fs_info->tree_root) 1857 btrfs_delalloc_release_metadata(inode, len, false); 1858 1859 /* For sanity tests. */ 1860 if (btrfs_is_testing(fs_info)) 1861 return; 1862 1863 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID && 1864 do_list && !(state->state & EXTENT_NORESERVE) && 1865 (*bits & EXTENT_CLEAR_DATA_RESV)) 1866 btrfs_free_reserved_data_space_noquota( 1867 &inode->vfs_inode, 1868 state->start, len); 1869 1870 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, 1871 fs_info->delalloc_batch); 1872 spin_lock(&inode->lock); 1873 inode->delalloc_bytes -= len; 1874 if (do_list && inode->delalloc_bytes == 0 && 1875 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1876 &inode->runtime_flags)) 1877 btrfs_del_delalloc_inode(root, inode); 1878 spin_unlock(&inode->lock); 1879 } 1880 1881 if ((state->state & EXTENT_DELALLOC_NEW) && 1882 (*bits & EXTENT_DELALLOC_NEW)) { 1883 spin_lock(&inode->lock); 1884 ASSERT(inode->new_delalloc_bytes >= len); 1885 inode->new_delalloc_bytes -= len; 1886 spin_unlock(&inode->lock); 1887 } 1888 } 1889 1890 /* 1891 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure 1892 * we don't create bios that span stripes or chunks 1893 * 1894 * return 1 if page cannot be merged to bio 1895 * return 0 if page can be merged to bio 1896 * return error otherwise 1897 */ 1898 int btrfs_merge_bio_hook(struct page *page, unsigned long offset, 1899 size_t size, struct bio *bio, 1900 unsigned long bio_flags) 1901 { 1902 struct inode *inode = page->mapping->host; 1903 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1904 u64 logical = (u64)bio->bi_iter.bi_sector << 9; 1905 u64 length = 0; 1906 u64 map_length; 1907 int ret; 1908 1909 if (bio_flags & EXTENT_BIO_COMPRESSED) 1910 return 0; 1911 1912 length = bio->bi_iter.bi_size; 1913 map_length = length; 1914 ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, 1915 NULL, 0); 1916 if (ret < 0) 1917 return ret; 1918 if (map_length < length + size) 1919 return 1; 1920 return 0; 1921 } 1922 1923 /* 1924 * in order to insert checksums into the metadata in large chunks, 1925 * we wait until bio submission time. All the pages in the bio are 1926 * checksummed and sums are attached onto the ordered extent record. 1927 * 1928 * At IO completion time the cums attached on the ordered extent record 1929 * are inserted into the btree 1930 */ 1931 static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio, 1932 u64 bio_offset) 1933 { 1934 struct inode *inode = private_data; 1935 blk_status_t ret = 0; 1936 1937 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 1938 BUG_ON(ret); /* -ENOMEM */ 1939 return 0; 1940 } 1941 1942 /* 1943 * in order to insert checksums into the metadata in large chunks, 1944 * we wait until bio submission time. All the pages in the bio are 1945 * checksummed and sums are attached onto the ordered extent record. 1946 * 1947 * At IO completion time the cums attached on the ordered extent record 1948 * are inserted into the btree 1949 */ 1950 static blk_status_t btrfs_submit_bio_done(void *private_data, struct bio *bio, 1951 int mirror_num) 1952 { 1953 struct inode *inode = private_data; 1954 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1955 blk_status_t ret; 1956 1957 ret = btrfs_map_bio(fs_info, bio, mirror_num, 1); 1958 if (ret) { 1959 bio->bi_status = ret; 1960 bio_endio(bio); 1961 } 1962 return ret; 1963 } 1964 1965 /* 1966 * extent_io.c submission hook. This does the right thing for csum calculation 1967 * on write, or reading the csums from the tree before a read. 1968 * 1969 * Rules about async/sync submit, 1970 * a) read: sync submit 1971 * 1972 * b) write without checksum: sync submit 1973 * 1974 * c) write with checksum: 1975 * c-1) if bio is issued by fsync: sync submit 1976 * (sync_writers != 0) 1977 * 1978 * c-2) if root is reloc root: sync submit 1979 * (only in case of buffered IO) 1980 * 1981 * c-3) otherwise: async submit 1982 */ 1983 static blk_status_t btrfs_submit_bio_hook(void *private_data, struct bio *bio, 1984 int mirror_num, unsigned long bio_flags, 1985 u64 bio_offset) 1986 { 1987 struct inode *inode = private_data; 1988 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1989 struct btrfs_root *root = BTRFS_I(inode)->root; 1990 enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA; 1991 blk_status_t ret = 0; 1992 int skip_sum; 1993 int async = !atomic_read(&BTRFS_I(inode)->sync_writers); 1994 1995 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 1996 1997 if (btrfs_is_free_space_inode(BTRFS_I(inode))) 1998 metadata = BTRFS_WQ_ENDIO_FREE_SPACE; 1999 2000 if (bio_op(bio) != REQ_OP_WRITE) { 2001 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); 2002 if (ret) 2003 goto out; 2004 2005 if (bio_flags & EXTENT_BIO_COMPRESSED) { 2006 ret = btrfs_submit_compressed_read(inode, bio, 2007 mirror_num, 2008 bio_flags); 2009 goto out; 2010 } else if (!skip_sum) { 2011 ret = btrfs_lookup_bio_sums(inode, bio, NULL); 2012 if (ret) 2013 goto out; 2014 } 2015 goto mapit; 2016 } else if (async && !skip_sum) { 2017 /* csum items have already been cloned */ 2018 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 2019 goto mapit; 2020 /* we're doing a write, do the async checksumming */ 2021 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags, 2022 bio_offset, inode, 2023 btrfs_submit_bio_start, 2024 btrfs_submit_bio_done); 2025 goto out; 2026 } else if (!skip_sum) { 2027 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 2028 if (ret) 2029 goto out; 2030 } 2031 2032 mapit: 2033 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 2034 2035 out: 2036 if (ret) { 2037 bio->bi_status = ret; 2038 bio_endio(bio); 2039 } 2040 return ret; 2041 } 2042 2043 /* 2044 * given a list of ordered sums record them in the inode. This happens 2045 * at IO completion time based on sums calculated at bio submission time. 2046 */ 2047 static noinline int add_pending_csums(struct btrfs_trans_handle *trans, 2048 struct inode *inode, struct list_head *list) 2049 { 2050 struct btrfs_ordered_sum *sum; 2051 int ret; 2052 2053 list_for_each_entry(sum, list, list) { 2054 trans->adding_csums = true; 2055 ret = btrfs_csum_file_blocks(trans, 2056 BTRFS_I(inode)->root->fs_info->csum_root, sum); 2057 trans->adding_csums = false; 2058 if (ret) 2059 return ret; 2060 } 2061 return 0; 2062 } 2063 2064 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end, 2065 unsigned int extra_bits, 2066 struct extent_state **cached_state, int dedupe) 2067 { 2068 WARN_ON((end & (PAGE_SIZE - 1)) == 0); 2069 return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end, 2070 extra_bits, cached_state); 2071 } 2072 2073 /* see btrfs_writepage_start_hook for details on why this is required */ 2074 struct btrfs_writepage_fixup { 2075 struct page *page; 2076 struct btrfs_work work; 2077 }; 2078 2079 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2080 { 2081 struct btrfs_writepage_fixup *fixup; 2082 struct btrfs_ordered_extent *ordered; 2083 struct extent_state *cached_state = NULL; 2084 struct extent_changeset *data_reserved = NULL; 2085 struct page *page; 2086 struct inode *inode; 2087 u64 page_start; 2088 u64 page_end; 2089 int ret; 2090 2091 fixup = container_of(work, struct btrfs_writepage_fixup, work); 2092 page = fixup->page; 2093 again: 2094 lock_page(page); 2095 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 2096 ClearPageChecked(page); 2097 goto out_page; 2098 } 2099 2100 inode = page->mapping->host; 2101 page_start = page_offset(page); 2102 page_end = page_offset(page) + PAGE_SIZE - 1; 2103 2104 lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end, 2105 &cached_state); 2106 2107 /* already ordered? We're done */ 2108 if (PagePrivate2(page)) 2109 goto out; 2110 2111 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 2112 PAGE_SIZE); 2113 if (ordered) { 2114 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, 2115 page_end, &cached_state); 2116 unlock_page(page); 2117 btrfs_start_ordered_extent(inode, ordered, 1); 2118 btrfs_put_ordered_extent(ordered); 2119 goto again; 2120 } 2121 2122 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 2123 PAGE_SIZE); 2124 if (ret) { 2125 mapping_set_error(page->mapping, ret); 2126 end_extent_writepage(page, ret, page_start, page_end); 2127 ClearPageChecked(page); 2128 goto out; 2129 } 2130 2131 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, 2132 &cached_state, 0); 2133 if (ret) { 2134 mapping_set_error(page->mapping, ret); 2135 end_extent_writepage(page, ret, page_start, page_end); 2136 ClearPageChecked(page); 2137 goto out; 2138 } 2139 2140 ClearPageChecked(page); 2141 set_page_dirty(page); 2142 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, false); 2143 out: 2144 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end, 2145 &cached_state); 2146 out_page: 2147 unlock_page(page); 2148 put_page(page); 2149 kfree(fixup); 2150 extent_changeset_free(data_reserved); 2151 } 2152 2153 /* 2154 * There are a few paths in the higher layers of the kernel that directly 2155 * set the page dirty bit without asking the filesystem if it is a 2156 * good idea. This causes problems because we want to make sure COW 2157 * properly happens and the data=ordered rules are followed. 2158 * 2159 * In our case any range that doesn't have the ORDERED bit set 2160 * hasn't been properly setup for IO. We kick off an async process 2161 * to fix it up. The async helper will wait for ordered extents, set 2162 * the delalloc bit and make it safe to write the page. 2163 */ 2164 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end) 2165 { 2166 struct inode *inode = page->mapping->host; 2167 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2168 struct btrfs_writepage_fixup *fixup; 2169 2170 /* this page is properly in the ordered list */ 2171 if (TestClearPagePrivate2(page)) 2172 return 0; 2173 2174 if (PageChecked(page)) 2175 return -EAGAIN; 2176 2177 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2178 if (!fixup) 2179 return -EAGAIN; 2180 2181 SetPageChecked(page); 2182 get_page(page); 2183 btrfs_init_work(&fixup->work, btrfs_fixup_helper, 2184 btrfs_writepage_fixup_worker, NULL, NULL); 2185 fixup->page = page; 2186 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2187 return -EBUSY; 2188 } 2189 2190 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2191 struct inode *inode, u64 file_pos, 2192 u64 disk_bytenr, u64 disk_num_bytes, 2193 u64 num_bytes, u64 ram_bytes, 2194 u8 compression, u8 encryption, 2195 u16 other_encoding, int extent_type) 2196 { 2197 struct btrfs_root *root = BTRFS_I(inode)->root; 2198 struct btrfs_file_extent_item *fi; 2199 struct btrfs_path *path; 2200 struct extent_buffer *leaf; 2201 struct btrfs_key ins; 2202 u64 qg_released; 2203 int extent_inserted = 0; 2204 int ret; 2205 2206 path = btrfs_alloc_path(); 2207 if (!path) 2208 return -ENOMEM; 2209 2210 /* 2211 * we may be replacing one extent in the tree with another. 2212 * The new extent is pinned in the extent map, and we don't want 2213 * to drop it from the cache until it is completely in the btree. 2214 * 2215 * So, tell btrfs_drop_extents to leave this extent in the cache. 2216 * the caller is expected to unpin it and allow it to be merged 2217 * with the others. 2218 */ 2219 ret = __btrfs_drop_extents(trans, root, inode, path, file_pos, 2220 file_pos + num_bytes, NULL, 0, 2221 1, sizeof(*fi), &extent_inserted); 2222 if (ret) 2223 goto out; 2224 2225 if (!extent_inserted) { 2226 ins.objectid = btrfs_ino(BTRFS_I(inode)); 2227 ins.offset = file_pos; 2228 ins.type = BTRFS_EXTENT_DATA_KEY; 2229 2230 path->leave_spinning = 1; 2231 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2232 sizeof(*fi)); 2233 if (ret) 2234 goto out; 2235 } 2236 leaf = path->nodes[0]; 2237 fi = btrfs_item_ptr(leaf, path->slots[0], 2238 struct btrfs_file_extent_item); 2239 btrfs_set_file_extent_generation(leaf, fi, trans->transid); 2240 btrfs_set_file_extent_type(leaf, fi, extent_type); 2241 btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr); 2242 btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes); 2243 btrfs_set_file_extent_offset(leaf, fi, 0); 2244 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); 2245 btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes); 2246 btrfs_set_file_extent_compression(leaf, fi, compression); 2247 btrfs_set_file_extent_encryption(leaf, fi, encryption); 2248 btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding); 2249 2250 btrfs_mark_buffer_dirty(leaf); 2251 btrfs_release_path(path); 2252 2253 inode_add_bytes(inode, num_bytes); 2254 2255 ins.objectid = disk_bytenr; 2256 ins.offset = disk_num_bytes; 2257 ins.type = BTRFS_EXTENT_ITEM_KEY; 2258 2259 /* 2260 * Release the reserved range from inode dirty range map, as it is 2261 * already moved into delayed_ref_head 2262 */ 2263 ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes); 2264 if (ret < 0) 2265 goto out; 2266 qg_released = ret; 2267 ret = btrfs_alloc_reserved_file_extent(trans, root, 2268 btrfs_ino(BTRFS_I(inode)), 2269 file_pos, qg_released, &ins); 2270 out: 2271 btrfs_free_path(path); 2272 2273 return ret; 2274 } 2275 2276 /* snapshot-aware defrag */ 2277 struct sa_defrag_extent_backref { 2278 struct rb_node node; 2279 struct old_sa_defrag_extent *old; 2280 u64 root_id; 2281 u64 inum; 2282 u64 file_pos; 2283 u64 extent_offset; 2284 u64 num_bytes; 2285 u64 generation; 2286 }; 2287 2288 struct old_sa_defrag_extent { 2289 struct list_head list; 2290 struct new_sa_defrag_extent *new; 2291 2292 u64 extent_offset; 2293 u64 bytenr; 2294 u64 offset; 2295 u64 len; 2296 int count; 2297 }; 2298 2299 struct new_sa_defrag_extent { 2300 struct rb_root root; 2301 struct list_head head; 2302 struct btrfs_path *path; 2303 struct inode *inode; 2304 u64 file_pos; 2305 u64 len; 2306 u64 bytenr; 2307 u64 disk_len; 2308 u8 compress_type; 2309 }; 2310 2311 static int backref_comp(struct sa_defrag_extent_backref *b1, 2312 struct sa_defrag_extent_backref *b2) 2313 { 2314 if (b1->root_id < b2->root_id) 2315 return -1; 2316 else if (b1->root_id > b2->root_id) 2317 return 1; 2318 2319 if (b1->inum < b2->inum) 2320 return -1; 2321 else if (b1->inum > b2->inum) 2322 return 1; 2323 2324 if (b1->file_pos < b2->file_pos) 2325 return -1; 2326 else if (b1->file_pos > b2->file_pos) 2327 return 1; 2328 2329 /* 2330 * [------------------------------] ===> (a range of space) 2331 * |<--->| |<---->| =============> (fs/file tree A) 2332 * |<---------------------------->| ===> (fs/file tree B) 2333 * 2334 * A range of space can refer to two file extents in one tree while 2335 * refer to only one file extent in another tree. 2336 * 2337 * So we may process a disk offset more than one time(two extents in A) 2338 * and locate at the same extent(one extent in B), then insert two same 2339 * backrefs(both refer to the extent in B). 2340 */ 2341 return 0; 2342 } 2343 2344 static void backref_insert(struct rb_root *root, 2345 struct sa_defrag_extent_backref *backref) 2346 { 2347 struct rb_node **p = &root->rb_node; 2348 struct rb_node *parent = NULL; 2349 struct sa_defrag_extent_backref *entry; 2350 int ret; 2351 2352 while (*p) { 2353 parent = *p; 2354 entry = rb_entry(parent, struct sa_defrag_extent_backref, node); 2355 2356 ret = backref_comp(backref, entry); 2357 if (ret < 0) 2358 p = &(*p)->rb_left; 2359 else 2360 p = &(*p)->rb_right; 2361 } 2362 2363 rb_link_node(&backref->node, parent, p); 2364 rb_insert_color(&backref->node, root); 2365 } 2366 2367 /* 2368 * Note the backref might has changed, and in this case we just return 0. 2369 */ 2370 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id, 2371 void *ctx) 2372 { 2373 struct btrfs_file_extent_item *extent; 2374 struct old_sa_defrag_extent *old = ctx; 2375 struct new_sa_defrag_extent *new = old->new; 2376 struct btrfs_path *path = new->path; 2377 struct btrfs_key key; 2378 struct btrfs_root *root; 2379 struct sa_defrag_extent_backref *backref; 2380 struct extent_buffer *leaf; 2381 struct inode *inode = new->inode; 2382 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2383 int slot; 2384 int ret; 2385 u64 extent_offset; 2386 u64 num_bytes; 2387 2388 if (BTRFS_I(inode)->root->root_key.objectid == root_id && 2389 inum == btrfs_ino(BTRFS_I(inode))) 2390 return 0; 2391 2392 key.objectid = root_id; 2393 key.type = BTRFS_ROOT_ITEM_KEY; 2394 key.offset = (u64)-1; 2395 2396 root = btrfs_read_fs_root_no_name(fs_info, &key); 2397 if (IS_ERR(root)) { 2398 if (PTR_ERR(root) == -ENOENT) 2399 return 0; 2400 WARN_ON(1); 2401 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu", 2402 inum, offset, root_id); 2403 return PTR_ERR(root); 2404 } 2405 2406 key.objectid = inum; 2407 key.type = BTRFS_EXTENT_DATA_KEY; 2408 if (offset > (u64)-1 << 32) 2409 key.offset = 0; 2410 else 2411 key.offset = offset; 2412 2413 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2414 if (WARN_ON(ret < 0)) 2415 return ret; 2416 ret = 0; 2417 2418 while (1) { 2419 cond_resched(); 2420 2421 leaf = path->nodes[0]; 2422 slot = path->slots[0]; 2423 2424 if (slot >= btrfs_header_nritems(leaf)) { 2425 ret = btrfs_next_leaf(root, path); 2426 if (ret < 0) { 2427 goto out; 2428 } else if (ret > 0) { 2429 ret = 0; 2430 goto out; 2431 } 2432 continue; 2433 } 2434 2435 path->slots[0]++; 2436 2437 btrfs_item_key_to_cpu(leaf, &key, slot); 2438 2439 if (key.objectid > inum) 2440 goto out; 2441 2442 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY) 2443 continue; 2444 2445 extent = btrfs_item_ptr(leaf, slot, 2446 struct btrfs_file_extent_item); 2447 2448 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr) 2449 continue; 2450 2451 /* 2452 * 'offset' refers to the exact key.offset, 2453 * NOT the 'offset' field in btrfs_extent_data_ref, ie. 2454 * (key.offset - extent_offset). 2455 */ 2456 if (key.offset != offset) 2457 continue; 2458 2459 extent_offset = btrfs_file_extent_offset(leaf, extent); 2460 num_bytes = btrfs_file_extent_num_bytes(leaf, extent); 2461 2462 if (extent_offset >= old->extent_offset + old->offset + 2463 old->len || extent_offset + num_bytes <= 2464 old->extent_offset + old->offset) 2465 continue; 2466 break; 2467 } 2468 2469 backref = kmalloc(sizeof(*backref), GFP_NOFS); 2470 if (!backref) { 2471 ret = -ENOENT; 2472 goto out; 2473 } 2474 2475 backref->root_id = root_id; 2476 backref->inum = inum; 2477 backref->file_pos = offset; 2478 backref->num_bytes = num_bytes; 2479 backref->extent_offset = extent_offset; 2480 backref->generation = btrfs_file_extent_generation(leaf, extent); 2481 backref->old = old; 2482 backref_insert(&new->root, backref); 2483 old->count++; 2484 out: 2485 btrfs_release_path(path); 2486 WARN_ON(ret); 2487 return ret; 2488 } 2489 2490 static noinline bool record_extent_backrefs(struct btrfs_path *path, 2491 struct new_sa_defrag_extent *new) 2492 { 2493 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2494 struct old_sa_defrag_extent *old, *tmp; 2495 int ret; 2496 2497 new->path = path; 2498 2499 list_for_each_entry_safe(old, tmp, &new->head, list) { 2500 ret = iterate_inodes_from_logical(old->bytenr + 2501 old->extent_offset, fs_info, 2502 path, record_one_backref, 2503 old, false); 2504 if (ret < 0 && ret != -ENOENT) 2505 return false; 2506 2507 /* no backref to be processed for this extent */ 2508 if (!old->count) { 2509 list_del(&old->list); 2510 kfree(old); 2511 } 2512 } 2513 2514 if (list_empty(&new->head)) 2515 return false; 2516 2517 return true; 2518 } 2519 2520 static int relink_is_mergable(struct extent_buffer *leaf, 2521 struct btrfs_file_extent_item *fi, 2522 struct new_sa_defrag_extent *new) 2523 { 2524 if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr) 2525 return 0; 2526 2527 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG) 2528 return 0; 2529 2530 if (btrfs_file_extent_compression(leaf, fi) != new->compress_type) 2531 return 0; 2532 2533 if (btrfs_file_extent_encryption(leaf, fi) || 2534 btrfs_file_extent_other_encoding(leaf, fi)) 2535 return 0; 2536 2537 return 1; 2538 } 2539 2540 /* 2541 * Note the backref might has changed, and in this case we just return 0. 2542 */ 2543 static noinline int relink_extent_backref(struct btrfs_path *path, 2544 struct sa_defrag_extent_backref *prev, 2545 struct sa_defrag_extent_backref *backref) 2546 { 2547 struct btrfs_file_extent_item *extent; 2548 struct btrfs_file_extent_item *item; 2549 struct btrfs_ordered_extent *ordered; 2550 struct btrfs_trans_handle *trans; 2551 struct btrfs_root *root; 2552 struct btrfs_key key; 2553 struct extent_buffer *leaf; 2554 struct old_sa_defrag_extent *old = backref->old; 2555 struct new_sa_defrag_extent *new = old->new; 2556 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2557 struct inode *inode; 2558 struct extent_state *cached = NULL; 2559 int ret = 0; 2560 u64 start; 2561 u64 len; 2562 u64 lock_start; 2563 u64 lock_end; 2564 bool merge = false; 2565 int index; 2566 2567 if (prev && prev->root_id == backref->root_id && 2568 prev->inum == backref->inum && 2569 prev->file_pos + prev->num_bytes == backref->file_pos) 2570 merge = true; 2571 2572 /* step 1: get root */ 2573 key.objectid = backref->root_id; 2574 key.type = BTRFS_ROOT_ITEM_KEY; 2575 key.offset = (u64)-1; 2576 2577 index = srcu_read_lock(&fs_info->subvol_srcu); 2578 2579 root = btrfs_read_fs_root_no_name(fs_info, &key); 2580 if (IS_ERR(root)) { 2581 srcu_read_unlock(&fs_info->subvol_srcu, index); 2582 if (PTR_ERR(root) == -ENOENT) 2583 return 0; 2584 return PTR_ERR(root); 2585 } 2586 2587 if (btrfs_root_readonly(root)) { 2588 srcu_read_unlock(&fs_info->subvol_srcu, index); 2589 return 0; 2590 } 2591 2592 /* step 2: get inode */ 2593 key.objectid = backref->inum; 2594 key.type = BTRFS_INODE_ITEM_KEY; 2595 key.offset = 0; 2596 2597 inode = btrfs_iget(fs_info->sb, &key, root, NULL); 2598 if (IS_ERR(inode)) { 2599 srcu_read_unlock(&fs_info->subvol_srcu, index); 2600 return 0; 2601 } 2602 2603 srcu_read_unlock(&fs_info->subvol_srcu, index); 2604 2605 /* step 3: relink backref */ 2606 lock_start = backref->file_pos; 2607 lock_end = backref->file_pos + backref->num_bytes - 1; 2608 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2609 &cached); 2610 2611 ordered = btrfs_lookup_first_ordered_extent(inode, lock_end); 2612 if (ordered) { 2613 btrfs_put_ordered_extent(ordered); 2614 goto out_unlock; 2615 } 2616 2617 trans = btrfs_join_transaction(root); 2618 if (IS_ERR(trans)) { 2619 ret = PTR_ERR(trans); 2620 goto out_unlock; 2621 } 2622 2623 key.objectid = backref->inum; 2624 key.type = BTRFS_EXTENT_DATA_KEY; 2625 key.offset = backref->file_pos; 2626 2627 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2628 if (ret < 0) { 2629 goto out_free_path; 2630 } else if (ret > 0) { 2631 ret = 0; 2632 goto out_free_path; 2633 } 2634 2635 extent = btrfs_item_ptr(path->nodes[0], path->slots[0], 2636 struct btrfs_file_extent_item); 2637 2638 if (btrfs_file_extent_generation(path->nodes[0], extent) != 2639 backref->generation) 2640 goto out_free_path; 2641 2642 btrfs_release_path(path); 2643 2644 start = backref->file_pos; 2645 if (backref->extent_offset < old->extent_offset + old->offset) 2646 start += old->extent_offset + old->offset - 2647 backref->extent_offset; 2648 2649 len = min(backref->extent_offset + backref->num_bytes, 2650 old->extent_offset + old->offset + old->len); 2651 len -= max(backref->extent_offset, old->extent_offset + old->offset); 2652 2653 ret = btrfs_drop_extents(trans, root, inode, start, 2654 start + len, 1); 2655 if (ret) 2656 goto out_free_path; 2657 again: 2658 key.objectid = btrfs_ino(BTRFS_I(inode)); 2659 key.type = BTRFS_EXTENT_DATA_KEY; 2660 key.offset = start; 2661 2662 path->leave_spinning = 1; 2663 if (merge) { 2664 struct btrfs_file_extent_item *fi; 2665 u64 extent_len; 2666 struct btrfs_key found_key; 2667 2668 ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 2669 if (ret < 0) 2670 goto out_free_path; 2671 2672 path->slots[0]--; 2673 leaf = path->nodes[0]; 2674 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 2675 2676 fi = btrfs_item_ptr(leaf, path->slots[0], 2677 struct btrfs_file_extent_item); 2678 extent_len = btrfs_file_extent_num_bytes(leaf, fi); 2679 2680 if (extent_len + found_key.offset == start && 2681 relink_is_mergable(leaf, fi, new)) { 2682 btrfs_set_file_extent_num_bytes(leaf, fi, 2683 extent_len + len); 2684 btrfs_mark_buffer_dirty(leaf); 2685 inode_add_bytes(inode, len); 2686 2687 ret = 1; 2688 goto out_free_path; 2689 } else { 2690 merge = false; 2691 btrfs_release_path(path); 2692 goto again; 2693 } 2694 } 2695 2696 ret = btrfs_insert_empty_item(trans, root, path, &key, 2697 sizeof(*extent)); 2698 if (ret) { 2699 btrfs_abort_transaction(trans, ret); 2700 goto out_free_path; 2701 } 2702 2703 leaf = path->nodes[0]; 2704 item = btrfs_item_ptr(leaf, path->slots[0], 2705 struct btrfs_file_extent_item); 2706 btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr); 2707 btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len); 2708 btrfs_set_file_extent_offset(leaf, item, start - new->file_pos); 2709 btrfs_set_file_extent_num_bytes(leaf, item, len); 2710 btrfs_set_file_extent_ram_bytes(leaf, item, new->len); 2711 btrfs_set_file_extent_generation(leaf, item, trans->transid); 2712 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); 2713 btrfs_set_file_extent_compression(leaf, item, new->compress_type); 2714 btrfs_set_file_extent_encryption(leaf, item, 0); 2715 btrfs_set_file_extent_other_encoding(leaf, item, 0); 2716 2717 btrfs_mark_buffer_dirty(leaf); 2718 inode_add_bytes(inode, len); 2719 btrfs_release_path(path); 2720 2721 ret = btrfs_inc_extent_ref(trans, root, new->bytenr, 2722 new->disk_len, 0, 2723 backref->root_id, backref->inum, 2724 new->file_pos); /* start - extent_offset */ 2725 if (ret) { 2726 btrfs_abort_transaction(trans, ret); 2727 goto out_free_path; 2728 } 2729 2730 ret = 1; 2731 out_free_path: 2732 btrfs_release_path(path); 2733 path->leave_spinning = 0; 2734 btrfs_end_transaction(trans); 2735 out_unlock: 2736 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2737 &cached); 2738 iput(inode); 2739 return ret; 2740 } 2741 2742 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new) 2743 { 2744 struct old_sa_defrag_extent *old, *tmp; 2745 2746 if (!new) 2747 return; 2748 2749 list_for_each_entry_safe(old, tmp, &new->head, list) { 2750 kfree(old); 2751 } 2752 kfree(new); 2753 } 2754 2755 static void relink_file_extents(struct new_sa_defrag_extent *new) 2756 { 2757 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2758 struct btrfs_path *path; 2759 struct sa_defrag_extent_backref *backref; 2760 struct sa_defrag_extent_backref *prev = NULL; 2761 struct inode *inode; 2762 struct rb_node *node; 2763 int ret; 2764 2765 inode = new->inode; 2766 2767 path = btrfs_alloc_path(); 2768 if (!path) 2769 return; 2770 2771 if (!record_extent_backrefs(path, new)) { 2772 btrfs_free_path(path); 2773 goto out; 2774 } 2775 btrfs_release_path(path); 2776 2777 while (1) { 2778 node = rb_first(&new->root); 2779 if (!node) 2780 break; 2781 rb_erase(node, &new->root); 2782 2783 backref = rb_entry(node, struct sa_defrag_extent_backref, node); 2784 2785 ret = relink_extent_backref(path, prev, backref); 2786 WARN_ON(ret < 0); 2787 2788 kfree(prev); 2789 2790 if (ret == 1) 2791 prev = backref; 2792 else 2793 prev = NULL; 2794 cond_resched(); 2795 } 2796 kfree(prev); 2797 2798 btrfs_free_path(path); 2799 out: 2800 free_sa_defrag_extent(new); 2801 2802 atomic_dec(&fs_info->defrag_running); 2803 wake_up(&fs_info->transaction_wait); 2804 } 2805 2806 static struct new_sa_defrag_extent * 2807 record_old_file_extents(struct inode *inode, 2808 struct btrfs_ordered_extent *ordered) 2809 { 2810 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2811 struct btrfs_root *root = BTRFS_I(inode)->root; 2812 struct btrfs_path *path; 2813 struct btrfs_key key; 2814 struct old_sa_defrag_extent *old; 2815 struct new_sa_defrag_extent *new; 2816 int ret; 2817 2818 new = kmalloc(sizeof(*new), GFP_NOFS); 2819 if (!new) 2820 return NULL; 2821 2822 new->inode = inode; 2823 new->file_pos = ordered->file_offset; 2824 new->len = ordered->len; 2825 new->bytenr = ordered->start; 2826 new->disk_len = ordered->disk_len; 2827 new->compress_type = ordered->compress_type; 2828 new->root = RB_ROOT; 2829 INIT_LIST_HEAD(&new->head); 2830 2831 path = btrfs_alloc_path(); 2832 if (!path) 2833 goto out_kfree; 2834 2835 key.objectid = btrfs_ino(BTRFS_I(inode)); 2836 key.type = BTRFS_EXTENT_DATA_KEY; 2837 key.offset = new->file_pos; 2838 2839 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2840 if (ret < 0) 2841 goto out_free_path; 2842 if (ret > 0 && path->slots[0] > 0) 2843 path->slots[0]--; 2844 2845 /* find out all the old extents for the file range */ 2846 while (1) { 2847 struct btrfs_file_extent_item *extent; 2848 struct extent_buffer *l; 2849 int slot; 2850 u64 num_bytes; 2851 u64 offset; 2852 u64 end; 2853 u64 disk_bytenr; 2854 u64 extent_offset; 2855 2856 l = path->nodes[0]; 2857 slot = path->slots[0]; 2858 2859 if (slot >= btrfs_header_nritems(l)) { 2860 ret = btrfs_next_leaf(root, path); 2861 if (ret < 0) 2862 goto out_free_path; 2863 else if (ret > 0) 2864 break; 2865 continue; 2866 } 2867 2868 btrfs_item_key_to_cpu(l, &key, slot); 2869 2870 if (key.objectid != btrfs_ino(BTRFS_I(inode))) 2871 break; 2872 if (key.type != BTRFS_EXTENT_DATA_KEY) 2873 break; 2874 if (key.offset >= new->file_pos + new->len) 2875 break; 2876 2877 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item); 2878 2879 num_bytes = btrfs_file_extent_num_bytes(l, extent); 2880 if (key.offset + num_bytes < new->file_pos) 2881 goto next; 2882 2883 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent); 2884 if (!disk_bytenr) 2885 goto next; 2886 2887 extent_offset = btrfs_file_extent_offset(l, extent); 2888 2889 old = kmalloc(sizeof(*old), GFP_NOFS); 2890 if (!old) 2891 goto out_free_path; 2892 2893 offset = max(new->file_pos, key.offset); 2894 end = min(new->file_pos + new->len, key.offset + num_bytes); 2895 2896 old->bytenr = disk_bytenr; 2897 old->extent_offset = extent_offset; 2898 old->offset = offset - key.offset; 2899 old->len = end - offset; 2900 old->new = new; 2901 old->count = 0; 2902 list_add_tail(&old->list, &new->head); 2903 next: 2904 path->slots[0]++; 2905 cond_resched(); 2906 } 2907 2908 btrfs_free_path(path); 2909 atomic_inc(&fs_info->defrag_running); 2910 2911 return new; 2912 2913 out_free_path: 2914 btrfs_free_path(path); 2915 out_kfree: 2916 free_sa_defrag_extent(new); 2917 return NULL; 2918 } 2919 2920 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 2921 u64 start, u64 len) 2922 { 2923 struct btrfs_block_group_cache *cache; 2924 2925 cache = btrfs_lookup_block_group(fs_info, start); 2926 ASSERT(cache); 2927 2928 spin_lock(&cache->lock); 2929 cache->delalloc_bytes -= len; 2930 spin_unlock(&cache->lock); 2931 2932 btrfs_put_block_group(cache); 2933 } 2934 2935 /* as ordered data IO finishes, this gets called so we can finish 2936 * an ordered extent if the range of bytes in the file it covers are 2937 * fully written. 2938 */ 2939 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 2940 { 2941 struct inode *inode = ordered_extent->inode; 2942 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2943 struct btrfs_root *root = BTRFS_I(inode)->root; 2944 struct btrfs_trans_handle *trans = NULL; 2945 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2946 struct extent_state *cached_state = NULL; 2947 struct new_sa_defrag_extent *new = NULL; 2948 int compress_type = 0; 2949 int ret = 0; 2950 u64 logical_len = ordered_extent->len; 2951 bool nolock; 2952 bool truncated = false; 2953 bool range_locked = false; 2954 bool clear_new_delalloc_bytes = false; 2955 2956 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 2957 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && 2958 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags)) 2959 clear_new_delalloc_bytes = true; 2960 2961 nolock = btrfs_is_free_space_inode(BTRFS_I(inode)); 2962 2963 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 2964 ret = -EIO; 2965 goto out; 2966 } 2967 2968 btrfs_free_io_failure_record(BTRFS_I(inode), 2969 ordered_extent->file_offset, 2970 ordered_extent->file_offset + 2971 ordered_extent->len - 1); 2972 2973 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 2974 truncated = true; 2975 logical_len = ordered_extent->truncated_len; 2976 /* Truncated the entire extent, don't bother adding */ 2977 if (!logical_len) 2978 goto out; 2979 } 2980 2981 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 2982 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 2983 2984 /* 2985 * For mwrite(mmap + memset to write) case, we still reserve 2986 * space for NOCOW range. 2987 * As NOCOW won't cause a new delayed ref, just free the space 2988 */ 2989 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset, 2990 ordered_extent->len); 2991 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 2992 if (nolock) 2993 trans = btrfs_join_transaction_nolock(root); 2994 else 2995 trans = btrfs_join_transaction(root); 2996 if (IS_ERR(trans)) { 2997 ret = PTR_ERR(trans); 2998 trans = NULL; 2999 goto out; 3000 } 3001 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 3002 ret = btrfs_update_inode_fallback(trans, root, inode); 3003 if (ret) /* -ENOMEM or corruption */ 3004 btrfs_abort_transaction(trans, ret); 3005 goto out; 3006 } 3007 3008 range_locked = true; 3009 lock_extent_bits(io_tree, ordered_extent->file_offset, 3010 ordered_extent->file_offset + ordered_extent->len - 1, 3011 &cached_state); 3012 3013 ret = test_range_bit(io_tree, ordered_extent->file_offset, 3014 ordered_extent->file_offset + ordered_extent->len - 1, 3015 EXTENT_DEFRAG, 0, cached_state); 3016 if (ret) { 3017 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item); 3018 if (0 && last_snapshot >= BTRFS_I(inode)->generation) 3019 /* the inode is shared */ 3020 new = record_old_file_extents(inode, ordered_extent); 3021 3022 clear_extent_bit(io_tree, ordered_extent->file_offset, 3023 ordered_extent->file_offset + ordered_extent->len - 1, 3024 EXTENT_DEFRAG, 0, 0, &cached_state); 3025 } 3026 3027 if (nolock) 3028 trans = btrfs_join_transaction_nolock(root); 3029 else 3030 trans = btrfs_join_transaction(root); 3031 if (IS_ERR(trans)) { 3032 ret = PTR_ERR(trans); 3033 trans = NULL; 3034 goto out; 3035 } 3036 3037 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 3038 3039 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 3040 compress_type = ordered_extent->compress_type; 3041 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3042 BUG_ON(compress_type); 3043 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset, 3044 ordered_extent->len); 3045 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode), 3046 ordered_extent->file_offset, 3047 ordered_extent->file_offset + 3048 logical_len); 3049 } else { 3050 BUG_ON(root == fs_info->tree_root); 3051 ret = insert_reserved_file_extent(trans, inode, 3052 ordered_extent->file_offset, 3053 ordered_extent->start, 3054 ordered_extent->disk_len, 3055 logical_len, logical_len, 3056 compress_type, 0, 0, 3057 BTRFS_FILE_EXTENT_REG); 3058 if (!ret) 3059 btrfs_release_delalloc_bytes(fs_info, 3060 ordered_extent->start, 3061 ordered_extent->disk_len); 3062 } 3063 unpin_extent_cache(&BTRFS_I(inode)->extent_tree, 3064 ordered_extent->file_offset, ordered_extent->len, 3065 trans->transid); 3066 if (ret < 0) { 3067 btrfs_abort_transaction(trans, ret); 3068 goto out; 3069 } 3070 3071 ret = add_pending_csums(trans, inode, &ordered_extent->list); 3072 if (ret) { 3073 btrfs_abort_transaction(trans, ret); 3074 goto out; 3075 } 3076 3077 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 3078 ret = btrfs_update_inode_fallback(trans, root, inode); 3079 if (ret) { /* -ENOMEM or corruption */ 3080 btrfs_abort_transaction(trans, ret); 3081 goto out; 3082 } 3083 ret = 0; 3084 out: 3085 if (range_locked || clear_new_delalloc_bytes) { 3086 unsigned int clear_bits = 0; 3087 3088 if (range_locked) 3089 clear_bits |= EXTENT_LOCKED; 3090 if (clear_new_delalloc_bytes) 3091 clear_bits |= EXTENT_DELALLOC_NEW; 3092 clear_extent_bit(&BTRFS_I(inode)->io_tree, 3093 ordered_extent->file_offset, 3094 ordered_extent->file_offset + 3095 ordered_extent->len - 1, 3096 clear_bits, 3097 (clear_bits & EXTENT_LOCKED) ? 1 : 0, 3098 0, &cached_state); 3099 } 3100 3101 if (trans) 3102 btrfs_end_transaction(trans); 3103 3104 if (ret || truncated) { 3105 u64 start, end; 3106 3107 if (truncated) 3108 start = ordered_extent->file_offset + logical_len; 3109 else 3110 start = ordered_extent->file_offset; 3111 end = ordered_extent->file_offset + ordered_extent->len - 1; 3112 clear_extent_uptodate(io_tree, start, end, NULL); 3113 3114 /* Drop the cache for the part of the extent we didn't write. */ 3115 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0); 3116 3117 /* 3118 * If the ordered extent had an IOERR or something else went 3119 * wrong we need to return the space for this ordered extent 3120 * back to the allocator. We only free the extent in the 3121 * truncated case if we didn't write out the extent at all. 3122 */ 3123 if ((ret || !logical_len) && 3124 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3125 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) 3126 btrfs_free_reserved_extent(fs_info, 3127 ordered_extent->start, 3128 ordered_extent->disk_len, 1); 3129 } 3130 3131 3132 /* 3133 * This needs to be done to make sure anybody waiting knows we are done 3134 * updating everything for this ordered extent. 3135 */ 3136 btrfs_remove_ordered_extent(inode, ordered_extent); 3137 3138 /* for snapshot-aware defrag */ 3139 if (new) { 3140 if (ret) { 3141 free_sa_defrag_extent(new); 3142 atomic_dec(&fs_info->defrag_running); 3143 } else { 3144 relink_file_extents(new); 3145 } 3146 } 3147 3148 /* once for us */ 3149 btrfs_put_ordered_extent(ordered_extent); 3150 /* once for the tree */ 3151 btrfs_put_ordered_extent(ordered_extent); 3152 3153 return ret; 3154 } 3155 3156 static void finish_ordered_fn(struct btrfs_work *work) 3157 { 3158 struct btrfs_ordered_extent *ordered_extent; 3159 ordered_extent = container_of(work, struct btrfs_ordered_extent, work); 3160 btrfs_finish_ordered_io(ordered_extent); 3161 } 3162 3163 static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end, 3164 struct extent_state *state, int uptodate) 3165 { 3166 struct inode *inode = page->mapping->host; 3167 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3168 struct btrfs_ordered_extent *ordered_extent = NULL; 3169 struct btrfs_workqueue *wq; 3170 btrfs_work_func_t func; 3171 3172 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); 3173 3174 ClearPagePrivate2(page); 3175 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, 3176 end - start + 1, uptodate)) 3177 return; 3178 3179 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 3180 wq = fs_info->endio_freespace_worker; 3181 func = btrfs_freespace_write_helper; 3182 } else { 3183 wq = fs_info->endio_write_workers; 3184 func = btrfs_endio_write_helper; 3185 } 3186 3187 btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL, 3188 NULL); 3189 btrfs_queue_work(wq, &ordered_extent->work); 3190 } 3191 3192 static int __readpage_endio_check(struct inode *inode, 3193 struct btrfs_io_bio *io_bio, 3194 int icsum, struct page *page, 3195 int pgoff, u64 start, size_t len) 3196 { 3197 char *kaddr; 3198 u32 csum_expected; 3199 u32 csum = ~(u32)0; 3200 3201 csum_expected = *(((u32 *)io_bio->csum) + icsum); 3202 3203 kaddr = kmap_atomic(page); 3204 csum = btrfs_csum_data(kaddr + pgoff, csum, len); 3205 btrfs_csum_final(csum, (u8 *)&csum); 3206 if (csum != csum_expected) 3207 goto zeroit; 3208 3209 kunmap_atomic(kaddr); 3210 return 0; 3211 zeroit: 3212 btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected, 3213 io_bio->mirror_num); 3214 memset(kaddr + pgoff, 1, len); 3215 flush_dcache_page(page); 3216 kunmap_atomic(kaddr); 3217 return -EIO; 3218 } 3219 3220 /* 3221 * when reads are done, we need to check csums to verify the data is correct 3222 * if there's a match, we allow the bio to finish. If not, the code in 3223 * extent_io.c will try to find good copies for us. 3224 */ 3225 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, 3226 u64 phy_offset, struct page *page, 3227 u64 start, u64 end, int mirror) 3228 { 3229 size_t offset = start - page_offset(page); 3230 struct inode *inode = page->mapping->host; 3231 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 3232 struct btrfs_root *root = BTRFS_I(inode)->root; 3233 3234 if (PageChecked(page)) { 3235 ClearPageChecked(page); 3236 return 0; 3237 } 3238 3239 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 3240 return 0; 3241 3242 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID && 3243 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) { 3244 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM); 3245 return 0; 3246 } 3247 3248 phy_offset >>= inode->i_sb->s_blocksize_bits; 3249 return __readpage_endio_check(inode, io_bio, phy_offset, page, offset, 3250 start, (size_t)(end - start + 1)); 3251 } 3252 3253 /* 3254 * btrfs_add_delayed_iput - perform a delayed iput on @inode 3255 * 3256 * @inode: The inode we want to perform iput on 3257 * 3258 * This function uses the generic vfs_inode::i_count to track whether we should 3259 * just decrement it (in case it's > 1) or if this is the last iput then link 3260 * the inode to the delayed iput machinery. Delayed iputs are processed at 3261 * transaction commit time/superblock commit/cleaner kthread. 3262 */ 3263 void btrfs_add_delayed_iput(struct inode *inode) 3264 { 3265 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3266 struct btrfs_inode *binode = BTRFS_I(inode); 3267 3268 if (atomic_add_unless(&inode->i_count, -1, 1)) 3269 return; 3270 3271 spin_lock(&fs_info->delayed_iput_lock); 3272 ASSERT(list_empty(&binode->delayed_iput)); 3273 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); 3274 spin_unlock(&fs_info->delayed_iput_lock); 3275 } 3276 3277 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 3278 { 3279 3280 spin_lock(&fs_info->delayed_iput_lock); 3281 while (!list_empty(&fs_info->delayed_iputs)) { 3282 struct btrfs_inode *inode; 3283 3284 inode = list_first_entry(&fs_info->delayed_iputs, 3285 struct btrfs_inode, delayed_iput); 3286 list_del_init(&inode->delayed_iput); 3287 spin_unlock(&fs_info->delayed_iput_lock); 3288 iput(&inode->vfs_inode); 3289 spin_lock(&fs_info->delayed_iput_lock); 3290 } 3291 spin_unlock(&fs_info->delayed_iput_lock); 3292 } 3293 3294 /* 3295 * This is called in transaction commit time. If there are no orphan 3296 * files in the subvolume, it removes orphan item and frees block_rsv 3297 * structure. 3298 */ 3299 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans, 3300 struct btrfs_root *root) 3301 { 3302 struct btrfs_fs_info *fs_info = root->fs_info; 3303 struct btrfs_block_rsv *block_rsv; 3304 int ret; 3305 3306 if (atomic_read(&root->orphan_inodes) || 3307 root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) 3308 return; 3309 3310 spin_lock(&root->orphan_lock); 3311 if (atomic_read(&root->orphan_inodes)) { 3312 spin_unlock(&root->orphan_lock); 3313 return; 3314 } 3315 3316 if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) { 3317 spin_unlock(&root->orphan_lock); 3318 return; 3319 } 3320 3321 block_rsv = root->orphan_block_rsv; 3322 root->orphan_block_rsv = NULL; 3323 spin_unlock(&root->orphan_lock); 3324 3325 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) && 3326 btrfs_root_refs(&root->root_item) > 0) { 3327 ret = btrfs_del_orphan_item(trans, fs_info->tree_root, 3328 root->root_key.objectid); 3329 if (ret) 3330 btrfs_abort_transaction(trans, ret); 3331 else 3332 clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, 3333 &root->state); 3334 } 3335 3336 if (block_rsv) { 3337 WARN_ON(block_rsv->size > 0); 3338 btrfs_free_block_rsv(fs_info, block_rsv); 3339 } 3340 } 3341 3342 /* 3343 * This creates an orphan entry for the given inode in case something goes 3344 * wrong in the middle of an unlink/truncate. 3345 * 3346 * NOTE: caller of this function should reserve 5 units of metadata for 3347 * this function. 3348 */ 3349 int btrfs_orphan_add(struct btrfs_trans_handle *trans, 3350 struct btrfs_inode *inode) 3351 { 3352 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 3353 struct btrfs_root *root = inode->root; 3354 struct btrfs_block_rsv *block_rsv = NULL; 3355 int reserve = 0; 3356 bool insert = false; 3357 int ret; 3358 3359 if (!root->orphan_block_rsv) { 3360 block_rsv = btrfs_alloc_block_rsv(fs_info, 3361 BTRFS_BLOCK_RSV_TEMP); 3362 if (!block_rsv) 3363 return -ENOMEM; 3364 } 3365 3366 if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3367 &inode->runtime_flags)) 3368 insert = true; 3369 3370 if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3371 &inode->runtime_flags)) 3372 reserve = 1; 3373 3374 spin_lock(&root->orphan_lock); 3375 /* If someone has created ->orphan_block_rsv, be happy to use it. */ 3376 if (!root->orphan_block_rsv) { 3377 root->orphan_block_rsv = block_rsv; 3378 } else if (block_rsv) { 3379 btrfs_free_block_rsv(fs_info, block_rsv); 3380 block_rsv = NULL; 3381 } 3382 3383 if (insert) 3384 atomic_inc(&root->orphan_inodes); 3385 spin_unlock(&root->orphan_lock); 3386 3387 /* grab metadata reservation from transaction handle */ 3388 if (reserve) { 3389 ret = btrfs_orphan_reserve_metadata(trans, inode); 3390 ASSERT(!ret); 3391 if (ret) { 3392 /* 3393 * dec doesn't need spin_lock as ->orphan_block_rsv 3394 * would be released only if ->orphan_inodes is 3395 * zero. 3396 */ 3397 atomic_dec(&root->orphan_inodes); 3398 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3399 &inode->runtime_flags); 3400 if (insert) 3401 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3402 &inode->runtime_flags); 3403 return ret; 3404 } 3405 } 3406 3407 /* insert an orphan item to track this unlinked/truncated file */ 3408 if (insert) { 3409 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode)); 3410 if (ret) { 3411 if (reserve) { 3412 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3413 &inode->runtime_flags); 3414 btrfs_orphan_release_metadata(inode); 3415 } 3416 /* 3417 * btrfs_orphan_commit_root may race with us and set 3418 * ->orphan_block_rsv to zero, in order to avoid that, 3419 * decrease ->orphan_inodes after everything is done. 3420 */ 3421 atomic_dec(&root->orphan_inodes); 3422 if (ret != -EEXIST) { 3423 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3424 &inode->runtime_flags); 3425 btrfs_abort_transaction(trans, ret); 3426 return ret; 3427 } 3428 } 3429 ret = 0; 3430 } 3431 3432 return 0; 3433 } 3434 3435 /* 3436 * We have done the truncate/delete so we can go ahead and remove the orphan 3437 * item for this particular inode. 3438 */ 3439 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3440 struct btrfs_inode *inode) 3441 { 3442 struct btrfs_root *root = inode->root; 3443 int delete_item = 0; 3444 int ret = 0; 3445 3446 if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3447 &inode->runtime_flags)) 3448 delete_item = 1; 3449 3450 if (delete_item && trans) 3451 ret = btrfs_del_orphan_item(trans, root, btrfs_ino(inode)); 3452 3453 if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3454 &inode->runtime_flags)) 3455 btrfs_orphan_release_metadata(inode); 3456 3457 /* 3458 * btrfs_orphan_commit_root may race with us and set ->orphan_block_rsv 3459 * to zero, in order to avoid that, decrease ->orphan_inodes after 3460 * everything is done. 3461 */ 3462 if (delete_item) 3463 atomic_dec(&root->orphan_inodes); 3464 3465 return ret; 3466 } 3467 3468 /* 3469 * this cleans up any orphans that may be left on the list from the last use 3470 * of this root. 3471 */ 3472 int btrfs_orphan_cleanup(struct btrfs_root *root) 3473 { 3474 struct btrfs_fs_info *fs_info = root->fs_info; 3475 struct btrfs_path *path; 3476 struct extent_buffer *leaf; 3477 struct btrfs_key key, found_key; 3478 struct btrfs_trans_handle *trans; 3479 struct inode *inode; 3480 u64 last_objectid = 0; 3481 int ret = 0, nr_unlink = 0, nr_truncate = 0; 3482 3483 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED)) 3484 return 0; 3485 3486 path = btrfs_alloc_path(); 3487 if (!path) { 3488 ret = -ENOMEM; 3489 goto out; 3490 } 3491 path->reada = READA_BACK; 3492 3493 key.objectid = BTRFS_ORPHAN_OBJECTID; 3494 key.type = BTRFS_ORPHAN_ITEM_KEY; 3495 key.offset = (u64)-1; 3496 3497 while (1) { 3498 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3499 if (ret < 0) 3500 goto out; 3501 3502 /* 3503 * if ret == 0 means we found what we were searching for, which 3504 * is weird, but possible, so only screw with path if we didn't 3505 * find the key and see if we have stuff that matches 3506 */ 3507 if (ret > 0) { 3508 ret = 0; 3509 if (path->slots[0] == 0) 3510 break; 3511 path->slots[0]--; 3512 } 3513 3514 /* pull out the item */ 3515 leaf = path->nodes[0]; 3516 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3517 3518 /* make sure the item matches what we want */ 3519 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3520 break; 3521 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3522 break; 3523 3524 /* release the path since we're done with it */ 3525 btrfs_release_path(path); 3526 3527 /* 3528 * this is where we are basically btrfs_lookup, without the 3529 * crossing root thing. we store the inode number in the 3530 * offset of the orphan item. 3531 */ 3532 3533 if (found_key.offset == last_objectid) { 3534 btrfs_err(fs_info, 3535 "Error removing orphan entry, stopping orphan cleanup"); 3536 ret = -EINVAL; 3537 goto out; 3538 } 3539 3540 last_objectid = found_key.offset; 3541 3542 found_key.objectid = found_key.offset; 3543 found_key.type = BTRFS_INODE_ITEM_KEY; 3544 found_key.offset = 0; 3545 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL); 3546 ret = PTR_ERR_OR_ZERO(inode); 3547 if (ret && ret != -ENOENT) 3548 goto out; 3549 3550 if (ret == -ENOENT && root == fs_info->tree_root) { 3551 struct btrfs_root *dead_root; 3552 struct btrfs_fs_info *fs_info = root->fs_info; 3553 int is_dead_root = 0; 3554 3555 /* 3556 * this is an orphan in the tree root. Currently these 3557 * could come from 2 sources: 3558 * a) a snapshot deletion in progress 3559 * b) a free space cache inode 3560 * We need to distinguish those two, as the snapshot 3561 * orphan must not get deleted. 3562 * find_dead_roots already ran before us, so if this 3563 * is a snapshot deletion, we should find the root 3564 * in the dead_roots list 3565 */ 3566 spin_lock(&fs_info->trans_lock); 3567 list_for_each_entry(dead_root, &fs_info->dead_roots, 3568 root_list) { 3569 if (dead_root->root_key.objectid == 3570 found_key.objectid) { 3571 is_dead_root = 1; 3572 break; 3573 } 3574 } 3575 spin_unlock(&fs_info->trans_lock); 3576 if (is_dead_root) { 3577 /* prevent this orphan from being found again */ 3578 key.offset = found_key.objectid - 1; 3579 continue; 3580 } 3581 } 3582 /* 3583 * Inode is already gone but the orphan item is still there, 3584 * kill the orphan item. 3585 */ 3586 if (ret == -ENOENT) { 3587 trans = btrfs_start_transaction(root, 1); 3588 if (IS_ERR(trans)) { 3589 ret = PTR_ERR(trans); 3590 goto out; 3591 } 3592 btrfs_debug(fs_info, "auto deleting %Lu", 3593 found_key.objectid); 3594 ret = btrfs_del_orphan_item(trans, root, 3595 found_key.objectid); 3596 btrfs_end_transaction(trans); 3597 if (ret) 3598 goto out; 3599 continue; 3600 } 3601 3602 /* 3603 * add this inode to the orphan list so btrfs_orphan_del does 3604 * the proper thing when we hit it 3605 */ 3606 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3607 &BTRFS_I(inode)->runtime_flags); 3608 atomic_inc(&root->orphan_inodes); 3609 3610 /* if we have links, this was a truncate, lets do that */ 3611 if (inode->i_nlink) { 3612 if (WARN_ON(!S_ISREG(inode->i_mode))) { 3613 iput(inode); 3614 continue; 3615 } 3616 nr_truncate++; 3617 3618 /* 1 for the orphan item deletion. */ 3619 trans = btrfs_start_transaction(root, 1); 3620 if (IS_ERR(trans)) { 3621 iput(inode); 3622 ret = PTR_ERR(trans); 3623 goto out; 3624 } 3625 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 3626 btrfs_end_transaction(trans); 3627 if (ret) { 3628 iput(inode); 3629 goto out; 3630 } 3631 3632 ret = btrfs_truncate(inode, false); 3633 if (ret) 3634 btrfs_orphan_del(NULL, BTRFS_I(inode)); 3635 } else { 3636 nr_unlink++; 3637 } 3638 3639 /* this will do delete_inode and everything for us */ 3640 iput(inode); 3641 if (ret) 3642 goto out; 3643 } 3644 /* release the path since we're done with it */ 3645 btrfs_release_path(path); 3646 3647 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE; 3648 3649 if (root->orphan_block_rsv) 3650 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv, 3651 (u64)-1); 3652 3653 if (root->orphan_block_rsv || 3654 test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3655 trans = btrfs_join_transaction(root); 3656 if (!IS_ERR(trans)) 3657 btrfs_end_transaction(trans); 3658 } 3659 3660 if (nr_unlink) 3661 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3662 if (nr_truncate) 3663 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate); 3664 3665 out: 3666 if (ret) 3667 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3668 btrfs_free_path(path); 3669 return ret; 3670 } 3671 3672 /* 3673 * very simple check to peek ahead in the leaf looking for xattrs. If we 3674 * don't find any xattrs, we know there can't be any acls. 3675 * 3676 * slot is the slot the inode is in, objectid is the objectid of the inode 3677 */ 3678 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3679 int slot, u64 objectid, 3680 int *first_xattr_slot) 3681 { 3682 u32 nritems = btrfs_header_nritems(leaf); 3683 struct btrfs_key found_key; 3684 static u64 xattr_access = 0; 3685 static u64 xattr_default = 0; 3686 int scanned = 0; 3687 3688 if (!xattr_access) { 3689 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3690 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3691 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3692 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3693 } 3694 3695 slot++; 3696 *first_xattr_slot = -1; 3697 while (slot < nritems) { 3698 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3699 3700 /* we found a different objectid, there must not be acls */ 3701 if (found_key.objectid != objectid) 3702 return 0; 3703 3704 /* we found an xattr, assume we've got an acl */ 3705 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3706 if (*first_xattr_slot == -1) 3707 *first_xattr_slot = slot; 3708 if (found_key.offset == xattr_access || 3709 found_key.offset == xattr_default) 3710 return 1; 3711 } 3712 3713 /* 3714 * we found a key greater than an xattr key, there can't 3715 * be any acls later on 3716 */ 3717 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3718 return 0; 3719 3720 slot++; 3721 scanned++; 3722 3723 /* 3724 * it goes inode, inode backrefs, xattrs, extents, 3725 * so if there are a ton of hard links to an inode there can 3726 * be a lot of backrefs. Don't waste time searching too hard, 3727 * this is just an optimization 3728 */ 3729 if (scanned >= 8) 3730 break; 3731 } 3732 /* we hit the end of the leaf before we found an xattr or 3733 * something larger than an xattr. We have to assume the inode 3734 * has acls 3735 */ 3736 if (*first_xattr_slot == -1) 3737 *first_xattr_slot = slot; 3738 return 1; 3739 } 3740 3741 /* 3742 * read an inode from the btree into the in-memory inode 3743 */ 3744 static int btrfs_read_locked_inode(struct inode *inode) 3745 { 3746 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3747 struct btrfs_path *path; 3748 struct extent_buffer *leaf; 3749 struct btrfs_inode_item *inode_item; 3750 struct btrfs_root *root = BTRFS_I(inode)->root; 3751 struct btrfs_key location; 3752 unsigned long ptr; 3753 int maybe_acls; 3754 u32 rdev; 3755 int ret; 3756 bool filled = false; 3757 int first_xattr_slot; 3758 3759 ret = btrfs_fill_inode(inode, &rdev); 3760 if (!ret) 3761 filled = true; 3762 3763 path = btrfs_alloc_path(); 3764 if (!path) { 3765 ret = -ENOMEM; 3766 goto make_bad; 3767 } 3768 3769 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3770 3771 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3772 if (ret) { 3773 if (ret > 0) 3774 ret = -ENOENT; 3775 goto make_bad; 3776 } 3777 3778 leaf = path->nodes[0]; 3779 3780 if (filled) 3781 goto cache_index; 3782 3783 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3784 struct btrfs_inode_item); 3785 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3786 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3787 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3788 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3789 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item)); 3790 3791 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime); 3792 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime); 3793 3794 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime); 3795 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime); 3796 3797 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime); 3798 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime); 3799 3800 BTRFS_I(inode)->i_otime.tv_sec = 3801 btrfs_timespec_sec(leaf, &inode_item->otime); 3802 BTRFS_I(inode)->i_otime.tv_nsec = 3803 btrfs_timespec_nsec(leaf, &inode_item->otime); 3804 3805 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3806 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3807 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3808 3809 inode_set_iversion_queried(inode, 3810 btrfs_inode_sequence(leaf, inode_item)); 3811 inode->i_generation = BTRFS_I(inode)->generation; 3812 inode->i_rdev = 0; 3813 rdev = btrfs_inode_rdev(leaf, inode_item); 3814 3815 BTRFS_I(inode)->index_cnt = (u64)-1; 3816 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item); 3817 3818 cache_index: 3819 /* 3820 * If we were modified in the current generation and evicted from memory 3821 * and then re-read we need to do a full sync since we don't have any 3822 * idea about which extents were modified before we were evicted from 3823 * cache. 3824 * 3825 * This is required for both inode re-read from disk and delayed inode 3826 * in delayed_nodes_tree. 3827 */ 3828 if (BTRFS_I(inode)->last_trans == fs_info->generation) 3829 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3830 &BTRFS_I(inode)->runtime_flags); 3831 3832 /* 3833 * We don't persist the id of the transaction where an unlink operation 3834 * against the inode was last made. So here we assume the inode might 3835 * have been evicted, and therefore the exact value of last_unlink_trans 3836 * lost, and set it to last_trans to avoid metadata inconsistencies 3837 * between the inode and its parent if the inode is fsync'ed and the log 3838 * replayed. For example, in the scenario: 3839 * 3840 * touch mydir/foo 3841 * ln mydir/foo mydir/bar 3842 * sync 3843 * unlink mydir/bar 3844 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 3845 * xfs_io -c fsync mydir/foo 3846 * <power failure> 3847 * mount fs, triggers fsync log replay 3848 * 3849 * We must make sure that when we fsync our inode foo we also log its 3850 * parent inode, otherwise after log replay the parent still has the 3851 * dentry with the "bar" name but our inode foo has a link count of 1 3852 * and doesn't have an inode ref with the name "bar" anymore. 3853 * 3854 * Setting last_unlink_trans to last_trans is a pessimistic approach, 3855 * but it guarantees correctness at the expense of occasional full 3856 * transaction commits on fsync if our inode is a directory, or if our 3857 * inode is not a directory, logging its parent unnecessarily. 3858 */ 3859 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans; 3860 3861 path->slots[0]++; 3862 if (inode->i_nlink != 1 || 3863 path->slots[0] >= btrfs_header_nritems(leaf)) 3864 goto cache_acl; 3865 3866 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3867 if (location.objectid != btrfs_ino(BTRFS_I(inode))) 3868 goto cache_acl; 3869 3870 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3871 if (location.type == BTRFS_INODE_REF_KEY) { 3872 struct btrfs_inode_ref *ref; 3873 3874 ref = (struct btrfs_inode_ref *)ptr; 3875 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3876 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3877 struct btrfs_inode_extref *extref; 3878 3879 extref = (struct btrfs_inode_extref *)ptr; 3880 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3881 extref); 3882 } 3883 cache_acl: 3884 /* 3885 * try to precache a NULL acl entry for files that don't have 3886 * any xattrs or acls 3887 */ 3888 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3889 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot); 3890 if (first_xattr_slot != -1) { 3891 path->slots[0] = first_xattr_slot; 3892 ret = btrfs_load_inode_props(inode, path); 3893 if (ret) 3894 btrfs_err(fs_info, 3895 "error loading props for ino %llu (root %llu): %d", 3896 btrfs_ino(BTRFS_I(inode)), 3897 root->root_key.objectid, ret); 3898 } 3899 btrfs_free_path(path); 3900 3901 if (!maybe_acls) 3902 cache_no_acl(inode); 3903 3904 switch (inode->i_mode & S_IFMT) { 3905 case S_IFREG: 3906 inode->i_mapping->a_ops = &btrfs_aops; 3907 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 3908 inode->i_fop = &btrfs_file_operations; 3909 inode->i_op = &btrfs_file_inode_operations; 3910 break; 3911 case S_IFDIR: 3912 inode->i_fop = &btrfs_dir_file_operations; 3913 inode->i_op = &btrfs_dir_inode_operations; 3914 break; 3915 case S_IFLNK: 3916 inode->i_op = &btrfs_symlink_inode_operations; 3917 inode_nohighmem(inode); 3918 inode->i_mapping->a_ops = &btrfs_symlink_aops; 3919 break; 3920 default: 3921 inode->i_op = &btrfs_special_inode_operations; 3922 init_special_inode(inode, inode->i_mode, rdev); 3923 break; 3924 } 3925 3926 btrfs_update_iflags(inode); 3927 return 0; 3928 3929 make_bad: 3930 btrfs_free_path(path); 3931 make_bad_inode(inode); 3932 return ret; 3933 } 3934 3935 /* 3936 * given a leaf and an inode, copy the inode fields into the leaf 3937 */ 3938 static void fill_inode_item(struct btrfs_trans_handle *trans, 3939 struct extent_buffer *leaf, 3940 struct btrfs_inode_item *item, 3941 struct inode *inode) 3942 { 3943 struct btrfs_map_token token; 3944 3945 btrfs_init_map_token(&token); 3946 3947 btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token); 3948 btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token); 3949 btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size, 3950 &token); 3951 btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token); 3952 btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token); 3953 3954 btrfs_set_token_timespec_sec(leaf, &item->atime, 3955 inode->i_atime.tv_sec, &token); 3956 btrfs_set_token_timespec_nsec(leaf, &item->atime, 3957 inode->i_atime.tv_nsec, &token); 3958 3959 btrfs_set_token_timespec_sec(leaf, &item->mtime, 3960 inode->i_mtime.tv_sec, &token); 3961 btrfs_set_token_timespec_nsec(leaf, &item->mtime, 3962 inode->i_mtime.tv_nsec, &token); 3963 3964 btrfs_set_token_timespec_sec(leaf, &item->ctime, 3965 inode->i_ctime.tv_sec, &token); 3966 btrfs_set_token_timespec_nsec(leaf, &item->ctime, 3967 inode->i_ctime.tv_nsec, &token); 3968 3969 btrfs_set_token_timespec_sec(leaf, &item->otime, 3970 BTRFS_I(inode)->i_otime.tv_sec, &token); 3971 btrfs_set_token_timespec_nsec(leaf, &item->otime, 3972 BTRFS_I(inode)->i_otime.tv_nsec, &token); 3973 3974 btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode), 3975 &token); 3976 btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation, 3977 &token); 3978 btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode), 3979 &token); 3980 btrfs_set_token_inode_transid(leaf, item, trans->transid, &token); 3981 btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token); 3982 btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token); 3983 btrfs_set_token_inode_block_group(leaf, item, 0, &token); 3984 } 3985 3986 /* 3987 * copy everything in the in-memory inode into the btree. 3988 */ 3989 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 3990 struct btrfs_root *root, struct inode *inode) 3991 { 3992 struct btrfs_inode_item *inode_item; 3993 struct btrfs_path *path; 3994 struct extent_buffer *leaf; 3995 int ret; 3996 3997 path = btrfs_alloc_path(); 3998 if (!path) 3999 return -ENOMEM; 4000 4001 path->leave_spinning = 1; 4002 ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location, 4003 1); 4004 if (ret) { 4005 if (ret > 0) 4006 ret = -ENOENT; 4007 goto failed; 4008 } 4009 4010 leaf = path->nodes[0]; 4011 inode_item = btrfs_item_ptr(leaf, path->slots[0], 4012 struct btrfs_inode_item); 4013 4014 fill_inode_item(trans, leaf, inode_item, inode); 4015 btrfs_mark_buffer_dirty(leaf); 4016 btrfs_set_inode_last_trans(trans, inode); 4017 ret = 0; 4018 failed: 4019 btrfs_free_path(path); 4020 return ret; 4021 } 4022 4023 /* 4024 * copy everything in the in-memory inode into the btree. 4025 */ 4026 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 4027 struct btrfs_root *root, struct inode *inode) 4028 { 4029 struct btrfs_fs_info *fs_info = root->fs_info; 4030 int ret; 4031 4032 /* 4033 * If the inode is a free space inode, we can deadlock during commit 4034 * if we put it into the delayed code. 4035 * 4036 * The data relocation inode should also be directly updated 4037 * without delay 4038 */ 4039 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) 4040 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 4041 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 4042 btrfs_update_root_times(trans, root); 4043 4044 ret = btrfs_delayed_update_inode(trans, root, inode); 4045 if (!ret) 4046 btrfs_set_inode_last_trans(trans, inode); 4047 return ret; 4048 } 4049 4050 return btrfs_update_inode_item(trans, root, inode); 4051 } 4052 4053 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 4054 struct btrfs_root *root, 4055 struct inode *inode) 4056 { 4057 int ret; 4058 4059 ret = btrfs_update_inode(trans, root, inode); 4060 if (ret == -ENOSPC) 4061 return btrfs_update_inode_item(trans, root, inode); 4062 return ret; 4063 } 4064 4065 /* 4066 * unlink helper that gets used here in inode.c and in the tree logging 4067 * recovery code. It remove a link in a directory with a given name, and 4068 * also drops the back refs in the inode to the directory 4069 */ 4070 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4071 struct btrfs_root *root, 4072 struct btrfs_inode *dir, 4073 struct btrfs_inode *inode, 4074 const char *name, int name_len) 4075 { 4076 struct btrfs_fs_info *fs_info = root->fs_info; 4077 struct btrfs_path *path; 4078 int ret = 0; 4079 struct extent_buffer *leaf; 4080 struct btrfs_dir_item *di; 4081 struct btrfs_key key; 4082 u64 index; 4083 u64 ino = btrfs_ino(inode); 4084 u64 dir_ino = btrfs_ino(dir); 4085 4086 path = btrfs_alloc_path(); 4087 if (!path) { 4088 ret = -ENOMEM; 4089 goto out; 4090 } 4091 4092 path->leave_spinning = 1; 4093 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4094 name, name_len, -1); 4095 if (IS_ERR(di)) { 4096 ret = PTR_ERR(di); 4097 goto err; 4098 } 4099 if (!di) { 4100 ret = -ENOENT; 4101 goto err; 4102 } 4103 leaf = path->nodes[0]; 4104 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4105 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4106 if (ret) 4107 goto err; 4108 btrfs_release_path(path); 4109 4110 /* 4111 * If we don't have dir index, we have to get it by looking up 4112 * the inode ref, since we get the inode ref, remove it directly, 4113 * it is unnecessary to do delayed deletion. 4114 * 4115 * But if we have dir index, needn't search inode ref to get it. 4116 * Since the inode ref is close to the inode item, it is better 4117 * that we delay to delete it, and just do this deletion when 4118 * we update the inode item. 4119 */ 4120 if (inode->dir_index) { 4121 ret = btrfs_delayed_delete_inode_ref(inode); 4122 if (!ret) { 4123 index = inode->dir_index; 4124 goto skip_backref; 4125 } 4126 } 4127 4128 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino, 4129 dir_ino, &index); 4130 if (ret) { 4131 btrfs_info(fs_info, 4132 "failed to delete reference to %.*s, inode %llu parent %llu", 4133 name_len, name, ino, dir_ino); 4134 btrfs_abort_transaction(trans, ret); 4135 goto err; 4136 } 4137 skip_backref: 4138 ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index); 4139 if (ret) { 4140 btrfs_abort_transaction(trans, ret); 4141 goto err; 4142 } 4143 4144 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode, 4145 dir_ino); 4146 if (ret != 0 && ret != -ENOENT) { 4147 btrfs_abort_transaction(trans, ret); 4148 goto err; 4149 } 4150 4151 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir, 4152 index); 4153 if (ret == -ENOENT) 4154 ret = 0; 4155 else if (ret) 4156 btrfs_abort_transaction(trans, ret); 4157 err: 4158 btrfs_free_path(path); 4159 if (ret) 4160 goto out; 4161 4162 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2); 4163 inode_inc_iversion(&inode->vfs_inode); 4164 inode_inc_iversion(&dir->vfs_inode); 4165 inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime = 4166 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode); 4167 ret = btrfs_update_inode(trans, root, &dir->vfs_inode); 4168 out: 4169 return ret; 4170 } 4171 4172 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4173 struct btrfs_root *root, 4174 struct btrfs_inode *dir, struct btrfs_inode *inode, 4175 const char *name, int name_len) 4176 { 4177 int ret; 4178 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len); 4179 if (!ret) { 4180 drop_nlink(&inode->vfs_inode); 4181 ret = btrfs_update_inode(trans, root, &inode->vfs_inode); 4182 } 4183 return ret; 4184 } 4185 4186 /* 4187 * helper to start transaction for unlink and rmdir. 4188 * 4189 * unlink and rmdir are special in btrfs, they do not always free space, so 4190 * if we cannot make our reservations the normal way try and see if there is 4191 * plenty of slack room in the global reserve to migrate, otherwise we cannot 4192 * allow the unlink to occur. 4193 */ 4194 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) 4195 { 4196 struct btrfs_root *root = BTRFS_I(dir)->root; 4197 4198 /* 4199 * 1 for the possible orphan item 4200 * 1 for the dir item 4201 * 1 for the dir index 4202 * 1 for the inode ref 4203 * 1 for the inode 4204 */ 4205 return btrfs_start_transaction_fallback_global_rsv(root, 5, 5); 4206 } 4207 4208 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 4209 { 4210 struct btrfs_root *root = BTRFS_I(dir)->root; 4211 struct btrfs_trans_handle *trans; 4212 struct inode *inode = d_inode(dentry); 4213 int ret; 4214 4215 trans = __unlink_start_trans(dir); 4216 if (IS_ERR(trans)) 4217 return PTR_ERR(trans); 4218 4219 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4220 0); 4221 4222 ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4223 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4224 dentry->d_name.len); 4225 if (ret) 4226 goto out; 4227 4228 if (inode->i_nlink == 0) { 4229 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 4230 if (ret) 4231 goto out; 4232 } 4233 4234 out: 4235 btrfs_end_transaction(trans); 4236 btrfs_btree_balance_dirty(root->fs_info); 4237 return ret; 4238 } 4239 4240 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 4241 struct btrfs_root *root, 4242 struct inode *dir, u64 objectid, 4243 const char *name, int name_len) 4244 { 4245 struct btrfs_fs_info *fs_info = root->fs_info; 4246 struct btrfs_path *path; 4247 struct extent_buffer *leaf; 4248 struct btrfs_dir_item *di; 4249 struct btrfs_key key; 4250 u64 index; 4251 int ret; 4252 u64 dir_ino = btrfs_ino(BTRFS_I(dir)); 4253 4254 path = btrfs_alloc_path(); 4255 if (!path) 4256 return -ENOMEM; 4257 4258 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4259 name, name_len, -1); 4260 if (IS_ERR_OR_NULL(di)) { 4261 if (!di) 4262 ret = -ENOENT; 4263 else 4264 ret = PTR_ERR(di); 4265 goto out; 4266 } 4267 4268 leaf = path->nodes[0]; 4269 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4270 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 4271 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4272 if (ret) { 4273 btrfs_abort_transaction(trans, ret); 4274 goto out; 4275 } 4276 btrfs_release_path(path); 4277 4278 ret = btrfs_del_root_ref(trans, fs_info, objectid, 4279 root->root_key.objectid, dir_ino, 4280 &index, name, name_len); 4281 if (ret < 0) { 4282 if (ret != -ENOENT) { 4283 btrfs_abort_transaction(trans, ret); 4284 goto out; 4285 } 4286 di = btrfs_search_dir_index_item(root, path, dir_ino, 4287 name, name_len); 4288 if (IS_ERR_OR_NULL(di)) { 4289 if (!di) 4290 ret = -ENOENT; 4291 else 4292 ret = PTR_ERR(di); 4293 btrfs_abort_transaction(trans, ret); 4294 goto out; 4295 } 4296 4297 leaf = path->nodes[0]; 4298 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4299 btrfs_release_path(path); 4300 index = key.offset; 4301 } 4302 btrfs_release_path(path); 4303 4304 ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index); 4305 if (ret) { 4306 btrfs_abort_transaction(trans, ret); 4307 goto out; 4308 } 4309 4310 btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2); 4311 inode_inc_iversion(dir); 4312 dir->i_mtime = dir->i_ctime = current_time(dir); 4313 ret = btrfs_update_inode_fallback(trans, root, dir); 4314 if (ret) 4315 btrfs_abort_transaction(trans, ret); 4316 out: 4317 btrfs_free_path(path); 4318 return ret; 4319 } 4320 4321 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4322 { 4323 struct inode *inode = d_inode(dentry); 4324 int err = 0; 4325 struct btrfs_root *root = BTRFS_I(dir)->root; 4326 struct btrfs_trans_handle *trans; 4327 u64 last_unlink_trans; 4328 4329 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4330 return -ENOTEMPTY; 4331 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) 4332 return -EPERM; 4333 4334 trans = __unlink_start_trans(dir); 4335 if (IS_ERR(trans)) 4336 return PTR_ERR(trans); 4337 4338 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4339 err = btrfs_unlink_subvol(trans, root, dir, 4340 BTRFS_I(inode)->location.objectid, 4341 dentry->d_name.name, 4342 dentry->d_name.len); 4343 goto out; 4344 } 4345 4346 err = btrfs_orphan_add(trans, BTRFS_I(inode)); 4347 if (err) 4348 goto out; 4349 4350 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; 4351 4352 /* now the directory is empty */ 4353 err = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4354 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4355 dentry->d_name.len); 4356 if (!err) { 4357 btrfs_i_size_write(BTRFS_I(inode), 0); 4358 /* 4359 * Propagate the last_unlink_trans value of the deleted dir to 4360 * its parent directory. This is to prevent an unrecoverable 4361 * log tree in the case we do something like this: 4362 * 1) create dir foo 4363 * 2) create snapshot under dir foo 4364 * 3) delete the snapshot 4365 * 4) rmdir foo 4366 * 5) mkdir foo 4367 * 6) fsync foo or some file inside foo 4368 */ 4369 if (last_unlink_trans >= trans->transid) 4370 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; 4371 } 4372 out: 4373 btrfs_end_transaction(trans); 4374 btrfs_btree_balance_dirty(root->fs_info); 4375 4376 return err; 4377 } 4378 4379 static int truncate_space_check(struct btrfs_trans_handle *trans, 4380 struct btrfs_root *root, 4381 u64 bytes_deleted) 4382 { 4383 struct btrfs_fs_info *fs_info = root->fs_info; 4384 int ret; 4385 4386 /* 4387 * This is only used to apply pressure to the enospc system, we don't 4388 * intend to use this reservation at all. 4389 */ 4390 bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted); 4391 bytes_deleted *= fs_info->nodesize; 4392 ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv, 4393 bytes_deleted, BTRFS_RESERVE_NO_FLUSH); 4394 if (!ret) { 4395 trace_btrfs_space_reservation(fs_info, "transaction", 4396 trans->transid, 4397 bytes_deleted, 1); 4398 trans->bytes_reserved += bytes_deleted; 4399 } 4400 return ret; 4401 4402 } 4403 4404 /* 4405 * Return this if we need to call truncate_block for the last bit of the 4406 * truncate. 4407 */ 4408 #define NEED_TRUNCATE_BLOCK 1 4409 4410 /* 4411 * this can truncate away extent items, csum items and directory items. 4412 * It starts at a high offset and removes keys until it can't find 4413 * any higher than new_size 4414 * 4415 * csum items that cross the new i_size are truncated to the new size 4416 * as well. 4417 * 4418 * min_type is the minimum key type to truncate down to. If set to 0, this 4419 * will kill all the items on this inode, including the INODE_ITEM_KEY. 4420 */ 4421 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, 4422 struct btrfs_root *root, 4423 struct inode *inode, 4424 u64 new_size, u32 min_type) 4425 { 4426 struct btrfs_fs_info *fs_info = root->fs_info; 4427 struct btrfs_path *path; 4428 struct extent_buffer *leaf; 4429 struct btrfs_file_extent_item *fi; 4430 struct btrfs_key key; 4431 struct btrfs_key found_key; 4432 u64 extent_start = 0; 4433 u64 extent_num_bytes = 0; 4434 u64 extent_offset = 0; 4435 u64 item_end = 0; 4436 u64 last_size = new_size; 4437 u32 found_type = (u8)-1; 4438 int found_extent; 4439 int del_item; 4440 int pending_del_nr = 0; 4441 int pending_del_slot = 0; 4442 int extent_type = -1; 4443 int ret; 4444 int err = 0; 4445 u64 ino = btrfs_ino(BTRFS_I(inode)); 4446 u64 bytes_deleted = 0; 4447 bool be_nice = false; 4448 bool should_throttle = false; 4449 bool should_end = false; 4450 4451 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); 4452 4453 /* 4454 * for non-free space inodes and ref cows, we want to back off from 4455 * time to time 4456 */ 4457 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) && 4458 test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4459 be_nice = true; 4460 4461 path = btrfs_alloc_path(); 4462 if (!path) 4463 return -ENOMEM; 4464 path->reada = READA_BACK; 4465 4466 /* 4467 * We want to drop from the next block forward in case this new size is 4468 * not block aligned since we will be keeping the last block of the 4469 * extent just the way it is. 4470 */ 4471 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4472 root == fs_info->tree_root) 4473 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size, 4474 fs_info->sectorsize), 4475 (u64)-1, 0); 4476 4477 /* 4478 * This function is also used to drop the items in the log tree before 4479 * we relog the inode, so if root != BTRFS_I(inode)->root, it means 4480 * it is used to drop the loged items. So we shouldn't kill the delayed 4481 * items. 4482 */ 4483 if (min_type == 0 && root == BTRFS_I(inode)->root) 4484 btrfs_kill_delayed_inode_items(BTRFS_I(inode)); 4485 4486 key.objectid = ino; 4487 key.offset = (u64)-1; 4488 key.type = (u8)-1; 4489 4490 search_again: 4491 /* 4492 * with a 16K leaf size and 128MB extents, you can actually queue 4493 * up a huge file in a single leaf. Most of the time that 4494 * bytes_deleted is > 0, it will be huge by the time we get here 4495 */ 4496 if (be_nice && bytes_deleted > SZ_32M) { 4497 if (btrfs_should_end_transaction(trans)) { 4498 err = -EAGAIN; 4499 goto error; 4500 } 4501 } 4502 4503 4504 path->leave_spinning = 1; 4505 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 4506 if (ret < 0) { 4507 err = ret; 4508 goto out; 4509 } 4510 4511 if (ret > 0) { 4512 /* there are no items in the tree for us to truncate, we're 4513 * done 4514 */ 4515 if (path->slots[0] == 0) 4516 goto out; 4517 path->slots[0]--; 4518 } 4519 4520 while (1) { 4521 fi = NULL; 4522 leaf = path->nodes[0]; 4523 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4524 found_type = found_key.type; 4525 4526 if (found_key.objectid != ino) 4527 break; 4528 4529 if (found_type < min_type) 4530 break; 4531 4532 item_end = found_key.offset; 4533 if (found_type == BTRFS_EXTENT_DATA_KEY) { 4534 fi = btrfs_item_ptr(leaf, path->slots[0], 4535 struct btrfs_file_extent_item); 4536 extent_type = btrfs_file_extent_type(leaf, fi); 4537 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4538 item_end += 4539 btrfs_file_extent_num_bytes(leaf, fi); 4540 4541 trace_btrfs_truncate_show_fi_regular( 4542 BTRFS_I(inode), leaf, fi, 4543 found_key.offset); 4544 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4545 item_end += btrfs_file_extent_inline_len(leaf, 4546 path->slots[0], fi); 4547 4548 trace_btrfs_truncate_show_fi_inline( 4549 BTRFS_I(inode), leaf, fi, path->slots[0], 4550 found_key.offset); 4551 } 4552 item_end--; 4553 } 4554 if (found_type > min_type) { 4555 del_item = 1; 4556 } else { 4557 if (item_end < new_size) 4558 break; 4559 if (found_key.offset >= new_size) 4560 del_item = 1; 4561 else 4562 del_item = 0; 4563 } 4564 found_extent = 0; 4565 /* FIXME, shrink the extent if the ref count is only 1 */ 4566 if (found_type != BTRFS_EXTENT_DATA_KEY) 4567 goto delete; 4568 4569 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4570 u64 num_dec; 4571 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); 4572 if (!del_item) { 4573 u64 orig_num_bytes = 4574 btrfs_file_extent_num_bytes(leaf, fi); 4575 extent_num_bytes = ALIGN(new_size - 4576 found_key.offset, 4577 fs_info->sectorsize); 4578 btrfs_set_file_extent_num_bytes(leaf, fi, 4579 extent_num_bytes); 4580 num_dec = (orig_num_bytes - 4581 extent_num_bytes); 4582 if (test_bit(BTRFS_ROOT_REF_COWS, 4583 &root->state) && 4584 extent_start != 0) 4585 inode_sub_bytes(inode, num_dec); 4586 btrfs_mark_buffer_dirty(leaf); 4587 } else { 4588 extent_num_bytes = 4589 btrfs_file_extent_disk_num_bytes(leaf, 4590 fi); 4591 extent_offset = found_key.offset - 4592 btrfs_file_extent_offset(leaf, fi); 4593 4594 /* FIXME blocksize != 4096 */ 4595 num_dec = btrfs_file_extent_num_bytes(leaf, fi); 4596 if (extent_start != 0) { 4597 found_extent = 1; 4598 if (test_bit(BTRFS_ROOT_REF_COWS, 4599 &root->state)) 4600 inode_sub_bytes(inode, num_dec); 4601 } 4602 } 4603 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4604 /* 4605 * we can't truncate inline items that have had 4606 * special encodings 4607 */ 4608 if (!del_item && 4609 btrfs_file_extent_encryption(leaf, fi) == 0 && 4610 btrfs_file_extent_other_encoding(leaf, fi) == 0 && 4611 btrfs_file_extent_compression(leaf, fi) == 0) { 4612 u32 size = (u32)(new_size - found_key.offset); 4613 4614 btrfs_set_file_extent_ram_bytes(leaf, fi, size); 4615 size = btrfs_file_extent_calc_inline_size(size); 4616 btrfs_truncate_item(root->fs_info, path, size, 1); 4617 } else if (!del_item) { 4618 /* 4619 * We have to bail so the last_size is set to 4620 * just before this extent. 4621 */ 4622 err = NEED_TRUNCATE_BLOCK; 4623 break; 4624 } 4625 4626 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4627 inode_sub_bytes(inode, item_end + 1 - new_size); 4628 } 4629 delete: 4630 if (del_item) 4631 last_size = found_key.offset; 4632 else 4633 last_size = new_size; 4634 if (del_item) { 4635 if (!pending_del_nr) { 4636 /* no pending yet, add ourselves */ 4637 pending_del_slot = path->slots[0]; 4638 pending_del_nr = 1; 4639 } else if (pending_del_nr && 4640 path->slots[0] + 1 == pending_del_slot) { 4641 /* hop on the pending chunk */ 4642 pending_del_nr++; 4643 pending_del_slot = path->slots[0]; 4644 } else { 4645 BUG(); 4646 } 4647 } else { 4648 break; 4649 } 4650 should_throttle = false; 4651 4652 if (found_extent && 4653 (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4654 root == fs_info->tree_root)) { 4655 btrfs_set_path_blocking(path); 4656 bytes_deleted += extent_num_bytes; 4657 ret = btrfs_free_extent(trans, root, extent_start, 4658 extent_num_bytes, 0, 4659 btrfs_header_owner(leaf), 4660 ino, extent_offset); 4661 BUG_ON(ret); 4662 if (btrfs_should_throttle_delayed_refs(trans, fs_info)) 4663 btrfs_async_run_delayed_refs(fs_info, 4664 trans->delayed_ref_updates * 2, 4665 trans->transid, 0); 4666 if (be_nice) { 4667 if (truncate_space_check(trans, root, 4668 extent_num_bytes)) { 4669 should_end = true; 4670 } 4671 if (btrfs_should_throttle_delayed_refs(trans, 4672 fs_info)) 4673 should_throttle = true; 4674 } 4675 } 4676 4677 if (found_type == BTRFS_INODE_ITEM_KEY) 4678 break; 4679 4680 if (path->slots[0] == 0 || 4681 path->slots[0] != pending_del_slot || 4682 should_throttle || should_end) { 4683 if (pending_del_nr) { 4684 ret = btrfs_del_items(trans, root, path, 4685 pending_del_slot, 4686 pending_del_nr); 4687 if (ret) { 4688 btrfs_abort_transaction(trans, ret); 4689 goto error; 4690 } 4691 pending_del_nr = 0; 4692 } 4693 btrfs_release_path(path); 4694 if (should_throttle) { 4695 unsigned long updates = trans->delayed_ref_updates; 4696 if (updates) { 4697 trans->delayed_ref_updates = 0; 4698 ret = btrfs_run_delayed_refs(trans, 4699 updates * 2); 4700 if (ret && !err) 4701 err = ret; 4702 } 4703 } 4704 /* 4705 * if we failed to refill our space rsv, bail out 4706 * and let the transaction restart 4707 */ 4708 if (should_end) { 4709 err = -EAGAIN; 4710 goto error; 4711 } 4712 goto search_again; 4713 } else { 4714 path->slots[0]--; 4715 } 4716 } 4717 out: 4718 if (pending_del_nr) { 4719 ret = btrfs_del_items(trans, root, path, pending_del_slot, 4720 pending_del_nr); 4721 if (ret) 4722 btrfs_abort_transaction(trans, ret); 4723 } 4724 error: 4725 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4726 ASSERT(last_size >= new_size); 4727 if (!err && last_size > new_size) 4728 last_size = new_size; 4729 btrfs_ordered_update_i_size(inode, last_size, NULL); 4730 } 4731 4732 btrfs_free_path(path); 4733 4734 if (be_nice && bytes_deleted > SZ_32M) { 4735 unsigned long updates = trans->delayed_ref_updates; 4736 if (updates) { 4737 trans->delayed_ref_updates = 0; 4738 ret = btrfs_run_delayed_refs(trans, updates * 2); 4739 if (ret && !err) 4740 err = ret; 4741 } 4742 } 4743 return err; 4744 } 4745 4746 /* 4747 * btrfs_truncate_block - read, zero a chunk and write a block 4748 * @inode - inode that we're zeroing 4749 * @from - the offset to start zeroing 4750 * @len - the length to zero, 0 to zero the entire range respective to the 4751 * offset 4752 * @front - zero up to the offset instead of from the offset on 4753 * 4754 * This will find the block for the "from" offset and cow the block and zero the 4755 * part we want to zero. This is used with truncate and hole punching. 4756 */ 4757 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len, 4758 int front) 4759 { 4760 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4761 struct address_space *mapping = inode->i_mapping; 4762 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4763 struct btrfs_ordered_extent *ordered; 4764 struct extent_state *cached_state = NULL; 4765 struct extent_changeset *data_reserved = NULL; 4766 char *kaddr; 4767 u32 blocksize = fs_info->sectorsize; 4768 pgoff_t index = from >> PAGE_SHIFT; 4769 unsigned offset = from & (blocksize - 1); 4770 struct page *page; 4771 gfp_t mask = btrfs_alloc_write_mask(mapping); 4772 int ret = 0; 4773 u64 block_start; 4774 u64 block_end; 4775 4776 if (IS_ALIGNED(offset, blocksize) && 4777 (!len || IS_ALIGNED(len, blocksize))) 4778 goto out; 4779 4780 block_start = round_down(from, blocksize); 4781 block_end = block_start + blocksize - 1; 4782 4783 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, 4784 block_start, blocksize); 4785 if (ret) 4786 goto out; 4787 4788 again: 4789 page = find_or_create_page(mapping, index, mask); 4790 if (!page) { 4791 btrfs_delalloc_release_space(inode, data_reserved, 4792 block_start, blocksize, true); 4793 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, true); 4794 ret = -ENOMEM; 4795 goto out; 4796 } 4797 4798 if (!PageUptodate(page)) { 4799 ret = btrfs_readpage(NULL, page); 4800 lock_page(page); 4801 if (page->mapping != mapping) { 4802 unlock_page(page); 4803 put_page(page); 4804 goto again; 4805 } 4806 if (!PageUptodate(page)) { 4807 ret = -EIO; 4808 goto out_unlock; 4809 } 4810 } 4811 wait_on_page_writeback(page); 4812 4813 lock_extent_bits(io_tree, block_start, block_end, &cached_state); 4814 set_page_extent_mapped(page); 4815 4816 ordered = btrfs_lookup_ordered_extent(inode, block_start); 4817 if (ordered) { 4818 unlock_extent_cached(io_tree, block_start, block_end, 4819 &cached_state); 4820 unlock_page(page); 4821 put_page(page); 4822 btrfs_start_ordered_extent(inode, ordered, 1); 4823 btrfs_put_ordered_extent(ordered); 4824 goto again; 4825 } 4826 4827 clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end, 4828 EXTENT_DIRTY | EXTENT_DELALLOC | 4829 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4830 0, 0, &cached_state); 4831 4832 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, 4833 &cached_state, 0); 4834 if (ret) { 4835 unlock_extent_cached(io_tree, block_start, block_end, 4836 &cached_state); 4837 goto out_unlock; 4838 } 4839 4840 if (offset != blocksize) { 4841 if (!len) 4842 len = blocksize - offset; 4843 kaddr = kmap(page); 4844 if (front) 4845 memset(kaddr + (block_start - page_offset(page)), 4846 0, offset); 4847 else 4848 memset(kaddr + (block_start - page_offset(page)) + offset, 4849 0, len); 4850 flush_dcache_page(page); 4851 kunmap(page); 4852 } 4853 ClearPageChecked(page); 4854 set_page_dirty(page); 4855 unlock_extent_cached(io_tree, block_start, block_end, &cached_state); 4856 4857 out_unlock: 4858 if (ret) 4859 btrfs_delalloc_release_space(inode, data_reserved, block_start, 4860 blocksize, true); 4861 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, (ret != 0)); 4862 unlock_page(page); 4863 put_page(page); 4864 out: 4865 extent_changeset_free(data_reserved); 4866 return ret; 4867 } 4868 4869 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode, 4870 u64 offset, u64 len) 4871 { 4872 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4873 struct btrfs_trans_handle *trans; 4874 int ret; 4875 4876 /* 4877 * Still need to make sure the inode looks like it's been updated so 4878 * that any holes get logged if we fsync. 4879 */ 4880 if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 4881 BTRFS_I(inode)->last_trans = fs_info->generation; 4882 BTRFS_I(inode)->last_sub_trans = root->log_transid; 4883 BTRFS_I(inode)->last_log_commit = root->last_log_commit; 4884 return 0; 4885 } 4886 4887 /* 4888 * 1 - for the one we're dropping 4889 * 1 - for the one we're adding 4890 * 1 - for updating the inode. 4891 */ 4892 trans = btrfs_start_transaction(root, 3); 4893 if (IS_ERR(trans)) 4894 return PTR_ERR(trans); 4895 4896 ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1); 4897 if (ret) { 4898 btrfs_abort_transaction(trans, ret); 4899 btrfs_end_transaction(trans); 4900 return ret; 4901 } 4902 4903 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)), 4904 offset, 0, 0, len, 0, len, 0, 0, 0); 4905 if (ret) 4906 btrfs_abort_transaction(trans, ret); 4907 else 4908 btrfs_update_inode(trans, root, inode); 4909 btrfs_end_transaction(trans); 4910 return ret; 4911 } 4912 4913 /* 4914 * This function puts in dummy file extents for the area we're creating a hole 4915 * for. So if we are truncating this file to a larger size we need to insert 4916 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 4917 * the range between oldsize and size 4918 */ 4919 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size) 4920 { 4921 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4922 struct btrfs_root *root = BTRFS_I(inode)->root; 4923 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4924 struct extent_map *em = NULL; 4925 struct extent_state *cached_state = NULL; 4926 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 4927 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 4928 u64 block_end = ALIGN(size, fs_info->sectorsize); 4929 u64 last_byte; 4930 u64 cur_offset; 4931 u64 hole_size; 4932 int err = 0; 4933 4934 /* 4935 * If our size started in the middle of a block we need to zero out the 4936 * rest of the block before we expand the i_size, otherwise we could 4937 * expose stale data. 4938 */ 4939 err = btrfs_truncate_block(inode, oldsize, 0, 0); 4940 if (err) 4941 return err; 4942 4943 if (size <= hole_start) 4944 return 0; 4945 4946 while (1) { 4947 struct btrfs_ordered_extent *ordered; 4948 4949 lock_extent_bits(io_tree, hole_start, block_end - 1, 4950 &cached_state); 4951 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start, 4952 block_end - hole_start); 4953 if (!ordered) 4954 break; 4955 unlock_extent_cached(io_tree, hole_start, block_end - 1, 4956 &cached_state); 4957 btrfs_start_ordered_extent(inode, ordered, 1); 4958 btrfs_put_ordered_extent(ordered); 4959 } 4960 4961 cur_offset = hole_start; 4962 while (1) { 4963 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset, 4964 block_end - cur_offset, 0); 4965 if (IS_ERR(em)) { 4966 err = PTR_ERR(em); 4967 em = NULL; 4968 break; 4969 } 4970 last_byte = min(extent_map_end(em), block_end); 4971 last_byte = ALIGN(last_byte, fs_info->sectorsize); 4972 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 4973 struct extent_map *hole_em; 4974 hole_size = last_byte - cur_offset; 4975 4976 err = maybe_insert_hole(root, inode, cur_offset, 4977 hole_size); 4978 if (err) 4979 break; 4980 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 4981 cur_offset + hole_size - 1, 0); 4982 hole_em = alloc_extent_map(); 4983 if (!hole_em) { 4984 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 4985 &BTRFS_I(inode)->runtime_flags); 4986 goto next; 4987 } 4988 hole_em->start = cur_offset; 4989 hole_em->len = hole_size; 4990 hole_em->orig_start = cur_offset; 4991 4992 hole_em->block_start = EXTENT_MAP_HOLE; 4993 hole_em->block_len = 0; 4994 hole_em->orig_block_len = 0; 4995 hole_em->ram_bytes = hole_size; 4996 hole_em->bdev = fs_info->fs_devices->latest_bdev; 4997 hole_em->compress_type = BTRFS_COMPRESS_NONE; 4998 hole_em->generation = fs_info->generation; 4999 5000 while (1) { 5001 write_lock(&em_tree->lock); 5002 err = add_extent_mapping(em_tree, hole_em, 1); 5003 write_unlock(&em_tree->lock); 5004 if (err != -EEXIST) 5005 break; 5006 btrfs_drop_extent_cache(BTRFS_I(inode), 5007 cur_offset, 5008 cur_offset + 5009 hole_size - 1, 0); 5010 } 5011 free_extent_map(hole_em); 5012 } 5013 next: 5014 free_extent_map(em); 5015 em = NULL; 5016 cur_offset = last_byte; 5017 if (cur_offset >= block_end) 5018 break; 5019 } 5020 free_extent_map(em); 5021 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state); 5022 return err; 5023 } 5024 5025 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 5026 { 5027 struct btrfs_root *root = BTRFS_I(inode)->root; 5028 struct btrfs_trans_handle *trans; 5029 loff_t oldsize = i_size_read(inode); 5030 loff_t newsize = attr->ia_size; 5031 int mask = attr->ia_valid; 5032 int ret; 5033 5034 /* 5035 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 5036 * special case where we need to update the times despite not having 5037 * these flags set. For all other operations the VFS set these flags 5038 * explicitly if it wants a timestamp update. 5039 */ 5040 if (newsize != oldsize) { 5041 inode_inc_iversion(inode); 5042 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) 5043 inode->i_ctime = inode->i_mtime = 5044 current_time(inode); 5045 } 5046 5047 if (newsize > oldsize) { 5048 /* 5049 * Don't do an expanding truncate while snapshotting is ongoing. 5050 * This is to ensure the snapshot captures a fully consistent 5051 * state of this file - if the snapshot captures this expanding 5052 * truncation, it must capture all writes that happened before 5053 * this truncation. 5054 */ 5055 btrfs_wait_for_snapshot_creation(root); 5056 ret = btrfs_cont_expand(inode, oldsize, newsize); 5057 if (ret) { 5058 btrfs_end_write_no_snapshotting(root); 5059 return ret; 5060 } 5061 5062 trans = btrfs_start_transaction(root, 1); 5063 if (IS_ERR(trans)) { 5064 btrfs_end_write_no_snapshotting(root); 5065 return PTR_ERR(trans); 5066 } 5067 5068 i_size_write(inode, newsize); 5069 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL); 5070 pagecache_isize_extended(inode, oldsize, newsize); 5071 ret = btrfs_update_inode(trans, root, inode); 5072 btrfs_end_write_no_snapshotting(root); 5073 btrfs_end_transaction(trans); 5074 } else { 5075 5076 /* 5077 * We're truncating a file that used to have good data down to 5078 * zero. Make sure it gets into the ordered flush list so that 5079 * any new writes get down to disk quickly. 5080 */ 5081 if (newsize == 0) 5082 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE, 5083 &BTRFS_I(inode)->runtime_flags); 5084 5085 /* 5086 * 1 for the orphan item we're going to add 5087 * 1 for the orphan item deletion. 5088 */ 5089 trans = btrfs_start_transaction(root, 2); 5090 if (IS_ERR(trans)) 5091 return PTR_ERR(trans); 5092 5093 /* 5094 * We need to do this in case we fail at _any_ point during the 5095 * actual truncate. Once we do the truncate_setsize we could 5096 * invalidate pages which forces any outstanding ordered io to 5097 * be instantly completed which will give us extents that need 5098 * to be truncated. If we fail to get an orphan inode down we 5099 * could have left over extents that were never meant to live, 5100 * so we need to guarantee from this point on that everything 5101 * will be consistent. 5102 */ 5103 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 5104 btrfs_end_transaction(trans); 5105 if (ret) 5106 return ret; 5107 5108 /* we don't support swapfiles, so vmtruncate shouldn't fail */ 5109 truncate_setsize(inode, newsize); 5110 5111 /* Disable nonlocked read DIO to avoid the end less truncate */ 5112 btrfs_inode_block_unlocked_dio(BTRFS_I(inode)); 5113 inode_dio_wait(inode); 5114 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode)); 5115 5116 ret = btrfs_truncate(inode, newsize == oldsize); 5117 if (ret && inode->i_nlink) { 5118 int err; 5119 5120 /* To get a stable disk_i_size */ 5121 err = btrfs_wait_ordered_range(inode, 0, (u64)-1); 5122 if (err) { 5123 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5124 return err; 5125 } 5126 5127 /* 5128 * failed to truncate, disk_i_size is only adjusted down 5129 * as we remove extents, so it should represent the true 5130 * size of the inode, so reset the in memory size and 5131 * delete our orphan entry. 5132 */ 5133 trans = btrfs_join_transaction(root); 5134 if (IS_ERR(trans)) { 5135 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5136 return ret; 5137 } 5138 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 5139 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 5140 if (err) 5141 btrfs_abort_transaction(trans, err); 5142 btrfs_end_transaction(trans); 5143 } 5144 } 5145 5146 return ret; 5147 } 5148 5149 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) 5150 { 5151 struct inode *inode = d_inode(dentry); 5152 struct btrfs_root *root = BTRFS_I(inode)->root; 5153 int err; 5154 5155 if (btrfs_root_readonly(root)) 5156 return -EROFS; 5157 5158 err = setattr_prepare(dentry, attr); 5159 if (err) 5160 return err; 5161 5162 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 5163 err = btrfs_setsize(inode, attr); 5164 if (err) 5165 return err; 5166 } 5167 5168 if (attr->ia_valid) { 5169 setattr_copy(inode, attr); 5170 inode_inc_iversion(inode); 5171 err = btrfs_dirty_inode(inode); 5172 5173 if (!err && attr->ia_valid & ATTR_MODE) 5174 err = posix_acl_chmod(inode, inode->i_mode); 5175 } 5176 5177 return err; 5178 } 5179 5180 /* 5181 * While truncating the inode pages during eviction, we get the VFS calling 5182 * btrfs_invalidatepage() against each page of the inode. This is slow because 5183 * the calls to btrfs_invalidatepage() result in a huge amount of calls to 5184 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting 5185 * extent_state structures over and over, wasting lots of time. 5186 * 5187 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all 5188 * those expensive operations on a per page basis and do only the ordered io 5189 * finishing, while we release here the extent_map and extent_state structures, 5190 * without the excessive merging and splitting. 5191 */ 5192 static void evict_inode_truncate_pages(struct inode *inode) 5193 { 5194 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 5195 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; 5196 struct rb_node *node; 5197 5198 ASSERT(inode->i_state & I_FREEING); 5199 truncate_inode_pages_final(&inode->i_data); 5200 5201 write_lock(&map_tree->lock); 5202 while (!RB_EMPTY_ROOT(&map_tree->map)) { 5203 struct extent_map *em; 5204 5205 node = rb_first(&map_tree->map); 5206 em = rb_entry(node, struct extent_map, rb_node); 5207 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 5208 clear_bit(EXTENT_FLAG_LOGGING, &em->flags); 5209 remove_extent_mapping(map_tree, em); 5210 free_extent_map(em); 5211 if (need_resched()) { 5212 write_unlock(&map_tree->lock); 5213 cond_resched(); 5214 write_lock(&map_tree->lock); 5215 } 5216 } 5217 write_unlock(&map_tree->lock); 5218 5219 /* 5220 * Keep looping until we have no more ranges in the io tree. 5221 * We can have ongoing bios started by readpages (called from readahead) 5222 * that have their endio callback (extent_io.c:end_bio_extent_readpage) 5223 * still in progress (unlocked the pages in the bio but did not yet 5224 * unlocked the ranges in the io tree). Therefore this means some 5225 * ranges can still be locked and eviction started because before 5226 * submitting those bios, which are executed by a separate task (work 5227 * queue kthread), inode references (inode->i_count) were not taken 5228 * (which would be dropped in the end io callback of each bio). 5229 * Therefore here we effectively end up waiting for those bios and 5230 * anyone else holding locked ranges without having bumped the inode's 5231 * reference count - if we don't do it, when they access the inode's 5232 * io_tree to unlock a range it may be too late, leading to an 5233 * use-after-free issue. 5234 */ 5235 spin_lock(&io_tree->lock); 5236 while (!RB_EMPTY_ROOT(&io_tree->state)) { 5237 struct extent_state *state; 5238 struct extent_state *cached_state = NULL; 5239 u64 start; 5240 u64 end; 5241 5242 node = rb_first(&io_tree->state); 5243 state = rb_entry(node, struct extent_state, rb_node); 5244 start = state->start; 5245 end = state->end; 5246 spin_unlock(&io_tree->lock); 5247 5248 lock_extent_bits(io_tree, start, end, &cached_state); 5249 5250 /* 5251 * If still has DELALLOC flag, the extent didn't reach disk, 5252 * and its reserved space won't be freed by delayed_ref. 5253 * So we need to free its reserved space here. 5254 * (Refer to comment in btrfs_invalidatepage, case 2) 5255 * 5256 * Note, end is the bytenr of last byte, so we need + 1 here. 5257 */ 5258 if (state->state & EXTENT_DELALLOC) 5259 btrfs_qgroup_free_data(inode, NULL, start, end - start + 1); 5260 5261 clear_extent_bit(io_tree, start, end, 5262 EXTENT_LOCKED | EXTENT_DIRTY | 5263 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 5264 EXTENT_DEFRAG, 1, 1, &cached_state); 5265 5266 cond_resched(); 5267 spin_lock(&io_tree->lock); 5268 } 5269 spin_unlock(&io_tree->lock); 5270 } 5271 5272 void btrfs_evict_inode(struct inode *inode) 5273 { 5274 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5275 struct btrfs_trans_handle *trans; 5276 struct btrfs_root *root = BTRFS_I(inode)->root; 5277 struct btrfs_block_rsv *rsv, *global_rsv; 5278 int steal_from_global = 0; 5279 u64 min_size; 5280 int ret; 5281 5282 trace_btrfs_inode_evict(inode); 5283 5284 if (!root) { 5285 clear_inode(inode); 5286 return; 5287 } 5288 5289 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 5290 5291 evict_inode_truncate_pages(inode); 5292 5293 if (inode->i_nlink && 5294 ((btrfs_root_refs(&root->root_item) != 0 && 5295 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 5296 btrfs_is_free_space_inode(BTRFS_I(inode)))) 5297 goto no_delete; 5298 5299 if (is_bad_inode(inode)) { 5300 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5301 goto no_delete; 5302 } 5303 /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */ 5304 if (!special_file(inode->i_mode)) 5305 btrfs_wait_ordered_range(inode, 0, (u64)-1); 5306 5307 btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1); 5308 5309 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 5310 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 5311 &BTRFS_I(inode)->runtime_flags)); 5312 goto no_delete; 5313 } 5314 5315 if (inode->i_nlink > 0) { 5316 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5317 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 5318 goto no_delete; 5319 } 5320 5321 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); 5322 if (ret) { 5323 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5324 goto no_delete; 5325 } 5326 5327 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 5328 if (!rsv) { 5329 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5330 goto no_delete; 5331 } 5332 rsv->size = min_size; 5333 rsv->failfast = 1; 5334 global_rsv = &fs_info->global_block_rsv; 5335 5336 btrfs_i_size_write(BTRFS_I(inode), 0); 5337 5338 /* 5339 * This is a bit simpler than btrfs_truncate since we've already 5340 * reserved our space for our orphan item in the unlink, so we just 5341 * need to reserve some slack space in case we add bytes and update 5342 * inode item when doing the truncate. 5343 */ 5344 while (1) { 5345 ret = btrfs_block_rsv_refill(root, rsv, min_size, 5346 BTRFS_RESERVE_FLUSH_LIMIT); 5347 5348 /* 5349 * Try and steal from the global reserve since we will 5350 * likely not use this space anyway, we want to try as 5351 * hard as possible to get this to work. 5352 */ 5353 if (ret) 5354 steal_from_global++; 5355 else 5356 steal_from_global = 0; 5357 ret = 0; 5358 5359 /* 5360 * steal_from_global == 0: we reserved stuff, hooray! 5361 * steal_from_global == 1: we didn't reserve stuff, boo! 5362 * steal_from_global == 2: we've committed, still not a lot of 5363 * room but maybe we'll have room in the global reserve this 5364 * time. 5365 * steal_from_global == 3: abandon all hope! 5366 */ 5367 if (steal_from_global > 2) { 5368 btrfs_warn(fs_info, 5369 "Could not get space for a delete, will truncate on mount %d", 5370 ret); 5371 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5372 btrfs_free_block_rsv(fs_info, rsv); 5373 goto no_delete; 5374 } 5375 5376 trans = btrfs_join_transaction(root); 5377 if (IS_ERR(trans)) { 5378 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5379 btrfs_free_block_rsv(fs_info, rsv); 5380 goto no_delete; 5381 } 5382 5383 /* 5384 * We can't just steal from the global reserve, we need to make 5385 * sure there is room to do it, if not we need to commit and try 5386 * again. 5387 */ 5388 if (steal_from_global) { 5389 if (!btrfs_check_space_for_delayed_refs(trans, fs_info)) 5390 ret = btrfs_block_rsv_migrate(global_rsv, rsv, 5391 min_size, 0); 5392 else 5393 ret = -ENOSPC; 5394 } 5395 5396 /* 5397 * Couldn't steal from the global reserve, we have too much 5398 * pending stuff built up, commit the transaction and try it 5399 * again. 5400 */ 5401 if (ret) { 5402 ret = btrfs_commit_transaction(trans); 5403 if (ret) { 5404 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5405 btrfs_free_block_rsv(fs_info, rsv); 5406 goto no_delete; 5407 } 5408 continue; 5409 } else { 5410 steal_from_global = 0; 5411 } 5412 5413 trans->block_rsv = rsv; 5414 5415 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0); 5416 if (ret != -ENOSPC && ret != -EAGAIN) 5417 break; 5418 5419 trans->block_rsv = &fs_info->trans_block_rsv; 5420 btrfs_end_transaction(trans); 5421 trans = NULL; 5422 btrfs_btree_balance_dirty(fs_info); 5423 } 5424 5425 btrfs_free_block_rsv(fs_info, rsv); 5426 5427 /* 5428 * Errors here aren't a big deal, it just means we leave orphan items 5429 * in the tree. They will be cleaned up on the next mount. 5430 */ 5431 if (ret == 0) { 5432 trans->block_rsv = root->orphan_block_rsv; 5433 btrfs_orphan_del(trans, BTRFS_I(inode)); 5434 } else { 5435 btrfs_orphan_del(NULL, BTRFS_I(inode)); 5436 } 5437 5438 trans->block_rsv = &fs_info->trans_block_rsv; 5439 if (!(root == fs_info->tree_root || 5440 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)) 5441 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode))); 5442 5443 btrfs_end_transaction(trans); 5444 btrfs_btree_balance_dirty(fs_info); 5445 no_delete: 5446 btrfs_remove_delayed_node(BTRFS_I(inode)); 5447 clear_inode(inode); 5448 } 5449 5450 /* 5451 * this returns the key found in the dir entry in the location pointer. 5452 * If no dir entries were found, returns -ENOENT. 5453 * If found a corrupted location in dir entry, returns -EUCLEAN. 5454 */ 5455 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry, 5456 struct btrfs_key *location) 5457 { 5458 const char *name = dentry->d_name.name; 5459 int namelen = dentry->d_name.len; 5460 struct btrfs_dir_item *di; 5461 struct btrfs_path *path; 5462 struct btrfs_root *root = BTRFS_I(dir)->root; 5463 int ret = 0; 5464 5465 path = btrfs_alloc_path(); 5466 if (!path) 5467 return -ENOMEM; 5468 5469 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)), 5470 name, namelen, 0); 5471 if (!di) { 5472 ret = -ENOENT; 5473 goto out; 5474 } 5475 if (IS_ERR(di)) { 5476 ret = PTR_ERR(di); 5477 goto out; 5478 } 5479 5480 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5481 if (location->type != BTRFS_INODE_ITEM_KEY && 5482 location->type != BTRFS_ROOT_ITEM_KEY) { 5483 ret = -EUCLEAN; 5484 btrfs_warn(root->fs_info, 5485 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))", 5486 __func__, name, btrfs_ino(BTRFS_I(dir)), 5487 location->objectid, location->type, location->offset); 5488 } 5489 out: 5490 btrfs_free_path(path); 5491 return ret; 5492 } 5493 5494 /* 5495 * when we hit a tree root in a directory, the btrfs part of the inode 5496 * needs to be changed to reflect the root directory of the tree root. This 5497 * is kind of like crossing a mount point. 5498 */ 5499 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5500 struct inode *dir, 5501 struct dentry *dentry, 5502 struct btrfs_key *location, 5503 struct btrfs_root **sub_root) 5504 { 5505 struct btrfs_path *path; 5506 struct btrfs_root *new_root; 5507 struct btrfs_root_ref *ref; 5508 struct extent_buffer *leaf; 5509 struct btrfs_key key; 5510 int ret; 5511 int err = 0; 5512 5513 path = btrfs_alloc_path(); 5514 if (!path) { 5515 err = -ENOMEM; 5516 goto out; 5517 } 5518 5519 err = -ENOENT; 5520 key.objectid = BTRFS_I(dir)->root->root_key.objectid; 5521 key.type = BTRFS_ROOT_REF_KEY; 5522 key.offset = location->objectid; 5523 5524 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5525 if (ret) { 5526 if (ret < 0) 5527 err = ret; 5528 goto out; 5529 } 5530 5531 leaf = path->nodes[0]; 5532 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5533 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) || 5534 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len) 5535 goto out; 5536 5537 ret = memcmp_extent_buffer(leaf, dentry->d_name.name, 5538 (unsigned long)(ref + 1), 5539 dentry->d_name.len); 5540 if (ret) 5541 goto out; 5542 5543 btrfs_release_path(path); 5544 5545 new_root = btrfs_read_fs_root_no_name(fs_info, location); 5546 if (IS_ERR(new_root)) { 5547 err = PTR_ERR(new_root); 5548 goto out; 5549 } 5550 5551 *sub_root = new_root; 5552 location->objectid = btrfs_root_dirid(&new_root->root_item); 5553 location->type = BTRFS_INODE_ITEM_KEY; 5554 location->offset = 0; 5555 err = 0; 5556 out: 5557 btrfs_free_path(path); 5558 return err; 5559 } 5560 5561 static void inode_tree_add(struct inode *inode) 5562 { 5563 struct btrfs_root *root = BTRFS_I(inode)->root; 5564 struct btrfs_inode *entry; 5565 struct rb_node **p; 5566 struct rb_node *parent; 5567 struct rb_node *new = &BTRFS_I(inode)->rb_node; 5568 u64 ino = btrfs_ino(BTRFS_I(inode)); 5569 5570 if (inode_unhashed(inode)) 5571 return; 5572 parent = NULL; 5573 spin_lock(&root->inode_lock); 5574 p = &root->inode_tree.rb_node; 5575 while (*p) { 5576 parent = *p; 5577 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5578 5579 if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5580 p = &parent->rb_left; 5581 else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5582 p = &parent->rb_right; 5583 else { 5584 WARN_ON(!(entry->vfs_inode.i_state & 5585 (I_WILL_FREE | I_FREEING))); 5586 rb_replace_node(parent, new, &root->inode_tree); 5587 RB_CLEAR_NODE(parent); 5588 spin_unlock(&root->inode_lock); 5589 return; 5590 } 5591 } 5592 rb_link_node(new, parent, p); 5593 rb_insert_color(new, &root->inode_tree); 5594 spin_unlock(&root->inode_lock); 5595 } 5596 5597 static void inode_tree_del(struct inode *inode) 5598 { 5599 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5600 struct btrfs_root *root = BTRFS_I(inode)->root; 5601 int empty = 0; 5602 5603 spin_lock(&root->inode_lock); 5604 if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) { 5605 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree); 5606 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node); 5607 empty = RB_EMPTY_ROOT(&root->inode_tree); 5608 } 5609 spin_unlock(&root->inode_lock); 5610 5611 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5612 synchronize_srcu(&fs_info->subvol_srcu); 5613 spin_lock(&root->inode_lock); 5614 empty = RB_EMPTY_ROOT(&root->inode_tree); 5615 spin_unlock(&root->inode_lock); 5616 if (empty) 5617 btrfs_add_dead_root(root); 5618 } 5619 } 5620 5621 void btrfs_invalidate_inodes(struct btrfs_root *root) 5622 { 5623 struct btrfs_fs_info *fs_info = root->fs_info; 5624 struct rb_node *node; 5625 struct rb_node *prev; 5626 struct btrfs_inode *entry; 5627 struct inode *inode; 5628 u64 objectid = 0; 5629 5630 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 5631 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 5632 5633 spin_lock(&root->inode_lock); 5634 again: 5635 node = root->inode_tree.rb_node; 5636 prev = NULL; 5637 while (node) { 5638 prev = node; 5639 entry = rb_entry(node, struct btrfs_inode, rb_node); 5640 5641 if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5642 node = node->rb_left; 5643 else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode))) 5644 node = node->rb_right; 5645 else 5646 break; 5647 } 5648 if (!node) { 5649 while (prev) { 5650 entry = rb_entry(prev, struct btrfs_inode, rb_node); 5651 if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) { 5652 node = prev; 5653 break; 5654 } 5655 prev = rb_next(prev); 5656 } 5657 } 5658 while (node) { 5659 entry = rb_entry(node, struct btrfs_inode, rb_node); 5660 objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1; 5661 inode = igrab(&entry->vfs_inode); 5662 if (inode) { 5663 spin_unlock(&root->inode_lock); 5664 if (atomic_read(&inode->i_count) > 1) 5665 d_prune_aliases(inode); 5666 /* 5667 * btrfs_drop_inode will have it removed from 5668 * the inode cache when its usage count 5669 * hits zero. 5670 */ 5671 iput(inode); 5672 cond_resched(); 5673 spin_lock(&root->inode_lock); 5674 goto again; 5675 } 5676 5677 if (cond_resched_lock(&root->inode_lock)) 5678 goto again; 5679 5680 node = rb_next(node); 5681 } 5682 spin_unlock(&root->inode_lock); 5683 } 5684 5685 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5686 { 5687 struct btrfs_iget_args *args = p; 5688 inode->i_ino = args->location->objectid; 5689 memcpy(&BTRFS_I(inode)->location, args->location, 5690 sizeof(*args->location)); 5691 BTRFS_I(inode)->root = args->root; 5692 return 0; 5693 } 5694 5695 static int btrfs_find_actor(struct inode *inode, void *opaque) 5696 { 5697 struct btrfs_iget_args *args = opaque; 5698 return args->location->objectid == BTRFS_I(inode)->location.objectid && 5699 args->root == BTRFS_I(inode)->root; 5700 } 5701 5702 static struct inode *btrfs_iget_locked(struct super_block *s, 5703 struct btrfs_key *location, 5704 struct btrfs_root *root) 5705 { 5706 struct inode *inode; 5707 struct btrfs_iget_args args; 5708 unsigned long hashval = btrfs_inode_hash(location->objectid, root); 5709 5710 args.location = location; 5711 args.root = root; 5712 5713 inode = iget5_locked(s, hashval, btrfs_find_actor, 5714 btrfs_init_locked_inode, 5715 (void *)&args); 5716 return inode; 5717 } 5718 5719 /* Get an inode object given its location and corresponding root. 5720 * Returns in *is_new if the inode was read from disk 5721 */ 5722 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location, 5723 struct btrfs_root *root, int *new) 5724 { 5725 struct inode *inode; 5726 5727 inode = btrfs_iget_locked(s, location, root); 5728 if (!inode) 5729 return ERR_PTR(-ENOMEM); 5730 5731 if (inode->i_state & I_NEW) { 5732 int ret; 5733 5734 ret = btrfs_read_locked_inode(inode); 5735 if (!is_bad_inode(inode)) { 5736 inode_tree_add(inode); 5737 unlock_new_inode(inode); 5738 if (new) 5739 *new = 1; 5740 } else { 5741 unlock_new_inode(inode); 5742 iput(inode); 5743 ASSERT(ret < 0); 5744 inode = ERR_PTR(ret < 0 ? ret : -ESTALE); 5745 } 5746 } 5747 5748 return inode; 5749 } 5750 5751 static struct inode *new_simple_dir(struct super_block *s, 5752 struct btrfs_key *key, 5753 struct btrfs_root *root) 5754 { 5755 struct inode *inode = new_inode(s); 5756 5757 if (!inode) 5758 return ERR_PTR(-ENOMEM); 5759 5760 BTRFS_I(inode)->root = root; 5761 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5762 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5763 5764 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5765 inode->i_op = &btrfs_dir_ro_inode_operations; 5766 inode->i_opflags &= ~IOP_XATTR; 5767 inode->i_fop = &simple_dir_operations; 5768 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5769 inode->i_mtime = current_time(inode); 5770 inode->i_atime = inode->i_mtime; 5771 inode->i_ctime = inode->i_mtime; 5772 BTRFS_I(inode)->i_otime = inode->i_mtime; 5773 5774 return inode; 5775 } 5776 5777 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5778 { 5779 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 5780 struct inode *inode; 5781 struct btrfs_root *root = BTRFS_I(dir)->root; 5782 struct btrfs_root *sub_root = root; 5783 struct btrfs_key location; 5784 int index; 5785 int ret = 0; 5786 5787 if (dentry->d_name.len > BTRFS_NAME_LEN) 5788 return ERR_PTR(-ENAMETOOLONG); 5789 5790 ret = btrfs_inode_by_name(dir, dentry, &location); 5791 if (ret < 0) 5792 return ERR_PTR(ret); 5793 5794 if (location.type == BTRFS_INODE_ITEM_KEY) { 5795 inode = btrfs_iget(dir->i_sb, &location, root, NULL); 5796 return inode; 5797 } 5798 5799 index = srcu_read_lock(&fs_info->subvol_srcu); 5800 ret = fixup_tree_root_location(fs_info, dir, dentry, 5801 &location, &sub_root); 5802 if (ret < 0) { 5803 if (ret != -ENOENT) 5804 inode = ERR_PTR(ret); 5805 else 5806 inode = new_simple_dir(dir->i_sb, &location, sub_root); 5807 } else { 5808 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL); 5809 } 5810 srcu_read_unlock(&fs_info->subvol_srcu, index); 5811 5812 if (!IS_ERR(inode) && root != sub_root) { 5813 down_read(&fs_info->cleanup_work_sem); 5814 if (!sb_rdonly(inode->i_sb)) 5815 ret = btrfs_orphan_cleanup(sub_root); 5816 up_read(&fs_info->cleanup_work_sem); 5817 if (ret) { 5818 iput(inode); 5819 inode = ERR_PTR(ret); 5820 } 5821 } 5822 5823 return inode; 5824 } 5825 5826 static int btrfs_dentry_delete(const struct dentry *dentry) 5827 { 5828 struct btrfs_root *root; 5829 struct inode *inode = d_inode(dentry); 5830 5831 if (!inode && !IS_ROOT(dentry)) 5832 inode = d_inode(dentry->d_parent); 5833 5834 if (inode) { 5835 root = BTRFS_I(inode)->root; 5836 if (btrfs_root_refs(&root->root_item) == 0) 5837 return 1; 5838 5839 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5840 return 1; 5841 } 5842 return 0; 5843 } 5844 5845 static void btrfs_dentry_release(struct dentry *dentry) 5846 { 5847 kfree(dentry->d_fsdata); 5848 } 5849 5850 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5851 unsigned int flags) 5852 { 5853 struct inode *inode; 5854 5855 inode = btrfs_lookup_dentry(dir, dentry); 5856 if (IS_ERR(inode)) { 5857 if (PTR_ERR(inode) == -ENOENT) 5858 inode = NULL; 5859 else 5860 return ERR_CAST(inode); 5861 } 5862 5863 return d_splice_alias(inode, dentry); 5864 } 5865 5866 unsigned char btrfs_filetype_table[] = { 5867 DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK 5868 }; 5869 5870 /* 5871 * All this infrastructure exists because dir_emit can fault, and we are holding 5872 * the tree lock when doing readdir. For now just allocate a buffer and copy 5873 * our information into that, and then dir_emit from the buffer. This is 5874 * similar to what NFS does, only we don't keep the buffer around in pagecache 5875 * because I'm afraid I'll mess that up. Long term we need to make filldir do 5876 * copy_to_user_inatomic so we don't have to worry about page faulting under the 5877 * tree lock. 5878 */ 5879 static int btrfs_opendir(struct inode *inode, struct file *file) 5880 { 5881 struct btrfs_file_private *private; 5882 5883 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL); 5884 if (!private) 5885 return -ENOMEM; 5886 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 5887 if (!private->filldir_buf) { 5888 kfree(private); 5889 return -ENOMEM; 5890 } 5891 file->private_data = private; 5892 return 0; 5893 } 5894 5895 struct dir_entry { 5896 u64 ino; 5897 u64 offset; 5898 unsigned type; 5899 int name_len; 5900 }; 5901 5902 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) 5903 { 5904 while (entries--) { 5905 struct dir_entry *entry = addr; 5906 char *name = (char *)(entry + 1); 5907 5908 ctx->pos = entry->offset; 5909 if (!dir_emit(ctx, name, entry->name_len, entry->ino, 5910 entry->type)) 5911 return 1; 5912 addr += sizeof(struct dir_entry) + entry->name_len; 5913 ctx->pos++; 5914 } 5915 return 0; 5916 } 5917 5918 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5919 { 5920 struct inode *inode = file_inode(file); 5921 struct btrfs_root *root = BTRFS_I(inode)->root; 5922 struct btrfs_file_private *private = file->private_data; 5923 struct btrfs_dir_item *di; 5924 struct btrfs_key key; 5925 struct btrfs_key found_key; 5926 struct btrfs_path *path; 5927 void *addr; 5928 struct list_head ins_list; 5929 struct list_head del_list; 5930 int ret; 5931 struct extent_buffer *leaf; 5932 int slot; 5933 char *name_ptr; 5934 int name_len; 5935 int entries = 0; 5936 int total_len = 0; 5937 bool put = false; 5938 struct btrfs_key location; 5939 5940 if (!dir_emit_dots(file, ctx)) 5941 return 0; 5942 5943 path = btrfs_alloc_path(); 5944 if (!path) 5945 return -ENOMEM; 5946 5947 addr = private->filldir_buf; 5948 path->reada = READA_FORWARD; 5949 5950 INIT_LIST_HEAD(&ins_list); 5951 INIT_LIST_HEAD(&del_list); 5952 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list); 5953 5954 again: 5955 key.type = BTRFS_DIR_INDEX_KEY; 5956 key.offset = ctx->pos; 5957 key.objectid = btrfs_ino(BTRFS_I(inode)); 5958 5959 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5960 if (ret < 0) 5961 goto err; 5962 5963 while (1) { 5964 struct dir_entry *entry; 5965 5966 leaf = path->nodes[0]; 5967 slot = path->slots[0]; 5968 if (slot >= btrfs_header_nritems(leaf)) { 5969 ret = btrfs_next_leaf(root, path); 5970 if (ret < 0) 5971 goto err; 5972 else if (ret > 0) 5973 break; 5974 continue; 5975 } 5976 5977 btrfs_item_key_to_cpu(leaf, &found_key, slot); 5978 5979 if (found_key.objectid != key.objectid) 5980 break; 5981 if (found_key.type != BTRFS_DIR_INDEX_KEY) 5982 break; 5983 if (found_key.offset < ctx->pos) 5984 goto next; 5985 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 5986 goto next; 5987 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); 5988 name_len = btrfs_dir_name_len(leaf, di); 5989 if ((total_len + sizeof(struct dir_entry) + name_len) >= 5990 PAGE_SIZE) { 5991 btrfs_release_path(path); 5992 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 5993 if (ret) 5994 goto nopos; 5995 addr = private->filldir_buf; 5996 entries = 0; 5997 total_len = 0; 5998 goto again; 5999 } 6000 6001 entry = addr; 6002 entry->name_len = name_len; 6003 name_ptr = (char *)(entry + 1); 6004 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1), 6005 name_len); 6006 entry->type = btrfs_filetype_table[btrfs_dir_type(leaf, di)]; 6007 btrfs_dir_item_key_to_cpu(leaf, di, &location); 6008 entry->ino = location.objectid; 6009 entry->offset = found_key.offset; 6010 entries++; 6011 addr += sizeof(struct dir_entry) + name_len; 6012 total_len += sizeof(struct dir_entry) + name_len; 6013 next: 6014 path->slots[0]++; 6015 } 6016 btrfs_release_path(path); 6017 6018 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6019 if (ret) 6020 goto nopos; 6021 6022 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 6023 if (ret) 6024 goto nopos; 6025 6026 /* 6027 * Stop new entries from being returned after we return the last 6028 * entry. 6029 * 6030 * New directory entries are assigned a strictly increasing 6031 * offset. This means that new entries created during readdir 6032 * are *guaranteed* to be seen in the future by that readdir. 6033 * This has broken buggy programs which operate on names as 6034 * they're returned by readdir. Until we re-use freed offsets 6035 * we have this hack to stop new entries from being returned 6036 * under the assumption that they'll never reach this huge 6037 * offset. 6038 * 6039 * This is being careful not to overflow 32bit loff_t unless the 6040 * last entry requires it because doing so has broken 32bit apps 6041 * in the past. 6042 */ 6043 if (ctx->pos >= INT_MAX) 6044 ctx->pos = LLONG_MAX; 6045 else 6046 ctx->pos = INT_MAX; 6047 nopos: 6048 ret = 0; 6049 err: 6050 if (put) 6051 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list); 6052 btrfs_free_path(path); 6053 return ret; 6054 } 6055 6056 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc) 6057 { 6058 struct btrfs_root *root = BTRFS_I(inode)->root; 6059 struct btrfs_trans_handle *trans; 6060 int ret = 0; 6061 bool nolock = false; 6062 6063 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 6064 return 0; 6065 6066 if (btrfs_fs_closing(root->fs_info) && 6067 btrfs_is_free_space_inode(BTRFS_I(inode))) 6068 nolock = true; 6069 6070 if (wbc->sync_mode == WB_SYNC_ALL) { 6071 if (nolock) 6072 trans = btrfs_join_transaction_nolock(root); 6073 else 6074 trans = btrfs_join_transaction(root); 6075 if (IS_ERR(trans)) 6076 return PTR_ERR(trans); 6077 ret = btrfs_commit_transaction(trans); 6078 } 6079 return ret; 6080 } 6081 6082 /* 6083 * This is somewhat expensive, updating the tree every time the 6084 * inode changes. But, it is most likely to find the inode in cache. 6085 * FIXME, needs more benchmarking...there are no reasons other than performance 6086 * to keep or drop this code. 6087 */ 6088 static int btrfs_dirty_inode(struct inode *inode) 6089 { 6090 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6091 struct btrfs_root *root = BTRFS_I(inode)->root; 6092 struct btrfs_trans_handle *trans; 6093 int ret; 6094 6095 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 6096 return 0; 6097 6098 trans = btrfs_join_transaction(root); 6099 if (IS_ERR(trans)) 6100 return PTR_ERR(trans); 6101 6102 ret = btrfs_update_inode(trans, root, inode); 6103 if (ret && ret == -ENOSPC) { 6104 /* whoops, lets try again with the full transaction */ 6105 btrfs_end_transaction(trans); 6106 trans = btrfs_start_transaction(root, 1); 6107 if (IS_ERR(trans)) 6108 return PTR_ERR(trans); 6109 6110 ret = btrfs_update_inode(trans, root, inode); 6111 } 6112 btrfs_end_transaction(trans); 6113 if (BTRFS_I(inode)->delayed_node) 6114 btrfs_balance_delayed_items(fs_info); 6115 6116 return ret; 6117 } 6118 6119 /* 6120 * This is a copy of file_update_time. We need this so we can return error on 6121 * ENOSPC for updating the inode in the case of file write and mmap writes. 6122 */ 6123 static int btrfs_update_time(struct inode *inode, struct timespec *now, 6124 int flags) 6125 { 6126 struct btrfs_root *root = BTRFS_I(inode)->root; 6127 bool dirty = flags & ~S_VERSION; 6128 6129 if (btrfs_root_readonly(root)) 6130 return -EROFS; 6131 6132 if (flags & S_VERSION) 6133 dirty |= inode_maybe_inc_iversion(inode, dirty); 6134 if (flags & S_CTIME) 6135 inode->i_ctime = *now; 6136 if (flags & S_MTIME) 6137 inode->i_mtime = *now; 6138 if (flags & S_ATIME) 6139 inode->i_atime = *now; 6140 return dirty ? btrfs_dirty_inode(inode) : 0; 6141 } 6142 6143 /* 6144 * find the highest existing sequence number in a directory 6145 * and then set the in-memory index_cnt variable to reflect 6146 * free sequence numbers 6147 */ 6148 static int btrfs_set_inode_index_count(struct btrfs_inode *inode) 6149 { 6150 struct btrfs_root *root = inode->root; 6151 struct btrfs_key key, found_key; 6152 struct btrfs_path *path; 6153 struct extent_buffer *leaf; 6154 int ret; 6155 6156 key.objectid = btrfs_ino(inode); 6157 key.type = BTRFS_DIR_INDEX_KEY; 6158 key.offset = (u64)-1; 6159 6160 path = btrfs_alloc_path(); 6161 if (!path) 6162 return -ENOMEM; 6163 6164 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 6165 if (ret < 0) 6166 goto out; 6167 /* FIXME: we should be able to handle this */ 6168 if (ret == 0) 6169 goto out; 6170 ret = 0; 6171 6172 /* 6173 * MAGIC NUMBER EXPLANATION: 6174 * since we search a directory based on f_pos we have to start at 2 6175 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody 6176 * else has to start at 2 6177 */ 6178 if (path->slots[0] == 0) { 6179 inode->index_cnt = 2; 6180 goto out; 6181 } 6182 6183 path->slots[0]--; 6184 6185 leaf = path->nodes[0]; 6186 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6187 6188 if (found_key.objectid != btrfs_ino(inode) || 6189 found_key.type != BTRFS_DIR_INDEX_KEY) { 6190 inode->index_cnt = 2; 6191 goto out; 6192 } 6193 6194 inode->index_cnt = found_key.offset + 1; 6195 out: 6196 btrfs_free_path(path); 6197 return ret; 6198 } 6199 6200 /* 6201 * helper to find a free sequence number in a given directory. This current 6202 * code is very simple, later versions will do smarter things in the btree 6203 */ 6204 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) 6205 { 6206 int ret = 0; 6207 6208 if (dir->index_cnt == (u64)-1) { 6209 ret = btrfs_inode_delayed_dir_index_count(dir); 6210 if (ret) { 6211 ret = btrfs_set_inode_index_count(dir); 6212 if (ret) 6213 return ret; 6214 } 6215 } 6216 6217 *index = dir->index_cnt; 6218 dir->index_cnt++; 6219 6220 return ret; 6221 } 6222 6223 static int btrfs_insert_inode_locked(struct inode *inode) 6224 { 6225 struct btrfs_iget_args args; 6226 args.location = &BTRFS_I(inode)->location; 6227 args.root = BTRFS_I(inode)->root; 6228 6229 return insert_inode_locked4(inode, 6230 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 6231 btrfs_find_actor, &args); 6232 } 6233 6234 /* 6235 * Inherit flags from the parent inode. 6236 * 6237 * Currently only the compression flags and the cow flags are inherited. 6238 */ 6239 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir) 6240 { 6241 unsigned int flags; 6242 6243 if (!dir) 6244 return; 6245 6246 flags = BTRFS_I(dir)->flags; 6247 6248 if (flags & BTRFS_INODE_NOCOMPRESS) { 6249 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS; 6250 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 6251 } else if (flags & BTRFS_INODE_COMPRESS) { 6252 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS; 6253 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS; 6254 } 6255 6256 if (flags & BTRFS_INODE_NODATACOW) { 6257 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW; 6258 if (S_ISREG(inode->i_mode)) 6259 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6260 } 6261 6262 btrfs_update_iflags(inode); 6263 } 6264 6265 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, 6266 struct btrfs_root *root, 6267 struct inode *dir, 6268 const char *name, int name_len, 6269 u64 ref_objectid, u64 objectid, 6270 umode_t mode, u64 *index) 6271 { 6272 struct btrfs_fs_info *fs_info = root->fs_info; 6273 struct inode *inode; 6274 struct btrfs_inode_item *inode_item; 6275 struct btrfs_key *location; 6276 struct btrfs_path *path; 6277 struct btrfs_inode_ref *ref; 6278 struct btrfs_key key[2]; 6279 u32 sizes[2]; 6280 int nitems = name ? 2 : 1; 6281 unsigned long ptr; 6282 int ret; 6283 6284 path = btrfs_alloc_path(); 6285 if (!path) 6286 return ERR_PTR(-ENOMEM); 6287 6288 inode = new_inode(fs_info->sb); 6289 if (!inode) { 6290 btrfs_free_path(path); 6291 return ERR_PTR(-ENOMEM); 6292 } 6293 6294 /* 6295 * O_TMPFILE, set link count to 0, so that after this point, 6296 * we fill in an inode item with the correct link count. 6297 */ 6298 if (!name) 6299 set_nlink(inode, 0); 6300 6301 /* 6302 * we have to initialize this early, so we can reclaim the inode 6303 * number if we fail afterwards in this function. 6304 */ 6305 inode->i_ino = objectid; 6306 6307 if (dir && name) { 6308 trace_btrfs_inode_request(dir); 6309 6310 ret = btrfs_set_inode_index(BTRFS_I(dir), index); 6311 if (ret) { 6312 btrfs_free_path(path); 6313 iput(inode); 6314 return ERR_PTR(ret); 6315 } 6316 } else if (dir) { 6317 *index = 0; 6318 } 6319 /* 6320 * index_cnt is ignored for everything but a dir, 6321 * btrfs_set_inode_index_count has an explanation for the magic 6322 * number 6323 */ 6324 BTRFS_I(inode)->index_cnt = 2; 6325 BTRFS_I(inode)->dir_index = *index; 6326 BTRFS_I(inode)->root = root; 6327 BTRFS_I(inode)->generation = trans->transid; 6328 inode->i_generation = BTRFS_I(inode)->generation; 6329 6330 /* 6331 * We could have gotten an inode number from somebody who was fsynced 6332 * and then removed in this same transaction, so let's just set full 6333 * sync since it will be a full sync anyway and this will blow away the 6334 * old info in the log. 6335 */ 6336 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 6337 6338 key[0].objectid = objectid; 6339 key[0].type = BTRFS_INODE_ITEM_KEY; 6340 key[0].offset = 0; 6341 6342 sizes[0] = sizeof(struct btrfs_inode_item); 6343 6344 if (name) { 6345 /* 6346 * Start new inodes with an inode_ref. This is slightly more 6347 * efficient for small numbers of hard links since they will 6348 * be packed into one item. Extended refs will kick in if we 6349 * add more hard links than can fit in the ref item. 6350 */ 6351 key[1].objectid = objectid; 6352 key[1].type = BTRFS_INODE_REF_KEY; 6353 key[1].offset = ref_objectid; 6354 6355 sizes[1] = name_len + sizeof(*ref); 6356 } 6357 6358 location = &BTRFS_I(inode)->location; 6359 location->objectid = objectid; 6360 location->offset = 0; 6361 location->type = BTRFS_INODE_ITEM_KEY; 6362 6363 ret = btrfs_insert_inode_locked(inode); 6364 if (ret < 0) 6365 goto fail; 6366 6367 path->leave_spinning = 1; 6368 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems); 6369 if (ret != 0) 6370 goto fail_unlock; 6371 6372 inode_init_owner(inode, dir, mode); 6373 inode_set_bytes(inode, 0); 6374 6375 inode->i_mtime = current_time(inode); 6376 inode->i_atime = inode->i_mtime; 6377 inode->i_ctime = inode->i_mtime; 6378 BTRFS_I(inode)->i_otime = inode->i_mtime; 6379 6380 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6381 struct btrfs_inode_item); 6382 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6383 sizeof(*inode_item)); 6384 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6385 6386 if (name) { 6387 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6388 struct btrfs_inode_ref); 6389 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len); 6390 btrfs_set_inode_ref_index(path->nodes[0], ref, *index); 6391 ptr = (unsigned long)(ref + 1); 6392 write_extent_buffer(path->nodes[0], name, ptr, name_len); 6393 } 6394 6395 btrfs_mark_buffer_dirty(path->nodes[0]); 6396 btrfs_free_path(path); 6397 6398 btrfs_inherit_iflags(inode, dir); 6399 6400 if (S_ISREG(mode)) { 6401 if (btrfs_test_opt(fs_info, NODATASUM)) 6402 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6403 if (btrfs_test_opt(fs_info, NODATACOW)) 6404 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6405 BTRFS_INODE_NODATASUM; 6406 } 6407 6408 inode_tree_add(inode); 6409 6410 trace_btrfs_inode_new(inode); 6411 btrfs_set_inode_last_trans(trans, inode); 6412 6413 btrfs_update_root_times(trans, root); 6414 6415 ret = btrfs_inode_inherit_props(trans, inode, dir); 6416 if (ret) 6417 btrfs_err(fs_info, 6418 "error inheriting props for ino %llu (root %llu): %d", 6419 btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret); 6420 6421 return inode; 6422 6423 fail_unlock: 6424 unlock_new_inode(inode); 6425 fail: 6426 if (dir && name) 6427 BTRFS_I(dir)->index_cnt--; 6428 btrfs_free_path(path); 6429 iput(inode); 6430 return ERR_PTR(ret); 6431 } 6432 6433 static inline u8 btrfs_inode_type(struct inode *inode) 6434 { 6435 return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT]; 6436 } 6437 6438 /* 6439 * utility function to add 'inode' into 'parent_inode' with 6440 * a give name and a given sequence number. 6441 * if 'add_backref' is true, also insert a backref from the 6442 * inode to the parent directory. 6443 */ 6444 int btrfs_add_link(struct btrfs_trans_handle *trans, 6445 struct btrfs_inode *parent_inode, struct btrfs_inode *inode, 6446 const char *name, int name_len, int add_backref, u64 index) 6447 { 6448 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 6449 int ret = 0; 6450 struct btrfs_key key; 6451 struct btrfs_root *root = parent_inode->root; 6452 u64 ino = btrfs_ino(inode); 6453 u64 parent_ino = btrfs_ino(parent_inode); 6454 6455 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6456 memcpy(&key, &inode->root->root_key, sizeof(key)); 6457 } else { 6458 key.objectid = ino; 6459 key.type = BTRFS_INODE_ITEM_KEY; 6460 key.offset = 0; 6461 } 6462 6463 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6464 ret = btrfs_add_root_ref(trans, fs_info, key.objectid, 6465 root->root_key.objectid, parent_ino, 6466 index, name, name_len); 6467 } else if (add_backref) { 6468 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino, 6469 parent_ino, index); 6470 } 6471 6472 /* Nothing to clean up yet */ 6473 if (ret) 6474 return ret; 6475 6476 ret = btrfs_insert_dir_item(trans, root, name, name_len, 6477 parent_inode, &key, 6478 btrfs_inode_type(&inode->vfs_inode), index); 6479 if (ret == -EEXIST || ret == -EOVERFLOW) 6480 goto fail_dir_item; 6481 else if (ret) { 6482 btrfs_abort_transaction(trans, ret); 6483 return ret; 6484 } 6485 6486 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + 6487 name_len * 2); 6488 inode_inc_iversion(&parent_inode->vfs_inode); 6489 parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime = 6490 current_time(&parent_inode->vfs_inode); 6491 ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode); 6492 if (ret) 6493 btrfs_abort_transaction(trans, ret); 6494 return ret; 6495 6496 fail_dir_item: 6497 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6498 u64 local_index; 6499 int err; 6500 err = btrfs_del_root_ref(trans, fs_info, key.objectid, 6501 root->root_key.objectid, parent_ino, 6502 &local_index, name, name_len); 6503 6504 } else if (add_backref) { 6505 u64 local_index; 6506 int err; 6507 6508 err = btrfs_del_inode_ref(trans, root, name, name_len, 6509 ino, parent_ino, &local_index); 6510 } 6511 return ret; 6512 } 6513 6514 static int btrfs_add_nondir(struct btrfs_trans_handle *trans, 6515 struct btrfs_inode *dir, struct dentry *dentry, 6516 struct btrfs_inode *inode, int backref, u64 index) 6517 { 6518 int err = btrfs_add_link(trans, dir, inode, 6519 dentry->d_name.name, dentry->d_name.len, 6520 backref, index); 6521 if (err > 0) 6522 err = -EEXIST; 6523 return err; 6524 } 6525 6526 static int btrfs_mknod(struct inode *dir, struct dentry *dentry, 6527 umode_t mode, dev_t rdev) 6528 { 6529 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6530 struct btrfs_trans_handle *trans; 6531 struct btrfs_root *root = BTRFS_I(dir)->root; 6532 struct inode *inode = NULL; 6533 int err; 6534 int drop_inode = 0; 6535 u64 objectid; 6536 u64 index = 0; 6537 6538 /* 6539 * 2 for inode item and ref 6540 * 2 for dir items 6541 * 1 for xattr if selinux is on 6542 */ 6543 trans = btrfs_start_transaction(root, 5); 6544 if (IS_ERR(trans)) 6545 return PTR_ERR(trans); 6546 6547 err = btrfs_find_free_ino(root, &objectid); 6548 if (err) 6549 goto out_unlock; 6550 6551 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6552 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6553 mode, &index); 6554 if (IS_ERR(inode)) { 6555 err = PTR_ERR(inode); 6556 goto out_unlock; 6557 } 6558 6559 /* 6560 * If the active LSM wants to access the inode during 6561 * d_instantiate it needs these. Smack checks to see 6562 * if the filesystem supports xattrs by looking at the 6563 * ops vector. 6564 */ 6565 inode->i_op = &btrfs_special_inode_operations; 6566 init_special_inode(inode, inode->i_mode, rdev); 6567 6568 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6569 if (err) 6570 goto out_unlock_inode; 6571 6572 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6573 0, index); 6574 if (err) { 6575 goto out_unlock_inode; 6576 } else { 6577 btrfs_update_inode(trans, root, inode); 6578 unlock_new_inode(inode); 6579 d_instantiate(dentry, inode); 6580 } 6581 6582 out_unlock: 6583 btrfs_end_transaction(trans); 6584 btrfs_btree_balance_dirty(fs_info); 6585 if (drop_inode) { 6586 inode_dec_link_count(inode); 6587 iput(inode); 6588 } 6589 return err; 6590 6591 out_unlock_inode: 6592 drop_inode = 1; 6593 unlock_new_inode(inode); 6594 goto out_unlock; 6595 6596 } 6597 6598 static int btrfs_create(struct inode *dir, struct dentry *dentry, 6599 umode_t mode, bool excl) 6600 { 6601 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6602 struct btrfs_trans_handle *trans; 6603 struct btrfs_root *root = BTRFS_I(dir)->root; 6604 struct inode *inode = NULL; 6605 int drop_inode_on_err = 0; 6606 int err; 6607 u64 objectid; 6608 u64 index = 0; 6609 6610 /* 6611 * 2 for inode item and ref 6612 * 2 for dir items 6613 * 1 for xattr if selinux is on 6614 */ 6615 trans = btrfs_start_transaction(root, 5); 6616 if (IS_ERR(trans)) 6617 return PTR_ERR(trans); 6618 6619 err = btrfs_find_free_ino(root, &objectid); 6620 if (err) 6621 goto out_unlock; 6622 6623 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6624 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6625 mode, &index); 6626 if (IS_ERR(inode)) { 6627 err = PTR_ERR(inode); 6628 goto out_unlock; 6629 } 6630 drop_inode_on_err = 1; 6631 /* 6632 * If the active LSM wants to access the inode during 6633 * d_instantiate it needs these. Smack checks to see 6634 * if the filesystem supports xattrs by looking at the 6635 * ops vector. 6636 */ 6637 inode->i_fop = &btrfs_file_operations; 6638 inode->i_op = &btrfs_file_inode_operations; 6639 inode->i_mapping->a_ops = &btrfs_aops; 6640 6641 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6642 if (err) 6643 goto out_unlock_inode; 6644 6645 err = btrfs_update_inode(trans, root, inode); 6646 if (err) 6647 goto out_unlock_inode; 6648 6649 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6650 0, index); 6651 if (err) 6652 goto out_unlock_inode; 6653 6654 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 6655 unlock_new_inode(inode); 6656 d_instantiate(dentry, inode); 6657 6658 out_unlock: 6659 btrfs_end_transaction(trans); 6660 if (err && drop_inode_on_err) { 6661 inode_dec_link_count(inode); 6662 iput(inode); 6663 } 6664 btrfs_btree_balance_dirty(fs_info); 6665 return err; 6666 6667 out_unlock_inode: 6668 unlock_new_inode(inode); 6669 goto out_unlock; 6670 6671 } 6672 6673 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6674 struct dentry *dentry) 6675 { 6676 struct btrfs_trans_handle *trans = NULL; 6677 struct btrfs_root *root = BTRFS_I(dir)->root; 6678 struct inode *inode = d_inode(old_dentry); 6679 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6680 u64 index; 6681 int err; 6682 int drop_inode = 0; 6683 6684 /* do not allow sys_link's with other subvols of the same device */ 6685 if (root->objectid != BTRFS_I(inode)->root->objectid) 6686 return -EXDEV; 6687 6688 if (inode->i_nlink >= BTRFS_LINK_MAX) 6689 return -EMLINK; 6690 6691 err = btrfs_set_inode_index(BTRFS_I(dir), &index); 6692 if (err) 6693 goto fail; 6694 6695 /* 6696 * 2 items for inode and inode ref 6697 * 2 items for dir items 6698 * 1 item for parent inode 6699 */ 6700 trans = btrfs_start_transaction(root, 5); 6701 if (IS_ERR(trans)) { 6702 err = PTR_ERR(trans); 6703 trans = NULL; 6704 goto fail; 6705 } 6706 6707 /* There are several dir indexes for this inode, clear the cache. */ 6708 BTRFS_I(inode)->dir_index = 0ULL; 6709 inc_nlink(inode); 6710 inode_inc_iversion(inode); 6711 inode->i_ctime = current_time(inode); 6712 ihold(inode); 6713 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6714 6715 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6716 1, index); 6717 6718 if (err) { 6719 drop_inode = 1; 6720 } else { 6721 struct dentry *parent = dentry->d_parent; 6722 err = btrfs_update_inode(trans, root, inode); 6723 if (err) 6724 goto fail; 6725 if (inode->i_nlink == 1) { 6726 /* 6727 * If new hard link count is 1, it's a file created 6728 * with open(2) O_TMPFILE flag. 6729 */ 6730 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 6731 if (err) 6732 goto fail; 6733 } 6734 d_instantiate(dentry, inode); 6735 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent); 6736 } 6737 6738 fail: 6739 if (trans) 6740 btrfs_end_transaction(trans); 6741 if (drop_inode) { 6742 inode_dec_link_count(inode); 6743 iput(inode); 6744 } 6745 btrfs_btree_balance_dirty(fs_info); 6746 return err; 6747 } 6748 6749 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 6750 { 6751 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6752 struct inode *inode = NULL; 6753 struct btrfs_trans_handle *trans; 6754 struct btrfs_root *root = BTRFS_I(dir)->root; 6755 int err = 0; 6756 int drop_on_err = 0; 6757 u64 objectid = 0; 6758 u64 index = 0; 6759 6760 /* 6761 * 2 items for inode and ref 6762 * 2 items for dir items 6763 * 1 for xattr if selinux is on 6764 */ 6765 trans = btrfs_start_transaction(root, 5); 6766 if (IS_ERR(trans)) 6767 return PTR_ERR(trans); 6768 6769 err = btrfs_find_free_ino(root, &objectid); 6770 if (err) 6771 goto out_fail; 6772 6773 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6774 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6775 S_IFDIR | mode, &index); 6776 if (IS_ERR(inode)) { 6777 err = PTR_ERR(inode); 6778 goto out_fail; 6779 } 6780 6781 drop_on_err = 1; 6782 /* these must be set before we unlock the inode */ 6783 inode->i_op = &btrfs_dir_inode_operations; 6784 inode->i_fop = &btrfs_dir_file_operations; 6785 6786 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6787 if (err) 6788 goto out_fail_inode; 6789 6790 btrfs_i_size_write(BTRFS_I(inode), 0); 6791 err = btrfs_update_inode(trans, root, inode); 6792 if (err) 6793 goto out_fail_inode; 6794 6795 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), 6796 dentry->d_name.name, 6797 dentry->d_name.len, 0, index); 6798 if (err) 6799 goto out_fail_inode; 6800 6801 d_instantiate(dentry, inode); 6802 /* 6803 * mkdir is special. We're unlocking after we call d_instantiate 6804 * to avoid a race with nfsd calling d_instantiate. 6805 */ 6806 unlock_new_inode(inode); 6807 drop_on_err = 0; 6808 6809 out_fail: 6810 btrfs_end_transaction(trans); 6811 if (drop_on_err) { 6812 inode_dec_link_count(inode); 6813 iput(inode); 6814 } 6815 btrfs_btree_balance_dirty(fs_info); 6816 return err; 6817 6818 out_fail_inode: 6819 unlock_new_inode(inode); 6820 goto out_fail; 6821 } 6822 6823 static noinline int uncompress_inline(struct btrfs_path *path, 6824 struct page *page, 6825 size_t pg_offset, u64 extent_offset, 6826 struct btrfs_file_extent_item *item) 6827 { 6828 int ret; 6829 struct extent_buffer *leaf = path->nodes[0]; 6830 char *tmp; 6831 size_t max_size; 6832 unsigned long inline_size; 6833 unsigned long ptr; 6834 int compress_type; 6835 6836 WARN_ON(pg_offset != 0); 6837 compress_type = btrfs_file_extent_compression(leaf, item); 6838 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6839 inline_size = btrfs_file_extent_inline_item_len(leaf, 6840 btrfs_item_nr(path->slots[0])); 6841 tmp = kmalloc(inline_size, GFP_NOFS); 6842 if (!tmp) 6843 return -ENOMEM; 6844 ptr = btrfs_file_extent_inline_start(item); 6845 6846 read_extent_buffer(leaf, tmp, ptr, inline_size); 6847 6848 max_size = min_t(unsigned long, PAGE_SIZE, max_size); 6849 ret = btrfs_decompress(compress_type, tmp, page, 6850 extent_offset, inline_size, max_size); 6851 6852 /* 6853 * decompression code contains a memset to fill in any space between the end 6854 * of the uncompressed data and the end of max_size in case the decompressed 6855 * data ends up shorter than ram_bytes. That doesn't cover the hole between 6856 * the end of an inline extent and the beginning of the next block, so we 6857 * cover that region here. 6858 */ 6859 6860 if (max_size + pg_offset < PAGE_SIZE) { 6861 char *map = kmap(page); 6862 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset); 6863 kunmap(page); 6864 } 6865 kfree(tmp); 6866 return ret; 6867 } 6868 6869 /* 6870 * a bit scary, this does extent mapping from logical file offset to the disk. 6871 * the ugly parts come from merging extents from the disk with the in-ram 6872 * representation. This gets more complex because of the data=ordered code, 6873 * where the in-ram extents might be locked pending data=ordered completion. 6874 * 6875 * This also copies inline extents directly into the page. 6876 */ 6877 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, 6878 struct page *page, 6879 size_t pg_offset, u64 start, u64 len, 6880 int create) 6881 { 6882 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb); 6883 int ret; 6884 int err = 0; 6885 u64 extent_start = 0; 6886 u64 extent_end = 0; 6887 u64 objectid = btrfs_ino(inode); 6888 u32 found_type; 6889 struct btrfs_path *path = NULL; 6890 struct btrfs_root *root = inode->root; 6891 struct btrfs_file_extent_item *item; 6892 struct extent_buffer *leaf; 6893 struct btrfs_key found_key; 6894 struct extent_map *em = NULL; 6895 struct extent_map_tree *em_tree = &inode->extent_tree; 6896 struct extent_io_tree *io_tree = &inode->io_tree; 6897 const bool new_inline = !page || create; 6898 6899 read_lock(&em_tree->lock); 6900 em = lookup_extent_mapping(em_tree, start, len); 6901 if (em) 6902 em->bdev = fs_info->fs_devices->latest_bdev; 6903 read_unlock(&em_tree->lock); 6904 6905 if (em) { 6906 if (em->start > start || em->start + em->len <= start) 6907 free_extent_map(em); 6908 else if (em->block_start == EXTENT_MAP_INLINE && page) 6909 free_extent_map(em); 6910 else 6911 goto out; 6912 } 6913 em = alloc_extent_map(); 6914 if (!em) { 6915 err = -ENOMEM; 6916 goto out; 6917 } 6918 em->bdev = fs_info->fs_devices->latest_bdev; 6919 em->start = EXTENT_MAP_HOLE; 6920 em->orig_start = EXTENT_MAP_HOLE; 6921 em->len = (u64)-1; 6922 em->block_len = (u64)-1; 6923 6924 if (!path) { 6925 path = btrfs_alloc_path(); 6926 if (!path) { 6927 err = -ENOMEM; 6928 goto out; 6929 } 6930 /* 6931 * Chances are we'll be called again, so go ahead and do 6932 * readahead 6933 */ 6934 path->reada = READA_FORWARD; 6935 } 6936 6937 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); 6938 if (ret < 0) { 6939 err = ret; 6940 goto out; 6941 } 6942 6943 if (ret != 0) { 6944 if (path->slots[0] == 0) 6945 goto not_found; 6946 path->slots[0]--; 6947 } 6948 6949 leaf = path->nodes[0]; 6950 item = btrfs_item_ptr(leaf, path->slots[0], 6951 struct btrfs_file_extent_item); 6952 /* are we inside the extent that was found? */ 6953 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6954 found_type = found_key.type; 6955 if (found_key.objectid != objectid || 6956 found_type != BTRFS_EXTENT_DATA_KEY) { 6957 /* 6958 * If we backup past the first extent we want to move forward 6959 * and see if there is an extent in front of us, otherwise we'll 6960 * say there is a hole for our whole search range which can 6961 * cause problems. 6962 */ 6963 extent_end = start; 6964 goto next; 6965 } 6966 6967 found_type = btrfs_file_extent_type(leaf, item); 6968 extent_start = found_key.offset; 6969 if (found_type == BTRFS_FILE_EXTENT_REG || 6970 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6971 extent_end = extent_start + 6972 btrfs_file_extent_num_bytes(leaf, item); 6973 6974 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, 6975 extent_start); 6976 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 6977 size_t size; 6978 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 6979 extent_end = ALIGN(extent_start + size, 6980 fs_info->sectorsize); 6981 6982 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, 6983 path->slots[0], 6984 extent_start); 6985 } 6986 next: 6987 if (start >= extent_end) { 6988 path->slots[0]++; 6989 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 6990 ret = btrfs_next_leaf(root, path); 6991 if (ret < 0) { 6992 err = ret; 6993 goto out; 6994 } 6995 if (ret > 0) 6996 goto not_found; 6997 leaf = path->nodes[0]; 6998 } 6999 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 7000 if (found_key.objectid != objectid || 7001 found_key.type != BTRFS_EXTENT_DATA_KEY) 7002 goto not_found; 7003 if (start + len <= found_key.offset) 7004 goto not_found; 7005 if (start > found_key.offset) 7006 goto next; 7007 em->start = start; 7008 em->orig_start = start; 7009 em->len = found_key.offset - start; 7010 goto not_found_em; 7011 } 7012 7013 btrfs_extent_item_to_extent_map(inode, path, item, 7014 new_inline, em); 7015 7016 if (found_type == BTRFS_FILE_EXTENT_REG || 7017 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7018 goto insert; 7019 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 7020 unsigned long ptr; 7021 char *map; 7022 size_t size; 7023 size_t extent_offset; 7024 size_t copy_size; 7025 7026 if (new_inline) 7027 goto out; 7028 7029 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 7030 extent_offset = page_offset(page) + pg_offset - extent_start; 7031 copy_size = min_t(u64, PAGE_SIZE - pg_offset, 7032 size - extent_offset); 7033 em->start = extent_start + extent_offset; 7034 em->len = ALIGN(copy_size, fs_info->sectorsize); 7035 em->orig_block_len = em->len; 7036 em->orig_start = em->start; 7037 ptr = btrfs_file_extent_inline_start(item) + extent_offset; 7038 if (!PageUptodate(page)) { 7039 if (btrfs_file_extent_compression(leaf, item) != 7040 BTRFS_COMPRESS_NONE) { 7041 ret = uncompress_inline(path, page, pg_offset, 7042 extent_offset, item); 7043 if (ret) { 7044 err = ret; 7045 goto out; 7046 } 7047 } else { 7048 map = kmap(page); 7049 read_extent_buffer(leaf, map + pg_offset, ptr, 7050 copy_size); 7051 if (pg_offset + copy_size < PAGE_SIZE) { 7052 memset(map + pg_offset + copy_size, 0, 7053 PAGE_SIZE - pg_offset - 7054 copy_size); 7055 } 7056 kunmap(page); 7057 } 7058 flush_dcache_page(page); 7059 } 7060 set_extent_uptodate(io_tree, em->start, 7061 extent_map_end(em) - 1, NULL, GFP_NOFS); 7062 goto insert; 7063 } 7064 not_found: 7065 em->start = start; 7066 em->orig_start = start; 7067 em->len = len; 7068 not_found_em: 7069 em->block_start = EXTENT_MAP_HOLE; 7070 insert: 7071 btrfs_release_path(path); 7072 if (em->start > start || extent_map_end(em) <= start) { 7073 btrfs_err(fs_info, 7074 "bad extent! em: [%llu %llu] passed [%llu %llu]", 7075 em->start, em->len, start, len); 7076 err = -EIO; 7077 goto out; 7078 } 7079 7080 err = 0; 7081 write_lock(&em_tree->lock); 7082 err = btrfs_add_extent_mapping(em_tree, &em, start, len); 7083 write_unlock(&em_tree->lock); 7084 out: 7085 7086 trace_btrfs_get_extent(root, inode, em); 7087 7088 btrfs_free_path(path); 7089 if (err) { 7090 free_extent_map(em); 7091 return ERR_PTR(err); 7092 } 7093 BUG_ON(!em); /* Error is always set */ 7094 return em; 7095 } 7096 7097 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, 7098 struct page *page, 7099 size_t pg_offset, u64 start, u64 len, 7100 int create) 7101 { 7102 struct extent_map *em; 7103 struct extent_map *hole_em = NULL; 7104 u64 range_start = start; 7105 u64 end; 7106 u64 found; 7107 u64 found_end; 7108 int err = 0; 7109 7110 em = btrfs_get_extent(inode, page, pg_offset, start, len, create); 7111 if (IS_ERR(em)) 7112 return em; 7113 /* 7114 * If our em maps to: 7115 * - a hole or 7116 * - a pre-alloc extent, 7117 * there might actually be delalloc bytes behind it. 7118 */ 7119 if (em->block_start != EXTENT_MAP_HOLE && 7120 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7121 return em; 7122 else 7123 hole_em = em; 7124 7125 /* check to see if we've wrapped (len == -1 or similar) */ 7126 end = start + len; 7127 if (end < start) 7128 end = (u64)-1; 7129 else 7130 end -= 1; 7131 7132 em = NULL; 7133 7134 /* ok, we didn't find anything, lets look for delalloc */ 7135 found = count_range_bits(&inode->io_tree, &range_start, 7136 end, len, EXTENT_DELALLOC, 1); 7137 found_end = range_start + found; 7138 if (found_end < range_start) 7139 found_end = (u64)-1; 7140 7141 /* 7142 * we didn't find anything useful, return 7143 * the original results from get_extent() 7144 */ 7145 if (range_start > end || found_end <= start) { 7146 em = hole_em; 7147 hole_em = NULL; 7148 goto out; 7149 } 7150 7151 /* adjust the range_start to make sure it doesn't 7152 * go backwards from the start they passed in 7153 */ 7154 range_start = max(start, range_start); 7155 found = found_end - range_start; 7156 7157 if (found > 0) { 7158 u64 hole_start = start; 7159 u64 hole_len = len; 7160 7161 em = alloc_extent_map(); 7162 if (!em) { 7163 err = -ENOMEM; 7164 goto out; 7165 } 7166 /* 7167 * when btrfs_get_extent can't find anything it 7168 * returns one huge hole 7169 * 7170 * make sure what it found really fits our range, and 7171 * adjust to make sure it is based on the start from 7172 * the caller 7173 */ 7174 if (hole_em) { 7175 u64 calc_end = extent_map_end(hole_em); 7176 7177 if (calc_end <= start || (hole_em->start > end)) { 7178 free_extent_map(hole_em); 7179 hole_em = NULL; 7180 } else { 7181 hole_start = max(hole_em->start, start); 7182 hole_len = calc_end - hole_start; 7183 } 7184 } 7185 em->bdev = NULL; 7186 if (hole_em && range_start > hole_start) { 7187 /* our hole starts before our delalloc, so we 7188 * have to return just the parts of the hole 7189 * that go until the delalloc starts 7190 */ 7191 em->len = min(hole_len, 7192 range_start - hole_start); 7193 em->start = hole_start; 7194 em->orig_start = hole_start; 7195 /* 7196 * don't adjust block start at all, 7197 * it is fixed at EXTENT_MAP_HOLE 7198 */ 7199 em->block_start = hole_em->block_start; 7200 em->block_len = hole_len; 7201 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) 7202 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 7203 } else { 7204 em->start = range_start; 7205 em->len = found; 7206 em->orig_start = range_start; 7207 em->block_start = EXTENT_MAP_DELALLOC; 7208 em->block_len = found; 7209 } 7210 } else { 7211 return hole_em; 7212 } 7213 out: 7214 7215 free_extent_map(hole_em); 7216 if (err) { 7217 free_extent_map(em); 7218 return ERR_PTR(err); 7219 } 7220 return em; 7221 } 7222 7223 static struct extent_map *btrfs_create_dio_extent(struct inode *inode, 7224 const u64 start, 7225 const u64 len, 7226 const u64 orig_start, 7227 const u64 block_start, 7228 const u64 block_len, 7229 const u64 orig_block_len, 7230 const u64 ram_bytes, 7231 const int type) 7232 { 7233 struct extent_map *em = NULL; 7234 int ret; 7235 7236 if (type != BTRFS_ORDERED_NOCOW) { 7237 em = create_io_em(inode, start, len, orig_start, 7238 block_start, block_len, orig_block_len, 7239 ram_bytes, 7240 BTRFS_COMPRESS_NONE, /* compress_type */ 7241 type); 7242 if (IS_ERR(em)) 7243 goto out; 7244 } 7245 ret = btrfs_add_ordered_extent_dio(inode, start, block_start, 7246 len, block_len, type); 7247 if (ret) { 7248 if (em) { 7249 free_extent_map(em); 7250 btrfs_drop_extent_cache(BTRFS_I(inode), start, 7251 start + len - 1, 0); 7252 } 7253 em = ERR_PTR(ret); 7254 } 7255 out: 7256 7257 return em; 7258 } 7259 7260 static struct extent_map *btrfs_new_extent_direct(struct inode *inode, 7261 u64 start, u64 len) 7262 { 7263 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7264 struct btrfs_root *root = BTRFS_I(inode)->root; 7265 struct extent_map *em; 7266 struct btrfs_key ins; 7267 u64 alloc_hint; 7268 int ret; 7269 7270 alloc_hint = get_extent_allocation_hint(inode, start, len); 7271 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, 7272 0, alloc_hint, &ins, 1, 1); 7273 if (ret) 7274 return ERR_PTR(ret); 7275 7276 em = btrfs_create_dio_extent(inode, start, ins.offset, start, 7277 ins.objectid, ins.offset, ins.offset, 7278 ins.offset, BTRFS_ORDERED_REGULAR); 7279 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 7280 if (IS_ERR(em)) 7281 btrfs_free_reserved_extent(fs_info, ins.objectid, 7282 ins.offset, 1); 7283 7284 return em; 7285 } 7286 7287 /* 7288 * returns 1 when the nocow is safe, < 1 on error, 0 if the 7289 * block must be cow'd 7290 */ 7291 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 7292 u64 *orig_start, u64 *orig_block_len, 7293 u64 *ram_bytes) 7294 { 7295 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7296 struct btrfs_path *path; 7297 int ret; 7298 struct extent_buffer *leaf; 7299 struct btrfs_root *root = BTRFS_I(inode)->root; 7300 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7301 struct btrfs_file_extent_item *fi; 7302 struct btrfs_key key; 7303 u64 disk_bytenr; 7304 u64 backref_offset; 7305 u64 extent_end; 7306 u64 num_bytes; 7307 int slot; 7308 int found_type; 7309 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW); 7310 7311 path = btrfs_alloc_path(); 7312 if (!path) 7313 return -ENOMEM; 7314 7315 ret = btrfs_lookup_file_extent(NULL, root, path, 7316 btrfs_ino(BTRFS_I(inode)), offset, 0); 7317 if (ret < 0) 7318 goto out; 7319 7320 slot = path->slots[0]; 7321 if (ret == 1) { 7322 if (slot == 0) { 7323 /* can't find the item, must cow */ 7324 ret = 0; 7325 goto out; 7326 } 7327 slot--; 7328 } 7329 ret = 0; 7330 leaf = path->nodes[0]; 7331 btrfs_item_key_to_cpu(leaf, &key, slot); 7332 if (key.objectid != btrfs_ino(BTRFS_I(inode)) || 7333 key.type != BTRFS_EXTENT_DATA_KEY) { 7334 /* not our file or wrong item type, must cow */ 7335 goto out; 7336 } 7337 7338 if (key.offset > offset) { 7339 /* Wrong offset, must cow */ 7340 goto out; 7341 } 7342 7343 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 7344 found_type = btrfs_file_extent_type(leaf, fi); 7345 if (found_type != BTRFS_FILE_EXTENT_REG && 7346 found_type != BTRFS_FILE_EXTENT_PREALLOC) { 7347 /* not a regular extent, must cow */ 7348 goto out; 7349 } 7350 7351 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG) 7352 goto out; 7353 7354 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 7355 if (extent_end <= offset) 7356 goto out; 7357 7358 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 7359 if (disk_bytenr == 0) 7360 goto out; 7361 7362 if (btrfs_file_extent_compression(leaf, fi) || 7363 btrfs_file_extent_encryption(leaf, fi) || 7364 btrfs_file_extent_other_encoding(leaf, fi)) 7365 goto out; 7366 7367 backref_offset = btrfs_file_extent_offset(leaf, fi); 7368 7369 if (orig_start) { 7370 *orig_start = key.offset - backref_offset; 7371 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 7372 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 7373 } 7374 7375 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 7376 goto out; 7377 7378 num_bytes = min(offset + *len, extent_end) - offset; 7379 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7380 u64 range_end; 7381 7382 range_end = round_up(offset + num_bytes, 7383 root->fs_info->sectorsize) - 1; 7384 ret = test_range_bit(io_tree, offset, range_end, 7385 EXTENT_DELALLOC, 0, NULL); 7386 if (ret) { 7387 ret = -EAGAIN; 7388 goto out; 7389 } 7390 } 7391 7392 btrfs_release_path(path); 7393 7394 /* 7395 * look for other files referencing this extent, if we 7396 * find any we must cow 7397 */ 7398 7399 ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)), 7400 key.offset - backref_offset, disk_bytenr); 7401 if (ret) { 7402 ret = 0; 7403 goto out; 7404 } 7405 7406 /* 7407 * adjust disk_bytenr and num_bytes to cover just the bytes 7408 * in this extent we are about to write. If there 7409 * are any csums in that range we have to cow in order 7410 * to keep the csums correct 7411 */ 7412 disk_bytenr += backref_offset; 7413 disk_bytenr += offset - key.offset; 7414 if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes)) 7415 goto out; 7416 /* 7417 * all of the above have passed, it is safe to overwrite this extent 7418 * without cow 7419 */ 7420 *len = num_bytes; 7421 ret = 1; 7422 out: 7423 btrfs_free_path(path); 7424 return ret; 7425 } 7426 7427 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7428 struct extent_state **cached_state, int writing) 7429 { 7430 struct btrfs_ordered_extent *ordered; 7431 int ret = 0; 7432 7433 while (1) { 7434 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7435 cached_state); 7436 /* 7437 * We're concerned with the entire range that we're going to be 7438 * doing DIO to, so we need to make sure there's no ordered 7439 * extents in this range. 7440 */ 7441 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart, 7442 lockend - lockstart + 1); 7443 7444 /* 7445 * We need to make sure there are no buffered pages in this 7446 * range either, we could have raced between the invalidate in 7447 * generic_file_direct_write and locking the extent. The 7448 * invalidate needs to happen so that reads after a write do not 7449 * get stale data. 7450 */ 7451 if (!ordered && 7452 (!writing || !filemap_range_has_page(inode->i_mapping, 7453 lockstart, lockend))) 7454 break; 7455 7456 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7457 cached_state); 7458 7459 if (ordered) { 7460 /* 7461 * If we are doing a DIO read and the ordered extent we 7462 * found is for a buffered write, we can not wait for it 7463 * to complete and retry, because if we do so we can 7464 * deadlock with concurrent buffered writes on page 7465 * locks. This happens only if our DIO read covers more 7466 * than one extent map, if at this point has already 7467 * created an ordered extent for a previous extent map 7468 * and locked its range in the inode's io tree, and a 7469 * concurrent write against that previous extent map's 7470 * range and this range started (we unlock the ranges 7471 * in the io tree only when the bios complete and 7472 * buffered writes always lock pages before attempting 7473 * to lock range in the io tree). 7474 */ 7475 if (writing || 7476 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) 7477 btrfs_start_ordered_extent(inode, ordered, 1); 7478 else 7479 ret = -ENOTBLK; 7480 btrfs_put_ordered_extent(ordered); 7481 } else { 7482 /* 7483 * We could trigger writeback for this range (and wait 7484 * for it to complete) and then invalidate the pages for 7485 * this range (through invalidate_inode_pages2_range()), 7486 * but that can lead us to a deadlock with a concurrent 7487 * call to readpages() (a buffered read or a defrag call 7488 * triggered a readahead) on a page lock due to an 7489 * ordered dio extent we created before but did not have 7490 * yet a corresponding bio submitted (whence it can not 7491 * complete), which makes readpages() wait for that 7492 * ordered extent to complete while holding a lock on 7493 * that page. 7494 */ 7495 ret = -ENOTBLK; 7496 } 7497 7498 if (ret) 7499 break; 7500 7501 cond_resched(); 7502 } 7503 7504 return ret; 7505 } 7506 7507 /* The callers of this must take lock_extent() */ 7508 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, 7509 u64 orig_start, u64 block_start, 7510 u64 block_len, u64 orig_block_len, 7511 u64 ram_bytes, int compress_type, 7512 int type) 7513 { 7514 struct extent_map_tree *em_tree; 7515 struct extent_map *em; 7516 struct btrfs_root *root = BTRFS_I(inode)->root; 7517 int ret; 7518 7519 ASSERT(type == BTRFS_ORDERED_PREALLOC || 7520 type == BTRFS_ORDERED_COMPRESSED || 7521 type == BTRFS_ORDERED_NOCOW || 7522 type == BTRFS_ORDERED_REGULAR); 7523 7524 em_tree = &BTRFS_I(inode)->extent_tree; 7525 em = alloc_extent_map(); 7526 if (!em) 7527 return ERR_PTR(-ENOMEM); 7528 7529 em->start = start; 7530 em->orig_start = orig_start; 7531 em->len = len; 7532 em->block_len = block_len; 7533 em->block_start = block_start; 7534 em->bdev = root->fs_info->fs_devices->latest_bdev; 7535 em->orig_block_len = orig_block_len; 7536 em->ram_bytes = ram_bytes; 7537 em->generation = -1; 7538 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7539 if (type == BTRFS_ORDERED_PREALLOC) { 7540 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7541 } else if (type == BTRFS_ORDERED_COMPRESSED) { 7542 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 7543 em->compress_type = compress_type; 7544 } 7545 7546 do { 7547 btrfs_drop_extent_cache(BTRFS_I(inode), em->start, 7548 em->start + em->len - 1, 0); 7549 write_lock(&em_tree->lock); 7550 ret = add_extent_mapping(em_tree, em, 1); 7551 write_unlock(&em_tree->lock); 7552 /* 7553 * The caller has taken lock_extent(), who could race with us 7554 * to add em? 7555 */ 7556 } while (ret == -EEXIST); 7557 7558 if (ret) { 7559 free_extent_map(em); 7560 return ERR_PTR(ret); 7561 } 7562 7563 /* em got 2 refs now, callers needs to do free_extent_map once. */ 7564 return em; 7565 } 7566 7567 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, 7568 struct buffer_head *bh_result, int create) 7569 { 7570 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7571 struct extent_map *em; 7572 struct extent_state *cached_state = NULL; 7573 struct btrfs_dio_data *dio_data = NULL; 7574 u64 start = iblock << inode->i_blkbits; 7575 u64 lockstart, lockend; 7576 u64 len = bh_result->b_size; 7577 int unlock_bits = EXTENT_LOCKED; 7578 int ret = 0; 7579 7580 if (create) 7581 unlock_bits |= EXTENT_DIRTY; 7582 else 7583 len = min_t(u64, len, fs_info->sectorsize); 7584 7585 lockstart = start; 7586 lockend = start + len - 1; 7587 7588 if (current->journal_info) { 7589 /* 7590 * Need to pull our outstanding extents and set journal_info to NULL so 7591 * that anything that needs to check if there's a transaction doesn't get 7592 * confused. 7593 */ 7594 dio_data = current->journal_info; 7595 current->journal_info = NULL; 7596 } 7597 7598 /* 7599 * If this errors out it's because we couldn't invalidate pagecache for 7600 * this range and we need to fallback to buffered. 7601 */ 7602 if (lock_extent_direct(inode, lockstart, lockend, &cached_state, 7603 create)) { 7604 ret = -ENOTBLK; 7605 goto err; 7606 } 7607 7608 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0); 7609 if (IS_ERR(em)) { 7610 ret = PTR_ERR(em); 7611 goto unlock_err; 7612 } 7613 7614 /* 7615 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7616 * io. INLINE is special, and we could probably kludge it in here, but 7617 * it's still buffered so for safety lets just fall back to the generic 7618 * buffered path. 7619 * 7620 * For COMPRESSED we _have_ to read the entire extent in so we can 7621 * decompress it, so there will be buffering required no matter what we 7622 * do, so go ahead and fallback to buffered. 7623 * 7624 * We return -ENOTBLK because that's what makes DIO go ahead and go back 7625 * to buffered IO. Don't blame me, this is the price we pay for using 7626 * the generic code. 7627 */ 7628 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7629 em->block_start == EXTENT_MAP_INLINE) { 7630 free_extent_map(em); 7631 ret = -ENOTBLK; 7632 goto unlock_err; 7633 } 7634 7635 /* Just a good old fashioned hole, return */ 7636 if (!create && (em->block_start == EXTENT_MAP_HOLE || 7637 test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) { 7638 free_extent_map(em); 7639 goto unlock_err; 7640 } 7641 7642 /* 7643 * We don't allocate a new extent in the following cases 7644 * 7645 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7646 * existing extent. 7647 * 2) The extent is marked as PREALLOC. We're good to go here and can 7648 * just use the extent. 7649 * 7650 */ 7651 if (!create) { 7652 len = min(len, em->len - (start - em->start)); 7653 lockstart = start + len; 7654 goto unlock; 7655 } 7656 7657 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7658 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7659 em->block_start != EXTENT_MAP_HOLE)) { 7660 int type; 7661 u64 block_start, orig_start, orig_block_len, ram_bytes; 7662 7663 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7664 type = BTRFS_ORDERED_PREALLOC; 7665 else 7666 type = BTRFS_ORDERED_NOCOW; 7667 len = min(len, em->len - (start - em->start)); 7668 block_start = em->block_start + (start - em->start); 7669 7670 if (can_nocow_extent(inode, start, &len, &orig_start, 7671 &orig_block_len, &ram_bytes) == 1 && 7672 btrfs_inc_nocow_writers(fs_info, block_start)) { 7673 struct extent_map *em2; 7674 7675 em2 = btrfs_create_dio_extent(inode, start, len, 7676 orig_start, block_start, 7677 len, orig_block_len, 7678 ram_bytes, type); 7679 btrfs_dec_nocow_writers(fs_info, block_start); 7680 if (type == BTRFS_ORDERED_PREALLOC) { 7681 free_extent_map(em); 7682 em = em2; 7683 } 7684 if (em2 && IS_ERR(em2)) { 7685 ret = PTR_ERR(em2); 7686 goto unlock_err; 7687 } 7688 /* 7689 * For inode marked NODATACOW or extent marked PREALLOC, 7690 * use the existing or preallocated extent, so does not 7691 * need to adjust btrfs_space_info's bytes_may_use. 7692 */ 7693 btrfs_free_reserved_data_space_noquota(inode, 7694 start, len); 7695 goto unlock; 7696 } 7697 } 7698 7699 /* 7700 * this will cow the extent, reset the len in case we changed 7701 * it above 7702 */ 7703 len = bh_result->b_size; 7704 free_extent_map(em); 7705 em = btrfs_new_extent_direct(inode, start, len); 7706 if (IS_ERR(em)) { 7707 ret = PTR_ERR(em); 7708 goto unlock_err; 7709 } 7710 len = min(len, em->len - (start - em->start)); 7711 unlock: 7712 bh_result->b_blocknr = (em->block_start + (start - em->start)) >> 7713 inode->i_blkbits; 7714 bh_result->b_size = len; 7715 bh_result->b_bdev = em->bdev; 7716 set_buffer_mapped(bh_result); 7717 if (create) { 7718 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7719 set_buffer_new(bh_result); 7720 7721 /* 7722 * Need to update the i_size under the extent lock so buffered 7723 * readers will get the updated i_size when we unlock. 7724 */ 7725 if (!dio_data->overwrite && start + len > i_size_read(inode)) 7726 i_size_write(inode, start + len); 7727 7728 WARN_ON(dio_data->reserve < len); 7729 dio_data->reserve -= len; 7730 dio_data->unsubmitted_oe_range_end = start + len; 7731 current->journal_info = dio_data; 7732 } 7733 7734 /* 7735 * In the case of write we need to clear and unlock the entire range, 7736 * in the case of read we need to unlock only the end area that we 7737 * aren't using if there is any left over space. 7738 */ 7739 if (lockstart < lockend) { 7740 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, 7741 lockend, unlock_bits, 1, 0, 7742 &cached_state); 7743 } else { 7744 free_extent_state(cached_state); 7745 } 7746 7747 free_extent_map(em); 7748 7749 return 0; 7750 7751 unlock_err: 7752 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7753 unlock_bits, 1, 0, &cached_state); 7754 err: 7755 if (dio_data) 7756 current->journal_info = dio_data; 7757 return ret; 7758 } 7759 7760 static inline blk_status_t submit_dio_repair_bio(struct inode *inode, 7761 struct bio *bio, 7762 int mirror_num) 7763 { 7764 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7765 blk_status_t ret; 7766 7767 BUG_ON(bio_op(bio) == REQ_OP_WRITE); 7768 7769 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR); 7770 if (ret) 7771 return ret; 7772 7773 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 7774 7775 return ret; 7776 } 7777 7778 static int btrfs_check_dio_repairable(struct inode *inode, 7779 struct bio *failed_bio, 7780 struct io_failure_record *failrec, 7781 int failed_mirror) 7782 { 7783 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7784 int num_copies; 7785 7786 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len); 7787 if (num_copies == 1) { 7788 /* 7789 * we only have a single copy of the data, so don't bother with 7790 * all the retry and error correction code that follows. no 7791 * matter what the error is, it is very likely to persist. 7792 */ 7793 btrfs_debug(fs_info, 7794 "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d", 7795 num_copies, failrec->this_mirror, failed_mirror); 7796 return 0; 7797 } 7798 7799 failrec->failed_mirror = failed_mirror; 7800 failrec->this_mirror++; 7801 if (failrec->this_mirror == failed_mirror) 7802 failrec->this_mirror++; 7803 7804 if (failrec->this_mirror > num_copies) { 7805 btrfs_debug(fs_info, 7806 "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d", 7807 num_copies, failrec->this_mirror, failed_mirror); 7808 return 0; 7809 } 7810 7811 return 1; 7812 } 7813 7814 static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio, 7815 struct page *page, unsigned int pgoff, 7816 u64 start, u64 end, int failed_mirror, 7817 bio_end_io_t *repair_endio, void *repair_arg) 7818 { 7819 struct io_failure_record *failrec; 7820 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7821 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 7822 struct bio *bio; 7823 int isector; 7824 unsigned int read_mode = 0; 7825 int segs; 7826 int ret; 7827 blk_status_t status; 7828 struct bio_vec bvec; 7829 7830 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); 7831 7832 ret = btrfs_get_io_failure_record(inode, start, end, &failrec); 7833 if (ret) 7834 return errno_to_blk_status(ret); 7835 7836 ret = btrfs_check_dio_repairable(inode, failed_bio, failrec, 7837 failed_mirror); 7838 if (!ret) { 7839 free_io_failure(failure_tree, io_tree, failrec); 7840 return BLK_STS_IOERR; 7841 } 7842 7843 segs = bio_segments(failed_bio); 7844 bio_get_first_bvec(failed_bio, &bvec); 7845 if (segs > 1 || 7846 (bvec.bv_len > btrfs_inode_sectorsize(inode))) 7847 read_mode |= REQ_FAILFAST_DEV; 7848 7849 isector = start - btrfs_io_bio(failed_bio)->logical; 7850 isector >>= inode->i_sb->s_blocksize_bits; 7851 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, 7852 pgoff, isector, repair_endio, repair_arg); 7853 bio_set_op_attrs(bio, REQ_OP_READ, read_mode); 7854 7855 btrfs_debug(BTRFS_I(inode)->root->fs_info, 7856 "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d", 7857 read_mode, failrec->this_mirror, failrec->in_validation); 7858 7859 status = submit_dio_repair_bio(inode, bio, failrec->this_mirror); 7860 if (status) { 7861 free_io_failure(failure_tree, io_tree, failrec); 7862 bio_put(bio); 7863 } 7864 7865 return status; 7866 } 7867 7868 struct btrfs_retry_complete { 7869 struct completion done; 7870 struct inode *inode; 7871 u64 start; 7872 int uptodate; 7873 }; 7874 7875 static void btrfs_retry_endio_nocsum(struct bio *bio) 7876 { 7877 struct btrfs_retry_complete *done = bio->bi_private; 7878 struct inode *inode = done->inode; 7879 struct bio_vec *bvec; 7880 struct extent_io_tree *io_tree, *failure_tree; 7881 int i; 7882 7883 if (bio->bi_status) 7884 goto end; 7885 7886 ASSERT(bio->bi_vcnt == 1); 7887 io_tree = &BTRFS_I(inode)->io_tree; 7888 failure_tree = &BTRFS_I(inode)->io_failure_tree; 7889 ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(inode)); 7890 7891 done->uptodate = 1; 7892 ASSERT(!bio_flagged(bio, BIO_CLONED)); 7893 bio_for_each_segment_all(bvec, bio, i) 7894 clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, 7895 io_tree, done->start, bvec->bv_page, 7896 btrfs_ino(BTRFS_I(inode)), 0); 7897 end: 7898 complete(&done->done); 7899 bio_put(bio); 7900 } 7901 7902 static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode, 7903 struct btrfs_io_bio *io_bio) 7904 { 7905 struct btrfs_fs_info *fs_info; 7906 struct bio_vec bvec; 7907 struct bvec_iter iter; 7908 struct btrfs_retry_complete done; 7909 u64 start; 7910 unsigned int pgoff; 7911 u32 sectorsize; 7912 int nr_sectors; 7913 blk_status_t ret; 7914 blk_status_t err = BLK_STS_OK; 7915 7916 fs_info = BTRFS_I(inode)->root->fs_info; 7917 sectorsize = fs_info->sectorsize; 7918 7919 start = io_bio->logical; 7920 done.inode = inode; 7921 io_bio->bio.bi_iter = io_bio->iter; 7922 7923 bio_for_each_segment(bvec, &io_bio->bio, iter) { 7924 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 7925 pgoff = bvec.bv_offset; 7926 7927 next_block_or_try_again: 7928 done.uptodate = 0; 7929 done.start = start; 7930 init_completion(&done.done); 7931 7932 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page, 7933 pgoff, start, start + sectorsize - 1, 7934 io_bio->mirror_num, 7935 btrfs_retry_endio_nocsum, &done); 7936 if (ret) { 7937 err = ret; 7938 goto next; 7939 } 7940 7941 wait_for_completion_io(&done.done); 7942 7943 if (!done.uptodate) { 7944 /* We might have another mirror, so try again */ 7945 goto next_block_or_try_again; 7946 } 7947 7948 next: 7949 start += sectorsize; 7950 7951 nr_sectors--; 7952 if (nr_sectors) { 7953 pgoff += sectorsize; 7954 ASSERT(pgoff < PAGE_SIZE); 7955 goto next_block_or_try_again; 7956 } 7957 } 7958 7959 return err; 7960 } 7961 7962 static void btrfs_retry_endio(struct bio *bio) 7963 { 7964 struct btrfs_retry_complete *done = bio->bi_private; 7965 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7966 struct extent_io_tree *io_tree, *failure_tree; 7967 struct inode *inode = done->inode; 7968 struct bio_vec *bvec; 7969 int uptodate; 7970 int ret; 7971 int i; 7972 7973 if (bio->bi_status) 7974 goto end; 7975 7976 uptodate = 1; 7977 7978 ASSERT(bio->bi_vcnt == 1); 7979 ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(done->inode)); 7980 7981 io_tree = &BTRFS_I(inode)->io_tree; 7982 failure_tree = &BTRFS_I(inode)->io_failure_tree; 7983 7984 ASSERT(!bio_flagged(bio, BIO_CLONED)); 7985 bio_for_each_segment_all(bvec, bio, i) { 7986 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page, 7987 bvec->bv_offset, done->start, 7988 bvec->bv_len); 7989 if (!ret) 7990 clean_io_failure(BTRFS_I(inode)->root->fs_info, 7991 failure_tree, io_tree, done->start, 7992 bvec->bv_page, 7993 btrfs_ino(BTRFS_I(inode)), 7994 bvec->bv_offset); 7995 else 7996 uptodate = 0; 7997 } 7998 7999 done->uptodate = uptodate; 8000 end: 8001 complete(&done->done); 8002 bio_put(bio); 8003 } 8004 8005 static blk_status_t __btrfs_subio_endio_read(struct inode *inode, 8006 struct btrfs_io_bio *io_bio, blk_status_t err) 8007 { 8008 struct btrfs_fs_info *fs_info; 8009 struct bio_vec bvec; 8010 struct bvec_iter iter; 8011 struct btrfs_retry_complete done; 8012 u64 start; 8013 u64 offset = 0; 8014 u32 sectorsize; 8015 int nr_sectors; 8016 unsigned int pgoff; 8017 int csum_pos; 8018 bool uptodate = (err == 0); 8019 int ret; 8020 blk_status_t status; 8021 8022 fs_info = BTRFS_I(inode)->root->fs_info; 8023 sectorsize = fs_info->sectorsize; 8024 8025 err = BLK_STS_OK; 8026 start = io_bio->logical; 8027 done.inode = inode; 8028 io_bio->bio.bi_iter = io_bio->iter; 8029 8030 bio_for_each_segment(bvec, &io_bio->bio, iter) { 8031 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 8032 8033 pgoff = bvec.bv_offset; 8034 next_block: 8035 if (uptodate) { 8036 csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset); 8037 ret = __readpage_endio_check(inode, io_bio, csum_pos, 8038 bvec.bv_page, pgoff, start, sectorsize); 8039 if (likely(!ret)) 8040 goto next; 8041 } 8042 try_again: 8043 done.uptodate = 0; 8044 done.start = start; 8045 init_completion(&done.done); 8046 8047 status = dio_read_error(inode, &io_bio->bio, bvec.bv_page, 8048 pgoff, start, start + sectorsize - 1, 8049 io_bio->mirror_num, btrfs_retry_endio, 8050 &done); 8051 if (status) { 8052 err = status; 8053 goto next; 8054 } 8055 8056 wait_for_completion_io(&done.done); 8057 8058 if (!done.uptodate) { 8059 /* We might have another mirror, so try again */ 8060 goto try_again; 8061 } 8062 next: 8063 offset += sectorsize; 8064 start += sectorsize; 8065 8066 ASSERT(nr_sectors); 8067 8068 nr_sectors--; 8069 if (nr_sectors) { 8070 pgoff += sectorsize; 8071 ASSERT(pgoff < PAGE_SIZE); 8072 goto next_block; 8073 } 8074 } 8075 8076 return err; 8077 } 8078 8079 static blk_status_t btrfs_subio_endio_read(struct inode *inode, 8080 struct btrfs_io_bio *io_bio, blk_status_t err) 8081 { 8082 bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 8083 8084 if (skip_csum) { 8085 if (unlikely(err)) 8086 return __btrfs_correct_data_nocsum(inode, io_bio); 8087 else 8088 return BLK_STS_OK; 8089 } else { 8090 return __btrfs_subio_endio_read(inode, io_bio, err); 8091 } 8092 } 8093 8094 static void btrfs_endio_direct_read(struct bio *bio) 8095 { 8096 struct btrfs_dio_private *dip = bio->bi_private; 8097 struct inode *inode = dip->inode; 8098 struct bio *dio_bio; 8099 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8100 blk_status_t err = bio->bi_status; 8101 8102 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) 8103 err = btrfs_subio_endio_read(inode, io_bio, err); 8104 8105 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset, 8106 dip->logical_offset + dip->bytes - 1); 8107 dio_bio = dip->dio_bio; 8108 8109 kfree(dip); 8110 8111 dio_bio->bi_status = err; 8112 dio_end_io(dio_bio); 8113 8114 if (io_bio->end_io) 8115 io_bio->end_io(io_bio, blk_status_to_errno(err)); 8116 bio_put(bio); 8117 } 8118 8119 static void __endio_write_update_ordered(struct inode *inode, 8120 const u64 offset, const u64 bytes, 8121 const bool uptodate) 8122 { 8123 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8124 struct btrfs_ordered_extent *ordered = NULL; 8125 struct btrfs_workqueue *wq; 8126 btrfs_work_func_t func; 8127 u64 ordered_offset = offset; 8128 u64 ordered_bytes = bytes; 8129 u64 last_offset; 8130 int ret; 8131 8132 if (btrfs_is_free_space_inode(BTRFS_I(inode))) { 8133 wq = fs_info->endio_freespace_worker; 8134 func = btrfs_freespace_write_helper; 8135 } else { 8136 wq = fs_info->endio_write_workers; 8137 func = btrfs_endio_write_helper; 8138 } 8139 8140 again: 8141 last_offset = ordered_offset; 8142 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, 8143 &ordered_offset, 8144 ordered_bytes, 8145 uptodate); 8146 if (!ret) 8147 goto out_test; 8148 8149 btrfs_init_work(&ordered->work, func, finish_ordered_fn, NULL, NULL); 8150 btrfs_queue_work(wq, &ordered->work); 8151 out_test: 8152 /* 8153 * If btrfs_dec_test_ordered_pending does not find any ordered extent 8154 * in the range, we can exit. 8155 */ 8156 if (ordered_offset == last_offset) 8157 return; 8158 /* 8159 * our bio might span multiple ordered extents. If we haven't 8160 * completed the accounting for the whole dio, go back and try again 8161 */ 8162 if (ordered_offset < offset + bytes) { 8163 ordered_bytes = offset + bytes - ordered_offset; 8164 ordered = NULL; 8165 goto again; 8166 } 8167 } 8168 8169 static void btrfs_endio_direct_write(struct bio *bio) 8170 { 8171 struct btrfs_dio_private *dip = bio->bi_private; 8172 struct bio *dio_bio = dip->dio_bio; 8173 8174 __endio_write_update_ordered(dip->inode, dip->logical_offset, 8175 dip->bytes, !bio->bi_status); 8176 8177 kfree(dip); 8178 8179 dio_bio->bi_status = bio->bi_status; 8180 dio_end_io(dio_bio); 8181 bio_put(bio); 8182 } 8183 8184 static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data, 8185 struct bio *bio, u64 offset) 8186 { 8187 struct inode *inode = private_data; 8188 blk_status_t ret; 8189 ret = btrfs_csum_one_bio(inode, bio, offset, 1); 8190 BUG_ON(ret); /* -ENOMEM */ 8191 return 0; 8192 } 8193 8194 static void btrfs_end_dio_bio(struct bio *bio) 8195 { 8196 struct btrfs_dio_private *dip = bio->bi_private; 8197 blk_status_t err = bio->bi_status; 8198 8199 if (err) 8200 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 8201 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", 8202 btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio), 8203 bio->bi_opf, 8204 (unsigned long long)bio->bi_iter.bi_sector, 8205 bio->bi_iter.bi_size, err); 8206 8207 if (dip->subio_endio) 8208 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err); 8209 8210 if (err) { 8211 /* 8212 * We want to perceive the errors flag being set before 8213 * decrementing the reference count. We don't need a barrier 8214 * since atomic operations with a return value are fully 8215 * ordered as per atomic_t.txt 8216 */ 8217 dip->errors = 1; 8218 } 8219 8220 /* if there are more bios still pending for this dio, just exit */ 8221 if (!atomic_dec_and_test(&dip->pending_bios)) 8222 goto out; 8223 8224 if (dip->errors) { 8225 bio_io_error(dip->orig_bio); 8226 } else { 8227 dip->dio_bio->bi_status = BLK_STS_OK; 8228 bio_endio(dip->orig_bio); 8229 } 8230 out: 8231 bio_put(bio); 8232 } 8233 8234 static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode, 8235 struct btrfs_dio_private *dip, 8236 struct bio *bio, 8237 u64 file_offset) 8238 { 8239 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8240 struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio); 8241 blk_status_t ret; 8242 8243 /* 8244 * We load all the csum data we need when we submit 8245 * the first bio to reduce the csum tree search and 8246 * contention. 8247 */ 8248 if (dip->logical_offset == file_offset) { 8249 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio, 8250 file_offset); 8251 if (ret) 8252 return ret; 8253 } 8254 8255 if (bio == dip->orig_bio) 8256 return 0; 8257 8258 file_offset -= dip->logical_offset; 8259 file_offset >>= inode->i_sb->s_blocksize_bits; 8260 io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset); 8261 8262 return 0; 8263 } 8264 8265 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, 8266 struct inode *inode, u64 file_offset, int async_submit) 8267 { 8268 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8269 struct btrfs_dio_private *dip = bio->bi_private; 8270 bool write = bio_op(bio) == REQ_OP_WRITE; 8271 blk_status_t ret; 8272 8273 /* Check btrfs_submit_bio_hook() for rules about async submit. */ 8274 if (async_submit) 8275 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers); 8276 8277 if (!write) { 8278 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 8279 if (ret) 8280 goto err; 8281 } 8282 8283 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 8284 goto map; 8285 8286 if (write && async_submit) { 8287 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0, 8288 file_offset, inode, 8289 btrfs_submit_bio_start_direct_io, 8290 btrfs_submit_bio_done); 8291 goto err; 8292 } else if (write) { 8293 /* 8294 * If we aren't doing async submit, calculate the csum of the 8295 * bio now. 8296 */ 8297 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1); 8298 if (ret) 8299 goto err; 8300 } else { 8301 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio, 8302 file_offset); 8303 if (ret) 8304 goto err; 8305 } 8306 map: 8307 ret = btrfs_map_bio(fs_info, bio, 0, 0); 8308 err: 8309 return ret; 8310 } 8311 8312 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) 8313 { 8314 struct inode *inode = dip->inode; 8315 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8316 struct bio *bio; 8317 struct bio *orig_bio = dip->orig_bio; 8318 u64 start_sector = orig_bio->bi_iter.bi_sector; 8319 u64 file_offset = dip->logical_offset; 8320 u64 map_length; 8321 int async_submit = 0; 8322 u64 submit_len; 8323 int clone_offset = 0; 8324 int clone_len; 8325 int ret; 8326 blk_status_t status; 8327 8328 map_length = orig_bio->bi_iter.bi_size; 8329 submit_len = map_length; 8330 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9, 8331 &map_length, NULL, 0); 8332 if (ret) 8333 return -EIO; 8334 8335 if (map_length >= submit_len) { 8336 bio = orig_bio; 8337 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED; 8338 goto submit; 8339 } 8340 8341 /* async crcs make it difficult to collect full stripe writes. */ 8342 if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK) 8343 async_submit = 0; 8344 else 8345 async_submit = 1; 8346 8347 /* bio split */ 8348 ASSERT(map_length <= INT_MAX); 8349 atomic_inc(&dip->pending_bios); 8350 do { 8351 clone_len = min_t(int, submit_len, map_length); 8352 8353 /* 8354 * This will never fail as it's passing GPF_NOFS and 8355 * the allocation is backed by btrfs_bioset. 8356 */ 8357 bio = btrfs_bio_clone_partial(orig_bio, clone_offset, 8358 clone_len); 8359 bio->bi_private = dip; 8360 bio->bi_end_io = btrfs_end_dio_bio; 8361 btrfs_io_bio(bio)->logical = file_offset; 8362 8363 ASSERT(submit_len >= clone_len); 8364 submit_len -= clone_len; 8365 if (submit_len == 0) 8366 break; 8367 8368 /* 8369 * Increase the count before we submit the bio so we know 8370 * the end IO handler won't happen before we increase the 8371 * count. Otherwise, the dip might get freed before we're 8372 * done setting it up. 8373 */ 8374 atomic_inc(&dip->pending_bios); 8375 8376 status = btrfs_submit_dio_bio(bio, inode, file_offset, 8377 async_submit); 8378 if (status) { 8379 bio_put(bio); 8380 atomic_dec(&dip->pending_bios); 8381 goto out_err; 8382 } 8383 8384 clone_offset += clone_len; 8385 start_sector += clone_len >> 9; 8386 file_offset += clone_len; 8387 8388 map_length = submit_len; 8389 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), 8390 start_sector << 9, &map_length, NULL, 0); 8391 if (ret) 8392 goto out_err; 8393 } while (submit_len > 0); 8394 8395 submit: 8396 status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit); 8397 if (!status) 8398 return 0; 8399 8400 bio_put(bio); 8401 out_err: 8402 dip->errors = 1; 8403 /* 8404 * Before atomic variable goto zero, we must make sure dip->errors is 8405 * perceived to be set. This ordering is ensured by the fact that an 8406 * atomic operations with a return value are fully ordered as per 8407 * atomic_t.txt 8408 */ 8409 if (atomic_dec_and_test(&dip->pending_bios)) 8410 bio_io_error(dip->orig_bio); 8411 8412 /* bio_end_io() will handle error, so we needn't return it */ 8413 return 0; 8414 } 8415 8416 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, 8417 loff_t file_offset) 8418 { 8419 struct btrfs_dio_private *dip = NULL; 8420 struct bio *bio = NULL; 8421 struct btrfs_io_bio *io_bio; 8422 bool write = (bio_op(dio_bio) == REQ_OP_WRITE); 8423 int ret = 0; 8424 8425 bio = btrfs_bio_clone(dio_bio); 8426 8427 dip = kzalloc(sizeof(*dip), GFP_NOFS); 8428 if (!dip) { 8429 ret = -ENOMEM; 8430 goto free_ordered; 8431 } 8432 8433 dip->private = dio_bio->bi_private; 8434 dip->inode = inode; 8435 dip->logical_offset = file_offset; 8436 dip->bytes = dio_bio->bi_iter.bi_size; 8437 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; 8438 bio->bi_private = dip; 8439 dip->orig_bio = bio; 8440 dip->dio_bio = dio_bio; 8441 atomic_set(&dip->pending_bios, 0); 8442 io_bio = btrfs_io_bio(bio); 8443 io_bio->logical = file_offset; 8444 8445 if (write) { 8446 bio->bi_end_io = btrfs_endio_direct_write; 8447 } else { 8448 bio->bi_end_io = btrfs_endio_direct_read; 8449 dip->subio_endio = btrfs_subio_endio_read; 8450 } 8451 8452 /* 8453 * Reset the range for unsubmitted ordered extents (to a 0 length range) 8454 * even if we fail to submit a bio, because in such case we do the 8455 * corresponding error handling below and it must not be done a second 8456 * time by btrfs_direct_IO(). 8457 */ 8458 if (write) { 8459 struct btrfs_dio_data *dio_data = current->journal_info; 8460 8461 dio_data->unsubmitted_oe_range_end = dip->logical_offset + 8462 dip->bytes; 8463 dio_data->unsubmitted_oe_range_start = 8464 dio_data->unsubmitted_oe_range_end; 8465 } 8466 8467 ret = btrfs_submit_direct_hook(dip); 8468 if (!ret) 8469 return; 8470 8471 if (io_bio->end_io) 8472 io_bio->end_io(io_bio, ret); 8473 8474 free_ordered: 8475 /* 8476 * If we arrived here it means either we failed to submit the dip 8477 * or we either failed to clone the dio_bio or failed to allocate the 8478 * dip. If we cloned the dio_bio and allocated the dip, we can just 8479 * call bio_endio against our io_bio so that we get proper resource 8480 * cleanup if we fail to submit the dip, otherwise, we must do the 8481 * same as btrfs_endio_direct_[write|read] because we can't call these 8482 * callbacks - they require an allocated dip and a clone of dio_bio. 8483 */ 8484 if (bio && dip) { 8485 bio_io_error(bio); 8486 /* 8487 * The end io callbacks free our dip, do the final put on bio 8488 * and all the cleanup and final put for dio_bio (through 8489 * dio_end_io()). 8490 */ 8491 dip = NULL; 8492 bio = NULL; 8493 } else { 8494 if (write) 8495 __endio_write_update_ordered(inode, 8496 file_offset, 8497 dio_bio->bi_iter.bi_size, 8498 false); 8499 else 8500 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, 8501 file_offset + dio_bio->bi_iter.bi_size - 1); 8502 8503 dio_bio->bi_status = BLK_STS_IOERR; 8504 /* 8505 * Releases and cleans up our dio_bio, no need to bio_put() 8506 * nor bio_endio()/bio_io_error() against dio_bio. 8507 */ 8508 dio_end_io(dio_bio); 8509 } 8510 if (bio) 8511 bio_put(bio); 8512 kfree(dip); 8513 } 8514 8515 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, 8516 const struct iov_iter *iter, loff_t offset) 8517 { 8518 int seg; 8519 int i; 8520 unsigned int blocksize_mask = fs_info->sectorsize - 1; 8521 ssize_t retval = -EINVAL; 8522 8523 if (offset & blocksize_mask) 8524 goto out; 8525 8526 if (iov_iter_alignment(iter) & blocksize_mask) 8527 goto out; 8528 8529 /* If this is a write we don't need to check anymore */ 8530 if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter)) 8531 return 0; 8532 /* 8533 * Check to make sure we don't have duplicate iov_base's in this 8534 * iovec, if so return EINVAL, otherwise we'll get csum errors 8535 * when reading back. 8536 */ 8537 for (seg = 0; seg < iter->nr_segs; seg++) { 8538 for (i = seg + 1; i < iter->nr_segs; i++) { 8539 if (iter->iov[seg].iov_base == iter->iov[i].iov_base) 8540 goto out; 8541 } 8542 } 8543 retval = 0; 8544 out: 8545 return retval; 8546 } 8547 8548 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 8549 { 8550 struct file *file = iocb->ki_filp; 8551 struct inode *inode = file->f_mapping->host; 8552 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8553 struct btrfs_dio_data dio_data = { 0 }; 8554 struct extent_changeset *data_reserved = NULL; 8555 loff_t offset = iocb->ki_pos; 8556 size_t count = 0; 8557 int flags = 0; 8558 bool wakeup = true; 8559 bool relock = false; 8560 ssize_t ret; 8561 8562 if (check_direct_IO(fs_info, iter, offset)) 8563 return 0; 8564 8565 inode_dio_begin(inode); 8566 8567 /* 8568 * The generic stuff only does filemap_write_and_wait_range, which 8569 * isn't enough if we've written compressed pages to this area, so 8570 * we need to flush the dirty pages again to make absolutely sure 8571 * that any outstanding dirty pages are on disk. 8572 */ 8573 count = iov_iter_count(iter); 8574 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 8575 &BTRFS_I(inode)->runtime_flags)) 8576 filemap_fdatawrite_range(inode->i_mapping, offset, 8577 offset + count - 1); 8578 8579 if (iov_iter_rw(iter) == WRITE) { 8580 /* 8581 * If the write DIO is beyond the EOF, we need update 8582 * the isize, but it is protected by i_mutex. So we can 8583 * not unlock the i_mutex at this case. 8584 */ 8585 if (offset + count <= inode->i_size) { 8586 dio_data.overwrite = 1; 8587 inode_unlock(inode); 8588 relock = true; 8589 } else if (iocb->ki_flags & IOCB_NOWAIT) { 8590 ret = -EAGAIN; 8591 goto out; 8592 } 8593 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, 8594 offset, count); 8595 if (ret) 8596 goto out; 8597 8598 /* 8599 * We need to know how many extents we reserved so that we can 8600 * do the accounting properly if we go over the number we 8601 * originally calculated. Abuse current->journal_info for this. 8602 */ 8603 dio_data.reserve = round_up(count, 8604 fs_info->sectorsize); 8605 dio_data.unsubmitted_oe_range_start = (u64)offset; 8606 dio_data.unsubmitted_oe_range_end = (u64)offset; 8607 current->journal_info = &dio_data; 8608 down_read(&BTRFS_I(inode)->dio_sem); 8609 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK, 8610 &BTRFS_I(inode)->runtime_flags)) { 8611 inode_dio_end(inode); 8612 flags = DIO_LOCKING | DIO_SKIP_HOLES; 8613 wakeup = false; 8614 } 8615 8616 ret = __blockdev_direct_IO(iocb, inode, 8617 fs_info->fs_devices->latest_bdev, 8618 iter, btrfs_get_blocks_direct, NULL, 8619 btrfs_submit_direct, flags); 8620 if (iov_iter_rw(iter) == WRITE) { 8621 up_read(&BTRFS_I(inode)->dio_sem); 8622 current->journal_info = NULL; 8623 if (ret < 0 && ret != -EIOCBQUEUED) { 8624 if (dio_data.reserve) 8625 btrfs_delalloc_release_space(inode, data_reserved, 8626 offset, dio_data.reserve, true); 8627 /* 8628 * On error we might have left some ordered extents 8629 * without submitting corresponding bios for them, so 8630 * cleanup them up to avoid other tasks getting them 8631 * and waiting for them to complete forever. 8632 */ 8633 if (dio_data.unsubmitted_oe_range_start < 8634 dio_data.unsubmitted_oe_range_end) 8635 __endio_write_update_ordered(inode, 8636 dio_data.unsubmitted_oe_range_start, 8637 dio_data.unsubmitted_oe_range_end - 8638 dio_data.unsubmitted_oe_range_start, 8639 false); 8640 } else if (ret >= 0 && (size_t)ret < count) 8641 btrfs_delalloc_release_space(inode, data_reserved, 8642 offset, count - (size_t)ret, true); 8643 btrfs_delalloc_release_extents(BTRFS_I(inode), count, false); 8644 } 8645 out: 8646 if (wakeup) 8647 inode_dio_end(inode); 8648 if (relock) 8649 inode_lock(inode); 8650 8651 extent_changeset_free(data_reserved); 8652 return ret; 8653 } 8654 8655 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC) 8656 8657 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 8658 __u64 start, __u64 len) 8659 { 8660 int ret; 8661 8662 ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS); 8663 if (ret) 8664 return ret; 8665 8666 return extent_fiemap(inode, fieinfo, start, len); 8667 } 8668 8669 int btrfs_readpage(struct file *file, struct page *page) 8670 { 8671 struct extent_io_tree *tree; 8672 tree = &BTRFS_I(page->mapping->host)->io_tree; 8673 return extent_read_full_page(tree, page, btrfs_get_extent, 0); 8674 } 8675 8676 static int btrfs_writepage(struct page *page, struct writeback_control *wbc) 8677 { 8678 struct inode *inode = page->mapping->host; 8679 int ret; 8680 8681 if (current->flags & PF_MEMALLOC) { 8682 redirty_page_for_writepage(wbc, page); 8683 unlock_page(page); 8684 return 0; 8685 } 8686 8687 /* 8688 * If we are under memory pressure we will call this directly from the 8689 * VM, we need to make sure we have the inode referenced for the ordered 8690 * extent. If not just return like we didn't do anything. 8691 */ 8692 if (!igrab(inode)) { 8693 redirty_page_for_writepage(wbc, page); 8694 return AOP_WRITEPAGE_ACTIVATE; 8695 } 8696 ret = extent_write_full_page(page, wbc); 8697 btrfs_add_delayed_iput(inode); 8698 return ret; 8699 } 8700 8701 static int btrfs_writepages(struct address_space *mapping, 8702 struct writeback_control *wbc) 8703 { 8704 struct extent_io_tree *tree; 8705 8706 tree = &BTRFS_I(mapping->host)->io_tree; 8707 return extent_writepages(tree, mapping, wbc); 8708 } 8709 8710 static int 8711 btrfs_readpages(struct file *file, struct address_space *mapping, 8712 struct list_head *pages, unsigned nr_pages) 8713 { 8714 struct extent_io_tree *tree; 8715 tree = &BTRFS_I(mapping->host)->io_tree; 8716 return extent_readpages(tree, mapping, pages, nr_pages); 8717 } 8718 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8719 { 8720 struct extent_io_tree *tree; 8721 struct extent_map_tree *map; 8722 int ret; 8723 8724 tree = &BTRFS_I(page->mapping->host)->io_tree; 8725 map = &BTRFS_I(page->mapping->host)->extent_tree; 8726 ret = try_release_extent_mapping(map, tree, page, gfp_flags); 8727 if (ret == 1) { 8728 ClearPagePrivate(page); 8729 set_page_private(page, 0); 8730 put_page(page); 8731 } 8732 return ret; 8733 } 8734 8735 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8736 { 8737 if (PageWriteback(page) || PageDirty(page)) 8738 return 0; 8739 return __btrfs_releasepage(page, gfp_flags); 8740 } 8741 8742 static void btrfs_invalidatepage(struct page *page, unsigned int offset, 8743 unsigned int length) 8744 { 8745 struct inode *inode = page->mapping->host; 8746 struct extent_io_tree *tree; 8747 struct btrfs_ordered_extent *ordered; 8748 struct extent_state *cached_state = NULL; 8749 u64 page_start = page_offset(page); 8750 u64 page_end = page_start + PAGE_SIZE - 1; 8751 u64 start; 8752 u64 end; 8753 int inode_evicting = inode->i_state & I_FREEING; 8754 8755 /* 8756 * we have the page locked, so new writeback can't start, 8757 * and the dirty bit won't be cleared while we are here. 8758 * 8759 * Wait for IO on this page so that we can safely clear 8760 * the PagePrivate2 bit and do ordered accounting 8761 */ 8762 wait_on_page_writeback(page); 8763 8764 tree = &BTRFS_I(inode)->io_tree; 8765 if (offset) { 8766 btrfs_releasepage(page, GFP_NOFS); 8767 return; 8768 } 8769 8770 if (!inode_evicting) 8771 lock_extent_bits(tree, page_start, page_end, &cached_state); 8772 again: 8773 start = page_start; 8774 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start, 8775 page_end - start + 1); 8776 if (ordered) { 8777 end = min(page_end, ordered->file_offset + ordered->len - 1); 8778 /* 8779 * IO on this page will never be started, so we need 8780 * to account for any ordered extents now 8781 */ 8782 if (!inode_evicting) 8783 clear_extent_bit(tree, start, end, 8784 EXTENT_DIRTY | EXTENT_DELALLOC | 8785 EXTENT_DELALLOC_NEW | 8786 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 8787 EXTENT_DEFRAG, 1, 0, &cached_state); 8788 /* 8789 * whoever cleared the private bit is responsible 8790 * for the finish_ordered_io 8791 */ 8792 if (TestClearPagePrivate2(page)) { 8793 struct btrfs_ordered_inode_tree *tree; 8794 u64 new_len; 8795 8796 tree = &BTRFS_I(inode)->ordered_tree; 8797 8798 spin_lock_irq(&tree->lock); 8799 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 8800 new_len = start - ordered->file_offset; 8801 if (new_len < ordered->truncated_len) 8802 ordered->truncated_len = new_len; 8803 spin_unlock_irq(&tree->lock); 8804 8805 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8806 start, 8807 end - start + 1, 1)) 8808 btrfs_finish_ordered_io(ordered); 8809 } 8810 btrfs_put_ordered_extent(ordered); 8811 if (!inode_evicting) { 8812 cached_state = NULL; 8813 lock_extent_bits(tree, start, end, 8814 &cached_state); 8815 } 8816 8817 start = end + 1; 8818 if (start < page_end) 8819 goto again; 8820 } 8821 8822 /* 8823 * Qgroup reserved space handler 8824 * Page here will be either 8825 * 1) Already written to disk 8826 * In this case, its reserved space is released from data rsv map 8827 * and will be freed by delayed_ref handler finally. 8828 * So even we call qgroup_free_data(), it won't decrease reserved 8829 * space. 8830 * 2) Not written to disk 8831 * This means the reserved space should be freed here. However, 8832 * if a truncate invalidates the page (by clearing PageDirty) 8833 * and the page is accounted for while allocating extent 8834 * in btrfs_check_data_free_space() we let delayed_ref to 8835 * free the entire extent. 8836 */ 8837 if (PageDirty(page)) 8838 btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE); 8839 if (!inode_evicting) { 8840 clear_extent_bit(tree, page_start, page_end, 8841 EXTENT_LOCKED | EXTENT_DIRTY | 8842 EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 8843 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1, 8844 &cached_state); 8845 8846 __btrfs_releasepage(page, GFP_NOFS); 8847 } 8848 8849 ClearPageChecked(page); 8850 if (PagePrivate(page)) { 8851 ClearPagePrivate(page); 8852 set_page_private(page, 0); 8853 put_page(page); 8854 } 8855 } 8856 8857 /* 8858 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8859 * called from a page fault handler when a page is first dirtied. Hence we must 8860 * be careful to check for EOF conditions here. We set the page up correctly 8861 * for a written page which means we get ENOSPC checking when writing into 8862 * holes and correct delalloc and unwritten extent mapping on filesystems that 8863 * support these features. 8864 * 8865 * We are not allowed to take the i_mutex here so we have to play games to 8866 * protect against truncate races as the page could now be beyond EOF. Because 8867 * vmtruncate() writes the inode size before removing pages, once we have the 8868 * page lock we can determine safely if the page is beyond EOF. If it is not 8869 * beyond EOF, then the page is guaranteed safe against truncation until we 8870 * unlock the page. 8871 */ 8872 int btrfs_page_mkwrite(struct vm_fault *vmf) 8873 { 8874 struct page *page = vmf->page; 8875 struct inode *inode = file_inode(vmf->vma->vm_file); 8876 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8877 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8878 struct btrfs_ordered_extent *ordered; 8879 struct extent_state *cached_state = NULL; 8880 struct extent_changeset *data_reserved = NULL; 8881 char *kaddr; 8882 unsigned long zero_start; 8883 loff_t size; 8884 int ret; 8885 int reserved = 0; 8886 u64 reserved_space; 8887 u64 page_start; 8888 u64 page_end; 8889 u64 end; 8890 8891 reserved_space = PAGE_SIZE; 8892 8893 sb_start_pagefault(inode->i_sb); 8894 page_start = page_offset(page); 8895 page_end = page_start + PAGE_SIZE - 1; 8896 end = page_end; 8897 8898 /* 8899 * Reserving delalloc space after obtaining the page lock can lead to 8900 * deadlock. For example, if a dirty page is locked by this function 8901 * and the call to btrfs_delalloc_reserve_space() ends up triggering 8902 * dirty page write out, then the btrfs_writepage() function could 8903 * end up waiting indefinitely to get a lock on the page currently 8904 * being processed by btrfs_page_mkwrite() function. 8905 */ 8906 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 8907 reserved_space); 8908 if (!ret) { 8909 ret = file_update_time(vmf->vma->vm_file); 8910 reserved = 1; 8911 } 8912 if (ret) { 8913 if (ret == -ENOMEM) 8914 ret = VM_FAULT_OOM; 8915 else /* -ENOSPC, -EIO, etc */ 8916 ret = VM_FAULT_SIGBUS; 8917 if (reserved) 8918 goto out; 8919 goto out_noreserve; 8920 } 8921 8922 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 8923 again: 8924 lock_page(page); 8925 size = i_size_read(inode); 8926 8927 if ((page->mapping != inode->i_mapping) || 8928 (page_start >= size)) { 8929 /* page got truncated out from underneath us */ 8930 goto out_unlock; 8931 } 8932 wait_on_page_writeback(page); 8933 8934 lock_extent_bits(io_tree, page_start, page_end, &cached_state); 8935 set_page_extent_mapped(page); 8936 8937 /* 8938 * we can't set the delalloc bits if there are pending ordered 8939 * extents. Drop our locks and wait for them to finish 8940 */ 8941 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 8942 PAGE_SIZE); 8943 if (ordered) { 8944 unlock_extent_cached(io_tree, page_start, page_end, 8945 &cached_state); 8946 unlock_page(page); 8947 btrfs_start_ordered_extent(inode, ordered, 1); 8948 btrfs_put_ordered_extent(ordered); 8949 goto again; 8950 } 8951 8952 if (page->index == ((size - 1) >> PAGE_SHIFT)) { 8953 reserved_space = round_up(size - page_start, 8954 fs_info->sectorsize); 8955 if (reserved_space < PAGE_SIZE) { 8956 end = page_start + reserved_space - 1; 8957 btrfs_delalloc_release_space(inode, data_reserved, 8958 page_start, PAGE_SIZE - reserved_space, 8959 true); 8960 } 8961 } 8962 8963 /* 8964 * page_mkwrite gets called when the page is firstly dirtied after it's 8965 * faulted in, but write(2) could also dirty a page and set delalloc 8966 * bits, thus in this case for space account reason, we still need to 8967 * clear any delalloc bits within this page range since we have to 8968 * reserve data&meta space before lock_page() (see above comments). 8969 */ 8970 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, 8971 EXTENT_DIRTY | EXTENT_DELALLOC | 8972 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 8973 0, 0, &cached_state); 8974 8975 ret = btrfs_set_extent_delalloc(inode, page_start, end, 0, 8976 &cached_state, 0); 8977 if (ret) { 8978 unlock_extent_cached(io_tree, page_start, page_end, 8979 &cached_state); 8980 ret = VM_FAULT_SIGBUS; 8981 goto out_unlock; 8982 } 8983 ret = 0; 8984 8985 /* page is wholly or partially inside EOF */ 8986 if (page_start + PAGE_SIZE > size) 8987 zero_start = size & ~PAGE_MASK; 8988 else 8989 zero_start = PAGE_SIZE; 8990 8991 if (zero_start != PAGE_SIZE) { 8992 kaddr = kmap(page); 8993 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start); 8994 flush_dcache_page(page); 8995 kunmap(page); 8996 } 8997 ClearPageChecked(page); 8998 set_page_dirty(page); 8999 SetPageUptodate(page); 9000 9001 BTRFS_I(inode)->last_trans = fs_info->generation; 9002 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; 9003 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; 9004 9005 unlock_extent_cached(io_tree, page_start, page_end, &cached_state); 9006 9007 out_unlock: 9008 if (!ret) { 9009 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, true); 9010 sb_end_pagefault(inode->i_sb); 9011 extent_changeset_free(data_reserved); 9012 return VM_FAULT_LOCKED; 9013 } 9014 unlock_page(page); 9015 out: 9016 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, (ret != 0)); 9017 btrfs_delalloc_release_space(inode, data_reserved, page_start, 9018 reserved_space, (ret != 0)); 9019 out_noreserve: 9020 sb_end_pagefault(inode->i_sb); 9021 extent_changeset_free(data_reserved); 9022 return ret; 9023 } 9024 9025 static int btrfs_truncate(struct inode *inode, bool skip_writeback) 9026 { 9027 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9028 struct btrfs_root *root = BTRFS_I(inode)->root; 9029 struct btrfs_block_rsv *rsv; 9030 int ret = 0; 9031 int err = 0; 9032 struct btrfs_trans_handle *trans; 9033 u64 mask = fs_info->sectorsize - 1; 9034 u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 9035 9036 if (!skip_writeback) { 9037 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), 9038 (u64)-1); 9039 if (ret) 9040 return ret; 9041 } 9042 9043 /* 9044 * Yes ladies and gentlemen, this is indeed ugly. The fact is we have 9045 * 3 things going on here 9046 * 9047 * 1) We need to reserve space for our orphan item and the space to 9048 * delete our orphan item. Lord knows we don't want to have a dangling 9049 * orphan item because we didn't reserve space to remove it. 9050 * 9051 * 2) We need to reserve space to update our inode. 9052 * 9053 * 3) We need to have something to cache all the space that is going to 9054 * be free'd up by the truncate operation, but also have some slack 9055 * space reserved in case it uses space during the truncate (thank you 9056 * very much snapshotting). 9057 * 9058 * And we need these to all be separate. The fact is we can use a lot of 9059 * space doing the truncate, and we have no earthly idea how much space 9060 * we will use, so we need the truncate reservation to be separate so it 9061 * doesn't end up using space reserved for updating the inode or 9062 * removing the orphan item. We also need to be able to stop the 9063 * transaction and start a new one, which means we need to be able to 9064 * update the inode several times, and we have no idea of knowing how 9065 * many times that will be, so we can't just reserve 1 item for the 9066 * entirety of the operation, so that has to be done separately as well. 9067 * Then there is the orphan item, which does indeed need to be held on 9068 * to for the whole operation, and we need nobody to touch this reserved 9069 * space except the orphan code. 9070 * 9071 * So that leaves us with 9072 * 9073 * 1) root->orphan_block_rsv - for the orphan deletion. 9074 * 2) rsv - for the truncate reservation, which we will steal from the 9075 * transaction reservation. 9076 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for 9077 * updating the inode. 9078 */ 9079 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 9080 if (!rsv) 9081 return -ENOMEM; 9082 rsv->size = min_size; 9083 rsv->failfast = 1; 9084 9085 /* 9086 * 1 for the truncate slack space 9087 * 1 for updating the inode. 9088 */ 9089 trans = btrfs_start_transaction(root, 2); 9090 if (IS_ERR(trans)) { 9091 err = PTR_ERR(trans); 9092 goto out; 9093 } 9094 9095 /* Migrate the slack space for the truncate to our reserve */ 9096 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv, 9097 min_size, 0); 9098 BUG_ON(ret); 9099 9100 /* 9101 * So if we truncate and then write and fsync we normally would just 9102 * write the extents that changed, which is a problem if we need to 9103 * first truncate that entire inode. So set this flag so we write out 9104 * all of the extents in the inode to the sync log so we're completely 9105 * safe. 9106 */ 9107 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 9108 trans->block_rsv = rsv; 9109 9110 while (1) { 9111 ret = btrfs_truncate_inode_items(trans, root, inode, 9112 inode->i_size, 9113 BTRFS_EXTENT_DATA_KEY); 9114 trans->block_rsv = &fs_info->trans_block_rsv; 9115 if (ret != -ENOSPC && ret != -EAGAIN) { 9116 err = ret; 9117 break; 9118 } 9119 9120 ret = btrfs_update_inode(trans, root, inode); 9121 if (ret) { 9122 err = ret; 9123 break; 9124 } 9125 9126 btrfs_end_transaction(trans); 9127 btrfs_btree_balance_dirty(fs_info); 9128 9129 trans = btrfs_start_transaction(root, 2); 9130 if (IS_ERR(trans)) { 9131 ret = err = PTR_ERR(trans); 9132 trans = NULL; 9133 break; 9134 } 9135 9136 btrfs_block_rsv_release(fs_info, rsv, -1); 9137 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 9138 rsv, min_size, 0); 9139 BUG_ON(ret); /* shouldn't happen */ 9140 trans->block_rsv = rsv; 9141 } 9142 9143 /* 9144 * We can't call btrfs_truncate_block inside a trans handle as we could 9145 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know 9146 * we've truncated everything except the last little bit, and can do 9147 * btrfs_truncate_block and then update the disk_i_size. 9148 */ 9149 if (ret == NEED_TRUNCATE_BLOCK) { 9150 btrfs_end_transaction(trans); 9151 btrfs_btree_balance_dirty(fs_info); 9152 9153 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0); 9154 if (ret) 9155 goto out; 9156 trans = btrfs_start_transaction(root, 1); 9157 if (IS_ERR(trans)) { 9158 ret = PTR_ERR(trans); 9159 goto out; 9160 } 9161 btrfs_ordered_update_i_size(inode, inode->i_size, NULL); 9162 } 9163 9164 if (ret == 0 && inode->i_nlink > 0) { 9165 trans->block_rsv = root->orphan_block_rsv; 9166 ret = btrfs_orphan_del(trans, BTRFS_I(inode)); 9167 if (ret) 9168 err = ret; 9169 } 9170 9171 if (trans) { 9172 trans->block_rsv = &fs_info->trans_block_rsv; 9173 ret = btrfs_update_inode(trans, root, inode); 9174 if (ret && !err) 9175 err = ret; 9176 9177 ret = btrfs_end_transaction(trans); 9178 btrfs_btree_balance_dirty(fs_info); 9179 } 9180 out: 9181 btrfs_free_block_rsv(fs_info, rsv); 9182 9183 if (ret && !err) 9184 err = ret; 9185 9186 return err; 9187 } 9188 9189 /* 9190 * create a new subvolume directory/inode (helper for the ioctl). 9191 */ 9192 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, 9193 struct btrfs_root *new_root, 9194 struct btrfs_root *parent_root, 9195 u64 new_dirid) 9196 { 9197 struct inode *inode; 9198 int err; 9199 u64 index = 0; 9200 9201 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, 9202 new_dirid, new_dirid, 9203 S_IFDIR | (~current_umask() & S_IRWXUGO), 9204 &index); 9205 if (IS_ERR(inode)) 9206 return PTR_ERR(inode); 9207 inode->i_op = &btrfs_dir_inode_operations; 9208 inode->i_fop = &btrfs_dir_file_operations; 9209 9210 set_nlink(inode, 1); 9211 btrfs_i_size_write(BTRFS_I(inode), 0); 9212 unlock_new_inode(inode); 9213 9214 err = btrfs_subvol_inherit_props(trans, new_root, parent_root); 9215 if (err) 9216 btrfs_err(new_root->fs_info, 9217 "error inheriting subvolume %llu properties: %d", 9218 new_root->root_key.objectid, err); 9219 9220 err = btrfs_update_inode(trans, new_root, inode); 9221 9222 iput(inode); 9223 return err; 9224 } 9225 9226 struct inode *btrfs_alloc_inode(struct super_block *sb) 9227 { 9228 struct btrfs_fs_info *fs_info = btrfs_sb(sb); 9229 struct btrfs_inode *ei; 9230 struct inode *inode; 9231 9232 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL); 9233 if (!ei) 9234 return NULL; 9235 9236 ei->root = NULL; 9237 ei->generation = 0; 9238 ei->last_trans = 0; 9239 ei->last_sub_trans = 0; 9240 ei->logged_trans = 0; 9241 ei->delalloc_bytes = 0; 9242 ei->new_delalloc_bytes = 0; 9243 ei->defrag_bytes = 0; 9244 ei->disk_i_size = 0; 9245 ei->flags = 0; 9246 ei->csum_bytes = 0; 9247 ei->index_cnt = (u64)-1; 9248 ei->dir_index = 0; 9249 ei->last_unlink_trans = 0; 9250 ei->last_log_commit = 0; 9251 9252 spin_lock_init(&ei->lock); 9253 ei->outstanding_extents = 0; 9254 if (sb->s_magic != BTRFS_TEST_MAGIC) 9255 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, 9256 BTRFS_BLOCK_RSV_DELALLOC); 9257 ei->runtime_flags = 0; 9258 ei->prop_compress = BTRFS_COMPRESS_NONE; 9259 ei->defrag_compress = BTRFS_COMPRESS_NONE; 9260 9261 ei->delayed_node = NULL; 9262 9263 ei->i_otime.tv_sec = 0; 9264 ei->i_otime.tv_nsec = 0; 9265 9266 inode = &ei->vfs_inode; 9267 extent_map_tree_init(&ei->extent_tree); 9268 extent_io_tree_init(&ei->io_tree, inode); 9269 extent_io_tree_init(&ei->io_failure_tree, inode); 9270 ei->io_tree.track_uptodate = 1; 9271 ei->io_failure_tree.track_uptodate = 1; 9272 atomic_set(&ei->sync_writers, 0); 9273 mutex_init(&ei->log_mutex); 9274 mutex_init(&ei->delalloc_mutex); 9275 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 9276 INIT_LIST_HEAD(&ei->delalloc_inodes); 9277 INIT_LIST_HEAD(&ei->delayed_iput); 9278 RB_CLEAR_NODE(&ei->rb_node); 9279 init_rwsem(&ei->dio_sem); 9280 9281 return inode; 9282 } 9283 9284 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 9285 void btrfs_test_destroy_inode(struct inode *inode) 9286 { 9287 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 9288 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9289 } 9290 #endif 9291 9292 static void btrfs_i_callback(struct rcu_head *head) 9293 { 9294 struct inode *inode = container_of(head, struct inode, i_rcu); 9295 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9296 } 9297 9298 void btrfs_destroy_inode(struct inode *inode) 9299 { 9300 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9301 struct btrfs_ordered_extent *ordered; 9302 struct btrfs_root *root = BTRFS_I(inode)->root; 9303 9304 WARN_ON(!hlist_empty(&inode->i_dentry)); 9305 WARN_ON(inode->i_data.nrpages); 9306 WARN_ON(BTRFS_I(inode)->block_rsv.reserved); 9307 WARN_ON(BTRFS_I(inode)->block_rsv.size); 9308 WARN_ON(BTRFS_I(inode)->outstanding_extents); 9309 WARN_ON(BTRFS_I(inode)->delalloc_bytes); 9310 WARN_ON(BTRFS_I(inode)->new_delalloc_bytes); 9311 WARN_ON(BTRFS_I(inode)->csum_bytes); 9312 WARN_ON(BTRFS_I(inode)->defrag_bytes); 9313 9314 /* 9315 * This can happen where we create an inode, but somebody else also 9316 * created the same inode and we need to destroy the one we already 9317 * created. 9318 */ 9319 if (!root) 9320 goto free; 9321 9322 if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 9323 &BTRFS_I(inode)->runtime_flags)) { 9324 btrfs_info(fs_info, "inode %llu still on the orphan list", 9325 btrfs_ino(BTRFS_I(inode))); 9326 atomic_dec(&root->orphan_inodes); 9327 } 9328 9329 while (1) { 9330 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 9331 if (!ordered) 9332 break; 9333 else { 9334 btrfs_err(fs_info, 9335 "found ordered extent %llu %llu on inode cleanup", 9336 ordered->file_offset, ordered->len); 9337 btrfs_remove_ordered_extent(inode, ordered); 9338 btrfs_put_ordered_extent(ordered); 9339 btrfs_put_ordered_extent(ordered); 9340 } 9341 } 9342 btrfs_qgroup_check_reserved_leak(inode); 9343 inode_tree_del(inode); 9344 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 9345 free: 9346 call_rcu(&inode->i_rcu, btrfs_i_callback); 9347 } 9348 9349 int btrfs_drop_inode(struct inode *inode) 9350 { 9351 struct btrfs_root *root = BTRFS_I(inode)->root; 9352 9353 if (root == NULL) 9354 return 1; 9355 9356 /* the snap/subvol tree is on deleting */ 9357 if (btrfs_root_refs(&root->root_item) == 0) 9358 return 1; 9359 else 9360 return generic_drop_inode(inode); 9361 } 9362 9363 static void init_once(void *foo) 9364 { 9365 struct btrfs_inode *ei = (struct btrfs_inode *) foo; 9366 9367 inode_init_once(&ei->vfs_inode); 9368 } 9369 9370 void __cold btrfs_destroy_cachep(void) 9371 { 9372 /* 9373 * Make sure all delayed rcu free inodes are flushed before we 9374 * destroy cache. 9375 */ 9376 rcu_barrier(); 9377 kmem_cache_destroy(btrfs_inode_cachep); 9378 kmem_cache_destroy(btrfs_trans_handle_cachep); 9379 kmem_cache_destroy(btrfs_path_cachep); 9380 kmem_cache_destroy(btrfs_free_space_cachep); 9381 } 9382 9383 int __init btrfs_init_cachep(void) 9384 { 9385 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 9386 sizeof(struct btrfs_inode), 0, 9387 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, 9388 init_once); 9389 if (!btrfs_inode_cachep) 9390 goto fail; 9391 9392 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", 9393 sizeof(struct btrfs_trans_handle), 0, 9394 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); 9395 if (!btrfs_trans_handle_cachep) 9396 goto fail; 9397 9398 btrfs_path_cachep = kmem_cache_create("btrfs_path", 9399 sizeof(struct btrfs_path), 0, 9400 SLAB_MEM_SPREAD, NULL); 9401 if (!btrfs_path_cachep) 9402 goto fail; 9403 9404 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space", 9405 sizeof(struct btrfs_free_space), 0, 9406 SLAB_MEM_SPREAD, NULL); 9407 if (!btrfs_free_space_cachep) 9408 goto fail; 9409 9410 return 0; 9411 fail: 9412 btrfs_destroy_cachep(); 9413 return -ENOMEM; 9414 } 9415 9416 static int btrfs_getattr(const struct path *path, struct kstat *stat, 9417 u32 request_mask, unsigned int flags) 9418 { 9419 u64 delalloc_bytes; 9420 struct inode *inode = d_inode(path->dentry); 9421 u32 blocksize = inode->i_sb->s_blocksize; 9422 u32 bi_flags = BTRFS_I(inode)->flags; 9423 9424 stat->result_mask |= STATX_BTIME; 9425 stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec; 9426 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec; 9427 if (bi_flags & BTRFS_INODE_APPEND) 9428 stat->attributes |= STATX_ATTR_APPEND; 9429 if (bi_flags & BTRFS_INODE_COMPRESS) 9430 stat->attributes |= STATX_ATTR_COMPRESSED; 9431 if (bi_flags & BTRFS_INODE_IMMUTABLE) 9432 stat->attributes |= STATX_ATTR_IMMUTABLE; 9433 if (bi_flags & BTRFS_INODE_NODUMP) 9434 stat->attributes |= STATX_ATTR_NODUMP; 9435 9436 stat->attributes_mask |= (STATX_ATTR_APPEND | 9437 STATX_ATTR_COMPRESSED | 9438 STATX_ATTR_IMMUTABLE | 9439 STATX_ATTR_NODUMP); 9440 9441 generic_fillattr(inode, stat); 9442 stat->dev = BTRFS_I(inode)->root->anon_dev; 9443 9444 spin_lock(&BTRFS_I(inode)->lock); 9445 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; 9446 spin_unlock(&BTRFS_I(inode)->lock); 9447 stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) + 9448 ALIGN(delalloc_bytes, blocksize)) >> 9; 9449 return 0; 9450 } 9451 9452 static int btrfs_rename_exchange(struct inode *old_dir, 9453 struct dentry *old_dentry, 9454 struct inode *new_dir, 9455 struct dentry *new_dentry) 9456 { 9457 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9458 struct btrfs_trans_handle *trans; 9459 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9460 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9461 struct inode *new_inode = new_dentry->d_inode; 9462 struct inode *old_inode = old_dentry->d_inode; 9463 struct timespec ctime = current_time(old_inode); 9464 struct dentry *parent; 9465 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9466 u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); 9467 u64 old_idx = 0; 9468 u64 new_idx = 0; 9469 u64 root_objectid; 9470 int ret; 9471 bool root_log_pinned = false; 9472 bool dest_log_pinned = false; 9473 9474 /* we only allow rename subvolume link between subvolumes */ 9475 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9476 return -EXDEV; 9477 9478 /* close the race window with snapshot create/destroy ioctl */ 9479 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9480 down_read(&fs_info->subvol_sem); 9481 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9482 down_read(&fs_info->subvol_sem); 9483 9484 /* 9485 * We want to reserve the absolute worst case amount of items. So if 9486 * both inodes are subvols and we need to unlink them then that would 9487 * require 4 item modifications, but if they are both normal inodes it 9488 * would require 5 item modifications, so we'll assume their normal 9489 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items 9490 * should cover the worst case number of items we'll modify. 9491 */ 9492 trans = btrfs_start_transaction(root, 12); 9493 if (IS_ERR(trans)) { 9494 ret = PTR_ERR(trans); 9495 goto out_notrans; 9496 } 9497 9498 /* 9499 * We need to find a free sequence number both in the source and 9500 * in the destination directory for the exchange. 9501 */ 9502 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); 9503 if (ret) 9504 goto out_fail; 9505 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); 9506 if (ret) 9507 goto out_fail; 9508 9509 BTRFS_I(old_inode)->dir_index = 0ULL; 9510 BTRFS_I(new_inode)->dir_index = 0ULL; 9511 9512 /* Reference for the source. */ 9513 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9514 /* force full log commit if subvolume involved. */ 9515 btrfs_set_log_full_commit(fs_info, trans); 9516 } else { 9517 btrfs_pin_log_trans(root); 9518 root_log_pinned = true; 9519 ret = btrfs_insert_inode_ref(trans, dest, 9520 new_dentry->d_name.name, 9521 new_dentry->d_name.len, 9522 old_ino, 9523 btrfs_ino(BTRFS_I(new_dir)), 9524 old_idx); 9525 if (ret) 9526 goto out_fail; 9527 } 9528 9529 /* And now for the dest. */ 9530 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9531 /* force full log commit if subvolume involved. */ 9532 btrfs_set_log_full_commit(fs_info, trans); 9533 } else { 9534 btrfs_pin_log_trans(dest); 9535 dest_log_pinned = true; 9536 ret = btrfs_insert_inode_ref(trans, root, 9537 old_dentry->d_name.name, 9538 old_dentry->d_name.len, 9539 new_ino, 9540 btrfs_ino(BTRFS_I(old_dir)), 9541 new_idx); 9542 if (ret) 9543 goto out_fail; 9544 } 9545 9546 /* Update inode version and ctime/mtime. */ 9547 inode_inc_iversion(old_dir); 9548 inode_inc_iversion(new_dir); 9549 inode_inc_iversion(old_inode); 9550 inode_inc_iversion(new_inode); 9551 old_dir->i_ctime = old_dir->i_mtime = ctime; 9552 new_dir->i_ctime = new_dir->i_mtime = ctime; 9553 old_inode->i_ctime = ctime; 9554 new_inode->i_ctime = ctime; 9555 9556 if (old_dentry->d_parent != new_dentry->d_parent) { 9557 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9558 BTRFS_I(old_inode), 1); 9559 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), 9560 BTRFS_I(new_inode), 1); 9561 } 9562 9563 /* src is a subvolume */ 9564 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9565 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9566 ret = btrfs_unlink_subvol(trans, root, old_dir, 9567 root_objectid, 9568 old_dentry->d_name.name, 9569 old_dentry->d_name.len); 9570 } else { /* src is an inode */ 9571 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9572 BTRFS_I(old_dentry->d_inode), 9573 old_dentry->d_name.name, 9574 old_dentry->d_name.len); 9575 if (!ret) 9576 ret = btrfs_update_inode(trans, root, old_inode); 9577 } 9578 if (ret) { 9579 btrfs_abort_transaction(trans, ret); 9580 goto out_fail; 9581 } 9582 9583 /* dest is a subvolume */ 9584 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9585 root_objectid = BTRFS_I(new_inode)->root->root_key.objectid; 9586 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9587 root_objectid, 9588 new_dentry->d_name.name, 9589 new_dentry->d_name.len); 9590 } else { /* dest is an inode */ 9591 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9592 BTRFS_I(new_dentry->d_inode), 9593 new_dentry->d_name.name, 9594 new_dentry->d_name.len); 9595 if (!ret) 9596 ret = btrfs_update_inode(trans, dest, new_inode); 9597 } 9598 if (ret) { 9599 btrfs_abort_transaction(trans, ret); 9600 goto out_fail; 9601 } 9602 9603 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9604 new_dentry->d_name.name, 9605 new_dentry->d_name.len, 0, old_idx); 9606 if (ret) { 9607 btrfs_abort_transaction(trans, ret); 9608 goto out_fail; 9609 } 9610 9611 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), 9612 old_dentry->d_name.name, 9613 old_dentry->d_name.len, 0, new_idx); 9614 if (ret) { 9615 btrfs_abort_transaction(trans, ret); 9616 goto out_fail; 9617 } 9618 9619 if (old_inode->i_nlink == 1) 9620 BTRFS_I(old_inode)->dir_index = old_idx; 9621 if (new_inode->i_nlink == 1) 9622 BTRFS_I(new_inode)->dir_index = new_idx; 9623 9624 if (root_log_pinned) { 9625 parent = new_dentry->d_parent; 9626 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9627 parent); 9628 btrfs_end_log_trans(root); 9629 root_log_pinned = false; 9630 } 9631 if (dest_log_pinned) { 9632 parent = old_dentry->d_parent; 9633 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir), 9634 parent); 9635 btrfs_end_log_trans(dest); 9636 dest_log_pinned = false; 9637 } 9638 out_fail: 9639 /* 9640 * If we have pinned a log and an error happened, we unpin tasks 9641 * trying to sync the log and force them to fallback to a transaction 9642 * commit if the log currently contains any of the inodes involved in 9643 * this rename operation (to ensure we do not persist a log with an 9644 * inconsistent state for any of these inodes or leading to any 9645 * inconsistencies when replayed). If the transaction was aborted, the 9646 * abortion reason is propagated to userspace when attempting to commit 9647 * the transaction. If the log does not contain any of these inodes, we 9648 * allow the tasks to sync it. 9649 */ 9650 if (ret && (root_log_pinned || dest_log_pinned)) { 9651 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9652 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9653 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9654 (new_inode && 9655 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9656 btrfs_set_log_full_commit(fs_info, trans); 9657 9658 if (root_log_pinned) { 9659 btrfs_end_log_trans(root); 9660 root_log_pinned = false; 9661 } 9662 if (dest_log_pinned) { 9663 btrfs_end_log_trans(dest); 9664 dest_log_pinned = false; 9665 } 9666 } 9667 ret = btrfs_end_transaction(trans); 9668 out_notrans: 9669 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9670 up_read(&fs_info->subvol_sem); 9671 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9672 up_read(&fs_info->subvol_sem); 9673 9674 return ret; 9675 } 9676 9677 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans, 9678 struct btrfs_root *root, 9679 struct inode *dir, 9680 struct dentry *dentry) 9681 { 9682 int ret; 9683 struct inode *inode; 9684 u64 objectid; 9685 u64 index; 9686 9687 ret = btrfs_find_free_ino(root, &objectid); 9688 if (ret) 9689 return ret; 9690 9691 inode = btrfs_new_inode(trans, root, dir, 9692 dentry->d_name.name, 9693 dentry->d_name.len, 9694 btrfs_ino(BTRFS_I(dir)), 9695 objectid, 9696 S_IFCHR | WHITEOUT_MODE, 9697 &index); 9698 9699 if (IS_ERR(inode)) { 9700 ret = PTR_ERR(inode); 9701 return ret; 9702 } 9703 9704 inode->i_op = &btrfs_special_inode_operations; 9705 init_special_inode(inode, inode->i_mode, 9706 WHITEOUT_DEV); 9707 9708 ret = btrfs_init_inode_security(trans, inode, dir, 9709 &dentry->d_name); 9710 if (ret) 9711 goto out; 9712 9713 ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 9714 BTRFS_I(inode), 0, index); 9715 if (ret) 9716 goto out; 9717 9718 ret = btrfs_update_inode(trans, root, inode); 9719 out: 9720 unlock_new_inode(inode); 9721 if (ret) 9722 inode_dec_link_count(inode); 9723 iput(inode); 9724 9725 return ret; 9726 } 9727 9728 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry, 9729 struct inode *new_dir, struct dentry *new_dentry, 9730 unsigned int flags) 9731 { 9732 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9733 struct btrfs_trans_handle *trans; 9734 unsigned int trans_num_items; 9735 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9736 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9737 struct inode *new_inode = d_inode(new_dentry); 9738 struct inode *old_inode = d_inode(old_dentry); 9739 u64 index = 0; 9740 u64 root_objectid; 9741 int ret; 9742 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9743 bool log_pinned = false; 9744 9745 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 9746 return -EPERM; 9747 9748 /* we only allow rename subvolume link between subvolumes */ 9749 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9750 return -EXDEV; 9751 9752 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 9753 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) 9754 return -ENOTEMPTY; 9755 9756 if (S_ISDIR(old_inode->i_mode) && new_inode && 9757 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 9758 return -ENOTEMPTY; 9759 9760 9761 /* check for collisions, even if the name isn't there */ 9762 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, 9763 new_dentry->d_name.name, 9764 new_dentry->d_name.len); 9765 9766 if (ret) { 9767 if (ret == -EEXIST) { 9768 /* we shouldn't get 9769 * eexist without a new_inode */ 9770 if (WARN_ON(!new_inode)) { 9771 return ret; 9772 } 9773 } else { 9774 /* maybe -EOVERFLOW */ 9775 return ret; 9776 } 9777 } 9778 ret = 0; 9779 9780 /* 9781 * we're using rename to replace one file with another. Start IO on it 9782 * now so we don't add too much work to the end of the transaction 9783 */ 9784 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 9785 filemap_flush(old_inode->i_mapping); 9786 9787 /* close the racy window with snapshot create/destroy ioctl */ 9788 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9789 down_read(&fs_info->subvol_sem); 9790 /* 9791 * We want to reserve the absolute worst case amount of items. So if 9792 * both inodes are subvols and we need to unlink them then that would 9793 * require 4 item modifications, but if they are both normal inodes it 9794 * would require 5 item modifications, so we'll assume they are normal 9795 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items 9796 * should cover the worst case number of items we'll modify. 9797 * If our rename has the whiteout flag, we need more 5 units for the 9798 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item 9799 * when selinux is enabled). 9800 */ 9801 trans_num_items = 11; 9802 if (flags & RENAME_WHITEOUT) 9803 trans_num_items += 5; 9804 trans = btrfs_start_transaction(root, trans_num_items); 9805 if (IS_ERR(trans)) { 9806 ret = PTR_ERR(trans); 9807 goto out_notrans; 9808 } 9809 9810 if (dest != root) 9811 btrfs_record_root_in_trans(trans, dest); 9812 9813 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); 9814 if (ret) 9815 goto out_fail; 9816 9817 BTRFS_I(old_inode)->dir_index = 0ULL; 9818 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9819 /* force full log commit if subvolume involved. */ 9820 btrfs_set_log_full_commit(fs_info, trans); 9821 } else { 9822 btrfs_pin_log_trans(root); 9823 log_pinned = true; 9824 ret = btrfs_insert_inode_ref(trans, dest, 9825 new_dentry->d_name.name, 9826 new_dentry->d_name.len, 9827 old_ino, 9828 btrfs_ino(BTRFS_I(new_dir)), index); 9829 if (ret) 9830 goto out_fail; 9831 } 9832 9833 inode_inc_iversion(old_dir); 9834 inode_inc_iversion(new_dir); 9835 inode_inc_iversion(old_inode); 9836 old_dir->i_ctime = old_dir->i_mtime = 9837 new_dir->i_ctime = new_dir->i_mtime = 9838 old_inode->i_ctime = current_time(old_dir); 9839 9840 if (old_dentry->d_parent != new_dentry->d_parent) 9841 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9842 BTRFS_I(old_inode), 1); 9843 9844 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9845 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9846 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid, 9847 old_dentry->d_name.name, 9848 old_dentry->d_name.len); 9849 } else { 9850 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9851 BTRFS_I(d_inode(old_dentry)), 9852 old_dentry->d_name.name, 9853 old_dentry->d_name.len); 9854 if (!ret) 9855 ret = btrfs_update_inode(trans, root, old_inode); 9856 } 9857 if (ret) { 9858 btrfs_abort_transaction(trans, ret); 9859 goto out_fail; 9860 } 9861 9862 if (new_inode) { 9863 inode_inc_iversion(new_inode); 9864 new_inode->i_ctime = current_time(new_inode); 9865 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == 9866 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 9867 root_objectid = BTRFS_I(new_inode)->location.objectid; 9868 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9869 root_objectid, 9870 new_dentry->d_name.name, 9871 new_dentry->d_name.len); 9872 BUG_ON(new_inode->i_nlink == 0); 9873 } else { 9874 ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9875 BTRFS_I(d_inode(new_dentry)), 9876 new_dentry->d_name.name, 9877 new_dentry->d_name.len); 9878 } 9879 if (!ret && new_inode->i_nlink == 0) 9880 ret = btrfs_orphan_add(trans, 9881 BTRFS_I(d_inode(new_dentry))); 9882 if (ret) { 9883 btrfs_abort_transaction(trans, ret); 9884 goto out_fail; 9885 } 9886 } 9887 9888 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9889 new_dentry->d_name.name, 9890 new_dentry->d_name.len, 0, index); 9891 if (ret) { 9892 btrfs_abort_transaction(trans, ret); 9893 goto out_fail; 9894 } 9895 9896 if (old_inode->i_nlink == 1) 9897 BTRFS_I(old_inode)->dir_index = index; 9898 9899 if (log_pinned) { 9900 struct dentry *parent = new_dentry->d_parent; 9901 9902 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9903 parent); 9904 btrfs_end_log_trans(root); 9905 log_pinned = false; 9906 } 9907 9908 if (flags & RENAME_WHITEOUT) { 9909 ret = btrfs_whiteout_for_rename(trans, root, old_dir, 9910 old_dentry); 9911 9912 if (ret) { 9913 btrfs_abort_transaction(trans, ret); 9914 goto out_fail; 9915 } 9916 } 9917 out_fail: 9918 /* 9919 * If we have pinned the log and an error happened, we unpin tasks 9920 * trying to sync the log and force them to fallback to a transaction 9921 * commit if the log currently contains any of the inodes involved in 9922 * this rename operation (to ensure we do not persist a log with an 9923 * inconsistent state for any of these inodes or leading to any 9924 * inconsistencies when replayed). If the transaction was aborted, the 9925 * abortion reason is propagated to userspace when attempting to commit 9926 * the transaction. If the log does not contain any of these inodes, we 9927 * allow the tasks to sync it. 9928 */ 9929 if (ret && log_pinned) { 9930 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9931 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9932 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9933 (new_inode && 9934 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9935 btrfs_set_log_full_commit(fs_info, trans); 9936 9937 btrfs_end_log_trans(root); 9938 log_pinned = false; 9939 } 9940 btrfs_end_transaction(trans); 9941 out_notrans: 9942 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9943 up_read(&fs_info->subvol_sem); 9944 9945 return ret; 9946 } 9947 9948 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, 9949 struct inode *new_dir, struct dentry *new_dentry, 9950 unsigned int flags) 9951 { 9952 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 9953 return -EINVAL; 9954 9955 if (flags & RENAME_EXCHANGE) 9956 return btrfs_rename_exchange(old_dir, old_dentry, new_dir, 9957 new_dentry); 9958 9959 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags); 9960 } 9961 9962 static void btrfs_run_delalloc_work(struct btrfs_work *work) 9963 { 9964 struct btrfs_delalloc_work *delalloc_work; 9965 struct inode *inode; 9966 9967 delalloc_work = container_of(work, struct btrfs_delalloc_work, 9968 work); 9969 inode = delalloc_work->inode; 9970 filemap_flush(inode->i_mapping); 9971 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9972 &BTRFS_I(inode)->runtime_flags)) 9973 filemap_flush(inode->i_mapping); 9974 9975 if (delalloc_work->delay_iput) 9976 btrfs_add_delayed_iput(inode); 9977 else 9978 iput(inode); 9979 complete(&delalloc_work->completion); 9980 } 9981 9982 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode, 9983 int delay_iput) 9984 { 9985 struct btrfs_delalloc_work *work; 9986 9987 work = kmalloc(sizeof(*work), GFP_NOFS); 9988 if (!work) 9989 return NULL; 9990 9991 init_completion(&work->completion); 9992 INIT_LIST_HEAD(&work->list); 9993 work->inode = inode; 9994 work->delay_iput = delay_iput; 9995 WARN_ON_ONCE(!inode); 9996 btrfs_init_work(&work->work, btrfs_flush_delalloc_helper, 9997 btrfs_run_delalloc_work, NULL, NULL); 9998 9999 return work; 10000 } 10001 10002 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work) 10003 { 10004 wait_for_completion(&work->completion); 10005 kfree(work); 10006 } 10007 10008 /* 10009 * some fairly slow code that needs optimization. This walks the list 10010 * of all the inodes with pending delalloc and forces them to disk. 10011 */ 10012 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput, 10013 int nr) 10014 { 10015 struct btrfs_inode *binode; 10016 struct inode *inode; 10017 struct btrfs_delalloc_work *work, *next; 10018 struct list_head works; 10019 struct list_head splice; 10020 int ret = 0; 10021 10022 INIT_LIST_HEAD(&works); 10023 INIT_LIST_HEAD(&splice); 10024 10025 mutex_lock(&root->delalloc_mutex); 10026 spin_lock(&root->delalloc_lock); 10027 list_splice_init(&root->delalloc_inodes, &splice); 10028 while (!list_empty(&splice)) { 10029 binode = list_entry(splice.next, struct btrfs_inode, 10030 delalloc_inodes); 10031 10032 list_move_tail(&binode->delalloc_inodes, 10033 &root->delalloc_inodes); 10034 inode = igrab(&binode->vfs_inode); 10035 if (!inode) { 10036 cond_resched_lock(&root->delalloc_lock); 10037 continue; 10038 } 10039 spin_unlock(&root->delalloc_lock); 10040 10041 work = btrfs_alloc_delalloc_work(inode, delay_iput); 10042 if (!work) { 10043 if (delay_iput) 10044 btrfs_add_delayed_iput(inode); 10045 else 10046 iput(inode); 10047 ret = -ENOMEM; 10048 goto out; 10049 } 10050 list_add_tail(&work->list, &works); 10051 btrfs_queue_work(root->fs_info->flush_workers, 10052 &work->work); 10053 ret++; 10054 if (nr != -1 && ret >= nr) 10055 goto out; 10056 cond_resched(); 10057 spin_lock(&root->delalloc_lock); 10058 } 10059 spin_unlock(&root->delalloc_lock); 10060 10061 out: 10062 list_for_each_entry_safe(work, next, &works, list) { 10063 list_del_init(&work->list); 10064 btrfs_wait_and_free_delalloc_work(work); 10065 } 10066 10067 if (!list_empty_careful(&splice)) { 10068 spin_lock(&root->delalloc_lock); 10069 list_splice_tail(&splice, &root->delalloc_inodes); 10070 spin_unlock(&root->delalloc_lock); 10071 } 10072 mutex_unlock(&root->delalloc_mutex); 10073 return ret; 10074 } 10075 10076 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput) 10077 { 10078 struct btrfs_fs_info *fs_info = root->fs_info; 10079 int ret; 10080 10081 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10082 return -EROFS; 10083 10084 ret = __start_delalloc_inodes(root, delay_iput, -1); 10085 if (ret > 0) 10086 ret = 0; 10087 return ret; 10088 } 10089 10090 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput, 10091 int nr) 10092 { 10093 struct btrfs_root *root; 10094 struct list_head splice; 10095 int ret; 10096 10097 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10098 return -EROFS; 10099 10100 INIT_LIST_HEAD(&splice); 10101 10102 mutex_lock(&fs_info->delalloc_root_mutex); 10103 spin_lock(&fs_info->delalloc_root_lock); 10104 list_splice_init(&fs_info->delalloc_roots, &splice); 10105 while (!list_empty(&splice) && nr) { 10106 root = list_first_entry(&splice, struct btrfs_root, 10107 delalloc_root); 10108 root = btrfs_grab_fs_root(root); 10109 BUG_ON(!root); 10110 list_move_tail(&root->delalloc_root, 10111 &fs_info->delalloc_roots); 10112 spin_unlock(&fs_info->delalloc_root_lock); 10113 10114 ret = __start_delalloc_inodes(root, delay_iput, nr); 10115 btrfs_put_fs_root(root); 10116 if (ret < 0) 10117 goto out; 10118 10119 if (nr != -1) { 10120 nr -= ret; 10121 WARN_ON(nr < 0); 10122 } 10123 spin_lock(&fs_info->delalloc_root_lock); 10124 } 10125 spin_unlock(&fs_info->delalloc_root_lock); 10126 10127 ret = 0; 10128 out: 10129 if (!list_empty_careful(&splice)) { 10130 spin_lock(&fs_info->delalloc_root_lock); 10131 list_splice_tail(&splice, &fs_info->delalloc_roots); 10132 spin_unlock(&fs_info->delalloc_root_lock); 10133 } 10134 mutex_unlock(&fs_info->delalloc_root_mutex); 10135 return ret; 10136 } 10137 10138 static int btrfs_symlink(struct inode *dir, struct dentry *dentry, 10139 const char *symname) 10140 { 10141 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10142 struct btrfs_trans_handle *trans; 10143 struct btrfs_root *root = BTRFS_I(dir)->root; 10144 struct btrfs_path *path; 10145 struct btrfs_key key; 10146 struct inode *inode = NULL; 10147 int err; 10148 int drop_inode = 0; 10149 u64 objectid; 10150 u64 index = 0; 10151 int name_len; 10152 int datasize; 10153 unsigned long ptr; 10154 struct btrfs_file_extent_item *ei; 10155 struct extent_buffer *leaf; 10156 10157 name_len = strlen(symname); 10158 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 10159 return -ENAMETOOLONG; 10160 10161 /* 10162 * 2 items for inode item and ref 10163 * 2 items for dir items 10164 * 1 item for updating parent inode item 10165 * 1 item for the inline extent item 10166 * 1 item for xattr if selinux is on 10167 */ 10168 trans = btrfs_start_transaction(root, 7); 10169 if (IS_ERR(trans)) 10170 return PTR_ERR(trans); 10171 10172 err = btrfs_find_free_ino(root, &objectid); 10173 if (err) 10174 goto out_unlock; 10175 10176 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 10177 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), 10178 objectid, S_IFLNK|S_IRWXUGO, &index); 10179 if (IS_ERR(inode)) { 10180 err = PTR_ERR(inode); 10181 goto out_unlock; 10182 } 10183 10184 /* 10185 * If the active LSM wants to access the inode during 10186 * d_instantiate it needs these. Smack checks to see 10187 * if the filesystem supports xattrs by looking at the 10188 * ops vector. 10189 */ 10190 inode->i_fop = &btrfs_file_operations; 10191 inode->i_op = &btrfs_file_inode_operations; 10192 inode->i_mapping->a_ops = &btrfs_aops; 10193 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10194 10195 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 10196 if (err) 10197 goto out_unlock_inode; 10198 10199 path = btrfs_alloc_path(); 10200 if (!path) { 10201 err = -ENOMEM; 10202 goto out_unlock_inode; 10203 } 10204 key.objectid = btrfs_ino(BTRFS_I(inode)); 10205 key.offset = 0; 10206 key.type = BTRFS_EXTENT_DATA_KEY; 10207 datasize = btrfs_file_extent_calc_inline_size(name_len); 10208 err = btrfs_insert_empty_item(trans, root, path, &key, 10209 datasize); 10210 if (err) { 10211 btrfs_free_path(path); 10212 goto out_unlock_inode; 10213 } 10214 leaf = path->nodes[0]; 10215 ei = btrfs_item_ptr(leaf, path->slots[0], 10216 struct btrfs_file_extent_item); 10217 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 10218 btrfs_set_file_extent_type(leaf, ei, 10219 BTRFS_FILE_EXTENT_INLINE); 10220 btrfs_set_file_extent_encryption(leaf, ei, 0); 10221 btrfs_set_file_extent_compression(leaf, ei, 0); 10222 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 10223 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 10224 10225 ptr = btrfs_file_extent_inline_start(ei); 10226 write_extent_buffer(leaf, symname, ptr, name_len); 10227 btrfs_mark_buffer_dirty(leaf); 10228 btrfs_free_path(path); 10229 10230 inode->i_op = &btrfs_symlink_inode_operations; 10231 inode_nohighmem(inode); 10232 inode->i_mapping->a_ops = &btrfs_symlink_aops; 10233 inode_set_bytes(inode, name_len); 10234 btrfs_i_size_write(BTRFS_I(inode), name_len); 10235 err = btrfs_update_inode(trans, root, inode); 10236 /* 10237 * Last step, add directory indexes for our symlink inode. This is the 10238 * last step to avoid extra cleanup of these indexes if an error happens 10239 * elsewhere above. 10240 */ 10241 if (!err) 10242 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 10243 BTRFS_I(inode), 0, index); 10244 if (err) { 10245 drop_inode = 1; 10246 goto out_unlock_inode; 10247 } 10248 10249 unlock_new_inode(inode); 10250 d_instantiate(dentry, inode); 10251 10252 out_unlock: 10253 btrfs_end_transaction(trans); 10254 if (drop_inode) { 10255 inode_dec_link_count(inode); 10256 iput(inode); 10257 } 10258 btrfs_btree_balance_dirty(fs_info); 10259 return err; 10260 10261 out_unlock_inode: 10262 drop_inode = 1; 10263 unlock_new_inode(inode); 10264 goto out_unlock; 10265 } 10266 10267 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 10268 u64 start, u64 num_bytes, u64 min_size, 10269 loff_t actual_len, u64 *alloc_hint, 10270 struct btrfs_trans_handle *trans) 10271 { 10272 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 10273 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 10274 struct extent_map *em; 10275 struct btrfs_root *root = BTRFS_I(inode)->root; 10276 struct btrfs_key ins; 10277 u64 cur_offset = start; 10278 u64 i_size; 10279 u64 cur_bytes; 10280 u64 last_alloc = (u64)-1; 10281 int ret = 0; 10282 bool own_trans = true; 10283 u64 end = start + num_bytes - 1; 10284 10285 if (trans) 10286 own_trans = false; 10287 while (num_bytes > 0) { 10288 if (own_trans) { 10289 trans = btrfs_start_transaction(root, 3); 10290 if (IS_ERR(trans)) { 10291 ret = PTR_ERR(trans); 10292 break; 10293 } 10294 } 10295 10296 cur_bytes = min_t(u64, num_bytes, SZ_256M); 10297 cur_bytes = max(cur_bytes, min_size); 10298 /* 10299 * If we are severely fragmented we could end up with really 10300 * small allocations, so if the allocator is returning small 10301 * chunks lets make its job easier by only searching for those 10302 * sized chunks. 10303 */ 10304 cur_bytes = min(cur_bytes, last_alloc); 10305 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 10306 min_size, 0, *alloc_hint, &ins, 1, 0); 10307 if (ret) { 10308 if (own_trans) 10309 btrfs_end_transaction(trans); 10310 break; 10311 } 10312 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10313 10314 last_alloc = ins.offset; 10315 ret = insert_reserved_file_extent(trans, inode, 10316 cur_offset, ins.objectid, 10317 ins.offset, ins.offset, 10318 ins.offset, 0, 0, 0, 10319 BTRFS_FILE_EXTENT_PREALLOC); 10320 if (ret) { 10321 btrfs_free_reserved_extent(fs_info, ins.objectid, 10322 ins.offset, 0); 10323 btrfs_abort_transaction(trans, ret); 10324 if (own_trans) 10325 btrfs_end_transaction(trans); 10326 break; 10327 } 10328 10329 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 10330 cur_offset + ins.offset -1, 0); 10331 10332 em = alloc_extent_map(); 10333 if (!em) { 10334 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 10335 &BTRFS_I(inode)->runtime_flags); 10336 goto next; 10337 } 10338 10339 em->start = cur_offset; 10340 em->orig_start = cur_offset; 10341 em->len = ins.offset; 10342 em->block_start = ins.objectid; 10343 em->block_len = ins.offset; 10344 em->orig_block_len = ins.offset; 10345 em->ram_bytes = ins.offset; 10346 em->bdev = fs_info->fs_devices->latest_bdev; 10347 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 10348 em->generation = trans->transid; 10349 10350 while (1) { 10351 write_lock(&em_tree->lock); 10352 ret = add_extent_mapping(em_tree, em, 1); 10353 write_unlock(&em_tree->lock); 10354 if (ret != -EEXIST) 10355 break; 10356 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 10357 cur_offset + ins.offset - 1, 10358 0); 10359 } 10360 free_extent_map(em); 10361 next: 10362 num_bytes -= ins.offset; 10363 cur_offset += ins.offset; 10364 *alloc_hint = ins.objectid + ins.offset; 10365 10366 inode_inc_iversion(inode); 10367 inode->i_ctime = current_time(inode); 10368 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 10369 if (!(mode & FALLOC_FL_KEEP_SIZE) && 10370 (actual_len > inode->i_size) && 10371 (cur_offset > inode->i_size)) { 10372 if (cur_offset > actual_len) 10373 i_size = actual_len; 10374 else 10375 i_size = cur_offset; 10376 i_size_write(inode, i_size); 10377 btrfs_ordered_update_i_size(inode, i_size, NULL); 10378 } 10379 10380 ret = btrfs_update_inode(trans, root, inode); 10381 10382 if (ret) { 10383 btrfs_abort_transaction(trans, ret); 10384 if (own_trans) 10385 btrfs_end_transaction(trans); 10386 break; 10387 } 10388 10389 if (own_trans) 10390 btrfs_end_transaction(trans); 10391 } 10392 if (cur_offset < end) 10393 btrfs_free_reserved_data_space(inode, NULL, cur_offset, 10394 end - cur_offset + 1); 10395 return ret; 10396 } 10397 10398 int btrfs_prealloc_file_range(struct inode *inode, int mode, 10399 u64 start, u64 num_bytes, u64 min_size, 10400 loff_t actual_len, u64 *alloc_hint) 10401 { 10402 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10403 min_size, actual_len, alloc_hint, 10404 NULL); 10405 } 10406 10407 int btrfs_prealloc_file_range_trans(struct inode *inode, 10408 struct btrfs_trans_handle *trans, int mode, 10409 u64 start, u64 num_bytes, u64 min_size, 10410 loff_t actual_len, u64 *alloc_hint) 10411 { 10412 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10413 min_size, actual_len, alloc_hint, trans); 10414 } 10415 10416 static int btrfs_set_page_dirty(struct page *page) 10417 { 10418 return __set_page_dirty_nobuffers(page); 10419 } 10420 10421 static int btrfs_permission(struct inode *inode, int mask) 10422 { 10423 struct btrfs_root *root = BTRFS_I(inode)->root; 10424 umode_t mode = inode->i_mode; 10425 10426 if (mask & MAY_WRITE && 10427 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 10428 if (btrfs_root_readonly(root)) 10429 return -EROFS; 10430 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 10431 return -EACCES; 10432 } 10433 return generic_permission(inode, mask); 10434 } 10435 10436 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 10437 { 10438 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10439 struct btrfs_trans_handle *trans; 10440 struct btrfs_root *root = BTRFS_I(dir)->root; 10441 struct inode *inode = NULL; 10442 u64 objectid; 10443 u64 index; 10444 int ret = 0; 10445 10446 /* 10447 * 5 units required for adding orphan entry 10448 */ 10449 trans = btrfs_start_transaction(root, 5); 10450 if (IS_ERR(trans)) 10451 return PTR_ERR(trans); 10452 10453 ret = btrfs_find_free_ino(root, &objectid); 10454 if (ret) 10455 goto out; 10456 10457 inode = btrfs_new_inode(trans, root, dir, NULL, 0, 10458 btrfs_ino(BTRFS_I(dir)), objectid, mode, &index); 10459 if (IS_ERR(inode)) { 10460 ret = PTR_ERR(inode); 10461 inode = NULL; 10462 goto out; 10463 } 10464 10465 inode->i_fop = &btrfs_file_operations; 10466 inode->i_op = &btrfs_file_inode_operations; 10467 10468 inode->i_mapping->a_ops = &btrfs_aops; 10469 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10470 10471 ret = btrfs_init_inode_security(trans, inode, dir, NULL); 10472 if (ret) 10473 goto out_inode; 10474 10475 ret = btrfs_update_inode(trans, root, inode); 10476 if (ret) 10477 goto out_inode; 10478 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 10479 if (ret) 10480 goto out_inode; 10481 10482 /* 10483 * We set number of links to 0 in btrfs_new_inode(), and here we set 10484 * it to 1 because d_tmpfile() will issue a warning if the count is 0, 10485 * through: 10486 * 10487 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 10488 */ 10489 set_nlink(inode, 1); 10490 unlock_new_inode(inode); 10491 d_tmpfile(dentry, inode); 10492 mark_inode_dirty(inode); 10493 10494 out: 10495 btrfs_end_transaction(trans); 10496 if (ret) 10497 iput(inode); 10498 btrfs_btree_balance_dirty(fs_info); 10499 return ret; 10500 10501 out_inode: 10502 unlock_new_inode(inode); 10503 goto out; 10504 10505 } 10506 10507 __attribute__((const)) 10508 static int btrfs_readpage_io_failed_hook(struct page *page, int failed_mirror) 10509 { 10510 return -EAGAIN; 10511 } 10512 10513 static struct btrfs_fs_info *iotree_fs_info(void *private_data) 10514 { 10515 struct inode *inode = private_data; 10516 return btrfs_sb(inode->i_sb); 10517 } 10518 10519 static void btrfs_check_extent_io_range(void *private_data, const char *caller, 10520 u64 start, u64 end) 10521 { 10522 struct inode *inode = private_data; 10523 u64 isize; 10524 10525 isize = i_size_read(inode); 10526 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { 10527 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info, 10528 "%s: ino %llu isize %llu odd range [%llu,%llu]", 10529 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end); 10530 } 10531 } 10532 10533 void btrfs_set_range_writeback(void *private_data, u64 start, u64 end) 10534 { 10535 struct inode *inode = private_data; 10536 unsigned long index = start >> PAGE_SHIFT; 10537 unsigned long end_index = end >> PAGE_SHIFT; 10538 struct page *page; 10539 10540 while (index <= end_index) { 10541 page = find_get_page(inode->i_mapping, index); 10542 ASSERT(page); /* Pages should be in the extent_io_tree */ 10543 set_page_writeback(page); 10544 put_page(page); 10545 index++; 10546 } 10547 } 10548 10549 static const struct inode_operations btrfs_dir_inode_operations = { 10550 .getattr = btrfs_getattr, 10551 .lookup = btrfs_lookup, 10552 .create = btrfs_create, 10553 .unlink = btrfs_unlink, 10554 .link = btrfs_link, 10555 .mkdir = btrfs_mkdir, 10556 .rmdir = btrfs_rmdir, 10557 .rename = btrfs_rename2, 10558 .symlink = btrfs_symlink, 10559 .setattr = btrfs_setattr, 10560 .mknod = btrfs_mknod, 10561 .listxattr = btrfs_listxattr, 10562 .permission = btrfs_permission, 10563 .get_acl = btrfs_get_acl, 10564 .set_acl = btrfs_set_acl, 10565 .update_time = btrfs_update_time, 10566 .tmpfile = btrfs_tmpfile, 10567 }; 10568 static const struct inode_operations btrfs_dir_ro_inode_operations = { 10569 .lookup = btrfs_lookup, 10570 .permission = btrfs_permission, 10571 .update_time = btrfs_update_time, 10572 }; 10573 10574 static const struct file_operations btrfs_dir_file_operations = { 10575 .llseek = generic_file_llseek, 10576 .read = generic_read_dir, 10577 .iterate_shared = btrfs_real_readdir, 10578 .open = btrfs_opendir, 10579 .unlocked_ioctl = btrfs_ioctl, 10580 #ifdef CONFIG_COMPAT 10581 .compat_ioctl = btrfs_compat_ioctl, 10582 #endif 10583 .release = btrfs_release_file, 10584 .fsync = btrfs_sync_file, 10585 }; 10586 10587 static const struct extent_io_ops btrfs_extent_io_ops = { 10588 /* mandatory callbacks */ 10589 .submit_bio_hook = btrfs_submit_bio_hook, 10590 .readpage_end_io_hook = btrfs_readpage_end_io_hook, 10591 .merge_bio_hook = btrfs_merge_bio_hook, 10592 .readpage_io_failed_hook = btrfs_readpage_io_failed_hook, 10593 .tree_fs_info = iotree_fs_info, 10594 .set_range_writeback = btrfs_set_range_writeback, 10595 10596 /* optional callbacks */ 10597 .fill_delalloc = run_delalloc_range, 10598 .writepage_end_io_hook = btrfs_writepage_end_io_hook, 10599 .writepage_start_hook = btrfs_writepage_start_hook, 10600 .set_bit_hook = btrfs_set_bit_hook, 10601 .clear_bit_hook = btrfs_clear_bit_hook, 10602 .merge_extent_hook = btrfs_merge_extent_hook, 10603 .split_extent_hook = btrfs_split_extent_hook, 10604 .check_extent_io_range = btrfs_check_extent_io_range, 10605 }; 10606 10607 /* 10608 * btrfs doesn't support the bmap operation because swapfiles 10609 * use bmap to make a mapping of extents in the file. They assume 10610 * these extents won't change over the life of the file and they 10611 * use the bmap result to do IO directly to the drive. 10612 * 10613 * the btrfs bmap call would return logical addresses that aren't 10614 * suitable for IO and they also will change frequently as COW 10615 * operations happen. So, swapfile + btrfs == corruption. 10616 * 10617 * For now we're avoiding this by dropping bmap. 10618 */ 10619 static const struct address_space_operations btrfs_aops = { 10620 .readpage = btrfs_readpage, 10621 .writepage = btrfs_writepage, 10622 .writepages = btrfs_writepages, 10623 .readpages = btrfs_readpages, 10624 .direct_IO = btrfs_direct_IO, 10625 .invalidatepage = btrfs_invalidatepage, 10626 .releasepage = btrfs_releasepage, 10627 .set_page_dirty = btrfs_set_page_dirty, 10628 .error_remove_page = generic_error_remove_page, 10629 }; 10630 10631 static const struct address_space_operations btrfs_symlink_aops = { 10632 .readpage = btrfs_readpage, 10633 .writepage = btrfs_writepage, 10634 .invalidatepage = btrfs_invalidatepage, 10635 .releasepage = btrfs_releasepage, 10636 }; 10637 10638 static const struct inode_operations btrfs_file_inode_operations = { 10639 .getattr = btrfs_getattr, 10640 .setattr = btrfs_setattr, 10641 .listxattr = btrfs_listxattr, 10642 .permission = btrfs_permission, 10643 .fiemap = btrfs_fiemap, 10644 .get_acl = btrfs_get_acl, 10645 .set_acl = btrfs_set_acl, 10646 .update_time = btrfs_update_time, 10647 }; 10648 static const struct inode_operations btrfs_special_inode_operations = { 10649 .getattr = btrfs_getattr, 10650 .setattr = btrfs_setattr, 10651 .permission = btrfs_permission, 10652 .listxattr = btrfs_listxattr, 10653 .get_acl = btrfs_get_acl, 10654 .set_acl = btrfs_set_acl, 10655 .update_time = btrfs_update_time, 10656 }; 10657 static const struct inode_operations btrfs_symlink_inode_operations = { 10658 .get_link = page_get_link, 10659 .getattr = btrfs_getattr, 10660 .setattr = btrfs_setattr, 10661 .permission = btrfs_permission, 10662 .listxattr = btrfs_listxattr, 10663 .update_time = btrfs_update_time, 10664 }; 10665 10666 const struct dentry_operations btrfs_dentry_operations = { 10667 .d_delete = btrfs_dentry_delete, 10668 .d_release = btrfs_dentry_release, 10669 }; 10670