1 /* 2 * Copyright (C) 2007 Oracle. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public 6 * License v2 as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public 14 * License along with this program; if not, write to the 15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 16 * Boston, MA 021110-1307, USA. 17 */ 18 19 #include <linux/kernel.h> 20 #include <linux/bio.h> 21 #include <linux/buffer_head.h> 22 #include <linux/file.h> 23 #include <linux/fs.h> 24 #include <linux/pagemap.h> 25 #include <linux/highmem.h> 26 #include <linux/time.h> 27 #include <linux/init.h> 28 #include <linux/string.h> 29 #include <linux/backing-dev.h> 30 #include <linux/mpage.h> 31 #include <linux/swap.h> 32 #include <linux/writeback.h> 33 #include <linux/compat.h> 34 #include <linux/bit_spinlock.h> 35 #include <linux/xattr.h> 36 #include <linux/posix_acl.h> 37 #include <linux/falloc.h> 38 #include <linux/slab.h> 39 #include <linux/ratelimit.h> 40 #include <linux/mount.h> 41 #include <linux/btrfs.h> 42 #include <linux/blkdev.h> 43 #include <linux/posix_acl_xattr.h> 44 #include <linux/uio.h> 45 #include "ctree.h" 46 #include "disk-io.h" 47 #include "transaction.h" 48 #include "btrfs_inode.h" 49 #include "print-tree.h" 50 #include "ordered-data.h" 51 #include "xattr.h" 52 #include "tree-log.h" 53 #include "volumes.h" 54 #include "compression.h" 55 #include "locking.h" 56 #include "free-space-cache.h" 57 #include "inode-map.h" 58 #include "backref.h" 59 #include "hash.h" 60 #include "props.h" 61 #include "qgroup.h" 62 #include "dedupe.h" 63 64 struct btrfs_iget_args { 65 struct btrfs_key *location; 66 struct btrfs_root *root; 67 }; 68 69 struct btrfs_dio_data { 70 u64 outstanding_extents; 71 u64 reserve; 72 u64 unsubmitted_oe_range_start; 73 u64 unsubmitted_oe_range_end; 74 }; 75 76 static const struct inode_operations btrfs_dir_inode_operations; 77 static const struct inode_operations btrfs_symlink_inode_operations; 78 static const struct inode_operations btrfs_dir_ro_inode_operations; 79 static const struct inode_operations btrfs_special_inode_operations; 80 static const struct inode_operations btrfs_file_inode_operations; 81 static const struct address_space_operations btrfs_aops; 82 static const struct address_space_operations btrfs_symlink_aops; 83 static const struct file_operations btrfs_dir_file_operations; 84 static const struct extent_io_ops btrfs_extent_io_ops; 85 86 static struct kmem_cache *btrfs_inode_cachep; 87 struct kmem_cache *btrfs_trans_handle_cachep; 88 struct kmem_cache *btrfs_transaction_cachep; 89 struct kmem_cache *btrfs_path_cachep; 90 struct kmem_cache *btrfs_free_space_cachep; 91 92 #define S_SHIFT 12 93 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = { 94 [S_IFREG >> S_SHIFT] = BTRFS_FT_REG_FILE, 95 [S_IFDIR >> S_SHIFT] = BTRFS_FT_DIR, 96 [S_IFCHR >> S_SHIFT] = BTRFS_FT_CHRDEV, 97 [S_IFBLK >> S_SHIFT] = BTRFS_FT_BLKDEV, 98 [S_IFIFO >> S_SHIFT] = BTRFS_FT_FIFO, 99 [S_IFSOCK >> S_SHIFT] = BTRFS_FT_SOCK, 100 [S_IFLNK >> S_SHIFT] = BTRFS_FT_SYMLINK, 101 }; 102 103 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 104 static int btrfs_truncate(struct inode *inode); 105 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); 106 static noinline int cow_file_range(struct inode *inode, 107 struct page *locked_page, 108 u64 start, u64 end, u64 delalloc_end, 109 int *page_started, unsigned long *nr_written, 110 int unlock, struct btrfs_dedupe_hash *hash); 111 static struct extent_map *create_pinned_em(struct inode *inode, u64 start, 112 u64 len, u64 orig_start, 113 u64 block_start, u64 block_len, 114 u64 orig_block_len, u64 ram_bytes, 115 int type); 116 117 static int btrfs_dirty_inode(struct inode *inode); 118 119 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 120 void btrfs_test_inode_set_ops(struct inode *inode) 121 { 122 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 123 } 124 #endif 125 126 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 127 struct inode *inode, struct inode *dir, 128 const struct qstr *qstr) 129 { 130 int err; 131 132 err = btrfs_init_acl(trans, inode, dir); 133 if (!err) 134 err = btrfs_xattr_security_init(trans, inode, dir, qstr); 135 return err; 136 } 137 138 /* 139 * this does all the hard work for inserting an inline extent into 140 * the btree. The caller should have done a btrfs_drop_extents so that 141 * no overlapping inline items exist in the btree 142 */ 143 static int insert_inline_extent(struct btrfs_trans_handle *trans, 144 struct btrfs_path *path, int extent_inserted, 145 struct btrfs_root *root, struct inode *inode, 146 u64 start, size_t size, size_t compressed_size, 147 int compress_type, 148 struct page **compressed_pages) 149 { 150 struct extent_buffer *leaf; 151 struct page *page = NULL; 152 char *kaddr; 153 unsigned long ptr; 154 struct btrfs_file_extent_item *ei; 155 int err = 0; 156 int ret; 157 size_t cur_size = size; 158 unsigned long offset; 159 160 if (compressed_size && compressed_pages) 161 cur_size = compressed_size; 162 163 inode_add_bytes(inode, size); 164 165 if (!extent_inserted) { 166 struct btrfs_key key; 167 size_t datasize; 168 169 key.objectid = btrfs_ino(inode); 170 key.offset = start; 171 key.type = BTRFS_EXTENT_DATA_KEY; 172 173 datasize = btrfs_file_extent_calc_inline_size(cur_size); 174 path->leave_spinning = 1; 175 ret = btrfs_insert_empty_item(trans, root, path, &key, 176 datasize); 177 if (ret) { 178 err = ret; 179 goto fail; 180 } 181 } 182 leaf = path->nodes[0]; 183 ei = btrfs_item_ptr(leaf, path->slots[0], 184 struct btrfs_file_extent_item); 185 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 186 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 187 btrfs_set_file_extent_encryption(leaf, ei, 0); 188 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 189 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 190 ptr = btrfs_file_extent_inline_start(ei); 191 192 if (compress_type != BTRFS_COMPRESS_NONE) { 193 struct page *cpage; 194 int i = 0; 195 while (compressed_size > 0) { 196 cpage = compressed_pages[i]; 197 cur_size = min_t(unsigned long, compressed_size, 198 PAGE_SIZE); 199 200 kaddr = kmap_atomic(cpage); 201 write_extent_buffer(leaf, kaddr, ptr, cur_size); 202 kunmap_atomic(kaddr); 203 204 i++; 205 ptr += cur_size; 206 compressed_size -= cur_size; 207 } 208 btrfs_set_file_extent_compression(leaf, ei, 209 compress_type); 210 } else { 211 page = find_get_page(inode->i_mapping, 212 start >> PAGE_SHIFT); 213 btrfs_set_file_extent_compression(leaf, ei, 0); 214 kaddr = kmap_atomic(page); 215 offset = start & (PAGE_SIZE - 1); 216 write_extent_buffer(leaf, kaddr + offset, ptr, size); 217 kunmap_atomic(kaddr); 218 put_page(page); 219 } 220 btrfs_mark_buffer_dirty(leaf); 221 btrfs_release_path(path); 222 223 /* 224 * we're an inline extent, so nobody can 225 * extend the file past i_size without locking 226 * a page we already have locked. 227 * 228 * We must do any isize and inode updates 229 * before we unlock the pages. Otherwise we 230 * could end up racing with unlink. 231 */ 232 BTRFS_I(inode)->disk_i_size = inode->i_size; 233 ret = btrfs_update_inode(trans, root, inode); 234 235 return ret; 236 fail: 237 return err; 238 } 239 240 241 /* 242 * conditionally insert an inline extent into the file. This 243 * does the checks required to make sure the data is small enough 244 * to fit as an inline extent. 245 */ 246 static noinline int cow_file_range_inline(struct btrfs_root *root, 247 struct inode *inode, u64 start, 248 u64 end, size_t compressed_size, 249 int compress_type, 250 struct page **compressed_pages) 251 { 252 struct btrfs_fs_info *fs_info = root->fs_info; 253 struct btrfs_trans_handle *trans; 254 u64 isize = i_size_read(inode); 255 u64 actual_end = min(end + 1, isize); 256 u64 inline_len = actual_end - start; 257 u64 aligned_end = ALIGN(end, fs_info->sectorsize); 258 u64 data_len = inline_len; 259 int ret; 260 struct btrfs_path *path; 261 int extent_inserted = 0; 262 u32 extent_item_size; 263 264 if (compressed_size) 265 data_len = compressed_size; 266 267 if (start > 0 || 268 actual_end > fs_info->sectorsize || 269 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 270 (!compressed_size && 271 (actual_end & (fs_info->sectorsize - 1)) == 0) || 272 end + 1 < isize || 273 data_len > fs_info->max_inline) { 274 return 1; 275 } 276 277 path = btrfs_alloc_path(); 278 if (!path) 279 return -ENOMEM; 280 281 trans = btrfs_join_transaction(root); 282 if (IS_ERR(trans)) { 283 btrfs_free_path(path); 284 return PTR_ERR(trans); 285 } 286 trans->block_rsv = &fs_info->delalloc_block_rsv; 287 288 if (compressed_size && compressed_pages) 289 extent_item_size = btrfs_file_extent_calc_inline_size( 290 compressed_size); 291 else 292 extent_item_size = btrfs_file_extent_calc_inline_size( 293 inline_len); 294 295 ret = __btrfs_drop_extents(trans, root, inode, path, 296 start, aligned_end, NULL, 297 1, 1, extent_item_size, &extent_inserted); 298 if (ret) { 299 btrfs_abort_transaction(trans, ret); 300 goto out; 301 } 302 303 if (isize > actual_end) 304 inline_len = min_t(u64, isize, actual_end); 305 ret = insert_inline_extent(trans, path, extent_inserted, 306 root, inode, start, 307 inline_len, compressed_size, 308 compress_type, compressed_pages); 309 if (ret && ret != -ENOSPC) { 310 btrfs_abort_transaction(trans, ret); 311 goto out; 312 } else if (ret == -ENOSPC) { 313 ret = 1; 314 goto out; 315 } 316 317 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 318 btrfs_delalloc_release_metadata(inode, end + 1 - start); 319 btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0); 320 out: 321 /* 322 * Don't forget to free the reserved space, as for inlined extent 323 * it won't count as data extent, free them directly here. 324 * And at reserve time, it's always aligned to page size, so 325 * just free one page here. 326 */ 327 btrfs_qgroup_free_data(inode, 0, PAGE_SIZE); 328 btrfs_free_path(path); 329 btrfs_end_transaction(trans); 330 return ret; 331 } 332 333 struct async_extent { 334 u64 start; 335 u64 ram_size; 336 u64 compressed_size; 337 struct page **pages; 338 unsigned long nr_pages; 339 int compress_type; 340 struct list_head list; 341 }; 342 343 struct async_cow { 344 struct inode *inode; 345 struct btrfs_root *root; 346 struct page *locked_page; 347 u64 start; 348 u64 end; 349 struct list_head extents; 350 struct btrfs_work work; 351 }; 352 353 static noinline int add_async_extent(struct async_cow *cow, 354 u64 start, u64 ram_size, 355 u64 compressed_size, 356 struct page **pages, 357 unsigned long nr_pages, 358 int compress_type) 359 { 360 struct async_extent *async_extent; 361 362 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 363 BUG_ON(!async_extent); /* -ENOMEM */ 364 async_extent->start = start; 365 async_extent->ram_size = ram_size; 366 async_extent->compressed_size = compressed_size; 367 async_extent->pages = pages; 368 async_extent->nr_pages = nr_pages; 369 async_extent->compress_type = compress_type; 370 list_add_tail(&async_extent->list, &cow->extents); 371 return 0; 372 } 373 374 static inline int inode_need_compress(struct inode *inode) 375 { 376 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 377 378 /* force compress */ 379 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 380 return 1; 381 /* bad compression ratios */ 382 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS) 383 return 0; 384 if (btrfs_test_opt(fs_info, COMPRESS) || 385 BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS || 386 BTRFS_I(inode)->force_compress) 387 return 1; 388 return 0; 389 } 390 391 /* 392 * we create compressed extents in two phases. The first 393 * phase compresses a range of pages that have already been 394 * locked (both pages and state bits are locked). 395 * 396 * This is done inside an ordered work queue, and the compression 397 * is spread across many cpus. The actual IO submission is step 398 * two, and the ordered work queue takes care of making sure that 399 * happens in the same order things were put onto the queue by 400 * writepages and friends. 401 * 402 * If this code finds it can't get good compression, it puts an 403 * entry onto the work queue to write the uncompressed bytes. This 404 * makes sure that both compressed inodes and uncompressed inodes 405 * are written in the same order that the flusher thread sent them 406 * down. 407 */ 408 static noinline void compress_file_range(struct inode *inode, 409 struct page *locked_page, 410 u64 start, u64 end, 411 struct async_cow *async_cow, 412 int *num_added) 413 { 414 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 415 struct btrfs_root *root = BTRFS_I(inode)->root; 416 u64 num_bytes; 417 u64 blocksize = fs_info->sectorsize; 418 u64 actual_end; 419 u64 isize = i_size_read(inode); 420 int ret = 0; 421 struct page **pages = NULL; 422 unsigned long nr_pages; 423 unsigned long nr_pages_ret = 0; 424 unsigned long total_compressed = 0; 425 unsigned long total_in = 0; 426 unsigned long max_compressed = SZ_128K; 427 unsigned long max_uncompressed = SZ_128K; 428 int i; 429 int will_compress; 430 int compress_type = fs_info->compress_type; 431 int redirty = 0; 432 433 /* if this is a small write inside eof, kick off a defrag */ 434 if ((end - start + 1) < SZ_16K && 435 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 436 btrfs_add_inode_defrag(NULL, inode); 437 438 actual_end = min_t(u64, isize, end + 1); 439 again: 440 will_compress = 0; 441 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; 442 nr_pages = min_t(unsigned long, nr_pages, SZ_128K / PAGE_SIZE); 443 444 /* 445 * we don't want to send crud past the end of i_size through 446 * compression, that's just a waste of CPU time. So, if the 447 * end of the file is before the start of our current 448 * requested range of bytes, we bail out to the uncompressed 449 * cleanup code that can deal with all of this. 450 * 451 * It isn't really the fastest way to fix things, but this is a 452 * very uncommon corner. 453 */ 454 if (actual_end <= start) 455 goto cleanup_and_bail_uncompressed; 456 457 total_compressed = actual_end - start; 458 459 /* 460 * skip compression for a small file range(<=blocksize) that 461 * isn't an inline extent, since it doesn't save disk space at all. 462 */ 463 if (total_compressed <= blocksize && 464 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 465 goto cleanup_and_bail_uncompressed; 466 467 /* we want to make sure that amount of ram required to uncompress 468 * an extent is reasonable, so we limit the total size in ram 469 * of a compressed extent to 128k. This is a crucial number 470 * because it also controls how easily we can spread reads across 471 * cpus for decompression. 472 * 473 * We also want to make sure the amount of IO required to do 474 * a random read is reasonably small, so we limit the size of 475 * a compressed extent to 128k. 476 */ 477 total_compressed = min(total_compressed, max_uncompressed); 478 num_bytes = ALIGN(end - start + 1, blocksize); 479 num_bytes = max(blocksize, num_bytes); 480 total_in = 0; 481 ret = 0; 482 483 /* 484 * we do compression for mount -o compress and when the 485 * inode has not been flagged as nocompress. This flag can 486 * change at any time if we discover bad compression ratios. 487 */ 488 if (inode_need_compress(inode)) { 489 WARN_ON(pages); 490 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 491 if (!pages) { 492 /* just bail out to the uncompressed code */ 493 goto cont; 494 } 495 496 if (BTRFS_I(inode)->force_compress) 497 compress_type = BTRFS_I(inode)->force_compress; 498 499 /* 500 * we need to call clear_page_dirty_for_io on each 501 * page in the range. Otherwise applications with the file 502 * mmap'd can wander in and change the page contents while 503 * we are compressing them. 504 * 505 * If the compression fails for any reason, we set the pages 506 * dirty again later on. 507 */ 508 extent_range_clear_dirty_for_io(inode, start, end); 509 redirty = 1; 510 ret = btrfs_compress_pages(compress_type, 511 inode->i_mapping, start, 512 total_compressed, pages, 513 nr_pages, &nr_pages_ret, 514 &total_in, 515 &total_compressed, 516 max_compressed); 517 518 if (!ret) { 519 unsigned long offset = total_compressed & 520 (PAGE_SIZE - 1); 521 struct page *page = pages[nr_pages_ret - 1]; 522 char *kaddr; 523 524 /* zero the tail end of the last page, we might be 525 * sending it down to disk 526 */ 527 if (offset) { 528 kaddr = kmap_atomic(page); 529 memset(kaddr + offset, 0, 530 PAGE_SIZE - offset); 531 kunmap_atomic(kaddr); 532 } 533 will_compress = 1; 534 } 535 } 536 cont: 537 if (start == 0) { 538 /* lets try to make an inline extent */ 539 if (ret || total_in < (actual_end - start)) { 540 /* we didn't compress the entire range, try 541 * to make an uncompressed inline extent. 542 */ 543 ret = cow_file_range_inline(root, inode, start, end, 544 0, 0, NULL); 545 } else { 546 /* try making a compressed inline extent */ 547 ret = cow_file_range_inline(root, inode, start, end, 548 total_compressed, 549 compress_type, pages); 550 } 551 if (ret <= 0) { 552 unsigned long clear_flags = EXTENT_DELALLOC | 553 EXTENT_DEFRAG; 554 unsigned long page_error_op; 555 556 clear_flags |= (ret < 0) ? EXTENT_DO_ACCOUNTING : 0; 557 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 558 559 /* 560 * inline extent creation worked or returned error, 561 * we don't need to create any more async work items. 562 * Unlock and free up our temp pages. 563 */ 564 extent_clear_unlock_delalloc(inode, start, end, end, 565 NULL, clear_flags, 566 PAGE_UNLOCK | 567 PAGE_CLEAR_DIRTY | 568 PAGE_SET_WRITEBACK | 569 page_error_op | 570 PAGE_END_WRITEBACK); 571 btrfs_free_reserved_data_space_noquota(inode, start, 572 end - start + 1); 573 goto free_pages_out; 574 } 575 } 576 577 if (will_compress) { 578 /* 579 * we aren't doing an inline extent round the compressed size 580 * up to a block size boundary so the allocator does sane 581 * things 582 */ 583 total_compressed = ALIGN(total_compressed, blocksize); 584 585 /* 586 * one last check to make sure the compression is really a 587 * win, compare the page count read with the blocks on disk 588 */ 589 total_in = ALIGN(total_in, PAGE_SIZE); 590 if (total_compressed >= total_in) { 591 will_compress = 0; 592 } else { 593 num_bytes = total_in; 594 *num_added += 1; 595 596 /* 597 * The async work queues will take care of doing actual 598 * allocation on disk for these compressed pages, and 599 * will submit them to the elevator. 600 */ 601 add_async_extent(async_cow, start, num_bytes, 602 total_compressed, pages, nr_pages_ret, 603 compress_type); 604 605 if (start + num_bytes < end) { 606 start += num_bytes; 607 pages = NULL; 608 cond_resched(); 609 goto again; 610 } 611 return; 612 } 613 } 614 if (pages) { 615 /* 616 * the compression code ran but failed to make things smaller, 617 * free any pages it allocated and our page pointer array 618 */ 619 for (i = 0; i < nr_pages_ret; i++) { 620 WARN_ON(pages[i]->mapping); 621 put_page(pages[i]); 622 } 623 kfree(pages); 624 pages = NULL; 625 total_compressed = 0; 626 nr_pages_ret = 0; 627 628 /* flag the file so we don't compress in the future */ 629 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && 630 !(BTRFS_I(inode)->force_compress)) { 631 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 632 } 633 } 634 cleanup_and_bail_uncompressed: 635 /* 636 * No compression, but we still need to write the pages in the file 637 * we've been given so far. redirty the locked page if it corresponds 638 * to our extent and set things up for the async work queue to run 639 * cow_file_range to do the normal delalloc dance. 640 */ 641 if (page_offset(locked_page) >= start && 642 page_offset(locked_page) <= end) 643 __set_page_dirty_nobuffers(locked_page); 644 /* unlocked later on in the async handlers */ 645 646 if (redirty) 647 extent_range_redirty_for_io(inode, start, end); 648 add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0, 649 BTRFS_COMPRESS_NONE); 650 *num_added += 1; 651 652 return; 653 654 free_pages_out: 655 for (i = 0; i < nr_pages_ret; i++) { 656 WARN_ON(pages[i]->mapping); 657 put_page(pages[i]); 658 } 659 kfree(pages); 660 } 661 662 static void free_async_extent_pages(struct async_extent *async_extent) 663 { 664 int i; 665 666 if (!async_extent->pages) 667 return; 668 669 for (i = 0; i < async_extent->nr_pages; i++) { 670 WARN_ON(async_extent->pages[i]->mapping); 671 put_page(async_extent->pages[i]); 672 } 673 kfree(async_extent->pages); 674 async_extent->nr_pages = 0; 675 async_extent->pages = NULL; 676 } 677 678 /* 679 * phase two of compressed writeback. This is the ordered portion 680 * of the code, which only gets called in the order the work was 681 * queued. We walk all the async extents created by compress_file_range 682 * and send them down to the disk. 683 */ 684 static noinline void submit_compressed_extents(struct inode *inode, 685 struct async_cow *async_cow) 686 { 687 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 688 struct async_extent *async_extent; 689 u64 alloc_hint = 0; 690 struct btrfs_key ins; 691 struct extent_map *em; 692 struct btrfs_root *root = BTRFS_I(inode)->root; 693 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 694 struct extent_io_tree *io_tree; 695 int ret = 0; 696 697 again: 698 while (!list_empty(&async_cow->extents)) { 699 async_extent = list_entry(async_cow->extents.next, 700 struct async_extent, list); 701 list_del(&async_extent->list); 702 703 io_tree = &BTRFS_I(inode)->io_tree; 704 705 retry: 706 /* did the compression code fall back to uncompressed IO? */ 707 if (!async_extent->pages) { 708 int page_started = 0; 709 unsigned long nr_written = 0; 710 711 lock_extent(io_tree, async_extent->start, 712 async_extent->start + 713 async_extent->ram_size - 1); 714 715 /* allocate blocks */ 716 ret = cow_file_range(inode, async_cow->locked_page, 717 async_extent->start, 718 async_extent->start + 719 async_extent->ram_size - 1, 720 async_extent->start + 721 async_extent->ram_size - 1, 722 &page_started, &nr_written, 0, 723 NULL); 724 725 /* JDM XXX */ 726 727 /* 728 * if page_started, cow_file_range inserted an 729 * inline extent and took care of all the unlocking 730 * and IO for us. Otherwise, we need to submit 731 * all those pages down to the drive. 732 */ 733 if (!page_started && !ret) 734 extent_write_locked_range(io_tree, 735 inode, async_extent->start, 736 async_extent->start + 737 async_extent->ram_size - 1, 738 btrfs_get_extent, 739 WB_SYNC_ALL); 740 else if (ret) 741 unlock_page(async_cow->locked_page); 742 kfree(async_extent); 743 cond_resched(); 744 continue; 745 } 746 747 lock_extent(io_tree, async_extent->start, 748 async_extent->start + async_extent->ram_size - 1); 749 750 ret = btrfs_reserve_extent(root, async_extent->ram_size, 751 async_extent->compressed_size, 752 async_extent->compressed_size, 753 0, alloc_hint, &ins, 1, 1); 754 if (ret) { 755 free_async_extent_pages(async_extent); 756 757 if (ret == -ENOSPC) { 758 unlock_extent(io_tree, async_extent->start, 759 async_extent->start + 760 async_extent->ram_size - 1); 761 762 /* 763 * we need to redirty the pages if we decide to 764 * fallback to uncompressed IO, otherwise we 765 * will not submit these pages down to lower 766 * layers. 767 */ 768 extent_range_redirty_for_io(inode, 769 async_extent->start, 770 async_extent->start + 771 async_extent->ram_size - 1); 772 773 goto retry; 774 } 775 goto out_free; 776 } 777 /* 778 * here we're doing allocation and writeback of the 779 * compressed pages 780 */ 781 btrfs_drop_extent_cache(inode, async_extent->start, 782 async_extent->start + 783 async_extent->ram_size - 1, 0); 784 785 em = alloc_extent_map(); 786 if (!em) { 787 ret = -ENOMEM; 788 goto out_free_reserve; 789 } 790 em->start = async_extent->start; 791 em->len = async_extent->ram_size; 792 em->orig_start = em->start; 793 em->mod_start = em->start; 794 em->mod_len = em->len; 795 796 em->block_start = ins.objectid; 797 em->block_len = ins.offset; 798 em->orig_block_len = ins.offset; 799 em->ram_bytes = async_extent->ram_size; 800 em->bdev = fs_info->fs_devices->latest_bdev; 801 em->compress_type = async_extent->compress_type; 802 set_bit(EXTENT_FLAG_PINNED, &em->flags); 803 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 804 em->generation = -1; 805 806 while (1) { 807 write_lock(&em_tree->lock); 808 ret = add_extent_mapping(em_tree, em, 1); 809 write_unlock(&em_tree->lock); 810 if (ret != -EEXIST) { 811 free_extent_map(em); 812 break; 813 } 814 btrfs_drop_extent_cache(inode, async_extent->start, 815 async_extent->start + 816 async_extent->ram_size - 1, 0); 817 } 818 819 if (ret) 820 goto out_free_reserve; 821 822 ret = btrfs_add_ordered_extent_compress(inode, 823 async_extent->start, 824 ins.objectid, 825 async_extent->ram_size, 826 ins.offset, 827 BTRFS_ORDERED_COMPRESSED, 828 async_extent->compress_type); 829 if (ret) { 830 btrfs_drop_extent_cache(inode, async_extent->start, 831 async_extent->start + 832 async_extent->ram_size - 1, 0); 833 goto out_free_reserve; 834 } 835 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 836 837 /* 838 * clear dirty, set writeback and unlock the pages. 839 */ 840 extent_clear_unlock_delalloc(inode, async_extent->start, 841 async_extent->start + 842 async_extent->ram_size - 1, 843 async_extent->start + 844 async_extent->ram_size - 1, 845 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 846 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 847 PAGE_SET_WRITEBACK); 848 ret = btrfs_submit_compressed_write(inode, 849 async_extent->start, 850 async_extent->ram_size, 851 ins.objectid, 852 ins.offset, async_extent->pages, 853 async_extent->nr_pages); 854 if (ret) { 855 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 856 struct page *p = async_extent->pages[0]; 857 const u64 start = async_extent->start; 858 const u64 end = start + async_extent->ram_size - 1; 859 860 p->mapping = inode->i_mapping; 861 tree->ops->writepage_end_io_hook(p, start, end, 862 NULL, 0); 863 p->mapping = NULL; 864 extent_clear_unlock_delalloc(inode, start, end, end, 865 NULL, 0, 866 PAGE_END_WRITEBACK | 867 PAGE_SET_ERROR); 868 free_async_extent_pages(async_extent); 869 } 870 alloc_hint = ins.objectid + ins.offset; 871 kfree(async_extent); 872 cond_resched(); 873 } 874 return; 875 out_free_reserve: 876 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 877 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 878 out_free: 879 extent_clear_unlock_delalloc(inode, async_extent->start, 880 async_extent->start + 881 async_extent->ram_size - 1, 882 async_extent->start + 883 async_extent->ram_size - 1, 884 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 885 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 886 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 887 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | 888 PAGE_SET_ERROR); 889 free_async_extent_pages(async_extent); 890 kfree(async_extent); 891 goto again; 892 } 893 894 static u64 get_extent_allocation_hint(struct inode *inode, u64 start, 895 u64 num_bytes) 896 { 897 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 898 struct extent_map *em; 899 u64 alloc_hint = 0; 900 901 read_lock(&em_tree->lock); 902 em = search_extent_mapping(em_tree, start, num_bytes); 903 if (em) { 904 /* 905 * if block start isn't an actual block number then find the 906 * first block in this inode and use that as a hint. If that 907 * block is also bogus then just don't worry about it. 908 */ 909 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 910 free_extent_map(em); 911 em = search_extent_mapping(em_tree, 0, 0); 912 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 913 alloc_hint = em->block_start; 914 if (em) 915 free_extent_map(em); 916 } else { 917 alloc_hint = em->block_start; 918 free_extent_map(em); 919 } 920 } 921 read_unlock(&em_tree->lock); 922 923 return alloc_hint; 924 } 925 926 /* 927 * when extent_io.c finds a delayed allocation range in the file, 928 * the call backs end up in this code. The basic idea is to 929 * allocate extents on disk for the range, and create ordered data structs 930 * in ram to track those extents. 931 * 932 * locked_page is the page that writepage had locked already. We use 933 * it to make sure we don't do extra locks or unlocks. 934 * 935 * *page_started is set to one if we unlock locked_page and do everything 936 * required to start IO on it. It may be clean and already done with 937 * IO when we return. 938 */ 939 static noinline int cow_file_range(struct inode *inode, 940 struct page *locked_page, 941 u64 start, u64 end, u64 delalloc_end, 942 int *page_started, unsigned long *nr_written, 943 int unlock, struct btrfs_dedupe_hash *hash) 944 { 945 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 946 struct btrfs_root *root = BTRFS_I(inode)->root; 947 u64 alloc_hint = 0; 948 u64 num_bytes; 949 unsigned long ram_size; 950 u64 disk_num_bytes; 951 u64 cur_alloc_size; 952 u64 blocksize = fs_info->sectorsize; 953 struct btrfs_key ins; 954 struct extent_map *em; 955 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 956 int ret = 0; 957 958 if (btrfs_is_free_space_inode(inode)) { 959 WARN_ON_ONCE(1); 960 ret = -EINVAL; 961 goto out_unlock; 962 } 963 964 num_bytes = ALIGN(end - start + 1, blocksize); 965 num_bytes = max(blocksize, num_bytes); 966 disk_num_bytes = num_bytes; 967 968 /* if this is a small write inside eof, kick off defrag */ 969 if (num_bytes < SZ_64K && 970 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 971 btrfs_add_inode_defrag(NULL, inode); 972 973 if (start == 0) { 974 /* lets try to make an inline extent */ 975 ret = cow_file_range_inline(root, inode, start, end, 0, 0, 976 NULL); 977 if (ret == 0) { 978 extent_clear_unlock_delalloc(inode, start, end, 979 delalloc_end, NULL, 980 EXTENT_LOCKED | EXTENT_DELALLOC | 981 EXTENT_DEFRAG, PAGE_UNLOCK | 982 PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 983 PAGE_END_WRITEBACK); 984 btrfs_free_reserved_data_space_noquota(inode, start, 985 end - start + 1); 986 *nr_written = *nr_written + 987 (end - start + PAGE_SIZE) / PAGE_SIZE; 988 *page_started = 1; 989 goto out; 990 } else if (ret < 0) { 991 goto out_unlock; 992 } 993 } 994 995 BUG_ON(disk_num_bytes > 996 btrfs_super_total_bytes(fs_info->super_copy)); 997 998 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 999 btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0); 1000 1001 while (disk_num_bytes > 0) { 1002 unsigned long op; 1003 1004 cur_alloc_size = disk_num_bytes; 1005 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size, 1006 fs_info->sectorsize, 0, alloc_hint, 1007 &ins, 1, 1); 1008 if (ret < 0) 1009 goto out_unlock; 1010 1011 em = alloc_extent_map(); 1012 if (!em) { 1013 ret = -ENOMEM; 1014 goto out_reserve; 1015 } 1016 em->start = start; 1017 em->orig_start = em->start; 1018 ram_size = ins.offset; 1019 em->len = ins.offset; 1020 em->mod_start = em->start; 1021 em->mod_len = em->len; 1022 1023 em->block_start = ins.objectid; 1024 em->block_len = ins.offset; 1025 em->orig_block_len = ins.offset; 1026 em->ram_bytes = ram_size; 1027 em->bdev = fs_info->fs_devices->latest_bdev; 1028 set_bit(EXTENT_FLAG_PINNED, &em->flags); 1029 em->generation = -1; 1030 1031 while (1) { 1032 write_lock(&em_tree->lock); 1033 ret = add_extent_mapping(em_tree, em, 1); 1034 write_unlock(&em_tree->lock); 1035 if (ret != -EEXIST) { 1036 free_extent_map(em); 1037 break; 1038 } 1039 btrfs_drop_extent_cache(inode, start, 1040 start + ram_size - 1, 0); 1041 } 1042 if (ret) 1043 goto out_reserve; 1044 1045 cur_alloc_size = ins.offset; 1046 ret = btrfs_add_ordered_extent(inode, start, ins.objectid, 1047 ram_size, cur_alloc_size, 0); 1048 if (ret) 1049 goto out_drop_extent_cache; 1050 1051 if (root->root_key.objectid == 1052 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1053 ret = btrfs_reloc_clone_csums(inode, start, 1054 cur_alloc_size); 1055 if (ret) 1056 goto out_drop_extent_cache; 1057 } 1058 1059 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1060 1061 if (disk_num_bytes < cur_alloc_size) 1062 break; 1063 1064 /* we're not doing compressed IO, don't unlock the first 1065 * page (which the caller expects to stay locked), don't 1066 * clear any dirty bits and don't set any writeback bits 1067 * 1068 * Do set the Private2 bit so we know this page was properly 1069 * setup for writepage 1070 */ 1071 op = unlock ? PAGE_UNLOCK : 0; 1072 op |= PAGE_SET_PRIVATE2; 1073 1074 extent_clear_unlock_delalloc(inode, start, 1075 start + ram_size - 1, 1076 delalloc_end, locked_page, 1077 EXTENT_LOCKED | EXTENT_DELALLOC, 1078 op); 1079 disk_num_bytes -= cur_alloc_size; 1080 num_bytes -= cur_alloc_size; 1081 alloc_hint = ins.objectid + ins.offset; 1082 start += cur_alloc_size; 1083 } 1084 out: 1085 return ret; 1086 1087 out_drop_extent_cache: 1088 btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0); 1089 out_reserve: 1090 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1091 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1092 out_unlock: 1093 extent_clear_unlock_delalloc(inode, start, end, delalloc_end, 1094 locked_page, 1095 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 1096 EXTENT_DELALLOC | EXTENT_DEFRAG, 1097 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 1098 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK); 1099 goto out; 1100 } 1101 1102 /* 1103 * work queue call back to started compression on a file and pages 1104 */ 1105 static noinline void async_cow_start(struct btrfs_work *work) 1106 { 1107 struct async_cow *async_cow; 1108 int num_added = 0; 1109 async_cow = container_of(work, struct async_cow, work); 1110 1111 compress_file_range(async_cow->inode, async_cow->locked_page, 1112 async_cow->start, async_cow->end, async_cow, 1113 &num_added); 1114 if (num_added == 0) { 1115 btrfs_add_delayed_iput(async_cow->inode); 1116 async_cow->inode = NULL; 1117 } 1118 } 1119 1120 /* 1121 * work queue call back to submit previously compressed pages 1122 */ 1123 static noinline void async_cow_submit(struct btrfs_work *work) 1124 { 1125 struct btrfs_fs_info *fs_info; 1126 struct async_cow *async_cow; 1127 struct btrfs_root *root; 1128 unsigned long nr_pages; 1129 1130 async_cow = container_of(work, struct async_cow, work); 1131 1132 root = async_cow->root; 1133 fs_info = root->fs_info; 1134 nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >> 1135 PAGE_SHIFT; 1136 1137 /* 1138 * atomic_sub_return implies a barrier for waitqueue_active 1139 */ 1140 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1141 5 * SZ_1M && 1142 waitqueue_active(&fs_info->async_submit_wait)) 1143 wake_up(&fs_info->async_submit_wait); 1144 1145 if (async_cow->inode) 1146 submit_compressed_extents(async_cow->inode, async_cow); 1147 } 1148 1149 static noinline void async_cow_free(struct btrfs_work *work) 1150 { 1151 struct async_cow *async_cow; 1152 async_cow = container_of(work, struct async_cow, work); 1153 if (async_cow->inode) 1154 btrfs_add_delayed_iput(async_cow->inode); 1155 kfree(async_cow); 1156 } 1157 1158 static int cow_file_range_async(struct inode *inode, struct page *locked_page, 1159 u64 start, u64 end, int *page_started, 1160 unsigned long *nr_written) 1161 { 1162 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1163 struct async_cow *async_cow; 1164 struct btrfs_root *root = BTRFS_I(inode)->root; 1165 unsigned long nr_pages; 1166 u64 cur_end; 1167 int limit = 10 * SZ_1M; 1168 1169 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED, 1170 1, 0, NULL, GFP_NOFS); 1171 while (start < end) { 1172 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS); 1173 BUG_ON(!async_cow); /* -ENOMEM */ 1174 async_cow->inode = igrab(inode); 1175 async_cow->root = root; 1176 async_cow->locked_page = locked_page; 1177 async_cow->start = start; 1178 1179 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS && 1180 !btrfs_test_opt(fs_info, FORCE_COMPRESS)) 1181 cur_end = end; 1182 else 1183 cur_end = min(end, start + SZ_512K - 1); 1184 1185 async_cow->end = cur_end; 1186 INIT_LIST_HEAD(&async_cow->extents); 1187 1188 btrfs_init_work(&async_cow->work, 1189 btrfs_delalloc_helper, 1190 async_cow_start, async_cow_submit, 1191 async_cow_free); 1192 1193 nr_pages = (cur_end - start + PAGE_SIZE) >> 1194 PAGE_SHIFT; 1195 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1196 1197 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work); 1198 1199 if (atomic_read(&fs_info->async_delalloc_pages) > limit) { 1200 wait_event(fs_info->async_submit_wait, 1201 (atomic_read(&fs_info->async_delalloc_pages) < 1202 limit)); 1203 } 1204 1205 while (atomic_read(&fs_info->async_submit_draining) && 1206 atomic_read(&fs_info->async_delalloc_pages)) { 1207 wait_event(fs_info->async_submit_wait, 1208 (atomic_read(&fs_info->async_delalloc_pages) == 1209 0)); 1210 } 1211 1212 *nr_written += nr_pages; 1213 start = cur_end + 1; 1214 } 1215 *page_started = 1; 1216 return 0; 1217 } 1218 1219 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, 1220 u64 bytenr, u64 num_bytes) 1221 { 1222 int ret; 1223 struct btrfs_ordered_sum *sums; 1224 LIST_HEAD(list); 1225 1226 ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr, 1227 bytenr + num_bytes - 1, &list, 0); 1228 if (ret == 0 && list_empty(&list)) 1229 return 0; 1230 1231 while (!list_empty(&list)) { 1232 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1233 list_del(&sums->list); 1234 kfree(sums); 1235 } 1236 return 1; 1237 } 1238 1239 /* 1240 * when nowcow writeback call back. This checks for snapshots or COW copies 1241 * of the extents that exist in the file, and COWs the file as required. 1242 * 1243 * If no cow copies or snapshots exist, we write directly to the existing 1244 * blocks on disk 1245 */ 1246 static noinline int run_delalloc_nocow(struct inode *inode, 1247 struct page *locked_page, 1248 u64 start, u64 end, int *page_started, int force, 1249 unsigned long *nr_written) 1250 { 1251 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1252 struct btrfs_root *root = BTRFS_I(inode)->root; 1253 struct btrfs_trans_handle *trans; 1254 struct extent_buffer *leaf; 1255 struct btrfs_path *path; 1256 struct btrfs_file_extent_item *fi; 1257 struct btrfs_key found_key; 1258 u64 cow_start; 1259 u64 cur_offset; 1260 u64 extent_end; 1261 u64 extent_offset; 1262 u64 disk_bytenr; 1263 u64 num_bytes; 1264 u64 disk_num_bytes; 1265 u64 ram_bytes; 1266 int extent_type; 1267 int ret, err; 1268 int type; 1269 int nocow; 1270 int check_prev = 1; 1271 bool nolock; 1272 u64 ino = btrfs_ino(inode); 1273 1274 path = btrfs_alloc_path(); 1275 if (!path) { 1276 extent_clear_unlock_delalloc(inode, start, end, end, 1277 locked_page, 1278 EXTENT_LOCKED | EXTENT_DELALLOC | 1279 EXTENT_DO_ACCOUNTING | 1280 EXTENT_DEFRAG, PAGE_UNLOCK | 1281 PAGE_CLEAR_DIRTY | 1282 PAGE_SET_WRITEBACK | 1283 PAGE_END_WRITEBACK); 1284 return -ENOMEM; 1285 } 1286 1287 nolock = btrfs_is_free_space_inode(inode); 1288 1289 if (nolock) 1290 trans = btrfs_join_transaction_nolock(root); 1291 else 1292 trans = btrfs_join_transaction(root); 1293 1294 if (IS_ERR(trans)) { 1295 extent_clear_unlock_delalloc(inode, start, end, end, 1296 locked_page, 1297 EXTENT_LOCKED | EXTENT_DELALLOC | 1298 EXTENT_DO_ACCOUNTING | 1299 EXTENT_DEFRAG, PAGE_UNLOCK | 1300 PAGE_CLEAR_DIRTY | 1301 PAGE_SET_WRITEBACK | 1302 PAGE_END_WRITEBACK); 1303 btrfs_free_path(path); 1304 return PTR_ERR(trans); 1305 } 1306 1307 trans->block_rsv = &fs_info->delalloc_block_rsv; 1308 1309 cow_start = (u64)-1; 1310 cur_offset = start; 1311 while (1) { 1312 ret = btrfs_lookup_file_extent(trans, root, path, ino, 1313 cur_offset, 0); 1314 if (ret < 0) 1315 goto error; 1316 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1317 leaf = path->nodes[0]; 1318 btrfs_item_key_to_cpu(leaf, &found_key, 1319 path->slots[0] - 1); 1320 if (found_key.objectid == ino && 1321 found_key.type == BTRFS_EXTENT_DATA_KEY) 1322 path->slots[0]--; 1323 } 1324 check_prev = 0; 1325 next_slot: 1326 leaf = path->nodes[0]; 1327 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1328 ret = btrfs_next_leaf(root, path); 1329 if (ret < 0) 1330 goto error; 1331 if (ret > 0) 1332 break; 1333 leaf = path->nodes[0]; 1334 } 1335 1336 nocow = 0; 1337 disk_bytenr = 0; 1338 num_bytes = 0; 1339 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1340 1341 if (found_key.objectid > ino) 1342 break; 1343 if (WARN_ON_ONCE(found_key.objectid < ino) || 1344 found_key.type < BTRFS_EXTENT_DATA_KEY) { 1345 path->slots[0]++; 1346 goto next_slot; 1347 } 1348 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 1349 found_key.offset > end) 1350 break; 1351 1352 if (found_key.offset > cur_offset) { 1353 extent_end = found_key.offset; 1354 extent_type = 0; 1355 goto out_check; 1356 } 1357 1358 fi = btrfs_item_ptr(leaf, path->slots[0], 1359 struct btrfs_file_extent_item); 1360 extent_type = btrfs_file_extent_type(leaf, fi); 1361 1362 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1363 if (extent_type == BTRFS_FILE_EXTENT_REG || 1364 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1365 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1366 extent_offset = btrfs_file_extent_offset(leaf, fi); 1367 extent_end = found_key.offset + 1368 btrfs_file_extent_num_bytes(leaf, fi); 1369 disk_num_bytes = 1370 btrfs_file_extent_disk_num_bytes(leaf, fi); 1371 if (extent_end <= start) { 1372 path->slots[0]++; 1373 goto next_slot; 1374 } 1375 if (disk_bytenr == 0) 1376 goto out_check; 1377 if (btrfs_file_extent_compression(leaf, fi) || 1378 btrfs_file_extent_encryption(leaf, fi) || 1379 btrfs_file_extent_other_encoding(leaf, fi)) 1380 goto out_check; 1381 if (extent_type == BTRFS_FILE_EXTENT_REG && !force) 1382 goto out_check; 1383 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 1384 goto out_check; 1385 if (btrfs_cross_ref_exist(trans, root, ino, 1386 found_key.offset - 1387 extent_offset, disk_bytenr)) 1388 goto out_check; 1389 disk_bytenr += extent_offset; 1390 disk_bytenr += cur_offset - found_key.offset; 1391 num_bytes = min(end + 1, extent_end) - cur_offset; 1392 /* 1393 * if there are pending snapshots for this root, 1394 * we fall into common COW way. 1395 */ 1396 if (!nolock) { 1397 err = btrfs_start_write_no_snapshoting(root); 1398 if (!err) 1399 goto out_check; 1400 } 1401 /* 1402 * force cow if csum exists in the range. 1403 * this ensure that csum for a given extent are 1404 * either valid or do not exist. 1405 */ 1406 if (csum_exist_in_range(fs_info, disk_bytenr, 1407 num_bytes)) 1408 goto out_check; 1409 if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) 1410 goto out_check; 1411 nocow = 1; 1412 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 1413 extent_end = found_key.offset + 1414 btrfs_file_extent_inline_len(leaf, 1415 path->slots[0], fi); 1416 extent_end = ALIGN(extent_end, 1417 fs_info->sectorsize); 1418 } else { 1419 BUG_ON(1); 1420 } 1421 out_check: 1422 if (extent_end <= start) { 1423 path->slots[0]++; 1424 if (!nolock && nocow) 1425 btrfs_end_write_no_snapshoting(root); 1426 if (nocow) 1427 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1428 goto next_slot; 1429 } 1430 if (!nocow) { 1431 if (cow_start == (u64)-1) 1432 cow_start = cur_offset; 1433 cur_offset = extent_end; 1434 if (cur_offset > end) 1435 break; 1436 path->slots[0]++; 1437 goto next_slot; 1438 } 1439 1440 btrfs_release_path(path); 1441 if (cow_start != (u64)-1) { 1442 ret = cow_file_range(inode, locked_page, 1443 cow_start, found_key.offset - 1, 1444 end, page_started, nr_written, 1, 1445 NULL); 1446 if (ret) { 1447 if (!nolock && nocow) 1448 btrfs_end_write_no_snapshoting(root); 1449 if (nocow) 1450 btrfs_dec_nocow_writers(fs_info, 1451 disk_bytenr); 1452 goto error; 1453 } 1454 cow_start = (u64)-1; 1455 } 1456 1457 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1458 struct extent_map *em; 1459 struct extent_map_tree *em_tree; 1460 em_tree = &BTRFS_I(inode)->extent_tree; 1461 em = alloc_extent_map(); 1462 BUG_ON(!em); /* -ENOMEM */ 1463 em->start = cur_offset; 1464 em->orig_start = found_key.offset - extent_offset; 1465 em->len = num_bytes; 1466 em->block_len = num_bytes; 1467 em->block_start = disk_bytenr; 1468 em->orig_block_len = disk_num_bytes; 1469 em->ram_bytes = ram_bytes; 1470 em->bdev = fs_info->fs_devices->latest_bdev; 1471 em->mod_start = em->start; 1472 em->mod_len = em->len; 1473 set_bit(EXTENT_FLAG_PINNED, &em->flags); 1474 set_bit(EXTENT_FLAG_FILLING, &em->flags); 1475 em->generation = -1; 1476 while (1) { 1477 write_lock(&em_tree->lock); 1478 ret = add_extent_mapping(em_tree, em, 1); 1479 write_unlock(&em_tree->lock); 1480 if (ret != -EEXIST) { 1481 free_extent_map(em); 1482 break; 1483 } 1484 btrfs_drop_extent_cache(inode, em->start, 1485 em->start + em->len - 1, 0); 1486 } 1487 type = BTRFS_ORDERED_PREALLOC; 1488 } else { 1489 type = BTRFS_ORDERED_NOCOW; 1490 } 1491 1492 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr, 1493 num_bytes, num_bytes, type); 1494 if (nocow) 1495 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1496 BUG_ON(ret); /* -ENOMEM */ 1497 1498 if (root->root_key.objectid == 1499 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1500 ret = btrfs_reloc_clone_csums(inode, cur_offset, 1501 num_bytes); 1502 if (ret) { 1503 if (!nolock && nocow) 1504 btrfs_end_write_no_snapshoting(root); 1505 goto error; 1506 } 1507 } 1508 1509 extent_clear_unlock_delalloc(inode, cur_offset, 1510 cur_offset + num_bytes - 1, end, 1511 locked_page, EXTENT_LOCKED | 1512 EXTENT_DELALLOC | 1513 EXTENT_CLEAR_DATA_RESV, 1514 PAGE_UNLOCK | PAGE_SET_PRIVATE2); 1515 1516 if (!nolock && nocow) 1517 btrfs_end_write_no_snapshoting(root); 1518 cur_offset = extent_end; 1519 if (cur_offset > end) 1520 break; 1521 } 1522 btrfs_release_path(path); 1523 1524 if (cur_offset <= end && cow_start == (u64)-1) { 1525 cow_start = cur_offset; 1526 cur_offset = end; 1527 } 1528 1529 if (cow_start != (u64)-1) { 1530 ret = cow_file_range(inode, locked_page, cow_start, end, end, 1531 page_started, nr_written, 1, NULL); 1532 if (ret) 1533 goto error; 1534 } 1535 1536 error: 1537 err = btrfs_end_transaction(trans); 1538 if (!ret) 1539 ret = err; 1540 1541 if (ret && cur_offset < end) 1542 extent_clear_unlock_delalloc(inode, cur_offset, end, end, 1543 locked_page, EXTENT_LOCKED | 1544 EXTENT_DELALLOC | EXTENT_DEFRAG | 1545 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1546 PAGE_CLEAR_DIRTY | 1547 PAGE_SET_WRITEBACK | 1548 PAGE_END_WRITEBACK); 1549 btrfs_free_path(path); 1550 return ret; 1551 } 1552 1553 static inline int need_force_cow(struct inode *inode, u64 start, u64 end) 1554 { 1555 1556 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 1557 !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC)) 1558 return 0; 1559 1560 /* 1561 * @defrag_bytes is a hint value, no spinlock held here, 1562 * if is not zero, it means the file is defragging. 1563 * Force cow if given extent needs to be defragged. 1564 */ 1565 if (BTRFS_I(inode)->defrag_bytes && 1566 test_range_bit(&BTRFS_I(inode)->io_tree, start, end, 1567 EXTENT_DEFRAG, 0, NULL)) 1568 return 1; 1569 1570 return 0; 1571 } 1572 1573 /* 1574 * extent_io.c call back to do delayed allocation processing 1575 */ 1576 static int run_delalloc_range(struct inode *inode, struct page *locked_page, 1577 u64 start, u64 end, int *page_started, 1578 unsigned long *nr_written) 1579 { 1580 int ret; 1581 int force_cow = need_force_cow(inode, start, end); 1582 1583 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) { 1584 ret = run_delalloc_nocow(inode, locked_page, start, end, 1585 page_started, 1, nr_written); 1586 } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) { 1587 ret = run_delalloc_nocow(inode, locked_page, start, end, 1588 page_started, 0, nr_written); 1589 } else if (!inode_need_compress(inode)) { 1590 ret = cow_file_range(inode, locked_page, start, end, end, 1591 page_started, nr_written, 1, NULL); 1592 } else { 1593 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 1594 &BTRFS_I(inode)->runtime_flags); 1595 ret = cow_file_range_async(inode, locked_page, start, end, 1596 page_started, nr_written); 1597 } 1598 return ret; 1599 } 1600 1601 static void btrfs_split_extent_hook(struct inode *inode, 1602 struct extent_state *orig, u64 split) 1603 { 1604 u64 size; 1605 1606 /* not delalloc, ignore it */ 1607 if (!(orig->state & EXTENT_DELALLOC)) 1608 return; 1609 1610 size = orig->end - orig->start + 1; 1611 if (size > BTRFS_MAX_EXTENT_SIZE) { 1612 u64 num_extents; 1613 u64 new_size; 1614 1615 /* 1616 * See the explanation in btrfs_merge_extent_hook, the same 1617 * applies here, just in reverse. 1618 */ 1619 new_size = orig->end - split + 1; 1620 num_extents = div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1, 1621 BTRFS_MAX_EXTENT_SIZE); 1622 new_size = split - orig->start; 1623 num_extents += div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1, 1624 BTRFS_MAX_EXTENT_SIZE); 1625 if (div64_u64(size + BTRFS_MAX_EXTENT_SIZE - 1, 1626 BTRFS_MAX_EXTENT_SIZE) >= num_extents) 1627 return; 1628 } 1629 1630 spin_lock(&BTRFS_I(inode)->lock); 1631 BTRFS_I(inode)->outstanding_extents++; 1632 spin_unlock(&BTRFS_I(inode)->lock); 1633 } 1634 1635 /* 1636 * extent_io.c merge_extent_hook, used to track merged delayed allocation 1637 * extents so we can keep track of new extents that are just merged onto old 1638 * extents, such as when we are doing sequential writes, so we can properly 1639 * account for the metadata space we'll need. 1640 */ 1641 static void btrfs_merge_extent_hook(struct inode *inode, 1642 struct extent_state *new, 1643 struct extent_state *other) 1644 { 1645 u64 new_size, old_size; 1646 u64 num_extents; 1647 1648 /* not delalloc, ignore it */ 1649 if (!(other->state & EXTENT_DELALLOC)) 1650 return; 1651 1652 if (new->start > other->start) 1653 new_size = new->end - other->start + 1; 1654 else 1655 new_size = other->end - new->start + 1; 1656 1657 /* we're not bigger than the max, unreserve the space and go */ 1658 if (new_size <= BTRFS_MAX_EXTENT_SIZE) { 1659 spin_lock(&BTRFS_I(inode)->lock); 1660 BTRFS_I(inode)->outstanding_extents--; 1661 spin_unlock(&BTRFS_I(inode)->lock); 1662 return; 1663 } 1664 1665 /* 1666 * We have to add up either side to figure out how many extents were 1667 * accounted for before we merged into one big extent. If the number of 1668 * extents we accounted for is <= the amount we need for the new range 1669 * then we can return, otherwise drop. Think of it like this 1670 * 1671 * [ 4k][MAX_SIZE] 1672 * 1673 * So we've grown the extent by a MAX_SIZE extent, this would mean we 1674 * need 2 outstanding extents, on one side we have 1 and the other side 1675 * we have 1 so they are == and we can return. But in this case 1676 * 1677 * [MAX_SIZE+4k][MAX_SIZE+4k] 1678 * 1679 * Each range on their own accounts for 2 extents, but merged together 1680 * they are only 3 extents worth of accounting, so we need to drop in 1681 * this case. 1682 */ 1683 old_size = other->end - other->start + 1; 1684 num_extents = div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1, 1685 BTRFS_MAX_EXTENT_SIZE); 1686 old_size = new->end - new->start + 1; 1687 num_extents += div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1, 1688 BTRFS_MAX_EXTENT_SIZE); 1689 1690 if (div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1, 1691 BTRFS_MAX_EXTENT_SIZE) >= num_extents) 1692 return; 1693 1694 spin_lock(&BTRFS_I(inode)->lock); 1695 BTRFS_I(inode)->outstanding_extents--; 1696 spin_unlock(&BTRFS_I(inode)->lock); 1697 } 1698 1699 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 1700 struct inode *inode) 1701 { 1702 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1703 1704 spin_lock(&root->delalloc_lock); 1705 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1706 list_add_tail(&BTRFS_I(inode)->delalloc_inodes, 1707 &root->delalloc_inodes); 1708 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1709 &BTRFS_I(inode)->runtime_flags); 1710 root->nr_delalloc_inodes++; 1711 if (root->nr_delalloc_inodes == 1) { 1712 spin_lock(&fs_info->delalloc_root_lock); 1713 BUG_ON(!list_empty(&root->delalloc_root)); 1714 list_add_tail(&root->delalloc_root, 1715 &fs_info->delalloc_roots); 1716 spin_unlock(&fs_info->delalloc_root_lock); 1717 } 1718 } 1719 spin_unlock(&root->delalloc_lock); 1720 } 1721 1722 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 1723 struct inode *inode) 1724 { 1725 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1726 1727 spin_lock(&root->delalloc_lock); 1728 if (!list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1729 list_del_init(&BTRFS_I(inode)->delalloc_inodes); 1730 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1731 &BTRFS_I(inode)->runtime_flags); 1732 root->nr_delalloc_inodes--; 1733 if (!root->nr_delalloc_inodes) { 1734 spin_lock(&fs_info->delalloc_root_lock); 1735 BUG_ON(list_empty(&root->delalloc_root)); 1736 list_del_init(&root->delalloc_root); 1737 spin_unlock(&fs_info->delalloc_root_lock); 1738 } 1739 } 1740 spin_unlock(&root->delalloc_lock); 1741 } 1742 1743 /* 1744 * extent_io.c set_bit_hook, used to track delayed allocation 1745 * bytes in this file, and to maintain the list of inodes that 1746 * have pending delalloc work to be done. 1747 */ 1748 static void btrfs_set_bit_hook(struct inode *inode, 1749 struct extent_state *state, unsigned *bits) 1750 { 1751 1752 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1753 1754 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC)) 1755 WARN_ON(1); 1756 /* 1757 * set_bit and clear bit hooks normally require _irqsave/restore 1758 * but in this case, we are only testing for the DELALLOC 1759 * bit, which is only set or cleared with irqs on 1760 */ 1761 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1762 struct btrfs_root *root = BTRFS_I(inode)->root; 1763 u64 len = state->end + 1 - state->start; 1764 bool do_list = !btrfs_is_free_space_inode(inode); 1765 1766 if (*bits & EXTENT_FIRST_DELALLOC) { 1767 *bits &= ~EXTENT_FIRST_DELALLOC; 1768 } else { 1769 spin_lock(&BTRFS_I(inode)->lock); 1770 BTRFS_I(inode)->outstanding_extents++; 1771 spin_unlock(&BTRFS_I(inode)->lock); 1772 } 1773 1774 /* For sanity tests */ 1775 if (btrfs_is_testing(fs_info)) 1776 return; 1777 1778 __percpu_counter_add(&fs_info->delalloc_bytes, len, 1779 fs_info->delalloc_batch); 1780 spin_lock(&BTRFS_I(inode)->lock); 1781 BTRFS_I(inode)->delalloc_bytes += len; 1782 if (*bits & EXTENT_DEFRAG) 1783 BTRFS_I(inode)->defrag_bytes += len; 1784 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1785 &BTRFS_I(inode)->runtime_flags)) 1786 btrfs_add_delalloc_inodes(root, inode); 1787 spin_unlock(&BTRFS_I(inode)->lock); 1788 } 1789 } 1790 1791 /* 1792 * extent_io.c clear_bit_hook, see set_bit_hook for why 1793 */ 1794 static void btrfs_clear_bit_hook(struct inode *inode, 1795 struct extent_state *state, 1796 unsigned *bits) 1797 { 1798 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1799 u64 len = state->end + 1 - state->start; 1800 u64 num_extents = div64_u64(len + BTRFS_MAX_EXTENT_SIZE -1, 1801 BTRFS_MAX_EXTENT_SIZE); 1802 1803 spin_lock(&BTRFS_I(inode)->lock); 1804 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) 1805 BTRFS_I(inode)->defrag_bytes -= len; 1806 spin_unlock(&BTRFS_I(inode)->lock); 1807 1808 /* 1809 * set_bit and clear bit hooks normally require _irqsave/restore 1810 * but in this case, we are only testing for the DELALLOC 1811 * bit, which is only set or cleared with irqs on 1812 */ 1813 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1814 struct btrfs_root *root = BTRFS_I(inode)->root; 1815 bool do_list = !btrfs_is_free_space_inode(inode); 1816 1817 if (*bits & EXTENT_FIRST_DELALLOC) { 1818 *bits &= ~EXTENT_FIRST_DELALLOC; 1819 } else if (!(*bits & EXTENT_DO_ACCOUNTING)) { 1820 spin_lock(&BTRFS_I(inode)->lock); 1821 BTRFS_I(inode)->outstanding_extents -= num_extents; 1822 spin_unlock(&BTRFS_I(inode)->lock); 1823 } 1824 1825 /* 1826 * We don't reserve metadata space for space cache inodes so we 1827 * don't need to call dellalloc_release_metadata if there is an 1828 * error. 1829 */ 1830 if (*bits & EXTENT_DO_ACCOUNTING && 1831 root != fs_info->tree_root) 1832 btrfs_delalloc_release_metadata(inode, len); 1833 1834 /* For sanity tests. */ 1835 if (btrfs_is_testing(fs_info)) 1836 return; 1837 1838 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 1839 && do_list && !(state->state & EXTENT_NORESERVE) 1840 && (*bits & (EXTENT_DO_ACCOUNTING | 1841 EXTENT_CLEAR_DATA_RESV))) 1842 btrfs_free_reserved_data_space_noquota(inode, 1843 state->start, len); 1844 1845 __percpu_counter_add(&fs_info->delalloc_bytes, -len, 1846 fs_info->delalloc_batch); 1847 spin_lock(&BTRFS_I(inode)->lock); 1848 BTRFS_I(inode)->delalloc_bytes -= len; 1849 if (do_list && BTRFS_I(inode)->delalloc_bytes == 0 && 1850 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1851 &BTRFS_I(inode)->runtime_flags)) 1852 btrfs_del_delalloc_inode(root, inode); 1853 spin_unlock(&BTRFS_I(inode)->lock); 1854 } 1855 } 1856 1857 /* 1858 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure 1859 * we don't create bios that span stripes or chunks 1860 * 1861 * return 1 if page cannot be merged to bio 1862 * return 0 if page can be merged to bio 1863 * return error otherwise 1864 */ 1865 int btrfs_merge_bio_hook(struct page *page, unsigned long offset, 1866 size_t size, struct bio *bio, 1867 unsigned long bio_flags) 1868 { 1869 struct inode *inode = page->mapping->host; 1870 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1871 u64 logical = (u64)bio->bi_iter.bi_sector << 9; 1872 u64 length = 0; 1873 u64 map_length; 1874 int ret; 1875 1876 if (bio_flags & EXTENT_BIO_COMPRESSED) 1877 return 0; 1878 1879 length = bio->bi_iter.bi_size; 1880 map_length = length; 1881 ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, 1882 NULL, 0); 1883 if (ret < 0) 1884 return ret; 1885 if (map_length < length + size) 1886 return 1; 1887 return 0; 1888 } 1889 1890 /* 1891 * in order to insert checksums into the metadata in large chunks, 1892 * we wait until bio submission time. All the pages in the bio are 1893 * checksummed and sums are attached onto the ordered extent record. 1894 * 1895 * At IO completion time the cums attached on the ordered extent record 1896 * are inserted into the btree 1897 */ 1898 static int __btrfs_submit_bio_start(struct inode *inode, struct bio *bio, 1899 int mirror_num, unsigned long bio_flags, 1900 u64 bio_offset) 1901 { 1902 int ret = 0; 1903 1904 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 1905 BUG_ON(ret); /* -ENOMEM */ 1906 return 0; 1907 } 1908 1909 /* 1910 * in order to insert checksums into the metadata in large chunks, 1911 * we wait until bio submission time. All the pages in the bio are 1912 * checksummed and sums are attached onto the ordered extent record. 1913 * 1914 * At IO completion time the cums attached on the ordered extent record 1915 * are inserted into the btree 1916 */ 1917 static int __btrfs_submit_bio_done(struct inode *inode, struct bio *bio, 1918 int mirror_num, unsigned long bio_flags, 1919 u64 bio_offset) 1920 { 1921 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1922 int ret; 1923 1924 ret = btrfs_map_bio(fs_info, bio, mirror_num, 1); 1925 if (ret) { 1926 bio->bi_error = ret; 1927 bio_endio(bio); 1928 } 1929 return ret; 1930 } 1931 1932 /* 1933 * extent_io.c submission hook. This does the right thing for csum calculation 1934 * on write, or reading the csums from the tree before a read 1935 */ 1936 static int btrfs_submit_bio_hook(struct inode *inode, struct bio *bio, 1937 int mirror_num, unsigned long bio_flags, 1938 u64 bio_offset) 1939 { 1940 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1941 struct btrfs_root *root = BTRFS_I(inode)->root; 1942 enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA; 1943 int ret = 0; 1944 int skip_sum; 1945 int async = !atomic_read(&BTRFS_I(inode)->sync_writers); 1946 1947 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 1948 1949 if (btrfs_is_free_space_inode(inode)) 1950 metadata = BTRFS_WQ_ENDIO_FREE_SPACE; 1951 1952 if (bio_op(bio) != REQ_OP_WRITE) { 1953 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); 1954 if (ret) 1955 goto out; 1956 1957 if (bio_flags & EXTENT_BIO_COMPRESSED) { 1958 ret = btrfs_submit_compressed_read(inode, bio, 1959 mirror_num, 1960 bio_flags); 1961 goto out; 1962 } else if (!skip_sum) { 1963 ret = btrfs_lookup_bio_sums(inode, bio, NULL); 1964 if (ret) 1965 goto out; 1966 } 1967 goto mapit; 1968 } else if (async && !skip_sum) { 1969 /* csum items have already been cloned */ 1970 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 1971 goto mapit; 1972 /* we're doing a write, do the async checksumming */ 1973 ret = btrfs_wq_submit_bio(fs_info, inode, bio, mirror_num, 1974 bio_flags, bio_offset, 1975 __btrfs_submit_bio_start, 1976 __btrfs_submit_bio_done); 1977 goto out; 1978 } else if (!skip_sum) { 1979 ret = btrfs_csum_one_bio(inode, bio, 0, 0); 1980 if (ret) 1981 goto out; 1982 } 1983 1984 mapit: 1985 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 1986 1987 out: 1988 if (ret < 0) { 1989 bio->bi_error = ret; 1990 bio_endio(bio); 1991 } 1992 return ret; 1993 } 1994 1995 /* 1996 * given a list of ordered sums record them in the inode. This happens 1997 * at IO completion time based on sums calculated at bio submission time. 1998 */ 1999 static noinline int add_pending_csums(struct btrfs_trans_handle *trans, 2000 struct inode *inode, u64 file_offset, 2001 struct list_head *list) 2002 { 2003 struct btrfs_ordered_sum *sum; 2004 2005 list_for_each_entry(sum, list, list) { 2006 trans->adding_csums = 1; 2007 btrfs_csum_file_blocks(trans, 2008 BTRFS_I(inode)->root->fs_info->csum_root, sum); 2009 trans->adding_csums = 0; 2010 } 2011 return 0; 2012 } 2013 2014 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end, 2015 struct extent_state **cached_state, int dedupe) 2016 { 2017 WARN_ON((end & (PAGE_SIZE - 1)) == 0); 2018 return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end, 2019 cached_state); 2020 } 2021 2022 /* see btrfs_writepage_start_hook for details on why this is required */ 2023 struct btrfs_writepage_fixup { 2024 struct page *page; 2025 struct btrfs_work work; 2026 }; 2027 2028 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2029 { 2030 struct btrfs_writepage_fixup *fixup; 2031 struct btrfs_ordered_extent *ordered; 2032 struct extent_state *cached_state = NULL; 2033 struct page *page; 2034 struct inode *inode; 2035 u64 page_start; 2036 u64 page_end; 2037 int ret; 2038 2039 fixup = container_of(work, struct btrfs_writepage_fixup, work); 2040 page = fixup->page; 2041 again: 2042 lock_page(page); 2043 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 2044 ClearPageChecked(page); 2045 goto out_page; 2046 } 2047 2048 inode = page->mapping->host; 2049 page_start = page_offset(page); 2050 page_end = page_offset(page) + PAGE_SIZE - 1; 2051 2052 lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end, 2053 &cached_state); 2054 2055 /* already ordered? We're done */ 2056 if (PagePrivate2(page)) 2057 goto out; 2058 2059 ordered = btrfs_lookup_ordered_range(inode, page_start, 2060 PAGE_SIZE); 2061 if (ordered) { 2062 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, 2063 page_end, &cached_state, GFP_NOFS); 2064 unlock_page(page); 2065 btrfs_start_ordered_extent(inode, ordered, 1); 2066 btrfs_put_ordered_extent(ordered); 2067 goto again; 2068 } 2069 2070 ret = btrfs_delalloc_reserve_space(inode, page_start, 2071 PAGE_SIZE); 2072 if (ret) { 2073 mapping_set_error(page->mapping, ret); 2074 end_extent_writepage(page, ret, page_start, page_end); 2075 ClearPageChecked(page); 2076 goto out; 2077 } 2078 2079 btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state, 2080 0); 2081 ClearPageChecked(page); 2082 set_page_dirty(page); 2083 out: 2084 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end, 2085 &cached_state, GFP_NOFS); 2086 out_page: 2087 unlock_page(page); 2088 put_page(page); 2089 kfree(fixup); 2090 } 2091 2092 /* 2093 * There are a few paths in the higher layers of the kernel that directly 2094 * set the page dirty bit without asking the filesystem if it is a 2095 * good idea. This causes problems because we want to make sure COW 2096 * properly happens and the data=ordered rules are followed. 2097 * 2098 * In our case any range that doesn't have the ORDERED bit set 2099 * hasn't been properly setup for IO. We kick off an async process 2100 * to fix it up. The async helper will wait for ordered extents, set 2101 * the delalloc bit and make it safe to write the page. 2102 */ 2103 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end) 2104 { 2105 struct inode *inode = page->mapping->host; 2106 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2107 struct btrfs_writepage_fixup *fixup; 2108 2109 /* this page is properly in the ordered list */ 2110 if (TestClearPagePrivate2(page)) 2111 return 0; 2112 2113 if (PageChecked(page)) 2114 return -EAGAIN; 2115 2116 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2117 if (!fixup) 2118 return -EAGAIN; 2119 2120 SetPageChecked(page); 2121 get_page(page); 2122 btrfs_init_work(&fixup->work, btrfs_fixup_helper, 2123 btrfs_writepage_fixup_worker, NULL, NULL); 2124 fixup->page = page; 2125 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2126 return -EBUSY; 2127 } 2128 2129 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2130 struct inode *inode, u64 file_pos, 2131 u64 disk_bytenr, u64 disk_num_bytes, 2132 u64 num_bytes, u64 ram_bytes, 2133 u8 compression, u8 encryption, 2134 u16 other_encoding, int extent_type) 2135 { 2136 struct btrfs_root *root = BTRFS_I(inode)->root; 2137 struct btrfs_file_extent_item *fi; 2138 struct btrfs_path *path; 2139 struct extent_buffer *leaf; 2140 struct btrfs_key ins; 2141 int extent_inserted = 0; 2142 int ret; 2143 2144 path = btrfs_alloc_path(); 2145 if (!path) 2146 return -ENOMEM; 2147 2148 /* 2149 * we may be replacing one extent in the tree with another. 2150 * The new extent is pinned in the extent map, and we don't want 2151 * to drop it from the cache until it is completely in the btree. 2152 * 2153 * So, tell btrfs_drop_extents to leave this extent in the cache. 2154 * the caller is expected to unpin it and allow it to be merged 2155 * with the others. 2156 */ 2157 ret = __btrfs_drop_extents(trans, root, inode, path, file_pos, 2158 file_pos + num_bytes, NULL, 0, 2159 1, sizeof(*fi), &extent_inserted); 2160 if (ret) 2161 goto out; 2162 2163 if (!extent_inserted) { 2164 ins.objectid = btrfs_ino(inode); 2165 ins.offset = file_pos; 2166 ins.type = BTRFS_EXTENT_DATA_KEY; 2167 2168 path->leave_spinning = 1; 2169 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2170 sizeof(*fi)); 2171 if (ret) 2172 goto out; 2173 } 2174 leaf = path->nodes[0]; 2175 fi = btrfs_item_ptr(leaf, path->slots[0], 2176 struct btrfs_file_extent_item); 2177 btrfs_set_file_extent_generation(leaf, fi, trans->transid); 2178 btrfs_set_file_extent_type(leaf, fi, extent_type); 2179 btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr); 2180 btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes); 2181 btrfs_set_file_extent_offset(leaf, fi, 0); 2182 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); 2183 btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes); 2184 btrfs_set_file_extent_compression(leaf, fi, compression); 2185 btrfs_set_file_extent_encryption(leaf, fi, encryption); 2186 btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding); 2187 2188 btrfs_mark_buffer_dirty(leaf); 2189 btrfs_release_path(path); 2190 2191 inode_add_bytes(inode, num_bytes); 2192 2193 ins.objectid = disk_bytenr; 2194 ins.offset = disk_num_bytes; 2195 ins.type = BTRFS_EXTENT_ITEM_KEY; 2196 ret = btrfs_alloc_reserved_file_extent(trans, root->root_key.objectid, 2197 btrfs_ino(inode), file_pos, 2198 ram_bytes, &ins); 2199 /* 2200 * Release the reserved range from inode dirty range map, as it is 2201 * already moved into delayed_ref_head 2202 */ 2203 btrfs_qgroup_release_data(inode, file_pos, ram_bytes); 2204 out: 2205 btrfs_free_path(path); 2206 2207 return ret; 2208 } 2209 2210 /* snapshot-aware defrag */ 2211 struct sa_defrag_extent_backref { 2212 struct rb_node node; 2213 struct old_sa_defrag_extent *old; 2214 u64 root_id; 2215 u64 inum; 2216 u64 file_pos; 2217 u64 extent_offset; 2218 u64 num_bytes; 2219 u64 generation; 2220 }; 2221 2222 struct old_sa_defrag_extent { 2223 struct list_head list; 2224 struct new_sa_defrag_extent *new; 2225 2226 u64 extent_offset; 2227 u64 bytenr; 2228 u64 offset; 2229 u64 len; 2230 int count; 2231 }; 2232 2233 struct new_sa_defrag_extent { 2234 struct rb_root root; 2235 struct list_head head; 2236 struct btrfs_path *path; 2237 struct inode *inode; 2238 u64 file_pos; 2239 u64 len; 2240 u64 bytenr; 2241 u64 disk_len; 2242 u8 compress_type; 2243 }; 2244 2245 static int backref_comp(struct sa_defrag_extent_backref *b1, 2246 struct sa_defrag_extent_backref *b2) 2247 { 2248 if (b1->root_id < b2->root_id) 2249 return -1; 2250 else if (b1->root_id > b2->root_id) 2251 return 1; 2252 2253 if (b1->inum < b2->inum) 2254 return -1; 2255 else if (b1->inum > b2->inum) 2256 return 1; 2257 2258 if (b1->file_pos < b2->file_pos) 2259 return -1; 2260 else if (b1->file_pos > b2->file_pos) 2261 return 1; 2262 2263 /* 2264 * [------------------------------] ===> (a range of space) 2265 * |<--->| |<---->| =============> (fs/file tree A) 2266 * |<---------------------------->| ===> (fs/file tree B) 2267 * 2268 * A range of space can refer to two file extents in one tree while 2269 * refer to only one file extent in another tree. 2270 * 2271 * So we may process a disk offset more than one time(two extents in A) 2272 * and locate at the same extent(one extent in B), then insert two same 2273 * backrefs(both refer to the extent in B). 2274 */ 2275 return 0; 2276 } 2277 2278 static void backref_insert(struct rb_root *root, 2279 struct sa_defrag_extent_backref *backref) 2280 { 2281 struct rb_node **p = &root->rb_node; 2282 struct rb_node *parent = NULL; 2283 struct sa_defrag_extent_backref *entry; 2284 int ret; 2285 2286 while (*p) { 2287 parent = *p; 2288 entry = rb_entry(parent, struct sa_defrag_extent_backref, node); 2289 2290 ret = backref_comp(backref, entry); 2291 if (ret < 0) 2292 p = &(*p)->rb_left; 2293 else 2294 p = &(*p)->rb_right; 2295 } 2296 2297 rb_link_node(&backref->node, parent, p); 2298 rb_insert_color(&backref->node, root); 2299 } 2300 2301 /* 2302 * Note the backref might has changed, and in this case we just return 0. 2303 */ 2304 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id, 2305 void *ctx) 2306 { 2307 struct btrfs_file_extent_item *extent; 2308 struct old_sa_defrag_extent *old = ctx; 2309 struct new_sa_defrag_extent *new = old->new; 2310 struct btrfs_path *path = new->path; 2311 struct btrfs_key key; 2312 struct btrfs_root *root; 2313 struct sa_defrag_extent_backref *backref; 2314 struct extent_buffer *leaf; 2315 struct inode *inode = new->inode; 2316 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2317 int slot; 2318 int ret; 2319 u64 extent_offset; 2320 u64 num_bytes; 2321 2322 if (BTRFS_I(inode)->root->root_key.objectid == root_id && 2323 inum == btrfs_ino(inode)) 2324 return 0; 2325 2326 key.objectid = root_id; 2327 key.type = BTRFS_ROOT_ITEM_KEY; 2328 key.offset = (u64)-1; 2329 2330 root = btrfs_read_fs_root_no_name(fs_info, &key); 2331 if (IS_ERR(root)) { 2332 if (PTR_ERR(root) == -ENOENT) 2333 return 0; 2334 WARN_ON(1); 2335 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu", 2336 inum, offset, root_id); 2337 return PTR_ERR(root); 2338 } 2339 2340 key.objectid = inum; 2341 key.type = BTRFS_EXTENT_DATA_KEY; 2342 if (offset > (u64)-1 << 32) 2343 key.offset = 0; 2344 else 2345 key.offset = offset; 2346 2347 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2348 if (WARN_ON(ret < 0)) 2349 return ret; 2350 ret = 0; 2351 2352 while (1) { 2353 cond_resched(); 2354 2355 leaf = path->nodes[0]; 2356 slot = path->slots[0]; 2357 2358 if (slot >= btrfs_header_nritems(leaf)) { 2359 ret = btrfs_next_leaf(root, path); 2360 if (ret < 0) { 2361 goto out; 2362 } else if (ret > 0) { 2363 ret = 0; 2364 goto out; 2365 } 2366 continue; 2367 } 2368 2369 path->slots[0]++; 2370 2371 btrfs_item_key_to_cpu(leaf, &key, slot); 2372 2373 if (key.objectid > inum) 2374 goto out; 2375 2376 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY) 2377 continue; 2378 2379 extent = btrfs_item_ptr(leaf, slot, 2380 struct btrfs_file_extent_item); 2381 2382 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr) 2383 continue; 2384 2385 /* 2386 * 'offset' refers to the exact key.offset, 2387 * NOT the 'offset' field in btrfs_extent_data_ref, ie. 2388 * (key.offset - extent_offset). 2389 */ 2390 if (key.offset != offset) 2391 continue; 2392 2393 extent_offset = btrfs_file_extent_offset(leaf, extent); 2394 num_bytes = btrfs_file_extent_num_bytes(leaf, extent); 2395 2396 if (extent_offset >= old->extent_offset + old->offset + 2397 old->len || extent_offset + num_bytes <= 2398 old->extent_offset + old->offset) 2399 continue; 2400 break; 2401 } 2402 2403 backref = kmalloc(sizeof(*backref), GFP_NOFS); 2404 if (!backref) { 2405 ret = -ENOENT; 2406 goto out; 2407 } 2408 2409 backref->root_id = root_id; 2410 backref->inum = inum; 2411 backref->file_pos = offset; 2412 backref->num_bytes = num_bytes; 2413 backref->extent_offset = extent_offset; 2414 backref->generation = btrfs_file_extent_generation(leaf, extent); 2415 backref->old = old; 2416 backref_insert(&new->root, backref); 2417 old->count++; 2418 out: 2419 btrfs_release_path(path); 2420 WARN_ON(ret); 2421 return ret; 2422 } 2423 2424 static noinline bool record_extent_backrefs(struct btrfs_path *path, 2425 struct new_sa_defrag_extent *new) 2426 { 2427 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2428 struct old_sa_defrag_extent *old, *tmp; 2429 int ret; 2430 2431 new->path = path; 2432 2433 list_for_each_entry_safe(old, tmp, &new->head, list) { 2434 ret = iterate_inodes_from_logical(old->bytenr + 2435 old->extent_offset, fs_info, 2436 path, record_one_backref, 2437 old); 2438 if (ret < 0 && ret != -ENOENT) 2439 return false; 2440 2441 /* no backref to be processed for this extent */ 2442 if (!old->count) { 2443 list_del(&old->list); 2444 kfree(old); 2445 } 2446 } 2447 2448 if (list_empty(&new->head)) 2449 return false; 2450 2451 return true; 2452 } 2453 2454 static int relink_is_mergable(struct extent_buffer *leaf, 2455 struct btrfs_file_extent_item *fi, 2456 struct new_sa_defrag_extent *new) 2457 { 2458 if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr) 2459 return 0; 2460 2461 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG) 2462 return 0; 2463 2464 if (btrfs_file_extent_compression(leaf, fi) != new->compress_type) 2465 return 0; 2466 2467 if (btrfs_file_extent_encryption(leaf, fi) || 2468 btrfs_file_extent_other_encoding(leaf, fi)) 2469 return 0; 2470 2471 return 1; 2472 } 2473 2474 /* 2475 * Note the backref might has changed, and in this case we just return 0. 2476 */ 2477 static noinline int relink_extent_backref(struct btrfs_path *path, 2478 struct sa_defrag_extent_backref *prev, 2479 struct sa_defrag_extent_backref *backref) 2480 { 2481 struct btrfs_file_extent_item *extent; 2482 struct btrfs_file_extent_item *item; 2483 struct btrfs_ordered_extent *ordered; 2484 struct btrfs_trans_handle *trans; 2485 struct btrfs_root *root; 2486 struct btrfs_key key; 2487 struct extent_buffer *leaf; 2488 struct old_sa_defrag_extent *old = backref->old; 2489 struct new_sa_defrag_extent *new = old->new; 2490 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2491 struct inode *inode; 2492 struct extent_state *cached = NULL; 2493 int ret = 0; 2494 u64 start; 2495 u64 len; 2496 u64 lock_start; 2497 u64 lock_end; 2498 bool merge = false; 2499 int index; 2500 2501 if (prev && prev->root_id == backref->root_id && 2502 prev->inum == backref->inum && 2503 prev->file_pos + prev->num_bytes == backref->file_pos) 2504 merge = true; 2505 2506 /* step 1: get root */ 2507 key.objectid = backref->root_id; 2508 key.type = BTRFS_ROOT_ITEM_KEY; 2509 key.offset = (u64)-1; 2510 2511 index = srcu_read_lock(&fs_info->subvol_srcu); 2512 2513 root = btrfs_read_fs_root_no_name(fs_info, &key); 2514 if (IS_ERR(root)) { 2515 srcu_read_unlock(&fs_info->subvol_srcu, index); 2516 if (PTR_ERR(root) == -ENOENT) 2517 return 0; 2518 return PTR_ERR(root); 2519 } 2520 2521 if (btrfs_root_readonly(root)) { 2522 srcu_read_unlock(&fs_info->subvol_srcu, index); 2523 return 0; 2524 } 2525 2526 /* step 2: get inode */ 2527 key.objectid = backref->inum; 2528 key.type = BTRFS_INODE_ITEM_KEY; 2529 key.offset = 0; 2530 2531 inode = btrfs_iget(fs_info->sb, &key, root, NULL); 2532 if (IS_ERR(inode)) { 2533 srcu_read_unlock(&fs_info->subvol_srcu, index); 2534 return 0; 2535 } 2536 2537 srcu_read_unlock(&fs_info->subvol_srcu, index); 2538 2539 /* step 3: relink backref */ 2540 lock_start = backref->file_pos; 2541 lock_end = backref->file_pos + backref->num_bytes - 1; 2542 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2543 &cached); 2544 2545 ordered = btrfs_lookup_first_ordered_extent(inode, lock_end); 2546 if (ordered) { 2547 btrfs_put_ordered_extent(ordered); 2548 goto out_unlock; 2549 } 2550 2551 trans = btrfs_join_transaction(root); 2552 if (IS_ERR(trans)) { 2553 ret = PTR_ERR(trans); 2554 goto out_unlock; 2555 } 2556 2557 key.objectid = backref->inum; 2558 key.type = BTRFS_EXTENT_DATA_KEY; 2559 key.offset = backref->file_pos; 2560 2561 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2562 if (ret < 0) { 2563 goto out_free_path; 2564 } else if (ret > 0) { 2565 ret = 0; 2566 goto out_free_path; 2567 } 2568 2569 extent = btrfs_item_ptr(path->nodes[0], path->slots[0], 2570 struct btrfs_file_extent_item); 2571 2572 if (btrfs_file_extent_generation(path->nodes[0], extent) != 2573 backref->generation) 2574 goto out_free_path; 2575 2576 btrfs_release_path(path); 2577 2578 start = backref->file_pos; 2579 if (backref->extent_offset < old->extent_offset + old->offset) 2580 start += old->extent_offset + old->offset - 2581 backref->extent_offset; 2582 2583 len = min(backref->extent_offset + backref->num_bytes, 2584 old->extent_offset + old->offset + old->len); 2585 len -= max(backref->extent_offset, old->extent_offset + old->offset); 2586 2587 ret = btrfs_drop_extents(trans, root, inode, start, 2588 start + len, 1); 2589 if (ret) 2590 goto out_free_path; 2591 again: 2592 key.objectid = btrfs_ino(inode); 2593 key.type = BTRFS_EXTENT_DATA_KEY; 2594 key.offset = start; 2595 2596 path->leave_spinning = 1; 2597 if (merge) { 2598 struct btrfs_file_extent_item *fi; 2599 u64 extent_len; 2600 struct btrfs_key found_key; 2601 2602 ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 2603 if (ret < 0) 2604 goto out_free_path; 2605 2606 path->slots[0]--; 2607 leaf = path->nodes[0]; 2608 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 2609 2610 fi = btrfs_item_ptr(leaf, path->slots[0], 2611 struct btrfs_file_extent_item); 2612 extent_len = btrfs_file_extent_num_bytes(leaf, fi); 2613 2614 if (extent_len + found_key.offset == start && 2615 relink_is_mergable(leaf, fi, new)) { 2616 btrfs_set_file_extent_num_bytes(leaf, fi, 2617 extent_len + len); 2618 btrfs_mark_buffer_dirty(leaf); 2619 inode_add_bytes(inode, len); 2620 2621 ret = 1; 2622 goto out_free_path; 2623 } else { 2624 merge = false; 2625 btrfs_release_path(path); 2626 goto again; 2627 } 2628 } 2629 2630 ret = btrfs_insert_empty_item(trans, root, path, &key, 2631 sizeof(*extent)); 2632 if (ret) { 2633 btrfs_abort_transaction(trans, ret); 2634 goto out_free_path; 2635 } 2636 2637 leaf = path->nodes[0]; 2638 item = btrfs_item_ptr(leaf, path->slots[0], 2639 struct btrfs_file_extent_item); 2640 btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr); 2641 btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len); 2642 btrfs_set_file_extent_offset(leaf, item, start - new->file_pos); 2643 btrfs_set_file_extent_num_bytes(leaf, item, len); 2644 btrfs_set_file_extent_ram_bytes(leaf, item, new->len); 2645 btrfs_set_file_extent_generation(leaf, item, trans->transid); 2646 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); 2647 btrfs_set_file_extent_compression(leaf, item, new->compress_type); 2648 btrfs_set_file_extent_encryption(leaf, item, 0); 2649 btrfs_set_file_extent_other_encoding(leaf, item, 0); 2650 2651 btrfs_mark_buffer_dirty(leaf); 2652 inode_add_bytes(inode, len); 2653 btrfs_release_path(path); 2654 2655 ret = btrfs_inc_extent_ref(trans, fs_info, new->bytenr, 2656 new->disk_len, 0, 2657 backref->root_id, backref->inum, 2658 new->file_pos); /* start - extent_offset */ 2659 if (ret) { 2660 btrfs_abort_transaction(trans, ret); 2661 goto out_free_path; 2662 } 2663 2664 ret = 1; 2665 out_free_path: 2666 btrfs_release_path(path); 2667 path->leave_spinning = 0; 2668 btrfs_end_transaction(trans); 2669 out_unlock: 2670 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2671 &cached, GFP_NOFS); 2672 iput(inode); 2673 return ret; 2674 } 2675 2676 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new) 2677 { 2678 struct old_sa_defrag_extent *old, *tmp; 2679 2680 if (!new) 2681 return; 2682 2683 list_for_each_entry_safe(old, tmp, &new->head, list) { 2684 kfree(old); 2685 } 2686 kfree(new); 2687 } 2688 2689 static void relink_file_extents(struct new_sa_defrag_extent *new) 2690 { 2691 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb); 2692 struct btrfs_path *path; 2693 struct sa_defrag_extent_backref *backref; 2694 struct sa_defrag_extent_backref *prev = NULL; 2695 struct inode *inode; 2696 struct btrfs_root *root; 2697 struct rb_node *node; 2698 int ret; 2699 2700 inode = new->inode; 2701 root = BTRFS_I(inode)->root; 2702 2703 path = btrfs_alloc_path(); 2704 if (!path) 2705 return; 2706 2707 if (!record_extent_backrefs(path, new)) { 2708 btrfs_free_path(path); 2709 goto out; 2710 } 2711 btrfs_release_path(path); 2712 2713 while (1) { 2714 node = rb_first(&new->root); 2715 if (!node) 2716 break; 2717 rb_erase(node, &new->root); 2718 2719 backref = rb_entry(node, struct sa_defrag_extent_backref, node); 2720 2721 ret = relink_extent_backref(path, prev, backref); 2722 WARN_ON(ret < 0); 2723 2724 kfree(prev); 2725 2726 if (ret == 1) 2727 prev = backref; 2728 else 2729 prev = NULL; 2730 cond_resched(); 2731 } 2732 kfree(prev); 2733 2734 btrfs_free_path(path); 2735 out: 2736 free_sa_defrag_extent(new); 2737 2738 atomic_dec(&fs_info->defrag_running); 2739 wake_up(&fs_info->transaction_wait); 2740 } 2741 2742 static struct new_sa_defrag_extent * 2743 record_old_file_extents(struct inode *inode, 2744 struct btrfs_ordered_extent *ordered) 2745 { 2746 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2747 struct btrfs_root *root = BTRFS_I(inode)->root; 2748 struct btrfs_path *path; 2749 struct btrfs_key key; 2750 struct old_sa_defrag_extent *old; 2751 struct new_sa_defrag_extent *new; 2752 int ret; 2753 2754 new = kmalloc(sizeof(*new), GFP_NOFS); 2755 if (!new) 2756 return NULL; 2757 2758 new->inode = inode; 2759 new->file_pos = ordered->file_offset; 2760 new->len = ordered->len; 2761 new->bytenr = ordered->start; 2762 new->disk_len = ordered->disk_len; 2763 new->compress_type = ordered->compress_type; 2764 new->root = RB_ROOT; 2765 INIT_LIST_HEAD(&new->head); 2766 2767 path = btrfs_alloc_path(); 2768 if (!path) 2769 goto out_kfree; 2770 2771 key.objectid = btrfs_ino(inode); 2772 key.type = BTRFS_EXTENT_DATA_KEY; 2773 key.offset = new->file_pos; 2774 2775 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2776 if (ret < 0) 2777 goto out_free_path; 2778 if (ret > 0 && path->slots[0] > 0) 2779 path->slots[0]--; 2780 2781 /* find out all the old extents for the file range */ 2782 while (1) { 2783 struct btrfs_file_extent_item *extent; 2784 struct extent_buffer *l; 2785 int slot; 2786 u64 num_bytes; 2787 u64 offset; 2788 u64 end; 2789 u64 disk_bytenr; 2790 u64 extent_offset; 2791 2792 l = path->nodes[0]; 2793 slot = path->slots[0]; 2794 2795 if (slot >= btrfs_header_nritems(l)) { 2796 ret = btrfs_next_leaf(root, path); 2797 if (ret < 0) 2798 goto out_free_path; 2799 else if (ret > 0) 2800 break; 2801 continue; 2802 } 2803 2804 btrfs_item_key_to_cpu(l, &key, slot); 2805 2806 if (key.objectid != btrfs_ino(inode)) 2807 break; 2808 if (key.type != BTRFS_EXTENT_DATA_KEY) 2809 break; 2810 if (key.offset >= new->file_pos + new->len) 2811 break; 2812 2813 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item); 2814 2815 num_bytes = btrfs_file_extent_num_bytes(l, extent); 2816 if (key.offset + num_bytes < new->file_pos) 2817 goto next; 2818 2819 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent); 2820 if (!disk_bytenr) 2821 goto next; 2822 2823 extent_offset = btrfs_file_extent_offset(l, extent); 2824 2825 old = kmalloc(sizeof(*old), GFP_NOFS); 2826 if (!old) 2827 goto out_free_path; 2828 2829 offset = max(new->file_pos, key.offset); 2830 end = min(new->file_pos + new->len, key.offset + num_bytes); 2831 2832 old->bytenr = disk_bytenr; 2833 old->extent_offset = extent_offset; 2834 old->offset = offset - key.offset; 2835 old->len = end - offset; 2836 old->new = new; 2837 old->count = 0; 2838 list_add_tail(&old->list, &new->head); 2839 next: 2840 path->slots[0]++; 2841 cond_resched(); 2842 } 2843 2844 btrfs_free_path(path); 2845 atomic_inc(&fs_info->defrag_running); 2846 2847 return new; 2848 2849 out_free_path: 2850 btrfs_free_path(path); 2851 out_kfree: 2852 free_sa_defrag_extent(new); 2853 return NULL; 2854 } 2855 2856 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 2857 u64 start, u64 len) 2858 { 2859 struct btrfs_block_group_cache *cache; 2860 2861 cache = btrfs_lookup_block_group(fs_info, start); 2862 ASSERT(cache); 2863 2864 spin_lock(&cache->lock); 2865 cache->delalloc_bytes -= len; 2866 spin_unlock(&cache->lock); 2867 2868 btrfs_put_block_group(cache); 2869 } 2870 2871 /* as ordered data IO finishes, this gets called so we can finish 2872 * an ordered extent if the range of bytes in the file it covers are 2873 * fully written. 2874 */ 2875 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 2876 { 2877 struct inode *inode = ordered_extent->inode; 2878 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2879 struct btrfs_root *root = BTRFS_I(inode)->root; 2880 struct btrfs_trans_handle *trans = NULL; 2881 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2882 struct extent_state *cached_state = NULL; 2883 struct new_sa_defrag_extent *new = NULL; 2884 int compress_type = 0; 2885 int ret = 0; 2886 u64 logical_len = ordered_extent->len; 2887 bool nolock; 2888 bool truncated = false; 2889 2890 nolock = btrfs_is_free_space_inode(inode); 2891 2892 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 2893 ret = -EIO; 2894 goto out; 2895 } 2896 2897 btrfs_free_io_failure_record(inode, ordered_extent->file_offset, 2898 ordered_extent->file_offset + 2899 ordered_extent->len - 1); 2900 2901 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 2902 truncated = true; 2903 logical_len = ordered_extent->truncated_len; 2904 /* Truncated the entire extent, don't bother adding */ 2905 if (!logical_len) 2906 goto out; 2907 } 2908 2909 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 2910 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 2911 2912 /* 2913 * For mwrite(mmap + memset to write) case, we still reserve 2914 * space for NOCOW range. 2915 * As NOCOW won't cause a new delayed ref, just free the space 2916 */ 2917 btrfs_qgroup_free_data(inode, ordered_extent->file_offset, 2918 ordered_extent->len); 2919 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 2920 if (nolock) 2921 trans = btrfs_join_transaction_nolock(root); 2922 else 2923 trans = btrfs_join_transaction(root); 2924 if (IS_ERR(trans)) { 2925 ret = PTR_ERR(trans); 2926 trans = NULL; 2927 goto out; 2928 } 2929 trans->block_rsv = &fs_info->delalloc_block_rsv; 2930 ret = btrfs_update_inode_fallback(trans, root, inode); 2931 if (ret) /* -ENOMEM or corruption */ 2932 btrfs_abort_transaction(trans, ret); 2933 goto out; 2934 } 2935 2936 lock_extent_bits(io_tree, ordered_extent->file_offset, 2937 ordered_extent->file_offset + ordered_extent->len - 1, 2938 &cached_state); 2939 2940 ret = test_range_bit(io_tree, ordered_extent->file_offset, 2941 ordered_extent->file_offset + ordered_extent->len - 1, 2942 EXTENT_DEFRAG, 1, cached_state); 2943 if (ret) { 2944 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item); 2945 if (0 && last_snapshot >= BTRFS_I(inode)->generation) 2946 /* the inode is shared */ 2947 new = record_old_file_extents(inode, ordered_extent); 2948 2949 clear_extent_bit(io_tree, ordered_extent->file_offset, 2950 ordered_extent->file_offset + ordered_extent->len - 1, 2951 EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS); 2952 } 2953 2954 if (nolock) 2955 trans = btrfs_join_transaction_nolock(root); 2956 else 2957 trans = btrfs_join_transaction(root); 2958 if (IS_ERR(trans)) { 2959 ret = PTR_ERR(trans); 2960 trans = NULL; 2961 goto out_unlock; 2962 } 2963 2964 trans->block_rsv = &fs_info->delalloc_block_rsv; 2965 2966 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 2967 compress_type = ordered_extent->compress_type; 2968 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 2969 BUG_ON(compress_type); 2970 ret = btrfs_mark_extent_written(trans, inode, 2971 ordered_extent->file_offset, 2972 ordered_extent->file_offset + 2973 logical_len); 2974 } else { 2975 BUG_ON(root == fs_info->tree_root); 2976 ret = insert_reserved_file_extent(trans, inode, 2977 ordered_extent->file_offset, 2978 ordered_extent->start, 2979 ordered_extent->disk_len, 2980 logical_len, logical_len, 2981 compress_type, 0, 0, 2982 BTRFS_FILE_EXTENT_REG); 2983 if (!ret) 2984 btrfs_release_delalloc_bytes(fs_info, 2985 ordered_extent->start, 2986 ordered_extent->disk_len); 2987 } 2988 unpin_extent_cache(&BTRFS_I(inode)->extent_tree, 2989 ordered_extent->file_offset, ordered_extent->len, 2990 trans->transid); 2991 if (ret < 0) { 2992 btrfs_abort_transaction(trans, ret); 2993 goto out_unlock; 2994 } 2995 2996 add_pending_csums(trans, inode, ordered_extent->file_offset, 2997 &ordered_extent->list); 2998 2999 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 3000 ret = btrfs_update_inode_fallback(trans, root, inode); 3001 if (ret) { /* -ENOMEM or corruption */ 3002 btrfs_abort_transaction(trans, ret); 3003 goto out_unlock; 3004 } 3005 ret = 0; 3006 out_unlock: 3007 unlock_extent_cached(io_tree, ordered_extent->file_offset, 3008 ordered_extent->file_offset + 3009 ordered_extent->len - 1, &cached_state, GFP_NOFS); 3010 out: 3011 if (root != fs_info->tree_root) 3012 btrfs_delalloc_release_metadata(inode, ordered_extent->len); 3013 if (trans) 3014 btrfs_end_transaction(trans); 3015 3016 if (ret || truncated) { 3017 u64 start, end; 3018 3019 if (truncated) 3020 start = ordered_extent->file_offset + logical_len; 3021 else 3022 start = ordered_extent->file_offset; 3023 end = ordered_extent->file_offset + ordered_extent->len - 1; 3024 clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS); 3025 3026 /* Drop the cache for the part of the extent we didn't write. */ 3027 btrfs_drop_extent_cache(inode, start, end, 0); 3028 3029 /* 3030 * If the ordered extent had an IOERR or something else went 3031 * wrong we need to return the space for this ordered extent 3032 * back to the allocator. We only free the extent in the 3033 * truncated case if we didn't write out the extent at all. 3034 */ 3035 if ((ret || !logical_len) && 3036 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3037 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) 3038 btrfs_free_reserved_extent(fs_info, 3039 ordered_extent->start, 3040 ordered_extent->disk_len, 1); 3041 } 3042 3043 3044 /* 3045 * This needs to be done to make sure anybody waiting knows we are done 3046 * updating everything for this ordered extent. 3047 */ 3048 btrfs_remove_ordered_extent(inode, ordered_extent); 3049 3050 /* for snapshot-aware defrag */ 3051 if (new) { 3052 if (ret) { 3053 free_sa_defrag_extent(new); 3054 atomic_dec(&fs_info->defrag_running); 3055 } else { 3056 relink_file_extents(new); 3057 } 3058 } 3059 3060 /* once for us */ 3061 btrfs_put_ordered_extent(ordered_extent); 3062 /* once for the tree */ 3063 btrfs_put_ordered_extent(ordered_extent); 3064 3065 return ret; 3066 } 3067 3068 static void finish_ordered_fn(struct btrfs_work *work) 3069 { 3070 struct btrfs_ordered_extent *ordered_extent; 3071 ordered_extent = container_of(work, struct btrfs_ordered_extent, work); 3072 btrfs_finish_ordered_io(ordered_extent); 3073 } 3074 3075 static int btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end, 3076 struct extent_state *state, int uptodate) 3077 { 3078 struct inode *inode = page->mapping->host; 3079 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3080 struct btrfs_ordered_extent *ordered_extent = NULL; 3081 struct btrfs_workqueue *wq; 3082 btrfs_work_func_t func; 3083 3084 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); 3085 3086 ClearPagePrivate2(page); 3087 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, 3088 end - start + 1, uptodate)) 3089 return 0; 3090 3091 if (btrfs_is_free_space_inode(inode)) { 3092 wq = fs_info->endio_freespace_worker; 3093 func = btrfs_freespace_write_helper; 3094 } else { 3095 wq = fs_info->endio_write_workers; 3096 func = btrfs_endio_write_helper; 3097 } 3098 3099 btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL, 3100 NULL); 3101 btrfs_queue_work(wq, &ordered_extent->work); 3102 3103 return 0; 3104 } 3105 3106 static int __readpage_endio_check(struct inode *inode, 3107 struct btrfs_io_bio *io_bio, 3108 int icsum, struct page *page, 3109 int pgoff, u64 start, size_t len) 3110 { 3111 char *kaddr; 3112 u32 csum_expected; 3113 u32 csum = ~(u32)0; 3114 3115 csum_expected = *(((u32 *)io_bio->csum) + icsum); 3116 3117 kaddr = kmap_atomic(page); 3118 csum = btrfs_csum_data(kaddr + pgoff, csum, len); 3119 btrfs_csum_final(csum, (u8 *)&csum); 3120 if (csum != csum_expected) 3121 goto zeroit; 3122 3123 kunmap_atomic(kaddr); 3124 return 0; 3125 zeroit: 3126 btrfs_warn_rl(BTRFS_I(inode)->root->fs_info, 3127 "csum failed ino %llu off %llu csum %u expected csum %u", 3128 btrfs_ino(inode), start, csum, csum_expected); 3129 memset(kaddr + pgoff, 1, len); 3130 flush_dcache_page(page); 3131 kunmap_atomic(kaddr); 3132 if (csum_expected == 0) 3133 return 0; 3134 return -EIO; 3135 } 3136 3137 /* 3138 * when reads are done, we need to check csums to verify the data is correct 3139 * if there's a match, we allow the bio to finish. If not, the code in 3140 * extent_io.c will try to find good copies for us. 3141 */ 3142 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, 3143 u64 phy_offset, struct page *page, 3144 u64 start, u64 end, int mirror) 3145 { 3146 size_t offset = start - page_offset(page); 3147 struct inode *inode = page->mapping->host; 3148 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 3149 struct btrfs_root *root = BTRFS_I(inode)->root; 3150 3151 if (PageChecked(page)) { 3152 ClearPageChecked(page); 3153 return 0; 3154 } 3155 3156 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 3157 return 0; 3158 3159 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID && 3160 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) { 3161 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM); 3162 return 0; 3163 } 3164 3165 phy_offset >>= inode->i_sb->s_blocksize_bits; 3166 return __readpage_endio_check(inode, io_bio, phy_offset, page, offset, 3167 start, (size_t)(end - start + 1)); 3168 } 3169 3170 void btrfs_add_delayed_iput(struct inode *inode) 3171 { 3172 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3173 struct btrfs_inode *binode = BTRFS_I(inode); 3174 3175 if (atomic_add_unless(&inode->i_count, -1, 1)) 3176 return; 3177 3178 spin_lock(&fs_info->delayed_iput_lock); 3179 if (binode->delayed_iput_count == 0) { 3180 ASSERT(list_empty(&binode->delayed_iput)); 3181 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); 3182 } else { 3183 binode->delayed_iput_count++; 3184 } 3185 spin_unlock(&fs_info->delayed_iput_lock); 3186 } 3187 3188 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 3189 { 3190 3191 spin_lock(&fs_info->delayed_iput_lock); 3192 while (!list_empty(&fs_info->delayed_iputs)) { 3193 struct btrfs_inode *inode; 3194 3195 inode = list_first_entry(&fs_info->delayed_iputs, 3196 struct btrfs_inode, delayed_iput); 3197 if (inode->delayed_iput_count) { 3198 inode->delayed_iput_count--; 3199 list_move_tail(&inode->delayed_iput, 3200 &fs_info->delayed_iputs); 3201 } else { 3202 list_del_init(&inode->delayed_iput); 3203 } 3204 spin_unlock(&fs_info->delayed_iput_lock); 3205 iput(&inode->vfs_inode); 3206 spin_lock(&fs_info->delayed_iput_lock); 3207 } 3208 spin_unlock(&fs_info->delayed_iput_lock); 3209 } 3210 3211 /* 3212 * This is called in transaction commit time. If there are no orphan 3213 * files in the subvolume, it removes orphan item and frees block_rsv 3214 * structure. 3215 */ 3216 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans, 3217 struct btrfs_root *root) 3218 { 3219 struct btrfs_fs_info *fs_info = root->fs_info; 3220 struct btrfs_block_rsv *block_rsv; 3221 int ret; 3222 3223 if (atomic_read(&root->orphan_inodes) || 3224 root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) 3225 return; 3226 3227 spin_lock(&root->orphan_lock); 3228 if (atomic_read(&root->orphan_inodes)) { 3229 spin_unlock(&root->orphan_lock); 3230 return; 3231 } 3232 3233 if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) { 3234 spin_unlock(&root->orphan_lock); 3235 return; 3236 } 3237 3238 block_rsv = root->orphan_block_rsv; 3239 root->orphan_block_rsv = NULL; 3240 spin_unlock(&root->orphan_lock); 3241 3242 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) && 3243 btrfs_root_refs(&root->root_item) > 0) { 3244 ret = btrfs_del_orphan_item(trans, fs_info->tree_root, 3245 root->root_key.objectid); 3246 if (ret) 3247 btrfs_abort_transaction(trans, ret); 3248 else 3249 clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, 3250 &root->state); 3251 } 3252 3253 if (block_rsv) { 3254 WARN_ON(block_rsv->size > 0); 3255 btrfs_free_block_rsv(fs_info, block_rsv); 3256 } 3257 } 3258 3259 /* 3260 * This creates an orphan entry for the given inode in case something goes 3261 * wrong in the middle of an unlink/truncate. 3262 * 3263 * NOTE: caller of this function should reserve 5 units of metadata for 3264 * this function. 3265 */ 3266 int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode) 3267 { 3268 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3269 struct btrfs_root *root = BTRFS_I(inode)->root; 3270 struct btrfs_block_rsv *block_rsv = NULL; 3271 int reserve = 0; 3272 int insert = 0; 3273 int ret; 3274 3275 if (!root->orphan_block_rsv) { 3276 block_rsv = btrfs_alloc_block_rsv(fs_info, 3277 BTRFS_BLOCK_RSV_TEMP); 3278 if (!block_rsv) 3279 return -ENOMEM; 3280 } 3281 3282 spin_lock(&root->orphan_lock); 3283 if (!root->orphan_block_rsv) { 3284 root->orphan_block_rsv = block_rsv; 3285 } else if (block_rsv) { 3286 btrfs_free_block_rsv(fs_info, block_rsv); 3287 block_rsv = NULL; 3288 } 3289 3290 if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3291 &BTRFS_I(inode)->runtime_flags)) { 3292 #if 0 3293 /* 3294 * For proper ENOSPC handling, we should do orphan 3295 * cleanup when mounting. But this introduces backward 3296 * compatibility issue. 3297 */ 3298 if (!xchg(&root->orphan_item_inserted, 1)) 3299 insert = 2; 3300 else 3301 insert = 1; 3302 #endif 3303 insert = 1; 3304 atomic_inc(&root->orphan_inodes); 3305 } 3306 3307 if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3308 &BTRFS_I(inode)->runtime_flags)) 3309 reserve = 1; 3310 spin_unlock(&root->orphan_lock); 3311 3312 /* grab metadata reservation from transaction handle */ 3313 if (reserve) { 3314 ret = btrfs_orphan_reserve_metadata(trans, inode); 3315 ASSERT(!ret); 3316 if (ret) { 3317 atomic_dec(&root->orphan_inodes); 3318 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3319 &BTRFS_I(inode)->runtime_flags); 3320 if (insert) 3321 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3322 &BTRFS_I(inode)->runtime_flags); 3323 return ret; 3324 } 3325 } 3326 3327 /* insert an orphan item to track this unlinked/truncated file */ 3328 if (insert >= 1) { 3329 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode)); 3330 if (ret) { 3331 atomic_dec(&root->orphan_inodes); 3332 if (reserve) { 3333 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3334 &BTRFS_I(inode)->runtime_flags); 3335 btrfs_orphan_release_metadata(inode); 3336 } 3337 if (ret != -EEXIST) { 3338 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3339 &BTRFS_I(inode)->runtime_flags); 3340 btrfs_abort_transaction(trans, ret); 3341 return ret; 3342 } 3343 } 3344 ret = 0; 3345 } 3346 3347 /* insert an orphan item to track subvolume contains orphan files */ 3348 if (insert >= 2) { 3349 ret = btrfs_insert_orphan_item(trans, fs_info->tree_root, 3350 root->root_key.objectid); 3351 if (ret && ret != -EEXIST) { 3352 btrfs_abort_transaction(trans, ret); 3353 return ret; 3354 } 3355 } 3356 return 0; 3357 } 3358 3359 /* 3360 * We have done the truncate/delete so we can go ahead and remove the orphan 3361 * item for this particular inode. 3362 */ 3363 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3364 struct inode *inode) 3365 { 3366 struct btrfs_root *root = BTRFS_I(inode)->root; 3367 int delete_item = 0; 3368 int release_rsv = 0; 3369 int ret = 0; 3370 3371 spin_lock(&root->orphan_lock); 3372 if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3373 &BTRFS_I(inode)->runtime_flags)) 3374 delete_item = 1; 3375 3376 if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3377 &BTRFS_I(inode)->runtime_flags)) 3378 release_rsv = 1; 3379 spin_unlock(&root->orphan_lock); 3380 3381 if (delete_item) { 3382 atomic_dec(&root->orphan_inodes); 3383 if (trans) 3384 ret = btrfs_del_orphan_item(trans, root, 3385 btrfs_ino(inode)); 3386 } 3387 3388 if (release_rsv) 3389 btrfs_orphan_release_metadata(inode); 3390 3391 return ret; 3392 } 3393 3394 /* 3395 * this cleans up any orphans that may be left on the list from the last use 3396 * of this root. 3397 */ 3398 int btrfs_orphan_cleanup(struct btrfs_root *root) 3399 { 3400 struct btrfs_fs_info *fs_info = root->fs_info; 3401 struct btrfs_path *path; 3402 struct extent_buffer *leaf; 3403 struct btrfs_key key, found_key; 3404 struct btrfs_trans_handle *trans; 3405 struct inode *inode; 3406 u64 last_objectid = 0; 3407 int ret = 0, nr_unlink = 0, nr_truncate = 0; 3408 3409 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED)) 3410 return 0; 3411 3412 path = btrfs_alloc_path(); 3413 if (!path) { 3414 ret = -ENOMEM; 3415 goto out; 3416 } 3417 path->reada = READA_BACK; 3418 3419 key.objectid = BTRFS_ORPHAN_OBJECTID; 3420 key.type = BTRFS_ORPHAN_ITEM_KEY; 3421 key.offset = (u64)-1; 3422 3423 while (1) { 3424 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3425 if (ret < 0) 3426 goto out; 3427 3428 /* 3429 * if ret == 0 means we found what we were searching for, which 3430 * is weird, but possible, so only screw with path if we didn't 3431 * find the key and see if we have stuff that matches 3432 */ 3433 if (ret > 0) { 3434 ret = 0; 3435 if (path->slots[0] == 0) 3436 break; 3437 path->slots[0]--; 3438 } 3439 3440 /* pull out the item */ 3441 leaf = path->nodes[0]; 3442 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3443 3444 /* make sure the item matches what we want */ 3445 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3446 break; 3447 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3448 break; 3449 3450 /* release the path since we're done with it */ 3451 btrfs_release_path(path); 3452 3453 /* 3454 * this is where we are basically btrfs_lookup, without the 3455 * crossing root thing. we store the inode number in the 3456 * offset of the orphan item. 3457 */ 3458 3459 if (found_key.offset == last_objectid) { 3460 btrfs_err(fs_info, 3461 "Error removing orphan entry, stopping orphan cleanup"); 3462 ret = -EINVAL; 3463 goto out; 3464 } 3465 3466 last_objectid = found_key.offset; 3467 3468 found_key.objectid = found_key.offset; 3469 found_key.type = BTRFS_INODE_ITEM_KEY; 3470 found_key.offset = 0; 3471 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL); 3472 ret = PTR_ERR_OR_ZERO(inode); 3473 if (ret && ret != -ENOENT) 3474 goto out; 3475 3476 if (ret == -ENOENT && root == fs_info->tree_root) { 3477 struct btrfs_root *dead_root; 3478 struct btrfs_fs_info *fs_info = root->fs_info; 3479 int is_dead_root = 0; 3480 3481 /* 3482 * this is an orphan in the tree root. Currently these 3483 * could come from 2 sources: 3484 * a) a snapshot deletion in progress 3485 * b) a free space cache inode 3486 * We need to distinguish those two, as the snapshot 3487 * orphan must not get deleted. 3488 * find_dead_roots already ran before us, so if this 3489 * is a snapshot deletion, we should find the root 3490 * in the dead_roots list 3491 */ 3492 spin_lock(&fs_info->trans_lock); 3493 list_for_each_entry(dead_root, &fs_info->dead_roots, 3494 root_list) { 3495 if (dead_root->root_key.objectid == 3496 found_key.objectid) { 3497 is_dead_root = 1; 3498 break; 3499 } 3500 } 3501 spin_unlock(&fs_info->trans_lock); 3502 if (is_dead_root) { 3503 /* prevent this orphan from being found again */ 3504 key.offset = found_key.objectid - 1; 3505 continue; 3506 } 3507 } 3508 /* 3509 * Inode is already gone but the orphan item is still there, 3510 * kill the orphan item. 3511 */ 3512 if (ret == -ENOENT) { 3513 trans = btrfs_start_transaction(root, 1); 3514 if (IS_ERR(trans)) { 3515 ret = PTR_ERR(trans); 3516 goto out; 3517 } 3518 btrfs_debug(fs_info, "auto deleting %Lu", 3519 found_key.objectid); 3520 ret = btrfs_del_orphan_item(trans, root, 3521 found_key.objectid); 3522 btrfs_end_transaction(trans); 3523 if (ret) 3524 goto out; 3525 continue; 3526 } 3527 3528 /* 3529 * add this inode to the orphan list so btrfs_orphan_del does 3530 * the proper thing when we hit it 3531 */ 3532 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3533 &BTRFS_I(inode)->runtime_flags); 3534 atomic_inc(&root->orphan_inodes); 3535 3536 /* if we have links, this was a truncate, lets do that */ 3537 if (inode->i_nlink) { 3538 if (WARN_ON(!S_ISREG(inode->i_mode))) { 3539 iput(inode); 3540 continue; 3541 } 3542 nr_truncate++; 3543 3544 /* 1 for the orphan item deletion. */ 3545 trans = btrfs_start_transaction(root, 1); 3546 if (IS_ERR(trans)) { 3547 iput(inode); 3548 ret = PTR_ERR(trans); 3549 goto out; 3550 } 3551 ret = btrfs_orphan_add(trans, inode); 3552 btrfs_end_transaction(trans); 3553 if (ret) { 3554 iput(inode); 3555 goto out; 3556 } 3557 3558 ret = btrfs_truncate(inode); 3559 if (ret) 3560 btrfs_orphan_del(NULL, inode); 3561 } else { 3562 nr_unlink++; 3563 } 3564 3565 /* this will do delete_inode and everything for us */ 3566 iput(inode); 3567 if (ret) 3568 goto out; 3569 } 3570 /* release the path since we're done with it */ 3571 btrfs_release_path(path); 3572 3573 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE; 3574 3575 if (root->orphan_block_rsv) 3576 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv, 3577 (u64)-1); 3578 3579 if (root->orphan_block_rsv || 3580 test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3581 trans = btrfs_join_transaction(root); 3582 if (!IS_ERR(trans)) 3583 btrfs_end_transaction(trans); 3584 } 3585 3586 if (nr_unlink) 3587 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3588 if (nr_truncate) 3589 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate); 3590 3591 out: 3592 if (ret) 3593 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3594 btrfs_free_path(path); 3595 return ret; 3596 } 3597 3598 /* 3599 * very simple check to peek ahead in the leaf looking for xattrs. If we 3600 * don't find any xattrs, we know there can't be any acls. 3601 * 3602 * slot is the slot the inode is in, objectid is the objectid of the inode 3603 */ 3604 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3605 int slot, u64 objectid, 3606 int *first_xattr_slot) 3607 { 3608 u32 nritems = btrfs_header_nritems(leaf); 3609 struct btrfs_key found_key; 3610 static u64 xattr_access = 0; 3611 static u64 xattr_default = 0; 3612 int scanned = 0; 3613 3614 if (!xattr_access) { 3615 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3616 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3617 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3618 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3619 } 3620 3621 slot++; 3622 *first_xattr_slot = -1; 3623 while (slot < nritems) { 3624 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3625 3626 /* we found a different objectid, there must not be acls */ 3627 if (found_key.objectid != objectid) 3628 return 0; 3629 3630 /* we found an xattr, assume we've got an acl */ 3631 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3632 if (*first_xattr_slot == -1) 3633 *first_xattr_slot = slot; 3634 if (found_key.offset == xattr_access || 3635 found_key.offset == xattr_default) 3636 return 1; 3637 } 3638 3639 /* 3640 * we found a key greater than an xattr key, there can't 3641 * be any acls later on 3642 */ 3643 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3644 return 0; 3645 3646 slot++; 3647 scanned++; 3648 3649 /* 3650 * it goes inode, inode backrefs, xattrs, extents, 3651 * so if there are a ton of hard links to an inode there can 3652 * be a lot of backrefs. Don't waste time searching too hard, 3653 * this is just an optimization 3654 */ 3655 if (scanned >= 8) 3656 break; 3657 } 3658 /* we hit the end of the leaf before we found an xattr or 3659 * something larger than an xattr. We have to assume the inode 3660 * has acls 3661 */ 3662 if (*first_xattr_slot == -1) 3663 *first_xattr_slot = slot; 3664 return 1; 3665 } 3666 3667 /* 3668 * read an inode from the btree into the in-memory inode 3669 */ 3670 static int btrfs_read_locked_inode(struct inode *inode) 3671 { 3672 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3673 struct btrfs_path *path; 3674 struct extent_buffer *leaf; 3675 struct btrfs_inode_item *inode_item; 3676 struct btrfs_root *root = BTRFS_I(inode)->root; 3677 struct btrfs_key location; 3678 unsigned long ptr; 3679 int maybe_acls; 3680 u32 rdev; 3681 int ret; 3682 bool filled = false; 3683 int first_xattr_slot; 3684 3685 ret = btrfs_fill_inode(inode, &rdev); 3686 if (!ret) 3687 filled = true; 3688 3689 path = btrfs_alloc_path(); 3690 if (!path) { 3691 ret = -ENOMEM; 3692 goto make_bad; 3693 } 3694 3695 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3696 3697 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3698 if (ret) { 3699 if (ret > 0) 3700 ret = -ENOENT; 3701 goto make_bad; 3702 } 3703 3704 leaf = path->nodes[0]; 3705 3706 if (filled) 3707 goto cache_index; 3708 3709 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3710 struct btrfs_inode_item); 3711 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3712 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3713 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3714 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3715 btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item)); 3716 3717 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime); 3718 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime); 3719 3720 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime); 3721 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime); 3722 3723 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime); 3724 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime); 3725 3726 BTRFS_I(inode)->i_otime.tv_sec = 3727 btrfs_timespec_sec(leaf, &inode_item->otime); 3728 BTRFS_I(inode)->i_otime.tv_nsec = 3729 btrfs_timespec_nsec(leaf, &inode_item->otime); 3730 3731 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3732 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3733 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3734 3735 inode->i_version = btrfs_inode_sequence(leaf, inode_item); 3736 inode->i_generation = BTRFS_I(inode)->generation; 3737 inode->i_rdev = 0; 3738 rdev = btrfs_inode_rdev(leaf, inode_item); 3739 3740 BTRFS_I(inode)->index_cnt = (u64)-1; 3741 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item); 3742 3743 cache_index: 3744 /* 3745 * If we were modified in the current generation and evicted from memory 3746 * and then re-read we need to do a full sync since we don't have any 3747 * idea about which extents were modified before we were evicted from 3748 * cache. 3749 * 3750 * This is required for both inode re-read from disk and delayed inode 3751 * in delayed_nodes_tree. 3752 */ 3753 if (BTRFS_I(inode)->last_trans == fs_info->generation) 3754 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3755 &BTRFS_I(inode)->runtime_flags); 3756 3757 /* 3758 * We don't persist the id of the transaction where an unlink operation 3759 * against the inode was last made. So here we assume the inode might 3760 * have been evicted, and therefore the exact value of last_unlink_trans 3761 * lost, and set it to last_trans to avoid metadata inconsistencies 3762 * between the inode and its parent if the inode is fsync'ed and the log 3763 * replayed. For example, in the scenario: 3764 * 3765 * touch mydir/foo 3766 * ln mydir/foo mydir/bar 3767 * sync 3768 * unlink mydir/bar 3769 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 3770 * xfs_io -c fsync mydir/foo 3771 * <power failure> 3772 * mount fs, triggers fsync log replay 3773 * 3774 * We must make sure that when we fsync our inode foo we also log its 3775 * parent inode, otherwise after log replay the parent still has the 3776 * dentry with the "bar" name but our inode foo has a link count of 1 3777 * and doesn't have an inode ref with the name "bar" anymore. 3778 * 3779 * Setting last_unlink_trans to last_trans is a pessimistic approach, 3780 * but it guarantees correctness at the expense of occasional full 3781 * transaction commits on fsync if our inode is a directory, or if our 3782 * inode is not a directory, logging its parent unnecessarily. 3783 */ 3784 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans; 3785 3786 path->slots[0]++; 3787 if (inode->i_nlink != 1 || 3788 path->slots[0] >= btrfs_header_nritems(leaf)) 3789 goto cache_acl; 3790 3791 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3792 if (location.objectid != btrfs_ino(inode)) 3793 goto cache_acl; 3794 3795 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3796 if (location.type == BTRFS_INODE_REF_KEY) { 3797 struct btrfs_inode_ref *ref; 3798 3799 ref = (struct btrfs_inode_ref *)ptr; 3800 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3801 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3802 struct btrfs_inode_extref *extref; 3803 3804 extref = (struct btrfs_inode_extref *)ptr; 3805 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3806 extref); 3807 } 3808 cache_acl: 3809 /* 3810 * try to precache a NULL acl entry for files that don't have 3811 * any xattrs or acls 3812 */ 3813 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3814 btrfs_ino(inode), &first_xattr_slot); 3815 if (first_xattr_slot != -1) { 3816 path->slots[0] = first_xattr_slot; 3817 ret = btrfs_load_inode_props(inode, path); 3818 if (ret) 3819 btrfs_err(fs_info, 3820 "error loading props for ino %llu (root %llu): %d", 3821 btrfs_ino(inode), 3822 root->root_key.objectid, ret); 3823 } 3824 btrfs_free_path(path); 3825 3826 if (!maybe_acls) 3827 cache_no_acl(inode); 3828 3829 switch (inode->i_mode & S_IFMT) { 3830 case S_IFREG: 3831 inode->i_mapping->a_ops = &btrfs_aops; 3832 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 3833 inode->i_fop = &btrfs_file_operations; 3834 inode->i_op = &btrfs_file_inode_operations; 3835 break; 3836 case S_IFDIR: 3837 inode->i_fop = &btrfs_dir_file_operations; 3838 if (root == fs_info->tree_root) 3839 inode->i_op = &btrfs_dir_ro_inode_operations; 3840 else 3841 inode->i_op = &btrfs_dir_inode_operations; 3842 break; 3843 case S_IFLNK: 3844 inode->i_op = &btrfs_symlink_inode_operations; 3845 inode_nohighmem(inode); 3846 inode->i_mapping->a_ops = &btrfs_symlink_aops; 3847 break; 3848 default: 3849 inode->i_op = &btrfs_special_inode_operations; 3850 init_special_inode(inode, inode->i_mode, rdev); 3851 break; 3852 } 3853 3854 btrfs_update_iflags(inode); 3855 return 0; 3856 3857 make_bad: 3858 btrfs_free_path(path); 3859 make_bad_inode(inode); 3860 return ret; 3861 } 3862 3863 /* 3864 * given a leaf and an inode, copy the inode fields into the leaf 3865 */ 3866 static void fill_inode_item(struct btrfs_trans_handle *trans, 3867 struct extent_buffer *leaf, 3868 struct btrfs_inode_item *item, 3869 struct inode *inode) 3870 { 3871 struct btrfs_map_token token; 3872 3873 btrfs_init_map_token(&token); 3874 3875 btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token); 3876 btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token); 3877 btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size, 3878 &token); 3879 btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token); 3880 btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token); 3881 3882 btrfs_set_token_timespec_sec(leaf, &item->atime, 3883 inode->i_atime.tv_sec, &token); 3884 btrfs_set_token_timespec_nsec(leaf, &item->atime, 3885 inode->i_atime.tv_nsec, &token); 3886 3887 btrfs_set_token_timespec_sec(leaf, &item->mtime, 3888 inode->i_mtime.tv_sec, &token); 3889 btrfs_set_token_timespec_nsec(leaf, &item->mtime, 3890 inode->i_mtime.tv_nsec, &token); 3891 3892 btrfs_set_token_timespec_sec(leaf, &item->ctime, 3893 inode->i_ctime.tv_sec, &token); 3894 btrfs_set_token_timespec_nsec(leaf, &item->ctime, 3895 inode->i_ctime.tv_nsec, &token); 3896 3897 btrfs_set_token_timespec_sec(leaf, &item->otime, 3898 BTRFS_I(inode)->i_otime.tv_sec, &token); 3899 btrfs_set_token_timespec_nsec(leaf, &item->otime, 3900 BTRFS_I(inode)->i_otime.tv_nsec, &token); 3901 3902 btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode), 3903 &token); 3904 btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation, 3905 &token); 3906 btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token); 3907 btrfs_set_token_inode_transid(leaf, item, trans->transid, &token); 3908 btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token); 3909 btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token); 3910 btrfs_set_token_inode_block_group(leaf, item, 0, &token); 3911 } 3912 3913 /* 3914 * copy everything in the in-memory inode into the btree. 3915 */ 3916 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 3917 struct btrfs_root *root, struct inode *inode) 3918 { 3919 struct btrfs_inode_item *inode_item; 3920 struct btrfs_path *path; 3921 struct extent_buffer *leaf; 3922 int ret; 3923 3924 path = btrfs_alloc_path(); 3925 if (!path) 3926 return -ENOMEM; 3927 3928 path->leave_spinning = 1; 3929 ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location, 3930 1); 3931 if (ret) { 3932 if (ret > 0) 3933 ret = -ENOENT; 3934 goto failed; 3935 } 3936 3937 leaf = path->nodes[0]; 3938 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3939 struct btrfs_inode_item); 3940 3941 fill_inode_item(trans, leaf, inode_item, inode); 3942 btrfs_mark_buffer_dirty(leaf); 3943 btrfs_set_inode_last_trans(trans, inode); 3944 ret = 0; 3945 failed: 3946 btrfs_free_path(path); 3947 return ret; 3948 } 3949 3950 /* 3951 * copy everything in the in-memory inode into the btree. 3952 */ 3953 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 3954 struct btrfs_root *root, struct inode *inode) 3955 { 3956 struct btrfs_fs_info *fs_info = root->fs_info; 3957 int ret; 3958 3959 /* 3960 * If the inode is a free space inode, we can deadlock during commit 3961 * if we put it into the delayed code. 3962 * 3963 * The data relocation inode should also be directly updated 3964 * without delay 3965 */ 3966 if (!btrfs_is_free_space_inode(inode) 3967 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 3968 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 3969 btrfs_update_root_times(trans, root); 3970 3971 ret = btrfs_delayed_update_inode(trans, root, inode); 3972 if (!ret) 3973 btrfs_set_inode_last_trans(trans, inode); 3974 return ret; 3975 } 3976 3977 return btrfs_update_inode_item(trans, root, inode); 3978 } 3979 3980 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 3981 struct btrfs_root *root, 3982 struct inode *inode) 3983 { 3984 int ret; 3985 3986 ret = btrfs_update_inode(trans, root, inode); 3987 if (ret == -ENOSPC) 3988 return btrfs_update_inode_item(trans, root, inode); 3989 return ret; 3990 } 3991 3992 /* 3993 * unlink helper that gets used here in inode.c and in the tree logging 3994 * recovery code. It remove a link in a directory with a given name, and 3995 * also drops the back refs in the inode to the directory 3996 */ 3997 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 3998 struct btrfs_root *root, 3999 struct inode *dir, struct inode *inode, 4000 const char *name, int name_len) 4001 { 4002 struct btrfs_fs_info *fs_info = root->fs_info; 4003 struct btrfs_path *path; 4004 int ret = 0; 4005 struct extent_buffer *leaf; 4006 struct btrfs_dir_item *di; 4007 struct btrfs_key key; 4008 u64 index; 4009 u64 ino = btrfs_ino(inode); 4010 u64 dir_ino = btrfs_ino(dir); 4011 4012 path = btrfs_alloc_path(); 4013 if (!path) { 4014 ret = -ENOMEM; 4015 goto out; 4016 } 4017 4018 path->leave_spinning = 1; 4019 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4020 name, name_len, -1); 4021 if (IS_ERR(di)) { 4022 ret = PTR_ERR(di); 4023 goto err; 4024 } 4025 if (!di) { 4026 ret = -ENOENT; 4027 goto err; 4028 } 4029 leaf = path->nodes[0]; 4030 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4031 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4032 if (ret) 4033 goto err; 4034 btrfs_release_path(path); 4035 4036 /* 4037 * If we don't have dir index, we have to get it by looking up 4038 * the inode ref, since we get the inode ref, remove it directly, 4039 * it is unnecessary to do delayed deletion. 4040 * 4041 * But if we have dir index, needn't search inode ref to get it. 4042 * Since the inode ref is close to the inode item, it is better 4043 * that we delay to delete it, and just do this deletion when 4044 * we update the inode item. 4045 */ 4046 if (BTRFS_I(inode)->dir_index) { 4047 ret = btrfs_delayed_delete_inode_ref(inode); 4048 if (!ret) { 4049 index = BTRFS_I(inode)->dir_index; 4050 goto skip_backref; 4051 } 4052 } 4053 4054 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino, 4055 dir_ino, &index); 4056 if (ret) { 4057 btrfs_info(fs_info, 4058 "failed to delete reference to %.*s, inode %llu parent %llu", 4059 name_len, name, ino, dir_ino); 4060 btrfs_abort_transaction(trans, ret); 4061 goto err; 4062 } 4063 skip_backref: 4064 ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index); 4065 if (ret) { 4066 btrfs_abort_transaction(trans, ret); 4067 goto err; 4068 } 4069 4070 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, 4071 inode, dir_ino); 4072 if (ret != 0 && ret != -ENOENT) { 4073 btrfs_abort_transaction(trans, ret); 4074 goto err; 4075 } 4076 4077 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, 4078 dir, index); 4079 if (ret == -ENOENT) 4080 ret = 0; 4081 else if (ret) 4082 btrfs_abort_transaction(trans, ret); 4083 err: 4084 btrfs_free_path(path); 4085 if (ret) 4086 goto out; 4087 4088 btrfs_i_size_write(dir, dir->i_size - name_len * 2); 4089 inode_inc_iversion(inode); 4090 inode_inc_iversion(dir); 4091 inode->i_ctime = dir->i_mtime = 4092 dir->i_ctime = current_time(inode); 4093 ret = btrfs_update_inode(trans, root, dir); 4094 out: 4095 return ret; 4096 } 4097 4098 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4099 struct btrfs_root *root, 4100 struct inode *dir, struct inode *inode, 4101 const char *name, int name_len) 4102 { 4103 int ret; 4104 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len); 4105 if (!ret) { 4106 drop_nlink(inode); 4107 ret = btrfs_update_inode(trans, root, inode); 4108 } 4109 return ret; 4110 } 4111 4112 /* 4113 * helper to start transaction for unlink and rmdir. 4114 * 4115 * unlink and rmdir are special in btrfs, they do not always free space, so 4116 * if we cannot make our reservations the normal way try and see if there is 4117 * plenty of slack room in the global reserve to migrate, otherwise we cannot 4118 * allow the unlink to occur. 4119 */ 4120 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) 4121 { 4122 struct btrfs_root *root = BTRFS_I(dir)->root; 4123 4124 /* 4125 * 1 for the possible orphan item 4126 * 1 for the dir item 4127 * 1 for the dir index 4128 * 1 for the inode ref 4129 * 1 for the inode 4130 */ 4131 return btrfs_start_transaction_fallback_global_rsv(root, 5, 5); 4132 } 4133 4134 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 4135 { 4136 struct btrfs_root *root = BTRFS_I(dir)->root; 4137 struct btrfs_trans_handle *trans; 4138 struct inode *inode = d_inode(dentry); 4139 int ret; 4140 4141 trans = __unlink_start_trans(dir); 4142 if (IS_ERR(trans)) 4143 return PTR_ERR(trans); 4144 4145 btrfs_record_unlink_dir(trans, dir, d_inode(dentry), 0); 4146 4147 ret = btrfs_unlink_inode(trans, root, dir, d_inode(dentry), 4148 dentry->d_name.name, dentry->d_name.len); 4149 if (ret) 4150 goto out; 4151 4152 if (inode->i_nlink == 0) { 4153 ret = btrfs_orphan_add(trans, inode); 4154 if (ret) 4155 goto out; 4156 } 4157 4158 out: 4159 btrfs_end_transaction(trans); 4160 btrfs_btree_balance_dirty(root->fs_info); 4161 return ret; 4162 } 4163 4164 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 4165 struct btrfs_root *root, 4166 struct inode *dir, u64 objectid, 4167 const char *name, int name_len) 4168 { 4169 struct btrfs_fs_info *fs_info = root->fs_info; 4170 struct btrfs_path *path; 4171 struct extent_buffer *leaf; 4172 struct btrfs_dir_item *di; 4173 struct btrfs_key key; 4174 u64 index; 4175 int ret; 4176 u64 dir_ino = btrfs_ino(dir); 4177 4178 path = btrfs_alloc_path(); 4179 if (!path) 4180 return -ENOMEM; 4181 4182 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4183 name, name_len, -1); 4184 if (IS_ERR_OR_NULL(di)) { 4185 if (!di) 4186 ret = -ENOENT; 4187 else 4188 ret = PTR_ERR(di); 4189 goto out; 4190 } 4191 4192 leaf = path->nodes[0]; 4193 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4194 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 4195 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4196 if (ret) { 4197 btrfs_abort_transaction(trans, ret); 4198 goto out; 4199 } 4200 btrfs_release_path(path); 4201 4202 ret = btrfs_del_root_ref(trans, fs_info, objectid, 4203 root->root_key.objectid, dir_ino, 4204 &index, name, name_len); 4205 if (ret < 0) { 4206 if (ret != -ENOENT) { 4207 btrfs_abort_transaction(trans, ret); 4208 goto out; 4209 } 4210 di = btrfs_search_dir_index_item(root, path, dir_ino, 4211 name, name_len); 4212 if (IS_ERR_OR_NULL(di)) { 4213 if (!di) 4214 ret = -ENOENT; 4215 else 4216 ret = PTR_ERR(di); 4217 btrfs_abort_transaction(trans, ret); 4218 goto out; 4219 } 4220 4221 leaf = path->nodes[0]; 4222 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4223 btrfs_release_path(path); 4224 index = key.offset; 4225 } 4226 btrfs_release_path(path); 4227 4228 ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index); 4229 if (ret) { 4230 btrfs_abort_transaction(trans, ret); 4231 goto out; 4232 } 4233 4234 btrfs_i_size_write(dir, dir->i_size - name_len * 2); 4235 inode_inc_iversion(dir); 4236 dir->i_mtime = dir->i_ctime = current_time(dir); 4237 ret = btrfs_update_inode_fallback(trans, root, dir); 4238 if (ret) 4239 btrfs_abort_transaction(trans, ret); 4240 out: 4241 btrfs_free_path(path); 4242 return ret; 4243 } 4244 4245 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4246 { 4247 struct inode *inode = d_inode(dentry); 4248 int err = 0; 4249 struct btrfs_root *root = BTRFS_I(dir)->root; 4250 struct btrfs_trans_handle *trans; 4251 u64 last_unlink_trans; 4252 4253 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4254 return -ENOTEMPTY; 4255 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) 4256 return -EPERM; 4257 4258 trans = __unlink_start_trans(dir); 4259 if (IS_ERR(trans)) 4260 return PTR_ERR(trans); 4261 4262 if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4263 err = btrfs_unlink_subvol(trans, root, dir, 4264 BTRFS_I(inode)->location.objectid, 4265 dentry->d_name.name, 4266 dentry->d_name.len); 4267 goto out; 4268 } 4269 4270 err = btrfs_orphan_add(trans, inode); 4271 if (err) 4272 goto out; 4273 4274 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; 4275 4276 /* now the directory is empty */ 4277 err = btrfs_unlink_inode(trans, root, dir, d_inode(dentry), 4278 dentry->d_name.name, dentry->d_name.len); 4279 if (!err) { 4280 btrfs_i_size_write(inode, 0); 4281 /* 4282 * Propagate the last_unlink_trans value of the deleted dir to 4283 * its parent directory. This is to prevent an unrecoverable 4284 * log tree in the case we do something like this: 4285 * 1) create dir foo 4286 * 2) create snapshot under dir foo 4287 * 3) delete the snapshot 4288 * 4) rmdir foo 4289 * 5) mkdir foo 4290 * 6) fsync foo or some file inside foo 4291 */ 4292 if (last_unlink_trans >= trans->transid) 4293 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; 4294 } 4295 out: 4296 btrfs_end_transaction(trans); 4297 btrfs_btree_balance_dirty(root->fs_info); 4298 4299 return err; 4300 } 4301 4302 static int truncate_space_check(struct btrfs_trans_handle *trans, 4303 struct btrfs_root *root, 4304 u64 bytes_deleted) 4305 { 4306 struct btrfs_fs_info *fs_info = root->fs_info; 4307 int ret; 4308 4309 /* 4310 * This is only used to apply pressure to the enospc system, we don't 4311 * intend to use this reservation at all. 4312 */ 4313 bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted); 4314 bytes_deleted *= fs_info->nodesize; 4315 ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv, 4316 bytes_deleted, BTRFS_RESERVE_NO_FLUSH); 4317 if (!ret) { 4318 trace_btrfs_space_reservation(fs_info, "transaction", 4319 trans->transid, 4320 bytes_deleted, 1); 4321 trans->bytes_reserved += bytes_deleted; 4322 } 4323 return ret; 4324 4325 } 4326 4327 static int truncate_inline_extent(struct inode *inode, 4328 struct btrfs_path *path, 4329 struct btrfs_key *found_key, 4330 const u64 item_end, 4331 const u64 new_size) 4332 { 4333 struct extent_buffer *leaf = path->nodes[0]; 4334 int slot = path->slots[0]; 4335 struct btrfs_file_extent_item *fi; 4336 u32 size = (u32)(new_size - found_key->offset); 4337 struct btrfs_root *root = BTRFS_I(inode)->root; 4338 4339 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 4340 4341 if (btrfs_file_extent_compression(leaf, fi) != BTRFS_COMPRESS_NONE) { 4342 loff_t offset = new_size; 4343 loff_t page_end = ALIGN(offset, PAGE_SIZE); 4344 4345 /* 4346 * Zero out the remaining of the last page of our inline extent, 4347 * instead of directly truncating our inline extent here - that 4348 * would be much more complex (decompressing all the data, then 4349 * compressing the truncated data, which might be bigger than 4350 * the size of the inline extent, resize the extent, etc). 4351 * We release the path because to get the page we might need to 4352 * read the extent item from disk (data not in the page cache). 4353 */ 4354 btrfs_release_path(path); 4355 return btrfs_truncate_block(inode, offset, page_end - offset, 4356 0); 4357 } 4358 4359 btrfs_set_file_extent_ram_bytes(leaf, fi, size); 4360 size = btrfs_file_extent_calc_inline_size(size); 4361 btrfs_truncate_item(root->fs_info, path, size, 1); 4362 4363 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4364 inode_sub_bytes(inode, item_end + 1 - new_size); 4365 4366 return 0; 4367 } 4368 4369 /* 4370 * this can truncate away extent items, csum items and directory items. 4371 * It starts at a high offset and removes keys until it can't find 4372 * any higher than new_size 4373 * 4374 * csum items that cross the new i_size are truncated to the new size 4375 * as well. 4376 * 4377 * min_type is the minimum key type to truncate down to. If set to 0, this 4378 * will kill all the items on this inode, including the INODE_ITEM_KEY. 4379 */ 4380 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, 4381 struct btrfs_root *root, 4382 struct inode *inode, 4383 u64 new_size, u32 min_type) 4384 { 4385 struct btrfs_fs_info *fs_info = root->fs_info; 4386 struct btrfs_path *path; 4387 struct extent_buffer *leaf; 4388 struct btrfs_file_extent_item *fi; 4389 struct btrfs_key key; 4390 struct btrfs_key found_key; 4391 u64 extent_start = 0; 4392 u64 extent_num_bytes = 0; 4393 u64 extent_offset = 0; 4394 u64 item_end = 0; 4395 u64 last_size = new_size; 4396 u32 found_type = (u8)-1; 4397 int found_extent; 4398 int del_item; 4399 int pending_del_nr = 0; 4400 int pending_del_slot = 0; 4401 int extent_type = -1; 4402 int ret; 4403 int err = 0; 4404 u64 ino = btrfs_ino(inode); 4405 u64 bytes_deleted = 0; 4406 bool be_nice = 0; 4407 bool should_throttle = 0; 4408 bool should_end = 0; 4409 4410 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); 4411 4412 /* 4413 * for non-free space inodes and ref cows, we want to back off from 4414 * time to time 4415 */ 4416 if (!btrfs_is_free_space_inode(inode) && 4417 test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4418 be_nice = 1; 4419 4420 path = btrfs_alloc_path(); 4421 if (!path) 4422 return -ENOMEM; 4423 path->reada = READA_BACK; 4424 4425 /* 4426 * We want to drop from the next block forward in case this new size is 4427 * not block aligned since we will be keeping the last block of the 4428 * extent just the way it is. 4429 */ 4430 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4431 root == fs_info->tree_root) 4432 btrfs_drop_extent_cache(inode, ALIGN(new_size, 4433 fs_info->sectorsize), 4434 (u64)-1, 0); 4435 4436 /* 4437 * This function is also used to drop the items in the log tree before 4438 * we relog the inode, so if root != BTRFS_I(inode)->root, it means 4439 * it is used to drop the loged items. So we shouldn't kill the delayed 4440 * items. 4441 */ 4442 if (min_type == 0 && root == BTRFS_I(inode)->root) 4443 btrfs_kill_delayed_inode_items(inode); 4444 4445 key.objectid = ino; 4446 key.offset = (u64)-1; 4447 key.type = (u8)-1; 4448 4449 search_again: 4450 /* 4451 * with a 16K leaf size and 128MB extents, you can actually queue 4452 * up a huge file in a single leaf. Most of the time that 4453 * bytes_deleted is > 0, it will be huge by the time we get here 4454 */ 4455 if (be_nice && bytes_deleted > SZ_32M) { 4456 if (btrfs_should_end_transaction(trans)) { 4457 err = -EAGAIN; 4458 goto error; 4459 } 4460 } 4461 4462 4463 path->leave_spinning = 1; 4464 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 4465 if (ret < 0) { 4466 err = ret; 4467 goto out; 4468 } 4469 4470 if (ret > 0) { 4471 /* there are no items in the tree for us to truncate, we're 4472 * done 4473 */ 4474 if (path->slots[0] == 0) 4475 goto out; 4476 path->slots[0]--; 4477 } 4478 4479 while (1) { 4480 fi = NULL; 4481 leaf = path->nodes[0]; 4482 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4483 found_type = found_key.type; 4484 4485 if (found_key.objectid != ino) 4486 break; 4487 4488 if (found_type < min_type) 4489 break; 4490 4491 item_end = found_key.offset; 4492 if (found_type == BTRFS_EXTENT_DATA_KEY) { 4493 fi = btrfs_item_ptr(leaf, path->slots[0], 4494 struct btrfs_file_extent_item); 4495 extent_type = btrfs_file_extent_type(leaf, fi); 4496 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4497 item_end += 4498 btrfs_file_extent_num_bytes(leaf, fi); 4499 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4500 item_end += btrfs_file_extent_inline_len(leaf, 4501 path->slots[0], fi); 4502 } 4503 item_end--; 4504 } 4505 if (found_type > min_type) { 4506 del_item = 1; 4507 } else { 4508 if (item_end < new_size) 4509 break; 4510 if (found_key.offset >= new_size) 4511 del_item = 1; 4512 else 4513 del_item = 0; 4514 } 4515 found_extent = 0; 4516 /* FIXME, shrink the extent if the ref count is only 1 */ 4517 if (found_type != BTRFS_EXTENT_DATA_KEY) 4518 goto delete; 4519 4520 if (del_item) 4521 last_size = found_key.offset; 4522 else 4523 last_size = new_size; 4524 4525 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4526 u64 num_dec; 4527 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); 4528 if (!del_item) { 4529 u64 orig_num_bytes = 4530 btrfs_file_extent_num_bytes(leaf, fi); 4531 extent_num_bytes = ALIGN(new_size - 4532 found_key.offset, 4533 fs_info->sectorsize); 4534 btrfs_set_file_extent_num_bytes(leaf, fi, 4535 extent_num_bytes); 4536 num_dec = (orig_num_bytes - 4537 extent_num_bytes); 4538 if (test_bit(BTRFS_ROOT_REF_COWS, 4539 &root->state) && 4540 extent_start != 0) 4541 inode_sub_bytes(inode, num_dec); 4542 btrfs_mark_buffer_dirty(leaf); 4543 } else { 4544 extent_num_bytes = 4545 btrfs_file_extent_disk_num_bytes(leaf, 4546 fi); 4547 extent_offset = found_key.offset - 4548 btrfs_file_extent_offset(leaf, fi); 4549 4550 /* FIXME blocksize != 4096 */ 4551 num_dec = btrfs_file_extent_num_bytes(leaf, fi); 4552 if (extent_start != 0) { 4553 found_extent = 1; 4554 if (test_bit(BTRFS_ROOT_REF_COWS, 4555 &root->state)) 4556 inode_sub_bytes(inode, num_dec); 4557 } 4558 } 4559 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4560 /* 4561 * we can't truncate inline items that have had 4562 * special encodings 4563 */ 4564 if (!del_item && 4565 btrfs_file_extent_encryption(leaf, fi) == 0 && 4566 btrfs_file_extent_other_encoding(leaf, fi) == 0) { 4567 4568 /* 4569 * Need to release path in order to truncate a 4570 * compressed extent. So delete any accumulated 4571 * extent items so far. 4572 */ 4573 if (btrfs_file_extent_compression(leaf, fi) != 4574 BTRFS_COMPRESS_NONE && pending_del_nr) { 4575 err = btrfs_del_items(trans, root, path, 4576 pending_del_slot, 4577 pending_del_nr); 4578 if (err) { 4579 btrfs_abort_transaction(trans, 4580 err); 4581 goto error; 4582 } 4583 pending_del_nr = 0; 4584 } 4585 4586 err = truncate_inline_extent(inode, path, 4587 &found_key, 4588 item_end, 4589 new_size); 4590 if (err) { 4591 btrfs_abort_transaction(trans, err); 4592 goto error; 4593 } 4594 } else if (test_bit(BTRFS_ROOT_REF_COWS, 4595 &root->state)) { 4596 inode_sub_bytes(inode, item_end + 1 - new_size); 4597 } 4598 } 4599 delete: 4600 if (del_item) { 4601 if (!pending_del_nr) { 4602 /* no pending yet, add ourselves */ 4603 pending_del_slot = path->slots[0]; 4604 pending_del_nr = 1; 4605 } else if (pending_del_nr && 4606 path->slots[0] + 1 == pending_del_slot) { 4607 /* hop on the pending chunk */ 4608 pending_del_nr++; 4609 pending_del_slot = path->slots[0]; 4610 } else { 4611 BUG(); 4612 } 4613 } else { 4614 break; 4615 } 4616 should_throttle = 0; 4617 4618 if (found_extent && 4619 (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4620 root == fs_info->tree_root)) { 4621 btrfs_set_path_blocking(path); 4622 bytes_deleted += extent_num_bytes; 4623 ret = btrfs_free_extent(trans, fs_info, extent_start, 4624 extent_num_bytes, 0, 4625 btrfs_header_owner(leaf), 4626 ino, extent_offset); 4627 BUG_ON(ret); 4628 if (btrfs_should_throttle_delayed_refs(trans, fs_info)) 4629 btrfs_async_run_delayed_refs(fs_info, 4630 trans->delayed_ref_updates * 2, 4631 trans->transid, 0); 4632 if (be_nice) { 4633 if (truncate_space_check(trans, root, 4634 extent_num_bytes)) { 4635 should_end = 1; 4636 } 4637 if (btrfs_should_throttle_delayed_refs(trans, 4638 fs_info)) 4639 should_throttle = 1; 4640 } 4641 } 4642 4643 if (found_type == BTRFS_INODE_ITEM_KEY) 4644 break; 4645 4646 if (path->slots[0] == 0 || 4647 path->slots[0] != pending_del_slot || 4648 should_throttle || should_end) { 4649 if (pending_del_nr) { 4650 ret = btrfs_del_items(trans, root, path, 4651 pending_del_slot, 4652 pending_del_nr); 4653 if (ret) { 4654 btrfs_abort_transaction(trans, ret); 4655 goto error; 4656 } 4657 pending_del_nr = 0; 4658 } 4659 btrfs_release_path(path); 4660 if (should_throttle) { 4661 unsigned long updates = trans->delayed_ref_updates; 4662 if (updates) { 4663 trans->delayed_ref_updates = 0; 4664 ret = btrfs_run_delayed_refs(trans, 4665 fs_info, 4666 updates * 2); 4667 if (ret && !err) 4668 err = ret; 4669 } 4670 } 4671 /* 4672 * if we failed to refill our space rsv, bail out 4673 * and let the transaction restart 4674 */ 4675 if (should_end) { 4676 err = -EAGAIN; 4677 goto error; 4678 } 4679 goto search_again; 4680 } else { 4681 path->slots[0]--; 4682 } 4683 } 4684 out: 4685 if (pending_del_nr) { 4686 ret = btrfs_del_items(trans, root, path, pending_del_slot, 4687 pending_del_nr); 4688 if (ret) 4689 btrfs_abort_transaction(trans, ret); 4690 } 4691 error: 4692 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) 4693 btrfs_ordered_update_i_size(inode, last_size, NULL); 4694 4695 btrfs_free_path(path); 4696 4697 if (be_nice && bytes_deleted > SZ_32M) { 4698 unsigned long updates = trans->delayed_ref_updates; 4699 if (updates) { 4700 trans->delayed_ref_updates = 0; 4701 ret = btrfs_run_delayed_refs(trans, fs_info, 4702 updates * 2); 4703 if (ret && !err) 4704 err = ret; 4705 } 4706 } 4707 return err; 4708 } 4709 4710 /* 4711 * btrfs_truncate_block - read, zero a chunk and write a block 4712 * @inode - inode that we're zeroing 4713 * @from - the offset to start zeroing 4714 * @len - the length to zero, 0 to zero the entire range respective to the 4715 * offset 4716 * @front - zero up to the offset instead of from the offset on 4717 * 4718 * This will find the block for the "from" offset and cow the block and zero the 4719 * part we want to zero. This is used with truncate and hole punching. 4720 */ 4721 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len, 4722 int front) 4723 { 4724 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4725 struct address_space *mapping = inode->i_mapping; 4726 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4727 struct btrfs_ordered_extent *ordered; 4728 struct extent_state *cached_state = NULL; 4729 char *kaddr; 4730 u32 blocksize = fs_info->sectorsize; 4731 pgoff_t index = from >> PAGE_SHIFT; 4732 unsigned offset = from & (blocksize - 1); 4733 struct page *page; 4734 gfp_t mask = btrfs_alloc_write_mask(mapping); 4735 int ret = 0; 4736 u64 block_start; 4737 u64 block_end; 4738 4739 if ((offset & (blocksize - 1)) == 0 && 4740 (!len || ((len & (blocksize - 1)) == 0))) 4741 goto out; 4742 4743 ret = btrfs_delalloc_reserve_space(inode, 4744 round_down(from, blocksize), blocksize); 4745 if (ret) 4746 goto out; 4747 4748 again: 4749 page = find_or_create_page(mapping, index, mask); 4750 if (!page) { 4751 btrfs_delalloc_release_space(inode, 4752 round_down(from, blocksize), 4753 blocksize); 4754 ret = -ENOMEM; 4755 goto out; 4756 } 4757 4758 block_start = round_down(from, blocksize); 4759 block_end = block_start + blocksize - 1; 4760 4761 if (!PageUptodate(page)) { 4762 ret = btrfs_readpage(NULL, page); 4763 lock_page(page); 4764 if (page->mapping != mapping) { 4765 unlock_page(page); 4766 put_page(page); 4767 goto again; 4768 } 4769 if (!PageUptodate(page)) { 4770 ret = -EIO; 4771 goto out_unlock; 4772 } 4773 } 4774 wait_on_page_writeback(page); 4775 4776 lock_extent_bits(io_tree, block_start, block_end, &cached_state); 4777 set_page_extent_mapped(page); 4778 4779 ordered = btrfs_lookup_ordered_extent(inode, block_start); 4780 if (ordered) { 4781 unlock_extent_cached(io_tree, block_start, block_end, 4782 &cached_state, GFP_NOFS); 4783 unlock_page(page); 4784 put_page(page); 4785 btrfs_start_ordered_extent(inode, ordered, 1); 4786 btrfs_put_ordered_extent(ordered); 4787 goto again; 4788 } 4789 4790 clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end, 4791 EXTENT_DIRTY | EXTENT_DELALLOC | 4792 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4793 0, 0, &cached_state, GFP_NOFS); 4794 4795 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 4796 &cached_state, 0); 4797 if (ret) { 4798 unlock_extent_cached(io_tree, block_start, block_end, 4799 &cached_state, GFP_NOFS); 4800 goto out_unlock; 4801 } 4802 4803 if (offset != blocksize) { 4804 if (!len) 4805 len = blocksize - offset; 4806 kaddr = kmap(page); 4807 if (front) 4808 memset(kaddr + (block_start - page_offset(page)), 4809 0, offset); 4810 else 4811 memset(kaddr + (block_start - page_offset(page)) + offset, 4812 0, len); 4813 flush_dcache_page(page); 4814 kunmap(page); 4815 } 4816 ClearPageChecked(page); 4817 set_page_dirty(page); 4818 unlock_extent_cached(io_tree, block_start, block_end, &cached_state, 4819 GFP_NOFS); 4820 4821 out_unlock: 4822 if (ret) 4823 btrfs_delalloc_release_space(inode, block_start, 4824 blocksize); 4825 unlock_page(page); 4826 put_page(page); 4827 out: 4828 return ret; 4829 } 4830 4831 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode, 4832 u64 offset, u64 len) 4833 { 4834 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4835 struct btrfs_trans_handle *trans; 4836 int ret; 4837 4838 /* 4839 * Still need to make sure the inode looks like it's been updated so 4840 * that any holes get logged if we fsync. 4841 */ 4842 if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 4843 BTRFS_I(inode)->last_trans = fs_info->generation; 4844 BTRFS_I(inode)->last_sub_trans = root->log_transid; 4845 BTRFS_I(inode)->last_log_commit = root->last_log_commit; 4846 return 0; 4847 } 4848 4849 /* 4850 * 1 - for the one we're dropping 4851 * 1 - for the one we're adding 4852 * 1 - for updating the inode. 4853 */ 4854 trans = btrfs_start_transaction(root, 3); 4855 if (IS_ERR(trans)) 4856 return PTR_ERR(trans); 4857 4858 ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1); 4859 if (ret) { 4860 btrfs_abort_transaction(trans, ret); 4861 btrfs_end_transaction(trans); 4862 return ret; 4863 } 4864 4865 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), offset, 4866 0, 0, len, 0, len, 0, 0, 0); 4867 if (ret) 4868 btrfs_abort_transaction(trans, ret); 4869 else 4870 btrfs_update_inode(trans, root, inode); 4871 btrfs_end_transaction(trans); 4872 return ret; 4873 } 4874 4875 /* 4876 * This function puts in dummy file extents for the area we're creating a hole 4877 * for. So if we are truncating this file to a larger size we need to insert 4878 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 4879 * the range between oldsize and size 4880 */ 4881 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size) 4882 { 4883 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4884 struct btrfs_root *root = BTRFS_I(inode)->root; 4885 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4886 struct extent_map *em = NULL; 4887 struct extent_state *cached_state = NULL; 4888 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 4889 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 4890 u64 block_end = ALIGN(size, fs_info->sectorsize); 4891 u64 last_byte; 4892 u64 cur_offset; 4893 u64 hole_size; 4894 int err = 0; 4895 4896 /* 4897 * If our size started in the middle of a block we need to zero out the 4898 * rest of the block before we expand the i_size, otherwise we could 4899 * expose stale data. 4900 */ 4901 err = btrfs_truncate_block(inode, oldsize, 0, 0); 4902 if (err) 4903 return err; 4904 4905 if (size <= hole_start) 4906 return 0; 4907 4908 while (1) { 4909 struct btrfs_ordered_extent *ordered; 4910 4911 lock_extent_bits(io_tree, hole_start, block_end - 1, 4912 &cached_state); 4913 ordered = btrfs_lookup_ordered_range(inode, hole_start, 4914 block_end - hole_start); 4915 if (!ordered) 4916 break; 4917 unlock_extent_cached(io_tree, hole_start, block_end - 1, 4918 &cached_state, GFP_NOFS); 4919 btrfs_start_ordered_extent(inode, ordered, 1); 4920 btrfs_put_ordered_extent(ordered); 4921 } 4922 4923 cur_offset = hole_start; 4924 while (1) { 4925 em = btrfs_get_extent(inode, NULL, 0, cur_offset, 4926 block_end - cur_offset, 0); 4927 if (IS_ERR(em)) { 4928 err = PTR_ERR(em); 4929 em = NULL; 4930 break; 4931 } 4932 last_byte = min(extent_map_end(em), block_end); 4933 last_byte = ALIGN(last_byte, fs_info->sectorsize); 4934 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 4935 struct extent_map *hole_em; 4936 hole_size = last_byte - cur_offset; 4937 4938 err = maybe_insert_hole(root, inode, cur_offset, 4939 hole_size); 4940 if (err) 4941 break; 4942 btrfs_drop_extent_cache(inode, cur_offset, 4943 cur_offset + hole_size - 1, 0); 4944 hole_em = alloc_extent_map(); 4945 if (!hole_em) { 4946 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 4947 &BTRFS_I(inode)->runtime_flags); 4948 goto next; 4949 } 4950 hole_em->start = cur_offset; 4951 hole_em->len = hole_size; 4952 hole_em->orig_start = cur_offset; 4953 4954 hole_em->block_start = EXTENT_MAP_HOLE; 4955 hole_em->block_len = 0; 4956 hole_em->orig_block_len = 0; 4957 hole_em->ram_bytes = hole_size; 4958 hole_em->bdev = fs_info->fs_devices->latest_bdev; 4959 hole_em->compress_type = BTRFS_COMPRESS_NONE; 4960 hole_em->generation = fs_info->generation; 4961 4962 while (1) { 4963 write_lock(&em_tree->lock); 4964 err = add_extent_mapping(em_tree, hole_em, 1); 4965 write_unlock(&em_tree->lock); 4966 if (err != -EEXIST) 4967 break; 4968 btrfs_drop_extent_cache(inode, cur_offset, 4969 cur_offset + 4970 hole_size - 1, 0); 4971 } 4972 free_extent_map(hole_em); 4973 } 4974 next: 4975 free_extent_map(em); 4976 em = NULL; 4977 cur_offset = last_byte; 4978 if (cur_offset >= block_end) 4979 break; 4980 } 4981 free_extent_map(em); 4982 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state, 4983 GFP_NOFS); 4984 return err; 4985 } 4986 4987 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 4988 { 4989 struct btrfs_root *root = BTRFS_I(inode)->root; 4990 struct btrfs_trans_handle *trans; 4991 loff_t oldsize = i_size_read(inode); 4992 loff_t newsize = attr->ia_size; 4993 int mask = attr->ia_valid; 4994 int ret; 4995 4996 /* 4997 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 4998 * special case where we need to update the times despite not having 4999 * these flags set. For all other operations the VFS set these flags 5000 * explicitly if it wants a timestamp update. 5001 */ 5002 if (newsize != oldsize) { 5003 inode_inc_iversion(inode); 5004 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) 5005 inode->i_ctime = inode->i_mtime = 5006 current_time(inode); 5007 } 5008 5009 if (newsize > oldsize) { 5010 /* 5011 * Don't do an expanding truncate while snapshoting is ongoing. 5012 * This is to ensure the snapshot captures a fully consistent 5013 * state of this file - if the snapshot captures this expanding 5014 * truncation, it must capture all writes that happened before 5015 * this truncation. 5016 */ 5017 btrfs_wait_for_snapshot_creation(root); 5018 ret = btrfs_cont_expand(inode, oldsize, newsize); 5019 if (ret) { 5020 btrfs_end_write_no_snapshoting(root); 5021 return ret; 5022 } 5023 5024 trans = btrfs_start_transaction(root, 1); 5025 if (IS_ERR(trans)) { 5026 btrfs_end_write_no_snapshoting(root); 5027 return PTR_ERR(trans); 5028 } 5029 5030 i_size_write(inode, newsize); 5031 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL); 5032 pagecache_isize_extended(inode, oldsize, newsize); 5033 ret = btrfs_update_inode(trans, root, inode); 5034 btrfs_end_write_no_snapshoting(root); 5035 btrfs_end_transaction(trans); 5036 } else { 5037 5038 /* 5039 * We're truncating a file that used to have good data down to 5040 * zero. Make sure it gets into the ordered flush list so that 5041 * any new writes get down to disk quickly. 5042 */ 5043 if (newsize == 0) 5044 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE, 5045 &BTRFS_I(inode)->runtime_flags); 5046 5047 /* 5048 * 1 for the orphan item we're going to add 5049 * 1 for the orphan item deletion. 5050 */ 5051 trans = btrfs_start_transaction(root, 2); 5052 if (IS_ERR(trans)) 5053 return PTR_ERR(trans); 5054 5055 /* 5056 * We need to do this in case we fail at _any_ point during the 5057 * actual truncate. Once we do the truncate_setsize we could 5058 * invalidate pages which forces any outstanding ordered io to 5059 * be instantly completed which will give us extents that need 5060 * to be truncated. If we fail to get an orphan inode down we 5061 * could have left over extents that were never meant to live, 5062 * so we need to guarantee from this point on that everything 5063 * will be consistent. 5064 */ 5065 ret = btrfs_orphan_add(trans, inode); 5066 btrfs_end_transaction(trans); 5067 if (ret) 5068 return ret; 5069 5070 /* we don't support swapfiles, so vmtruncate shouldn't fail */ 5071 truncate_setsize(inode, newsize); 5072 5073 /* Disable nonlocked read DIO to avoid the end less truncate */ 5074 btrfs_inode_block_unlocked_dio(inode); 5075 inode_dio_wait(inode); 5076 btrfs_inode_resume_unlocked_dio(inode); 5077 5078 ret = btrfs_truncate(inode); 5079 if (ret && inode->i_nlink) { 5080 int err; 5081 5082 /* 5083 * failed to truncate, disk_i_size is only adjusted down 5084 * as we remove extents, so it should represent the true 5085 * size of the inode, so reset the in memory size and 5086 * delete our orphan entry. 5087 */ 5088 trans = btrfs_join_transaction(root); 5089 if (IS_ERR(trans)) { 5090 btrfs_orphan_del(NULL, inode); 5091 return ret; 5092 } 5093 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 5094 err = btrfs_orphan_del(trans, inode); 5095 if (err) 5096 btrfs_abort_transaction(trans, err); 5097 btrfs_end_transaction(trans); 5098 } 5099 } 5100 5101 return ret; 5102 } 5103 5104 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) 5105 { 5106 struct inode *inode = d_inode(dentry); 5107 struct btrfs_root *root = BTRFS_I(inode)->root; 5108 int err; 5109 5110 if (btrfs_root_readonly(root)) 5111 return -EROFS; 5112 5113 err = setattr_prepare(dentry, attr); 5114 if (err) 5115 return err; 5116 5117 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 5118 err = btrfs_setsize(inode, attr); 5119 if (err) 5120 return err; 5121 } 5122 5123 if (attr->ia_valid) { 5124 setattr_copy(inode, attr); 5125 inode_inc_iversion(inode); 5126 err = btrfs_dirty_inode(inode); 5127 5128 if (!err && attr->ia_valid & ATTR_MODE) 5129 err = posix_acl_chmod(inode, inode->i_mode); 5130 } 5131 5132 return err; 5133 } 5134 5135 /* 5136 * While truncating the inode pages during eviction, we get the VFS calling 5137 * btrfs_invalidatepage() against each page of the inode. This is slow because 5138 * the calls to btrfs_invalidatepage() result in a huge amount of calls to 5139 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting 5140 * extent_state structures over and over, wasting lots of time. 5141 * 5142 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all 5143 * those expensive operations on a per page basis and do only the ordered io 5144 * finishing, while we release here the extent_map and extent_state structures, 5145 * without the excessive merging and splitting. 5146 */ 5147 static void evict_inode_truncate_pages(struct inode *inode) 5148 { 5149 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 5150 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; 5151 struct rb_node *node; 5152 5153 ASSERT(inode->i_state & I_FREEING); 5154 truncate_inode_pages_final(&inode->i_data); 5155 5156 write_lock(&map_tree->lock); 5157 while (!RB_EMPTY_ROOT(&map_tree->map)) { 5158 struct extent_map *em; 5159 5160 node = rb_first(&map_tree->map); 5161 em = rb_entry(node, struct extent_map, rb_node); 5162 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 5163 clear_bit(EXTENT_FLAG_LOGGING, &em->flags); 5164 remove_extent_mapping(map_tree, em); 5165 free_extent_map(em); 5166 if (need_resched()) { 5167 write_unlock(&map_tree->lock); 5168 cond_resched(); 5169 write_lock(&map_tree->lock); 5170 } 5171 } 5172 write_unlock(&map_tree->lock); 5173 5174 /* 5175 * Keep looping until we have no more ranges in the io tree. 5176 * We can have ongoing bios started by readpages (called from readahead) 5177 * that have their endio callback (extent_io.c:end_bio_extent_readpage) 5178 * still in progress (unlocked the pages in the bio but did not yet 5179 * unlocked the ranges in the io tree). Therefore this means some 5180 * ranges can still be locked and eviction started because before 5181 * submitting those bios, which are executed by a separate task (work 5182 * queue kthread), inode references (inode->i_count) were not taken 5183 * (which would be dropped in the end io callback of each bio). 5184 * Therefore here we effectively end up waiting for those bios and 5185 * anyone else holding locked ranges without having bumped the inode's 5186 * reference count - if we don't do it, when they access the inode's 5187 * io_tree to unlock a range it may be too late, leading to an 5188 * use-after-free issue. 5189 */ 5190 spin_lock(&io_tree->lock); 5191 while (!RB_EMPTY_ROOT(&io_tree->state)) { 5192 struct extent_state *state; 5193 struct extent_state *cached_state = NULL; 5194 u64 start; 5195 u64 end; 5196 5197 node = rb_first(&io_tree->state); 5198 state = rb_entry(node, struct extent_state, rb_node); 5199 start = state->start; 5200 end = state->end; 5201 spin_unlock(&io_tree->lock); 5202 5203 lock_extent_bits(io_tree, start, end, &cached_state); 5204 5205 /* 5206 * If still has DELALLOC flag, the extent didn't reach disk, 5207 * and its reserved space won't be freed by delayed_ref. 5208 * So we need to free its reserved space here. 5209 * (Refer to comment in btrfs_invalidatepage, case 2) 5210 * 5211 * Note, end is the bytenr of last byte, so we need + 1 here. 5212 */ 5213 if (state->state & EXTENT_DELALLOC) 5214 btrfs_qgroup_free_data(inode, start, end - start + 1); 5215 5216 clear_extent_bit(io_tree, start, end, 5217 EXTENT_LOCKED | EXTENT_DIRTY | 5218 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 5219 EXTENT_DEFRAG, 1, 1, 5220 &cached_state, GFP_NOFS); 5221 5222 cond_resched(); 5223 spin_lock(&io_tree->lock); 5224 } 5225 spin_unlock(&io_tree->lock); 5226 } 5227 5228 void btrfs_evict_inode(struct inode *inode) 5229 { 5230 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5231 struct btrfs_trans_handle *trans; 5232 struct btrfs_root *root = BTRFS_I(inode)->root; 5233 struct btrfs_block_rsv *rsv, *global_rsv; 5234 int steal_from_global = 0; 5235 u64 min_size; 5236 int ret; 5237 5238 trace_btrfs_inode_evict(inode); 5239 5240 if (!root) { 5241 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 5242 return; 5243 } 5244 5245 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 5246 5247 evict_inode_truncate_pages(inode); 5248 5249 if (inode->i_nlink && 5250 ((btrfs_root_refs(&root->root_item) != 0 && 5251 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 5252 btrfs_is_free_space_inode(inode))) 5253 goto no_delete; 5254 5255 if (is_bad_inode(inode)) { 5256 btrfs_orphan_del(NULL, inode); 5257 goto no_delete; 5258 } 5259 /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */ 5260 if (!special_file(inode->i_mode)) 5261 btrfs_wait_ordered_range(inode, 0, (u64)-1); 5262 5263 btrfs_free_io_failure_record(inode, 0, (u64)-1); 5264 5265 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 5266 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 5267 &BTRFS_I(inode)->runtime_flags)); 5268 goto no_delete; 5269 } 5270 5271 if (inode->i_nlink > 0) { 5272 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5273 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 5274 goto no_delete; 5275 } 5276 5277 ret = btrfs_commit_inode_delayed_inode(inode); 5278 if (ret) { 5279 btrfs_orphan_del(NULL, inode); 5280 goto no_delete; 5281 } 5282 5283 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 5284 if (!rsv) { 5285 btrfs_orphan_del(NULL, inode); 5286 goto no_delete; 5287 } 5288 rsv->size = min_size; 5289 rsv->failfast = 1; 5290 global_rsv = &fs_info->global_block_rsv; 5291 5292 btrfs_i_size_write(inode, 0); 5293 5294 /* 5295 * This is a bit simpler than btrfs_truncate since we've already 5296 * reserved our space for our orphan item in the unlink, so we just 5297 * need to reserve some slack space in case we add bytes and update 5298 * inode item when doing the truncate. 5299 */ 5300 while (1) { 5301 ret = btrfs_block_rsv_refill(root, rsv, min_size, 5302 BTRFS_RESERVE_FLUSH_LIMIT); 5303 5304 /* 5305 * Try and steal from the global reserve since we will 5306 * likely not use this space anyway, we want to try as 5307 * hard as possible to get this to work. 5308 */ 5309 if (ret) 5310 steal_from_global++; 5311 else 5312 steal_from_global = 0; 5313 ret = 0; 5314 5315 /* 5316 * steal_from_global == 0: we reserved stuff, hooray! 5317 * steal_from_global == 1: we didn't reserve stuff, boo! 5318 * steal_from_global == 2: we've committed, still not a lot of 5319 * room but maybe we'll have room in the global reserve this 5320 * time. 5321 * steal_from_global == 3: abandon all hope! 5322 */ 5323 if (steal_from_global > 2) { 5324 btrfs_warn(fs_info, 5325 "Could not get space for a delete, will truncate on mount %d", 5326 ret); 5327 btrfs_orphan_del(NULL, inode); 5328 btrfs_free_block_rsv(fs_info, rsv); 5329 goto no_delete; 5330 } 5331 5332 trans = btrfs_join_transaction(root); 5333 if (IS_ERR(trans)) { 5334 btrfs_orphan_del(NULL, inode); 5335 btrfs_free_block_rsv(fs_info, rsv); 5336 goto no_delete; 5337 } 5338 5339 /* 5340 * We can't just steal from the global reserve, we need to make 5341 * sure there is room to do it, if not we need to commit and try 5342 * again. 5343 */ 5344 if (steal_from_global) { 5345 if (!btrfs_check_space_for_delayed_refs(trans, fs_info)) 5346 ret = btrfs_block_rsv_migrate(global_rsv, rsv, 5347 min_size, 0); 5348 else 5349 ret = -ENOSPC; 5350 } 5351 5352 /* 5353 * Couldn't steal from the global reserve, we have too much 5354 * pending stuff built up, commit the transaction and try it 5355 * again. 5356 */ 5357 if (ret) { 5358 ret = btrfs_commit_transaction(trans); 5359 if (ret) { 5360 btrfs_orphan_del(NULL, inode); 5361 btrfs_free_block_rsv(fs_info, rsv); 5362 goto no_delete; 5363 } 5364 continue; 5365 } else { 5366 steal_from_global = 0; 5367 } 5368 5369 trans->block_rsv = rsv; 5370 5371 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0); 5372 if (ret != -ENOSPC && ret != -EAGAIN) 5373 break; 5374 5375 trans->block_rsv = &fs_info->trans_block_rsv; 5376 btrfs_end_transaction(trans); 5377 trans = NULL; 5378 btrfs_btree_balance_dirty(fs_info); 5379 } 5380 5381 btrfs_free_block_rsv(fs_info, rsv); 5382 5383 /* 5384 * Errors here aren't a big deal, it just means we leave orphan items 5385 * in the tree. They will be cleaned up on the next mount. 5386 */ 5387 if (ret == 0) { 5388 trans->block_rsv = root->orphan_block_rsv; 5389 btrfs_orphan_del(trans, inode); 5390 } else { 5391 btrfs_orphan_del(NULL, inode); 5392 } 5393 5394 trans->block_rsv = &fs_info->trans_block_rsv; 5395 if (!(root == fs_info->tree_root || 5396 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)) 5397 btrfs_return_ino(root, btrfs_ino(inode)); 5398 5399 btrfs_end_transaction(trans); 5400 btrfs_btree_balance_dirty(fs_info); 5401 no_delete: 5402 btrfs_remove_delayed_node(inode); 5403 clear_inode(inode); 5404 } 5405 5406 /* 5407 * this returns the key found in the dir entry in the location pointer. 5408 * If no dir entries were found, location->objectid is 0. 5409 */ 5410 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry, 5411 struct btrfs_key *location) 5412 { 5413 const char *name = dentry->d_name.name; 5414 int namelen = dentry->d_name.len; 5415 struct btrfs_dir_item *di; 5416 struct btrfs_path *path; 5417 struct btrfs_root *root = BTRFS_I(dir)->root; 5418 int ret = 0; 5419 5420 path = btrfs_alloc_path(); 5421 if (!path) 5422 return -ENOMEM; 5423 5424 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), name, 5425 namelen, 0); 5426 if (IS_ERR(di)) 5427 ret = PTR_ERR(di); 5428 5429 if (IS_ERR_OR_NULL(di)) 5430 goto out_err; 5431 5432 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5433 out: 5434 btrfs_free_path(path); 5435 return ret; 5436 out_err: 5437 location->objectid = 0; 5438 goto out; 5439 } 5440 5441 /* 5442 * when we hit a tree root in a directory, the btrfs part of the inode 5443 * needs to be changed to reflect the root directory of the tree root. This 5444 * is kind of like crossing a mount point. 5445 */ 5446 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5447 struct inode *dir, 5448 struct dentry *dentry, 5449 struct btrfs_key *location, 5450 struct btrfs_root **sub_root) 5451 { 5452 struct btrfs_path *path; 5453 struct btrfs_root *new_root; 5454 struct btrfs_root_ref *ref; 5455 struct extent_buffer *leaf; 5456 struct btrfs_key key; 5457 int ret; 5458 int err = 0; 5459 5460 path = btrfs_alloc_path(); 5461 if (!path) { 5462 err = -ENOMEM; 5463 goto out; 5464 } 5465 5466 err = -ENOENT; 5467 key.objectid = BTRFS_I(dir)->root->root_key.objectid; 5468 key.type = BTRFS_ROOT_REF_KEY; 5469 key.offset = location->objectid; 5470 5471 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5472 if (ret) { 5473 if (ret < 0) 5474 err = ret; 5475 goto out; 5476 } 5477 5478 leaf = path->nodes[0]; 5479 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5480 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) || 5481 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len) 5482 goto out; 5483 5484 ret = memcmp_extent_buffer(leaf, dentry->d_name.name, 5485 (unsigned long)(ref + 1), 5486 dentry->d_name.len); 5487 if (ret) 5488 goto out; 5489 5490 btrfs_release_path(path); 5491 5492 new_root = btrfs_read_fs_root_no_name(fs_info, location); 5493 if (IS_ERR(new_root)) { 5494 err = PTR_ERR(new_root); 5495 goto out; 5496 } 5497 5498 *sub_root = new_root; 5499 location->objectid = btrfs_root_dirid(&new_root->root_item); 5500 location->type = BTRFS_INODE_ITEM_KEY; 5501 location->offset = 0; 5502 err = 0; 5503 out: 5504 btrfs_free_path(path); 5505 return err; 5506 } 5507 5508 static void inode_tree_add(struct inode *inode) 5509 { 5510 struct btrfs_root *root = BTRFS_I(inode)->root; 5511 struct btrfs_inode *entry; 5512 struct rb_node **p; 5513 struct rb_node *parent; 5514 struct rb_node *new = &BTRFS_I(inode)->rb_node; 5515 u64 ino = btrfs_ino(inode); 5516 5517 if (inode_unhashed(inode)) 5518 return; 5519 parent = NULL; 5520 spin_lock(&root->inode_lock); 5521 p = &root->inode_tree.rb_node; 5522 while (*p) { 5523 parent = *p; 5524 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5525 5526 if (ino < btrfs_ino(&entry->vfs_inode)) 5527 p = &parent->rb_left; 5528 else if (ino > btrfs_ino(&entry->vfs_inode)) 5529 p = &parent->rb_right; 5530 else { 5531 WARN_ON(!(entry->vfs_inode.i_state & 5532 (I_WILL_FREE | I_FREEING))); 5533 rb_replace_node(parent, new, &root->inode_tree); 5534 RB_CLEAR_NODE(parent); 5535 spin_unlock(&root->inode_lock); 5536 return; 5537 } 5538 } 5539 rb_link_node(new, parent, p); 5540 rb_insert_color(new, &root->inode_tree); 5541 spin_unlock(&root->inode_lock); 5542 } 5543 5544 static void inode_tree_del(struct inode *inode) 5545 { 5546 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5547 struct btrfs_root *root = BTRFS_I(inode)->root; 5548 int empty = 0; 5549 5550 spin_lock(&root->inode_lock); 5551 if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) { 5552 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree); 5553 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node); 5554 empty = RB_EMPTY_ROOT(&root->inode_tree); 5555 } 5556 spin_unlock(&root->inode_lock); 5557 5558 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5559 synchronize_srcu(&fs_info->subvol_srcu); 5560 spin_lock(&root->inode_lock); 5561 empty = RB_EMPTY_ROOT(&root->inode_tree); 5562 spin_unlock(&root->inode_lock); 5563 if (empty) 5564 btrfs_add_dead_root(root); 5565 } 5566 } 5567 5568 void btrfs_invalidate_inodes(struct btrfs_root *root) 5569 { 5570 struct btrfs_fs_info *fs_info = root->fs_info; 5571 struct rb_node *node; 5572 struct rb_node *prev; 5573 struct btrfs_inode *entry; 5574 struct inode *inode; 5575 u64 objectid = 0; 5576 5577 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 5578 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 5579 5580 spin_lock(&root->inode_lock); 5581 again: 5582 node = root->inode_tree.rb_node; 5583 prev = NULL; 5584 while (node) { 5585 prev = node; 5586 entry = rb_entry(node, struct btrfs_inode, rb_node); 5587 5588 if (objectid < btrfs_ino(&entry->vfs_inode)) 5589 node = node->rb_left; 5590 else if (objectid > btrfs_ino(&entry->vfs_inode)) 5591 node = node->rb_right; 5592 else 5593 break; 5594 } 5595 if (!node) { 5596 while (prev) { 5597 entry = rb_entry(prev, struct btrfs_inode, rb_node); 5598 if (objectid <= btrfs_ino(&entry->vfs_inode)) { 5599 node = prev; 5600 break; 5601 } 5602 prev = rb_next(prev); 5603 } 5604 } 5605 while (node) { 5606 entry = rb_entry(node, struct btrfs_inode, rb_node); 5607 objectid = btrfs_ino(&entry->vfs_inode) + 1; 5608 inode = igrab(&entry->vfs_inode); 5609 if (inode) { 5610 spin_unlock(&root->inode_lock); 5611 if (atomic_read(&inode->i_count) > 1) 5612 d_prune_aliases(inode); 5613 /* 5614 * btrfs_drop_inode will have it removed from 5615 * the inode cache when its usage count 5616 * hits zero. 5617 */ 5618 iput(inode); 5619 cond_resched(); 5620 spin_lock(&root->inode_lock); 5621 goto again; 5622 } 5623 5624 if (cond_resched_lock(&root->inode_lock)) 5625 goto again; 5626 5627 node = rb_next(node); 5628 } 5629 spin_unlock(&root->inode_lock); 5630 } 5631 5632 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5633 { 5634 struct btrfs_iget_args *args = p; 5635 inode->i_ino = args->location->objectid; 5636 memcpy(&BTRFS_I(inode)->location, args->location, 5637 sizeof(*args->location)); 5638 BTRFS_I(inode)->root = args->root; 5639 return 0; 5640 } 5641 5642 static int btrfs_find_actor(struct inode *inode, void *opaque) 5643 { 5644 struct btrfs_iget_args *args = opaque; 5645 return args->location->objectid == BTRFS_I(inode)->location.objectid && 5646 args->root == BTRFS_I(inode)->root; 5647 } 5648 5649 static struct inode *btrfs_iget_locked(struct super_block *s, 5650 struct btrfs_key *location, 5651 struct btrfs_root *root) 5652 { 5653 struct inode *inode; 5654 struct btrfs_iget_args args; 5655 unsigned long hashval = btrfs_inode_hash(location->objectid, root); 5656 5657 args.location = location; 5658 args.root = root; 5659 5660 inode = iget5_locked(s, hashval, btrfs_find_actor, 5661 btrfs_init_locked_inode, 5662 (void *)&args); 5663 return inode; 5664 } 5665 5666 /* Get an inode object given its location and corresponding root. 5667 * Returns in *is_new if the inode was read from disk 5668 */ 5669 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location, 5670 struct btrfs_root *root, int *new) 5671 { 5672 struct inode *inode; 5673 5674 inode = btrfs_iget_locked(s, location, root); 5675 if (!inode) 5676 return ERR_PTR(-ENOMEM); 5677 5678 if (inode->i_state & I_NEW) { 5679 int ret; 5680 5681 ret = btrfs_read_locked_inode(inode); 5682 if (!is_bad_inode(inode)) { 5683 inode_tree_add(inode); 5684 unlock_new_inode(inode); 5685 if (new) 5686 *new = 1; 5687 } else { 5688 unlock_new_inode(inode); 5689 iput(inode); 5690 ASSERT(ret < 0); 5691 inode = ERR_PTR(ret < 0 ? ret : -ESTALE); 5692 } 5693 } 5694 5695 return inode; 5696 } 5697 5698 static struct inode *new_simple_dir(struct super_block *s, 5699 struct btrfs_key *key, 5700 struct btrfs_root *root) 5701 { 5702 struct inode *inode = new_inode(s); 5703 5704 if (!inode) 5705 return ERR_PTR(-ENOMEM); 5706 5707 BTRFS_I(inode)->root = root; 5708 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5709 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5710 5711 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5712 inode->i_op = &btrfs_dir_ro_inode_operations; 5713 inode->i_fop = &simple_dir_operations; 5714 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5715 inode->i_mtime = current_time(inode); 5716 inode->i_atime = inode->i_mtime; 5717 inode->i_ctime = inode->i_mtime; 5718 BTRFS_I(inode)->i_otime = inode->i_mtime; 5719 5720 return inode; 5721 } 5722 5723 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5724 { 5725 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 5726 struct inode *inode; 5727 struct btrfs_root *root = BTRFS_I(dir)->root; 5728 struct btrfs_root *sub_root = root; 5729 struct btrfs_key location; 5730 int index; 5731 int ret = 0; 5732 5733 if (dentry->d_name.len > BTRFS_NAME_LEN) 5734 return ERR_PTR(-ENAMETOOLONG); 5735 5736 ret = btrfs_inode_by_name(dir, dentry, &location); 5737 if (ret < 0) 5738 return ERR_PTR(ret); 5739 5740 if (location.objectid == 0) 5741 return ERR_PTR(-ENOENT); 5742 5743 if (location.type == BTRFS_INODE_ITEM_KEY) { 5744 inode = btrfs_iget(dir->i_sb, &location, root, NULL); 5745 return inode; 5746 } 5747 5748 BUG_ON(location.type != BTRFS_ROOT_ITEM_KEY); 5749 5750 index = srcu_read_lock(&fs_info->subvol_srcu); 5751 ret = fixup_tree_root_location(fs_info, dir, dentry, 5752 &location, &sub_root); 5753 if (ret < 0) { 5754 if (ret != -ENOENT) 5755 inode = ERR_PTR(ret); 5756 else 5757 inode = new_simple_dir(dir->i_sb, &location, sub_root); 5758 } else { 5759 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL); 5760 } 5761 srcu_read_unlock(&fs_info->subvol_srcu, index); 5762 5763 if (!IS_ERR(inode) && root != sub_root) { 5764 down_read(&fs_info->cleanup_work_sem); 5765 if (!(inode->i_sb->s_flags & MS_RDONLY)) 5766 ret = btrfs_orphan_cleanup(sub_root); 5767 up_read(&fs_info->cleanup_work_sem); 5768 if (ret) { 5769 iput(inode); 5770 inode = ERR_PTR(ret); 5771 } 5772 } 5773 5774 return inode; 5775 } 5776 5777 static int btrfs_dentry_delete(const struct dentry *dentry) 5778 { 5779 struct btrfs_root *root; 5780 struct inode *inode = d_inode(dentry); 5781 5782 if (!inode && !IS_ROOT(dentry)) 5783 inode = d_inode(dentry->d_parent); 5784 5785 if (inode) { 5786 root = BTRFS_I(inode)->root; 5787 if (btrfs_root_refs(&root->root_item) == 0) 5788 return 1; 5789 5790 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5791 return 1; 5792 } 5793 return 0; 5794 } 5795 5796 static void btrfs_dentry_release(struct dentry *dentry) 5797 { 5798 kfree(dentry->d_fsdata); 5799 } 5800 5801 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5802 unsigned int flags) 5803 { 5804 struct inode *inode; 5805 5806 inode = btrfs_lookup_dentry(dir, dentry); 5807 if (IS_ERR(inode)) { 5808 if (PTR_ERR(inode) == -ENOENT) 5809 inode = NULL; 5810 else 5811 return ERR_CAST(inode); 5812 } 5813 5814 return d_splice_alias(inode, dentry); 5815 } 5816 5817 unsigned char btrfs_filetype_table[] = { 5818 DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK 5819 }; 5820 5821 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5822 { 5823 struct inode *inode = file_inode(file); 5824 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5825 struct btrfs_root *root = BTRFS_I(inode)->root; 5826 struct btrfs_item *item; 5827 struct btrfs_dir_item *di; 5828 struct btrfs_key key; 5829 struct btrfs_key found_key; 5830 struct btrfs_path *path; 5831 struct list_head ins_list; 5832 struct list_head del_list; 5833 int ret; 5834 struct extent_buffer *leaf; 5835 int slot; 5836 unsigned char d_type; 5837 int over = 0; 5838 char tmp_name[32]; 5839 char *name_ptr; 5840 int name_len; 5841 bool put = false; 5842 struct btrfs_key location; 5843 5844 if (!dir_emit_dots(file, ctx)) 5845 return 0; 5846 5847 path = btrfs_alloc_path(); 5848 if (!path) 5849 return -ENOMEM; 5850 5851 path->reada = READA_FORWARD; 5852 5853 INIT_LIST_HEAD(&ins_list); 5854 INIT_LIST_HEAD(&del_list); 5855 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list); 5856 5857 key.type = BTRFS_DIR_INDEX_KEY; 5858 key.offset = ctx->pos; 5859 key.objectid = btrfs_ino(inode); 5860 5861 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5862 if (ret < 0) 5863 goto err; 5864 5865 while (1) { 5866 leaf = path->nodes[0]; 5867 slot = path->slots[0]; 5868 if (slot >= btrfs_header_nritems(leaf)) { 5869 ret = btrfs_next_leaf(root, path); 5870 if (ret < 0) 5871 goto err; 5872 else if (ret > 0) 5873 break; 5874 continue; 5875 } 5876 5877 item = btrfs_item_nr(slot); 5878 btrfs_item_key_to_cpu(leaf, &found_key, slot); 5879 5880 if (found_key.objectid != key.objectid) 5881 break; 5882 if (found_key.type != BTRFS_DIR_INDEX_KEY) 5883 break; 5884 if (found_key.offset < ctx->pos) 5885 goto next; 5886 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 5887 goto next; 5888 5889 ctx->pos = found_key.offset; 5890 5891 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); 5892 if (verify_dir_item(fs_info, leaf, di)) 5893 goto next; 5894 5895 name_len = btrfs_dir_name_len(leaf, di); 5896 if (name_len <= sizeof(tmp_name)) { 5897 name_ptr = tmp_name; 5898 } else { 5899 name_ptr = kmalloc(name_len, GFP_KERNEL); 5900 if (!name_ptr) { 5901 ret = -ENOMEM; 5902 goto err; 5903 } 5904 } 5905 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1), 5906 name_len); 5907 5908 d_type = btrfs_filetype_table[btrfs_dir_type(leaf, di)]; 5909 btrfs_dir_item_key_to_cpu(leaf, di, &location); 5910 5911 over = !dir_emit(ctx, name_ptr, name_len, location.objectid, 5912 d_type); 5913 5914 if (name_ptr != tmp_name) 5915 kfree(name_ptr); 5916 5917 if (over) 5918 goto nopos; 5919 ctx->pos++; 5920 next: 5921 path->slots[0]++; 5922 } 5923 5924 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 5925 if (ret) 5926 goto nopos; 5927 5928 /* 5929 * Stop new entries from being returned after we return the last 5930 * entry. 5931 * 5932 * New directory entries are assigned a strictly increasing 5933 * offset. This means that new entries created during readdir 5934 * are *guaranteed* to be seen in the future by that readdir. 5935 * This has broken buggy programs which operate on names as 5936 * they're returned by readdir. Until we re-use freed offsets 5937 * we have this hack to stop new entries from being returned 5938 * under the assumption that they'll never reach this huge 5939 * offset. 5940 * 5941 * This is being careful not to overflow 32bit loff_t unless the 5942 * last entry requires it because doing so has broken 32bit apps 5943 * in the past. 5944 */ 5945 if (ctx->pos >= INT_MAX) 5946 ctx->pos = LLONG_MAX; 5947 else 5948 ctx->pos = INT_MAX; 5949 nopos: 5950 ret = 0; 5951 err: 5952 if (put) 5953 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list); 5954 btrfs_free_path(path); 5955 return ret; 5956 } 5957 5958 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc) 5959 { 5960 struct btrfs_root *root = BTRFS_I(inode)->root; 5961 struct btrfs_trans_handle *trans; 5962 int ret = 0; 5963 bool nolock = false; 5964 5965 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 5966 return 0; 5967 5968 if (btrfs_fs_closing(root->fs_info) && btrfs_is_free_space_inode(inode)) 5969 nolock = true; 5970 5971 if (wbc->sync_mode == WB_SYNC_ALL) { 5972 if (nolock) 5973 trans = btrfs_join_transaction_nolock(root); 5974 else 5975 trans = btrfs_join_transaction(root); 5976 if (IS_ERR(trans)) 5977 return PTR_ERR(trans); 5978 ret = btrfs_commit_transaction(trans); 5979 } 5980 return ret; 5981 } 5982 5983 /* 5984 * This is somewhat expensive, updating the tree every time the 5985 * inode changes. But, it is most likely to find the inode in cache. 5986 * FIXME, needs more benchmarking...there are no reasons other than performance 5987 * to keep or drop this code. 5988 */ 5989 static int btrfs_dirty_inode(struct inode *inode) 5990 { 5991 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5992 struct btrfs_root *root = BTRFS_I(inode)->root; 5993 struct btrfs_trans_handle *trans; 5994 int ret; 5995 5996 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 5997 return 0; 5998 5999 trans = btrfs_join_transaction(root); 6000 if (IS_ERR(trans)) 6001 return PTR_ERR(trans); 6002 6003 ret = btrfs_update_inode(trans, root, inode); 6004 if (ret && ret == -ENOSPC) { 6005 /* whoops, lets try again with the full transaction */ 6006 btrfs_end_transaction(trans); 6007 trans = btrfs_start_transaction(root, 1); 6008 if (IS_ERR(trans)) 6009 return PTR_ERR(trans); 6010 6011 ret = btrfs_update_inode(trans, root, inode); 6012 } 6013 btrfs_end_transaction(trans); 6014 if (BTRFS_I(inode)->delayed_node) 6015 btrfs_balance_delayed_items(fs_info); 6016 6017 return ret; 6018 } 6019 6020 /* 6021 * This is a copy of file_update_time. We need this so we can return error on 6022 * ENOSPC for updating the inode in the case of file write and mmap writes. 6023 */ 6024 static int btrfs_update_time(struct inode *inode, struct timespec *now, 6025 int flags) 6026 { 6027 struct btrfs_root *root = BTRFS_I(inode)->root; 6028 6029 if (btrfs_root_readonly(root)) 6030 return -EROFS; 6031 6032 if (flags & S_VERSION) 6033 inode_inc_iversion(inode); 6034 if (flags & S_CTIME) 6035 inode->i_ctime = *now; 6036 if (flags & S_MTIME) 6037 inode->i_mtime = *now; 6038 if (flags & S_ATIME) 6039 inode->i_atime = *now; 6040 return btrfs_dirty_inode(inode); 6041 } 6042 6043 /* 6044 * find the highest existing sequence number in a directory 6045 * and then set the in-memory index_cnt variable to reflect 6046 * free sequence numbers 6047 */ 6048 static int btrfs_set_inode_index_count(struct inode *inode) 6049 { 6050 struct btrfs_root *root = BTRFS_I(inode)->root; 6051 struct btrfs_key key, found_key; 6052 struct btrfs_path *path; 6053 struct extent_buffer *leaf; 6054 int ret; 6055 6056 key.objectid = btrfs_ino(inode); 6057 key.type = BTRFS_DIR_INDEX_KEY; 6058 key.offset = (u64)-1; 6059 6060 path = btrfs_alloc_path(); 6061 if (!path) 6062 return -ENOMEM; 6063 6064 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 6065 if (ret < 0) 6066 goto out; 6067 /* FIXME: we should be able to handle this */ 6068 if (ret == 0) 6069 goto out; 6070 ret = 0; 6071 6072 /* 6073 * MAGIC NUMBER EXPLANATION: 6074 * since we search a directory based on f_pos we have to start at 2 6075 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody 6076 * else has to start at 2 6077 */ 6078 if (path->slots[0] == 0) { 6079 BTRFS_I(inode)->index_cnt = 2; 6080 goto out; 6081 } 6082 6083 path->slots[0]--; 6084 6085 leaf = path->nodes[0]; 6086 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6087 6088 if (found_key.objectid != btrfs_ino(inode) || 6089 found_key.type != BTRFS_DIR_INDEX_KEY) { 6090 BTRFS_I(inode)->index_cnt = 2; 6091 goto out; 6092 } 6093 6094 BTRFS_I(inode)->index_cnt = found_key.offset + 1; 6095 out: 6096 btrfs_free_path(path); 6097 return ret; 6098 } 6099 6100 /* 6101 * helper to find a free sequence number in a given directory. This current 6102 * code is very simple, later versions will do smarter things in the btree 6103 */ 6104 int btrfs_set_inode_index(struct inode *dir, u64 *index) 6105 { 6106 int ret = 0; 6107 6108 if (BTRFS_I(dir)->index_cnt == (u64)-1) { 6109 ret = btrfs_inode_delayed_dir_index_count(dir); 6110 if (ret) { 6111 ret = btrfs_set_inode_index_count(dir); 6112 if (ret) 6113 return ret; 6114 } 6115 } 6116 6117 *index = BTRFS_I(dir)->index_cnt; 6118 BTRFS_I(dir)->index_cnt++; 6119 6120 return ret; 6121 } 6122 6123 static int btrfs_insert_inode_locked(struct inode *inode) 6124 { 6125 struct btrfs_iget_args args; 6126 args.location = &BTRFS_I(inode)->location; 6127 args.root = BTRFS_I(inode)->root; 6128 6129 return insert_inode_locked4(inode, 6130 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 6131 btrfs_find_actor, &args); 6132 } 6133 6134 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, 6135 struct btrfs_root *root, 6136 struct inode *dir, 6137 const char *name, int name_len, 6138 u64 ref_objectid, u64 objectid, 6139 umode_t mode, u64 *index) 6140 { 6141 struct btrfs_fs_info *fs_info = root->fs_info; 6142 struct inode *inode; 6143 struct btrfs_inode_item *inode_item; 6144 struct btrfs_key *location; 6145 struct btrfs_path *path; 6146 struct btrfs_inode_ref *ref; 6147 struct btrfs_key key[2]; 6148 u32 sizes[2]; 6149 int nitems = name ? 2 : 1; 6150 unsigned long ptr; 6151 int ret; 6152 6153 path = btrfs_alloc_path(); 6154 if (!path) 6155 return ERR_PTR(-ENOMEM); 6156 6157 inode = new_inode(fs_info->sb); 6158 if (!inode) { 6159 btrfs_free_path(path); 6160 return ERR_PTR(-ENOMEM); 6161 } 6162 6163 /* 6164 * O_TMPFILE, set link count to 0, so that after this point, 6165 * we fill in an inode item with the correct link count. 6166 */ 6167 if (!name) 6168 set_nlink(inode, 0); 6169 6170 /* 6171 * we have to initialize this early, so we can reclaim the inode 6172 * number if we fail afterwards in this function. 6173 */ 6174 inode->i_ino = objectid; 6175 6176 if (dir && name) { 6177 trace_btrfs_inode_request(dir); 6178 6179 ret = btrfs_set_inode_index(dir, index); 6180 if (ret) { 6181 btrfs_free_path(path); 6182 iput(inode); 6183 return ERR_PTR(ret); 6184 } 6185 } else if (dir) { 6186 *index = 0; 6187 } 6188 /* 6189 * index_cnt is ignored for everything but a dir, 6190 * btrfs_get_inode_index_count has an explanation for the magic 6191 * number 6192 */ 6193 BTRFS_I(inode)->index_cnt = 2; 6194 BTRFS_I(inode)->dir_index = *index; 6195 BTRFS_I(inode)->root = root; 6196 BTRFS_I(inode)->generation = trans->transid; 6197 inode->i_generation = BTRFS_I(inode)->generation; 6198 6199 /* 6200 * We could have gotten an inode number from somebody who was fsynced 6201 * and then removed in this same transaction, so let's just set full 6202 * sync since it will be a full sync anyway and this will blow away the 6203 * old info in the log. 6204 */ 6205 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 6206 6207 key[0].objectid = objectid; 6208 key[0].type = BTRFS_INODE_ITEM_KEY; 6209 key[0].offset = 0; 6210 6211 sizes[0] = sizeof(struct btrfs_inode_item); 6212 6213 if (name) { 6214 /* 6215 * Start new inodes with an inode_ref. This is slightly more 6216 * efficient for small numbers of hard links since they will 6217 * be packed into one item. Extended refs will kick in if we 6218 * add more hard links than can fit in the ref item. 6219 */ 6220 key[1].objectid = objectid; 6221 key[1].type = BTRFS_INODE_REF_KEY; 6222 key[1].offset = ref_objectid; 6223 6224 sizes[1] = name_len + sizeof(*ref); 6225 } 6226 6227 location = &BTRFS_I(inode)->location; 6228 location->objectid = objectid; 6229 location->offset = 0; 6230 location->type = BTRFS_INODE_ITEM_KEY; 6231 6232 ret = btrfs_insert_inode_locked(inode); 6233 if (ret < 0) 6234 goto fail; 6235 6236 path->leave_spinning = 1; 6237 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems); 6238 if (ret != 0) 6239 goto fail_unlock; 6240 6241 inode_init_owner(inode, dir, mode); 6242 inode_set_bytes(inode, 0); 6243 6244 inode->i_mtime = current_time(inode); 6245 inode->i_atime = inode->i_mtime; 6246 inode->i_ctime = inode->i_mtime; 6247 BTRFS_I(inode)->i_otime = inode->i_mtime; 6248 6249 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6250 struct btrfs_inode_item); 6251 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6252 sizeof(*inode_item)); 6253 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6254 6255 if (name) { 6256 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6257 struct btrfs_inode_ref); 6258 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len); 6259 btrfs_set_inode_ref_index(path->nodes[0], ref, *index); 6260 ptr = (unsigned long)(ref + 1); 6261 write_extent_buffer(path->nodes[0], name, ptr, name_len); 6262 } 6263 6264 btrfs_mark_buffer_dirty(path->nodes[0]); 6265 btrfs_free_path(path); 6266 6267 btrfs_inherit_iflags(inode, dir); 6268 6269 if (S_ISREG(mode)) { 6270 if (btrfs_test_opt(fs_info, NODATASUM)) 6271 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6272 if (btrfs_test_opt(fs_info, NODATACOW)) 6273 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6274 BTRFS_INODE_NODATASUM; 6275 } 6276 6277 inode_tree_add(inode); 6278 6279 trace_btrfs_inode_new(inode); 6280 btrfs_set_inode_last_trans(trans, inode); 6281 6282 btrfs_update_root_times(trans, root); 6283 6284 ret = btrfs_inode_inherit_props(trans, inode, dir); 6285 if (ret) 6286 btrfs_err(fs_info, 6287 "error inheriting props for ino %llu (root %llu): %d", 6288 btrfs_ino(inode), root->root_key.objectid, ret); 6289 6290 return inode; 6291 6292 fail_unlock: 6293 unlock_new_inode(inode); 6294 fail: 6295 if (dir && name) 6296 BTRFS_I(dir)->index_cnt--; 6297 btrfs_free_path(path); 6298 iput(inode); 6299 return ERR_PTR(ret); 6300 } 6301 6302 static inline u8 btrfs_inode_type(struct inode *inode) 6303 { 6304 return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT]; 6305 } 6306 6307 /* 6308 * utility function to add 'inode' into 'parent_inode' with 6309 * a give name and a given sequence number. 6310 * if 'add_backref' is true, also insert a backref from the 6311 * inode to the parent directory. 6312 */ 6313 int btrfs_add_link(struct btrfs_trans_handle *trans, 6314 struct inode *parent_inode, struct inode *inode, 6315 const char *name, int name_len, int add_backref, u64 index) 6316 { 6317 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6318 int ret = 0; 6319 struct btrfs_key key; 6320 struct btrfs_root *root = BTRFS_I(parent_inode)->root; 6321 u64 ino = btrfs_ino(inode); 6322 u64 parent_ino = btrfs_ino(parent_inode); 6323 6324 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6325 memcpy(&key, &BTRFS_I(inode)->root->root_key, sizeof(key)); 6326 } else { 6327 key.objectid = ino; 6328 key.type = BTRFS_INODE_ITEM_KEY; 6329 key.offset = 0; 6330 } 6331 6332 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6333 ret = btrfs_add_root_ref(trans, fs_info, key.objectid, 6334 root->root_key.objectid, parent_ino, 6335 index, name, name_len); 6336 } else if (add_backref) { 6337 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino, 6338 parent_ino, index); 6339 } 6340 6341 /* Nothing to clean up yet */ 6342 if (ret) 6343 return ret; 6344 6345 ret = btrfs_insert_dir_item(trans, root, name, name_len, 6346 parent_inode, &key, 6347 btrfs_inode_type(inode), index); 6348 if (ret == -EEXIST || ret == -EOVERFLOW) 6349 goto fail_dir_item; 6350 else if (ret) { 6351 btrfs_abort_transaction(trans, ret); 6352 return ret; 6353 } 6354 6355 btrfs_i_size_write(parent_inode, parent_inode->i_size + 6356 name_len * 2); 6357 inode_inc_iversion(parent_inode); 6358 parent_inode->i_mtime = parent_inode->i_ctime = 6359 current_time(parent_inode); 6360 ret = btrfs_update_inode(trans, root, parent_inode); 6361 if (ret) 6362 btrfs_abort_transaction(trans, ret); 6363 return ret; 6364 6365 fail_dir_item: 6366 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6367 u64 local_index; 6368 int err; 6369 err = btrfs_del_root_ref(trans, fs_info, key.objectid, 6370 root->root_key.objectid, parent_ino, 6371 &local_index, name, name_len); 6372 6373 } else if (add_backref) { 6374 u64 local_index; 6375 int err; 6376 6377 err = btrfs_del_inode_ref(trans, root, name, name_len, 6378 ino, parent_ino, &local_index); 6379 } 6380 return ret; 6381 } 6382 6383 static int btrfs_add_nondir(struct btrfs_trans_handle *trans, 6384 struct inode *dir, struct dentry *dentry, 6385 struct inode *inode, int backref, u64 index) 6386 { 6387 int err = btrfs_add_link(trans, dir, inode, 6388 dentry->d_name.name, dentry->d_name.len, 6389 backref, index); 6390 if (err > 0) 6391 err = -EEXIST; 6392 return err; 6393 } 6394 6395 static int btrfs_mknod(struct inode *dir, struct dentry *dentry, 6396 umode_t mode, dev_t rdev) 6397 { 6398 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6399 struct btrfs_trans_handle *trans; 6400 struct btrfs_root *root = BTRFS_I(dir)->root; 6401 struct inode *inode = NULL; 6402 int err; 6403 int drop_inode = 0; 6404 u64 objectid; 6405 u64 index = 0; 6406 6407 /* 6408 * 2 for inode item and ref 6409 * 2 for dir items 6410 * 1 for xattr if selinux is on 6411 */ 6412 trans = btrfs_start_transaction(root, 5); 6413 if (IS_ERR(trans)) 6414 return PTR_ERR(trans); 6415 6416 err = btrfs_find_free_ino(root, &objectid); 6417 if (err) 6418 goto out_unlock; 6419 6420 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6421 dentry->d_name.len, btrfs_ino(dir), objectid, 6422 mode, &index); 6423 if (IS_ERR(inode)) { 6424 err = PTR_ERR(inode); 6425 goto out_unlock; 6426 } 6427 6428 /* 6429 * If the active LSM wants to access the inode during 6430 * d_instantiate it needs these. Smack checks to see 6431 * if the filesystem supports xattrs by looking at the 6432 * ops vector. 6433 */ 6434 inode->i_op = &btrfs_special_inode_operations; 6435 init_special_inode(inode, inode->i_mode, rdev); 6436 6437 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6438 if (err) 6439 goto out_unlock_inode; 6440 6441 err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index); 6442 if (err) { 6443 goto out_unlock_inode; 6444 } else { 6445 btrfs_update_inode(trans, root, inode); 6446 unlock_new_inode(inode); 6447 d_instantiate(dentry, inode); 6448 } 6449 6450 out_unlock: 6451 btrfs_end_transaction(trans); 6452 btrfs_balance_delayed_items(fs_info); 6453 btrfs_btree_balance_dirty(fs_info); 6454 if (drop_inode) { 6455 inode_dec_link_count(inode); 6456 iput(inode); 6457 } 6458 return err; 6459 6460 out_unlock_inode: 6461 drop_inode = 1; 6462 unlock_new_inode(inode); 6463 goto out_unlock; 6464 6465 } 6466 6467 static int btrfs_create(struct inode *dir, struct dentry *dentry, 6468 umode_t mode, bool excl) 6469 { 6470 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6471 struct btrfs_trans_handle *trans; 6472 struct btrfs_root *root = BTRFS_I(dir)->root; 6473 struct inode *inode = NULL; 6474 int drop_inode_on_err = 0; 6475 int err; 6476 u64 objectid; 6477 u64 index = 0; 6478 6479 /* 6480 * 2 for inode item and ref 6481 * 2 for dir items 6482 * 1 for xattr if selinux is on 6483 */ 6484 trans = btrfs_start_transaction(root, 5); 6485 if (IS_ERR(trans)) 6486 return PTR_ERR(trans); 6487 6488 err = btrfs_find_free_ino(root, &objectid); 6489 if (err) 6490 goto out_unlock; 6491 6492 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6493 dentry->d_name.len, btrfs_ino(dir), objectid, 6494 mode, &index); 6495 if (IS_ERR(inode)) { 6496 err = PTR_ERR(inode); 6497 goto out_unlock; 6498 } 6499 drop_inode_on_err = 1; 6500 /* 6501 * If the active LSM wants to access the inode during 6502 * d_instantiate it needs these. Smack checks to see 6503 * if the filesystem supports xattrs by looking at the 6504 * ops vector. 6505 */ 6506 inode->i_fop = &btrfs_file_operations; 6507 inode->i_op = &btrfs_file_inode_operations; 6508 inode->i_mapping->a_ops = &btrfs_aops; 6509 6510 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6511 if (err) 6512 goto out_unlock_inode; 6513 6514 err = btrfs_update_inode(trans, root, inode); 6515 if (err) 6516 goto out_unlock_inode; 6517 6518 err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index); 6519 if (err) 6520 goto out_unlock_inode; 6521 6522 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 6523 unlock_new_inode(inode); 6524 d_instantiate(dentry, inode); 6525 6526 out_unlock: 6527 btrfs_end_transaction(trans); 6528 if (err && drop_inode_on_err) { 6529 inode_dec_link_count(inode); 6530 iput(inode); 6531 } 6532 btrfs_balance_delayed_items(fs_info); 6533 btrfs_btree_balance_dirty(fs_info); 6534 return err; 6535 6536 out_unlock_inode: 6537 unlock_new_inode(inode); 6538 goto out_unlock; 6539 6540 } 6541 6542 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6543 struct dentry *dentry) 6544 { 6545 struct btrfs_trans_handle *trans = NULL; 6546 struct btrfs_root *root = BTRFS_I(dir)->root; 6547 struct inode *inode = d_inode(old_dentry); 6548 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6549 u64 index; 6550 int err; 6551 int drop_inode = 0; 6552 6553 /* do not allow sys_link's with other subvols of the same device */ 6554 if (root->objectid != BTRFS_I(inode)->root->objectid) 6555 return -EXDEV; 6556 6557 if (inode->i_nlink >= BTRFS_LINK_MAX) 6558 return -EMLINK; 6559 6560 err = btrfs_set_inode_index(dir, &index); 6561 if (err) 6562 goto fail; 6563 6564 /* 6565 * 2 items for inode and inode ref 6566 * 2 items for dir items 6567 * 1 item for parent inode 6568 */ 6569 trans = btrfs_start_transaction(root, 5); 6570 if (IS_ERR(trans)) { 6571 err = PTR_ERR(trans); 6572 trans = NULL; 6573 goto fail; 6574 } 6575 6576 /* There are several dir indexes for this inode, clear the cache. */ 6577 BTRFS_I(inode)->dir_index = 0ULL; 6578 inc_nlink(inode); 6579 inode_inc_iversion(inode); 6580 inode->i_ctime = current_time(inode); 6581 ihold(inode); 6582 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6583 6584 err = btrfs_add_nondir(trans, dir, dentry, inode, 1, index); 6585 6586 if (err) { 6587 drop_inode = 1; 6588 } else { 6589 struct dentry *parent = dentry->d_parent; 6590 err = btrfs_update_inode(trans, root, inode); 6591 if (err) 6592 goto fail; 6593 if (inode->i_nlink == 1) { 6594 /* 6595 * If new hard link count is 1, it's a file created 6596 * with open(2) O_TMPFILE flag. 6597 */ 6598 err = btrfs_orphan_del(trans, inode); 6599 if (err) 6600 goto fail; 6601 } 6602 d_instantiate(dentry, inode); 6603 btrfs_log_new_name(trans, inode, NULL, parent); 6604 } 6605 6606 btrfs_balance_delayed_items(fs_info); 6607 fail: 6608 if (trans) 6609 btrfs_end_transaction(trans); 6610 if (drop_inode) { 6611 inode_dec_link_count(inode); 6612 iput(inode); 6613 } 6614 btrfs_btree_balance_dirty(fs_info); 6615 return err; 6616 } 6617 6618 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 6619 { 6620 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6621 struct inode *inode = NULL; 6622 struct btrfs_trans_handle *trans; 6623 struct btrfs_root *root = BTRFS_I(dir)->root; 6624 int err = 0; 6625 int drop_on_err = 0; 6626 u64 objectid = 0; 6627 u64 index = 0; 6628 6629 /* 6630 * 2 items for inode and ref 6631 * 2 items for dir items 6632 * 1 for xattr if selinux is on 6633 */ 6634 trans = btrfs_start_transaction(root, 5); 6635 if (IS_ERR(trans)) 6636 return PTR_ERR(trans); 6637 6638 err = btrfs_find_free_ino(root, &objectid); 6639 if (err) 6640 goto out_fail; 6641 6642 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6643 dentry->d_name.len, btrfs_ino(dir), objectid, 6644 S_IFDIR | mode, &index); 6645 if (IS_ERR(inode)) { 6646 err = PTR_ERR(inode); 6647 goto out_fail; 6648 } 6649 6650 drop_on_err = 1; 6651 /* these must be set before we unlock the inode */ 6652 inode->i_op = &btrfs_dir_inode_operations; 6653 inode->i_fop = &btrfs_dir_file_operations; 6654 6655 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6656 if (err) 6657 goto out_fail_inode; 6658 6659 btrfs_i_size_write(inode, 0); 6660 err = btrfs_update_inode(trans, root, inode); 6661 if (err) 6662 goto out_fail_inode; 6663 6664 err = btrfs_add_link(trans, dir, inode, dentry->d_name.name, 6665 dentry->d_name.len, 0, index); 6666 if (err) 6667 goto out_fail_inode; 6668 6669 d_instantiate(dentry, inode); 6670 /* 6671 * mkdir is special. We're unlocking after we call d_instantiate 6672 * to avoid a race with nfsd calling d_instantiate. 6673 */ 6674 unlock_new_inode(inode); 6675 drop_on_err = 0; 6676 6677 out_fail: 6678 btrfs_end_transaction(trans); 6679 if (drop_on_err) { 6680 inode_dec_link_count(inode); 6681 iput(inode); 6682 } 6683 btrfs_balance_delayed_items(fs_info); 6684 btrfs_btree_balance_dirty(fs_info); 6685 return err; 6686 6687 out_fail_inode: 6688 unlock_new_inode(inode); 6689 goto out_fail; 6690 } 6691 6692 /* Find next extent map of a given extent map, caller needs to ensure locks */ 6693 static struct extent_map *next_extent_map(struct extent_map *em) 6694 { 6695 struct rb_node *next; 6696 6697 next = rb_next(&em->rb_node); 6698 if (!next) 6699 return NULL; 6700 return container_of(next, struct extent_map, rb_node); 6701 } 6702 6703 static struct extent_map *prev_extent_map(struct extent_map *em) 6704 { 6705 struct rb_node *prev; 6706 6707 prev = rb_prev(&em->rb_node); 6708 if (!prev) 6709 return NULL; 6710 return container_of(prev, struct extent_map, rb_node); 6711 } 6712 6713 /* helper for btfs_get_extent. Given an existing extent in the tree, 6714 * the existing extent is the nearest extent to map_start, 6715 * and an extent that you want to insert, deal with overlap and insert 6716 * the best fitted new extent into the tree. 6717 */ 6718 static int merge_extent_mapping(struct extent_map_tree *em_tree, 6719 struct extent_map *existing, 6720 struct extent_map *em, 6721 u64 map_start) 6722 { 6723 struct extent_map *prev; 6724 struct extent_map *next; 6725 u64 start; 6726 u64 end; 6727 u64 start_diff; 6728 6729 BUG_ON(map_start < em->start || map_start >= extent_map_end(em)); 6730 6731 if (existing->start > map_start) { 6732 next = existing; 6733 prev = prev_extent_map(next); 6734 } else { 6735 prev = existing; 6736 next = next_extent_map(prev); 6737 } 6738 6739 start = prev ? extent_map_end(prev) : em->start; 6740 start = max_t(u64, start, em->start); 6741 end = next ? next->start : extent_map_end(em); 6742 end = min_t(u64, end, extent_map_end(em)); 6743 start_diff = start - em->start; 6744 em->start = start; 6745 em->len = end - start; 6746 if (em->block_start < EXTENT_MAP_LAST_BYTE && 6747 !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 6748 em->block_start += start_diff; 6749 em->block_len -= start_diff; 6750 } 6751 return add_extent_mapping(em_tree, em, 0); 6752 } 6753 6754 static noinline int uncompress_inline(struct btrfs_path *path, 6755 struct page *page, 6756 size_t pg_offset, u64 extent_offset, 6757 struct btrfs_file_extent_item *item) 6758 { 6759 int ret; 6760 struct extent_buffer *leaf = path->nodes[0]; 6761 char *tmp; 6762 size_t max_size; 6763 unsigned long inline_size; 6764 unsigned long ptr; 6765 int compress_type; 6766 6767 WARN_ON(pg_offset != 0); 6768 compress_type = btrfs_file_extent_compression(leaf, item); 6769 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6770 inline_size = btrfs_file_extent_inline_item_len(leaf, 6771 btrfs_item_nr(path->slots[0])); 6772 tmp = kmalloc(inline_size, GFP_NOFS); 6773 if (!tmp) 6774 return -ENOMEM; 6775 ptr = btrfs_file_extent_inline_start(item); 6776 6777 read_extent_buffer(leaf, tmp, ptr, inline_size); 6778 6779 max_size = min_t(unsigned long, PAGE_SIZE, max_size); 6780 ret = btrfs_decompress(compress_type, tmp, page, 6781 extent_offset, inline_size, max_size); 6782 kfree(tmp); 6783 return ret; 6784 } 6785 6786 /* 6787 * a bit scary, this does extent mapping from logical file offset to the disk. 6788 * the ugly parts come from merging extents from the disk with the in-ram 6789 * representation. This gets more complex because of the data=ordered code, 6790 * where the in-ram extents might be locked pending data=ordered completion. 6791 * 6792 * This also copies inline extents directly into the page. 6793 */ 6794 6795 struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page, 6796 size_t pg_offset, u64 start, u64 len, 6797 int create) 6798 { 6799 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6800 int ret; 6801 int err = 0; 6802 u64 extent_start = 0; 6803 u64 extent_end = 0; 6804 u64 objectid = btrfs_ino(inode); 6805 u32 found_type; 6806 struct btrfs_path *path = NULL; 6807 struct btrfs_root *root = BTRFS_I(inode)->root; 6808 struct btrfs_file_extent_item *item; 6809 struct extent_buffer *leaf; 6810 struct btrfs_key found_key; 6811 struct extent_map *em = NULL; 6812 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 6813 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 6814 struct btrfs_trans_handle *trans = NULL; 6815 const bool new_inline = !page || create; 6816 6817 again: 6818 read_lock(&em_tree->lock); 6819 em = lookup_extent_mapping(em_tree, start, len); 6820 if (em) 6821 em->bdev = fs_info->fs_devices->latest_bdev; 6822 read_unlock(&em_tree->lock); 6823 6824 if (em) { 6825 if (em->start > start || em->start + em->len <= start) 6826 free_extent_map(em); 6827 else if (em->block_start == EXTENT_MAP_INLINE && page) 6828 free_extent_map(em); 6829 else 6830 goto out; 6831 } 6832 em = alloc_extent_map(); 6833 if (!em) { 6834 err = -ENOMEM; 6835 goto out; 6836 } 6837 em->bdev = fs_info->fs_devices->latest_bdev; 6838 em->start = EXTENT_MAP_HOLE; 6839 em->orig_start = EXTENT_MAP_HOLE; 6840 em->len = (u64)-1; 6841 em->block_len = (u64)-1; 6842 6843 if (!path) { 6844 path = btrfs_alloc_path(); 6845 if (!path) { 6846 err = -ENOMEM; 6847 goto out; 6848 } 6849 /* 6850 * Chances are we'll be called again, so go ahead and do 6851 * readahead 6852 */ 6853 path->reada = READA_FORWARD; 6854 } 6855 6856 ret = btrfs_lookup_file_extent(trans, root, path, 6857 objectid, start, trans != NULL); 6858 if (ret < 0) { 6859 err = ret; 6860 goto out; 6861 } 6862 6863 if (ret != 0) { 6864 if (path->slots[0] == 0) 6865 goto not_found; 6866 path->slots[0]--; 6867 } 6868 6869 leaf = path->nodes[0]; 6870 item = btrfs_item_ptr(leaf, path->slots[0], 6871 struct btrfs_file_extent_item); 6872 /* are we inside the extent that was found? */ 6873 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6874 found_type = found_key.type; 6875 if (found_key.objectid != objectid || 6876 found_type != BTRFS_EXTENT_DATA_KEY) { 6877 /* 6878 * If we backup past the first extent we want to move forward 6879 * and see if there is an extent in front of us, otherwise we'll 6880 * say there is a hole for our whole search range which can 6881 * cause problems. 6882 */ 6883 extent_end = start; 6884 goto next; 6885 } 6886 6887 found_type = btrfs_file_extent_type(leaf, item); 6888 extent_start = found_key.offset; 6889 if (found_type == BTRFS_FILE_EXTENT_REG || 6890 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6891 extent_end = extent_start + 6892 btrfs_file_extent_num_bytes(leaf, item); 6893 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 6894 size_t size; 6895 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 6896 extent_end = ALIGN(extent_start + size, 6897 fs_info->sectorsize); 6898 } 6899 next: 6900 if (start >= extent_end) { 6901 path->slots[0]++; 6902 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 6903 ret = btrfs_next_leaf(root, path); 6904 if (ret < 0) { 6905 err = ret; 6906 goto out; 6907 } 6908 if (ret > 0) 6909 goto not_found; 6910 leaf = path->nodes[0]; 6911 } 6912 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6913 if (found_key.objectid != objectid || 6914 found_key.type != BTRFS_EXTENT_DATA_KEY) 6915 goto not_found; 6916 if (start + len <= found_key.offset) 6917 goto not_found; 6918 if (start > found_key.offset) 6919 goto next; 6920 em->start = start; 6921 em->orig_start = start; 6922 em->len = found_key.offset - start; 6923 goto not_found_em; 6924 } 6925 6926 btrfs_extent_item_to_extent_map(inode, path, item, new_inline, em); 6927 6928 if (found_type == BTRFS_FILE_EXTENT_REG || 6929 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6930 goto insert; 6931 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 6932 unsigned long ptr; 6933 char *map; 6934 size_t size; 6935 size_t extent_offset; 6936 size_t copy_size; 6937 6938 if (new_inline) 6939 goto out; 6940 6941 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 6942 extent_offset = page_offset(page) + pg_offset - extent_start; 6943 copy_size = min_t(u64, PAGE_SIZE - pg_offset, 6944 size - extent_offset); 6945 em->start = extent_start + extent_offset; 6946 em->len = ALIGN(copy_size, fs_info->sectorsize); 6947 em->orig_block_len = em->len; 6948 em->orig_start = em->start; 6949 ptr = btrfs_file_extent_inline_start(item) + extent_offset; 6950 if (create == 0 && !PageUptodate(page)) { 6951 if (btrfs_file_extent_compression(leaf, item) != 6952 BTRFS_COMPRESS_NONE) { 6953 ret = uncompress_inline(path, page, pg_offset, 6954 extent_offset, item); 6955 if (ret) { 6956 err = ret; 6957 goto out; 6958 } 6959 } else { 6960 map = kmap(page); 6961 read_extent_buffer(leaf, map + pg_offset, ptr, 6962 copy_size); 6963 if (pg_offset + copy_size < PAGE_SIZE) { 6964 memset(map + pg_offset + copy_size, 0, 6965 PAGE_SIZE - pg_offset - 6966 copy_size); 6967 } 6968 kunmap(page); 6969 } 6970 flush_dcache_page(page); 6971 } else if (create && PageUptodate(page)) { 6972 BUG(); 6973 if (!trans) { 6974 kunmap(page); 6975 free_extent_map(em); 6976 em = NULL; 6977 6978 btrfs_release_path(path); 6979 trans = btrfs_join_transaction(root); 6980 6981 if (IS_ERR(trans)) 6982 return ERR_CAST(trans); 6983 goto again; 6984 } 6985 map = kmap(page); 6986 write_extent_buffer(leaf, map + pg_offset, ptr, 6987 copy_size); 6988 kunmap(page); 6989 btrfs_mark_buffer_dirty(leaf); 6990 } 6991 set_extent_uptodate(io_tree, em->start, 6992 extent_map_end(em) - 1, NULL, GFP_NOFS); 6993 goto insert; 6994 } 6995 not_found: 6996 em->start = start; 6997 em->orig_start = start; 6998 em->len = len; 6999 not_found_em: 7000 em->block_start = EXTENT_MAP_HOLE; 7001 set_bit(EXTENT_FLAG_VACANCY, &em->flags); 7002 insert: 7003 btrfs_release_path(path); 7004 if (em->start > start || extent_map_end(em) <= start) { 7005 btrfs_err(fs_info, 7006 "bad extent! em: [%llu %llu] passed [%llu %llu]", 7007 em->start, em->len, start, len); 7008 err = -EIO; 7009 goto out; 7010 } 7011 7012 err = 0; 7013 write_lock(&em_tree->lock); 7014 ret = add_extent_mapping(em_tree, em, 0); 7015 /* it is possible that someone inserted the extent into the tree 7016 * while we had the lock dropped. It is also possible that 7017 * an overlapping map exists in the tree 7018 */ 7019 if (ret == -EEXIST) { 7020 struct extent_map *existing; 7021 7022 ret = 0; 7023 7024 existing = search_extent_mapping(em_tree, start, len); 7025 /* 7026 * existing will always be non-NULL, since there must be 7027 * extent causing the -EEXIST. 7028 */ 7029 if (existing->start == em->start && 7030 extent_map_end(existing) >= extent_map_end(em) && 7031 em->block_start == existing->block_start) { 7032 /* 7033 * The existing extent map already encompasses the 7034 * entire extent map we tried to add. 7035 */ 7036 free_extent_map(em); 7037 em = existing; 7038 err = 0; 7039 7040 } else if (start >= extent_map_end(existing) || 7041 start <= existing->start) { 7042 /* 7043 * The existing extent map is the one nearest to 7044 * the [start, start + len) range which overlaps 7045 */ 7046 err = merge_extent_mapping(em_tree, existing, 7047 em, start); 7048 free_extent_map(existing); 7049 if (err) { 7050 free_extent_map(em); 7051 em = NULL; 7052 } 7053 } else { 7054 free_extent_map(em); 7055 em = existing; 7056 err = 0; 7057 } 7058 } 7059 write_unlock(&em_tree->lock); 7060 out: 7061 7062 trace_btrfs_get_extent(root, em); 7063 7064 btrfs_free_path(path); 7065 if (trans) { 7066 ret = btrfs_end_transaction(trans); 7067 if (!err) 7068 err = ret; 7069 } 7070 if (err) { 7071 free_extent_map(em); 7072 return ERR_PTR(err); 7073 } 7074 BUG_ON(!em); /* Error is always set */ 7075 return em; 7076 } 7077 7078 struct extent_map *btrfs_get_extent_fiemap(struct inode *inode, struct page *page, 7079 size_t pg_offset, u64 start, u64 len, 7080 int create) 7081 { 7082 struct extent_map *em; 7083 struct extent_map *hole_em = NULL; 7084 u64 range_start = start; 7085 u64 end; 7086 u64 found; 7087 u64 found_end; 7088 int err = 0; 7089 7090 em = btrfs_get_extent(inode, page, pg_offset, start, len, create); 7091 if (IS_ERR(em)) 7092 return em; 7093 if (em) { 7094 /* 7095 * if our em maps to 7096 * - a hole or 7097 * - a pre-alloc extent, 7098 * there might actually be delalloc bytes behind it. 7099 */ 7100 if (em->block_start != EXTENT_MAP_HOLE && 7101 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7102 return em; 7103 else 7104 hole_em = em; 7105 } 7106 7107 /* check to see if we've wrapped (len == -1 or similar) */ 7108 end = start + len; 7109 if (end < start) 7110 end = (u64)-1; 7111 else 7112 end -= 1; 7113 7114 em = NULL; 7115 7116 /* ok, we didn't find anything, lets look for delalloc */ 7117 found = count_range_bits(&BTRFS_I(inode)->io_tree, &range_start, 7118 end, len, EXTENT_DELALLOC, 1); 7119 found_end = range_start + found; 7120 if (found_end < range_start) 7121 found_end = (u64)-1; 7122 7123 /* 7124 * we didn't find anything useful, return 7125 * the original results from get_extent() 7126 */ 7127 if (range_start > end || found_end <= start) { 7128 em = hole_em; 7129 hole_em = NULL; 7130 goto out; 7131 } 7132 7133 /* adjust the range_start to make sure it doesn't 7134 * go backwards from the start they passed in 7135 */ 7136 range_start = max(start, range_start); 7137 found = found_end - range_start; 7138 7139 if (found > 0) { 7140 u64 hole_start = start; 7141 u64 hole_len = len; 7142 7143 em = alloc_extent_map(); 7144 if (!em) { 7145 err = -ENOMEM; 7146 goto out; 7147 } 7148 /* 7149 * when btrfs_get_extent can't find anything it 7150 * returns one huge hole 7151 * 7152 * make sure what it found really fits our range, and 7153 * adjust to make sure it is based on the start from 7154 * the caller 7155 */ 7156 if (hole_em) { 7157 u64 calc_end = extent_map_end(hole_em); 7158 7159 if (calc_end <= start || (hole_em->start > end)) { 7160 free_extent_map(hole_em); 7161 hole_em = NULL; 7162 } else { 7163 hole_start = max(hole_em->start, start); 7164 hole_len = calc_end - hole_start; 7165 } 7166 } 7167 em->bdev = NULL; 7168 if (hole_em && range_start > hole_start) { 7169 /* our hole starts before our delalloc, so we 7170 * have to return just the parts of the hole 7171 * that go until the delalloc starts 7172 */ 7173 em->len = min(hole_len, 7174 range_start - hole_start); 7175 em->start = hole_start; 7176 em->orig_start = hole_start; 7177 /* 7178 * don't adjust block start at all, 7179 * it is fixed at EXTENT_MAP_HOLE 7180 */ 7181 em->block_start = hole_em->block_start; 7182 em->block_len = hole_len; 7183 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) 7184 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 7185 } else { 7186 em->start = range_start; 7187 em->len = found; 7188 em->orig_start = range_start; 7189 em->block_start = EXTENT_MAP_DELALLOC; 7190 em->block_len = found; 7191 } 7192 } else if (hole_em) { 7193 return hole_em; 7194 } 7195 out: 7196 7197 free_extent_map(hole_em); 7198 if (err) { 7199 free_extent_map(em); 7200 return ERR_PTR(err); 7201 } 7202 return em; 7203 } 7204 7205 static struct extent_map *btrfs_create_dio_extent(struct inode *inode, 7206 const u64 start, 7207 const u64 len, 7208 const u64 orig_start, 7209 const u64 block_start, 7210 const u64 block_len, 7211 const u64 orig_block_len, 7212 const u64 ram_bytes, 7213 const int type) 7214 { 7215 struct extent_map *em = NULL; 7216 int ret; 7217 7218 down_read(&BTRFS_I(inode)->dio_sem); 7219 if (type != BTRFS_ORDERED_NOCOW) { 7220 em = create_pinned_em(inode, start, len, orig_start, 7221 block_start, block_len, orig_block_len, 7222 ram_bytes, type); 7223 if (IS_ERR(em)) 7224 goto out; 7225 } 7226 ret = btrfs_add_ordered_extent_dio(inode, start, block_start, 7227 len, block_len, type); 7228 if (ret) { 7229 if (em) { 7230 free_extent_map(em); 7231 btrfs_drop_extent_cache(inode, start, 7232 start + len - 1, 0); 7233 } 7234 em = ERR_PTR(ret); 7235 } 7236 out: 7237 up_read(&BTRFS_I(inode)->dio_sem); 7238 7239 return em; 7240 } 7241 7242 static struct extent_map *btrfs_new_extent_direct(struct inode *inode, 7243 u64 start, u64 len) 7244 { 7245 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7246 struct btrfs_root *root = BTRFS_I(inode)->root; 7247 struct extent_map *em; 7248 struct btrfs_key ins; 7249 u64 alloc_hint; 7250 int ret; 7251 7252 alloc_hint = get_extent_allocation_hint(inode, start, len); 7253 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, 7254 0, alloc_hint, &ins, 1, 1); 7255 if (ret) 7256 return ERR_PTR(ret); 7257 7258 em = btrfs_create_dio_extent(inode, start, ins.offset, start, 7259 ins.objectid, ins.offset, ins.offset, 7260 ins.offset, 0); 7261 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 7262 if (IS_ERR(em)) 7263 btrfs_free_reserved_extent(fs_info, ins.objectid, 7264 ins.offset, 1); 7265 7266 return em; 7267 } 7268 7269 /* 7270 * returns 1 when the nocow is safe, < 1 on error, 0 if the 7271 * block must be cow'd 7272 */ 7273 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 7274 u64 *orig_start, u64 *orig_block_len, 7275 u64 *ram_bytes) 7276 { 7277 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7278 struct btrfs_trans_handle *trans; 7279 struct btrfs_path *path; 7280 int ret; 7281 struct extent_buffer *leaf; 7282 struct btrfs_root *root = BTRFS_I(inode)->root; 7283 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7284 struct btrfs_file_extent_item *fi; 7285 struct btrfs_key key; 7286 u64 disk_bytenr; 7287 u64 backref_offset; 7288 u64 extent_end; 7289 u64 num_bytes; 7290 int slot; 7291 int found_type; 7292 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW); 7293 7294 path = btrfs_alloc_path(); 7295 if (!path) 7296 return -ENOMEM; 7297 7298 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), 7299 offset, 0); 7300 if (ret < 0) 7301 goto out; 7302 7303 slot = path->slots[0]; 7304 if (ret == 1) { 7305 if (slot == 0) { 7306 /* can't find the item, must cow */ 7307 ret = 0; 7308 goto out; 7309 } 7310 slot--; 7311 } 7312 ret = 0; 7313 leaf = path->nodes[0]; 7314 btrfs_item_key_to_cpu(leaf, &key, slot); 7315 if (key.objectid != btrfs_ino(inode) || 7316 key.type != BTRFS_EXTENT_DATA_KEY) { 7317 /* not our file or wrong item type, must cow */ 7318 goto out; 7319 } 7320 7321 if (key.offset > offset) { 7322 /* Wrong offset, must cow */ 7323 goto out; 7324 } 7325 7326 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 7327 found_type = btrfs_file_extent_type(leaf, fi); 7328 if (found_type != BTRFS_FILE_EXTENT_REG && 7329 found_type != BTRFS_FILE_EXTENT_PREALLOC) { 7330 /* not a regular extent, must cow */ 7331 goto out; 7332 } 7333 7334 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG) 7335 goto out; 7336 7337 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 7338 if (extent_end <= offset) 7339 goto out; 7340 7341 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 7342 if (disk_bytenr == 0) 7343 goto out; 7344 7345 if (btrfs_file_extent_compression(leaf, fi) || 7346 btrfs_file_extent_encryption(leaf, fi) || 7347 btrfs_file_extent_other_encoding(leaf, fi)) 7348 goto out; 7349 7350 backref_offset = btrfs_file_extent_offset(leaf, fi); 7351 7352 if (orig_start) { 7353 *orig_start = key.offset - backref_offset; 7354 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 7355 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 7356 } 7357 7358 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 7359 goto out; 7360 7361 num_bytes = min(offset + *len, extent_end) - offset; 7362 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7363 u64 range_end; 7364 7365 range_end = round_up(offset + num_bytes, 7366 root->fs_info->sectorsize) - 1; 7367 ret = test_range_bit(io_tree, offset, range_end, 7368 EXTENT_DELALLOC, 0, NULL); 7369 if (ret) { 7370 ret = -EAGAIN; 7371 goto out; 7372 } 7373 } 7374 7375 btrfs_release_path(path); 7376 7377 /* 7378 * look for other files referencing this extent, if we 7379 * find any we must cow 7380 */ 7381 trans = btrfs_join_transaction(root); 7382 if (IS_ERR(trans)) { 7383 ret = 0; 7384 goto out; 7385 } 7386 7387 ret = btrfs_cross_ref_exist(trans, root, btrfs_ino(inode), 7388 key.offset - backref_offset, disk_bytenr); 7389 btrfs_end_transaction(trans); 7390 if (ret) { 7391 ret = 0; 7392 goto out; 7393 } 7394 7395 /* 7396 * adjust disk_bytenr and num_bytes to cover just the bytes 7397 * in this extent we are about to write. If there 7398 * are any csums in that range we have to cow in order 7399 * to keep the csums correct 7400 */ 7401 disk_bytenr += backref_offset; 7402 disk_bytenr += offset - key.offset; 7403 if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes)) 7404 goto out; 7405 /* 7406 * all of the above have passed, it is safe to overwrite this extent 7407 * without cow 7408 */ 7409 *len = num_bytes; 7410 ret = 1; 7411 out: 7412 btrfs_free_path(path); 7413 return ret; 7414 } 7415 7416 bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end) 7417 { 7418 struct radix_tree_root *root = &inode->i_mapping->page_tree; 7419 int found = false; 7420 void **pagep = NULL; 7421 struct page *page = NULL; 7422 int start_idx; 7423 int end_idx; 7424 7425 start_idx = start >> PAGE_SHIFT; 7426 7427 /* 7428 * end is the last byte in the last page. end == start is legal 7429 */ 7430 end_idx = end >> PAGE_SHIFT; 7431 7432 rcu_read_lock(); 7433 7434 /* Most of the code in this while loop is lifted from 7435 * find_get_page. It's been modified to begin searching from a 7436 * page and return just the first page found in that range. If the 7437 * found idx is less than or equal to the end idx then we know that 7438 * a page exists. If no pages are found or if those pages are 7439 * outside of the range then we're fine (yay!) */ 7440 while (page == NULL && 7441 radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) { 7442 page = radix_tree_deref_slot(pagep); 7443 if (unlikely(!page)) 7444 break; 7445 7446 if (radix_tree_exception(page)) { 7447 if (radix_tree_deref_retry(page)) { 7448 page = NULL; 7449 continue; 7450 } 7451 /* 7452 * Otherwise, shmem/tmpfs must be storing a swap entry 7453 * here as an exceptional entry: so return it without 7454 * attempting to raise page count. 7455 */ 7456 page = NULL; 7457 break; /* TODO: Is this relevant for this use case? */ 7458 } 7459 7460 if (!page_cache_get_speculative(page)) { 7461 page = NULL; 7462 continue; 7463 } 7464 7465 /* 7466 * Has the page moved? 7467 * This is part of the lockless pagecache protocol. See 7468 * include/linux/pagemap.h for details. 7469 */ 7470 if (unlikely(page != *pagep)) { 7471 put_page(page); 7472 page = NULL; 7473 } 7474 } 7475 7476 if (page) { 7477 if (page->index <= end_idx) 7478 found = true; 7479 put_page(page); 7480 } 7481 7482 rcu_read_unlock(); 7483 return found; 7484 } 7485 7486 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7487 struct extent_state **cached_state, int writing) 7488 { 7489 struct btrfs_ordered_extent *ordered; 7490 int ret = 0; 7491 7492 while (1) { 7493 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7494 cached_state); 7495 /* 7496 * We're concerned with the entire range that we're going to be 7497 * doing DIO to, so we need to make sure there's no ordered 7498 * extents in this range. 7499 */ 7500 ordered = btrfs_lookup_ordered_range(inode, lockstart, 7501 lockend - lockstart + 1); 7502 7503 /* 7504 * We need to make sure there are no buffered pages in this 7505 * range either, we could have raced between the invalidate in 7506 * generic_file_direct_write and locking the extent. The 7507 * invalidate needs to happen so that reads after a write do not 7508 * get stale data. 7509 */ 7510 if (!ordered && 7511 (!writing || 7512 !btrfs_page_exists_in_range(inode, lockstart, lockend))) 7513 break; 7514 7515 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7516 cached_state, GFP_NOFS); 7517 7518 if (ordered) { 7519 /* 7520 * If we are doing a DIO read and the ordered extent we 7521 * found is for a buffered write, we can not wait for it 7522 * to complete and retry, because if we do so we can 7523 * deadlock with concurrent buffered writes on page 7524 * locks. This happens only if our DIO read covers more 7525 * than one extent map, if at this point has already 7526 * created an ordered extent for a previous extent map 7527 * and locked its range in the inode's io tree, and a 7528 * concurrent write against that previous extent map's 7529 * range and this range started (we unlock the ranges 7530 * in the io tree only when the bios complete and 7531 * buffered writes always lock pages before attempting 7532 * to lock range in the io tree). 7533 */ 7534 if (writing || 7535 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) 7536 btrfs_start_ordered_extent(inode, ordered, 1); 7537 else 7538 ret = -ENOTBLK; 7539 btrfs_put_ordered_extent(ordered); 7540 } else { 7541 /* 7542 * We could trigger writeback for this range (and wait 7543 * for it to complete) and then invalidate the pages for 7544 * this range (through invalidate_inode_pages2_range()), 7545 * but that can lead us to a deadlock with a concurrent 7546 * call to readpages() (a buffered read or a defrag call 7547 * triggered a readahead) on a page lock due to an 7548 * ordered dio extent we created before but did not have 7549 * yet a corresponding bio submitted (whence it can not 7550 * complete), which makes readpages() wait for that 7551 * ordered extent to complete while holding a lock on 7552 * that page. 7553 */ 7554 ret = -ENOTBLK; 7555 } 7556 7557 if (ret) 7558 break; 7559 7560 cond_resched(); 7561 } 7562 7563 return ret; 7564 } 7565 7566 static struct extent_map *create_pinned_em(struct inode *inode, u64 start, 7567 u64 len, u64 orig_start, 7568 u64 block_start, u64 block_len, 7569 u64 orig_block_len, u64 ram_bytes, 7570 int type) 7571 { 7572 struct extent_map_tree *em_tree; 7573 struct extent_map *em; 7574 struct btrfs_root *root = BTRFS_I(inode)->root; 7575 int ret; 7576 7577 em_tree = &BTRFS_I(inode)->extent_tree; 7578 em = alloc_extent_map(); 7579 if (!em) 7580 return ERR_PTR(-ENOMEM); 7581 7582 em->start = start; 7583 em->orig_start = orig_start; 7584 em->mod_start = start; 7585 em->mod_len = len; 7586 em->len = len; 7587 em->block_len = block_len; 7588 em->block_start = block_start; 7589 em->bdev = root->fs_info->fs_devices->latest_bdev; 7590 em->orig_block_len = orig_block_len; 7591 em->ram_bytes = ram_bytes; 7592 em->generation = -1; 7593 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7594 if (type == BTRFS_ORDERED_PREALLOC) 7595 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7596 7597 do { 7598 btrfs_drop_extent_cache(inode, em->start, 7599 em->start + em->len - 1, 0); 7600 write_lock(&em_tree->lock); 7601 ret = add_extent_mapping(em_tree, em, 1); 7602 write_unlock(&em_tree->lock); 7603 } while (ret == -EEXIST); 7604 7605 if (ret) { 7606 free_extent_map(em); 7607 return ERR_PTR(ret); 7608 } 7609 7610 return em; 7611 } 7612 7613 static void adjust_dio_outstanding_extents(struct inode *inode, 7614 struct btrfs_dio_data *dio_data, 7615 const u64 len) 7616 { 7617 unsigned num_extents; 7618 7619 num_extents = (unsigned) div64_u64(len + BTRFS_MAX_EXTENT_SIZE - 1, 7620 BTRFS_MAX_EXTENT_SIZE); 7621 /* 7622 * If we have an outstanding_extents count still set then we're 7623 * within our reservation, otherwise we need to adjust our inode 7624 * counter appropriately. 7625 */ 7626 if (dio_data->outstanding_extents) { 7627 dio_data->outstanding_extents -= num_extents; 7628 } else { 7629 spin_lock(&BTRFS_I(inode)->lock); 7630 BTRFS_I(inode)->outstanding_extents += num_extents; 7631 spin_unlock(&BTRFS_I(inode)->lock); 7632 } 7633 } 7634 7635 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, 7636 struct buffer_head *bh_result, int create) 7637 { 7638 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7639 struct extent_map *em; 7640 struct extent_state *cached_state = NULL; 7641 struct btrfs_dio_data *dio_data = NULL; 7642 u64 start = iblock << inode->i_blkbits; 7643 u64 lockstart, lockend; 7644 u64 len = bh_result->b_size; 7645 int unlock_bits = EXTENT_LOCKED; 7646 int ret = 0; 7647 7648 if (create) 7649 unlock_bits |= EXTENT_DIRTY; 7650 else 7651 len = min_t(u64, len, fs_info->sectorsize); 7652 7653 lockstart = start; 7654 lockend = start + len - 1; 7655 7656 if (current->journal_info) { 7657 /* 7658 * Need to pull our outstanding extents and set journal_info to NULL so 7659 * that anything that needs to check if there's a transaction doesn't get 7660 * confused. 7661 */ 7662 dio_data = current->journal_info; 7663 current->journal_info = NULL; 7664 } 7665 7666 /* 7667 * If this errors out it's because we couldn't invalidate pagecache for 7668 * this range and we need to fallback to buffered. 7669 */ 7670 if (lock_extent_direct(inode, lockstart, lockend, &cached_state, 7671 create)) { 7672 ret = -ENOTBLK; 7673 goto err; 7674 } 7675 7676 em = btrfs_get_extent(inode, NULL, 0, start, len, 0); 7677 if (IS_ERR(em)) { 7678 ret = PTR_ERR(em); 7679 goto unlock_err; 7680 } 7681 7682 /* 7683 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7684 * io. INLINE is special, and we could probably kludge it in here, but 7685 * it's still buffered so for safety lets just fall back to the generic 7686 * buffered path. 7687 * 7688 * For COMPRESSED we _have_ to read the entire extent in so we can 7689 * decompress it, so there will be buffering required no matter what we 7690 * do, so go ahead and fallback to buffered. 7691 * 7692 * We return -ENOTBLK because that's what makes DIO go ahead and go back 7693 * to buffered IO. Don't blame me, this is the price we pay for using 7694 * the generic code. 7695 */ 7696 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7697 em->block_start == EXTENT_MAP_INLINE) { 7698 free_extent_map(em); 7699 ret = -ENOTBLK; 7700 goto unlock_err; 7701 } 7702 7703 /* Just a good old fashioned hole, return */ 7704 if (!create && (em->block_start == EXTENT_MAP_HOLE || 7705 test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) { 7706 free_extent_map(em); 7707 goto unlock_err; 7708 } 7709 7710 /* 7711 * We don't allocate a new extent in the following cases 7712 * 7713 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7714 * existing extent. 7715 * 2) The extent is marked as PREALLOC. We're good to go here and can 7716 * just use the extent. 7717 * 7718 */ 7719 if (!create) { 7720 len = min(len, em->len - (start - em->start)); 7721 lockstart = start + len; 7722 goto unlock; 7723 } 7724 7725 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7726 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7727 em->block_start != EXTENT_MAP_HOLE)) { 7728 int type; 7729 u64 block_start, orig_start, orig_block_len, ram_bytes; 7730 7731 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7732 type = BTRFS_ORDERED_PREALLOC; 7733 else 7734 type = BTRFS_ORDERED_NOCOW; 7735 len = min(len, em->len - (start - em->start)); 7736 block_start = em->block_start + (start - em->start); 7737 7738 if (can_nocow_extent(inode, start, &len, &orig_start, 7739 &orig_block_len, &ram_bytes) == 1 && 7740 btrfs_inc_nocow_writers(fs_info, block_start)) { 7741 struct extent_map *em2; 7742 7743 em2 = btrfs_create_dio_extent(inode, start, len, 7744 orig_start, block_start, 7745 len, orig_block_len, 7746 ram_bytes, type); 7747 btrfs_dec_nocow_writers(fs_info, block_start); 7748 if (type == BTRFS_ORDERED_PREALLOC) { 7749 free_extent_map(em); 7750 em = em2; 7751 } 7752 if (em2 && IS_ERR(em2)) { 7753 ret = PTR_ERR(em2); 7754 goto unlock_err; 7755 } 7756 /* 7757 * For inode marked NODATACOW or extent marked PREALLOC, 7758 * use the existing or preallocated extent, so does not 7759 * need to adjust btrfs_space_info's bytes_may_use. 7760 */ 7761 btrfs_free_reserved_data_space_noquota(inode, 7762 start, len); 7763 goto unlock; 7764 } 7765 } 7766 7767 /* 7768 * this will cow the extent, reset the len in case we changed 7769 * it above 7770 */ 7771 len = bh_result->b_size; 7772 free_extent_map(em); 7773 em = btrfs_new_extent_direct(inode, start, len); 7774 if (IS_ERR(em)) { 7775 ret = PTR_ERR(em); 7776 goto unlock_err; 7777 } 7778 len = min(len, em->len - (start - em->start)); 7779 unlock: 7780 bh_result->b_blocknr = (em->block_start + (start - em->start)) >> 7781 inode->i_blkbits; 7782 bh_result->b_size = len; 7783 bh_result->b_bdev = em->bdev; 7784 set_buffer_mapped(bh_result); 7785 if (create) { 7786 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7787 set_buffer_new(bh_result); 7788 7789 /* 7790 * Need to update the i_size under the extent lock so buffered 7791 * readers will get the updated i_size when we unlock. 7792 */ 7793 if (start + len > i_size_read(inode)) 7794 i_size_write(inode, start + len); 7795 7796 adjust_dio_outstanding_extents(inode, dio_data, len); 7797 WARN_ON(dio_data->reserve < len); 7798 dio_data->reserve -= len; 7799 dio_data->unsubmitted_oe_range_end = start + len; 7800 current->journal_info = dio_data; 7801 } 7802 7803 /* 7804 * In the case of write we need to clear and unlock the entire range, 7805 * in the case of read we need to unlock only the end area that we 7806 * aren't using if there is any left over space. 7807 */ 7808 if (lockstart < lockend) { 7809 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, 7810 lockend, unlock_bits, 1, 0, 7811 &cached_state, GFP_NOFS); 7812 } else { 7813 free_extent_state(cached_state); 7814 } 7815 7816 free_extent_map(em); 7817 7818 return 0; 7819 7820 unlock_err: 7821 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7822 unlock_bits, 1, 0, &cached_state, GFP_NOFS); 7823 err: 7824 if (dio_data) 7825 current->journal_info = dio_data; 7826 /* 7827 * Compensate the delalloc release we do in btrfs_direct_IO() when we 7828 * write less data then expected, so that we don't underflow our inode's 7829 * outstanding extents counter. 7830 */ 7831 if (create && dio_data) 7832 adjust_dio_outstanding_extents(inode, dio_data, len); 7833 7834 return ret; 7835 } 7836 7837 static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio, 7838 int mirror_num) 7839 { 7840 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7841 int ret; 7842 7843 BUG_ON(bio_op(bio) == REQ_OP_WRITE); 7844 7845 bio_get(bio); 7846 7847 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR); 7848 if (ret) 7849 goto err; 7850 7851 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0); 7852 err: 7853 bio_put(bio); 7854 return ret; 7855 } 7856 7857 static int btrfs_check_dio_repairable(struct inode *inode, 7858 struct bio *failed_bio, 7859 struct io_failure_record *failrec, 7860 int failed_mirror) 7861 { 7862 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7863 int num_copies; 7864 7865 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len); 7866 if (num_copies == 1) { 7867 /* 7868 * we only have a single copy of the data, so don't bother with 7869 * all the retry and error correction code that follows. no 7870 * matter what the error is, it is very likely to persist. 7871 */ 7872 btrfs_debug(fs_info, 7873 "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d", 7874 num_copies, failrec->this_mirror, failed_mirror); 7875 return 0; 7876 } 7877 7878 failrec->failed_mirror = failed_mirror; 7879 failrec->this_mirror++; 7880 if (failrec->this_mirror == failed_mirror) 7881 failrec->this_mirror++; 7882 7883 if (failrec->this_mirror > num_copies) { 7884 btrfs_debug(fs_info, 7885 "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d", 7886 num_copies, failrec->this_mirror, failed_mirror); 7887 return 0; 7888 } 7889 7890 return 1; 7891 } 7892 7893 static int dio_read_error(struct inode *inode, struct bio *failed_bio, 7894 struct page *page, unsigned int pgoff, 7895 u64 start, u64 end, int failed_mirror, 7896 bio_end_io_t *repair_endio, void *repair_arg) 7897 { 7898 struct io_failure_record *failrec; 7899 struct bio *bio; 7900 int isector; 7901 int read_mode = 0; 7902 int ret; 7903 7904 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); 7905 7906 ret = btrfs_get_io_failure_record(inode, start, end, &failrec); 7907 if (ret) 7908 return ret; 7909 7910 ret = btrfs_check_dio_repairable(inode, failed_bio, failrec, 7911 failed_mirror); 7912 if (!ret) { 7913 free_io_failure(inode, failrec); 7914 return -EIO; 7915 } 7916 7917 if ((failed_bio->bi_vcnt > 1) 7918 || (failed_bio->bi_io_vec->bv_len 7919 > btrfs_inode_sectorsize(inode))) 7920 read_mode |= REQ_FAILFAST_DEV; 7921 7922 isector = start - btrfs_io_bio(failed_bio)->logical; 7923 isector >>= inode->i_sb->s_blocksize_bits; 7924 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, 7925 pgoff, isector, repair_endio, repair_arg); 7926 if (!bio) { 7927 free_io_failure(inode, failrec); 7928 return -EIO; 7929 } 7930 bio_set_op_attrs(bio, REQ_OP_READ, read_mode); 7931 7932 btrfs_debug(BTRFS_I(inode)->root->fs_info, 7933 "Repair DIO Read Error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d\n", 7934 read_mode, failrec->this_mirror, failrec->in_validation); 7935 7936 ret = submit_dio_repair_bio(inode, bio, failrec->this_mirror); 7937 if (ret) { 7938 free_io_failure(inode, failrec); 7939 bio_put(bio); 7940 } 7941 7942 return ret; 7943 } 7944 7945 struct btrfs_retry_complete { 7946 struct completion done; 7947 struct inode *inode; 7948 u64 start; 7949 int uptodate; 7950 }; 7951 7952 static void btrfs_retry_endio_nocsum(struct bio *bio) 7953 { 7954 struct btrfs_retry_complete *done = bio->bi_private; 7955 struct inode *inode; 7956 struct bio_vec *bvec; 7957 int i; 7958 7959 if (bio->bi_error) 7960 goto end; 7961 7962 ASSERT(bio->bi_vcnt == 1); 7963 inode = bio->bi_io_vec->bv_page->mapping->host; 7964 ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode)); 7965 7966 done->uptodate = 1; 7967 bio_for_each_segment_all(bvec, bio, i) 7968 clean_io_failure(done->inode, done->start, bvec->bv_page, 0); 7969 end: 7970 complete(&done->done); 7971 bio_put(bio); 7972 } 7973 7974 static int __btrfs_correct_data_nocsum(struct inode *inode, 7975 struct btrfs_io_bio *io_bio) 7976 { 7977 struct btrfs_fs_info *fs_info; 7978 struct bio_vec *bvec; 7979 struct btrfs_retry_complete done; 7980 u64 start; 7981 unsigned int pgoff; 7982 u32 sectorsize; 7983 int nr_sectors; 7984 int i; 7985 int ret; 7986 7987 fs_info = BTRFS_I(inode)->root->fs_info; 7988 sectorsize = fs_info->sectorsize; 7989 7990 start = io_bio->logical; 7991 done.inode = inode; 7992 7993 bio_for_each_segment_all(bvec, &io_bio->bio, i) { 7994 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len); 7995 pgoff = bvec->bv_offset; 7996 7997 next_block_or_try_again: 7998 done.uptodate = 0; 7999 done.start = start; 8000 init_completion(&done.done); 8001 8002 ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, 8003 pgoff, start, start + sectorsize - 1, 8004 io_bio->mirror_num, 8005 btrfs_retry_endio_nocsum, &done); 8006 if (ret) 8007 return ret; 8008 8009 wait_for_completion(&done.done); 8010 8011 if (!done.uptodate) { 8012 /* We might have another mirror, so try again */ 8013 goto next_block_or_try_again; 8014 } 8015 8016 start += sectorsize; 8017 8018 if (nr_sectors--) { 8019 pgoff += sectorsize; 8020 goto next_block_or_try_again; 8021 } 8022 } 8023 8024 return 0; 8025 } 8026 8027 static void btrfs_retry_endio(struct bio *bio) 8028 { 8029 struct btrfs_retry_complete *done = bio->bi_private; 8030 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8031 struct inode *inode; 8032 struct bio_vec *bvec; 8033 u64 start; 8034 int uptodate; 8035 int ret; 8036 int i; 8037 8038 if (bio->bi_error) 8039 goto end; 8040 8041 uptodate = 1; 8042 8043 start = done->start; 8044 8045 ASSERT(bio->bi_vcnt == 1); 8046 inode = bio->bi_io_vec->bv_page->mapping->host; 8047 ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode)); 8048 8049 bio_for_each_segment_all(bvec, bio, i) { 8050 ret = __readpage_endio_check(done->inode, io_bio, i, 8051 bvec->bv_page, bvec->bv_offset, 8052 done->start, bvec->bv_len); 8053 if (!ret) 8054 clean_io_failure(done->inode, done->start, 8055 bvec->bv_page, bvec->bv_offset); 8056 else 8057 uptodate = 0; 8058 } 8059 8060 done->uptodate = uptodate; 8061 end: 8062 complete(&done->done); 8063 bio_put(bio); 8064 } 8065 8066 static int __btrfs_subio_endio_read(struct inode *inode, 8067 struct btrfs_io_bio *io_bio, int err) 8068 { 8069 struct btrfs_fs_info *fs_info; 8070 struct bio_vec *bvec; 8071 struct btrfs_retry_complete done; 8072 u64 start; 8073 u64 offset = 0; 8074 u32 sectorsize; 8075 int nr_sectors; 8076 unsigned int pgoff; 8077 int csum_pos; 8078 int i; 8079 int ret; 8080 8081 fs_info = BTRFS_I(inode)->root->fs_info; 8082 sectorsize = fs_info->sectorsize; 8083 8084 err = 0; 8085 start = io_bio->logical; 8086 done.inode = inode; 8087 8088 bio_for_each_segment_all(bvec, &io_bio->bio, i) { 8089 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len); 8090 8091 pgoff = bvec->bv_offset; 8092 next_block: 8093 csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset); 8094 ret = __readpage_endio_check(inode, io_bio, csum_pos, 8095 bvec->bv_page, pgoff, start, 8096 sectorsize); 8097 if (likely(!ret)) 8098 goto next; 8099 try_again: 8100 done.uptodate = 0; 8101 done.start = start; 8102 init_completion(&done.done); 8103 8104 ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, 8105 pgoff, start, start + sectorsize - 1, 8106 io_bio->mirror_num, 8107 btrfs_retry_endio, &done); 8108 if (ret) { 8109 err = ret; 8110 goto next; 8111 } 8112 8113 wait_for_completion(&done.done); 8114 8115 if (!done.uptodate) { 8116 /* We might have another mirror, so try again */ 8117 goto try_again; 8118 } 8119 next: 8120 offset += sectorsize; 8121 start += sectorsize; 8122 8123 ASSERT(nr_sectors); 8124 8125 if (--nr_sectors) { 8126 pgoff += sectorsize; 8127 goto next_block; 8128 } 8129 } 8130 8131 return err; 8132 } 8133 8134 static int btrfs_subio_endio_read(struct inode *inode, 8135 struct btrfs_io_bio *io_bio, int err) 8136 { 8137 bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 8138 8139 if (skip_csum) { 8140 if (unlikely(err)) 8141 return __btrfs_correct_data_nocsum(inode, io_bio); 8142 else 8143 return 0; 8144 } else { 8145 return __btrfs_subio_endio_read(inode, io_bio, err); 8146 } 8147 } 8148 8149 static void btrfs_endio_direct_read(struct bio *bio) 8150 { 8151 struct btrfs_dio_private *dip = bio->bi_private; 8152 struct inode *inode = dip->inode; 8153 struct bio *dio_bio; 8154 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8155 int err = bio->bi_error; 8156 8157 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) 8158 err = btrfs_subio_endio_read(inode, io_bio, err); 8159 8160 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset, 8161 dip->logical_offset + dip->bytes - 1); 8162 dio_bio = dip->dio_bio; 8163 8164 kfree(dip); 8165 8166 dio_bio->bi_error = bio->bi_error; 8167 dio_end_io(dio_bio, bio->bi_error); 8168 8169 if (io_bio->end_io) 8170 io_bio->end_io(io_bio, err); 8171 bio_put(bio); 8172 } 8173 8174 static void btrfs_endio_direct_write_update_ordered(struct inode *inode, 8175 const u64 offset, 8176 const u64 bytes, 8177 const int uptodate) 8178 { 8179 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8180 struct btrfs_ordered_extent *ordered = NULL; 8181 u64 ordered_offset = offset; 8182 u64 ordered_bytes = bytes; 8183 int ret; 8184 8185 again: 8186 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, 8187 &ordered_offset, 8188 ordered_bytes, 8189 uptodate); 8190 if (!ret) 8191 goto out_test; 8192 8193 btrfs_init_work(&ordered->work, btrfs_endio_write_helper, 8194 finish_ordered_fn, NULL, NULL); 8195 btrfs_queue_work(fs_info->endio_write_workers, &ordered->work); 8196 out_test: 8197 /* 8198 * our bio might span multiple ordered extents. If we haven't 8199 * completed the accounting for the whole dio, go back and try again 8200 */ 8201 if (ordered_offset < offset + bytes) { 8202 ordered_bytes = offset + bytes - ordered_offset; 8203 ordered = NULL; 8204 goto again; 8205 } 8206 } 8207 8208 static void btrfs_endio_direct_write(struct bio *bio) 8209 { 8210 struct btrfs_dio_private *dip = bio->bi_private; 8211 struct bio *dio_bio = dip->dio_bio; 8212 8213 btrfs_endio_direct_write_update_ordered(dip->inode, 8214 dip->logical_offset, 8215 dip->bytes, 8216 !bio->bi_error); 8217 8218 kfree(dip); 8219 8220 dio_bio->bi_error = bio->bi_error; 8221 dio_end_io(dio_bio, bio->bi_error); 8222 bio_put(bio); 8223 } 8224 8225 static int __btrfs_submit_bio_start_direct_io(struct inode *inode, 8226 struct bio *bio, int mirror_num, 8227 unsigned long bio_flags, u64 offset) 8228 { 8229 int ret; 8230 ret = btrfs_csum_one_bio(inode, bio, offset, 1); 8231 BUG_ON(ret); /* -ENOMEM */ 8232 return 0; 8233 } 8234 8235 static void btrfs_end_dio_bio(struct bio *bio) 8236 { 8237 struct btrfs_dio_private *dip = bio->bi_private; 8238 int err = bio->bi_error; 8239 8240 if (err) 8241 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 8242 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", 8243 btrfs_ino(dip->inode), bio_op(bio), bio->bi_opf, 8244 (unsigned long long)bio->bi_iter.bi_sector, 8245 bio->bi_iter.bi_size, err); 8246 8247 if (dip->subio_endio) 8248 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err); 8249 8250 if (err) { 8251 dip->errors = 1; 8252 8253 /* 8254 * before atomic variable goto zero, we must make sure 8255 * dip->errors is perceived to be set. 8256 */ 8257 smp_mb__before_atomic(); 8258 } 8259 8260 /* if there are more bios still pending for this dio, just exit */ 8261 if (!atomic_dec_and_test(&dip->pending_bios)) 8262 goto out; 8263 8264 if (dip->errors) { 8265 bio_io_error(dip->orig_bio); 8266 } else { 8267 dip->dio_bio->bi_error = 0; 8268 bio_endio(dip->orig_bio); 8269 } 8270 out: 8271 bio_put(bio); 8272 } 8273 8274 static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev, 8275 u64 first_sector, gfp_t gfp_flags) 8276 { 8277 struct bio *bio; 8278 bio = btrfs_bio_alloc(bdev, first_sector, BIO_MAX_PAGES, gfp_flags); 8279 if (bio) 8280 bio_associate_current(bio); 8281 return bio; 8282 } 8283 8284 static inline int btrfs_lookup_and_bind_dio_csum(struct inode *inode, 8285 struct btrfs_dio_private *dip, 8286 struct bio *bio, 8287 u64 file_offset) 8288 { 8289 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 8290 struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio); 8291 int ret; 8292 8293 /* 8294 * We load all the csum data we need when we submit 8295 * the first bio to reduce the csum tree search and 8296 * contention. 8297 */ 8298 if (dip->logical_offset == file_offset) { 8299 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio, 8300 file_offset); 8301 if (ret) 8302 return ret; 8303 } 8304 8305 if (bio == dip->orig_bio) 8306 return 0; 8307 8308 file_offset -= dip->logical_offset; 8309 file_offset >>= inode->i_sb->s_blocksize_bits; 8310 io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset); 8311 8312 return 0; 8313 } 8314 8315 static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode, 8316 u64 file_offset, int skip_sum, 8317 int async_submit) 8318 { 8319 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8320 struct btrfs_dio_private *dip = bio->bi_private; 8321 bool write = bio_op(bio) == REQ_OP_WRITE; 8322 int ret; 8323 8324 if (async_submit) 8325 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers); 8326 8327 bio_get(bio); 8328 8329 if (!write) { 8330 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 8331 if (ret) 8332 goto err; 8333 } 8334 8335 if (skip_sum) 8336 goto map; 8337 8338 if (write && async_submit) { 8339 ret = btrfs_wq_submit_bio(fs_info, inode, bio, 0, 0, 8340 file_offset, 8341 __btrfs_submit_bio_start_direct_io, 8342 __btrfs_submit_bio_done); 8343 goto err; 8344 } else if (write) { 8345 /* 8346 * If we aren't doing async submit, calculate the csum of the 8347 * bio now. 8348 */ 8349 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1); 8350 if (ret) 8351 goto err; 8352 } else { 8353 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio, 8354 file_offset); 8355 if (ret) 8356 goto err; 8357 } 8358 map: 8359 ret = btrfs_map_bio(fs_info, bio, 0, async_submit); 8360 err: 8361 bio_put(bio); 8362 return ret; 8363 } 8364 8365 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip, 8366 int skip_sum) 8367 { 8368 struct inode *inode = dip->inode; 8369 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8370 struct btrfs_root *root = BTRFS_I(inode)->root; 8371 struct bio *bio; 8372 struct bio *orig_bio = dip->orig_bio; 8373 struct bio_vec *bvec; 8374 u64 start_sector = orig_bio->bi_iter.bi_sector; 8375 u64 file_offset = dip->logical_offset; 8376 u64 submit_len = 0; 8377 u64 map_length; 8378 u32 blocksize = fs_info->sectorsize; 8379 int async_submit = 0; 8380 int nr_sectors; 8381 int ret; 8382 int i, j; 8383 8384 map_length = orig_bio->bi_iter.bi_size; 8385 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9, 8386 &map_length, NULL, 0); 8387 if (ret) 8388 return -EIO; 8389 8390 if (map_length >= orig_bio->bi_iter.bi_size) { 8391 bio = orig_bio; 8392 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED; 8393 goto submit; 8394 } 8395 8396 /* async crcs make it difficult to collect full stripe writes. */ 8397 if (btrfs_get_alloc_profile(root, 1) & BTRFS_BLOCK_GROUP_RAID56_MASK) 8398 async_submit = 0; 8399 else 8400 async_submit = 1; 8401 8402 bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, start_sector, GFP_NOFS); 8403 if (!bio) 8404 return -ENOMEM; 8405 8406 bio->bi_opf = orig_bio->bi_opf; 8407 bio->bi_private = dip; 8408 bio->bi_end_io = btrfs_end_dio_bio; 8409 btrfs_io_bio(bio)->logical = file_offset; 8410 atomic_inc(&dip->pending_bios); 8411 8412 bio_for_each_segment_all(bvec, orig_bio, j) { 8413 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len); 8414 i = 0; 8415 next_block: 8416 if (unlikely(map_length < submit_len + blocksize || 8417 bio_add_page(bio, bvec->bv_page, blocksize, 8418 bvec->bv_offset + (i * blocksize)) < blocksize)) { 8419 /* 8420 * inc the count before we submit the bio so 8421 * we know the end IO handler won't happen before 8422 * we inc the count. Otherwise, the dip might get freed 8423 * before we're done setting it up 8424 */ 8425 atomic_inc(&dip->pending_bios); 8426 ret = __btrfs_submit_dio_bio(bio, inode, 8427 file_offset, skip_sum, 8428 async_submit); 8429 if (ret) { 8430 bio_put(bio); 8431 atomic_dec(&dip->pending_bios); 8432 goto out_err; 8433 } 8434 8435 start_sector += submit_len >> 9; 8436 file_offset += submit_len; 8437 8438 submit_len = 0; 8439 8440 bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, 8441 start_sector, GFP_NOFS); 8442 if (!bio) 8443 goto out_err; 8444 bio->bi_opf = orig_bio->bi_opf; 8445 bio->bi_private = dip; 8446 bio->bi_end_io = btrfs_end_dio_bio; 8447 btrfs_io_bio(bio)->logical = file_offset; 8448 8449 map_length = orig_bio->bi_iter.bi_size; 8450 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), 8451 start_sector << 9, 8452 &map_length, NULL, 0); 8453 if (ret) { 8454 bio_put(bio); 8455 goto out_err; 8456 } 8457 8458 goto next_block; 8459 } else { 8460 submit_len += blocksize; 8461 if (--nr_sectors) { 8462 i++; 8463 goto next_block; 8464 } 8465 } 8466 } 8467 8468 submit: 8469 ret = __btrfs_submit_dio_bio(bio, inode, file_offset, skip_sum, 8470 async_submit); 8471 if (!ret) 8472 return 0; 8473 8474 bio_put(bio); 8475 out_err: 8476 dip->errors = 1; 8477 /* 8478 * before atomic variable goto zero, we must 8479 * make sure dip->errors is perceived to be set. 8480 */ 8481 smp_mb__before_atomic(); 8482 if (atomic_dec_and_test(&dip->pending_bios)) 8483 bio_io_error(dip->orig_bio); 8484 8485 /* bio_end_io() will handle error, so we needn't return it */ 8486 return 0; 8487 } 8488 8489 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, 8490 loff_t file_offset) 8491 { 8492 struct btrfs_dio_private *dip = NULL; 8493 struct bio *io_bio = NULL; 8494 struct btrfs_io_bio *btrfs_bio; 8495 int skip_sum; 8496 bool write = (bio_op(dio_bio) == REQ_OP_WRITE); 8497 int ret = 0; 8498 8499 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 8500 8501 io_bio = btrfs_bio_clone(dio_bio, GFP_NOFS); 8502 if (!io_bio) { 8503 ret = -ENOMEM; 8504 goto free_ordered; 8505 } 8506 8507 dip = kzalloc(sizeof(*dip), GFP_NOFS); 8508 if (!dip) { 8509 ret = -ENOMEM; 8510 goto free_ordered; 8511 } 8512 8513 dip->private = dio_bio->bi_private; 8514 dip->inode = inode; 8515 dip->logical_offset = file_offset; 8516 dip->bytes = dio_bio->bi_iter.bi_size; 8517 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; 8518 io_bio->bi_private = dip; 8519 dip->orig_bio = io_bio; 8520 dip->dio_bio = dio_bio; 8521 atomic_set(&dip->pending_bios, 0); 8522 btrfs_bio = btrfs_io_bio(io_bio); 8523 btrfs_bio->logical = file_offset; 8524 8525 if (write) { 8526 io_bio->bi_end_io = btrfs_endio_direct_write; 8527 } else { 8528 io_bio->bi_end_io = btrfs_endio_direct_read; 8529 dip->subio_endio = btrfs_subio_endio_read; 8530 } 8531 8532 /* 8533 * Reset the range for unsubmitted ordered extents (to a 0 length range) 8534 * even if we fail to submit a bio, because in such case we do the 8535 * corresponding error handling below and it must not be done a second 8536 * time by btrfs_direct_IO(). 8537 */ 8538 if (write) { 8539 struct btrfs_dio_data *dio_data = current->journal_info; 8540 8541 dio_data->unsubmitted_oe_range_end = dip->logical_offset + 8542 dip->bytes; 8543 dio_data->unsubmitted_oe_range_start = 8544 dio_data->unsubmitted_oe_range_end; 8545 } 8546 8547 ret = btrfs_submit_direct_hook(dip, skip_sum); 8548 if (!ret) 8549 return; 8550 8551 if (btrfs_bio->end_io) 8552 btrfs_bio->end_io(btrfs_bio, ret); 8553 8554 free_ordered: 8555 /* 8556 * If we arrived here it means either we failed to submit the dip 8557 * or we either failed to clone the dio_bio or failed to allocate the 8558 * dip. If we cloned the dio_bio and allocated the dip, we can just 8559 * call bio_endio against our io_bio so that we get proper resource 8560 * cleanup if we fail to submit the dip, otherwise, we must do the 8561 * same as btrfs_endio_direct_[write|read] because we can't call these 8562 * callbacks - they require an allocated dip and a clone of dio_bio. 8563 */ 8564 if (io_bio && dip) { 8565 io_bio->bi_error = -EIO; 8566 bio_endio(io_bio); 8567 /* 8568 * The end io callbacks free our dip, do the final put on io_bio 8569 * and all the cleanup and final put for dio_bio (through 8570 * dio_end_io()). 8571 */ 8572 dip = NULL; 8573 io_bio = NULL; 8574 } else { 8575 if (write) 8576 btrfs_endio_direct_write_update_ordered(inode, 8577 file_offset, 8578 dio_bio->bi_iter.bi_size, 8579 0); 8580 else 8581 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, 8582 file_offset + dio_bio->bi_iter.bi_size - 1); 8583 8584 dio_bio->bi_error = -EIO; 8585 /* 8586 * Releases and cleans up our dio_bio, no need to bio_put() 8587 * nor bio_endio()/bio_io_error() against dio_bio. 8588 */ 8589 dio_end_io(dio_bio, ret); 8590 } 8591 if (io_bio) 8592 bio_put(io_bio); 8593 kfree(dip); 8594 } 8595 8596 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, 8597 struct kiocb *iocb, 8598 const struct iov_iter *iter, loff_t offset) 8599 { 8600 int seg; 8601 int i; 8602 unsigned int blocksize_mask = fs_info->sectorsize - 1; 8603 ssize_t retval = -EINVAL; 8604 8605 if (offset & blocksize_mask) 8606 goto out; 8607 8608 if (iov_iter_alignment(iter) & blocksize_mask) 8609 goto out; 8610 8611 /* If this is a write we don't need to check anymore */ 8612 if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter)) 8613 return 0; 8614 /* 8615 * Check to make sure we don't have duplicate iov_base's in this 8616 * iovec, if so return EINVAL, otherwise we'll get csum errors 8617 * when reading back. 8618 */ 8619 for (seg = 0; seg < iter->nr_segs; seg++) { 8620 for (i = seg + 1; i < iter->nr_segs; i++) { 8621 if (iter->iov[seg].iov_base == iter->iov[i].iov_base) 8622 goto out; 8623 } 8624 } 8625 retval = 0; 8626 out: 8627 return retval; 8628 } 8629 8630 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 8631 { 8632 struct file *file = iocb->ki_filp; 8633 struct inode *inode = file->f_mapping->host; 8634 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8635 struct btrfs_dio_data dio_data = { 0 }; 8636 loff_t offset = iocb->ki_pos; 8637 size_t count = 0; 8638 int flags = 0; 8639 bool wakeup = true; 8640 bool relock = false; 8641 ssize_t ret; 8642 8643 if (check_direct_IO(fs_info, iocb, iter, offset)) 8644 return 0; 8645 8646 inode_dio_begin(inode); 8647 smp_mb__after_atomic(); 8648 8649 /* 8650 * The generic stuff only does filemap_write_and_wait_range, which 8651 * isn't enough if we've written compressed pages to this area, so 8652 * we need to flush the dirty pages again to make absolutely sure 8653 * that any outstanding dirty pages are on disk. 8654 */ 8655 count = iov_iter_count(iter); 8656 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 8657 &BTRFS_I(inode)->runtime_flags)) 8658 filemap_fdatawrite_range(inode->i_mapping, offset, 8659 offset + count - 1); 8660 8661 if (iov_iter_rw(iter) == WRITE) { 8662 /* 8663 * If the write DIO is beyond the EOF, we need update 8664 * the isize, but it is protected by i_mutex. So we can 8665 * not unlock the i_mutex at this case. 8666 */ 8667 if (offset + count <= inode->i_size) { 8668 inode_unlock(inode); 8669 relock = true; 8670 } 8671 ret = btrfs_delalloc_reserve_space(inode, offset, count); 8672 if (ret) 8673 goto out; 8674 dio_data.outstanding_extents = div64_u64(count + 8675 BTRFS_MAX_EXTENT_SIZE - 1, 8676 BTRFS_MAX_EXTENT_SIZE); 8677 8678 /* 8679 * We need to know how many extents we reserved so that we can 8680 * do the accounting properly if we go over the number we 8681 * originally calculated. Abuse current->journal_info for this. 8682 */ 8683 dio_data.reserve = round_up(count, 8684 fs_info->sectorsize); 8685 dio_data.unsubmitted_oe_range_start = (u64)offset; 8686 dio_data.unsubmitted_oe_range_end = (u64)offset; 8687 current->journal_info = &dio_data; 8688 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK, 8689 &BTRFS_I(inode)->runtime_flags)) { 8690 inode_dio_end(inode); 8691 flags = DIO_LOCKING | DIO_SKIP_HOLES; 8692 wakeup = false; 8693 } 8694 8695 ret = __blockdev_direct_IO(iocb, inode, 8696 fs_info->fs_devices->latest_bdev, 8697 iter, btrfs_get_blocks_direct, NULL, 8698 btrfs_submit_direct, flags); 8699 if (iov_iter_rw(iter) == WRITE) { 8700 current->journal_info = NULL; 8701 if (ret < 0 && ret != -EIOCBQUEUED) { 8702 if (dio_data.reserve) 8703 btrfs_delalloc_release_space(inode, offset, 8704 dio_data.reserve); 8705 /* 8706 * On error we might have left some ordered extents 8707 * without submitting corresponding bios for them, so 8708 * cleanup them up to avoid other tasks getting them 8709 * and waiting for them to complete forever. 8710 */ 8711 if (dio_data.unsubmitted_oe_range_start < 8712 dio_data.unsubmitted_oe_range_end) 8713 btrfs_endio_direct_write_update_ordered(inode, 8714 dio_data.unsubmitted_oe_range_start, 8715 dio_data.unsubmitted_oe_range_end - 8716 dio_data.unsubmitted_oe_range_start, 8717 0); 8718 } else if (ret >= 0 && (size_t)ret < count) 8719 btrfs_delalloc_release_space(inode, offset, 8720 count - (size_t)ret); 8721 } 8722 out: 8723 if (wakeup) 8724 inode_dio_end(inode); 8725 if (relock) 8726 inode_lock(inode); 8727 8728 return ret; 8729 } 8730 8731 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC) 8732 8733 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 8734 __u64 start, __u64 len) 8735 { 8736 int ret; 8737 8738 ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS); 8739 if (ret) 8740 return ret; 8741 8742 return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap); 8743 } 8744 8745 int btrfs_readpage(struct file *file, struct page *page) 8746 { 8747 struct extent_io_tree *tree; 8748 tree = &BTRFS_I(page->mapping->host)->io_tree; 8749 return extent_read_full_page(tree, page, btrfs_get_extent, 0); 8750 } 8751 8752 static int btrfs_writepage(struct page *page, struct writeback_control *wbc) 8753 { 8754 struct extent_io_tree *tree; 8755 struct inode *inode = page->mapping->host; 8756 int ret; 8757 8758 if (current->flags & PF_MEMALLOC) { 8759 redirty_page_for_writepage(wbc, page); 8760 unlock_page(page); 8761 return 0; 8762 } 8763 8764 /* 8765 * If we are under memory pressure we will call this directly from the 8766 * VM, we need to make sure we have the inode referenced for the ordered 8767 * extent. If not just return like we didn't do anything. 8768 */ 8769 if (!igrab(inode)) { 8770 redirty_page_for_writepage(wbc, page); 8771 return AOP_WRITEPAGE_ACTIVATE; 8772 } 8773 tree = &BTRFS_I(page->mapping->host)->io_tree; 8774 ret = extent_write_full_page(tree, page, btrfs_get_extent, wbc); 8775 btrfs_add_delayed_iput(inode); 8776 return ret; 8777 } 8778 8779 static int btrfs_writepages(struct address_space *mapping, 8780 struct writeback_control *wbc) 8781 { 8782 struct extent_io_tree *tree; 8783 8784 tree = &BTRFS_I(mapping->host)->io_tree; 8785 return extent_writepages(tree, mapping, btrfs_get_extent, wbc); 8786 } 8787 8788 static int 8789 btrfs_readpages(struct file *file, struct address_space *mapping, 8790 struct list_head *pages, unsigned nr_pages) 8791 { 8792 struct extent_io_tree *tree; 8793 tree = &BTRFS_I(mapping->host)->io_tree; 8794 return extent_readpages(tree, mapping, pages, nr_pages, 8795 btrfs_get_extent); 8796 } 8797 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8798 { 8799 struct extent_io_tree *tree; 8800 struct extent_map_tree *map; 8801 int ret; 8802 8803 tree = &BTRFS_I(page->mapping->host)->io_tree; 8804 map = &BTRFS_I(page->mapping->host)->extent_tree; 8805 ret = try_release_extent_mapping(map, tree, page, gfp_flags); 8806 if (ret == 1) { 8807 ClearPagePrivate(page); 8808 set_page_private(page, 0); 8809 put_page(page); 8810 } 8811 return ret; 8812 } 8813 8814 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8815 { 8816 if (PageWriteback(page) || PageDirty(page)) 8817 return 0; 8818 return __btrfs_releasepage(page, gfp_flags & GFP_NOFS); 8819 } 8820 8821 static void btrfs_invalidatepage(struct page *page, unsigned int offset, 8822 unsigned int length) 8823 { 8824 struct inode *inode = page->mapping->host; 8825 struct extent_io_tree *tree; 8826 struct btrfs_ordered_extent *ordered; 8827 struct extent_state *cached_state = NULL; 8828 u64 page_start = page_offset(page); 8829 u64 page_end = page_start + PAGE_SIZE - 1; 8830 u64 start; 8831 u64 end; 8832 int inode_evicting = inode->i_state & I_FREEING; 8833 8834 /* 8835 * we have the page locked, so new writeback can't start, 8836 * and the dirty bit won't be cleared while we are here. 8837 * 8838 * Wait for IO on this page so that we can safely clear 8839 * the PagePrivate2 bit and do ordered accounting 8840 */ 8841 wait_on_page_writeback(page); 8842 8843 tree = &BTRFS_I(inode)->io_tree; 8844 if (offset) { 8845 btrfs_releasepage(page, GFP_NOFS); 8846 return; 8847 } 8848 8849 if (!inode_evicting) 8850 lock_extent_bits(tree, page_start, page_end, &cached_state); 8851 again: 8852 start = page_start; 8853 ordered = btrfs_lookup_ordered_range(inode, start, 8854 page_end - start + 1); 8855 if (ordered) { 8856 end = min(page_end, ordered->file_offset + ordered->len - 1); 8857 /* 8858 * IO on this page will never be started, so we need 8859 * to account for any ordered extents now 8860 */ 8861 if (!inode_evicting) 8862 clear_extent_bit(tree, start, end, 8863 EXTENT_DIRTY | EXTENT_DELALLOC | 8864 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 8865 EXTENT_DEFRAG, 1, 0, &cached_state, 8866 GFP_NOFS); 8867 /* 8868 * whoever cleared the private bit is responsible 8869 * for the finish_ordered_io 8870 */ 8871 if (TestClearPagePrivate2(page)) { 8872 struct btrfs_ordered_inode_tree *tree; 8873 u64 new_len; 8874 8875 tree = &BTRFS_I(inode)->ordered_tree; 8876 8877 spin_lock_irq(&tree->lock); 8878 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 8879 new_len = start - ordered->file_offset; 8880 if (new_len < ordered->truncated_len) 8881 ordered->truncated_len = new_len; 8882 spin_unlock_irq(&tree->lock); 8883 8884 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8885 start, 8886 end - start + 1, 1)) 8887 btrfs_finish_ordered_io(ordered); 8888 } 8889 btrfs_put_ordered_extent(ordered); 8890 if (!inode_evicting) { 8891 cached_state = NULL; 8892 lock_extent_bits(tree, start, end, 8893 &cached_state); 8894 } 8895 8896 start = end + 1; 8897 if (start < page_end) 8898 goto again; 8899 } 8900 8901 /* 8902 * Qgroup reserved space handler 8903 * Page here will be either 8904 * 1) Already written to disk 8905 * In this case, its reserved space is released from data rsv map 8906 * and will be freed by delayed_ref handler finally. 8907 * So even we call qgroup_free_data(), it won't decrease reserved 8908 * space. 8909 * 2) Not written to disk 8910 * This means the reserved space should be freed here. However, 8911 * if a truncate invalidates the page (by clearing PageDirty) 8912 * and the page is accounted for while allocating extent 8913 * in btrfs_check_data_free_space() we let delayed_ref to 8914 * free the entire extent. 8915 */ 8916 if (PageDirty(page)) 8917 btrfs_qgroup_free_data(inode, page_start, PAGE_SIZE); 8918 if (!inode_evicting) { 8919 clear_extent_bit(tree, page_start, page_end, 8920 EXTENT_LOCKED | EXTENT_DIRTY | 8921 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 8922 EXTENT_DEFRAG, 1, 1, 8923 &cached_state, GFP_NOFS); 8924 8925 __btrfs_releasepage(page, GFP_NOFS); 8926 } 8927 8928 ClearPageChecked(page); 8929 if (PagePrivate(page)) { 8930 ClearPagePrivate(page); 8931 set_page_private(page, 0); 8932 put_page(page); 8933 } 8934 } 8935 8936 /* 8937 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8938 * called from a page fault handler when a page is first dirtied. Hence we must 8939 * be careful to check for EOF conditions here. We set the page up correctly 8940 * for a written page which means we get ENOSPC checking when writing into 8941 * holes and correct delalloc and unwritten extent mapping on filesystems that 8942 * support these features. 8943 * 8944 * We are not allowed to take the i_mutex here so we have to play games to 8945 * protect against truncate races as the page could now be beyond EOF. Because 8946 * vmtruncate() writes the inode size before removing pages, once we have the 8947 * page lock we can determine safely if the page is beyond EOF. If it is not 8948 * beyond EOF, then the page is guaranteed safe against truncation until we 8949 * unlock the page. 8950 */ 8951 int btrfs_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 8952 { 8953 struct page *page = vmf->page; 8954 struct inode *inode = file_inode(vma->vm_file); 8955 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8956 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8957 struct btrfs_ordered_extent *ordered; 8958 struct extent_state *cached_state = NULL; 8959 char *kaddr; 8960 unsigned long zero_start; 8961 loff_t size; 8962 int ret; 8963 int reserved = 0; 8964 u64 reserved_space; 8965 u64 page_start; 8966 u64 page_end; 8967 u64 end; 8968 8969 reserved_space = PAGE_SIZE; 8970 8971 sb_start_pagefault(inode->i_sb); 8972 page_start = page_offset(page); 8973 page_end = page_start + PAGE_SIZE - 1; 8974 end = page_end; 8975 8976 /* 8977 * Reserving delalloc space after obtaining the page lock can lead to 8978 * deadlock. For example, if a dirty page is locked by this function 8979 * and the call to btrfs_delalloc_reserve_space() ends up triggering 8980 * dirty page write out, then the btrfs_writepage() function could 8981 * end up waiting indefinitely to get a lock on the page currently 8982 * being processed by btrfs_page_mkwrite() function. 8983 */ 8984 ret = btrfs_delalloc_reserve_space(inode, page_start, 8985 reserved_space); 8986 if (!ret) { 8987 ret = file_update_time(vma->vm_file); 8988 reserved = 1; 8989 } 8990 if (ret) { 8991 if (ret == -ENOMEM) 8992 ret = VM_FAULT_OOM; 8993 else /* -ENOSPC, -EIO, etc */ 8994 ret = VM_FAULT_SIGBUS; 8995 if (reserved) 8996 goto out; 8997 goto out_noreserve; 8998 } 8999 9000 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 9001 again: 9002 lock_page(page); 9003 size = i_size_read(inode); 9004 9005 if ((page->mapping != inode->i_mapping) || 9006 (page_start >= size)) { 9007 /* page got truncated out from underneath us */ 9008 goto out_unlock; 9009 } 9010 wait_on_page_writeback(page); 9011 9012 lock_extent_bits(io_tree, page_start, page_end, &cached_state); 9013 set_page_extent_mapped(page); 9014 9015 /* 9016 * we can't set the delalloc bits if there are pending ordered 9017 * extents. Drop our locks and wait for them to finish 9018 */ 9019 ordered = btrfs_lookup_ordered_range(inode, page_start, page_end); 9020 if (ordered) { 9021 unlock_extent_cached(io_tree, page_start, page_end, 9022 &cached_state, GFP_NOFS); 9023 unlock_page(page); 9024 btrfs_start_ordered_extent(inode, ordered, 1); 9025 btrfs_put_ordered_extent(ordered); 9026 goto again; 9027 } 9028 9029 if (page->index == ((size - 1) >> PAGE_SHIFT)) { 9030 reserved_space = round_up(size - page_start, 9031 fs_info->sectorsize); 9032 if (reserved_space < PAGE_SIZE) { 9033 end = page_start + reserved_space - 1; 9034 spin_lock(&BTRFS_I(inode)->lock); 9035 BTRFS_I(inode)->outstanding_extents++; 9036 spin_unlock(&BTRFS_I(inode)->lock); 9037 btrfs_delalloc_release_space(inode, page_start, 9038 PAGE_SIZE - reserved_space); 9039 } 9040 } 9041 9042 /* 9043 * XXX - page_mkwrite gets called every time the page is dirtied, even 9044 * if it was already dirty, so for space accounting reasons we need to 9045 * clear any delalloc bits for the range we are fixing to save. There 9046 * is probably a better way to do this, but for now keep consistent with 9047 * prepare_pages in the normal write path. 9048 */ 9049 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, 9050 EXTENT_DIRTY | EXTENT_DELALLOC | 9051 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 9052 0, 0, &cached_state, GFP_NOFS); 9053 9054 ret = btrfs_set_extent_delalloc(inode, page_start, end, 9055 &cached_state, 0); 9056 if (ret) { 9057 unlock_extent_cached(io_tree, page_start, page_end, 9058 &cached_state, GFP_NOFS); 9059 ret = VM_FAULT_SIGBUS; 9060 goto out_unlock; 9061 } 9062 ret = 0; 9063 9064 /* page is wholly or partially inside EOF */ 9065 if (page_start + PAGE_SIZE > size) 9066 zero_start = size & ~PAGE_MASK; 9067 else 9068 zero_start = PAGE_SIZE; 9069 9070 if (zero_start != PAGE_SIZE) { 9071 kaddr = kmap(page); 9072 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start); 9073 flush_dcache_page(page); 9074 kunmap(page); 9075 } 9076 ClearPageChecked(page); 9077 set_page_dirty(page); 9078 SetPageUptodate(page); 9079 9080 BTRFS_I(inode)->last_trans = fs_info->generation; 9081 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; 9082 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; 9083 9084 unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS); 9085 9086 out_unlock: 9087 if (!ret) { 9088 sb_end_pagefault(inode->i_sb); 9089 return VM_FAULT_LOCKED; 9090 } 9091 unlock_page(page); 9092 out: 9093 btrfs_delalloc_release_space(inode, page_start, reserved_space); 9094 out_noreserve: 9095 sb_end_pagefault(inode->i_sb); 9096 return ret; 9097 } 9098 9099 static int btrfs_truncate(struct inode *inode) 9100 { 9101 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9102 struct btrfs_root *root = BTRFS_I(inode)->root; 9103 struct btrfs_block_rsv *rsv; 9104 int ret = 0; 9105 int err = 0; 9106 struct btrfs_trans_handle *trans; 9107 u64 mask = fs_info->sectorsize - 1; 9108 u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1); 9109 9110 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), 9111 (u64)-1); 9112 if (ret) 9113 return ret; 9114 9115 /* 9116 * Yes ladies and gentlemen, this is indeed ugly. The fact is we have 9117 * 3 things going on here 9118 * 9119 * 1) We need to reserve space for our orphan item and the space to 9120 * delete our orphan item. Lord knows we don't want to have a dangling 9121 * orphan item because we didn't reserve space to remove it. 9122 * 9123 * 2) We need to reserve space to update our inode. 9124 * 9125 * 3) We need to have something to cache all the space that is going to 9126 * be free'd up by the truncate operation, but also have some slack 9127 * space reserved in case it uses space during the truncate (thank you 9128 * very much snapshotting). 9129 * 9130 * And we need these to all be separate. The fact is we can use a lot of 9131 * space doing the truncate, and we have no earthly idea how much space 9132 * we will use, so we need the truncate reservation to be separate so it 9133 * doesn't end up using space reserved for updating the inode or 9134 * removing the orphan item. We also need to be able to stop the 9135 * transaction and start a new one, which means we need to be able to 9136 * update the inode several times, and we have no idea of knowing how 9137 * many times that will be, so we can't just reserve 1 item for the 9138 * entirety of the operation, so that has to be done separately as well. 9139 * Then there is the orphan item, which does indeed need to be held on 9140 * to for the whole operation, and we need nobody to touch this reserved 9141 * space except the orphan code. 9142 * 9143 * So that leaves us with 9144 * 9145 * 1) root->orphan_block_rsv - for the orphan deletion. 9146 * 2) rsv - for the truncate reservation, which we will steal from the 9147 * transaction reservation. 9148 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for 9149 * updating the inode. 9150 */ 9151 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 9152 if (!rsv) 9153 return -ENOMEM; 9154 rsv->size = min_size; 9155 rsv->failfast = 1; 9156 9157 /* 9158 * 1 for the truncate slack space 9159 * 1 for updating the inode. 9160 */ 9161 trans = btrfs_start_transaction(root, 2); 9162 if (IS_ERR(trans)) { 9163 err = PTR_ERR(trans); 9164 goto out; 9165 } 9166 9167 /* Migrate the slack space for the truncate to our reserve */ 9168 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv, 9169 min_size, 0); 9170 BUG_ON(ret); 9171 9172 /* 9173 * So if we truncate and then write and fsync we normally would just 9174 * write the extents that changed, which is a problem if we need to 9175 * first truncate that entire inode. So set this flag so we write out 9176 * all of the extents in the inode to the sync log so we're completely 9177 * safe. 9178 */ 9179 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 9180 trans->block_rsv = rsv; 9181 9182 while (1) { 9183 ret = btrfs_truncate_inode_items(trans, root, inode, 9184 inode->i_size, 9185 BTRFS_EXTENT_DATA_KEY); 9186 if (ret != -ENOSPC && ret != -EAGAIN) { 9187 err = ret; 9188 break; 9189 } 9190 9191 trans->block_rsv = &fs_info->trans_block_rsv; 9192 ret = btrfs_update_inode(trans, root, inode); 9193 if (ret) { 9194 err = ret; 9195 break; 9196 } 9197 9198 btrfs_end_transaction(trans); 9199 btrfs_btree_balance_dirty(fs_info); 9200 9201 trans = btrfs_start_transaction(root, 2); 9202 if (IS_ERR(trans)) { 9203 ret = err = PTR_ERR(trans); 9204 trans = NULL; 9205 break; 9206 } 9207 9208 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 9209 rsv, min_size, 0); 9210 BUG_ON(ret); /* shouldn't happen */ 9211 trans->block_rsv = rsv; 9212 } 9213 9214 if (ret == 0 && inode->i_nlink > 0) { 9215 trans->block_rsv = root->orphan_block_rsv; 9216 ret = btrfs_orphan_del(trans, inode); 9217 if (ret) 9218 err = ret; 9219 } 9220 9221 if (trans) { 9222 trans->block_rsv = &fs_info->trans_block_rsv; 9223 ret = btrfs_update_inode(trans, root, inode); 9224 if (ret && !err) 9225 err = ret; 9226 9227 ret = btrfs_end_transaction(trans); 9228 btrfs_btree_balance_dirty(fs_info); 9229 } 9230 out: 9231 btrfs_free_block_rsv(fs_info, rsv); 9232 9233 if (ret && !err) 9234 err = ret; 9235 9236 return err; 9237 } 9238 9239 /* 9240 * create a new subvolume directory/inode (helper for the ioctl). 9241 */ 9242 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, 9243 struct btrfs_root *new_root, 9244 struct btrfs_root *parent_root, 9245 u64 new_dirid) 9246 { 9247 struct inode *inode; 9248 int err; 9249 u64 index = 0; 9250 9251 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, 9252 new_dirid, new_dirid, 9253 S_IFDIR | (~current_umask() & S_IRWXUGO), 9254 &index); 9255 if (IS_ERR(inode)) 9256 return PTR_ERR(inode); 9257 inode->i_op = &btrfs_dir_inode_operations; 9258 inode->i_fop = &btrfs_dir_file_operations; 9259 9260 set_nlink(inode, 1); 9261 btrfs_i_size_write(inode, 0); 9262 unlock_new_inode(inode); 9263 9264 err = btrfs_subvol_inherit_props(trans, new_root, parent_root); 9265 if (err) 9266 btrfs_err(new_root->fs_info, 9267 "error inheriting subvolume %llu properties: %d", 9268 new_root->root_key.objectid, err); 9269 9270 err = btrfs_update_inode(trans, new_root, inode); 9271 9272 iput(inode); 9273 return err; 9274 } 9275 9276 struct inode *btrfs_alloc_inode(struct super_block *sb) 9277 { 9278 struct btrfs_inode *ei; 9279 struct inode *inode; 9280 9281 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS); 9282 if (!ei) 9283 return NULL; 9284 9285 ei->root = NULL; 9286 ei->generation = 0; 9287 ei->last_trans = 0; 9288 ei->last_sub_trans = 0; 9289 ei->logged_trans = 0; 9290 ei->delalloc_bytes = 0; 9291 ei->defrag_bytes = 0; 9292 ei->disk_i_size = 0; 9293 ei->flags = 0; 9294 ei->csum_bytes = 0; 9295 ei->index_cnt = (u64)-1; 9296 ei->dir_index = 0; 9297 ei->last_unlink_trans = 0; 9298 ei->last_log_commit = 0; 9299 ei->delayed_iput_count = 0; 9300 9301 spin_lock_init(&ei->lock); 9302 ei->outstanding_extents = 0; 9303 ei->reserved_extents = 0; 9304 9305 ei->runtime_flags = 0; 9306 ei->force_compress = BTRFS_COMPRESS_NONE; 9307 9308 ei->delayed_node = NULL; 9309 9310 ei->i_otime.tv_sec = 0; 9311 ei->i_otime.tv_nsec = 0; 9312 9313 inode = &ei->vfs_inode; 9314 extent_map_tree_init(&ei->extent_tree); 9315 extent_io_tree_init(&ei->io_tree, &inode->i_data); 9316 extent_io_tree_init(&ei->io_failure_tree, &inode->i_data); 9317 ei->io_tree.track_uptodate = 1; 9318 ei->io_failure_tree.track_uptodate = 1; 9319 atomic_set(&ei->sync_writers, 0); 9320 mutex_init(&ei->log_mutex); 9321 mutex_init(&ei->delalloc_mutex); 9322 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 9323 INIT_LIST_HEAD(&ei->delalloc_inodes); 9324 INIT_LIST_HEAD(&ei->delayed_iput); 9325 RB_CLEAR_NODE(&ei->rb_node); 9326 init_rwsem(&ei->dio_sem); 9327 9328 return inode; 9329 } 9330 9331 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 9332 void btrfs_test_destroy_inode(struct inode *inode) 9333 { 9334 btrfs_drop_extent_cache(inode, 0, (u64)-1, 0); 9335 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9336 } 9337 #endif 9338 9339 static void btrfs_i_callback(struct rcu_head *head) 9340 { 9341 struct inode *inode = container_of(head, struct inode, i_rcu); 9342 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 9343 } 9344 9345 void btrfs_destroy_inode(struct inode *inode) 9346 { 9347 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9348 struct btrfs_ordered_extent *ordered; 9349 struct btrfs_root *root = BTRFS_I(inode)->root; 9350 9351 WARN_ON(!hlist_empty(&inode->i_dentry)); 9352 WARN_ON(inode->i_data.nrpages); 9353 WARN_ON(BTRFS_I(inode)->outstanding_extents); 9354 WARN_ON(BTRFS_I(inode)->reserved_extents); 9355 WARN_ON(BTRFS_I(inode)->delalloc_bytes); 9356 WARN_ON(BTRFS_I(inode)->csum_bytes); 9357 WARN_ON(BTRFS_I(inode)->defrag_bytes); 9358 9359 /* 9360 * This can happen where we create an inode, but somebody else also 9361 * created the same inode and we need to destroy the one we already 9362 * created. 9363 */ 9364 if (!root) 9365 goto free; 9366 9367 if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 9368 &BTRFS_I(inode)->runtime_flags)) { 9369 btrfs_info(fs_info, "inode %llu still on the orphan list", 9370 btrfs_ino(inode)); 9371 atomic_dec(&root->orphan_inodes); 9372 } 9373 9374 while (1) { 9375 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 9376 if (!ordered) 9377 break; 9378 else { 9379 btrfs_err(fs_info, 9380 "found ordered extent %llu %llu on inode cleanup", 9381 ordered->file_offset, ordered->len); 9382 btrfs_remove_ordered_extent(inode, ordered); 9383 btrfs_put_ordered_extent(ordered); 9384 btrfs_put_ordered_extent(ordered); 9385 } 9386 } 9387 btrfs_qgroup_check_reserved_leak(inode); 9388 inode_tree_del(inode); 9389 btrfs_drop_extent_cache(inode, 0, (u64)-1, 0); 9390 free: 9391 call_rcu(&inode->i_rcu, btrfs_i_callback); 9392 } 9393 9394 int btrfs_drop_inode(struct inode *inode) 9395 { 9396 struct btrfs_root *root = BTRFS_I(inode)->root; 9397 9398 if (root == NULL) 9399 return 1; 9400 9401 /* the snap/subvol tree is on deleting */ 9402 if (btrfs_root_refs(&root->root_item) == 0) 9403 return 1; 9404 else 9405 return generic_drop_inode(inode); 9406 } 9407 9408 static void init_once(void *foo) 9409 { 9410 struct btrfs_inode *ei = (struct btrfs_inode *) foo; 9411 9412 inode_init_once(&ei->vfs_inode); 9413 } 9414 9415 void btrfs_destroy_cachep(void) 9416 { 9417 /* 9418 * Make sure all delayed rcu free inodes are flushed before we 9419 * destroy cache. 9420 */ 9421 rcu_barrier(); 9422 kmem_cache_destroy(btrfs_inode_cachep); 9423 kmem_cache_destroy(btrfs_trans_handle_cachep); 9424 kmem_cache_destroy(btrfs_transaction_cachep); 9425 kmem_cache_destroy(btrfs_path_cachep); 9426 kmem_cache_destroy(btrfs_free_space_cachep); 9427 } 9428 9429 int btrfs_init_cachep(void) 9430 { 9431 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 9432 sizeof(struct btrfs_inode), 0, 9433 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, 9434 init_once); 9435 if (!btrfs_inode_cachep) 9436 goto fail; 9437 9438 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", 9439 sizeof(struct btrfs_trans_handle), 0, 9440 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); 9441 if (!btrfs_trans_handle_cachep) 9442 goto fail; 9443 9444 btrfs_transaction_cachep = kmem_cache_create("btrfs_transaction", 9445 sizeof(struct btrfs_transaction), 0, 9446 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); 9447 if (!btrfs_transaction_cachep) 9448 goto fail; 9449 9450 btrfs_path_cachep = kmem_cache_create("btrfs_path", 9451 sizeof(struct btrfs_path), 0, 9452 SLAB_MEM_SPREAD, NULL); 9453 if (!btrfs_path_cachep) 9454 goto fail; 9455 9456 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space", 9457 sizeof(struct btrfs_free_space), 0, 9458 SLAB_MEM_SPREAD, NULL); 9459 if (!btrfs_free_space_cachep) 9460 goto fail; 9461 9462 return 0; 9463 fail: 9464 btrfs_destroy_cachep(); 9465 return -ENOMEM; 9466 } 9467 9468 static int btrfs_getattr(struct vfsmount *mnt, 9469 struct dentry *dentry, struct kstat *stat) 9470 { 9471 u64 delalloc_bytes; 9472 struct inode *inode = d_inode(dentry); 9473 u32 blocksize = inode->i_sb->s_blocksize; 9474 9475 generic_fillattr(inode, stat); 9476 stat->dev = BTRFS_I(inode)->root->anon_dev; 9477 9478 spin_lock(&BTRFS_I(inode)->lock); 9479 delalloc_bytes = BTRFS_I(inode)->delalloc_bytes; 9480 spin_unlock(&BTRFS_I(inode)->lock); 9481 stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) + 9482 ALIGN(delalloc_bytes, blocksize)) >> 9; 9483 return 0; 9484 } 9485 9486 static int btrfs_rename_exchange(struct inode *old_dir, 9487 struct dentry *old_dentry, 9488 struct inode *new_dir, 9489 struct dentry *new_dentry) 9490 { 9491 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9492 struct btrfs_trans_handle *trans; 9493 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9494 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9495 struct inode *new_inode = new_dentry->d_inode; 9496 struct inode *old_inode = old_dentry->d_inode; 9497 struct timespec ctime = current_time(old_inode); 9498 struct dentry *parent; 9499 u64 old_ino = btrfs_ino(old_inode); 9500 u64 new_ino = btrfs_ino(new_inode); 9501 u64 old_idx = 0; 9502 u64 new_idx = 0; 9503 u64 root_objectid; 9504 int ret; 9505 bool root_log_pinned = false; 9506 bool dest_log_pinned = false; 9507 9508 /* we only allow rename subvolume link between subvolumes */ 9509 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9510 return -EXDEV; 9511 9512 /* close the race window with snapshot create/destroy ioctl */ 9513 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9514 down_read(&fs_info->subvol_sem); 9515 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9516 down_read(&fs_info->subvol_sem); 9517 9518 /* 9519 * We want to reserve the absolute worst case amount of items. So if 9520 * both inodes are subvols and we need to unlink them then that would 9521 * require 4 item modifications, but if they are both normal inodes it 9522 * would require 5 item modifications, so we'll assume their normal 9523 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items 9524 * should cover the worst case number of items we'll modify. 9525 */ 9526 trans = btrfs_start_transaction(root, 12); 9527 if (IS_ERR(trans)) { 9528 ret = PTR_ERR(trans); 9529 goto out_notrans; 9530 } 9531 9532 /* 9533 * We need to find a free sequence number both in the source and 9534 * in the destination directory for the exchange. 9535 */ 9536 ret = btrfs_set_inode_index(new_dir, &old_idx); 9537 if (ret) 9538 goto out_fail; 9539 ret = btrfs_set_inode_index(old_dir, &new_idx); 9540 if (ret) 9541 goto out_fail; 9542 9543 BTRFS_I(old_inode)->dir_index = 0ULL; 9544 BTRFS_I(new_inode)->dir_index = 0ULL; 9545 9546 /* Reference for the source. */ 9547 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9548 /* force full log commit if subvolume involved. */ 9549 btrfs_set_log_full_commit(fs_info, trans); 9550 } else { 9551 btrfs_pin_log_trans(root); 9552 root_log_pinned = true; 9553 ret = btrfs_insert_inode_ref(trans, dest, 9554 new_dentry->d_name.name, 9555 new_dentry->d_name.len, 9556 old_ino, 9557 btrfs_ino(new_dir), old_idx); 9558 if (ret) 9559 goto out_fail; 9560 } 9561 9562 /* And now for the dest. */ 9563 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9564 /* force full log commit if subvolume involved. */ 9565 btrfs_set_log_full_commit(fs_info, trans); 9566 } else { 9567 btrfs_pin_log_trans(dest); 9568 dest_log_pinned = true; 9569 ret = btrfs_insert_inode_ref(trans, root, 9570 old_dentry->d_name.name, 9571 old_dentry->d_name.len, 9572 new_ino, 9573 btrfs_ino(old_dir), new_idx); 9574 if (ret) 9575 goto out_fail; 9576 } 9577 9578 /* Update inode version and ctime/mtime. */ 9579 inode_inc_iversion(old_dir); 9580 inode_inc_iversion(new_dir); 9581 inode_inc_iversion(old_inode); 9582 inode_inc_iversion(new_inode); 9583 old_dir->i_ctime = old_dir->i_mtime = ctime; 9584 new_dir->i_ctime = new_dir->i_mtime = ctime; 9585 old_inode->i_ctime = ctime; 9586 new_inode->i_ctime = ctime; 9587 9588 if (old_dentry->d_parent != new_dentry->d_parent) { 9589 btrfs_record_unlink_dir(trans, old_dir, old_inode, 1); 9590 btrfs_record_unlink_dir(trans, new_dir, new_inode, 1); 9591 } 9592 9593 /* src is a subvolume */ 9594 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9595 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9596 ret = btrfs_unlink_subvol(trans, root, old_dir, 9597 root_objectid, 9598 old_dentry->d_name.name, 9599 old_dentry->d_name.len); 9600 } else { /* src is an inode */ 9601 ret = __btrfs_unlink_inode(trans, root, old_dir, 9602 old_dentry->d_inode, 9603 old_dentry->d_name.name, 9604 old_dentry->d_name.len); 9605 if (!ret) 9606 ret = btrfs_update_inode(trans, root, old_inode); 9607 } 9608 if (ret) { 9609 btrfs_abort_transaction(trans, ret); 9610 goto out_fail; 9611 } 9612 9613 /* dest is a subvolume */ 9614 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9615 root_objectid = BTRFS_I(new_inode)->root->root_key.objectid; 9616 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9617 root_objectid, 9618 new_dentry->d_name.name, 9619 new_dentry->d_name.len); 9620 } else { /* dest is an inode */ 9621 ret = __btrfs_unlink_inode(trans, dest, new_dir, 9622 new_dentry->d_inode, 9623 new_dentry->d_name.name, 9624 new_dentry->d_name.len); 9625 if (!ret) 9626 ret = btrfs_update_inode(trans, dest, new_inode); 9627 } 9628 if (ret) { 9629 btrfs_abort_transaction(trans, ret); 9630 goto out_fail; 9631 } 9632 9633 ret = btrfs_add_link(trans, new_dir, old_inode, 9634 new_dentry->d_name.name, 9635 new_dentry->d_name.len, 0, old_idx); 9636 if (ret) { 9637 btrfs_abort_transaction(trans, ret); 9638 goto out_fail; 9639 } 9640 9641 ret = btrfs_add_link(trans, old_dir, new_inode, 9642 old_dentry->d_name.name, 9643 old_dentry->d_name.len, 0, new_idx); 9644 if (ret) { 9645 btrfs_abort_transaction(trans, ret); 9646 goto out_fail; 9647 } 9648 9649 if (old_inode->i_nlink == 1) 9650 BTRFS_I(old_inode)->dir_index = old_idx; 9651 if (new_inode->i_nlink == 1) 9652 BTRFS_I(new_inode)->dir_index = new_idx; 9653 9654 if (root_log_pinned) { 9655 parent = new_dentry->d_parent; 9656 btrfs_log_new_name(trans, old_inode, old_dir, parent); 9657 btrfs_end_log_trans(root); 9658 root_log_pinned = false; 9659 } 9660 if (dest_log_pinned) { 9661 parent = old_dentry->d_parent; 9662 btrfs_log_new_name(trans, new_inode, new_dir, parent); 9663 btrfs_end_log_trans(dest); 9664 dest_log_pinned = false; 9665 } 9666 out_fail: 9667 /* 9668 * If we have pinned a log and an error happened, we unpin tasks 9669 * trying to sync the log and force them to fallback to a transaction 9670 * commit if the log currently contains any of the inodes involved in 9671 * this rename operation (to ensure we do not persist a log with an 9672 * inconsistent state for any of these inodes or leading to any 9673 * inconsistencies when replayed). If the transaction was aborted, the 9674 * abortion reason is propagated to userspace when attempting to commit 9675 * the transaction. If the log does not contain any of these inodes, we 9676 * allow the tasks to sync it. 9677 */ 9678 if (ret && (root_log_pinned || dest_log_pinned)) { 9679 if (btrfs_inode_in_log(old_dir, fs_info->generation) || 9680 btrfs_inode_in_log(new_dir, fs_info->generation) || 9681 btrfs_inode_in_log(old_inode, fs_info->generation) || 9682 (new_inode && 9683 btrfs_inode_in_log(new_inode, fs_info->generation))) 9684 btrfs_set_log_full_commit(fs_info, trans); 9685 9686 if (root_log_pinned) { 9687 btrfs_end_log_trans(root); 9688 root_log_pinned = false; 9689 } 9690 if (dest_log_pinned) { 9691 btrfs_end_log_trans(dest); 9692 dest_log_pinned = false; 9693 } 9694 } 9695 ret = btrfs_end_transaction(trans); 9696 out_notrans: 9697 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 9698 up_read(&fs_info->subvol_sem); 9699 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9700 up_read(&fs_info->subvol_sem); 9701 9702 return ret; 9703 } 9704 9705 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans, 9706 struct btrfs_root *root, 9707 struct inode *dir, 9708 struct dentry *dentry) 9709 { 9710 int ret; 9711 struct inode *inode; 9712 u64 objectid; 9713 u64 index; 9714 9715 ret = btrfs_find_free_ino(root, &objectid); 9716 if (ret) 9717 return ret; 9718 9719 inode = btrfs_new_inode(trans, root, dir, 9720 dentry->d_name.name, 9721 dentry->d_name.len, 9722 btrfs_ino(dir), 9723 objectid, 9724 S_IFCHR | WHITEOUT_MODE, 9725 &index); 9726 9727 if (IS_ERR(inode)) { 9728 ret = PTR_ERR(inode); 9729 return ret; 9730 } 9731 9732 inode->i_op = &btrfs_special_inode_operations; 9733 init_special_inode(inode, inode->i_mode, 9734 WHITEOUT_DEV); 9735 9736 ret = btrfs_init_inode_security(trans, inode, dir, 9737 &dentry->d_name); 9738 if (ret) 9739 goto out; 9740 9741 ret = btrfs_add_nondir(trans, dir, dentry, 9742 inode, 0, index); 9743 if (ret) 9744 goto out; 9745 9746 ret = btrfs_update_inode(trans, root, inode); 9747 out: 9748 unlock_new_inode(inode); 9749 if (ret) 9750 inode_dec_link_count(inode); 9751 iput(inode); 9752 9753 return ret; 9754 } 9755 9756 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry, 9757 struct inode *new_dir, struct dentry *new_dentry, 9758 unsigned int flags) 9759 { 9760 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9761 struct btrfs_trans_handle *trans; 9762 unsigned int trans_num_items; 9763 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9764 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9765 struct inode *new_inode = d_inode(new_dentry); 9766 struct inode *old_inode = d_inode(old_dentry); 9767 u64 index = 0; 9768 u64 root_objectid; 9769 int ret; 9770 u64 old_ino = btrfs_ino(old_inode); 9771 bool log_pinned = false; 9772 9773 if (btrfs_ino(new_dir) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 9774 return -EPERM; 9775 9776 /* we only allow rename subvolume link between subvolumes */ 9777 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9778 return -EXDEV; 9779 9780 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 9781 (new_inode && btrfs_ino(new_inode) == BTRFS_FIRST_FREE_OBJECTID)) 9782 return -ENOTEMPTY; 9783 9784 if (S_ISDIR(old_inode->i_mode) && new_inode && 9785 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 9786 return -ENOTEMPTY; 9787 9788 9789 /* check for collisions, even if the name isn't there */ 9790 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, 9791 new_dentry->d_name.name, 9792 new_dentry->d_name.len); 9793 9794 if (ret) { 9795 if (ret == -EEXIST) { 9796 /* we shouldn't get 9797 * eexist without a new_inode */ 9798 if (WARN_ON(!new_inode)) { 9799 return ret; 9800 } 9801 } else { 9802 /* maybe -EOVERFLOW */ 9803 return ret; 9804 } 9805 } 9806 ret = 0; 9807 9808 /* 9809 * we're using rename to replace one file with another. Start IO on it 9810 * now so we don't add too much work to the end of the transaction 9811 */ 9812 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 9813 filemap_flush(old_inode->i_mapping); 9814 9815 /* close the racy window with snapshot create/destroy ioctl */ 9816 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9817 down_read(&fs_info->subvol_sem); 9818 /* 9819 * We want to reserve the absolute worst case amount of items. So if 9820 * both inodes are subvols and we need to unlink them then that would 9821 * require 4 item modifications, but if they are both normal inodes it 9822 * would require 5 item modifications, so we'll assume they are normal 9823 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items 9824 * should cover the worst case number of items we'll modify. 9825 * If our rename has the whiteout flag, we need more 5 units for the 9826 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item 9827 * when selinux is enabled). 9828 */ 9829 trans_num_items = 11; 9830 if (flags & RENAME_WHITEOUT) 9831 trans_num_items += 5; 9832 trans = btrfs_start_transaction(root, trans_num_items); 9833 if (IS_ERR(trans)) { 9834 ret = PTR_ERR(trans); 9835 goto out_notrans; 9836 } 9837 9838 if (dest != root) 9839 btrfs_record_root_in_trans(trans, dest); 9840 9841 ret = btrfs_set_inode_index(new_dir, &index); 9842 if (ret) 9843 goto out_fail; 9844 9845 BTRFS_I(old_inode)->dir_index = 0ULL; 9846 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9847 /* force full log commit if subvolume involved. */ 9848 btrfs_set_log_full_commit(fs_info, trans); 9849 } else { 9850 btrfs_pin_log_trans(root); 9851 log_pinned = true; 9852 ret = btrfs_insert_inode_ref(trans, dest, 9853 new_dentry->d_name.name, 9854 new_dentry->d_name.len, 9855 old_ino, 9856 btrfs_ino(new_dir), index); 9857 if (ret) 9858 goto out_fail; 9859 } 9860 9861 inode_inc_iversion(old_dir); 9862 inode_inc_iversion(new_dir); 9863 inode_inc_iversion(old_inode); 9864 old_dir->i_ctime = old_dir->i_mtime = 9865 new_dir->i_ctime = new_dir->i_mtime = 9866 old_inode->i_ctime = current_time(old_dir); 9867 9868 if (old_dentry->d_parent != new_dentry->d_parent) 9869 btrfs_record_unlink_dir(trans, old_dir, old_inode, 1); 9870 9871 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9872 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 9873 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid, 9874 old_dentry->d_name.name, 9875 old_dentry->d_name.len); 9876 } else { 9877 ret = __btrfs_unlink_inode(trans, root, old_dir, 9878 d_inode(old_dentry), 9879 old_dentry->d_name.name, 9880 old_dentry->d_name.len); 9881 if (!ret) 9882 ret = btrfs_update_inode(trans, root, old_inode); 9883 } 9884 if (ret) { 9885 btrfs_abort_transaction(trans, ret); 9886 goto out_fail; 9887 } 9888 9889 if (new_inode) { 9890 inode_inc_iversion(new_inode); 9891 new_inode->i_ctime = current_time(new_inode); 9892 if (unlikely(btrfs_ino(new_inode) == 9893 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 9894 root_objectid = BTRFS_I(new_inode)->location.objectid; 9895 ret = btrfs_unlink_subvol(trans, dest, new_dir, 9896 root_objectid, 9897 new_dentry->d_name.name, 9898 new_dentry->d_name.len); 9899 BUG_ON(new_inode->i_nlink == 0); 9900 } else { 9901 ret = btrfs_unlink_inode(trans, dest, new_dir, 9902 d_inode(new_dentry), 9903 new_dentry->d_name.name, 9904 new_dentry->d_name.len); 9905 } 9906 if (!ret && new_inode->i_nlink == 0) 9907 ret = btrfs_orphan_add(trans, d_inode(new_dentry)); 9908 if (ret) { 9909 btrfs_abort_transaction(trans, ret); 9910 goto out_fail; 9911 } 9912 } 9913 9914 ret = btrfs_add_link(trans, new_dir, old_inode, 9915 new_dentry->d_name.name, 9916 new_dentry->d_name.len, 0, index); 9917 if (ret) { 9918 btrfs_abort_transaction(trans, ret); 9919 goto out_fail; 9920 } 9921 9922 if (old_inode->i_nlink == 1) 9923 BTRFS_I(old_inode)->dir_index = index; 9924 9925 if (log_pinned) { 9926 struct dentry *parent = new_dentry->d_parent; 9927 9928 btrfs_log_new_name(trans, old_inode, old_dir, parent); 9929 btrfs_end_log_trans(root); 9930 log_pinned = false; 9931 } 9932 9933 if (flags & RENAME_WHITEOUT) { 9934 ret = btrfs_whiteout_for_rename(trans, root, old_dir, 9935 old_dentry); 9936 9937 if (ret) { 9938 btrfs_abort_transaction(trans, ret); 9939 goto out_fail; 9940 } 9941 } 9942 out_fail: 9943 /* 9944 * If we have pinned the log and an error happened, we unpin tasks 9945 * trying to sync the log and force them to fallback to a transaction 9946 * commit if the log currently contains any of the inodes involved in 9947 * this rename operation (to ensure we do not persist a log with an 9948 * inconsistent state for any of these inodes or leading to any 9949 * inconsistencies when replayed). If the transaction was aborted, the 9950 * abortion reason is propagated to userspace when attempting to commit 9951 * the transaction. If the log does not contain any of these inodes, we 9952 * allow the tasks to sync it. 9953 */ 9954 if (ret && log_pinned) { 9955 if (btrfs_inode_in_log(old_dir, fs_info->generation) || 9956 btrfs_inode_in_log(new_dir, fs_info->generation) || 9957 btrfs_inode_in_log(old_inode, fs_info->generation) || 9958 (new_inode && 9959 btrfs_inode_in_log(new_inode, fs_info->generation))) 9960 btrfs_set_log_full_commit(fs_info, trans); 9961 9962 btrfs_end_log_trans(root); 9963 log_pinned = false; 9964 } 9965 btrfs_end_transaction(trans); 9966 out_notrans: 9967 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9968 up_read(&fs_info->subvol_sem); 9969 9970 return ret; 9971 } 9972 9973 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, 9974 struct inode *new_dir, struct dentry *new_dentry, 9975 unsigned int flags) 9976 { 9977 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 9978 return -EINVAL; 9979 9980 if (flags & RENAME_EXCHANGE) 9981 return btrfs_rename_exchange(old_dir, old_dentry, new_dir, 9982 new_dentry); 9983 9984 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags); 9985 } 9986 9987 static void btrfs_run_delalloc_work(struct btrfs_work *work) 9988 { 9989 struct btrfs_delalloc_work *delalloc_work; 9990 struct inode *inode; 9991 9992 delalloc_work = container_of(work, struct btrfs_delalloc_work, 9993 work); 9994 inode = delalloc_work->inode; 9995 filemap_flush(inode->i_mapping); 9996 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9997 &BTRFS_I(inode)->runtime_flags)) 9998 filemap_flush(inode->i_mapping); 9999 10000 if (delalloc_work->delay_iput) 10001 btrfs_add_delayed_iput(inode); 10002 else 10003 iput(inode); 10004 complete(&delalloc_work->completion); 10005 } 10006 10007 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode, 10008 int delay_iput) 10009 { 10010 struct btrfs_delalloc_work *work; 10011 10012 work = kmalloc(sizeof(*work), GFP_NOFS); 10013 if (!work) 10014 return NULL; 10015 10016 init_completion(&work->completion); 10017 INIT_LIST_HEAD(&work->list); 10018 work->inode = inode; 10019 work->delay_iput = delay_iput; 10020 WARN_ON_ONCE(!inode); 10021 btrfs_init_work(&work->work, btrfs_flush_delalloc_helper, 10022 btrfs_run_delalloc_work, NULL, NULL); 10023 10024 return work; 10025 } 10026 10027 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work) 10028 { 10029 wait_for_completion(&work->completion); 10030 kfree(work); 10031 } 10032 10033 /* 10034 * some fairly slow code that needs optimization. This walks the list 10035 * of all the inodes with pending delalloc and forces them to disk. 10036 */ 10037 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput, 10038 int nr) 10039 { 10040 struct btrfs_inode *binode; 10041 struct inode *inode; 10042 struct btrfs_delalloc_work *work, *next; 10043 struct list_head works; 10044 struct list_head splice; 10045 int ret = 0; 10046 10047 INIT_LIST_HEAD(&works); 10048 INIT_LIST_HEAD(&splice); 10049 10050 mutex_lock(&root->delalloc_mutex); 10051 spin_lock(&root->delalloc_lock); 10052 list_splice_init(&root->delalloc_inodes, &splice); 10053 while (!list_empty(&splice)) { 10054 binode = list_entry(splice.next, struct btrfs_inode, 10055 delalloc_inodes); 10056 10057 list_move_tail(&binode->delalloc_inodes, 10058 &root->delalloc_inodes); 10059 inode = igrab(&binode->vfs_inode); 10060 if (!inode) { 10061 cond_resched_lock(&root->delalloc_lock); 10062 continue; 10063 } 10064 spin_unlock(&root->delalloc_lock); 10065 10066 work = btrfs_alloc_delalloc_work(inode, delay_iput); 10067 if (!work) { 10068 if (delay_iput) 10069 btrfs_add_delayed_iput(inode); 10070 else 10071 iput(inode); 10072 ret = -ENOMEM; 10073 goto out; 10074 } 10075 list_add_tail(&work->list, &works); 10076 btrfs_queue_work(root->fs_info->flush_workers, 10077 &work->work); 10078 ret++; 10079 if (nr != -1 && ret >= nr) 10080 goto out; 10081 cond_resched(); 10082 spin_lock(&root->delalloc_lock); 10083 } 10084 spin_unlock(&root->delalloc_lock); 10085 10086 out: 10087 list_for_each_entry_safe(work, next, &works, list) { 10088 list_del_init(&work->list); 10089 btrfs_wait_and_free_delalloc_work(work); 10090 } 10091 10092 if (!list_empty_careful(&splice)) { 10093 spin_lock(&root->delalloc_lock); 10094 list_splice_tail(&splice, &root->delalloc_inodes); 10095 spin_unlock(&root->delalloc_lock); 10096 } 10097 mutex_unlock(&root->delalloc_mutex); 10098 return ret; 10099 } 10100 10101 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput) 10102 { 10103 struct btrfs_fs_info *fs_info = root->fs_info; 10104 int ret; 10105 10106 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10107 return -EROFS; 10108 10109 ret = __start_delalloc_inodes(root, delay_iput, -1); 10110 if (ret > 0) 10111 ret = 0; 10112 /* 10113 * the filemap_flush will queue IO into the worker threads, but 10114 * we have to make sure the IO is actually started and that 10115 * ordered extents get created before we return 10116 */ 10117 atomic_inc(&fs_info->async_submit_draining); 10118 while (atomic_read(&fs_info->nr_async_submits) || 10119 atomic_read(&fs_info->async_delalloc_pages)) { 10120 wait_event(fs_info->async_submit_wait, 10121 (atomic_read(&fs_info->nr_async_submits) == 0 && 10122 atomic_read(&fs_info->async_delalloc_pages) == 0)); 10123 } 10124 atomic_dec(&fs_info->async_submit_draining); 10125 return ret; 10126 } 10127 10128 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput, 10129 int nr) 10130 { 10131 struct btrfs_root *root; 10132 struct list_head splice; 10133 int ret; 10134 10135 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 10136 return -EROFS; 10137 10138 INIT_LIST_HEAD(&splice); 10139 10140 mutex_lock(&fs_info->delalloc_root_mutex); 10141 spin_lock(&fs_info->delalloc_root_lock); 10142 list_splice_init(&fs_info->delalloc_roots, &splice); 10143 while (!list_empty(&splice) && nr) { 10144 root = list_first_entry(&splice, struct btrfs_root, 10145 delalloc_root); 10146 root = btrfs_grab_fs_root(root); 10147 BUG_ON(!root); 10148 list_move_tail(&root->delalloc_root, 10149 &fs_info->delalloc_roots); 10150 spin_unlock(&fs_info->delalloc_root_lock); 10151 10152 ret = __start_delalloc_inodes(root, delay_iput, nr); 10153 btrfs_put_fs_root(root); 10154 if (ret < 0) 10155 goto out; 10156 10157 if (nr != -1) { 10158 nr -= ret; 10159 WARN_ON(nr < 0); 10160 } 10161 spin_lock(&fs_info->delalloc_root_lock); 10162 } 10163 spin_unlock(&fs_info->delalloc_root_lock); 10164 10165 ret = 0; 10166 atomic_inc(&fs_info->async_submit_draining); 10167 while (atomic_read(&fs_info->nr_async_submits) || 10168 atomic_read(&fs_info->async_delalloc_pages)) { 10169 wait_event(fs_info->async_submit_wait, 10170 (atomic_read(&fs_info->nr_async_submits) == 0 && 10171 atomic_read(&fs_info->async_delalloc_pages) == 0)); 10172 } 10173 atomic_dec(&fs_info->async_submit_draining); 10174 out: 10175 if (!list_empty_careful(&splice)) { 10176 spin_lock(&fs_info->delalloc_root_lock); 10177 list_splice_tail(&splice, &fs_info->delalloc_roots); 10178 spin_unlock(&fs_info->delalloc_root_lock); 10179 } 10180 mutex_unlock(&fs_info->delalloc_root_mutex); 10181 return ret; 10182 } 10183 10184 static int btrfs_symlink(struct inode *dir, struct dentry *dentry, 10185 const char *symname) 10186 { 10187 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10188 struct btrfs_trans_handle *trans; 10189 struct btrfs_root *root = BTRFS_I(dir)->root; 10190 struct btrfs_path *path; 10191 struct btrfs_key key; 10192 struct inode *inode = NULL; 10193 int err; 10194 int drop_inode = 0; 10195 u64 objectid; 10196 u64 index = 0; 10197 int name_len; 10198 int datasize; 10199 unsigned long ptr; 10200 struct btrfs_file_extent_item *ei; 10201 struct extent_buffer *leaf; 10202 10203 name_len = strlen(symname); 10204 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 10205 return -ENAMETOOLONG; 10206 10207 /* 10208 * 2 items for inode item and ref 10209 * 2 items for dir items 10210 * 1 item for updating parent inode item 10211 * 1 item for the inline extent item 10212 * 1 item for xattr if selinux is on 10213 */ 10214 trans = btrfs_start_transaction(root, 7); 10215 if (IS_ERR(trans)) 10216 return PTR_ERR(trans); 10217 10218 err = btrfs_find_free_ino(root, &objectid); 10219 if (err) 10220 goto out_unlock; 10221 10222 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 10223 dentry->d_name.len, btrfs_ino(dir), objectid, 10224 S_IFLNK|S_IRWXUGO, &index); 10225 if (IS_ERR(inode)) { 10226 err = PTR_ERR(inode); 10227 goto out_unlock; 10228 } 10229 10230 /* 10231 * If the active LSM wants to access the inode during 10232 * d_instantiate it needs these. Smack checks to see 10233 * if the filesystem supports xattrs by looking at the 10234 * ops vector. 10235 */ 10236 inode->i_fop = &btrfs_file_operations; 10237 inode->i_op = &btrfs_file_inode_operations; 10238 inode->i_mapping->a_ops = &btrfs_aops; 10239 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10240 10241 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 10242 if (err) 10243 goto out_unlock_inode; 10244 10245 path = btrfs_alloc_path(); 10246 if (!path) { 10247 err = -ENOMEM; 10248 goto out_unlock_inode; 10249 } 10250 key.objectid = btrfs_ino(inode); 10251 key.offset = 0; 10252 key.type = BTRFS_EXTENT_DATA_KEY; 10253 datasize = btrfs_file_extent_calc_inline_size(name_len); 10254 err = btrfs_insert_empty_item(trans, root, path, &key, 10255 datasize); 10256 if (err) { 10257 btrfs_free_path(path); 10258 goto out_unlock_inode; 10259 } 10260 leaf = path->nodes[0]; 10261 ei = btrfs_item_ptr(leaf, path->slots[0], 10262 struct btrfs_file_extent_item); 10263 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 10264 btrfs_set_file_extent_type(leaf, ei, 10265 BTRFS_FILE_EXTENT_INLINE); 10266 btrfs_set_file_extent_encryption(leaf, ei, 0); 10267 btrfs_set_file_extent_compression(leaf, ei, 0); 10268 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 10269 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 10270 10271 ptr = btrfs_file_extent_inline_start(ei); 10272 write_extent_buffer(leaf, symname, ptr, name_len); 10273 btrfs_mark_buffer_dirty(leaf); 10274 btrfs_free_path(path); 10275 10276 inode->i_op = &btrfs_symlink_inode_operations; 10277 inode_nohighmem(inode); 10278 inode->i_mapping->a_ops = &btrfs_symlink_aops; 10279 inode_set_bytes(inode, name_len); 10280 btrfs_i_size_write(inode, name_len); 10281 err = btrfs_update_inode(trans, root, inode); 10282 /* 10283 * Last step, add directory indexes for our symlink inode. This is the 10284 * last step to avoid extra cleanup of these indexes if an error happens 10285 * elsewhere above. 10286 */ 10287 if (!err) 10288 err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index); 10289 if (err) { 10290 drop_inode = 1; 10291 goto out_unlock_inode; 10292 } 10293 10294 unlock_new_inode(inode); 10295 d_instantiate(dentry, inode); 10296 10297 out_unlock: 10298 btrfs_end_transaction(trans); 10299 if (drop_inode) { 10300 inode_dec_link_count(inode); 10301 iput(inode); 10302 } 10303 btrfs_btree_balance_dirty(fs_info); 10304 return err; 10305 10306 out_unlock_inode: 10307 drop_inode = 1; 10308 unlock_new_inode(inode); 10309 goto out_unlock; 10310 } 10311 10312 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 10313 u64 start, u64 num_bytes, u64 min_size, 10314 loff_t actual_len, u64 *alloc_hint, 10315 struct btrfs_trans_handle *trans) 10316 { 10317 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 10318 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 10319 struct extent_map *em; 10320 struct btrfs_root *root = BTRFS_I(inode)->root; 10321 struct btrfs_key ins; 10322 u64 cur_offset = start; 10323 u64 i_size; 10324 u64 cur_bytes; 10325 u64 last_alloc = (u64)-1; 10326 int ret = 0; 10327 bool own_trans = true; 10328 u64 end = start + num_bytes - 1; 10329 10330 if (trans) 10331 own_trans = false; 10332 while (num_bytes > 0) { 10333 if (own_trans) { 10334 trans = btrfs_start_transaction(root, 3); 10335 if (IS_ERR(trans)) { 10336 ret = PTR_ERR(trans); 10337 break; 10338 } 10339 } 10340 10341 cur_bytes = min_t(u64, num_bytes, SZ_256M); 10342 cur_bytes = max(cur_bytes, min_size); 10343 /* 10344 * If we are severely fragmented we could end up with really 10345 * small allocations, so if the allocator is returning small 10346 * chunks lets make its job easier by only searching for those 10347 * sized chunks. 10348 */ 10349 cur_bytes = min(cur_bytes, last_alloc); 10350 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 10351 min_size, 0, *alloc_hint, &ins, 1, 0); 10352 if (ret) { 10353 if (own_trans) 10354 btrfs_end_transaction(trans); 10355 break; 10356 } 10357 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10358 10359 last_alloc = ins.offset; 10360 ret = insert_reserved_file_extent(trans, inode, 10361 cur_offset, ins.objectid, 10362 ins.offset, ins.offset, 10363 ins.offset, 0, 0, 0, 10364 BTRFS_FILE_EXTENT_PREALLOC); 10365 if (ret) { 10366 btrfs_free_reserved_extent(fs_info, ins.objectid, 10367 ins.offset, 0); 10368 btrfs_abort_transaction(trans, ret); 10369 if (own_trans) 10370 btrfs_end_transaction(trans); 10371 break; 10372 } 10373 10374 btrfs_drop_extent_cache(inode, cur_offset, 10375 cur_offset + ins.offset -1, 0); 10376 10377 em = alloc_extent_map(); 10378 if (!em) { 10379 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 10380 &BTRFS_I(inode)->runtime_flags); 10381 goto next; 10382 } 10383 10384 em->start = cur_offset; 10385 em->orig_start = cur_offset; 10386 em->len = ins.offset; 10387 em->block_start = ins.objectid; 10388 em->block_len = ins.offset; 10389 em->orig_block_len = ins.offset; 10390 em->ram_bytes = ins.offset; 10391 em->bdev = fs_info->fs_devices->latest_bdev; 10392 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 10393 em->generation = trans->transid; 10394 10395 while (1) { 10396 write_lock(&em_tree->lock); 10397 ret = add_extent_mapping(em_tree, em, 1); 10398 write_unlock(&em_tree->lock); 10399 if (ret != -EEXIST) 10400 break; 10401 btrfs_drop_extent_cache(inode, cur_offset, 10402 cur_offset + ins.offset - 1, 10403 0); 10404 } 10405 free_extent_map(em); 10406 next: 10407 num_bytes -= ins.offset; 10408 cur_offset += ins.offset; 10409 *alloc_hint = ins.objectid + ins.offset; 10410 10411 inode_inc_iversion(inode); 10412 inode->i_ctime = current_time(inode); 10413 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 10414 if (!(mode & FALLOC_FL_KEEP_SIZE) && 10415 (actual_len > inode->i_size) && 10416 (cur_offset > inode->i_size)) { 10417 if (cur_offset > actual_len) 10418 i_size = actual_len; 10419 else 10420 i_size = cur_offset; 10421 i_size_write(inode, i_size); 10422 btrfs_ordered_update_i_size(inode, i_size, NULL); 10423 } 10424 10425 ret = btrfs_update_inode(trans, root, inode); 10426 10427 if (ret) { 10428 btrfs_abort_transaction(trans, ret); 10429 if (own_trans) 10430 btrfs_end_transaction(trans); 10431 break; 10432 } 10433 10434 if (own_trans) 10435 btrfs_end_transaction(trans); 10436 } 10437 if (cur_offset < end) 10438 btrfs_free_reserved_data_space(inode, cur_offset, 10439 end - cur_offset + 1); 10440 return ret; 10441 } 10442 10443 int btrfs_prealloc_file_range(struct inode *inode, int mode, 10444 u64 start, u64 num_bytes, u64 min_size, 10445 loff_t actual_len, u64 *alloc_hint) 10446 { 10447 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10448 min_size, actual_len, alloc_hint, 10449 NULL); 10450 } 10451 10452 int btrfs_prealloc_file_range_trans(struct inode *inode, 10453 struct btrfs_trans_handle *trans, int mode, 10454 u64 start, u64 num_bytes, u64 min_size, 10455 loff_t actual_len, u64 *alloc_hint) 10456 { 10457 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10458 min_size, actual_len, alloc_hint, trans); 10459 } 10460 10461 static int btrfs_set_page_dirty(struct page *page) 10462 { 10463 return __set_page_dirty_nobuffers(page); 10464 } 10465 10466 static int btrfs_permission(struct inode *inode, int mask) 10467 { 10468 struct btrfs_root *root = BTRFS_I(inode)->root; 10469 umode_t mode = inode->i_mode; 10470 10471 if (mask & MAY_WRITE && 10472 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 10473 if (btrfs_root_readonly(root)) 10474 return -EROFS; 10475 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 10476 return -EACCES; 10477 } 10478 return generic_permission(inode, mask); 10479 } 10480 10481 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 10482 { 10483 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10484 struct btrfs_trans_handle *trans; 10485 struct btrfs_root *root = BTRFS_I(dir)->root; 10486 struct inode *inode = NULL; 10487 u64 objectid; 10488 u64 index; 10489 int ret = 0; 10490 10491 /* 10492 * 5 units required for adding orphan entry 10493 */ 10494 trans = btrfs_start_transaction(root, 5); 10495 if (IS_ERR(trans)) 10496 return PTR_ERR(trans); 10497 10498 ret = btrfs_find_free_ino(root, &objectid); 10499 if (ret) 10500 goto out; 10501 10502 inode = btrfs_new_inode(trans, root, dir, NULL, 0, 10503 btrfs_ino(dir), objectid, mode, &index); 10504 if (IS_ERR(inode)) { 10505 ret = PTR_ERR(inode); 10506 inode = NULL; 10507 goto out; 10508 } 10509 10510 inode->i_fop = &btrfs_file_operations; 10511 inode->i_op = &btrfs_file_inode_operations; 10512 10513 inode->i_mapping->a_ops = &btrfs_aops; 10514 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 10515 10516 ret = btrfs_init_inode_security(trans, inode, dir, NULL); 10517 if (ret) 10518 goto out_inode; 10519 10520 ret = btrfs_update_inode(trans, root, inode); 10521 if (ret) 10522 goto out_inode; 10523 ret = btrfs_orphan_add(trans, inode); 10524 if (ret) 10525 goto out_inode; 10526 10527 /* 10528 * We set number of links to 0 in btrfs_new_inode(), and here we set 10529 * it to 1 because d_tmpfile() will issue a warning if the count is 0, 10530 * through: 10531 * 10532 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 10533 */ 10534 set_nlink(inode, 1); 10535 unlock_new_inode(inode); 10536 d_tmpfile(dentry, inode); 10537 mark_inode_dirty(inode); 10538 10539 out: 10540 btrfs_end_transaction(trans); 10541 if (ret) 10542 iput(inode); 10543 btrfs_balance_delayed_items(fs_info); 10544 btrfs_btree_balance_dirty(fs_info); 10545 return ret; 10546 10547 out_inode: 10548 unlock_new_inode(inode); 10549 goto out; 10550 10551 } 10552 10553 static const struct inode_operations btrfs_dir_inode_operations = { 10554 .getattr = btrfs_getattr, 10555 .lookup = btrfs_lookup, 10556 .create = btrfs_create, 10557 .unlink = btrfs_unlink, 10558 .link = btrfs_link, 10559 .mkdir = btrfs_mkdir, 10560 .rmdir = btrfs_rmdir, 10561 .rename = btrfs_rename2, 10562 .symlink = btrfs_symlink, 10563 .setattr = btrfs_setattr, 10564 .mknod = btrfs_mknod, 10565 .listxattr = btrfs_listxattr, 10566 .permission = btrfs_permission, 10567 .get_acl = btrfs_get_acl, 10568 .set_acl = btrfs_set_acl, 10569 .update_time = btrfs_update_time, 10570 .tmpfile = btrfs_tmpfile, 10571 }; 10572 static const struct inode_operations btrfs_dir_ro_inode_operations = { 10573 .lookup = btrfs_lookup, 10574 .permission = btrfs_permission, 10575 .get_acl = btrfs_get_acl, 10576 .set_acl = btrfs_set_acl, 10577 .update_time = btrfs_update_time, 10578 }; 10579 10580 static const struct file_operations btrfs_dir_file_operations = { 10581 .llseek = generic_file_llseek, 10582 .read = generic_read_dir, 10583 .iterate_shared = btrfs_real_readdir, 10584 .unlocked_ioctl = btrfs_ioctl, 10585 #ifdef CONFIG_COMPAT 10586 .compat_ioctl = btrfs_compat_ioctl, 10587 #endif 10588 .release = btrfs_release_file, 10589 .fsync = btrfs_sync_file, 10590 }; 10591 10592 static const struct extent_io_ops btrfs_extent_io_ops = { 10593 .fill_delalloc = run_delalloc_range, 10594 .submit_bio_hook = btrfs_submit_bio_hook, 10595 .merge_bio_hook = btrfs_merge_bio_hook, 10596 .readpage_end_io_hook = btrfs_readpage_end_io_hook, 10597 .writepage_end_io_hook = btrfs_writepage_end_io_hook, 10598 .writepage_start_hook = btrfs_writepage_start_hook, 10599 .set_bit_hook = btrfs_set_bit_hook, 10600 .clear_bit_hook = btrfs_clear_bit_hook, 10601 .merge_extent_hook = btrfs_merge_extent_hook, 10602 .split_extent_hook = btrfs_split_extent_hook, 10603 }; 10604 10605 /* 10606 * btrfs doesn't support the bmap operation because swapfiles 10607 * use bmap to make a mapping of extents in the file. They assume 10608 * these extents won't change over the life of the file and they 10609 * use the bmap result to do IO directly to the drive. 10610 * 10611 * the btrfs bmap call would return logical addresses that aren't 10612 * suitable for IO and they also will change frequently as COW 10613 * operations happen. So, swapfile + btrfs == corruption. 10614 * 10615 * For now we're avoiding this by dropping bmap. 10616 */ 10617 static const struct address_space_operations btrfs_aops = { 10618 .readpage = btrfs_readpage, 10619 .writepage = btrfs_writepage, 10620 .writepages = btrfs_writepages, 10621 .readpages = btrfs_readpages, 10622 .direct_IO = btrfs_direct_IO, 10623 .invalidatepage = btrfs_invalidatepage, 10624 .releasepage = btrfs_releasepage, 10625 .set_page_dirty = btrfs_set_page_dirty, 10626 .error_remove_page = generic_error_remove_page, 10627 }; 10628 10629 static const struct address_space_operations btrfs_symlink_aops = { 10630 .readpage = btrfs_readpage, 10631 .writepage = btrfs_writepage, 10632 .invalidatepage = btrfs_invalidatepage, 10633 .releasepage = btrfs_releasepage, 10634 }; 10635 10636 static const struct inode_operations btrfs_file_inode_operations = { 10637 .getattr = btrfs_getattr, 10638 .setattr = btrfs_setattr, 10639 .listxattr = btrfs_listxattr, 10640 .permission = btrfs_permission, 10641 .fiemap = btrfs_fiemap, 10642 .get_acl = btrfs_get_acl, 10643 .set_acl = btrfs_set_acl, 10644 .update_time = btrfs_update_time, 10645 }; 10646 static const struct inode_operations btrfs_special_inode_operations = { 10647 .getattr = btrfs_getattr, 10648 .setattr = btrfs_setattr, 10649 .permission = btrfs_permission, 10650 .listxattr = btrfs_listxattr, 10651 .get_acl = btrfs_get_acl, 10652 .set_acl = btrfs_set_acl, 10653 .update_time = btrfs_update_time, 10654 }; 10655 static const struct inode_operations btrfs_symlink_inode_operations = { 10656 .get_link = page_get_link, 10657 .getattr = btrfs_getattr, 10658 .setattr = btrfs_setattr, 10659 .permission = btrfs_permission, 10660 .listxattr = btrfs_listxattr, 10661 .update_time = btrfs_update_time, 10662 }; 10663 10664 const struct dentry_operations btrfs_dentry_operations = { 10665 .d_delete = btrfs_dentry_delete, 10666 .d_release = btrfs_dentry_release, 10667 }; 10668