1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 */ 5 6 #include <crypto/hash.h> 7 #include <linux/kernel.h> 8 #include <linux/bio.h> 9 #include <linux/buffer_head.h> 10 #include <linux/file.h> 11 #include <linux/fs.h> 12 #include <linux/pagemap.h> 13 #include <linux/highmem.h> 14 #include <linux/time.h> 15 #include <linux/init.h> 16 #include <linux/string.h> 17 #include <linux/backing-dev.h> 18 #include <linux/writeback.h> 19 #include <linux/compat.h> 20 #include <linux/xattr.h> 21 #include <linux/posix_acl.h> 22 #include <linux/falloc.h> 23 #include <linux/slab.h> 24 #include <linux/ratelimit.h> 25 #include <linux/btrfs.h> 26 #include <linux/blkdev.h> 27 #include <linux/posix_acl_xattr.h> 28 #include <linux/uio.h> 29 #include <linux/magic.h> 30 #include <linux/iversion.h> 31 #include <linux/swap.h> 32 #include <linux/migrate.h> 33 #include <linux/sched/mm.h> 34 #include <asm/unaligned.h> 35 #include "misc.h" 36 #include "ctree.h" 37 #include "disk-io.h" 38 #include "transaction.h" 39 #include "btrfs_inode.h" 40 #include "print-tree.h" 41 #include "ordered-data.h" 42 #include "xattr.h" 43 #include "tree-log.h" 44 #include "volumes.h" 45 #include "compression.h" 46 #include "locking.h" 47 #include "free-space-cache.h" 48 #include "inode-map.h" 49 #include "props.h" 50 #include "qgroup.h" 51 #include "delalloc-space.h" 52 #include "block-group.h" 53 #include "space-info.h" 54 55 struct btrfs_iget_args { 56 u64 ino; 57 struct btrfs_root *root; 58 }; 59 60 struct btrfs_dio_data { 61 u64 reserve; 62 u64 unsubmitted_oe_range_start; 63 u64 unsubmitted_oe_range_end; 64 int overwrite; 65 }; 66 67 static const struct inode_operations btrfs_dir_inode_operations; 68 static const struct inode_operations btrfs_symlink_inode_operations; 69 static const struct inode_operations btrfs_special_inode_operations; 70 static const struct inode_operations btrfs_file_inode_operations; 71 static const struct address_space_operations btrfs_aops; 72 static const struct file_operations btrfs_dir_file_operations; 73 static const struct extent_io_ops btrfs_extent_io_ops; 74 75 static struct kmem_cache *btrfs_inode_cachep; 76 struct kmem_cache *btrfs_trans_handle_cachep; 77 struct kmem_cache *btrfs_path_cachep; 78 struct kmem_cache *btrfs_free_space_cachep; 79 struct kmem_cache *btrfs_free_space_bitmap_cachep; 80 81 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 82 static int btrfs_truncate(struct inode *inode, bool skip_writeback); 83 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); 84 static noinline int cow_file_range(struct btrfs_inode *inode, 85 struct page *locked_page, 86 u64 start, u64 end, int *page_started, 87 unsigned long *nr_written, int unlock); 88 static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, 89 u64 len, u64 orig_start, u64 block_start, 90 u64 block_len, u64 orig_block_len, 91 u64 ram_bytes, int compress_type, 92 int type); 93 94 static void __endio_write_update_ordered(struct btrfs_inode *inode, 95 const u64 offset, const u64 bytes, 96 const bool uptodate); 97 98 /* 99 * Cleanup all submitted ordered extents in specified range to handle errors 100 * from the btrfs_run_delalloc_range() callback. 101 * 102 * NOTE: caller must ensure that when an error happens, it can not call 103 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING 104 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata 105 * to be released, which we want to happen only when finishing the ordered 106 * extent (btrfs_finish_ordered_io()). 107 */ 108 static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, 109 struct page *locked_page, 110 u64 offset, u64 bytes) 111 { 112 unsigned long index = offset >> PAGE_SHIFT; 113 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT; 114 u64 page_start = page_offset(locked_page); 115 u64 page_end = page_start + PAGE_SIZE - 1; 116 117 struct page *page; 118 119 while (index <= end_index) { 120 page = find_get_page(inode->vfs_inode.i_mapping, index); 121 index++; 122 if (!page) 123 continue; 124 ClearPagePrivate2(page); 125 put_page(page); 126 } 127 128 /* 129 * In case this page belongs to the delalloc range being instantiated 130 * then skip it, since the first page of a range is going to be 131 * properly cleaned up by the caller of run_delalloc_range 132 */ 133 if (page_start >= offset && page_end <= (offset + bytes - 1)) { 134 offset += PAGE_SIZE; 135 bytes -= PAGE_SIZE; 136 } 137 138 return __endio_write_update_ordered(inode, offset, bytes, false); 139 } 140 141 static int btrfs_dirty_inode(struct inode *inode); 142 143 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 144 void btrfs_test_inode_set_ops(struct inode *inode) 145 { 146 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 147 } 148 #endif 149 150 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 151 struct inode *inode, struct inode *dir, 152 const struct qstr *qstr) 153 { 154 int err; 155 156 err = btrfs_init_acl(trans, inode, dir); 157 if (!err) 158 err = btrfs_xattr_security_init(trans, inode, dir, qstr); 159 return err; 160 } 161 162 /* 163 * this does all the hard work for inserting an inline extent into 164 * the btree. The caller should have done a btrfs_drop_extents so that 165 * no overlapping inline items exist in the btree 166 */ 167 static int insert_inline_extent(struct btrfs_trans_handle *trans, 168 struct btrfs_path *path, int extent_inserted, 169 struct btrfs_root *root, struct inode *inode, 170 u64 start, size_t size, size_t compressed_size, 171 int compress_type, 172 struct page **compressed_pages) 173 { 174 struct extent_buffer *leaf; 175 struct page *page = NULL; 176 char *kaddr; 177 unsigned long ptr; 178 struct btrfs_file_extent_item *ei; 179 int ret; 180 size_t cur_size = size; 181 unsigned long offset; 182 183 ASSERT((compressed_size > 0 && compressed_pages) || 184 (compressed_size == 0 && !compressed_pages)); 185 186 if (compressed_size && compressed_pages) 187 cur_size = compressed_size; 188 189 inode_add_bytes(inode, size); 190 191 if (!extent_inserted) { 192 struct btrfs_key key; 193 size_t datasize; 194 195 key.objectid = btrfs_ino(BTRFS_I(inode)); 196 key.offset = start; 197 key.type = BTRFS_EXTENT_DATA_KEY; 198 199 datasize = btrfs_file_extent_calc_inline_size(cur_size); 200 path->leave_spinning = 1; 201 ret = btrfs_insert_empty_item(trans, root, path, &key, 202 datasize); 203 if (ret) 204 goto fail; 205 } 206 leaf = path->nodes[0]; 207 ei = btrfs_item_ptr(leaf, path->slots[0], 208 struct btrfs_file_extent_item); 209 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 210 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 211 btrfs_set_file_extent_encryption(leaf, ei, 0); 212 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 213 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 214 ptr = btrfs_file_extent_inline_start(ei); 215 216 if (compress_type != BTRFS_COMPRESS_NONE) { 217 struct page *cpage; 218 int i = 0; 219 while (compressed_size > 0) { 220 cpage = compressed_pages[i]; 221 cur_size = min_t(unsigned long, compressed_size, 222 PAGE_SIZE); 223 224 kaddr = kmap_atomic(cpage); 225 write_extent_buffer(leaf, kaddr, ptr, cur_size); 226 kunmap_atomic(kaddr); 227 228 i++; 229 ptr += cur_size; 230 compressed_size -= cur_size; 231 } 232 btrfs_set_file_extent_compression(leaf, ei, 233 compress_type); 234 } else { 235 page = find_get_page(inode->i_mapping, 236 start >> PAGE_SHIFT); 237 btrfs_set_file_extent_compression(leaf, ei, 0); 238 kaddr = kmap_atomic(page); 239 offset = offset_in_page(start); 240 write_extent_buffer(leaf, kaddr + offset, ptr, size); 241 kunmap_atomic(kaddr); 242 put_page(page); 243 } 244 btrfs_mark_buffer_dirty(leaf); 245 btrfs_release_path(path); 246 247 /* 248 * We align size to sectorsize for inline extents just for simplicity 249 * sake. 250 */ 251 size = ALIGN(size, root->fs_info->sectorsize); 252 ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), start, size); 253 if (ret) 254 goto fail; 255 256 /* 257 * we're an inline extent, so nobody can 258 * extend the file past i_size without locking 259 * a page we already have locked. 260 * 261 * We must do any isize and inode updates 262 * before we unlock the pages. Otherwise we 263 * could end up racing with unlink. 264 */ 265 BTRFS_I(inode)->disk_i_size = inode->i_size; 266 ret = btrfs_update_inode(trans, root, inode); 267 268 fail: 269 return ret; 270 } 271 272 273 /* 274 * conditionally insert an inline extent into the file. This 275 * does the checks required to make sure the data is small enough 276 * to fit as an inline extent. 277 */ 278 static noinline int cow_file_range_inline(struct btrfs_inode *inode, u64 start, 279 u64 end, size_t compressed_size, 280 int compress_type, 281 struct page **compressed_pages) 282 { 283 struct btrfs_root *root = inode->root; 284 struct btrfs_fs_info *fs_info = root->fs_info; 285 struct btrfs_trans_handle *trans; 286 u64 isize = i_size_read(&inode->vfs_inode); 287 u64 actual_end = min(end + 1, isize); 288 u64 inline_len = actual_end - start; 289 u64 aligned_end = ALIGN(end, fs_info->sectorsize); 290 u64 data_len = inline_len; 291 int ret; 292 struct btrfs_path *path; 293 int extent_inserted = 0; 294 u32 extent_item_size; 295 296 if (compressed_size) 297 data_len = compressed_size; 298 299 if (start > 0 || 300 actual_end > fs_info->sectorsize || 301 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 302 (!compressed_size && 303 (actual_end & (fs_info->sectorsize - 1)) == 0) || 304 end + 1 < isize || 305 data_len > fs_info->max_inline) { 306 return 1; 307 } 308 309 path = btrfs_alloc_path(); 310 if (!path) 311 return -ENOMEM; 312 313 trans = btrfs_join_transaction(root); 314 if (IS_ERR(trans)) { 315 btrfs_free_path(path); 316 return PTR_ERR(trans); 317 } 318 trans->block_rsv = &inode->block_rsv; 319 320 if (compressed_size && compressed_pages) 321 extent_item_size = btrfs_file_extent_calc_inline_size( 322 compressed_size); 323 else 324 extent_item_size = btrfs_file_extent_calc_inline_size( 325 inline_len); 326 327 ret = __btrfs_drop_extents(trans, root, inode, path, start, aligned_end, 328 NULL, 1, 1, extent_item_size, 329 &extent_inserted); 330 if (ret) { 331 btrfs_abort_transaction(trans, ret); 332 goto out; 333 } 334 335 if (isize > actual_end) 336 inline_len = min_t(u64, isize, actual_end); 337 ret = insert_inline_extent(trans, path, extent_inserted, 338 root, &inode->vfs_inode, start, 339 inline_len, compressed_size, 340 compress_type, compressed_pages); 341 if (ret && ret != -ENOSPC) { 342 btrfs_abort_transaction(trans, ret); 343 goto out; 344 } else if (ret == -ENOSPC) { 345 ret = 1; 346 goto out; 347 } 348 349 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); 350 btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0); 351 out: 352 /* 353 * Don't forget to free the reserved space, as for inlined extent 354 * it won't count as data extent, free them directly here. 355 * And at reserve time, it's always aligned to page size, so 356 * just free one page here. 357 */ 358 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE); 359 btrfs_free_path(path); 360 btrfs_end_transaction(trans); 361 return ret; 362 } 363 364 struct async_extent { 365 u64 start; 366 u64 ram_size; 367 u64 compressed_size; 368 struct page **pages; 369 unsigned long nr_pages; 370 int compress_type; 371 struct list_head list; 372 }; 373 374 struct async_chunk { 375 struct inode *inode; 376 struct page *locked_page; 377 u64 start; 378 u64 end; 379 unsigned int write_flags; 380 struct list_head extents; 381 struct cgroup_subsys_state *blkcg_css; 382 struct btrfs_work work; 383 atomic_t *pending; 384 }; 385 386 struct async_cow { 387 /* Number of chunks in flight; must be first in the structure */ 388 atomic_t num_chunks; 389 struct async_chunk chunks[]; 390 }; 391 392 static noinline int add_async_extent(struct async_chunk *cow, 393 u64 start, u64 ram_size, 394 u64 compressed_size, 395 struct page **pages, 396 unsigned long nr_pages, 397 int compress_type) 398 { 399 struct async_extent *async_extent; 400 401 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 402 BUG_ON(!async_extent); /* -ENOMEM */ 403 async_extent->start = start; 404 async_extent->ram_size = ram_size; 405 async_extent->compressed_size = compressed_size; 406 async_extent->pages = pages; 407 async_extent->nr_pages = nr_pages; 408 async_extent->compress_type = compress_type; 409 list_add_tail(&async_extent->list, &cow->extents); 410 return 0; 411 } 412 413 /* 414 * Check if the inode has flags compatible with compression 415 */ 416 static inline bool inode_can_compress(struct btrfs_inode *inode) 417 { 418 if (inode->flags & BTRFS_INODE_NODATACOW || 419 inode->flags & BTRFS_INODE_NODATASUM) 420 return false; 421 return true; 422 } 423 424 /* 425 * Check if the inode needs to be submitted to compression, based on mount 426 * options, defragmentation, properties or heuristics. 427 */ 428 static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, 429 u64 end) 430 { 431 struct btrfs_fs_info *fs_info = inode->root->fs_info; 432 433 if (!inode_can_compress(inode)) { 434 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG), 435 KERN_ERR "BTRFS: unexpected compression for ino %llu\n", 436 btrfs_ino(inode)); 437 return 0; 438 } 439 /* force compress */ 440 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 441 return 1; 442 /* defrag ioctl */ 443 if (inode->defrag_compress) 444 return 1; 445 /* bad compression ratios */ 446 if (inode->flags & BTRFS_INODE_NOCOMPRESS) 447 return 0; 448 if (btrfs_test_opt(fs_info, COMPRESS) || 449 inode->flags & BTRFS_INODE_COMPRESS || 450 inode->prop_compress) 451 return btrfs_compress_heuristic(&inode->vfs_inode, start, end); 452 return 0; 453 } 454 455 static inline void inode_should_defrag(struct btrfs_inode *inode, 456 u64 start, u64 end, u64 num_bytes, u64 small_write) 457 { 458 /* If this is a small write inside eof, kick off a defrag */ 459 if (num_bytes < small_write && 460 (start > 0 || end + 1 < inode->disk_i_size)) 461 btrfs_add_inode_defrag(NULL, inode); 462 } 463 464 /* 465 * we create compressed extents in two phases. The first 466 * phase compresses a range of pages that have already been 467 * locked (both pages and state bits are locked). 468 * 469 * This is done inside an ordered work queue, and the compression 470 * is spread across many cpus. The actual IO submission is step 471 * two, and the ordered work queue takes care of making sure that 472 * happens in the same order things were put onto the queue by 473 * writepages and friends. 474 * 475 * If this code finds it can't get good compression, it puts an 476 * entry onto the work queue to write the uncompressed bytes. This 477 * makes sure that both compressed inodes and uncompressed inodes 478 * are written in the same order that the flusher thread sent them 479 * down. 480 */ 481 static noinline int compress_file_range(struct async_chunk *async_chunk) 482 { 483 struct inode *inode = async_chunk->inode; 484 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 485 u64 blocksize = fs_info->sectorsize; 486 u64 start = async_chunk->start; 487 u64 end = async_chunk->end; 488 u64 actual_end; 489 u64 i_size; 490 int ret = 0; 491 struct page **pages = NULL; 492 unsigned long nr_pages; 493 unsigned long total_compressed = 0; 494 unsigned long total_in = 0; 495 int i; 496 int will_compress; 497 int compress_type = fs_info->compress_type; 498 int compressed_extents = 0; 499 int redirty = 0; 500 501 inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1, 502 SZ_16K); 503 504 /* 505 * We need to save i_size before now because it could change in between 506 * us evaluating the size and assigning it. This is because we lock and 507 * unlock the page in truncate and fallocate, and then modify the i_size 508 * later on. 509 * 510 * The barriers are to emulate READ_ONCE, remove that once i_size_read 511 * does that for us. 512 */ 513 barrier(); 514 i_size = i_size_read(inode); 515 barrier(); 516 actual_end = min_t(u64, i_size, end + 1); 517 again: 518 will_compress = 0; 519 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; 520 BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0); 521 nr_pages = min_t(unsigned long, nr_pages, 522 BTRFS_MAX_COMPRESSED / PAGE_SIZE); 523 524 /* 525 * we don't want to send crud past the end of i_size through 526 * compression, that's just a waste of CPU time. So, if the 527 * end of the file is before the start of our current 528 * requested range of bytes, we bail out to the uncompressed 529 * cleanup code that can deal with all of this. 530 * 531 * It isn't really the fastest way to fix things, but this is a 532 * very uncommon corner. 533 */ 534 if (actual_end <= start) 535 goto cleanup_and_bail_uncompressed; 536 537 total_compressed = actual_end - start; 538 539 /* 540 * skip compression for a small file range(<=blocksize) that 541 * isn't an inline extent, since it doesn't save disk space at all. 542 */ 543 if (total_compressed <= blocksize && 544 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 545 goto cleanup_and_bail_uncompressed; 546 547 total_compressed = min_t(unsigned long, total_compressed, 548 BTRFS_MAX_UNCOMPRESSED); 549 total_in = 0; 550 ret = 0; 551 552 /* 553 * we do compression for mount -o compress and when the 554 * inode has not been flagged as nocompress. This flag can 555 * change at any time if we discover bad compression ratios. 556 */ 557 if (inode_need_compress(BTRFS_I(inode), start, end)) { 558 WARN_ON(pages); 559 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 560 if (!pages) { 561 /* just bail out to the uncompressed code */ 562 nr_pages = 0; 563 goto cont; 564 } 565 566 if (BTRFS_I(inode)->defrag_compress) 567 compress_type = BTRFS_I(inode)->defrag_compress; 568 else if (BTRFS_I(inode)->prop_compress) 569 compress_type = BTRFS_I(inode)->prop_compress; 570 571 /* 572 * we need to call clear_page_dirty_for_io on each 573 * page in the range. Otherwise applications with the file 574 * mmap'd can wander in and change the page contents while 575 * we are compressing them. 576 * 577 * If the compression fails for any reason, we set the pages 578 * dirty again later on. 579 * 580 * Note that the remaining part is redirtied, the start pointer 581 * has moved, the end is the original one. 582 */ 583 if (!redirty) { 584 extent_range_clear_dirty_for_io(inode, start, end); 585 redirty = 1; 586 } 587 588 /* Compression level is applied here and only here */ 589 ret = btrfs_compress_pages( 590 compress_type | (fs_info->compress_level << 4), 591 inode->i_mapping, start, 592 pages, 593 &nr_pages, 594 &total_in, 595 &total_compressed); 596 597 if (!ret) { 598 unsigned long offset = offset_in_page(total_compressed); 599 struct page *page = pages[nr_pages - 1]; 600 char *kaddr; 601 602 /* zero the tail end of the last page, we might be 603 * sending it down to disk 604 */ 605 if (offset) { 606 kaddr = kmap_atomic(page); 607 memset(kaddr + offset, 0, 608 PAGE_SIZE - offset); 609 kunmap_atomic(kaddr); 610 } 611 will_compress = 1; 612 } 613 } 614 cont: 615 if (start == 0) { 616 /* lets try to make an inline extent */ 617 if (ret || total_in < actual_end) { 618 /* we didn't compress the entire range, try 619 * to make an uncompressed inline extent. 620 */ 621 ret = cow_file_range_inline(BTRFS_I(inode), start, end, 622 0, BTRFS_COMPRESS_NONE, 623 NULL); 624 } else { 625 /* try making a compressed inline extent */ 626 ret = cow_file_range_inline(BTRFS_I(inode), start, end, 627 total_compressed, 628 compress_type, pages); 629 } 630 if (ret <= 0) { 631 unsigned long clear_flags = EXTENT_DELALLOC | 632 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 633 EXTENT_DO_ACCOUNTING; 634 unsigned long page_error_op; 635 636 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 637 638 /* 639 * inline extent creation worked or returned error, 640 * we don't need to create any more async work items. 641 * Unlock and free up our temp pages. 642 * 643 * We use DO_ACCOUNTING here because we need the 644 * delalloc_release_metadata to be done _after_ we drop 645 * our outstanding extent for clearing delalloc for this 646 * range. 647 */ 648 extent_clear_unlock_delalloc(BTRFS_I(inode), start, end, 649 NULL, 650 clear_flags, 651 PAGE_UNLOCK | 652 PAGE_CLEAR_DIRTY | 653 PAGE_SET_WRITEBACK | 654 page_error_op | 655 PAGE_END_WRITEBACK); 656 657 for (i = 0; i < nr_pages; i++) { 658 WARN_ON(pages[i]->mapping); 659 put_page(pages[i]); 660 } 661 kfree(pages); 662 663 return 0; 664 } 665 } 666 667 if (will_compress) { 668 /* 669 * we aren't doing an inline extent round the compressed size 670 * up to a block size boundary so the allocator does sane 671 * things 672 */ 673 total_compressed = ALIGN(total_compressed, blocksize); 674 675 /* 676 * one last check to make sure the compression is really a 677 * win, compare the page count read with the blocks on disk, 678 * compression must free at least one sector size 679 */ 680 total_in = ALIGN(total_in, PAGE_SIZE); 681 if (total_compressed + blocksize <= total_in) { 682 compressed_extents++; 683 684 /* 685 * The async work queues will take care of doing actual 686 * allocation on disk for these compressed pages, and 687 * will submit them to the elevator. 688 */ 689 add_async_extent(async_chunk, start, total_in, 690 total_compressed, pages, nr_pages, 691 compress_type); 692 693 if (start + total_in < end) { 694 start += total_in; 695 pages = NULL; 696 cond_resched(); 697 goto again; 698 } 699 return compressed_extents; 700 } 701 } 702 if (pages) { 703 /* 704 * the compression code ran but failed to make things smaller, 705 * free any pages it allocated and our page pointer array 706 */ 707 for (i = 0; i < nr_pages; i++) { 708 WARN_ON(pages[i]->mapping); 709 put_page(pages[i]); 710 } 711 kfree(pages); 712 pages = NULL; 713 total_compressed = 0; 714 nr_pages = 0; 715 716 /* flag the file so we don't compress in the future */ 717 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && 718 !(BTRFS_I(inode)->prop_compress)) { 719 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 720 } 721 } 722 cleanup_and_bail_uncompressed: 723 /* 724 * No compression, but we still need to write the pages in the file 725 * we've been given so far. redirty the locked page if it corresponds 726 * to our extent and set things up for the async work queue to run 727 * cow_file_range to do the normal delalloc dance. 728 */ 729 if (async_chunk->locked_page && 730 (page_offset(async_chunk->locked_page) >= start && 731 page_offset(async_chunk->locked_page)) <= end) { 732 __set_page_dirty_nobuffers(async_chunk->locked_page); 733 /* unlocked later on in the async handlers */ 734 } 735 736 if (redirty) 737 extent_range_redirty_for_io(inode, start, end); 738 add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0, 739 BTRFS_COMPRESS_NONE); 740 compressed_extents++; 741 742 return compressed_extents; 743 } 744 745 static void free_async_extent_pages(struct async_extent *async_extent) 746 { 747 int i; 748 749 if (!async_extent->pages) 750 return; 751 752 for (i = 0; i < async_extent->nr_pages; i++) { 753 WARN_ON(async_extent->pages[i]->mapping); 754 put_page(async_extent->pages[i]); 755 } 756 kfree(async_extent->pages); 757 async_extent->nr_pages = 0; 758 async_extent->pages = NULL; 759 } 760 761 /* 762 * phase two of compressed writeback. This is the ordered portion 763 * of the code, which only gets called in the order the work was 764 * queued. We walk all the async extents created by compress_file_range 765 * and send them down to the disk. 766 */ 767 static noinline void submit_compressed_extents(struct async_chunk *async_chunk) 768 { 769 struct btrfs_inode *inode = BTRFS_I(async_chunk->inode); 770 struct btrfs_fs_info *fs_info = inode->root->fs_info; 771 struct async_extent *async_extent; 772 u64 alloc_hint = 0; 773 struct btrfs_key ins; 774 struct extent_map *em; 775 struct btrfs_root *root = inode->root; 776 struct extent_io_tree *io_tree = &inode->io_tree; 777 int ret = 0; 778 779 again: 780 while (!list_empty(&async_chunk->extents)) { 781 async_extent = list_entry(async_chunk->extents.next, 782 struct async_extent, list); 783 list_del(&async_extent->list); 784 785 retry: 786 lock_extent(io_tree, async_extent->start, 787 async_extent->start + async_extent->ram_size - 1); 788 /* did the compression code fall back to uncompressed IO? */ 789 if (!async_extent->pages) { 790 int page_started = 0; 791 unsigned long nr_written = 0; 792 793 /* allocate blocks */ 794 ret = cow_file_range(inode, async_chunk->locked_page, 795 async_extent->start, 796 async_extent->start + 797 async_extent->ram_size - 1, 798 &page_started, &nr_written, 0); 799 800 /* JDM XXX */ 801 802 /* 803 * if page_started, cow_file_range inserted an 804 * inline extent and took care of all the unlocking 805 * and IO for us. Otherwise, we need to submit 806 * all those pages down to the drive. 807 */ 808 if (!page_started && !ret) 809 extent_write_locked_range(&inode->vfs_inode, 810 async_extent->start, 811 async_extent->start + 812 async_extent->ram_size - 1, 813 WB_SYNC_ALL); 814 else if (ret && async_chunk->locked_page) 815 unlock_page(async_chunk->locked_page); 816 kfree(async_extent); 817 cond_resched(); 818 continue; 819 } 820 821 ret = btrfs_reserve_extent(root, async_extent->ram_size, 822 async_extent->compressed_size, 823 async_extent->compressed_size, 824 0, alloc_hint, &ins, 1, 1); 825 if (ret) { 826 free_async_extent_pages(async_extent); 827 828 if (ret == -ENOSPC) { 829 unlock_extent(io_tree, async_extent->start, 830 async_extent->start + 831 async_extent->ram_size - 1); 832 833 /* 834 * we need to redirty the pages if we decide to 835 * fallback to uncompressed IO, otherwise we 836 * will not submit these pages down to lower 837 * layers. 838 */ 839 extent_range_redirty_for_io(&inode->vfs_inode, 840 async_extent->start, 841 async_extent->start + 842 async_extent->ram_size - 1); 843 844 goto retry; 845 } 846 goto out_free; 847 } 848 /* 849 * here we're doing allocation and writeback of the 850 * compressed pages 851 */ 852 em = create_io_em(inode, async_extent->start, 853 async_extent->ram_size, /* len */ 854 async_extent->start, /* orig_start */ 855 ins.objectid, /* block_start */ 856 ins.offset, /* block_len */ 857 ins.offset, /* orig_block_len */ 858 async_extent->ram_size, /* ram_bytes */ 859 async_extent->compress_type, 860 BTRFS_ORDERED_COMPRESSED); 861 if (IS_ERR(em)) 862 /* ret value is not necessary due to void function */ 863 goto out_free_reserve; 864 free_extent_map(em); 865 866 ret = btrfs_add_ordered_extent_compress(inode, 867 async_extent->start, 868 ins.objectid, 869 async_extent->ram_size, 870 ins.offset, 871 BTRFS_ORDERED_COMPRESSED, 872 async_extent->compress_type); 873 if (ret) { 874 btrfs_drop_extent_cache(inode, async_extent->start, 875 async_extent->start + 876 async_extent->ram_size - 1, 0); 877 goto out_free_reserve; 878 } 879 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 880 881 /* 882 * clear dirty, set writeback and unlock the pages. 883 */ 884 extent_clear_unlock_delalloc(inode, async_extent->start, 885 async_extent->start + 886 async_extent->ram_size - 1, 887 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 888 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 889 PAGE_SET_WRITEBACK); 890 if (btrfs_submit_compressed_write(inode, async_extent->start, 891 async_extent->ram_size, 892 ins.objectid, 893 ins.offset, async_extent->pages, 894 async_extent->nr_pages, 895 async_chunk->write_flags, 896 async_chunk->blkcg_css)) { 897 struct page *p = async_extent->pages[0]; 898 const u64 start = async_extent->start; 899 const u64 end = start + async_extent->ram_size - 1; 900 901 p->mapping = inode->vfs_inode.i_mapping; 902 btrfs_writepage_endio_finish_ordered(p, start, end, 0); 903 904 p->mapping = NULL; 905 extent_clear_unlock_delalloc(inode, start, end, NULL, 0, 906 PAGE_END_WRITEBACK | 907 PAGE_SET_ERROR); 908 free_async_extent_pages(async_extent); 909 } 910 alloc_hint = ins.objectid + ins.offset; 911 kfree(async_extent); 912 cond_resched(); 913 } 914 return; 915 out_free_reserve: 916 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 917 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 918 out_free: 919 extent_clear_unlock_delalloc(inode, async_extent->start, 920 async_extent->start + 921 async_extent->ram_size - 1, 922 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 923 EXTENT_DELALLOC_NEW | 924 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 925 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 926 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | 927 PAGE_SET_ERROR); 928 free_async_extent_pages(async_extent); 929 kfree(async_extent); 930 goto again; 931 } 932 933 static u64 get_extent_allocation_hint(struct btrfs_inode *inode, u64 start, 934 u64 num_bytes) 935 { 936 struct extent_map_tree *em_tree = &inode->extent_tree; 937 struct extent_map *em; 938 u64 alloc_hint = 0; 939 940 read_lock(&em_tree->lock); 941 em = search_extent_mapping(em_tree, start, num_bytes); 942 if (em) { 943 /* 944 * if block start isn't an actual block number then find the 945 * first block in this inode and use that as a hint. If that 946 * block is also bogus then just don't worry about it. 947 */ 948 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 949 free_extent_map(em); 950 em = search_extent_mapping(em_tree, 0, 0); 951 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 952 alloc_hint = em->block_start; 953 if (em) 954 free_extent_map(em); 955 } else { 956 alloc_hint = em->block_start; 957 free_extent_map(em); 958 } 959 } 960 read_unlock(&em_tree->lock); 961 962 return alloc_hint; 963 } 964 965 /* 966 * when extent_io.c finds a delayed allocation range in the file, 967 * the call backs end up in this code. The basic idea is to 968 * allocate extents on disk for the range, and create ordered data structs 969 * in ram to track those extents. 970 * 971 * locked_page is the page that writepage had locked already. We use 972 * it to make sure we don't do extra locks or unlocks. 973 * 974 * *page_started is set to one if we unlock locked_page and do everything 975 * required to start IO on it. It may be clean and already done with 976 * IO when we return. 977 */ 978 static noinline int cow_file_range(struct btrfs_inode *inode, 979 struct page *locked_page, 980 u64 start, u64 end, int *page_started, 981 unsigned long *nr_written, int unlock) 982 { 983 struct btrfs_root *root = inode->root; 984 struct btrfs_fs_info *fs_info = root->fs_info; 985 u64 alloc_hint = 0; 986 u64 num_bytes; 987 unsigned long ram_size; 988 u64 cur_alloc_size = 0; 989 u64 min_alloc_size; 990 u64 blocksize = fs_info->sectorsize; 991 struct btrfs_key ins; 992 struct extent_map *em; 993 unsigned clear_bits; 994 unsigned long page_ops; 995 bool extent_reserved = false; 996 int ret = 0; 997 998 if (btrfs_is_free_space_inode(inode)) { 999 WARN_ON_ONCE(1); 1000 ret = -EINVAL; 1001 goto out_unlock; 1002 } 1003 1004 num_bytes = ALIGN(end - start + 1, blocksize); 1005 num_bytes = max(blocksize, num_bytes); 1006 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); 1007 1008 inode_should_defrag(inode, start, end, num_bytes, SZ_64K); 1009 1010 if (start == 0) { 1011 /* lets try to make an inline extent */ 1012 ret = cow_file_range_inline(inode, start, end, 0, 1013 BTRFS_COMPRESS_NONE, NULL); 1014 if (ret == 0) { 1015 /* 1016 * We use DO_ACCOUNTING here because we need the 1017 * delalloc_release_metadata to be run _after_ we drop 1018 * our outstanding extent for clearing delalloc for this 1019 * range. 1020 */ 1021 extent_clear_unlock_delalloc(inode, start, end, NULL, 1022 EXTENT_LOCKED | EXTENT_DELALLOC | 1023 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 1024 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1025 PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 1026 PAGE_END_WRITEBACK); 1027 *nr_written = *nr_written + 1028 (end - start + PAGE_SIZE) / PAGE_SIZE; 1029 *page_started = 1; 1030 goto out; 1031 } else if (ret < 0) { 1032 goto out_unlock; 1033 } 1034 } 1035 1036 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 1037 btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0); 1038 1039 /* 1040 * Relocation relies on the relocated extents to have exactly the same 1041 * size as the original extents. Normally writeback for relocation data 1042 * extents follows a NOCOW path because relocation preallocates the 1043 * extents. However, due to an operation such as scrub turning a block 1044 * group to RO mode, it may fallback to COW mode, so we must make sure 1045 * an extent allocated during COW has exactly the requested size and can 1046 * not be split into smaller extents, otherwise relocation breaks and 1047 * fails during the stage where it updates the bytenr of file extent 1048 * items. 1049 */ 1050 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 1051 min_alloc_size = num_bytes; 1052 else 1053 min_alloc_size = fs_info->sectorsize; 1054 1055 while (num_bytes > 0) { 1056 cur_alloc_size = num_bytes; 1057 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size, 1058 min_alloc_size, 0, alloc_hint, 1059 &ins, 1, 1); 1060 if (ret < 0) 1061 goto out_unlock; 1062 cur_alloc_size = ins.offset; 1063 extent_reserved = true; 1064 1065 ram_size = ins.offset; 1066 em = create_io_em(inode, start, ins.offset, /* len */ 1067 start, /* orig_start */ 1068 ins.objectid, /* block_start */ 1069 ins.offset, /* block_len */ 1070 ins.offset, /* orig_block_len */ 1071 ram_size, /* ram_bytes */ 1072 BTRFS_COMPRESS_NONE, /* compress_type */ 1073 BTRFS_ORDERED_REGULAR /* type */); 1074 if (IS_ERR(em)) { 1075 ret = PTR_ERR(em); 1076 goto out_reserve; 1077 } 1078 free_extent_map(em); 1079 1080 ret = btrfs_add_ordered_extent(inode, start, ins.objectid, 1081 ram_size, cur_alloc_size, 0); 1082 if (ret) 1083 goto out_drop_extent_cache; 1084 1085 if (root->root_key.objectid == 1086 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1087 ret = btrfs_reloc_clone_csums(inode, start, 1088 cur_alloc_size); 1089 /* 1090 * Only drop cache here, and process as normal. 1091 * 1092 * We must not allow extent_clear_unlock_delalloc() 1093 * at out_unlock label to free meta of this ordered 1094 * extent, as its meta should be freed by 1095 * btrfs_finish_ordered_io(). 1096 * 1097 * So we must continue until @start is increased to 1098 * skip current ordered extent. 1099 */ 1100 if (ret) 1101 btrfs_drop_extent_cache(inode, start, 1102 start + ram_size - 1, 0); 1103 } 1104 1105 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1106 1107 /* we're not doing compressed IO, don't unlock the first 1108 * page (which the caller expects to stay locked), don't 1109 * clear any dirty bits and don't set any writeback bits 1110 * 1111 * Do set the Private2 bit so we know this page was properly 1112 * setup for writepage 1113 */ 1114 page_ops = unlock ? PAGE_UNLOCK : 0; 1115 page_ops |= PAGE_SET_PRIVATE2; 1116 1117 extent_clear_unlock_delalloc(inode, start, start + ram_size - 1, 1118 locked_page, 1119 EXTENT_LOCKED | EXTENT_DELALLOC, 1120 page_ops); 1121 if (num_bytes < cur_alloc_size) 1122 num_bytes = 0; 1123 else 1124 num_bytes -= cur_alloc_size; 1125 alloc_hint = ins.objectid + ins.offset; 1126 start += cur_alloc_size; 1127 extent_reserved = false; 1128 1129 /* 1130 * btrfs_reloc_clone_csums() error, since start is increased 1131 * extent_clear_unlock_delalloc() at out_unlock label won't 1132 * free metadata of current ordered extent, we're OK to exit. 1133 */ 1134 if (ret) 1135 goto out_unlock; 1136 } 1137 out: 1138 return ret; 1139 1140 out_drop_extent_cache: 1141 btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0); 1142 out_reserve: 1143 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1144 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1145 out_unlock: 1146 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 1147 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; 1148 page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 1149 PAGE_END_WRITEBACK; 1150 /* 1151 * If we reserved an extent for our delalloc range (or a subrange) and 1152 * failed to create the respective ordered extent, then it means that 1153 * when we reserved the extent we decremented the extent's size from 1154 * the data space_info's bytes_may_use counter and incremented the 1155 * space_info's bytes_reserved counter by the same amount. We must make 1156 * sure extent_clear_unlock_delalloc() does not try to decrement again 1157 * the data space_info's bytes_may_use counter, therefore we do not pass 1158 * it the flag EXTENT_CLEAR_DATA_RESV. 1159 */ 1160 if (extent_reserved) { 1161 extent_clear_unlock_delalloc(inode, start, 1162 start + cur_alloc_size - 1, 1163 locked_page, 1164 clear_bits, 1165 page_ops); 1166 start += cur_alloc_size; 1167 if (start >= end) 1168 goto out; 1169 } 1170 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1171 clear_bits | EXTENT_CLEAR_DATA_RESV, 1172 page_ops); 1173 goto out; 1174 } 1175 1176 /* 1177 * work queue call back to started compression on a file and pages 1178 */ 1179 static noinline void async_cow_start(struct btrfs_work *work) 1180 { 1181 struct async_chunk *async_chunk; 1182 int compressed_extents; 1183 1184 async_chunk = container_of(work, struct async_chunk, work); 1185 1186 compressed_extents = compress_file_range(async_chunk); 1187 if (compressed_extents == 0) { 1188 btrfs_add_delayed_iput(async_chunk->inode); 1189 async_chunk->inode = NULL; 1190 } 1191 } 1192 1193 /* 1194 * work queue call back to submit previously compressed pages 1195 */ 1196 static noinline void async_cow_submit(struct btrfs_work *work) 1197 { 1198 struct async_chunk *async_chunk = container_of(work, struct async_chunk, 1199 work); 1200 struct btrfs_fs_info *fs_info = btrfs_work_owner(work); 1201 unsigned long nr_pages; 1202 1203 nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >> 1204 PAGE_SHIFT; 1205 1206 /* atomic_sub_return implies a barrier */ 1207 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1208 5 * SZ_1M) 1209 cond_wake_up_nomb(&fs_info->async_submit_wait); 1210 1211 /* 1212 * ->inode could be NULL if async_chunk_start has failed to compress, 1213 * in which case we don't have anything to submit, yet we need to 1214 * always adjust ->async_delalloc_pages as its paired with the init 1215 * happening in cow_file_range_async 1216 */ 1217 if (async_chunk->inode) 1218 submit_compressed_extents(async_chunk); 1219 } 1220 1221 static noinline void async_cow_free(struct btrfs_work *work) 1222 { 1223 struct async_chunk *async_chunk; 1224 1225 async_chunk = container_of(work, struct async_chunk, work); 1226 if (async_chunk->inode) 1227 btrfs_add_delayed_iput(async_chunk->inode); 1228 if (async_chunk->blkcg_css) 1229 css_put(async_chunk->blkcg_css); 1230 /* 1231 * Since the pointer to 'pending' is at the beginning of the array of 1232 * async_chunk's, freeing it ensures the whole array has been freed. 1233 */ 1234 if (atomic_dec_and_test(async_chunk->pending)) 1235 kvfree(async_chunk->pending); 1236 } 1237 1238 static int cow_file_range_async(struct btrfs_inode *inode, 1239 struct writeback_control *wbc, 1240 struct page *locked_page, 1241 u64 start, u64 end, int *page_started, 1242 unsigned long *nr_written) 1243 { 1244 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1245 struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc); 1246 struct async_cow *ctx; 1247 struct async_chunk *async_chunk; 1248 unsigned long nr_pages; 1249 u64 cur_end; 1250 u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K); 1251 int i; 1252 bool should_compress; 1253 unsigned nofs_flag; 1254 const unsigned int write_flags = wbc_to_write_flags(wbc); 1255 1256 unlock_extent(&inode->io_tree, start, end); 1257 1258 if (inode->flags & BTRFS_INODE_NOCOMPRESS && 1259 !btrfs_test_opt(fs_info, FORCE_COMPRESS)) { 1260 num_chunks = 1; 1261 should_compress = false; 1262 } else { 1263 should_compress = true; 1264 } 1265 1266 nofs_flag = memalloc_nofs_save(); 1267 ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL); 1268 memalloc_nofs_restore(nofs_flag); 1269 1270 if (!ctx) { 1271 unsigned clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | 1272 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 1273 EXTENT_DO_ACCOUNTING; 1274 unsigned long page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 1275 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | 1276 PAGE_SET_ERROR; 1277 1278 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1279 clear_bits, page_ops); 1280 return -ENOMEM; 1281 } 1282 1283 async_chunk = ctx->chunks; 1284 atomic_set(&ctx->num_chunks, num_chunks); 1285 1286 for (i = 0; i < num_chunks; i++) { 1287 if (should_compress) 1288 cur_end = min(end, start + SZ_512K - 1); 1289 else 1290 cur_end = end; 1291 1292 /* 1293 * igrab is called higher up in the call chain, take only the 1294 * lightweight reference for the callback lifetime 1295 */ 1296 ihold(&inode->vfs_inode); 1297 async_chunk[i].pending = &ctx->num_chunks; 1298 async_chunk[i].inode = &inode->vfs_inode; 1299 async_chunk[i].start = start; 1300 async_chunk[i].end = cur_end; 1301 async_chunk[i].write_flags = write_flags; 1302 INIT_LIST_HEAD(&async_chunk[i].extents); 1303 1304 /* 1305 * The locked_page comes all the way from writepage and its 1306 * the original page we were actually given. As we spread 1307 * this large delalloc region across multiple async_chunk 1308 * structs, only the first struct needs a pointer to locked_page 1309 * 1310 * This way we don't need racey decisions about who is supposed 1311 * to unlock it. 1312 */ 1313 if (locked_page) { 1314 /* 1315 * Depending on the compressibility, the pages might or 1316 * might not go through async. We want all of them to 1317 * be accounted against wbc once. Let's do it here 1318 * before the paths diverge. wbc accounting is used 1319 * only for foreign writeback detection and doesn't 1320 * need full accuracy. Just account the whole thing 1321 * against the first page. 1322 */ 1323 wbc_account_cgroup_owner(wbc, locked_page, 1324 cur_end - start); 1325 async_chunk[i].locked_page = locked_page; 1326 locked_page = NULL; 1327 } else { 1328 async_chunk[i].locked_page = NULL; 1329 } 1330 1331 if (blkcg_css != blkcg_root_css) { 1332 css_get(blkcg_css); 1333 async_chunk[i].blkcg_css = blkcg_css; 1334 } else { 1335 async_chunk[i].blkcg_css = NULL; 1336 } 1337 1338 btrfs_init_work(&async_chunk[i].work, async_cow_start, 1339 async_cow_submit, async_cow_free); 1340 1341 nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE); 1342 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1343 1344 btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work); 1345 1346 *nr_written += nr_pages; 1347 start = cur_end + 1; 1348 } 1349 *page_started = 1; 1350 return 0; 1351 } 1352 1353 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, 1354 u64 bytenr, u64 num_bytes) 1355 { 1356 int ret; 1357 struct btrfs_ordered_sum *sums; 1358 LIST_HEAD(list); 1359 1360 ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr, 1361 bytenr + num_bytes - 1, &list, 0); 1362 if (ret == 0 && list_empty(&list)) 1363 return 0; 1364 1365 while (!list_empty(&list)) { 1366 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1367 list_del(&sums->list); 1368 kfree(sums); 1369 } 1370 if (ret < 0) 1371 return ret; 1372 return 1; 1373 } 1374 1375 static int fallback_to_cow(struct btrfs_inode *inode, struct page *locked_page, 1376 const u64 start, const u64 end, 1377 int *page_started, unsigned long *nr_written) 1378 { 1379 const bool is_space_ino = btrfs_is_free_space_inode(inode); 1380 const bool is_reloc_ino = (inode->root->root_key.objectid == 1381 BTRFS_DATA_RELOC_TREE_OBJECTID); 1382 const u64 range_bytes = end + 1 - start; 1383 struct extent_io_tree *io_tree = &inode->io_tree; 1384 u64 range_start = start; 1385 u64 count; 1386 1387 /* 1388 * If EXTENT_NORESERVE is set it means that when the buffered write was 1389 * made we had not enough available data space and therefore we did not 1390 * reserve data space for it, since we though we could do NOCOW for the 1391 * respective file range (either there is prealloc extent or the inode 1392 * has the NOCOW bit set). 1393 * 1394 * However when we need to fallback to COW mode (because for example the 1395 * block group for the corresponding extent was turned to RO mode by a 1396 * scrub or relocation) we need to do the following: 1397 * 1398 * 1) We increment the bytes_may_use counter of the data space info. 1399 * If COW succeeds, it allocates a new data extent and after doing 1400 * that it decrements the space info's bytes_may_use counter and 1401 * increments its bytes_reserved counter by the same amount (we do 1402 * this at btrfs_add_reserved_bytes()). So we need to increment the 1403 * bytes_may_use counter to compensate (when space is reserved at 1404 * buffered write time, the bytes_may_use counter is incremented); 1405 * 1406 * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so 1407 * that if the COW path fails for any reason, it decrements (through 1408 * extent_clear_unlock_delalloc()) the bytes_may_use counter of the 1409 * data space info, which we incremented in the step above. 1410 * 1411 * If we need to fallback to cow and the inode corresponds to a free 1412 * space cache inode or an inode of the data relocation tree, we must 1413 * also increment bytes_may_use of the data space_info for the same 1414 * reason. Space caches and relocated data extents always get a prealloc 1415 * extent for them, however scrub or balance may have set the block 1416 * group that contains that extent to RO mode and therefore force COW 1417 * when starting writeback. 1418 */ 1419 count = count_range_bits(io_tree, &range_start, end, range_bytes, 1420 EXTENT_NORESERVE, 0); 1421 if (count > 0 || is_space_ino || is_reloc_ino) { 1422 u64 bytes = count; 1423 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1424 struct btrfs_space_info *sinfo = fs_info->data_sinfo; 1425 1426 if (is_space_ino || is_reloc_ino) 1427 bytes = range_bytes; 1428 1429 spin_lock(&sinfo->lock); 1430 btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes); 1431 spin_unlock(&sinfo->lock); 1432 1433 if (count > 0) 1434 clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, 1435 0, 0, NULL); 1436 } 1437 1438 return cow_file_range(inode, locked_page, start, end, page_started, 1439 nr_written, 1); 1440 } 1441 1442 /* 1443 * when nowcow writeback call back. This checks for snapshots or COW copies 1444 * of the extents that exist in the file, and COWs the file as required. 1445 * 1446 * If no cow copies or snapshots exist, we write directly to the existing 1447 * blocks on disk 1448 */ 1449 static noinline int run_delalloc_nocow(struct btrfs_inode *inode, 1450 struct page *locked_page, 1451 const u64 start, const u64 end, 1452 int *page_started, int force, 1453 unsigned long *nr_written) 1454 { 1455 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1456 struct btrfs_root *root = inode->root; 1457 struct btrfs_path *path; 1458 u64 cow_start = (u64)-1; 1459 u64 cur_offset = start; 1460 int ret; 1461 bool check_prev = true; 1462 const bool freespace_inode = btrfs_is_free_space_inode(inode); 1463 u64 ino = btrfs_ino(inode); 1464 bool nocow = false; 1465 u64 disk_bytenr = 0; 1466 1467 path = btrfs_alloc_path(); 1468 if (!path) { 1469 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1470 EXTENT_LOCKED | EXTENT_DELALLOC | 1471 EXTENT_DO_ACCOUNTING | 1472 EXTENT_DEFRAG, PAGE_UNLOCK | 1473 PAGE_CLEAR_DIRTY | 1474 PAGE_SET_WRITEBACK | 1475 PAGE_END_WRITEBACK); 1476 return -ENOMEM; 1477 } 1478 1479 while (1) { 1480 struct btrfs_key found_key; 1481 struct btrfs_file_extent_item *fi; 1482 struct extent_buffer *leaf; 1483 u64 extent_end; 1484 u64 extent_offset; 1485 u64 num_bytes = 0; 1486 u64 disk_num_bytes; 1487 u64 ram_bytes; 1488 int extent_type; 1489 1490 nocow = false; 1491 1492 ret = btrfs_lookup_file_extent(NULL, root, path, ino, 1493 cur_offset, 0); 1494 if (ret < 0) 1495 goto error; 1496 1497 /* 1498 * If there is no extent for our range when doing the initial 1499 * search, then go back to the previous slot as it will be the 1500 * one containing the search offset 1501 */ 1502 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1503 leaf = path->nodes[0]; 1504 btrfs_item_key_to_cpu(leaf, &found_key, 1505 path->slots[0] - 1); 1506 if (found_key.objectid == ino && 1507 found_key.type == BTRFS_EXTENT_DATA_KEY) 1508 path->slots[0]--; 1509 } 1510 check_prev = false; 1511 next_slot: 1512 /* Go to next leaf if we have exhausted the current one */ 1513 leaf = path->nodes[0]; 1514 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1515 ret = btrfs_next_leaf(root, path); 1516 if (ret < 0) { 1517 if (cow_start != (u64)-1) 1518 cur_offset = cow_start; 1519 goto error; 1520 } 1521 if (ret > 0) 1522 break; 1523 leaf = path->nodes[0]; 1524 } 1525 1526 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1527 1528 /* Didn't find anything for our INO */ 1529 if (found_key.objectid > ino) 1530 break; 1531 /* 1532 * Keep searching until we find an EXTENT_ITEM or there are no 1533 * more extents for this inode 1534 */ 1535 if (WARN_ON_ONCE(found_key.objectid < ino) || 1536 found_key.type < BTRFS_EXTENT_DATA_KEY) { 1537 path->slots[0]++; 1538 goto next_slot; 1539 } 1540 1541 /* Found key is not EXTENT_DATA_KEY or starts after req range */ 1542 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 1543 found_key.offset > end) 1544 break; 1545 1546 /* 1547 * If the found extent starts after requested offset, then 1548 * adjust extent_end to be right before this extent begins 1549 */ 1550 if (found_key.offset > cur_offset) { 1551 extent_end = found_key.offset; 1552 extent_type = 0; 1553 goto out_check; 1554 } 1555 1556 /* 1557 * Found extent which begins before our range and potentially 1558 * intersect it 1559 */ 1560 fi = btrfs_item_ptr(leaf, path->slots[0], 1561 struct btrfs_file_extent_item); 1562 extent_type = btrfs_file_extent_type(leaf, fi); 1563 1564 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1565 if (extent_type == BTRFS_FILE_EXTENT_REG || 1566 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1567 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1568 extent_offset = btrfs_file_extent_offset(leaf, fi); 1569 extent_end = found_key.offset + 1570 btrfs_file_extent_num_bytes(leaf, fi); 1571 disk_num_bytes = 1572 btrfs_file_extent_disk_num_bytes(leaf, fi); 1573 /* 1574 * If the extent we got ends before our current offset, 1575 * skip to the next extent. 1576 */ 1577 if (extent_end <= cur_offset) { 1578 path->slots[0]++; 1579 goto next_slot; 1580 } 1581 /* Skip holes */ 1582 if (disk_bytenr == 0) 1583 goto out_check; 1584 /* Skip compressed/encrypted/encoded extents */ 1585 if (btrfs_file_extent_compression(leaf, fi) || 1586 btrfs_file_extent_encryption(leaf, fi) || 1587 btrfs_file_extent_other_encoding(leaf, fi)) 1588 goto out_check; 1589 /* 1590 * If extent is created before the last volume's snapshot 1591 * this implies the extent is shared, hence we can't do 1592 * nocow. This is the same check as in 1593 * btrfs_cross_ref_exist but without calling 1594 * btrfs_search_slot. 1595 */ 1596 if (!freespace_inode && 1597 btrfs_file_extent_generation(leaf, fi) <= 1598 btrfs_root_last_snapshot(&root->root_item)) 1599 goto out_check; 1600 if (extent_type == BTRFS_FILE_EXTENT_REG && !force) 1601 goto out_check; 1602 /* If extent is RO, we must COW it */ 1603 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 1604 goto out_check; 1605 ret = btrfs_cross_ref_exist(root, ino, 1606 found_key.offset - 1607 extent_offset, disk_bytenr); 1608 if (ret) { 1609 /* 1610 * ret could be -EIO if the above fails to read 1611 * metadata. 1612 */ 1613 if (ret < 0) { 1614 if (cow_start != (u64)-1) 1615 cur_offset = cow_start; 1616 goto error; 1617 } 1618 1619 WARN_ON_ONCE(freespace_inode); 1620 goto out_check; 1621 } 1622 disk_bytenr += extent_offset; 1623 disk_bytenr += cur_offset - found_key.offset; 1624 num_bytes = min(end + 1, extent_end) - cur_offset; 1625 /* 1626 * If there are pending snapshots for this root, we 1627 * fall into common COW way 1628 */ 1629 if (!freespace_inode && atomic_read(&root->snapshot_force_cow)) 1630 goto out_check; 1631 /* 1632 * force cow if csum exists in the range. 1633 * this ensure that csum for a given extent are 1634 * either valid or do not exist. 1635 */ 1636 ret = csum_exist_in_range(fs_info, disk_bytenr, 1637 num_bytes); 1638 if (ret) { 1639 /* 1640 * ret could be -EIO if the above fails to read 1641 * metadata. 1642 */ 1643 if (ret < 0) { 1644 if (cow_start != (u64)-1) 1645 cur_offset = cow_start; 1646 goto error; 1647 } 1648 WARN_ON_ONCE(freespace_inode); 1649 goto out_check; 1650 } 1651 if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) 1652 goto out_check; 1653 nocow = true; 1654 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 1655 extent_end = found_key.offset + ram_bytes; 1656 extent_end = ALIGN(extent_end, fs_info->sectorsize); 1657 /* Skip extents outside of our requested range */ 1658 if (extent_end <= start) { 1659 path->slots[0]++; 1660 goto next_slot; 1661 } 1662 } else { 1663 /* If this triggers then we have a memory corruption */ 1664 BUG(); 1665 } 1666 out_check: 1667 /* 1668 * If nocow is false then record the beginning of the range 1669 * that needs to be COWed 1670 */ 1671 if (!nocow) { 1672 if (cow_start == (u64)-1) 1673 cow_start = cur_offset; 1674 cur_offset = extent_end; 1675 if (cur_offset > end) 1676 break; 1677 path->slots[0]++; 1678 goto next_slot; 1679 } 1680 1681 btrfs_release_path(path); 1682 1683 /* 1684 * COW range from cow_start to found_key.offset - 1. As the key 1685 * will contain the beginning of the first extent that can be 1686 * NOCOW, following one which needs to be COW'ed 1687 */ 1688 if (cow_start != (u64)-1) { 1689 ret = fallback_to_cow(inode, locked_page, 1690 cow_start, found_key.offset - 1, 1691 page_started, nr_written); 1692 if (ret) 1693 goto error; 1694 cow_start = (u64)-1; 1695 } 1696 1697 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1698 u64 orig_start = found_key.offset - extent_offset; 1699 struct extent_map *em; 1700 1701 em = create_io_em(inode, cur_offset, num_bytes, 1702 orig_start, 1703 disk_bytenr, /* block_start */ 1704 num_bytes, /* block_len */ 1705 disk_num_bytes, /* orig_block_len */ 1706 ram_bytes, BTRFS_COMPRESS_NONE, 1707 BTRFS_ORDERED_PREALLOC); 1708 if (IS_ERR(em)) { 1709 ret = PTR_ERR(em); 1710 goto error; 1711 } 1712 free_extent_map(em); 1713 ret = btrfs_add_ordered_extent(inode, cur_offset, 1714 disk_bytenr, num_bytes, 1715 num_bytes, 1716 BTRFS_ORDERED_PREALLOC); 1717 if (ret) { 1718 btrfs_drop_extent_cache(inode, cur_offset, 1719 cur_offset + num_bytes - 1, 1720 0); 1721 goto error; 1722 } 1723 } else { 1724 ret = btrfs_add_ordered_extent(inode, cur_offset, 1725 disk_bytenr, num_bytes, 1726 num_bytes, 1727 BTRFS_ORDERED_NOCOW); 1728 if (ret) 1729 goto error; 1730 } 1731 1732 if (nocow) 1733 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1734 nocow = false; 1735 1736 if (root->root_key.objectid == 1737 BTRFS_DATA_RELOC_TREE_OBJECTID) 1738 /* 1739 * Error handled later, as we must prevent 1740 * extent_clear_unlock_delalloc() in error handler 1741 * from freeing metadata of created ordered extent. 1742 */ 1743 ret = btrfs_reloc_clone_csums(inode, cur_offset, 1744 num_bytes); 1745 1746 extent_clear_unlock_delalloc(inode, cur_offset, 1747 cur_offset + num_bytes - 1, 1748 locked_page, EXTENT_LOCKED | 1749 EXTENT_DELALLOC | 1750 EXTENT_CLEAR_DATA_RESV, 1751 PAGE_UNLOCK | PAGE_SET_PRIVATE2); 1752 1753 cur_offset = extent_end; 1754 1755 /* 1756 * btrfs_reloc_clone_csums() error, now we're OK to call error 1757 * handler, as metadata for created ordered extent will only 1758 * be freed by btrfs_finish_ordered_io(). 1759 */ 1760 if (ret) 1761 goto error; 1762 if (cur_offset > end) 1763 break; 1764 } 1765 btrfs_release_path(path); 1766 1767 if (cur_offset <= end && cow_start == (u64)-1) 1768 cow_start = cur_offset; 1769 1770 if (cow_start != (u64)-1) { 1771 cur_offset = end; 1772 ret = fallback_to_cow(inode, locked_page, cow_start, end, 1773 page_started, nr_written); 1774 if (ret) 1775 goto error; 1776 } 1777 1778 error: 1779 if (nocow) 1780 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1781 1782 if (ret && cur_offset < end) 1783 extent_clear_unlock_delalloc(inode, cur_offset, end, 1784 locked_page, EXTENT_LOCKED | 1785 EXTENT_DELALLOC | EXTENT_DEFRAG | 1786 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1787 PAGE_CLEAR_DIRTY | 1788 PAGE_SET_WRITEBACK | 1789 PAGE_END_WRITEBACK); 1790 btrfs_free_path(path); 1791 return ret; 1792 } 1793 1794 static inline int need_force_cow(struct btrfs_inode *inode, u64 start, u64 end) 1795 { 1796 1797 if (!(inode->flags & BTRFS_INODE_NODATACOW) && 1798 !(inode->flags & BTRFS_INODE_PREALLOC)) 1799 return 0; 1800 1801 /* 1802 * @defrag_bytes is a hint value, no spinlock held here, 1803 * if is not zero, it means the file is defragging. 1804 * Force cow if given extent needs to be defragged. 1805 */ 1806 if (inode->defrag_bytes && 1807 test_range_bit(&inode->io_tree, start, end, EXTENT_DEFRAG, 0, NULL)) 1808 return 1; 1809 1810 return 0; 1811 } 1812 1813 /* 1814 * Function to process delayed allocation (create CoW) for ranges which are 1815 * being touched for the first time. 1816 */ 1817 int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct page *locked_page, 1818 u64 start, u64 end, int *page_started, unsigned long *nr_written, 1819 struct writeback_control *wbc) 1820 { 1821 int ret; 1822 int force_cow = need_force_cow(inode, start, end); 1823 1824 if (inode->flags & BTRFS_INODE_NODATACOW && !force_cow) { 1825 ret = run_delalloc_nocow(inode, locked_page, start, end, 1826 page_started, 1, nr_written); 1827 } else if (inode->flags & BTRFS_INODE_PREALLOC && !force_cow) { 1828 ret = run_delalloc_nocow(inode, locked_page, start, end, 1829 page_started, 0, nr_written); 1830 } else if (!inode_can_compress(inode) || 1831 !inode_need_compress(inode, start, end)) { 1832 ret = cow_file_range(inode, locked_page, start, end, 1833 page_started, nr_written, 1); 1834 } else { 1835 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags); 1836 ret = cow_file_range_async(inode, wbc, locked_page, start, end, 1837 page_started, nr_written); 1838 } 1839 if (ret) 1840 btrfs_cleanup_ordered_extents(inode, locked_page, start, 1841 end - start + 1); 1842 return ret; 1843 } 1844 1845 void btrfs_split_delalloc_extent(struct inode *inode, 1846 struct extent_state *orig, u64 split) 1847 { 1848 u64 size; 1849 1850 /* not delalloc, ignore it */ 1851 if (!(orig->state & EXTENT_DELALLOC)) 1852 return; 1853 1854 size = orig->end - orig->start + 1; 1855 if (size > BTRFS_MAX_EXTENT_SIZE) { 1856 u32 num_extents; 1857 u64 new_size; 1858 1859 /* 1860 * See the explanation in btrfs_merge_delalloc_extent, the same 1861 * applies here, just in reverse. 1862 */ 1863 new_size = orig->end - split + 1; 1864 num_extents = count_max_extents(new_size); 1865 new_size = split - orig->start; 1866 num_extents += count_max_extents(new_size); 1867 if (count_max_extents(size) >= num_extents) 1868 return; 1869 } 1870 1871 spin_lock(&BTRFS_I(inode)->lock); 1872 btrfs_mod_outstanding_extents(BTRFS_I(inode), 1); 1873 spin_unlock(&BTRFS_I(inode)->lock); 1874 } 1875 1876 /* 1877 * Handle merged delayed allocation extents so we can keep track of new extents 1878 * that are just merged onto old extents, such as when we are doing sequential 1879 * writes, so we can properly account for the metadata space we'll need. 1880 */ 1881 void btrfs_merge_delalloc_extent(struct inode *inode, struct extent_state *new, 1882 struct extent_state *other) 1883 { 1884 u64 new_size, old_size; 1885 u32 num_extents; 1886 1887 /* not delalloc, ignore it */ 1888 if (!(other->state & EXTENT_DELALLOC)) 1889 return; 1890 1891 if (new->start > other->start) 1892 new_size = new->end - other->start + 1; 1893 else 1894 new_size = other->end - new->start + 1; 1895 1896 /* we're not bigger than the max, unreserve the space and go */ 1897 if (new_size <= BTRFS_MAX_EXTENT_SIZE) { 1898 spin_lock(&BTRFS_I(inode)->lock); 1899 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1900 spin_unlock(&BTRFS_I(inode)->lock); 1901 return; 1902 } 1903 1904 /* 1905 * We have to add up either side to figure out how many extents were 1906 * accounted for before we merged into one big extent. If the number of 1907 * extents we accounted for is <= the amount we need for the new range 1908 * then we can return, otherwise drop. Think of it like this 1909 * 1910 * [ 4k][MAX_SIZE] 1911 * 1912 * So we've grown the extent by a MAX_SIZE extent, this would mean we 1913 * need 2 outstanding extents, on one side we have 1 and the other side 1914 * we have 1 so they are == and we can return. But in this case 1915 * 1916 * [MAX_SIZE+4k][MAX_SIZE+4k] 1917 * 1918 * Each range on their own accounts for 2 extents, but merged together 1919 * they are only 3 extents worth of accounting, so we need to drop in 1920 * this case. 1921 */ 1922 old_size = other->end - other->start + 1; 1923 num_extents = count_max_extents(old_size); 1924 old_size = new->end - new->start + 1; 1925 num_extents += count_max_extents(old_size); 1926 if (count_max_extents(new_size) >= num_extents) 1927 return; 1928 1929 spin_lock(&BTRFS_I(inode)->lock); 1930 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1931 spin_unlock(&BTRFS_I(inode)->lock); 1932 } 1933 1934 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 1935 struct inode *inode) 1936 { 1937 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1938 1939 spin_lock(&root->delalloc_lock); 1940 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1941 list_add_tail(&BTRFS_I(inode)->delalloc_inodes, 1942 &root->delalloc_inodes); 1943 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1944 &BTRFS_I(inode)->runtime_flags); 1945 root->nr_delalloc_inodes++; 1946 if (root->nr_delalloc_inodes == 1) { 1947 spin_lock(&fs_info->delalloc_root_lock); 1948 BUG_ON(!list_empty(&root->delalloc_root)); 1949 list_add_tail(&root->delalloc_root, 1950 &fs_info->delalloc_roots); 1951 spin_unlock(&fs_info->delalloc_root_lock); 1952 } 1953 } 1954 spin_unlock(&root->delalloc_lock); 1955 } 1956 1957 1958 void __btrfs_del_delalloc_inode(struct btrfs_root *root, 1959 struct btrfs_inode *inode) 1960 { 1961 struct btrfs_fs_info *fs_info = root->fs_info; 1962 1963 if (!list_empty(&inode->delalloc_inodes)) { 1964 list_del_init(&inode->delalloc_inodes); 1965 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1966 &inode->runtime_flags); 1967 root->nr_delalloc_inodes--; 1968 if (!root->nr_delalloc_inodes) { 1969 ASSERT(list_empty(&root->delalloc_inodes)); 1970 spin_lock(&fs_info->delalloc_root_lock); 1971 BUG_ON(list_empty(&root->delalloc_root)); 1972 list_del_init(&root->delalloc_root); 1973 spin_unlock(&fs_info->delalloc_root_lock); 1974 } 1975 } 1976 } 1977 1978 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 1979 struct btrfs_inode *inode) 1980 { 1981 spin_lock(&root->delalloc_lock); 1982 __btrfs_del_delalloc_inode(root, inode); 1983 spin_unlock(&root->delalloc_lock); 1984 } 1985 1986 /* 1987 * Properly track delayed allocation bytes in the inode and to maintain the 1988 * list of inodes that have pending delalloc work to be done. 1989 */ 1990 void btrfs_set_delalloc_extent(struct inode *inode, struct extent_state *state, 1991 unsigned *bits) 1992 { 1993 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 1994 1995 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC)) 1996 WARN_ON(1); 1997 /* 1998 * set_bit and clear bit hooks normally require _irqsave/restore 1999 * but in this case, we are only testing for the DELALLOC 2000 * bit, which is only set or cleared with irqs on 2001 */ 2002 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 2003 struct btrfs_root *root = BTRFS_I(inode)->root; 2004 u64 len = state->end + 1 - state->start; 2005 u32 num_extents = count_max_extents(len); 2006 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode)); 2007 2008 spin_lock(&BTRFS_I(inode)->lock); 2009 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents); 2010 spin_unlock(&BTRFS_I(inode)->lock); 2011 2012 /* For sanity tests */ 2013 if (btrfs_is_testing(fs_info)) 2014 return; 2015 2016 percpu_counter_add_batch(&fs_info->delalloc_bytes, len, 2017 fs_info->delalloc_batch); 2018 spin_lock(&BTRFS_I(inode)->lock); 2019 BTRFS_I(inode)->delalloc_bytes += len; 2020 if (*bits & EXTENT_DEFRAG) 2021 BTRFS_I(inode)->defrag_bytes += len; 2022 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2023 &BTRFS_I(inode)->runtime_flags)) 2024 btrfs_add_delalloc_inodes(root, inode); 2025 spin_unlock(&BTRFS_I(inode)->lock); 2026 } 2027 2028 if (!(state->state & EXTENT_DELALLOC_NEW) && 2029 (*bits & EXTENT_DELALLOC_NEW)) { 2030 spin_lock(&BTRFS_I(inode)->lock); 2031 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 - 2032 state->start; 2033 spin_unlock(&BTRFS_I(inode)->lock); 2034 } 2035 } 2036 2037 /* 2038 * Once a range is no longer delalloc this function ensures that proper 2039 * accounting happens. 2040 */ 2041 void btrfs_clear_delalloc_extent(struct inode *vfs_inode, 2042 struct extent_state *state, unsigned *bits) 2043 { 2044 struct btrfs_inode *inode = BTRFS_I(vfs_inode); 2045 struct btrfs_fs_info *fs_info = btrfs_sb(vfs_inode->i_sb); 2046 u64 len = state->end + 1 - state->start; 2047 u32 num_extents = count_max_extents(len); 2048 2049 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) { 2050 spin_lock(&inode->lock); 2051 inode->defrag_bytes -= len; 2052 spin_unlock(&inode->lock); 2053 } 2054 2055 /* 2056 * set_bit and clear bit hooks normally require _irqsave/restore 2057 * but in this case, we are only testing for the DELALLOC 2058 * bit, which is only set or cleared with irqs on 2059 */ 2060 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 2061 struct btrfs_root *root = inode->root; 2062 bool do_list = !btrfs_is_free_space_inode(inode); 2063 2064 spin_lock(&inode->lock); 2065 btrfs_mod_outstanding_extents(inode, -num_extents); 2066 spin_unlock(&inode->lock); 2067 2068 /* 2069 * We don't reserve metadata space for space cache inodes so we 2070 * don't need to call delalloc_release_metadata if there is an 2071 * error. 2072 */ 2073 if (*bits & EXTENT_CLEAR_META_RESV && 2074 root != fs_info->tree_root) 2075 btrfs_delalloc_release_metadata(inode, len, false); 2076 2077 /* For sanity tests. */ 2078 if (btrfs_is_testing(fs_info)) 2079 return; 2080 2081 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID && 2082 do_list && !(state->state & EXTENT_NORESERVE) && 2083 (*bits & EXTENT_CLEAR_DATA_RESV)) 2084 btrfs_free_reserved_data_space_noquota(fs_info, len); 2085 2086 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, 2087 fs_info->delalloc_batch); 2088 spin_lock(&inode->lock); 2089 inode->delalloc_bytes -= len; 2090 if (do_list && inode->delalloc_bytes == 0 && 2091 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2092 &inode->runtime_flags)) 2093 btrfs_del_delalloc_inode(root, inode); 2094 spin_unlock(&inode->lock); 2095 } 2096 2097 if ((state->state & EXTENT_DELALLOC_NEW) && 2098 (*bits & EXTENT_DELALLOC_NEW)) { 2099 spin_lock(&inode->lock); 2100 ASSERT(inode->new_delalloc_bytes >= len); 2101 inode->new_delalloc_bytes -= len; 2102 spin_unlock(&inode->lock); 2103 } 2104 } 2105 2106 /* 2107 * btrfs_bio_fits_in_stripe - Checks whether the size of the given bio will fit 2108 * in a chunk's stripe. This function ensures that bios do not span a 2109 * stripe/chunk 2110 * 2111 * @page - The page we are about to add to the bio 2112 * @size - size we want to add to the bio 2113 * @bio - bio we want to ensure is smaller than a stripe 2114 * @bio_flags - flags of the bio 2115 * 2116 * return 1 if page cannot be added to the bio 2117 * return 0 if page can be added to the bio 2118 * return error otherwise 2119 */ 2120 int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, 2121 unsigned long bio_flags) 2122 { 2123 struct inode *inode = page->mapping->host; 2124 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2125 u64 logical = (u64)bio->bi_iter.bi_sector << 9; 2126 u64 length = 0; 2127 u64 map_length; 2128 int ret; 2129 struct btrfs_io_geometry geom; 2130 2131 if (bio_flags & EXTENT_BIO_COMPRESSED) 2132 return 0; 2133 2134 length = bio->bi_iter.bi_size; 2135 map_length = length; 2136 ret = btrfs_get_io_geometry(fs_info, btrfs_op(bio), logical, map_length, 2137 &geom); 2138 if (ret < 0) 2139 return ret; 2140 2141 if (geom.len < length + size) 2142 return 1; 2143 return 0; 2144 } 2145 2146 /* 2147 * in order to insert checksums into the metadata in large chunks, 2148 * we wait until bio submission time. All the pages in the bio are 2149 * checksummed and sums are attached onto the ordered extent record. 2150 * 2151 * At IO completion time the cums attached on the ordered extent record 2152 * are inserted into the btree 2153 */ 2154 static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio, 2155 u64 bio_offset) 2156 { 2157 struct inode *inode = private_data; 2158 blk_status_t ret = 0; 2159 2160 ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0); 2161 BUG_ON(ret); /* -ENOMEM */ 2162 return 0; 2163 } 2164 2165 /* 2166 * extent_io.c submission hook. This does the right thing for csum calculation 2167 * on write, or reading the csums from the tree before a read. 2168 * 2169 * Rules about async/sync submit, 2170 * a) read: sync submit 2171 * 2172 * b) write without checksum: sync submit 2173 * 2174 * c) write with checksum: 2175 * c-1) if bio is issued by fsync: sync submit 2176 * (sync_writers != 0) 2177 * 2178 * c-2) if root is reloc root: sync submit 2179 * (only in case of buffered IO) 2180 * 2181 * c-3) otherwise: async submit 2182 */ 2183 static blk_status_t btrfs_submit_bio_hook(struct inode *inode, struct bio *bio, 2184 int mirror_num, 2185 unsigned long bio_flags) 2186 2187 { 2188 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2189 struct btrfs_root *root = BTRFS_I(inode)->root; 2190 enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA; 2191 blk_status_t ret = 0; 2192 int skip_sum; 2193 int async = !atomic_read(&BTRFS_I(inode)->sync_writers); 2194 2195 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 2196 2197 if (btrfs_is_free_space_inode(BTRFS_I(inode))) 2198 metadata = BTRFS_WQ_ENDIO_FREE_SPACE; 2199 2200 if (bio_op(bio) != REQ_OP_WRITE) { 2201 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); 2202 if (ret) 2203 goto out; 2204 2205 if (bio_flags & EXTENT_BIO_COMPRESSED) { 2206 ret = btrfs_submit_compressed_read(inode, bio, 2207 mirror_num, 2208 bio_flags); 2209 goto out; 2210 } else if (!skip_sum) { 2211 ret = btrfs_lookup_bio_sums(inode, bio, (u64)-1, NULL); 2212 if (ret) 2213 goto out; 2214 } 2215 goto mapit; 2216 } else if (async && !skip_sum) { 2217 /* csum items have already been cloned */ 2218 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 2219 goto mapit; 2220 /* we're doing a write, do the async checksumming */ 2221 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags, 2222 0, inode, btrfs_submit_bio_start); 2223 goto out; 2224 } else if (!skip_sum) { 2225 ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0); 2226 if (ret) 2227 goto out; 2228 } 2229 2230 mapit: 2231 ret = btrfs_map_bio(fs_info, bio, mirror_num); 2232 2233 out: 2234 if (ret) { 2235 bio->bi_status = ret; 2236 bio_endio(bio); 2237 } 2238 return ret; 2239 } 2240 2241 /* 2242 * given a list of ordered sums record them in the inode. This happens 2243 * at IO completion time based on sums calculated at bio submission time. 2244 */ 2245 static noinline int add_pending_csums(struct btrfs_trans_handle *trans, 2246 struct inode *inode, struct list_head *list) 2247 { 2248 struct btrfs_ordered_sum *sum; 2249 int ret; 2250 2251 list_for_each_entry(sum, list, list) { 2252 trans->adding_csums = true; 2253 ret = btrfs_csum_file_blocks(trans, 2254 BTRFS_I(inode)->root->fs_info->csum_root, sum); 2255 trans->adding_csums = false; 2256 if (ret) 2257 return ret; 2258 } 2259 return 0; 2260 } 2261 2262 int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, 2263 unsigned int extra_bits, 2264 struct extent_state **cached_state) 2265 { 2266 WARN_ON(PAGE_ALIGNED(end)); 2267 return set_extent_delalloc(&inode->io_tree, start, end, extra_bits, 2268 cached_state); 2269 } 2270 2271 /* see btrfs_writepage_start_hook for details on why this is required */ 2272 struct btrfs_writepage_fixup { 2273 struct page *page; 2274 struct inode *inode; 2275 struct btrfs_work work; 2276 }; 2277 2278 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2279 { 2280 struct btrfs_writepage_fixup *fixup; 2281 struct btrfs_ordered_extent *ordered; 2282 struct extent_state *cached_state = NULL; 2283 struct extent_changeset *data_reserved = NULL; 2284 struct page *page; 2285 struct btrfs_inode *inode; 2286 u64 page_start; 2287 u64 page_end; 2288 int ret = 0; 2289 bool free_delalloc_space = true; 2290 2291 fixup = container_of(work, struct btrfs_writepage_fixup, work); 2292 page = fixup->page; 2293 inode = BTRFS_I(fixup->inode); 2294 page_start = page_offset(page); 2295 page_end = page_offset(page) + PAGE_SIZE - 1; 2296 2297 /* 2298 * This is similar to page_mkwrite, we need to reserve the space before 2299 * we take the page lock. 2300 */ 2301 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 2302 PAGE_SIZE); 2303 again: 2304 lock_page(page); 2305 2306 /* 2307 * Before we queued this fixup, we took a reference on the page. 2308 * page->mapping may go NULL, but it shouldn't be moved to a different 2309 * address space. 2310 */ 2311 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 2312 /* 2313 * Unfortunately this is a little tricky, either 2314 * 2315 * 1) We got here and our page had already been dealt with and 2316 * we reserved our space, thus ret == 0, so we need to just 2317 * drop our space reservation and bail. This can happen the 2318 * first time we come into the fixup worker, or could happen 2319 * while waiting for the ordered extent. 2320 * 2) Our page was already dealt with, but we happened to get an 2321 * ENOSPC above from the btrfs_delalloc_reserve_space. In 2322 * this case we obviously don't have anything to release, but 2323 * because the page was already dealt with we don't want to 2324 * mark the page with an error, so make sure we're resetting 2325 * ret to 0. This is why we have this check _before_ the ret 2326 * check, because we do not want to have a surprise ENOSPC 2327 * when the page was already properly dealt with. 2328 */ 2329 if (!ret) { 2330 btrfs_delalloc_release_extents(inode, PAGE_SIZE); 2331 btrfs_delalloc_release_space(inode, data_reserved, 2332 page_start, PAGE_SIZE, 2333 true); 2334 } 2335 ret = 0; 2336 goto out_page; 2337 } 2338 2339 /* 2340 * We can't mess with the page state unless it is locked, so now that 2341 * it is locked bail if we failed to make our space reservation. 2342 */ 2343 if (ret) 2344 goto out_page; 2345 2346 lock_extent_bits(&inode->io_tree, page_start, page_end, &cached_state); 2347 2348 /* already ordered? We're done */ 2349 if (PagePrivate2(page)) 2350 goto out_reserved; 2351 2352 ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); 2353 if (ordered) { 2354 unlock_extent_cached(&inode->io_tree, page_start, page_end, 2355 &cached_state); 2356 unlock_page(page); 2357 btrfs_start_ordered_extent(&inode->vfs_inode, ordered, 1); 2358 btrfs_put_ordered_extent(ordered); 2359 goto again; 2360 } 2361 2362 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, 2363 &cached_state); 2364 if (ret) 2365 goto out_reserved; 2366 2367 /* 2368 * Everything went as planned, we're now the owner of a dirty page with 2369 * delayed allocation bits set and space reserved for our COW 2370 * destination. 2371 * 2372 * The page was dirty when we started, nothing should have cleaned it. 2373 */ 2374 BUG_ON(!PageDirty(page)); 2375 free_delalloc_space = false; 2376 out_reserved: 2377 btrfs_delalloc_release_extents(inode, PAGE_SIZE); 2378 if (free_delalloc_space) 2379 btrfs_delalloc_release_space(inode, data_reserved, page_start, 2380 PAGE_SIZE, true); 2381 unlock_extent_cached(&inode->io_tree, page_start, page_end, 2382 &cached_state); 2383 out_page: 2384 if (ret) { 2385 /* 2386 * We hit ENOSPC or other errors. Update the mapping and page 2387 * to reflect the errors and clean the page. 2388 */ 2389 mapping_set_error(page->mapping, ret); 2390 end_extent_writepage(page, ret, page_start, page_end); 2391 clear_page_dirty_for_io(page); 2392 SetPageError(page); 2393 } 2394 ClearPageChecked(page); 2395 unlock_page(page); 2396 put_page(page); 2397 kfree(fixup); 2398 extent_changeset_free(data_reserved); 2399 /* 2400 * As a precaution, do a delayed iput in case it would be the last iput 2401 * that could need flushing space. Recursing back to fixup worker would 2402 * deadlock. 2403 */ 2404 btrfs_add_delayed_iput(&inode->vfs_inode); 2405 } 2406 2407 /* 2408 * There are a few paths in the higher layers of the kernel that directly 2409 * set the page dirty bit without asking the filesystem if it is a 2410 * good idea. This causes problems because we want to make sure COW 2411 * properly happens and the data=ordered rules are followed. 2412 * 2413 * In our case any range that doesn't have the ORDERED bit set 2414 * hasn't been properly setup for IO. We kick off an async process 2415 * to fix it up. The async helper will wait for ordered extents, set 2416 * the delalloc bit and make it safe to write the page. 2417 */ 2418 int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end) 2419 { 2420 struct inode *inode = page->mapping->host; 2421 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2422 struct btrfs_writepage_fixup *fixup; 2423 2424 /* this page is properly in the ordered list */ 2425 if (TestClearPagePrivate2(page)) 2426 return 0; 2427 2428 /* 2429 * PageChecked is set below when we create a fixup worker for this page, 2430 * don't try to create another one if we're already PageChecked() 2431 * 2432 * The extent_io writepage code will redirty the page if we send back 2433 * EAGAIN. 2434 */ 2435 if (PageChecked(page)) 2436 return -EAGAIN; 2437 2438 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2439 if (!fixup) 2440 return -EAGAIN; 2441 2442 /* 2443 * We are already holding a reference to this inode from 2444 * write_cache_pages. We need to hold it because the space reservation 2445 * takes place outside of the page lock, and we can't trust 2446 * page->mapping outside of the page lock. 2447 */ 2448 ihold(inode); 2449 SetPageChecked(page); 2450 get_page(page); 2451 btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL, NULL); 2452 fixup->page = page; 2453 fixup->inode = inode; 2454 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2455 2456 return -EAGAIN; 2457 } 2458 2459 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2460 struct btrfs_inode *inode, u64 file_pos, 2461 struct btrfs_file_extent_item *stack_fi, 2462 u64 qgroup_reserved) 2463 { 2464 struct btrfs_root *root = inode->root; 2465 struct btrfs_path *path; 2466 struct extent_buffer *leaf; 2467 struct btrfs_key ins; 2468 u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi); 2469 u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi); 2470 u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi); 2471 u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi); 2472 int extent_inserted = 0; 2473 int ret; 2474 2475 path = btrfs_alloc_path(); 2476 if (!path) 2477 return -ENOMEM; 2478 2479 /* 2480 * we may be replacing one extent in the tree with another. 2481 * The new extent is pinned in the extent map, and we don't want 2482 * to drop it from the cache until it is completely in the btree. 2483 * 2484 * So, tell btrfs_drop_extents to leave this extent in the cache. 2485 * the caller is expected to unpin it and allow it to be merged 2486 * with the others. 2487 */ 2488 ret = __btrfs_drop_extents(trans, root, inode, path, file_pos, 2489 file_pos + num_bytes, NULL, 0, 2490 1, sizeof(*stack_fi), &extent_inserted); 2491 if (ret) 2492 goto out; 2493 2494 if (!extent_inserted) { 2495 ins.objectid = btrfs_ino(inode); 2496 ins.offset = file_pos; 2497 ins.type = BTRFS_EXTENT_DATA_KEY; 2498 2499 path->leave_spinning = 1; 2500 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2501 sizeof(*stack_fi)); 2502 if (ret) 2503 goto out; 2504 } 2505 leaf = path->nodes[0]; 2506 btrfs_set_stack_file_extent_generation(stack_fi, trans->transid); 2507 write_extent_buffer(leaf, stack_fi, 2508 btrfs_item_ptr_offset(leaf, path->slots[0]), 2509 sizeof(struct btrfs_file_extent_item)); 2510 2511 btrfs_mark_buffer_dirty(leaf); 2512 btrfs_release_path(path); 2513 2514 inode_add_bytes(&inode->vfs_inode, num_bytes); 2515 2516 ins.objectid = disk_bytenr; 2517 ins.offset = disk_num_bytes; 2518 ins.type = BTRFS_EXTENT_ITEM_KEY; 2519 2520 ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes); 2521 if (ret) 2522 goto out; 2523 2524 ret = btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode), 2525 file_pos, qgroup_reserved, &ins); 2526 out: 2527 btrfs_free_path(path); 2528 2529 return ret; 2530 } 2531 2532 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 2533 u64 start, u64 len) 2534 { 2535 struct btrfs_block_group *cache; 2536 2537 cache = btrfs_lookup_block_group(fs_info, start); 2538 ASSERT(cache); 2539 2540 spin_lock(&cache->lock); 2541 cache->delalloc_bytes -= len; 2542 spin_unlock(&cache->lock); 2543 2544 btrfs_put_block_group(cache); 2545 } 2546 2547 static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans, 2548 struct inode *inode, 2549 struct btrfs_ordered_extent *oe) 2550 { 2551 struct btrfs_file_extent_item stack_fi; 2552 u64 logical_len; 2553 2554 memset(&stack_fi, 0, sizeof(stack_fi)); 2555 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG); 2556 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr); 2557 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, 2558 oe->disk_num_bytes); 2559 if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags)) 2560 logical_len = oe->truncated_len; 2561 else 2562 logical_len = oe->num_bytes; 2563 btrfs_set_stack_file_extent_num_bytes(&stack_fi, logical_len); 2564 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, logical_len); 2565 btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type); 2566 /* Encryption and other encoding is reserved and all 0 */ 2567 2568 return insert_reserved_file_extent(trans, BTRFS_I(inode), oe->file_offset, 2569 &stack_fi, oe->qgroup_rsv); 2570 } 2571 2572 /* 2573 * As ordered data IO finishes, this gets called so we can finish 2574 * an ordered extent if the range of bytes in the file it covers are 2575 * fully written. 2576 */ 2577 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 2578 { 2579 struct inode *inode = ordered_extent->inode; 2580 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2581 struct btrfs_root *root = BTRFS_I(inode)->root; 2582 struct btrfs_trans_handle *trans = NULL; 2583 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2584 struct extent_state *cached_state = NULL; 2585 u64 start, end; 2586 int compress_type = 0; 2587 int ret = 0; 2588 u64 logical_len = ordered_extent->num_bytes; 2589 bool freespace_inode; 2590 bool truncated = false; 2591 bool range_locked = false; 2592 bool clear_new_delalloc_bytes = false; 2593 bool clear_reserved_extent = true; 2594 unsigned int clear_bits; 2595 2596 start = ordered_extent->file_offset; 2597 end = start + ordered_extent->num_bytes - 1; 2598 2599 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 2600 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && 2601 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags)) 2602 clear_new_delalloc_bytes = true; 2603 2604 freespace_inode = btrfs_is_free_space_inode(BTRFS_I(inode)); 2605 2606 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 2607 ret = -EIO; 2608 goto out; 2609 } 2610 2611 btrfs_free_io_failure_record(BTRFS_I(inode), start, end); 2612 2613 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 2614 truncated = true; 2615 logical_len = ordered_extent->truncated_len; 2616 /* Truncated the entire extent, don't bother adding */ 2617 if (!logical_len) 2618 goto out; 2619 } 2620 2621 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 2622 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 2623 2624 btrfs_inode_safe_disk_i_size_write(inode, 0); 2625 if (freespace_inode) 2626 trans = btrfs_join_transaction_spacecache(root); 2627 else 2628 trans = btrfs_join_transaction(root); 2629 if (IS_ERR(trans)) { 2630 ret = PTR_ERR(trans); 2631 trans = NULL; 2632 goto out; 2633 } 2634 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 2635 ret = btrfs_update_inode_fallback(trans, root, inode); 2636 if (ret) /* -ENOMEM or corruption */ 2637 btrfs_abort_transaction(trans, ret); 2638 goto out; 2639 } 2640 2641 range_locked = true; 2642 lock_extent_bits(io_tree, start, end, &cached_state); 2643 2644 if (freespace_inode) 2645 trans = btrfs_join_transaction_spacecache(root); 2646 else 2647 trans = btrfs_join_transaction(root); 2648 if (IS_ERR(trans)) { 2649 ret = PTR_ERR(trans); 2650 trans = NULL; 2651 goto out; 2652 } 2653 2654 trans->block_rsv = &BTRFS_I(inode)->block_rsv; 2655 2656 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 2657 compress_type = ordered_extent->compress_type; 2658 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 2659 BUG_ON(compress_type); 2660 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode), 2661 ordered_extent->file_offset, 2662 ordered_extent->file_offset + 2663 logical_len); 2664 } else { 2665 BUG_ON(root == fs_info->tree_root); 2666 ret = insert_ordered_extent_file_extent(trans, inode, 2667 ordered_extent); 2668 if (!ret) { 2669 clear_reserved_extent = false; 2670 btrfs_release_delalloc_bytes(fs_info, 2671 ordered_extent->disk_bytenr, 2672 ordered_extent->disk_num_bytes); 2673 } 2674 } 2675 unpin_extent_cache(&BTRFS_I(inode)->extent_tree, 2676 ordered_extent->file_offset, 2677 ordered_extent->num_bytes, trans->transid); 2678 if (ret < 0) { 2679 btrfs_abort_transaction(trans, ret); 2680 goto out; 2681 } 2682 2683 ret = add_pending_csums(trans, inode, &ordered_extent->list); 2684 if (ret) { 2685 btrfs_abort_transaction(trans, ret); 2686 goto out; 2687 } 2688 2689 btrfs_inode_safe_disk_i_size_write(inode, 0); 2690 ret = btrfs_update_inode_fallback(trans, root, inode); 2691 if (ret) { /* -ENOMEM or corruption */ 2692 btrfs_abort_transaction(trans, ret); 2693 goto out; 2694 } 2695 ret = 0; 2696 out: 2697 clear_bits = EXTENT_DEFRAG; 2698 if (range_locked) 2699 clear_bits |= EXTENT_LOCKED; 2700 if (clear_new_delalloc_bytes) 2701 clear_bits |= EXTENT_DELALLOC_NEW; 2702 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 2703 (clear_bits & EXTENT_LOCKED) ? 1 : 0, 0, 2704 &cached_state); 2705 2706 if (trans) 2707 btrfs_end_transaction(trans); 2708 2709 if (ret || truncated) { 2710 u64 unwritten_start = start; 2711 2712 if (truncated) 2713 unwritten_start += logical_len; 2714 clear_extent_uptodate(io_tree, unwritten_start, end, NULL); 2715 2716 /* Drop the cache for the part of the extent we didn't write. */ 2717 btrfs_drop_extent_cache(BTRFS_I(inode), unwritten_start, end, 0); 2718 2719 /* 2720 * If the ordered extent had an IOERR or something else went 2721 * wrong we need to return the space for this ordered extent 2722 * back to the allocator. We only free the extent in the 2723 * truncated case if we didn't write out the extent at all. 2724 * 2725 * If we made it past insert_reserved_file_extent before we 2726 * errored out then we don't need to do this as the accounting 2727 * has already been done. 2728 */ 2729 if ((ret || !logical_len) && 2730 clear_reserved_extent && 2731 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 2732 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 2733 /* 2734 * Discard the range before returning it back to the 2735 * free space pool 2736 */ 2737 if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC)) 2738 btrfs_discard_extent(fs_info, 2739 ordered_extent->disk_bytenr, 2740 ordered_extent->disk_num_bytes, 2741 NULL); 2742 btrfs_free_reserved_extent(fs_info, 2743 ordered_extent->disk_bytenr, 2744 ordered_extent->disk_num_bytes, 1); 2745 } 2746 } 2747 2748 /* 2749 * This needs to be done to make sure anybody waiting knows we are done 2750 * updating everything for this ordered extent. 2751 */ 2752 btrfs_remove_ordered_extent(inode, ordered_extent); 2753 2754 /* once for us */ 2755 btrfs_put_ordered_extent(ordered_extent); 2756 /* once for the tree */ 2757 btrfs_put_ordered_extent(ordered_extent); 2758 2759 return ret; 2760 } 2761 2762 static void finish_ordered_fn(struct btrfs_work *work) 2763 { 2764 struct btrfs_ordered_extent *ordered_extent; 2765 ordered_extent = container_of(work, struct btrfs_ordered_extent, work); 2766 btrfs_finish_ordered_io(ordered_extent); 2767 } 2768 2769 void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, 2770 u64 end, int uptodate) 2771 { 2772 struct inode *inode = page->mapping->host; 2773 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2774 struct btrfs_ordered_extent *ordered_extent = NULL; 2775 struct btrfs_workqueue *wq; 2776 2777 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); 2778 2779 ClearPagePrivate2(page); 2780 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, 2781 end - start + 1, uptodate)) 2782 return; 2783 2784 if (btrfs_is_free_space_inode(BTRFS_I(inode))) 2785 wq = fs_info->endio_freespace_worker; 2786 else 2787 wq = fs_info->endio_write_workers; 2788 2789 btrfs_init_work(&ordered_extent->work, finish_ordered_fn, NULL, NULL); 2790 btrfs_queue_work(wq, &ordered_extent->work); 2791 } 2792 2793 static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, 2794 int icsum, struct page *page, int pgoff, u64 start, 2795 size_t len) 2796 { 2797 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2798 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 2799 char *kaddr; 2800 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 2801 u8 *csum_expected; 2802 u8 csum[BTRFS_CSUM_SIZE]; 2803 2804 csum_expected = ((u8 *)io_bio->csum) + icsum * csum_size; 2805 2806 kaddr = kmap_atomic(page); 2807 shash->tfm = fs_info->csum_shash; 2808 2809 crypto_shash_digest(shash, kaddr + pgoff, len, csum); 2810 2811 if (memcmp(csum, csum_expected, csum_size)) 2812 goto zeroit; 2813 2814 kunmap_atomic(kaddr); 2815 return 0; 2816 zeroit: 2817 btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected, 2818 io_bio->mirror_num); 2819 if (io_bio->device) 2820 btrfs_dev_stat_inc_and_print(io_bio->device, 2821 BTRFS_DEV_STAT_CORRUPTION_ERRS); 2822 memset(kaddr + pgoff, 1, len); 2823 flush_dcache_page(page); 2824 kunmap_atomic(kaddr); 2825 return -EIO; 2826 } 2827 2828 /* 2829 * when reads are done, we need to check csums to verify the data is correct 2830 * if there's a match, we allow the bio to finish. If not, the code in 2831 * extent_io.c will try to find good copies for us. 2832 */ 2833 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, 2834 u64 phy_offset, struct page *page, 2835 u64 start, u64 end, int mirror) 2836 { 2837 size_t offset = start - page_offset(page); 2838 struct inode *inode = page->mapping->host; 2839 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2840 struct btrfs_root *root = BTRFS_I(inode)->root; 2841 2842 if (PageChecked(page)) { 2843 ClearPageChecked(page); 2844 return 0; 2845 } 2846 2847 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 2848 return 0; 2849 2850 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID && 2851 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) { 2852 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM); 2853 return 0; 2854 } 2855 2856 phy_offset >>= inode->i_sb->s_blocksize_bits; 2857 return check_data_csum(inode, io_bio, phy_offset, page, offset, start, 2858 (size_t)(end - start + 1)); 2859 } 2860 2861 /* 2862 * btrfs_add_delayed_iput - perform a delayed iput on @inode 2863 * 2864 * @inode: The inode we want to perform iput on 2865 * 2866 * This function uses the generic vfs_inode::i_count to track whether we should 2867 * just decrement it (in case it's > 1) or if this is the last iput then link 2868 * the inode to the delayed iput machinery. Delayed iputs are processed at 2869 * transaction commit time/superblock commit/cleaner kthread. 2870 */ 2871 void btrfs_add_delayed_iput(struct inode *inode) 2872 { 2873 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2874 struct btrfs_inode *binode = BTRFS_I(inode); 2875 2876 if (atomic_add_unless(&inode->i_count, -1, 1)) 2877 return; 2878 2879 atomic_inc(&fs_info->nr_delayed_iputs); 2880 spin_lock(&fs_info->delayed_iput_lock); 2881 ASSERT(list_empty(&binode->delayed_iput)); 2882 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); 2883 spin_unlock(&fs_info->delayed_iput_lock); 2884 if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags)) 2885 wake_up_process(fs_info->cleaner_kthread); 2886 } 2887 2888 static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info, 2889 struct btrfs_inode *inode) 2890 { 2891 list_del_init(&inode->delayed_iput); 2892 spin_unlock(&fs_info->delayed_iput_lock); 2893 iput(&inode->vfs_inode); 2894 if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) 2895 wake_up(&fs_info->delayed_iputs_wait); 2896 spin_lock(&fs_info->delayed_iput_lock); 2897 } 2898 2899 static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info, 2900 struct btrfs_inode *inode) 2901 { 2902 if (!list_empty(&inode->delayed_iput)) { 2903 spin_lock(&fs_info->delayed_iput_lock); 2904 if (!list_empty(&inode->delayed_iput)) 2905 run_delayed_iput_locked(fs_info, inode); 2906 spin_unlock(&fs_info->delayed_iput_lock); 2907 } 2908 } 2909 2910 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 2911 { 2912 2913 spin_lock(&fs_info->delayed_iput_lock); 2914 while (!list_empty(&fs_info->delayed_iputs)) { 2915 struct btrfs_inode *inode; 2916 2917 inode = list_first_entry(&fs_info->delayed_iputs, 2918 struct btrfs_inode, delayed_iput); 2919 run_delayed_iput_locked(fs_info, inode); 2920 } 2921 spin_unlock(&fs_info->delayed_iput_lock); 2922 } 2923 2924 /** 2925 * btrfs_wait_on_delayed_iputs - wait on the delayed iputs to be done running 2926 * @fs_info - the fs_info for this fs 2927 * @return - EINTR if we were killed, 0 if nothing's pending 2928 * 2929 * This will wait on any delayed iputs that are currently running with KILLABLE 2930 * set. Once they are all done running we will return, unless we are killed in 2931 * which case we return EINTR. This helps in user operations like fallocate etc 2932 * that might get blocked on the iputs. 2933 */ 2934 int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info) 2935 { 2936 int ret = wait_event_killable(fs_info->delayed_iputs_wait, 2937 atomic_read(&fs_info->nr_delayed_iputs) == 0); 2938 if (ret) 2939 return -EINTR; 2940 return 0; 2941 } 2942 2943 /* 2944 * This creates an orphan entry for the given inode in case something goes wrong 2945 * in the middle of an unlink. 2946 */ 2947 int btrfs_orphan_add(struct btrfs_trans_handle *trans, 2948 struct btrfs_inode *inode) 2949 { 2950 int ret; 2951 2952 ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode)); 2953 if (ret && ret != -EEXIST) { 2954 btrfs_abort_transaction(trans, ret); 2955 return ret; 2956 } 2957 2958 return 0; 2959 } 2960 2961 /* 2962 * We have done the delete so we can go ahead and remove the orphan item for 2963 * this particular inode. 2964 */ 2965 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 2966 struct btrfs_inode *inode) 2967 { 2968 return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode)); 2969 } 2970 2971 /* 2972 * this cleans up any orphans that may be left on the list from the last use 2973 * of this root. 2974 */ 2975 int btrfs_orphan_cleanup(struct btrfs_root *root) 2976 { 2977 struct btrfs_fs_info *fs_info = root->fs_info; 2978 struct btrfs_path *path; 2979 struct extent_buffer *leaf; 2980 struct btrfs_key key, found_key; 2981 struct btrfs_trans_handle *trans; 2982 struct inode *inode; 2983 u64 last_objectid = 0; 2984 int ret = 0, nr_unlink = 0; 2985 2986 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED)) 2987 return 0; 2988 2989 path = btrfs_alloc_path(); 2990 if (!path) { 2991 ret = -ENOMEM; 2992 goto out; 2993 } 2994 path->reada = READA_BACK; 2995 2996 key.objectid = BTRFS_ORPHAN_OBJECTID; 2997 key.type = BTRFS_ORPHAN_ITEM_KEY; 2998 key.offset = (u64)-1; 2999 3000 while (1) { 3001 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3002 if (ret < 0) 3003 goto out; 3004 3005 /* 3006 * if ret == 0 means we found what we were searching for, which 3007 * is weird, but possible, so only screw with path if we didn't 3008 * find the key and see if we have stuff that matches 3009 */ 3010 if (ret > 0) { 3011 ret = 0; 3012 if (path->slots[0] == 0) 3013 break; 3014 path->slots[0]--; 3015 } 3016 3017 /* pull out the item */ 3018 leaf = path->nodes[0]; 3019 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3020 3021 /* make sure the item matches what we want */ 3022 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3023 break; 3024 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3025 break; 3026 3027 /* release the path since we're done with it */ 3028 btrfs_release_path(path); 3029 3030 /* 3031 * this is where we are basically btrfs_lookup, without the 3032 * crossing root thing. we store the inode number in the 3033 * offset of the orphan item. 3034 */ 3035 3036 if (found_key.offset == last_objectid) { 3037 btrfs_err(fs_info, 3038 "Error removing orphan entry, stopping orphan cleanup"); 3039 ret = -EINVAL; 3040 goto out; 3041 } 3042 3043 last_objectid = found_key.offset; 3044 3045 found_key.objectid = found_key.offset; 3046 found_key.type = BTRFS_INODE_ITEM_KEY; 3047 found_key.offset = 0; 3048 inode = btrfs_iget(fs_info->sb, last_objectid, root); 3049 ret = PTR_ERR_OR_ZERO(inode); 3050 if (ret && ret != -ENOENT) 3051 goto out; 3052 3053 if (ret == -ENOENT && root == fs_info->tree_root) { 3054 struct btrfs_root *dead_root; 3055 struct btrfs_fs_info *fs_info = root->fs_info; 3056 int is_dead_root = 0; 3057 3058 /* 3059 * this is an orphan in the tree root. Currently these 3060 * could come from 2 sources: 3061 * a) a snapshot deletion in progress 3062 * b) a free space cache inode 3063 * We need to distinguish those two, as the snapshot 3064 * orphan must not get deleted. 3065 * find_dead_roots already ran before us, so if this 3066 * is a snapshot deletion, we should find the root 3067 * in the fs_roots radix tree. 3068 */ 3069 3070 spin_lock(&fs_info->fs_roots_radix_lock); 3071 dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, 3072 (unsigned long)found_key.objectid); 3073 if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) 3074 is_dead_root = 1; 3075 spin_unlock(&fs_info->fs_roots_radix_lock); 3076 3077 if (is_dead_root) { 3078 /* prevent this orphan from being found again */ 3079 key.offset = found_key.objectid - 1; 3080 continue; 3081 } 3082 3083 } 3084 3085 /* 3086 * If we have an inode with links, there are a couple of 3087 * possibilities. Old kernels (before v3.12) used to create an 3088 * orphan item for truncate indicating that there were possibly 3089 * extent items past i_size that needed to be deleted. In v3.12, 3090 * truncate was changed to update i_size in sync with the extent 3091 * items, but the (useless) orphan item was still created. Since 3092 * v4.18, we don't create the orphan item for truncate at all. 3093 * 3094 * So, this item could mean that we need to do a truncate, but 3095 * only if this filesystem was last used on a pre-v3.12 kernel 3096 * and was not cleanly unmounted. The odds of that are quite 3097 * slim, and it's a pain to do the truncate now, so just delete 3098 * the orphan item. 3099 * 3100 * It's also possible that this orphan item was supposed to be 3101 * deleted but wasn't. The inode number may have been reused, 3102 * but either way, we can delete the orphan item. 3103 */ 3104 if (ret == -ENOENT || inode->i_nlink) { 3105 if (!ret) 3106 iput(inode); 3107 trans = btrfs_start_transaction(root, 1); 3108 if (IS_ERR(trans)) { 3109 ret = PTR_ERR(trans); 3110 goto out; 3111 } 3112 btrfs_debug(fs_info, "auto deleting %Lu", 3113 found_key.objectid); 3114 ret = btrfs_del_orphan_item(trans, root, 3115 found_key.objectid); 3116 btrfs_end_transaction(trans); 3117 if (ret) 3118 goto out; 3119 continue; 3120 } 3121 3122 nr_unlink++; 3123 3124 /* this will do delete_inode and everything for us */ 3125 iput(inode); 3126 } 3127 /* release the path since we're done with it */ 3128 btrfs_release_path(path); 3129 3130 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE; 3131 3132 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3133 trans = btrfs_join_transaction(root); 3134 if (!IS_ERR(trans)) 3135 btrfs_end_transaction(trans); 3136 } 3137 3138 if (nr_unlink) 3139 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3140 3141 out: 3142 if (ret) 3143 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3144 btrfs_free_path(path); 3145 return ret; 3146 } 3147 3148 /* 3149 * very simple check to peek ahead in the leaf looking for xattrs. If we 3150 * don't find any xattrs, we know there can't be any acls. 3151 * 3152 * slot is the slot the inode is in, objectid is the objectid of the inode 3153 */ 3154 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3155 int slot, u64 objectid, 3156 int *first_xattr_slot) 3157 { 3158 u32 nritems = btrfs_header_nritems(leaf); 3159 struct btrfs_key found_key; 3160 static u64 xattr_access = 0; 3161 static u64 xattr_default = 0; 3162 int scanned = 0; 3163 3164 if (!xattr_access) { 3165 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3166 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3167 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3168 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3169 } 3170 3171 slot++; 3172 *first_xattr_slot = -1; 3173 while (slot < nritems) { 3174 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3175 3176 /* we found a different objectid, there must not be acls */ 3177 if (found_key.objectid != objectid) 3178 return 0; 3179 3180 /* we found an xattr, assume we've got an acl */ 3181 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3182 if (*first_xattr_slot == -1) 3183 *first_xattr_slot = slot; 3184 if (found_key.offset == xattr_access || 3185 found_key.offset == xattr_default) 3186 return 1; 3187 } 3188 3189 /* 3190 * we found a key greater than an xattr key, there can't 3191 * be any acls later on 3192 */ 3193 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3194 return 0; 3195 3196 slot++; 3197 scanned++; 3198 3199 /* 3200 * it goes inode, inode backrefs, xattrs, extents, 3201 * so if there are a ton of hard links to an inode there can 3202 * be a lot of backrefs. Don't waste time searching too hard, 3203 * this is just an optimization 3204 */ 3205 if (scanned >= 8) 3206 break; 3207 } 3208 /* we hit the end of the leaf before we found an xattr or 3209 * something larger than an xattr. We have to assume the inode 3210 * has acls 3211 */ 3212 if (*first_xattr_slot == -1) 3213 *first_xattr_slot = slot; 3214 return 1; 3215 } 3216 3217 /* 3218 * read an inode from the btree into the in-memory inode 3219 */ 3220 static int btrfs_read_locked_inode(struct inode *inode, 3221 struct btrfs_path *in_path) 3222 { 3223 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3224 struct btrfs_path *path = in_path; 3225 struct extent_buffer *leaf; 3226 struct btrfs_inode_item *inode_item; 3227 struct btrfs_root *root = BTRFS_I(inode)->root; 3228 struct btrfs_key location; 3229 unsigned long ptr; 3230 int maybe_acls; 3231 u32 rdev; 3232 int ret; 3233 bool filled = false; 3234 int first_xattr_slot; 3235 3236 ret = btrfs_fill_inode(inode, &rdev); 3237 if (!ret) 3238 filled = true; 3239 3240 if (!path) { 3241 path = btrfs_alloc_path(); 3242 if (!path) 3243 return -ENOMEM; 3244 } 3245 3246 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3247 3248 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3249 if (ret) { 3250 if (path != in_path) 3251 btrfs_free_path(path); 3252 return ret; 3253 } 3254 3255 leaf = path->nodes[0]; 3256 3257 if (filled) 3258 goto cache_index; 3259 3260 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3261 struct btrfs_inode_item); 3262 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3263 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3264 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3265 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3266 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item)); 3267 btrfs_inode_set_file_extent_range(BTRFS_I(inode), 0, 3268 round_up(i_size_read(inode), fs_info->sectorsize)); 3269 3270 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime); 3271 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime); 3272 3273 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime); 3274 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime); 3275 3276 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime); 3277 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime); 3278 3279 BTRFS_I(inode)->i_otime.tv_sec = 3280 btrfs_timespec_sec(leaf, &inode_item->otime); 3281 BTRFS_I(inode)->i_otime.tv_nsec = 3282 btrfs_timespec_nsec(leaf, &inode_item->otime); 3283 3284 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3285 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3286 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3287 3288 inode_set_iversion_queried(inode, 3289 btrfs_inode_sequence(leaf, inode_item)); 3290 inode->i_generation = BTRFS_I(inode)->generation; 3291 inode->i_rdev = 0; 3292 rdev = btrfs_inode_rdev(leaf, inode_item); 3293 3294 BTRFS_I(inode)->index_cnt = (u64)-1; 3295 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item); 3296 3297 cache_index: 3298 /* 3299 * If we were modified in the current generation and evicted from memory 3300 * and then re-read we need to do a full sync since we don't have any 3301 * idea about which extents were modified before we were evicted from 3302 * cache. 3303 * 3304 * This is required for both inode re-read from disk and delayed inode 3305 * in delayed_nodes_tree. 3306 */ 3307 if (BTRFS_I(inode)->last_trans == fs_info->generation) 3308 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3309 &BTRFS_I(inode)->runtime_flags); 3310 3311 /* 3312 * We don't persist the id of the transaction where an unlink operation 3313 * against the inode was last made. So here we assume the inode might 3314 * have been evicted, and therefore the exact value of last_unlink_trans 3315 * lost, and set it to last_trans to avoid metadata inconsistencies 3316 * between the inode and its parent if the inode is fsync'ed and the log 3317 * replayed. For example, in the scenario: 3318 * 3319 * touch mydir/foo 3320 * ln mydir/foo mydir/bar 3321 * sync 3322 * unlink mydir/bar 3323 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 3324 * xfs_io -c fsync mydir/foo 3325 * <power failure> 3326 * mount fs, triggers fsync log replay 3327 * 3328 * We must make sure that when we fsync our inode foo we also log its 3329 * parent inode, otherwise after log replay the parent still has the 3330 * dentry with the "bar" name but our inode foo has a link count of 1 3331 * and doesn't have an inode ref with the name "bar" anymore. 3332 * 3333 * Setting last_unlink_trans to last_trans is a pessimistic approach, 3334 * but it guarantees correctness at the expense of occasional full 3335 * transaction commits on fsync if our inode is a directory, or if our 3336 * inode is not a directory, logging its parent unnecessarily. 3337 */ 3338 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans; 3339 3340 /* 3341 * Same logic as for last_unlink_trans. We don't persist the generation 3342 * of the last transaction where this inode was used for a reflink 3343 * operation, so after eviction and reloading the inode we must be 3344 * pessimistic and assume the last transaction that modified the inode. 3345 */ 3346 BTRFS_I(inode)->last_reflink_trans = BTRFS_I(inode)->last_trans; 3347 3348 path->slots[0]++; 3349 if (inode->i_nlink != 1 || 3350 path->slots[0] >= btrfs_header_nritems(leaf)) 3351 goto cache_acl; 3352 3353 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3354 if (location.objectid != btrfs_ino(BTRFS_I(inode))) 3355 goto cache_acl; 3356 3357 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3358 if (location.type == BTRFS_INODE_REF_KEY) { 3359 struct btrfs_inode_ref *ref; 3360 3361 ref = (struct btrfs_inode_ref *)ptr; 3362 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3363 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3364 struct btrfs_inode_extref *extref; 3365 3366 extref = (struct btrfs_inode_extref *)ptr; 3367 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3368 extref); 3369 } 3370 cache_acl: 3371 /* 3372 * try to precache a NULL acl entry for files that don't have 3373 * any xattrs or acls 3374 */ 3375 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3376 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot); 3377 if (first_xattr_slot != -1) { 3378 path->slots[0] = first_xattr_slot; 3379 ret = btrfs_load_inode_props(inode, path); 3380 if (ret) 3381 btrfs_err(fs_info, 3382 "error loading props for ino %llu (root %llu): %d", 3383 btrfs_ino(BTRFS_I(inode)), 3384 root->root_key.objectid, ret); 3385 } 3386 if (path != in_path) 3387 btrfs_free_path(path); 3388 3389 if (!maybe_acls) 3390 cache_no_acl(inode); 3391 3392 switch (inode->i_mode & S_IFMT) { 3393 case S_IFREG: 3394 inode->i_mapping->a_ops = &btrfs_aops; 3395 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 3396 inode->i_fop = &btrfs_file_operations; 3397 inode->i_op = &btrfs_file_inode_operations; 3398 break; 3399 case S_IFDIR: 3400 inode->i_fop = &btrfs_dir_file_operations; 3401 inode->i_op = &btrfs_dir_inode_operations; 3402 break; 3403 case S_IFLNK: 3404 inode->i_op = &btrfs_symlink_inode_operations; 3405 inode_nohighmem(inode); 3406 inode->i_mapping->a_ops = &btrfs_aops; 3407 break; 3408 default: 3409 inode->i_op = &btrfs_special_inode_operations; 3410 init_special_inode(inode, inode->i_mode, rdev); 3411 break; 3412 } 3413 3414 btrfs_sync_inode_flags_to_i_flags(inode); 3415 return 0; 3416 } 3417 3418 /* 3419 * given a leaf and an inode, copy the inode fields into the leaf 3420 */ 3421 static void fill_inode_item(struct btrfs_trans_handle *trans, 3422 struct extent_buffer *leaf, 3423 struct btrfs_inode_item *item, 3424 struct inode *inode) 3425 { 3426 struct btrfs_map_token token; 3427 3428 btrfs_init_map_token(&token, leaf); 3429 3430 btrfs_set_token_inode_uid(&token, item, i_uid_read(inode)); 3431 btrfs_set_token_inode_gid(&token, item, i_gid_read(inode)); 3432 btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size); 3433 btrfs_set_token_inode_mode(&token, item, inode->i_mode); 3434 btrfs_set_token_inode_nlink(&token, item, inode->i_nlink); 3435 3436 btrfs_set_token_timespec_sec(&token, &item->atime, 3437 inode->i_atime.tv_sec); 3438 btrfs_set_token_timespec_nsec(&token, &item->atime, 3439 inode->i_atime.tv_nsec); 3440 3441 btrfs_set_token_timespec_sec(&token, &item->mtime, 3442 inode->i_mtime.tv_sec); 3443 btrfs_set_token_timespec_nsec(&token, &item->mtime, 3444 inode->i_mtime.tv_nsec); 3445 3446 btrfs_set_token_timespec_sec(&token, &item->ctime, 3447 inode->i_ctime.tv_sec); 3448 btrfs_set_token_timespec_nsec(&token, &item->ctime, 3449 inode->i_ctime.tv_nsec); 3450 3451 btrfs_set_token_timespec_sec(&token, &item->otime, 3452 BTRFS_I(inode)->i_otime.tv_sec); 3453 btrfs_set_token_timespec_nsec(&token, &item->otime, 3454 BTRFS_I(inode)->i_otime.tv_nsec); 3455 3456 btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode)); 3457 btrfs_set_token_inode_generation(&token, item, 3458 BTRFS_I(inode)->generation); 3459 btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode)); 3460 btrfs_set_token_inode_transid(&token, item, trans->transid); 3461 btrfs_set_token_inode_rdev(&token, item, inode->i_rdev); 3462 btrfs_set_token_inode_flags(&token, item, BTRFS_I(inode)->flags); 3463 btrfs_set_token_inode_block_group(&token, item, 0); 3464 } 3465 3466 /* 3467 * copy everything in the in-memory inode into the btree. 3468 */ 3469 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 3470 struct btrfs_root *root, struct inode *inode) 3471 { 3472 struct btrfs_inode_item *inode_item; 3473 struct btrfs_path *path; 3474 struct extent_buffer *leaf; 3475 int ret; 3476 3477 path = btrfs_alloc_path(); 3478 if (!path) 3479 return -ENOMEM; 3480 3481 path->leave_spinning = 1; 3482 ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location, 3483 1); 3484 if (ret) { 3485 if (ret > 0) 3486 ret = -ENOENT; 3487 goto failed; 3488 } 3489 3490 leaf = path->nodes[0]; 3491 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3492 struct btrfs_inode_item); 3493 3494 fill_inode_item(trans, leaf, inode_item, inode); 3495 btrfs_mark_buffer_dirty(leaf); 3496 btrfs_set_inode_last_trans(trans, BTRFS_I(inode)); 3497 ret = 0; 3498 failed: 3499 btrfs_free_path(path); 3500 return ret; 3501 } 3502 3503 /* 3504 * copy everything in the in-memory inode into the btree. 3505 */ 3506 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 3507 struct btrfs_root *root, struct inode *inode) 3508 { 3509 struct btrfs_fs_info *fs_info = root->fs_info; 3510 int ret; 3511 3512 /* 3513 * If the inode is a free space inode, we can deadlock during commit 3514 * if we put it into the delayed code. 3515 * 3516 * The data relocation inode should also be directly updated 3517 * without delay 3518 */ 3519 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) 3520 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 3521 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 3522 btrfs_update_root_times(trans, root); 3523 3524 ret = btrfs_delayed_update_inode(trans, root, inode); 3525 if (!ret) 3526 btrfs_set_inode_last_trans(trans, BTRFS_I(inode)); 3527 return ret; 3528 } 3529 3530 return btrfs_update_inode_item(trans, root, inode); 3531 } 3532 3533 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 3534 struct btrfs_root *root, 3535 struct inode *inode) 3536 { 3537 int ret; 3538 3539 ret = btrfs_update_inode(trans, root, inode); 3540 if (ret == -ENOSPC) 3541 return btrfs_update_inode_item(trans, root, inode); 3542 return ret; 3543 } 3544 3545 /* 3546 * unlink helper that gets used here in inode.c and in the tree logging 3547 * recovery code. It remove a link in a directory with a given name, and 3548 * also drops the back refs in the inode to the directory 3549 */ 3550 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 3551 struct btrfs_root *root, 3552 struct btrfs_inode *dir, 3553 struct btrfs_inode *inode, 3554 const char *name, int name_len) 3555 { 3556 struct btrfs_fs_info *fs_info = root->fs_info; 3557 struct btrfs_path *path; 3558 int ret = 0; 3559 struct btrfs_dir_item *di; 3560 u64 index; 3561 u64 ino = btrfs_ino(inode); 3562 u64 dir_ino = btrfs_ino(dir); 3563 3564 path = btrfs_alloc_path(); 3565 if (!path) { 3566 ret = -ENOMEM; 3567 goto out; 3568 } 3569 3570 path->leave_spinning = 1; 3571 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 3572 name, name_len, -1); 3573 if (IS_ERR_OR_NULL(di)) { 3574 ret = di ? PTR_ERR(di) : -ENOENT; 3575 goto err; 3576 } 3577 ret = btrfs_delete_one_dir_name(trans, root, path, di); 3578 if (ret) 3579 goto err; 3580 btrfs_release_path(path); 3581 3582 /* 3583 * If we don't have dir index, we have to get it by looking up 3584 * the inode ref, since we get the inode ref, remove it directly, 3585 * it is unnecessary to do delayed deletion. 3586 * 3587 * But if we have dir index, needn't search inode ref to get it. 3588 * Since the inode ref is close to the inode item, it is better 3589 * that we delay to delete it, and just do this deletion when 3590 * we update the inode item. 3591 */ 3592 if (inode->dir_index) { 3593 ret = btrfs_delayed_delete_inode_ref(inode); 3594 if (!ret) { 3595 index = inode->dir_index; 3596 goto skip_backref; 3597 } 3598 } 3599 3600 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino, 3601 dir_ino, &index); 3602 if (ret) { 3603 btrfs_info(fs_info, 3604 "failed to delete reference to %.*s, inode %llu parent %llu", 3605 name_len, name, ino, dir_ino); 3606 btrfs_abort_transaction(trans, ret); 3607 goto err; 3608 } 3609 skip_backref: 3610 ret = btrfs_delete_delayed_dir_index(trans, dir, index); 3611 if (ret) { 3612 btrfs_abort_transaction(trans, ret); 3613 goto err; 3614 } 3615 3616 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode, 3617 dir_ino); 3618 if (ret != 0 && ret != -ENOENT) { 3619 btrfs_abort_transaction(trans, ret); 3620 goto err; 3621 } 3622 3623 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir, 3624 index); 3625 if (ret == -ENOENT) 3626 ret = 0; 3627 else if (ret) 3628 btrfs_abort_transaction(trans, ret); 3629 3630 /* 3631 * If we have a pending delayed iput we could end up with the final iput 3632 * being run in btrfs-cleaner context. If we have enough of these built 3633 * up we can end up burning a lot of time in btrfs-cleaner without any 3634 * way to throttle the unlinks. Since we're currently holding a ref on 3635 * the inode we can run the delayed iput here without any issues as the 3636 * final iput won't be done until after we drop the ref we're currently 3637 * holding. 3638 */ 3639 btrfs_run_delayed_iput(fs_info, inode); 3640 err: 3641 btrfs_free_path(path); 3642 if (ret) 3643 goto out; 3644 3645 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2); 3646 inode_inc_iversion(&inode->vfs_inode); 3647 inode_inc_iversion(&dir->vfs_inode); 3648 inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime = 3649 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode); 3650 ret = btrfs_update_inode(trans, root, &dir->vfs_inode); 3651 out: 3652 return ret; 3653 } 3654 3655 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 3656 struct btrfs_root *root, 3657 struct btrfs_inode *dir, struct btrfs_inode *inode, 3658 const char *name, int name_len) 3659 { 3660 int ret; 3661 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len); 3662 if (!ret) { 3663 drop_nlink(&inode->vfs_inode); 3664 ret = btrfs_update_inode(trans, root, &inode->vfs_inode); 3665 } 3666 return ret; 3667 } 3668 3669 /* 3670 * helper to start transaction for unlink and rmdir. 3671 * 3672 * unlink and rmdir are special in btrfs, they do not always free space, so 3673 * if we cannot make our reservations the normal way try and see if there is 3674 * plenty of slack room in the global reserve to migrate, otherwise we cannot 3675 * allow the unlink to occur. 3676 */ 3677 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) 3678 { 3679 struct btrfs_root *root = BTRFS_I(dir)->root; 3680 3681 /* 3682 * 1 for the possible orphan item 3683 * 1 for the dir item 3684 * 1 for the dir index 3685 * 1 for the inode ref 3686 * 1 for the inode 3687 */ 3688 return btrfs_start_transaction_fallback_global_rsv(root, 5); 3689 } 3690 3691 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 3692 { 3693 struct btrfs_root *root = BTRFS_I(dir)->root; 3694 struct btrfs_trans_handle *trans; 3695 struct inode *inode = d_inode(dentry); 3696 int ret; 3697 3698 trans = __unlink_start_trans(dir); 3699 if (IS_ERR(trans)) 3700 return PTR_ERR(trans); 3701 3702 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 3703 0); 3704 3705 ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 3706 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 3707 dentry->d_name.len); 3708 if (ret) 3709 goto out; 3710 3711 if (inode->i_nlink == 0) { 3712 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 3713 if (ret) 3714 goto out; 3715 } 3716 3717 out: 3718 btrfs_end_transaction(trans); 3719 btrfs_btree_balance_dirty(root->fs_info); 3720 return ret; 3721 } 3722 3723 static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 3724 struct inode *dir, struct dentry *dentry) 3725 { 3726 struct btrfs_root *root = BTRFS_I(dir)->root; 3727 struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); 3728 struct btrfs_path *path; 3729 struct extent_buffer *leaf; 3730 struct btrfs_dir_item *di; 3731 struct btrfs_key key; 3732 const char *name = dentry->d_name.name; 3733 int name_len = dentry->d_name.len; 3734 u64 index; 3735 int ret; 3736 u64 objectid; 3737 u64 dir_ino = btrfs_ino(BTRFS_I(dir)); 3738 3739 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) { 3740 objectid = inode->root->root_key.objectid; 3741 } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 3742 objectid = inode->location.objectid; 3743 } else { 3744 WARN_ON(1); 3745 return -EINVAL; 3746 } 3747 3748 path = btrfs_alloc_path(); 3749 if (!path) 3750 return -ENOMEM; 3751 3752 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 3753 name, name_len, -1); 3754 if (IS_ERR_OR_NULL(di)) { 3755 ret = di ? PTR_ERR(di) : -ENOENT; 3756 goto out; 3757 } 3758 3759 leaf = path->nodes[0]; 3760 btrfs_dir_item_key_to_cpu(leaf, di, &key); 3761 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 3762 ret = btrfs_delete_one_dir_name(trans, root, path, di); 3763 if (ret) { 3764 btrfs_abort_transaction(trans, ret); 3765 goto out; 3766 } 3767 btrfs_release_path(path); 3768 3769 /* 3770 * This is a placeholder inode for a subvolume we didn't have a 3771 * reference to at the time of the snapshot creation. In the meantime 3772 * we could have renamed the real subvol link into our snapshot, so 3773 * depending on btrfs_del_root_ref to return -ENOENT here is incorret. 3774 * Instead simply lookup the dir_index_item for this entry so we can 3775 * remove it. Otherwise we know we have a ref to the root and we can 3776 * call btrfs_del_root_ref, and it _shouldn't_ fail. 3777 */ 3778 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 3779 di = btrfs_search_dir_index_item(root, path, dir_ino, 3780 name, name_len); 3781 if (IS_ERR_OR_NULL(di)) { 3782 if (!di) 3783 ret = -ENOENT; 3784 else 3785 ret = PTR_ERR(di); 3786 btrfs_abort_transaction(trans, ret); 3787 goto out; 3788 } 3789 3790 leaf = path->nodes[0]; 3791 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3792 index = key.offset; 3793 btrfs_release_path(path); 3794 } else { 3795 ret = btrfs_del_root_ref(trans, objectid, 3796 root->root_key.objectid, dir_ino, 3797 &index, name, name_len); 3798 if (ret) { 3799 btrfs_abort_transaction(trans, ret); 3800 goto out; 3801 } 3802 } 3803 3804 ret = btrfs_delete_delayed_dir_index(trans, BTRFS_I(dir), index); 3805 if (ret) { 3806 btrfs_abort_transaction(trans, ret); 3807 goto out; 3808 } 3809 3810 btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2); 3811 inode_inc_iversion(dir); 3812 dir->i_mtime = dir->i_ctime = current_time(dir); 3813 ret = btrfs_update_inode_fallback(trans, root, dir); 3814 if (ret) 3815 btrfs_abort_transaction(trans, ret); 3816 out: 3817 btrfs_free_path(path); 3818 return ret; 3819 } 3820 3821 /* 3822 * Helper to check if the subvolume references other subvolumes or if it's 3823 * default. 3824 */ 3825 static noinline int may_destroy_subvol(struct btrfs_root *root) 3826 { 3827 struct btrfs_fs_info *fs_info = root->fs_info; 3828 struct btrfs_path *path; 3829 struct btrfs_dir_item *di; 3830 struct btrfs_key key; 3831 u64 dir_id; 3832 int ret; 3833 3834 path = btrfs_alloc_path(); 3835 if (!path) 3836 return -ENOMEM; 3837 3838 /* Make sure this root isn't set as the default subvol */ 3839 dir_id = btrfs_super_root_dir(fs_info->super_copy); 3840 di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path, 3841 dir_id, "default", 7, 0); 3842 if (di && !IS_ERR(di)) { 3843 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key); 3844 if (key.objectid == root->root_key.objectid) { 3845 ret = -EPERM; 3846 btrfs_err(fs_info, 3847 "deleting default subvolume %llu is not allowed", 3848 key.objectid); 3849 goto out; 3850 } 3851 btrfs_release_path(path); 3852 } 3853 3854 key.objectid = root->root_key.objectid; 3855 key.type = BTRFS_ROOT_REF_KEY; 3856 key.offset = (u64)-1; 3857 3858 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 3859 if (ret < 0) 3860 goto out; 3861 BUG_ON(ret == 0); 3862 3863 ret = 0; 3864 if (path->slots[0] > 0) { 3865 path->slots[0]--; 3866 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); 3867 if (key.objectid == root->root_key.objectid && 3868 key.type == BTRFS_ROOT_REF_KEY) 3869 ret = -ENOTEMPTY; 3870 } 3871 out: 3872 btrfs_free_path(path); 3873 return ret; 3874 } 3875 3876 /* Delete all dentries for inodes belonging to the root */ 3877 static void btrfs_prune_dentries(struct btrfs_root *root) 3878 { 3879 struct btrfs_fs_info *fs_info = root->fs_info; 3880 struct rb_node *node; 3881 struct rb_node *prev; 3882 struct btrfs_inode *entry; 3883 struct inode *inode; 3884 u64 objectid = 0; 3885 3886 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 3887 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 3888 3889 spin_lock(&root->inode_lock); 3890 again: 3891 node = root->inode_tree.rb_node; 3892 prev = NULL; 3893 while (node) { 3894 prev = node; 3895 entry = rb_entry(node, struct btrfs_inode, rb_node); 3896 3897 if (objectid < btrfs_ino(entry)) 3898 node = node->rb_left; 3899 else if (objectid > btrfs_ino(entry)) 3900 node = node->rb_right; 3901 else 3902 break; 3903 } 3904 if (!node) { 3905 while (prev) { 3906 entry = rb_entry(prev, struct btrfs_inode, rb_node); 3907 if (objectid <= btrfs_ino(entry)) { 3908 node = prev; 3909 break; 3910 } 3911 prev = rb_next(prev); 3912 } 3913 } 3914 while (node) { 3915 entry = rb_entry(node, struct btrfs_inode, rb_node); 3916 objectid = btrfs_ino(entry) + 1; 3917 inode = igrab(&entry->vfs_inode); 3918 if (inode) { 3919 spin_unlock(&root->inode_lock); 3920 if (atomic_read(&inode->i_count) > 1) 3921 d_prune_aliases(inode); 3922 /* 3923 * btrfs_drop_inode will have it removed from the inode 3924 * cache when its usage count hits zero. 3925 */ 3926 iput(inode); 3927 cond_resched(); 3928 spin_lock(&root->inode_lock); 3929 goto again; 3930 } 3931 3932 if (cond_resched_lock(&root->inode_lock)) 3933 goto again; 3934 3935 node = rb_next(node); 3936 } 3937 spin_unlock(&root->inode_lock); 3938 } 3939 3940 int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) 3941 { 3942 struct btrfs_fs_info *fs_info = btrfs_sb(dentry->d_sb); 3943 struct btrfs_root *root = BTRFS_I(dir)->root; 3944 struct inode *inode = d_inode(dentry); 3945 struct btrfs_root *dest = BTRFS_I(inode)->root; 3946 struct btrfs_trans_handle *trans; 3947 struct btrfs_block_rsv block_rsv; 3948 u64 root_flags; 3949 int ret; 3950 int err; 3951 3952 /* 3953 * Don't allow to delete a subvolume with send in progress. This is 3954 * inside the inode lock so the error handling that has to drop the bit 3955 * again is not run concurrently. 3956 */ 3957 spin_lock(&dest->root_item_lock); 3958 if (dest->send_in_progress) { 3959 spin_unlock(&dest->root_item_lock); 3960 btrfs_warn(fs_info, 3961 "attempt to delete subvolume %llu during send", 3962 dest->root_key.objectid); 3963 return -EPERM; 3964 } 3965 root_flags = btrfs_root_flags(&dest->root_item); 3966 btrfs_set_root_flags(&dest->root_item, 3967 root_flags | BTRFS_ROOT_SUBVOL_DEAD); 3968 spin_unlock(&dest->root_item_lock); 3969 3970 down_write(&fs_info->subvol_sem); 3971 3972 err = may_destroy_subvol(dest); 3973 if (err) 3974 goto out_up_write; 3975 3976 btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP); 3977 /* 3978 * One for dir inode, 3979 * two for dir entries, 3980 * two for root ref/backref. 3981 */ 3982 err = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true); 3983 if (err) 3984 goto out_up_write; 3985 3986 trans = btrfs_start_transaction(root, 0); 3987 if (IS_ERR(trans)) { 3988 err = PTR_ERR(trans); 3989 goto out_release; 3990 } 3991 trans->block_rsv = &block_rsv; 3992 trans->bytes_reserved = block_rsv.size; 3993 3994 btrfs_record_snapshot_destroy(trans, BTRFS_I(dir)); 3995 3996 ret = btrfs_unlink_subvol(trans, dir, dentry); 3997 if (ret) { 3998 err = ret; 3999 btrfs_abort_transaction(trans, ret); 4000 goto out_end_trans; 4001 } 4002 4003 btrfs_record_root_in_trans(trans, dest); 4004 4005 memset(&dest->root_item.drop_progress, 0, 4006 sizeof(dest->root_item.drop_progress)); 4007 dest->root_item.drop_level = 0; 4008 btrfs_set_root_refs(&dest->root_item, 0); 4009 4010 if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) { 4011 ret = btrfs_insert_orphan_item(trans, 4012 fs_info->tree_root, 4013 dest->root_key.objectid); 4014 if (ret) { 4015 btrfs_abort_transaction(trans, ret); 4016 err = ret; 4017 goto out_end_trans; 4018 } 4019 } 4020 4021 ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid, 4022 BTRFS_UUID_KEY_SUBVOL, 4023 dest->root_key.objectid); 4024 if (ret && ret != -ENOENT) { 4025 btrfs_abort_transaction(trans, ret); 4026 err = ret; 4027 goto out_end_trans; 4028 } 4029 if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) { 4030 ret = btrfs_uuid_tree_remove(trans, 4031 dest->root_item.received_uuid, 4032 BTRFS_UUID_KEY_RECEIVED_SUBVOL, 4033 dest->root_key.objectid); 4034 if (ret && ret != -ENOENT) { 4035 btrfs_abort_transaction(trans, ret); 4036 err = ret; 4037 goto out_end_trans; 4038 } 4039 } 4040 4041 free_anon_bdev(dest->anon_dev); 4042 dest->anon_dev = 0; 4043 out_end_trans: 4044 trans->block_rsv = NULL; 4045 trans->bytes_reserved = 0; 4046 ret = btrfs_end_transaction(trans); 4047 if (ret && !err) 4048 err = ret; 4049 inode->i_flags |= S_DEAD; 4050 out_release: 4051 btrfs_subvolume_release_metadata(fs_info, &block_rsv); 4052 out_up_write: 4053 up_write(&fs_info->subvol_sem); 4054 if (err) { 4055 spin_lock(&dest->root_item_lock); 4056 root_flags = btrfs_root_flags(&dest->root_item); 4057 btrfs_set_root_flags(&dest->root_item, 4058 root_flags & ~BTRFS_ROOT_SUBVOL_DEAD); 4059 spin_unlock(&dest->root_item_lock); 4060 } else { 4061 d_invalidate(dentry); 4062 btrfs_prune_dentries(dest); 4063 ASSERT(dest->send_in_progress == 0); 4064 4065 /* the last ref */ 4066 if (dest->ino_cache_inode) { 4067 iput(dest->ino_cache_inode); 4068 dest->ino_cache_inode = NULL; 4069 } 4070 } 4071 4072 return err; 4073 } 4074 4075 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4076 { 4077 struct inode *inode = d_inode(dentry); 4078 int err = 0; 4079 struct btrfs_root *root = BTRFS_I(dir)->root; 4080 struct btrfs_trans_handle *trans; 4081 u64 last_unlink_trans; 4082 4083 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4084 return -ENOTEMPTY; 4085 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) 4086 return btrfs_delete_subvolume(dir, dentry); 4087 4088 trans = __unlink_start_trans(dir); 4089 if (IS_ERR(trans)) 4090 return PTR_ERR(trans); 4091 4092 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4093 err = btrfs_unlink_subvol(trans, dir, dentry); 4094 goto out; 4095 } 4096 4097 err = btrfs_orphan_add(trans, BTRFS_I(inode)); 4098 if (err) 4099 goto out; 4100 4101 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; 4102 4103 /* now the directory is empty */ 4104 err = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4105 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4106 dentry->d_name.len); 4107 if (!err) { 4108 btrfs_i_size_write(BTRFS_I(inode), 0); 4109 /* 4110 * Propagate the last_unlink_trans value of the deleted dir to 4111 * its parent directory. This is to prevent an unrecoverable 4112 * log tree in the case we do something like this: 4113 * 1) create dir foo 4114 * 2) create snapshot under dir foo 4115 * 3) delete the snapshot 4116 * 4) rmdir foo 4117 * 5) mkdir foo 4118 * 6) fsync foo or some file inside foo 4119 */ 4120 if (last_unlink_trans >= trans->transid) 4121 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; 4122 } 4123 out: 4124 btrfs_end_transaction(trans); 4125 btrfs_btree_balance_dirty(root->fs_info); 4126 4127 return err; 4128 } 4129 4130 /* 4131 * Return this if we need to call truncate_block for the last bit of the 4132 * truncate. 4133 */ 4134 #define NEED_TRUNCATE_BLOCK 1 4135 4136 /* 4137 * this can truncate away extent items, csum items and directory items. 4138 * It starts at a high offset and removes keys until it can't find 4139 * any higher than new_size 4140 * 4141 * csum items that cross the new i_size are truncated to the new size 4142 * as well. 4143 * 4144 * min_type is the minimum key type to truncate down to. If set to 0, this 4145 * will kill all the items on this inode, including the INODE_ITEM_KEY. 4146 */ 4147 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, 4148 struct btrfs_root *root, 4149 struct inode *inode, 4150 u64 new_size, u32 min_type) 4151 { 4152 struct btrfs_fs_info *fs_info = root->fs_info; 4153 struct btrfs_path *path; 4154 struct extent_buffer *leaf; 4155 struct btrfs_file_extent_item *fi; 4156 struct btrfs_key key; 4157 struct btrfs_key found_key; 4158 u64 extent_start = 0; 4159 u64 extent_num_bytes = 0; 4160 u64 extent_offset = 0; 4161 u64 item_end = 0; 4162 u64 last_size = new_size; 4163 u32 found_type = (u8)-1; 4164 int found_extent; 4165 int del_item; 4166 int pending_del_nr = 0; 4167 int pending_del_slot = 0; 4168 int extent_type = -1; 4169 int ret; 4170 u64 ino = btrfs_ino(BTRFS_I(inode)); 4171 u64 bytes_deleted = 0; 4172 bool be_nice = false; 4173 bool should_throttle = false; 4174 const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize); 4175 struct extent_state *cached_state = NULL; 4176 4177 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); 4178 4179 /* 4180 * For non-free space inodes and non-shareable roots, we want to back 4181 * off from time to time. This means all inodes in subvolume roots, 4182 * reloc roots, and data reloc roots. 4183 */ 4184 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) && 4185 test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) 4186 be_nice = true; 4187 4188 path = btrfs_alloc_path(); 4189 if (!path) 4190 return -ENOMEM; 4191 path->reada = READA_BACK; 4192 4193 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4194 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1, 4195 &cached_state); 4196 4197 /* 4198 * We want to drop from the next block forward in case this 4199 * new size is not block aligned since we will be keeping the 4200 * last block of the extent just the way it is. 4201 */ 4202 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size, 4203 fs_info->sectorsize), 4204 (u64)-1, 0); 4205 } 4206 4207 /* 4208 * This function is also used to drop the items in the log tree before 4209 * we relog the inode, so if root != BTRFS_I(inode)->root, it means 4210 * it is used to drop the logged items. So we shouldn't kill the delayed 4211 * items. 4212 */ 4213 if (min_type == 0 && root == BTRFS_I(inode)->root) 4214 btrfs_kill_delayed_inode_items(BTRFS_I(inode)); 4215 4216 key.objectid = ino; 4217 key.offset = (u64)-1; 4218 key.type = (u8)-1; 4219 4220 search_again: 4221 /* 4222 * with a 16K leaf size and 128MB extents, you can actually queue 4223 * up a huge file in a single leaf. Most of the time that 4224 * bytes_deleted is > 0, it will be huge by the time we get here 4225 */ 4226 if (be_nice && bytes_deleted > SZ_32M && 4227 btrfs_should_end_transaction(trans)) { 4228 ret = -EAGAIN; 4229 goto out; 4230 } 4231 4232 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 4233 if (ret < 0) 4234 goto out; 4235 4236 if (ret > 0) { 4237 ret = 0; 4238 /* there are no items in the tree for us to truncate, we're 4239 * done 4240 */ 4241 if (path->slots[0] == 0) 4242 goto out; 4243 path->slots[0]--; 4244 } 4245 4246 while (1) { 4247 u64 clear_start = 0, clear_len = 0; 4248 4249 fi = NULL; 4250 leaf = path->nodes[0]; 4251 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4252 found_type = found_key.type; 4253 4254 if (found_key.objectid != ino) 4255 break; 4256 4257 if (found_type < min_type) 4258 break; 4259 4260 item_end = found_key.offset; 4261 if (found_type == BTRFS_EXTENT_DATA_KEY) { 4262 fi = btrfs_item_ptr(leaf, path->slots[0], 4263 struct btrfs_file_extent_item); 4264 extent_type = btrfs_file_extent_type(leaf, fi); 4265 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4266 item_end += 4267 btrfs_file_extent_num_bytes(leaf, fi); 4268 4269 trace_btrfs_truncate_show_fi_regular( 4270 BTRFS_I(inode), leaf, fi, 4271 found_key.offset); 4272 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4273 item_end += btrfs_file_extent_ram_bytes(leaf, 4274 fi); 4275 4276 trace_btrfs_truncate_show_fi_inline( 4277 BTRFS_I(inode), leaf, fi, path->slots[0], 4278 found_key.offset); 4279 } 4280 item_end--; 4281 } 4282 if (found_type > min_type) { 4283 del_item = 1; 4284 } else { 4285 if (item_end < new_size) 4286 break; 4287 if (found_key.offset >= new_size) 4288 del_item = 1; 4289 else 4290 del_item = 0; 4291 } 4292 found_extent = 0; 4293 /* FIXME, shrink the extent if the ref count is only 1 */ 4294 if (found_type != BTRFS_EXTENT_DATA_KEY) 4295 goto delete; 4296 4297 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4298 u64 num_dec; 4299 4300 clear_start = found_key.offset; 4301 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); 4302 if (!del_item) { 4303 u64 orig_num_bytes = 4304 btrfs_file_extent_num_bytes(leaf, fi); 4305 extent_num_bytes = ALIGN(new_size - 4306 found_key.offset, 4307 fs_info->sectorsize); 4308 clear_start = ALIGN(new_size, fs_info->sectorsize); 4309 btrfs_set_file_extent_num_bytes(leaf, fi, 4310 extent_num_bytes); 4311 num_dec = (orig_num_bytes - 4312 extent_num_bytes); 4313 if (test_bit(BTRFS_ROOT_SHAREABLE, 4314 &root->state) && 4315 extent_start != 0) 4316 inode_sub_bytes(inode, num_dec); 4317 btrfs_mark_buffer_dirty(leaf); 4318 } else { 4319 extent_num_bytes = 4320 btrfs_file_extent_disk_num_bytes(leaf, 4321 fi); 4322 extent_offset = found_key.offset - 4323 btrfs_file_extent_offset(leaf, fi); 4324 4325 /* FIXME blocksize != 4096 */ 4326 num_dec = btrfs_file_extent_num_bytes(leaf, fi); 4327 if (extent_start != 0) { 4328 found_extent = 1; 4329 if (test_bit(BTRFS_ROOT_SHAREABLE, 4330 &root->state)) 4331 inode_sub_bytes(inode, num_dec); 4332 } 4333 } 4334 clear_len = num_dec; 4335 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4336 /* 4337 * we can't truncate inline items that have had 4338 * special encodings 4339 */ 4340 if (!del_item && 4341 btrfs_file_extent_encryption(leaf, fi) == 0 && 4342 btrfs_file_extent_other_encoding(leaf, fi) == 0 && 4343 btrfs_file_extent_compression(leaf, fi) == 0) { 4344 u32 size = (u32)(new_size - found_key.offset); 4345 4346 btrfs_set_file_extent_ram_bytes(leaf, fi, size); 4347 size = btrfs_file_extent_calc_inline_size(size); 4348 btrfs_truncate_item(path, size, 1); 4349 } else if (!del_item) { 4350 /* 4351 * We have to bail so the last_size is set to 4352 * just before this extent. 4353 */ 4354 ret = NEED_TRUNCATE_BLOCK; 4355 break; 4356 } else { 4357 /* 4358 * Inline extents are special, we just treat 4359 * them as a full sector worth in the file 4360 * extent tree just for simplicity sake. 4361 */ 4362 clear_len = fs_info->sectorsize; 4363 } 4364 4365 if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) 4366 inode_sub_bytes(inode, item_end + 1 - new_size); 4367 } 4368 delete: 4369 /* 4370 * We use btrfs_truncate_inode_items() to clean up log trees for 4371 * multiple fsyncs, and in this case we don't want to clear the 4372 * file extent range because it's just the log. 4373 */ 4374 if (root == BTRFS_I(inode)->root) { 4375 ret = btrfs_inode_clear_file_extent_range(BTRFS_I(inode), 4376 clear_start, clear_len); 4377 if (ret) { 4378 btrfs_abort_transaction(trans, ret); 4379 break; 4380 } 4381 } 4382 4383 if (del_item) 4384 last_size = found_key.offset; 4385 else 4386 last_size = new_size; 4387 if (del_item) { 4388 if (!pending_del_nr) { 4389 /* no pending yet, add ourselves */ 4390 pending_del_slot = path->slots[0]; 4391 pending_del_nr = 1; 4392 } else if (pending_del_nr && 4393 path->slots[0] + 1 == pending_del_slot) { 4394 /* hop on the pending chunk */ 4395 pending_del_nr++; 4396 pending_del_slot = path->slots[0]; 4397 } else { 4398 BUG(); 4399 } 4400 } else { 4401 break; 4402 } 4403 should_throttle = false; 4404 4405 if (found_extent && 4406 root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4407 struct btrfs_ref ref = { 0 }; 4408 4409 bytes_deleted += extent_num_bytes; 4410 4411 btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF, 4412 extent_start, extent_num_bytes, 0); 4413 ref.real_root = root->root_key.objectid; 4414 btrfs_init_data_ref(&ref, btrfs_header_owner(leaf), 4415 ino, extent_offset); 4416 ret = btrfs_free_extent(trans, &ref); 4417 if (ret) { 4418 btrfs_abort_transaction(trans, ret); 4419 break; 4420 } 4421 if (be_nice) { 4422 if (btrfs_should_throttle_delayed_refs(trans)) 4423 should_throttle = true; 4424 } 4425 } 4426 4427 if (found_type == BTRFS_INODE_ITEM_KEY) 4428 break; 4429 4430 if (path->slots[0] == 0 || 4431 path->slots[0] != pending_del_slot || 4432 should_throttle) { 4433 if (pending_del_nr) { 4434 ret = btrfs_del_items(trans, root, path, 4435 pending_del_slot, 4436 pending_del_nr); 4437 if (ret) { 4438 btrfs_abort_transaction(trans, ret); 4439 break; 4440 } 4441 pending_del_nr = 0; 4442 } 4443 btrfs_release_path(path); 4444 4445 /* 4446 * We can generate a lot of delayed refs, so we need to 4447 * throttle every once and a while and make sure we're 4448 * adding enough space to keep up with the work we are 4449 * generating. Since we hold a transaction here we 4450 * can't flush, and we don't want to FLUSH_LIMIT because 4451 * we could have generated too many delayed refs to 4452 * actually allocate, so just bail if we're short and 4453 * let the normal reservation dance happen higher up. 4454 */ 4455 if (should_throttle) { 4456 ret = btrfs_delayed_refs_rsv_refill(fs_info, 4457 BTRFS_RESERVE_NO_FLUSH); 4458 if (ret) { 4459 ret = -EAGAIN; 4460 break; 4461 } 4462 } 4463 goto search_again; 4464 } else { 4465 path->slots[0]--; 4466 } 4467 } 4468 out: 4469 if (ret >= 0 && pending_del_nr) { 4470 int err; 4471 4472 err = btrfs_del_items(trans, root, path, pending_del_slot, 4473 pending_del_nr); 4474 if (err) { 4475 btrfs_abort_transaction(trans, err); 4476 ret = err; 4477 } 4478 } 4479 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4480 ASSERT(last_size >= new_size); 4481 if (!ret && last_size > new_size) 4482 last_size = new_size; 4483 btrfs_inode_safe_disk_i_size_write(inode, last_size); 4484 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, 4485 (u64)-1, &cached_state); 4486 } 4487 4488 btrfs_free_path(path); 4489 return ret; 4490 } 4491 4492 /* 4493 * btrfs_truncate_block - read, zero a chunk and write a block 4494 * @inode - inode that we're zeroing 4495 * @from - the offset to start zeroing 4496 * @len - the length to zero, 0 to zero the entire range respective to the 4497 * offset 4498 * @front - zero up to the offset instead of from the offset on 4499 * 4500 * This will find the block for the "from" offset and cow the block and zero the 4501 * part we want to zero. This is used with truncate and hole punching. 4502 */ 4503 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len, 4504 int front) 4505 { 4506 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4507 struct address_space *mapping = inode->i_mapping; 4508 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4509 struct btrfs_ordered_extent *ordered; 4510 struct extent_state *cached_state = NULL; 4511 struct extent_changeset *data_reserved = NULL; 4512 char *kaddr; 4513 bool only_release_metadata = false; 4514 u32 blocksize = fs_info->sectorsize; 4515 pgoff_t index = from >> PAGE_SHIFT; 4516 unsigned offset = from & (blocksize - 1); 4517 struct page *page; 4518 gfp_t mask = btrfs_alloc_write_mask(mapping); 4519 size_t write_bytes = blocksize; 4520 int ret = 0; 4521 u64 block_start; 4522 u64 block_end; 4523 4524 if (IS_ALIGNED(offset, blocksize) && 4525 (!len || IS_ALIGNED(len, blocksize))) 4526 goto out; 4527 4528 block_start = round_down(from, blocksize); 4529 block_end = block_start + blocksize - 1; 4530 4531 ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 4532 block_start, blocksize); 4533 if (ret < 0) { 4534 if (btrfs_check_nocow_lock(BTRFS_I(inode), block_start, 4535 &write_bytes) > 0) { 4536 /* For nocow case, no need to reserve data space */ 4537 only_release_metadata = true; 4538 } else { 4539 goto out; 4540 } 4541 } 4542 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), blocksize); 4543 if (ret < 0) { 4544 if (!only_release_metadata) 4545 btrfs_free_reserved_data_space(BTRFS_I(inode), 4546 data_reserved, block_start, blocksize); 4547 goto out; 4548 } 4549 again: 4550 page = find_or_create_page(mapping, index, mask); 4551 if (!page) { 4552 btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved, 4553 block_start, blocksize, true); 4554 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize); 4555 ret = -ENOMEM; 4556 goto out; 4557 } 4558 4559 if (!PageUptodate(page)) { 4560 ret = btrfs_readpage(NULL, page); 4561 lock_page(page); 4562 if (page->mapping != mapping) { 4563 unlock_page(page); 4564 put_page(page); 4565 goto again; 4566 } 4567 if (!PageUptodate(page)) { 4568 ret = -EIO; 4569 goto out_unlock; 4570 } 4571 } 4572 wait_on_page_writeback(page); 4573 4574 lock_extent_bits(io_tree, block_start, block_end, &cached_state); 4575 set_page_extent_mapped(page); 4576 4577 ordered = btrfs_lookup_ordered_extent(BTRFS_I(inode), block_start); 4578 if (ordered) { 4579 unlock_extent_cached(io_tree, block_start, block_end, 4580 &cached_state); 4581 unlock_page(page); 4582 put_page(page); 4583 btrfs_start_ordered_extent(inode, ordered, 1); 4584 btrfs_put_ordered_extent(ordered); 4585 goto again; 4586 } 4587 4588 clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end, 4589 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4590 0, 0, &cached_state); 4591 4592 ret = btrfs_set_extent_delalloc(BTRFS_I(inode), block_start, block_end, 0, 4593 &cached_state); 4594 if (ret) { 4595 unlock_extent_cached(io_tree, block_start, block_end, 4596 &cached_state); 4597 goto out_unlock; 4598 } 4599 4600 if (offset != blocksize) { 4601 if (!len) 4602 len = blocksize - offset; 4603 kaddr = kmap(page); 4604 if (front) 4605 memset(kaddr + (block_start - page_offset(page)), 4606 0, offset); 4607 else 4608 memset(kaddr + (block_start - page_offset(page)) + offset, 4609 0, len); 4610 flush_dcache_page(page); 4611 kunmap(page); 4612 } 4613 ClearPageChecked(page); 4614 set_page_dirty(page); 4615 unlock_extent_cached(io_tree, block_start, block_end, &cached_state); 4616 4617 if (only_release_metadata) 4618 set_extent_bit(&BTRFS_I(inode)->io_tree, block_start, 4619 block_end, EXTENT_NORESERVE, NULL, NULL, 4620 GFP_NOFS); 4621 4622 out_unlock: 4623 if (ret) { 4624 if (only_release_metadata) 4625 btrfs_delalloc_release_metadata(BTRFS_I(inode), 4626 blocksize, true); 4627 else 4628 btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved, 4629 block_start, blocksize, true); 4630 } 4631 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize); 4632 unlock_page(page); 4633 put_page(page); 4634 out: 4635 if (only_release_metadata) 4636 btrfs_check_nocow_unlock(BTRFS_I(inode)); 4637 extent_changeset_free(data_reserved); 4638 return ret; 4639 } 4640 4641 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode, 4642 u64 offset, u64 len) 4643 { 4644 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4645 struct btrfs_trans_handle *trans; 4646 int ret; 4647 4648 /* 4649 * Still need to make sure the inode looks like it's been updated so 4650 * that any holes get logged if we fsync. 4651 */ 4652 if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 4653 BTRFS_I(inode)->last_trans = fs_info->generation; 4654 BTRFS_I(inode)->last_sub_trans = root->log_transid; 4655 BTRFS_I(inode)->last_log_commit = root->last_log_commit; 4656 return 0; 4657 } 4658 4659 /* 4660 * 1 - for the one we're dropping 4661 * 1 - for the one we're adding 4662 * 1 - for updating the inode. 4663 */ 4664 trans = btrfs_start_transaction(root, 3); 4665 if (IS_ERR(trans)) 4666 return PTR_ERR(trans); 4667 4668 ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1); 4669 if (ret) { 4670 btrfs_abort_transaction(trans, ret); 4671 btrfs_end_transaction(trans); 4672 return ret; 4673 } 4674 4675 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)), 4676 offset, 0, 0, len, 0, len, 0, 0, 0); 4677 if (ret) 4678 btrfs_abort_transaction(trans, ret); 4679 else 4680 btrfs_update_inode(trans, root, inode); 4681 btrfs_end_transaction(trans); 4682 return ret; 4683 } 4684 4685 /* 4686 * This function puts in dummy file extents for the area we're creating a hole 4687 * for. So if we are truncating this file to a larger size we need to insert 4688 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 4689 * the range between oldsize and size 4690 */ 4691 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size) 4692 { 4693 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 4694 struct btrfs_root *root = BTRFS_I(inode)->root; 4695 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4696 struct extent_map *em = NULL; 4697 struct extent_state *cached_state = NULL; 4698 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 4699 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 4700 u64 block_end = ALIGN(size, fs_info->sectorsize); 4701 u64 last_byte; 4702 u64 cur_offset; 4703 u64 hole_size; 4704 int err = 0; 4705 4706 /* 4707 * If our size started in the middle of a block we need to zero out the 4708 * rest of the block before we expand the i_size, otherwise we could 4709 * expose stale data. 4710 */ 4711 err = btrfs_truncate_block(inode, oldsize, 0, 0); 4712 if (err) 4713 return err; 4714 4715 if (size <= hole_start) 4716 return 0; 4717 4718 btrfs_lock_and_flush_ordered_range(BTRFS_I(inode), hole_start, 4719 block_end - 1, &cached_state); 4720 cur_offset = hole_start; 4721 while (1) { 4722 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset, 4723 block_end - cur_offset); 4724 if (IS_ERR(em)) { 4725 err = PTR_ERR(em); 4726 em = NULL; 4727 break; 4728 } 4729 last_byte = min(extent_map_end(em), block_end); 4730 last_byte = ALIGN(last_byte, fs_info->sectorsize); 4731 hole_size = last_byte - cur_offset; 4732 4733 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 4734 struct extent_map *hole_em; 4735 4736 err = maybe_insert_hole(root, inode, cur_offset, 4737 hole_size); 4738 if (err) 4739 break; 4740 4741 err = btrfs_inode_set_file_extent_range(BTRFS_I(inode), 4742 cur_offset, hole_size); 4743 if (err) 4744 break; 4745 4746 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 4747 cur_offset + hole_size - 1, 0); 4748 hole_em = alloc_extent_map(); 4749 if (!hole_em) { 4750 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 4751 &BTRFS_I(inode)->runtime_flags); 4752 goto next; 4753 } 4754 hole_em->start = cur_offset; 4755 hole_em->len = hole_size; 4756 hole_em->orig_start = cur_offset; 4757 4758 hole_em->block_start = EXTENT_MAP_HOLE; 4759 hole_em->block_len = 0; 4760 hole_em->orig_block_len = 0; 4761 hole_em->ram_bytes = hole_size; 4762 hole_em->compress_type = BTRFS_COMPRESS_NONE; 4763 hole_em->generation = fs_info->generation; 4764 4765 while (1) { 4766 write_lock(&em_tree->lock); 4767 err = add_extent_mapping(em_tree, hole_em, 1); 4768 write_unlock(&em_tree->lock); 4769 if (err != -EEXIST) 4770 break; 4771 btrfs_drop_extent_cache(BTRFS_I(inode), 4772 cur_offset, 4773 cur_offset + 4774 hole_size - 1, 0); 4775 } 4776 free_extent_map(hole_em); 4777 } else { 4778 err = btrfs_inode_set_file_extent_range(BTRFS_I(inode), 4779 cur_offset, hole_size); 4780 if (err) 4781 break; 4782 } 4783 next: 4784 free_extent_map(em); 4785 em = NULL; 4786 cur_offset = last_byte; 4787 if (cur_offset >= block_end) 4788 break; 4789 } 4790 free_extent_map(em); 4791 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state); 4792 return err; 4793 } 4794 4795 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 4796 { 4797 struct btrfs_root *root = BTRFS_I(inode)->root; 4798 struct btrfs_trans_handle *trans; 4799 loff_t oldsize = i_size_read(inode); 4800 loff_t newsize = attr->ia_size; 4801 int mask = attr->ia_valid; 4802 int ret; 4803 4804 /* 4805 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 4806 * special case where we need to update the times despite not having 4807 * these flags set. For all other operations the VFS set these flags 4808 * explicitly if it wants a timestamp update. 4809 */ 4810 if (newsize != oldsize) { 4811 inode_inc_iversion(inode); 4812 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) 4813 inode->i_ctime = inode->i_mtime = 4814 current_time(inode); 4815 } 4816 4817 if (newsize > oldsize) { 4818 /* 4819 * Don't do an expanding truncate while snapshotting is ongoing. 4820 * This is to ensure the snapshot captures a fully consistent 4821 * state of this file - if the snapshot captures this expanding 4822 * truncation, it must capture all writes that happened before 4823 * this truncation. 4824 */ 4825 btrfs_drew_write_lock(&root->snapshot_lock); 4826 ret = btrfs_cont_expand(inode, oldsize, newsize); 4827 if (ret) { 4828 btrfs_drew_write_unlock(&root->snapshot_lock); 4829 return ret; 4830 } 4831 4832 trans = btrfs_start_transaction(root, 1); 4833 if (IS_ERR(trans)) { 4834 btrfs_drew_write_unlock(&root->snapshot_lock); 4835 return PTR_ERR(trans); 4836 } 4837 4838 i_size_write(inode, newsize); 4839 btrfs_inode_safe_disk_i_size_write(inode, 0); 4840 pagecache_isize_extended(inode, oldsize, newsize); 4841 ret = btrfs_update_inode(trans, root, inode); 4842 btrfs_drew_write_unlock(&root->snapshot_lock); 4843 btrfs_end_transaction(trans); 4844 } else { 4845 4846 /* 4847 * We're truncating a file that used to have good data down to 4848 * zero. Make sure it gets into the ordered flush list so that 4849 * any new writes get down to disk quickly. 4850 */ 4851 if (newsize == 0) 4852 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE, 4853 &BTRFS_I(inode)->runtime_flags); 4854 4855 truncate_setsize(inode, newsize); 4856 4857 /* Disable nonlocked read DIO to avoid the endless truncate */ 4858 btrfs_inode_block_unlocked_dio(BTRFS_I(inode)); 4859 inode_dio_wait(inode); 4860 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode)); 4861 4862 ret = btrfs_truncate(inode, newsize == oldsize); 4863 if (ret && inode->i_nlink) { 4864 int err; 4865 4866 /* 4867 * Truncate failed, so fix up the in-memory size. We 4868 * adjusted disk_i_size down as we removed extents, so 4869 * wait for disk_i_size to be stable and then update the 4870 * in-memory size to match. 4871 */ 4872 err = btrfs_wait_ordered_range(inode, 0, (u64)-1); 4873 if (err) 4874 return err; 4875 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 4876 } 4877 } 4878 4879 return ret; 4880 } 4881 4882 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) 4883 { 4884 struct inode *inode = d_inode(dentry); 4885 struct btrfs_root *root = BTRFS_I(inode)->root; 4886 int err; 4887 4888 if (btrfs_root_readonly(root)) 4889 return -EROFS; 4890 4891 err = setattr_prepare(dentry, attr); 4892 if (err) 4893 return err; 4894 4895 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 4896 err = btrfs_setsize(inode, attr); 4897 if (err) 4898 return err; 4899 } 4900 4901 if (attr->ia_valid) { 4902 setattr_copy(inode, attr); 4903 inode_inc_iversion(inode); 4904 err = btrfs_dirty_inode(inode); 4905 4906 if (!err && attr->ia_valid & ATTR_MODE) 4907 err = posix_acl_chmod(inode, inode->i_mode); 4908 } 4909 4910 return err; 4911 } 4912 4913 /* 4914 * While truncating the inode pages during eviction, we get the VFS calling 4915 * btrfs_invalidatepage() against each page of the inode. This is slow because 4916 * the calls to btrfs_invalidatepage() result in a huge amount of calls to 4917 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting 4918 * extent_state structures over and over, wasting lots of time. 4919 * 4920 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all 4921 * those expensive operations on a per page basis and do only the ordered io 4922 * finishing, while we release here the extent_map and extent_state structures, 4923 * without the excessive merging and splitting. 4924 */ 4925 static void evict_inode_truncate_pages(struct inode *inode) 4926 { 4927 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4928 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; 4929 struct rb_node *node; 4930 4931 ASSERT(inode->i_state & I_FREEING); 4932 truncate_inode_pages_final(&inode->i_data); 4933 4934 write_lock(&map_tree->lock); 4935 while (!RB_EMPTY_ROOT(&map_tree->map.rb_root)) { 4936 struct extent_map *em; 4937 4938 node = rb_first_cached(&map_tree->map); 4939 em = rb_entry(node, struct extent_map, rb_node); 4940 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 4941 clear_bit(EXTENT_FLAG_LOGGING, &em->flags); 4942 remove_extent_mapping(map_tree, em); 4943 free_extent_map(em); 4944 if (need_resched()) { 4945 write_unlock(&map_tree->lock); 4946 cond_resched(); 4947 write_lock(&map_tree->lock); 4948 } 4949 } 4950 write_unlock(&map_tree->lock); 4951 4952 /* 4953 * Keep looping until we have no more ranges in the io tree. 4954 * We can have ongoing bios started by readahead that have 4955 * their endio callback (extent_io.c:end_bio_extent_readpage) 4956 * still in progress (unlocked the pages in the bio but did not yet 4957 * unlocked the ranges in the io tree). Therefore this means some 4958 * ranges can still be locked and eviction started because before 4959 * submitting those bios, which are executed by a separate task (work 4960 * queue kthread), inode references (inode->i_count) were not taken 4961 * (which would be dropped in the end io callback of each bio). 4962 * Therefore here we effectively end up waiting for those bios and 4963 * anyone else holding locked ranges without having bumped the inode's 4964 * reference count - if we don't do it, when they access the inode's 4965 * io_tree to unlock a range it may be too late, leading to an 4966 * use-after-free issue. 4967 */ 4968 spin_lock(&io_tree->lock); 4969 while (!RB_EMPTY_ROOT(&io_tree->state)) { 4970 struct extent_state *state; 4971 struct extent_state *cached_state = NULL; 4972 u64 start; 4973 u64 end; 4974 unsigned state_flags; 4975 4976 node = rb_first(&io_tree->state); 4977 state = rb_entry(node, struct extent_state, rb_node); 4978 start = state->start; 4979 end = state->end; 4980 state_flags = state->state; 4981 spin_unlock(&io_tree->lock); 4982 4983 lock_extent_bits(io_tree, start, end, &cached_state); 4984 4985 /* 4986 * If still has DELALLOC flag, the extent didn't reach disk, 4987 * and its reserved space won't be freed by delayed_ref. 4988 * So we need to free its reserved space here. 4989 * (Refer to comment in btrfs_invalidatepage, case 2) 4990 * 4991 * Note, end is the bytenr of last byte, so we need + 1 here. 4992 */ 4993 if (state_flags & EXTENT_DELALLOC) 4994 btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start, 4995 end - start + 1); 4996 4997 clear_extent_bit(io_tree, start, end, 4998 EXTENT_LOCKED | EXTENT_DELALLOC | 4999 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1, 5000 &cached_state); 5001 5002 cond_resched(); 5003 spin_lock(&io_tree->lock); 5004 } 5005 spin_unlock(&io_tree->lock); 5006 } 5007 5008 static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root, 5009 struct btrfs_block_rsv *rsv) 5010 { 5011 struct btrfs_fs_info *fs_info = root->fs_info; 5012 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 5013 struct btrfs_trans_handle *trans; 5014 u64 delayed_refs_extra = btrfs_calc_insert_metadata_size(fs_info, 1); 5015 int ret; 5016 5017 /* 5018 * Eviction should be taking place at some place safe because of our 5019 * delayed iputs. However the normal flushing code will run delayed 5020 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock. 5021 * 5022 * We reserve the delayed_refs_extra here again because we can't use 5023 * btrfs_start_transaction(root, 0) for the same deadlocky reason as 5024 * above. We reserve our extra bit here because we generate a ton of 5025 * delayed refs activity by truncating. 5026 * 5027 * If we cannot make our reservation we'll attempt to steal from the 5028 * global reserve, because we really want to be able to free up space. 5029 */ 5030 ret = btrfs_block_rsv_refill(root, rsv, rsv->size + delayed_refs_extra, 5031 BTRFS_RESERVE_FLUSH_EVICT); 5032 if (ret) { 5033 /* 5034 * Try to steal from the global reserve if there is space for 5035 * it. 5036 */ 5037 if (btrfs_check_space_for_delayed_refs(fs_info) || 5038 btrfs_block_rsv_migrate(global_rsv, rsv, rsv->size, 0)) { 5039 btrfs_warn(fs_info, 5040 "could not allocate space for delete; will truncate on mount"); 5041 return ERR_PTR(-ENOSPC); 5042 } 5043 delayed_refs_extra = 0; 5044 } 5045 5046 trans = btrfs_join_transaction(root); 5047 if (IS_ERR(trans)) 5048 return trans; 5049 5050 if (delayed_refs_extra) { 5051 trans->block_rsv = &fs_info->trans_block_rsv; 5052 trans->bytes_reserved = delayed_refs_extra; 5053 btrfs_block_rsv_migrate(rsv, trans->block_rsv, 5054 delayed_refs_extra, 1); 5055 } 5056 return trans; 5057 } 5058 5059 void btrfs_evict_inode(struct inode *inode) 5060 { 5061 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5062 struct btrfs_trans_handle *trans; 5063 struct btrfs_root *root = BTRFS_I(inode)->root; 5064 struct btrfs_block_rsv *rsv; 5065 int ret; 5066 5067 trace_btrfs_inode_evict(inode); 5068 5069 if (!root) { 5070 clear_inode(inode); 5071 return; 5072 } 5073 5074 evict_inode_truncate_pages(inode); 5075 5076 if (inode->i_nlink && 5077 ((btrfs_root_refs(&root->root_item) != 0 && 5078 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 5079 btrfs_is_free_space_inode(BTRFS_I(inode)))) 5080 goto no_delete; 5081 5082 if (is_bad_inode(inode)) 5083 goto no_delete; 5084 5085 btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1); 5086 5087 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) 5088 goto no_delete; 5089 5090 if (inode->i_nlink > 0) { 5091 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5092 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 5093 goto no_delete; 5094 } 5095 5096 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); 5097 if (ret) 5098 goto no_delete; 5099 5100 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 5101 if (!rsv) 5102 goto no_delete; 5103 rsv->size = btrfs_calc_metadata_size(fs_info, 1); 5104 rsv->failfast = 1; 5105 5106 btrfs_i_size_write(BTRFS_I(inode), 0); 5107 5108 while (1) { 5109 trans = evict_refill_and_join(root, rsv); 5110 if (IS_ERR(trans)) 5111 goto free_rsv; 5112 5113 trans->block_rsv = rsv; 5114 5115 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0); 5116 trans->block_rsv = &fs_info->trans_block_rsv; 5117 btrfs_end_transaction(trans); 5118 btrfs_btree_balance_dirty(fs_info); 5119 if (ret && ret != -ENOSPC && ret != -EAGAIN) 5120 goto free_rsv; 5121 else if (!ret) 5122 break; 5123 } 5124 5125 /* 5126 * Errors here aren't a big deal, it just means we leave orphan items in 5127 * the tree. They will be cleaned up on the next mount. If the inode 5128 * number gets reused, cleanup deletes the orphan item without doing 5129 * anything, and unlink reuses the existing orphan item. 5130 * 5131 * If it turns out that we are dropping too many of these, we might want 5132 * to add a mechanism for retrying these after a commit. 5133 */ 5134 trans = evict_refill_and_join(root, rsv); 5135 if (!IS_ERR(trans)) { 5136 trans->block_rsv = rsv; 5137 btrfs_orphan_del(trans, BTRFS_I(inode)); 5138 trans->block_rsv = &fs_info->trans_block_rsv; 5139 btrfs_end_transaction(trans); 5140 } 5141 5142 if (!(root == fs_info->tree_root || 5143 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)) 5144 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode))); 5145 5146 free_rsv: 5147 btrfs_free_block_rsv(fs_info, rsv); 5148 no_delete: 5149 /* 5150 * If we didn't successfully delete, the orphan item will still be in 5151 * the tree and we'll retry on the next mount. Again, we might also want 5152 * to retry these periodically in the future. 5153 */ 5154 btrfs_remove_delayed_node(BTRFS_I(inode)); 5155 clear_inode(inode); 5156 } 5157 5158 /* 5159 * Return the key found in the dir entry in the location pointer, fill @type 5160 * with BTRFS_FT_*, and return 0. 5161 * 5162 * If no dir entries were found, returns -ENOENT. 5163 * If found a corrupted location in dir entry, returns -EUCLEAN. 5164 */ 5165 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry, 5166 struct btrfs_key *location, u8 *type) 5167 { 5168 const char *name = dentry->d_name.name; 5169 int namelen = dentry->d_name.len; 5170 struct btrfs_dir_item *di; 5171 struct btrfs_path *path; 5172 struct btrfs_root *root = BTRFS_I(dir)->root; 5173 int ret = 0; 5174 5175 path = btrfs_alloc_path(); 5176 if (!path) 5177 return -ENOMEM; 5178 5179 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)), 5180 name, namelen, 0); 5181 if (IS_ERR_OR_NULL(di)) { 5182 ret = di ? PTR_ERR(di) : -ENOENT; 5183 goto out; 5184 } 5185 5186 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5187 if (location->type != BTRFS_INODE_ITEM_KEY && 5188 location->type != BTRFS_ROOT_ITEM_KEY) { 5189 ret = -EUCLEAN; 5190 btrfs_warn(root->fs_info, 5191 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))", 5192 __func__, name, btrfs_ino(BTRFS_I(dir)), 5193 location->objectid, location->type, location->offset); 5194 } 5195 if (!ret) 5196 *type = btrfs_dir_type(path->nodes[0], di); 5197 out: 5198 btrfs_free_path(path); 5199 return ret; 5200 } 5201 5202 /* 5203 * when we hit a tree root in a directory, the btrfs part of the inode 5204 * needs to be changed to reflect the root directory of the tree root. This 5205 * is kind of like crossing a mount point. 5206 */ 5207 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5208 struct inode *dir, 5209 struct dentry *dentry, 5210 struct btrfs_key *location, 5211 struct btrfs_root **sub_root) 5212 { 5213 struct btrfs_path *path; 5214 struct btrfs_root *new_root; 5215 struct btrfs_root_ref *ref; 5216 struct extent_buffer *leaf; 5217 struct btrfs_key key; 5218 int ret; 5219 int err = 0; 5220 5221 path = btrfs_alloc_path(); 5222 if (!path) { 5223 err = -ENOMEM; 5224 goto out; 5225 } 5226 5227 err = -ENOENT; 5228 key.objectid = BTRFS_I(dir)->root->root_key.objectid; 5229 key.type = BTRFS_ROOT_REF_KEY; 5230 key.offset = location->objectid; 5231 5232 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5233 if (ret) { 5234 if (ret < 0) 5235 err = ret; 5236 goto out; 5237 } 5238 5239 leaf = path->nodes[0]; 5240 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5241 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) || 5242 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len) 5243 goto out; 5244 5245 ret = memcmp_extent_buffer(leaf, dentry->d_name.name, 5246 (unsigned long)(ref + 1), 5247 dentry->d_name.len); 5248 if (ret) 5249 goto out; 5250 5251 btrfs_release_path(path); 5252 5253 new_root = btrfs_get_fs_root(fs_info, location->objectid, true); 5254 if (IS_ERR(new_root)) { 5255 err = PTR_ERR(new_root); 5256 goto out; 5257 } 5258 5259 *sub_root = new_root; 5260 location->objectid = btrfs_root_dirid(&new_root->root_item); 5261 location->type = BTRFS_INODE_ITEM_KEY; 5262 location->offset = 0; 5263 err = 0; 5264 out: 5265 btrfs_free_path(path); 5266 return err; 5267 } 5268 5269 static void inode_tree_add(struct inode *inode) 5270 { 5271 struct btrfs_root *root = BTRFS_I(inode)->root; 5272 struct btrfs_inode *entry; 5273 struct rb_node **p; 5274 struct rb_node *parent; 5275 struct rb_node *new = &BTRFS_I(inode)->rb_node; 5276 u64 ino = btrfs_ino(BTRFS_I(inode)); 5277 5278 if (inode_unhashed(inode)) 5279 return; 5280 parent = NULL; 5281 spin_lock(&root->inode_lock); 5282 p = &root->inode_tree.rb_node; 5283 while (*p) { 5284 parent = *p; 5285 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5286 5287 if (ino < btrfs_ino(entry)) 5288 p = &parent->rb_left; 5289 else if (ino > btrfs_ino(entry)) 5290 p = &parent->rb_right; 5291 else { 5292 WARN_ON(!(entry->vfs_inode.i_state & 5293 (I_WILL_FREE | I_FREEING))); 5294 rb_replace_node(parent, new, &root->inode_tree); 5295 RB_CLEAR_NODE(parent); 5296 spin_unlock(&root->inode_lock); 5297 return; 5298 } 5299 } 5300 rb_link_node(new, parent, p); 5301 rb_insert_color(new, &root->inode_tree); 5302 spin_unlock(&root->inode_lock); 5303 } 5304 5305 static void inode_tree_del(struct inode *inode) 5306 { 5307 struct btrfs_root *root = BTRFS_I(inode)->root; 5308 int empty = 0; 5309 5310 spin_lock(&root->inode_lock); 5311 if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) { 5312 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree); 5313 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node); 5314 empty = RB_EMPTY_ROOT(&root->inode_tree); 5315 } 5316 spin_unlock(&root->inode_lock); 5317 5318 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5319 spin_lock(&root->inode_lock); 5320 empty = RB_EMPTY_ROOT(&root->inode_tree); 5321 spin_unlock(&root->inode_lock); 5322 if (empty) 5323 btrfs_add_dead_root(root); 5324 } 5325 } 5326 5327 5328 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5329 { 5330 struct btrfs_iget_args *args = p; 5331 5332 inode->i_ino = args->ino; 5333 BTRFS_I(inode)->location.objectid = args->ino; 5334 BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY; 5335 BTRFS_I(inode)->location.offset = 0; 5336 BTRFS_I(inode)->root = btrfs_grab_root(args->root); 5337 BUG_ON(args->root && !BTRFS_I(inode)->root); 5338 return 0; 5339 } 5340 5341 static int btrfs_find_actor(struct inode *inode, void *opaque) 5342 { 5343 struct btrfs_iget_args *args = opaque; 5344 5345 return args->ino == BTRFS_I(inode)->location.objectid && 5346 args->root == BTRFS_I(inode)->root; 5347 } 5348 5349 static struct inode *btrfs_iget_locked(struct super_block *s, u64 ino, 5350 struct btrfs_root *root) 5351 { 5352 struct inode *inode; 5353 struct btrfs_iget_args args; 5354 unsigned long hashval = btrfs_inode_hash(ino, root); 5355 5356 args.ino = ino; 5357 args.root = root; 5358 5359 inode = iget5_locked(s, hashval, btrfs_find_actor, 5360 btrfs_init_locked_inode, 5361 (void *)&args); 5362 return inode; 5363 } 5364 5365 /* 5366 * Get an inode object given its inode number and corresponding root. 5367 * Path can be preallocated to prevent recursing back to iget through 5368 * allocator. NULL is also valid but may require an additional allocation 5369 * later. 5370 */ 5371 struct inode *btrfs_iget_path(struct super_block *s, u64 ino, 5372 struct btrfs_root *root, struct btrfs_path *path) 5373 { 5374 struct inode *inode; 5375 5376 inode = btrfs_iget_locked(s, ino, root); 5377 if (!inode) 5378 return ERR_PTR(-ENOMEM); 5379 5380 if (inode->i_state & I_NEW) { 5381 int ret; 5382 5383 ret = btrfs_read_locked_inode(inode, path); 5384 if (!ret) { 5385 inode_tree_add(inode); 5386 unlock_new_inode(inode); 5387 } else { 5388 iget_failed(inode); 5389 /* 5390 * ret > 0 can come from btrfs_search_slot called by 5391 * btrfs_read_locked_inode, this means the inode item 5392 * was not found. 5393 */ 5394 if (ret > 0) 5395 ret = -ENOENT; 5396 inode = ERR_PTR(ret); 5397 } 5398 } 5399 5400 return inode; 5401 } 5402 5403 struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root) 5404 { 5405 return btrfs_iget_path(s, ino, root, NULL); 5406 } 5407 5408 static struct inode *new_simple_dir(struct super_block *s, 5409 struct btrfs_key *key, 5410 struct btrfs_root *root) 5411 { 5412 struct inode *inode = new_inode(s); 5413 5414 if (!inode) 5415 return ERR_PTR(-ENOMEM); 5416 5417 BTRFS_I(inode)->root = btrfs_grab_root(root); 5418 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5419 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5420 5421 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5422 /* 5423 * We only need lookup, the rest is read-only and there's no inode 5424 * associated with the dentry 5425 */ 5426 inode->i_op = &simple_dir_inode_operations; 5427 inode->i_opflags &= ~IOP_XATTR; 5428 inode->i_fop = &simple_dir_operations; 5429 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5430 inode->i_mtime = current_time(inode); 5431 inode->i_atime = inode->i_mtime; 5432 inode->i_ctime = inode->i_mtime; 5433 BTRFS_I(inode)->i_otime = inode->i_mtime; 5434 5435 return inode; 5436 } 5437 5438 static inline u8 btrfs_inode_type(struct inode *inode) 5439 { 5440 /* 5441 * Compile-time asserts that generic FT_* types still match 5442 * BTRFS_FT_* types 5443 */ 5444 BUILD_BUG_ON(BTRFS_FT_UNKNOWN != FT_UNKNOWN); 5445 BUILD_BUG_ON(BTRFS_FT_REG_FILE != FT_REG_FILE); 5446 BUILD_BUG_ON(BTRFS_FT_DIR != FT_DIR); 5447 BUILD_BUG_ON(BTRFS_FT_CHRDEV != FT_CHRDEV); 5448 BUILD_BUG_ON(BTRFS_FT_BLKDEV != FT_BLKDEV); 5449 BUILD_BUG_ON(BTRFS_FT_FIFO != FT_FIFO); 5450 BUILD_BUG_ON(BTRFS_FT_SOCK != FT_SOCK); 5451 BUILD_BUG_ON(BTRFS_FT_SYMLINK != FT_SYMLINK); 5452 5453 return fs_umode_to_ftype(inode->i_mode); 5454 } 5455 5456 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5457 { 5458 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 5459 struct inode *inode; 5460 struct btrfs_root *root = BTRFS_I(dir)->root; 5461 struct btrfs_root *sub_root = root; 5462 struct btrfs_key location; 5463 u8 di_type = 0; 5464 int ret = 0; 5465 5466 if (dentry->d_name.len > BTRFS_NAME_LEN) 5467 return ERR_PTR(-ENAMETOOLONG); 5468 5469 ret = btrfs_inode_by_name(dir, dentry, &location, &di_type); 5470 if (ret < 0) 5471 return ERR_PTR(ret); 5472 5473 if (location.type == BTRFS_INODE_ITEM_KEY) { 5474 inode = btrfs_iget(dir->i_sb, location.objectid, root); 5475 if (IS_ERR(inode)) 5476 return inode; 5477 5478 /* Do extra check against inode mode with di_type */ 5479 if (btrfs_inode_type(inode) != di_type) { 5480 btrfs_crit(fs_info, 5481 "inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u", 5482 inode->i_mode, btrfs_inode_type(inode), 5483 di_type); 5484 iput(inode); 5485 return ERR_PTR(-EUCLEAN); 5486 } 5487 return inode; 5488 } 5489 5490 ret = fixup_tree_root_location(fs_info, dir, dentry, 5491 &location, &sub_root); 5492 if (ret < 0) { 5493 if (ret != -ENOENT) 5494 inode = ERR_PTR(ret); 5495 else 5496 inode = new_simple_dir(dir->i_sb, &location, sub_root); 5497 } else { 5498 inode = btrfs_iget(dir->i_sb, location.objectid, sub_root); 5499 } 5500 if (root != sub_root) 5501 btrfs_put_root(sub_root); 5502 5503 if (!IS_ERR(inode) && root != sub_root) { 5504 down_read(&fs_info->cleanup_work_sem); 5505 if (!sb_rdonly(inode->i_sb)) 5506 ret = btrfs_orphan_cleanup(sub_root); 5507 up_read(&fs_info->cleanup_work_sem); 5508 if (ret) { 5509 iput(inode); 5510 inode = ERR_PTR(ret); 5511 } 5512 } 5513 5514 return inode; 5515 } 5516 5517 static int btrfs_dentry_delete(const struct dentry *dentry) 5518 { 5519 struct btrfs_root *root; 5520 struct inode *inode = d_inode(dentry); 5521 5522 if (!inode && !IS_ROOT(dentry)) 5523 inode = d_inode(dentry->d_parent); 5524 5525 if (inode) { 5526 root = BTRFS_I(inode)->root; 5527 if (btrfs_root_refs(&root->root_item) == 0) 5528 return 1; 5529 5530 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5531 return 1; 5532 } 5533 return 0; 5534 } 5535 5536 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5537 unsigned int flags) 5538 { 5539 struct inode *inode = btrfs_lookup_dentry(dir, dentry); 5540 5541 if (inode == ERR_PTR(-ENOENT)) 5542 inode = NULL; 5543 return d_splice_alias(inode, dentry); 5544 } 5545 5546 /* 5547 * All this infrastructure exists because dir_emit can fault, and we are holding 5548 * the tree lock when doing readdir. For now just allocate a buffer and copy 5549 * our information into that, and then dir_emit from the buffer. This is 5550 * similar to what NFS does, only we don't keep the buffer around in pagecache 5551 * because I'm afraid I'll mess that up. Long term we need to make filldir do 5552 * copy_to_user_inatomic so we don't have to worry about page faulting under the 5553 * tree lock. 5554 */ 5555 static int btrfs_opendir(struct inode *inode, struct file *file) 5556 { 5557 struct btrfs_file_private *private; 5558 5559 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL); 5560 if (!private) 5561 return -ENOMEM; 5562 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 5563 if (!private->filldir_buf) { 5564 kfree(private); 5565 return -ENOMEM; 5566 } 5567 file->private_data = private; 5568 return 0; 5569 } 5570 5571 struct dir_entry { 5572 u64 ino; 5573 u64 offset; 5574 unsigned type; 5575 int name_len; 5576 }; 5577 5578 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) 5579 { 5580 while (entries--) { 5581 struct dir_entry *entry = addr; 5582 char *name = (char *)(entry + 1); 5583 5584 ctx->pos = get_unaligned(&entry->offset); 5585 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len), 5586 get_unaligned(&entry->ino), 5587 get_unaligned(&entry->type))) 5588 return 1; 5589 addr += sizeof(struct dir_entry) + 5590 get_unaligned(&entry->name_len); 5591 ctx->pos++; 5592 } 5593 return 0; 5594 } 5595 5596 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5597 { 5598 struct inode *inode = file_inode(file); 5599 struct btrfs_root *root = BTRFS_I(inode)->root; 5600 struct btrfs_file_private *private = file->private_data; 5601 struct btrfs_dir_item *di; 5602 struct btrfs_key key; 5603 struct btrfs_key found_key; 5604 struct btrfs_path *path; 5605 void *addr; 5606 struct list_head ins_list; 5607 struct list_head del_list; 5608 int ret; 5609 struct extent_buffer *leaf; 5610 int slot; 5611 char *name_ptr; 5612 int name_len; 5613 int entries = 0; 5614 int total_len = 0; 5615 bool put = false; 5616 struct btrfs_key location; 5617 5618 if (!dir_emit_dots(file, ctx)) 5619 return 0; 5620 5621 path = btrfs_alloc_path(); 5622 if (!path) 5623 return -ENOMEM; 5624 5625 addr = private->filldir_buf; 5626 path->reada = READA_FORWARD; 5627 5628 INIT_LIST_HEAD(&ins_list); 5629 INIT_LIST_HEAD(&del_list); 5630 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list); 5631 5632 again: 5633 key.type = BTRFS_DIR_INDEX_KEY; 5634 key.offset = ctx->pos; 5635 key.objectid = btrfs_ino(BTRFS_I(inode)); 5636 5637 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5638 if (ret < 0) 5639 goto err; 5640 5641 while (1) { 5642 struct dir_entry *entry; 5643 5644 leaf = path->nodes[0]; 5645 slot = path->slots[0]; 5646 if (slot >= btrfs_header_nritems(leaf)) { 5647 ret = btrfs_next_leaf(root, path); 5648 if (ret < 0) 5649 goto err; 5650 else if (ret > 0) 5651 break; 5652 continue; 5653 } 5654 5655 btrfs_item_key_to_cpu(leaf, &found_key, slot); 5656 5657 if (found_key.objectid != key.objectid) 5658 break; 5659 if (found_key.type != BTRFS_DIR_INDEX_KEY) 5660 break; 5661 if (found_key.offset < ctx->pos) 5662 goto next; 5663 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 5664 goto next; 5665 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); 5666 name_len = btrfs_dir_name_len(leaf, di); 5667 if ((total_len + sizeof(struct dir_entry) + name_len) >= 5668 PAGE_SIZE) { 5669 btrfs_release_path(path); 5670 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 5671 if (ret) 5672 goto nopos; 5673 addr = private->filldir_buf; 5674 entries = 0; 5675 total_len = 0; 5676 goto again; 5677 } 5678 5679 entry = addr; 5680 put_unaligned(name_len, &entry->name_len); 5681 name_ptr = (char *)(entry + 1); 5682 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1), 5683 name_len); 5684 put_unaligned(fs_ftype_to_dtype(btrfs_dir_type(leaf, di)), 5685 &entry->type); 5686 btrfs_dir_item_key_to_cpu(leaf, di, &location); 5687 put_unaligned(location.objectid, &entry->ino); 5688 put_unaligned(found_key.offset, &entry->offset); 5689 entries++; 5690 addr += sizeof(struct dir_entry) + name_len; 5691 total_len += sizeof(struct dir_entry) + name_len; 5692 next: 5693 path->slots[0]++; 5694 } 5695 btrfs_release_path(path); 5696 5697 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 5698 if (ret) 5699 goto nopos; 5700 5701 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 5702 if (ret) 5703 goto nopos; 5704 5705 /* 5706 * Stop new entries from being returned after we return the last 5707 * entry. 5708 * 5709 * New directory entries are assigned a strictly increasing 5710 * offset. This means that new entries created during readdir 5711 * are *guaranteed* to be seen in the future by that readdir. 5712 * This has broken buggy programs which operate on names as 5713 * they're returned by readdir. Until we re-use freed offsets 5714 * we have this hack to stop new entries from being returned 5715 * under the assumption that they'll never reach this huge 5716 * offset. 5717 * 5718 * This is being careful not to overflow 32bit loff_t unless the 5719 * last entry requires it because doing so has broken 32bit apps 5720 * in the past. 5721 */ 5722 if (ctx->pos >= INT_MAX) 5723 ctx->pos = LLONG_MAX; 5724 else 5725 ctx->pos = INT_MAX; 5726 nopos: 5727 ret = 0; 5728 err: 5729 if (put) 5730 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list); 5731 btrfs_free_path(path); 5732 return ret; 5733 } 5734 5735 /* 5736 * This is somewhat expensive, updating the tree every time the 5737 * inode changes. But, it is most likely to find the inode in cache. 5738 * FIXME, needs more benchmarking...there are no reasons other than performance 5739 * to keep or drop this code. 5740 */ 5741 static int btrfs_dirty_inode(struct inode *inode) 5742 { 5743 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5744 struct btrfs_root *root = BTRFS_I(inode)->root; 5745 struct btrfs_trans_handle *trans; 5746 int ret; 5747 5748 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 5749 return 0; 5750 5751 trans = btrfs_join_transaction(root); 5752 if (IS_ERR(trans)) 5753 return PTR_ERR(trans); 5754 5755 ret = btrfs_update_inode(trans, root, inode); 5756 if (ret && ret == -ENOSPC) { 5757 /* whoops, lets try again with the full transaction */ 5758 btrfs_end_transaction(trans); 5759 trans = btrfs_start_transaction(root, 1); 5760 if (IS_ERR(trans)) 5761 return PTR_ERR(trans); 5762 5763 ret = btrfs_update_inode(trans, root, inode); 5764 } 5765 btrfs_end_transaction(trans); 5766 if (BTRFS_I(inode)->delayed_node) 5767 btrfs_balance_delayed_items(fs_info); 5768 5769 return ret; 5770 } 5771 5772 /* 5773 * This is a copy of file_update_time. We need this so we can return error on 5774 * ENOSPC for updating the inode in the case of file write and mmap writes. 5775 */ 5776 static int btrfs_update_time(struct inode *inode, struct timespec64 *now, 5777 int flags) 5778 { 5779 struct btrfs_root *root = BTRFS_I(inode)->root; 5780 bool dirty = flags & ~S_VERSION; 5781 5782 if (btrfs_root_readonly(root)) 5783 return -EROFS; 5784 5785 if (flags & S_VERSION) 5786 dirty |= inode_maybe_inc_iversion(inode, dirty); 5787 if (flags & S_CTIME) 5788 inode->i_ctime = *now; 5789 if (flags & S_MTIME) 5790 inode->i_mtime = *now; 5791 if (flags & S_ATIME) 5792 inode->i_atime = *now; 5793 return dirty ? btrfs_dirty_inode(inode) : 0; 5794 } 5795 5796 /* 5797 * find the highest existing sequence number in a directory 5798 * and then set the in-memory index_cnt variable to reflect 5799 * free sequence numbers 5800 */ 5801 static int btrfs_set_inode_index_count(struct btrfs_inode *inode) 5802 { 5803 struct btrfs_root *root = inode->root; 5804 struct btrfs_key key, found_key; 5805 struct btrfs_path *path; 5806 struct extent_buffer *leaf; 5807 int ret; 5808 5809 key.objectid = btrfs_ino(inode); 5810 key.type = BTRFS_DIR_INDEX_KEY; 5811 key.offset = (u64)-1; 5812 5813 path = btrfs_alloc_path(); 5814 if (!path) 5815 return -ENOMEM; 5816 5817 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5818 if (ret < 0) 5819 goto out; 5820 /* FIXME: we should be able to handle this */ 5821 if (ret == 0) 5822 goto out; 5823 ret = 0; 5824 5825 /* 5826 * MAGIC NUMBER EXPLANATION: 5827 * since we search a directory based on f_pos we have to start at 2 5828 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody 5829 * else has to start at 2 5830 */ 5831 if (path->slots[0] == 0) { 5832 inode->index_cnt = 2; 5833 goto out; 5834 } 5835 5836 path->slots[0]--; 5837 5838 leaf = path->nodes[0]; 5839 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5840 5841 if (found_key.objectid != btrfs_ino(inode) || 5842 found_key.type != BTRFS_DIR_INDEX_KEY) { 5843 inode->index_cnt = 2; 5844 goto out; 5845 } 5846 5847 inode->index_cnt = found_key.offset + 1; 5848 out: 5849 btrfs_free_path(path); 5850 return ret; 5851 } 5852 5853 /* 5854 * helper to find a free sequence number in a given directory. This current 5855 * code is very simple, later versions will do smarter things in the btree 5856 */ 5857 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) 5858 { 5859 int ret = 0; 5860 5861 if (dir->index_cnt == (u64)-1) { 5862 ret = btrfs_inode_delayed_dir_index_count(dir); 5863 if (ret) { 5864 ret = btrfs_set_inode_index_count(dir); 5865 if (ret) 5866 return ret; 5867 } 5868 } 5869 5870 *index = dir->index_cnt; 5871 dir->index_cnt++; 5872 5873 return ret; 5874 } 5875 5876 static int btrfs_insert_inode_locked(struct inode *inode) 5877 { 5878 struct btrfs_iget_args args; 5879 5880 args.ino = BTRFS_I(inode)->location.objectid; 5881 args.root = BTRFS_I(inode)->root; 5882 5883 return insert_inode_locked4(inode, 5884 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 5885 btrfs_find_actor, &args); 5886 } 5887 5888 /* 5889 * Inherit flags from the parent inode. 5890 * 5891 * Currently only the compression flags and the cow flags are inherited. 5892 */ 5893 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir) 5894 { 5895 unsigned int flags; 5896 5897 if (!dir) 5898 return; 5899 5900 flags = BTRFS_I(dir)->flags; 5901 5902 if (flags & BTRFS_INODE_NOCOMPRESS) { 5903 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS; 5904 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 5905 } else if (flags & BTRFS_INODE_COMPRESS) { 5906 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS; 5907 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS; 5908 } 5909 5910 if (flags & BTRFS_INODE_NODATACOW) { 5911 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW; 5912 if (S_ISREG(inode->i_mode)) 5913 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 5914 } 5915 5916 btrfs_sync_inode_flags_to_i_flags(inode); 5917 } 5918 5919 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, 5920 struct btrfs_root *root, 5921 struct inode *dir, 5922 const char *name, int name_len, 5923 u64 ref_objectid, u64 objectid, 5924 umode_t mode, u64 *index) 5925 { 5926 struct btrfs_fs_info *fs_info = root->fs_info; 5927 struct inode *inode; 5928 struct btrfs_inode_item *inode_item; 5929 struct btrfs_key *location; 5930 struct btrfs_path *path; 5931 struct btrfs_inode_ref *ref; 5932 struct btrfs_key key[2]; 5933 u32 sizes[2]; 5934 int nitems = name ? 2 : 1; 5935 unsigned long ptr; 5936 unsigned int nofs_flag; 5937 int ret; 5938 5939 path = btrfs_alloc_path(); 5940 if (!path) 5941 return ERR_PTR(-ENOMEM); 5942 5943 nofs_flag = memalloc_nofs_save(); 5944 inode = new_inode(fs_info->sb); 5945 memalloc_nofs_restore(nofs_flag); 5946 if (!inode) { 5947 btrfs_free_path(path); 5948 return ERR_PTR(-ENOMEM); 5949 } 5950 5951 /* 5952 * O_TMPFILE, set link count to 0, so that after this point, 5953 * we fill in an inode item with the correct link count. 5954 */ 5955 if (!name) 5956 set_nlink(inode, 0); 5957 5958 /* 5959 * we have to initialize this early, so we can reclaim the inode 5960 * number if we fail afterwards in this function. 5961 */ 5962 inode->i_ino = objectid; 5963 5964 if (dir && name) { 5965 trace_btrfs_inode_request(dir); 5966 5967 ret = btrfs_set_inode_index(BTRFS_I(dir), index); 5968 if (ret) { 5969 btrfs_free_path(path); 5970 iput(inode); 5971 return ERR_PTR(ret); 5972 } 5973 } else if (dir) { 5974 *index = 0; 5975 } 5976 /* 5977 * index_cnt is ignored for everything but a dir, 5978 * btrfs_set_inode_index_count has an explanation for the magic 5979 * number 5980 */ 5981 BTRFS_I(inode)->index_cnt = 2; 5982 BTRFS_I(inode)->dir_index = *index; 5983 BTRFS_I(inode)->root = btrfs_grab_root(root); 5984 BTRFS_I(inode)->generation = trans->transid; 5985 inode->i_generation = BTRFS_I(inode)->generation; 5986 5987 /* 5988 * We could have gotten an inode number from somebody who was fsynced 5989 * and then removed in this same transaction, so let's just set full 5990 * sync since it will be a full sync anyway and this will blow away the 5991 * old info in the log. 5992 */ 5993 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 5994 5995 key[0].objectid = objectid; 5996 key[0].type = BTRFS_INODE_ITEM_KEY; 5997 key[0].offset = 0; 5998 5999 sizes[0] = sizeof(struct btrfs_inode_item); 6000 6001 if (name) { 6002 /* 6003 * Start new inodes with an inode_ref. This is slightly more 6004 * efficient for small numbers of hard links since they will 6005 * be packed into one item. Extended refs will kick in if we 6006 * add more hard links than can fit in the ref item. 6007 */ 6008 key[1].objectid = objectid; 6009 key[1].type = BTRFS_INODE_REF_KEY; 6010 key[1].offset = ref_objectid; 6011 6012 sizes[1] = name_len + sizeof(*ref); 6013 } 6014 6015 location = &BTRFS_I(inode)->location; 6016 location->objectid = objectid; 6017 location->offset = 0; 6018 location->type = BTRFS_INODE_ITEM_KEY; 6019 6020 ret = btrfs_insert_inode_locked(inode); 6021 if (ret < 0) { 6022 iput(inode); 6023 goto fail; 6024 } 6025 6026 path->leave_spinning = 1; 6027 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems); 6028 if (ret != 0) 6029 goto fail_unlock; 6030 6031 inode_init_owner(inode, dir, mode); 6032 inode_set_bytes(inode, 0); 6033 6034 inode->i_mtime = current_time(inode); 6035 inode->i_atime = inode->i_mtime; 6036 inode->i_ctime = inode->i_mtime; 6037 BTRFS_I(inode)->i_otime = inode->i_mtime; 6038 6039 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6040 struct btrfs_inode_item); 6041 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6042 sizeof(*inode_item)); 6043 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6044 6045 if (name) { 6046 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6047 struct btrfs_inode_ref); 6048 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len); 6049 btrfs_set_inode_ref_index(path->nodes[0], ref, *index); 6050 ptr = (unsigned long)(ref + 1); 6051 write_extent_buffer(path->nodes[0], name, ptr, name_len); 6052 } 6053 6054 btrfs_mark_buffer_dirty(path->nodes[0]); 6055 btrfs_free_path(path); 6056 6057 btrfs_inherit_iflags(inode, dir); 6058 6059 if (S_ISREG(mode)) { 6060 if (btrfs_test_opt(fs_info, NODATASUM)) 6061 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6062 if (btrfs_test_opt(fs_info, NODATACOW)) 6063 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6064 BTRFS_INODE_NODATASUM; 6065 } 6066 6067 inode_tree_add(inode); 6068 6069 trace_btrfs_inode_new(inode); 6070 btrfs_set_inode_last_trans(trans, BTRFS_I(inode)); 6071 6072 btrfs_update_root_times(trans, root); 6073 6074 ret = btrfs_inode_inherit_props(trans, inode, dir); 6075 if (ret) 6076 btrfs_err(fs_info, 6077 "error inheriting props for ino %llu (root %llu): %d", 6078 btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret); 6079 6080 return inode; 6081 6082 fail_unlock: 6083 discard_new_inode(inode); 6084 fail: 6085 if (dir && name) 6086 BTRFS_I(dir)->index_cnt--; 6087 btrfs_free_path(path); 6088 return ERR_PTR(ret); 6089 } 6090 6091 /* 6092 * utility function to add 'inode' into 'parent_inode' with 6093 * a give name and a given sequence number. 6094 * if 'add_backref' is true, also insert a backref from the 6095 * inode to the parent directory. 6096 */ 6097 int btrfs_add_link(struct btrfs_trans_handle *trans, 6098 struct btrfs_inode *parent_inode, struct btrfs_inode *inode, 6099 const char *name, int name_len, int add_backref, u64 index) 6100 { 6101 int ret = 0; 6102 struct btrfs_key key; 6103 struct btrfs_root *root = parent_inode->root; 6104 u64 ino = btrfs_ino(inode); 6105 u64 parent_ino = btrfs_ino(parent_inode); 6106 6107 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6108 memcpy(&key, &inode->root->root_key, sizeof(key)); 6109 } else { 6110 key.objectid = ino; 6111 key.type = BTRFS_INODE_ITEM_KEY; 6112 key.offset = 0; 6113 } 6114 6115 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6116 ret = btrfs_add_root_ref(trans, key.objectid, 6117 root->root_key.objectid, parent_ino, 6118 index, name, name_len); 6119 } else if (add_backref) { 6120 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino, 6121 parent_ino, index); 6122 } 6123 6124 /* Nothing to clean up yet */ 6125 if (ret) 6126 return ret; 6127 6128 ret = btrfs_insert_dir_item(trans, name, name_len, parent_inode, &key, 6129 btrfs_inode_type(&inode->vfs_inode), index); 6130 if (ret == -EEXIST || ret == -EOVERFLOW) 6131 goto fail_dir_item; 6132 else if (ret) { 6133 btrfs_abort_transaction(trans, ret); 6134 return ret; 6135 } 6136 6137 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + 6138 name_len * 2); 6139 inode_inc_iversion(&parent_inode->vfs_inode); 6140 /* 6141 * If we are replaying a log tree, we do not want to update the mtime 6142 * and ctime of the parent directory with the current time, since the 6143 * log replay procedure is responsible for setting them to their correct 6144 * values (the ones it had when the fsync was done). 6145 */ 6146 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags)) { 6147 struct timespec64 now = current_time(&parent_inode->vfs_inode); 6148 6149 parent_inode->vfs_inode.i_mtime = now; 6150 parent_inode->vfs_inode.i_ctime = now; 6151 } 6152 ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode); 6153 if (ret) 6154 btrfs_abort_transaction(trans, ret); 6155 return ret; 6156 6157 fail_dir_item: 6158 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6159 u64 local_index; 6160 int err; 6161 err = btrfs_del_root_ref(trans, key.objectid, 6162 root->root_key.objectid, parent_ino, 6163 &local_index, name, name_len); 6164 if (err) 6165 btrfs_abort_transaction(trans, err); 6166 } else if (add_backref) { 6167 u64 local_index; 6168 int err; 6169 6170 err = btrfs_del_inode_ref(trans, root, name, name_len, 6171 ino, parent_ino, &local_index); 6172 if (err) 6173 btrfs_abort_transaction(trans, err); 6174 } 6175 6176 /* Return the original error code */ 6177 return ret; 6178 } 6179 6180 static int btrfs_add_nondir(struct btrfs_trans_handle *trans, 6181 struct btrfs_inode *dir, struct dentry *dentry, 6182 struct btrfs_inode *inode, int backref, u64 index) 6183 { 6184 int err = btrfs_add_link(trans, dir, inode, 6185 dentry->d_name.name, dentry->d_name.len, 6186 backref, index); 6187 if (err > 0) 6188 err = -EEXIST; 6189 return err; 6190 } 6191 6192 static int btrfs_mknod(struct inode *dir, struct dentry *dentry, 6193 umode_t mode, dev_t rdev) 6194 { 6195 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6196 struct btrfs_trans_handle *trans; 6197 struct btrfs_root *root = BTRFS_I(dir)->root; 6198 struct inode *inode = NULL; 6199 int err; 6200 u64 objectid; 6201 u64 index = 0; 6202 6203 /* 6204 * 2 for inode item and ref 6205 * 2 for dir items 6206 * 1 for xattr if selinux is on 6207 */ 6208 trans = btrfs_start_transaction(root, 5); 6209 if (IS_ERR(trans)) 6210 return PTR_ERR(trans); 6211 6212 err = btrfs_find_free_ino(root, &objectid); 6213 if (err) 6214 goto out_unlock; 6215 6216 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6217 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6218 mode, &index); 6219 if (IS_ERR(inode)) { 6220 err = PTR_ERR(inode); 6221 inode = NULL; 6222 goto out_unlock; 6223 } 6224 6225 /* 6226 * If the active LSM wants to access the inode during 6227 * d_instantiate it needs these. Smack checks to see 6228 * if the filesystem supports xattrs by looking at the 6229 * ops vector. 6230 */ 6231 inode->i_op = &btrfs_special_inode_operations; 6232 init_special_inode(inode, inode->i_mode, rdev); 6233 6234 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6235 if (err) 6236 goto out_unlock; 6237 6238 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6239 0, index); 6240 if (err) 6241 goto out_unlock; 6242 6243 btrfs_update_inode(trans, root, inode); 6244 d_instantiate_new(dentry, inode); 6245 6246 out_unlock: 6247 btrfs_end_transaction(trans); 6248 btrfs_btree_balance_dirty(fs_info); 6249 if (err && inode) { 6250 inode_dec_link_count(inode); 6251 discard_new_inode(inode); 6252 } 6253 return err; 6254 } 6255 6256 static int btrfs_create(struct inode *dir, struct dentry *dentry, 6257 umode_t mode, bool excl) 6258 { 6259 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6260 struct btrfs_trans_handle *trans; 6261 struct btrfs_root *root = BTRFS_I(dir)->root; 6262 struct inode *inode = NULL; 6263 int err; 6264 u64 objectid; 6265 u64 index = 0; 6266 6267 /* 6268 * 2 for inode item and ref 6269 * 2 for dir items 6270 * 1 for xattr if selinux is on 6271 */ 6272 trans = btrfs_start_transaction(root, 5); 6273 if (IS_ERR(trans)) 6274 return PTR_ERR(trans); 6275 6276 err = btrfs_find_free_ino(root, &objectid); 6277 if (err) 6278 goto out_unlock; 6279 6280 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6281 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6282 mode, &index); 6283 if (IS_ERR(inode)) { 6284 err = PTR_ERR(inode); 6285 inode = NULL; 6286 goto out_unlock; 6287 } 6288 /* 6289 * If the active LSM wants to access the inode during 6290 * d_instantiate it needs these. Smack checks to see 6291 * if the filesystem supports xattrs by looking at the 6292 * ops vector. 6293 */ 6294 inode->i_fop = &btrfs_file_operations; 6295 inode->i_op = &btrfs_file_inode_operations; 6296 inode->i_mapping->a_ops = &btrfs_aops; 6297 6298 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6299 if (err) 6300 goto out_unlock; 6301 6302 err = btrfs_update_inode(trans, root, inode); 6303 if (err) 6304 goto out_unlock; 6305 6306 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6307 0, index); 6308 if (err) 6309 goto out_unlock; 6310 6311 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 6312 d_instantiate_new(dentry, inode); 6313 6314 out_unlock: 6315 btrfs_end_transaction(trans); 6316 if (err && inode) { 6317 inode_dec_link_count(inode); 6318 discard_new_inode(inode); 6319 } 6320 btrfs_btree_balance_dirty(fs_info); 6321 return err; 6322 } 6323 6324 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6325 struct dentry *dentry) 6326 { 6327 struct btrfs_trans_handle *trans = NULL; 6328 struct btrfs_root *root = BTRFS_I(dir)->root; 6329 struct inode *inode = d_inode(old_dentry); 6330 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6331 u64 index; 6332 int err; 6333 int drop_inode = 0; 6334 6335 /* do not allow sys_link's with other subvols of the same device */ 6336 if (root->root_key.objectid != BTRFS_I(inode)->root->root_key.objectid) 6337 return -EXDEV; 6338 6339 if (inode->i_nlink >= BTRFS_LINK_MAX) 6340 return -EMLINK; 6341 6342 err = btrfs_set_inode_index(BTRFS_I(dir), &index); 6343 if (err) 6344 goto fail; 6345 6346 /* 6347 * 2 items for inode and inode ref 6348 * 2 items for dir items 6349 * 1 item for parent inode 6350 * 1 item for orphan item deletion if O_TMPFILE 6351 */ 6352 trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6); 6353 if (IS_ERR(trans)) { 6354 err = PTR_ERR(trans); 6355 trans = NULL; 6356 goto fail; 6357 } 6358 6359 /* There are several dir indexes for this inode, clear the cache. */ 6360 BTRFS_I(inode)->dir_index = 0ULL; 6361 inc_nlink(inode); 6362 inode_inc_iversion(inode); 6363 inode->i_ctime = current_time(inode); 6364 ihold(inode); 6365 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6366 6367 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6368 1, index); 6369 6370 if (err) { 6371 drop_inode = 1; 6372 } else { 6373 struct dentry *parent = dentry->d_parent; 6374 int ret; 6375 6376 err = btrfs_update_inode(trans, root, inode); 6377 if (err) 6378 goto fail; 6379 if (inode->i_nlink == 1) { 6380 /* 6381 * If new hard link count is 1, it's a file created 6382 * with open(2) O_TMPFILE flag. 6383 */ 6384 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 6385 if (err) 6386 goto fail; 6387 } 6388 d_instantiate(dentry, inode); 6389 ret = btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent, 6390 true, NULL); 6391 if (ret == BTRFS_NEED_TRANS_COMMIT) { 6392 err = btrfs_commit_transaction(trans); 6393 trans = NULL; 6394 } 6395 } 6396 6397 fail: 6398 if (trans) 6399 btrfs_end_transaction(trans); 6400 if (drop_inode) { 6401 inode_dec_link_count(inode); 6402 iput(inode); 6403 } 6404 btrfs_btree_balance_dirty(fs_info); 6405 return err; 6406 } 6407 6408 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 6409 { 6410 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6411 struct inode *inode = NULL; 6412 struct btrfs_trans_handle *trans; 6413 struct btrfs_root *root = BTRFS_I(dir)->root; 6414 int err = 0; 6415 u64 objectid = 0; 6416 u64 index = 0; 6417 6418 /* 6419 * 2 items for inode and ref 6420 * 2 items for dir items 6421 * 1 for xattr if selinux is on 6422 */ 6423 trans = btrfs_start_transaction(root, 5); 6424 if (IS_ERR(trans)) 6425 return PTR_ERR(trans); 6426 6427 err = btrfs_find_free_ino(root, &objectid); 6428 if (err) 6429 goto out_fail; 6430 6431 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6432 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6433 S_IFDIR | mode, &index); 6434 if (IS_ERR(inode)) { 6435 err = PTR_ERR(inode); 6436 inode = NULL; 6437 goto out_fail; 6438 } 6439 6440 /* these must be set before we unlock the inode */ 6441 inode->i_op = &btrfs_dir_inode_operations; 6442 inode->i_fop = &btrfs_dir_file_operations; 6443 6444 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6445 if (err) 6446 goto out_fail; 6447 6448 btrfs_i_size_write(BTRFS_I(inode), 0); 6449 err = btrfs_update_inode(trans, root, inode); 6450 if (err) 6451 goto out_fail; 6452 6453 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), 6454 dentry->d_name.name, 6455 dentry->d_name.len, 0, index); 6456 if (err) 6457 goto out_fail; 6458 6459 d_instantiate_new(dentry, inode); 6460 6461 out_fail: 6462 btrfs_end_transaction(trans); 6463 if (err && inode) { 6464 inode_dec_link_count(inode); 6465 discard_new_inode(inode); 6466 } 6467 btrfs_btree_balance_dirty(fs_info); 6468 return err; 6469 } 6470 6471 static noinline int uncompress_inline(struct btrfs_path *path, 6472 struct page *page, 6473 size_t pg_offset, u64 extent_offset, 6474 struct btrfs_file_extent_item *item) 6475 { 6476 int ret; 6477 struct extent_buffer *leaf = path->nodes[0]; 6478 char *tmp; 6479 size_t max_size; 6480 unsigned long inline_size; 6481 unsigned long ptr; 6482 int compress_type; 6483 6484 WARN_ON(pg_offset != 0); 6485 compress_type = btrfs_file_extent_compression(leaf, item); 6486 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6487 inline_size = btrfs_file_extent_inline_item_len(leaf, 6488 btrfs_item_nr(path->slots[0])); 6489 tmp = kmalloc(inline_size, GFP_NOFS); 6490 if (!tmp) 6491 return -ENOMEM; 6492 ptr = btrfs_file_extent_inline_start(item); 6493 6494 read_extent_buffer(leaf, tmp, ptr, inline_size); 6495 6496 max_size = min_t(unsigned long, PAGE_SIZE, max_size); 6497 ret = btrfs_decompress(compress_type, tmp, page, 6498 extent_offset, inline_size, max_size); 6499 6500 /* 6501 * decompression code contains a memset to fill in any space between the end 6502 * of the uncompressed data and the end of max_size in case the decompressed 6503 * data ends up shorter than ram_bytes. That doesn't cover the hole between 6504 * the end of an inline extent and the beginning of the next block, so we 6505 * cover that region here. 6506 */ 6507 6508 if (max_size + pg_offset < PAGE_SIZE) { 6509 char *map = kmap(page); 6510 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset); 6511 kunmap(page); 6512 } 6513 kfree(tmp); 6514 return ret; 6515 } 6516 6517 /** 6518 * btrfs_get_extent - Lookup the first extent overlapping a range in a file. 6519 * @inode: file to search in 6520 * @page: page to read extent data into if the extent is inline 6521 * @pg_offset: offset into @page to copy to 6522 * @start: file offset 6523 * @len: length of range starting at @start 6524 * 6525 * This returns the first &struct extent_map which overlaps with the given 6526 * range, reading it from the B-tree and caching it if necessary. Note that 6527 * there may be more extents which overlap the given range after the returned 6528 * extent_map. 6529 * 6530 * If @page is not NULL and the extent is inline, this also reads the extent 6531 * data directly into the page and marks the extent up to date in the io_tree. 6532 * 6533 * Return: ERR_PTR on error, non-NULL extent_map on success. 6534 */ 6535 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, 6536 struct page *page, size_t pg_offset, 6537 u64 start, u64 len) 6538 { 6539 struct btrfs_fs_info *fs_info = inode->root->fs_info; 6540 int ret; 6541 int err = 0; 6542 u64 extent_start = 0; 6543 u64 extent_end = 0; 6544 u64 objectid = btrfs_ino(inode); 6545 int extent_type = -1; 6546 struct btrfs_path *path = NULL; 6547 struct btrfs_root *root = inode->root; 6548 struct btrfs_file_extent_item *item; 6549 struct extent_buffer *leaf; 6550 struct btrfs_key found_key; 6551 struct extent_map *em = NULL; 6552 struct extent_map_tree *em_tree = &inode->extent_tree; 6553 struct extent_io_tree *io_tree = &inode->io_tree; 6554 6555 read_lock(&em_tree->lock); 6556 em = lookup_extent_mapping(em_tree, start, len); 6557 read_unlock(&em_tree->lock); 6558 6559 if (em) { 6560 if (em->start > start || em->start + em->len <= start) 6561 free_extent_map(em); 6562 else if (em->block_start == EXTENT_MAP_INLINE && page) 6563 free_extent_map(em); 6564 else 6565 goto out; 6566 } 6567 em = alloc_extent_map(); 6568 if (!em) { 6569 err = -ENOMEM; 6570 goto out; 6571 } 6572 em->start = EXTENT_MAP_HOLE; 6573 em->orig_start = EXTENT_MAP_HOLE; 6574 em->len = (u64)-1; 6575 em->block_len = (u64)-1; 6576 6577 path = btrfs_alloc_path(); 6578 if (!path) { 6579 err = -ENOMEM; 6580 goto out; 6581 } 6582 6583 /* Chances are we'll be called again, so go ahead and do readahead */ 6584 path->reada = READA_FORWARD; 6585 6586 /* 6587 * Unless we're going to uncompress the inline extent, no sleep would 6588 * happen. 6589 */ 6590 path->leave_spinning = 1; 6591 6592 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); 6593 if (ret < 0) { 6594 err = ret; 6595 goto out; 6596 } else if (ret > 0) { 6597 if (path->slots[0] == 0) 6598 goto not_found; 6599 path->slots[0]--; 6600 } 6601 6602 leaf = path->nodes[0]; 6603 item = btrfs_item_ptr(leaf, path->slots[0], 6604 struct btrfs_file_extent_item); 6605 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6606 if (found_key.objectid != objectid || 6607 found_key.type != BTRFS_EXTENT_DATA_KEY) { 6608 /* 6609 * If we backup past the first extent we want to move forward 6610 * and see if there is an extent in front of us, otherwise we'll 6611 * say there is a hole for our whole search range which can 6612 * cause problems. 6613 */ 6614 extent_end = start; 6615 goto next; 6616 } 6617 6618 extent_type = btrfs_file_extent_type(leaf, item); 6619 extent_start = found_key.offset; 6620 extent_end = btrfs_file_extent_end(path); 6621 if (extent_type == BTRFS_FILE_EXTENT_REG || 6622 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 6623 /* Only regular file could have regular/prealloc extent */ 6624 if (!S_ISREG(inode->vfs_inode.i_mode)) { 6625 ret = -EUCLEAN; 6626 btrfs_crit(fs_info, 6627 "regular/prealloc extent found for non-regular inode %llu", 6628 btrfs_ino(inode)); 6629 goto out; 6630 } 6631 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, 6632 extent_start); 6633 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 6634 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, 6635 path->slots[0], 6636 extent_start); 6637 } 6638 next: 6639 if (start >= extent_end) { 6640 path->slots[0]++; 6641 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 6642 ret = btrfs_next_leaf(root, path); 6643 if (ret < 0) { 6644 err = ret; 6645 goto out; 6646 } else if (ret > 0) { 6647 goto not_found; 6648 } 6649 leaf = path->nodes[0]; 6650 } 6651 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6652 if (found_key.objectid != objectid || 6653 found_key.type != BTRFS_EXTENT_DATA_KEY) 6654 goto not_found; 6655 if (start + len <= found_key.offset) 6656 goto not_found; 6657 if (start > found_key.offset) 6658 goto next; 6659 6660 /* New extent overlaps with existing one */ 6661 em->start = start; 6662 em->orig_start = start; 6663 em->len = found_key.offset - start; 6664 em->block_start = EXTENT_MAP_HOLE; 6665 goto insert; 6666 } 6667 6668 btrfs_extent_item_to_extent_map(inode, path, item, !page, em); 6669 6670 if (extent_type == BTRFS_FILE_EXTENT_REG || 6671 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 6672 goto insert; 6673 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 6674 unsigned long ptr; 6675 char *map; 6676 size_t size; 6677 size_t extent_offset; 6678 size_t copy_size; 6679 6680 if (!page) 6681 goto out; 6682 6683 size = btrfs_file_extent_ram_bytes(leaf, item); 6684 extent_offset = page_offset(page) + pg_offset - extent_start; 6685 copy_size = min_t(u64, PAGE_SIZE - pg_offset, 6686 size - extent_offset); 6687 em->start = extent_start + extent_offset; 6688 em->len = ALIGN(copy_size, fs_info->sectorsize); 6689 em->orig_block_len = em->len; 6690 em->orig_start = em->start; 6691 ptr = btrfs_file_extent_inline_start(item) + extent_offset; 6692 6693 btrfs_set_path_blocking(path); 6694 if (!PageUptodate(page)) { 6695 if (btrfs_file_extent_compression(leaf, item) != 6696 BTRFS_COMPRESS_NONE) { 6697 ret = uncompress_inline(path, page, pg_offset, 6698 extent_offset, item); 6699 if (ret) { 6700 err = ret; 6701 goto out; 6702 } 6703 } else { 6704 map = kmap(page); 6705 read_extent_buffer(leaf, map + pg_offset, ptr, 6706 copy_size); 6707 if (pg_offset + copy_size < PAGE_SIZE) { 6708 memset(map + pg_offset + copy_size, 0, 6709 PAGE_SIZE - pg_offset - 6710 copy_size); 6711 } 6712 kunmap(page); 6713 } 6714 flush_dcache_page(page); 6715 } 6716 set_extent_uptodate(io_tree, em->start, 6717 extent_map_end(em) - 1, NULL, GFP_NOFS); 6718 goto insert; 6719 } 6720 not_found: 6721 em->start = start; 6722 em->orig_start = start; 6723 em->len = len; 6724 em->block_start = EXTENT_MAP_HOLE; 6725 insert: 6726 btrfs_release_path(path); 6727 if (em->start > start || extent_map_end(em) <= start) { 6728 btrfs_err(fs_info, 6729 "bad extent! em: [%llu %llu] passed [%llu %llu]", 6730 em->start, em->len, start, len); 6731 err = -EIO; 6732 goto out; 6733 } 6734 6735 err = 0; 6736 write_lock(&em_tree->lock); 6737 err = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len); 6738 write_unlock(&em_tree->lock); 6739 out: 6740 btrfs_free_path(path); 6741 6742 trace_btrfs_get_extent(root, inode, em); 6743 6744 if (err) { 6745 free_extent_map(em); 6746 return ERR_PTR(err); 6747 } 6748 BUG_ON(!em); /* Error is always set */ 6749 return em; 6750 } 6751 6752 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, 6753 u64 start, u64 len) 6754 { 6755 struct extent_map *em; 6756 struct extent_map *hole_em = NULL; 6757 u64 delalloc_start = start; 6758 u64 end; 6759 u64 delalloc_len; 6760 u64 delalloc_end; 6761 int err = 0; 6762 6763 em = btrfs_get_extent(inode, NULL, 0, start, len); 6764 if (IS_ERR(em)) 6765 return em; 6766 /* 6767 * If our em maps to: 6768 * - a hole or 6769 * - a pre-alloc extent, 6770 * there might actually be delalloc bytes behind it. 6771 */ 6772 if (em->block_start != EXTENT_MAP_HOLE && 6773 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 6774 return em; 6775 else 6776 hole_em = em; 6777 6778 /* check to see if we've wrapped (len == -1 or similar) */ 6779 end = start + len; 6780 if (end < start) 6781 end = (u64)-1; 6782 else 6783 end -= 1; 6784 6785 em = NULL; 6786 6787 /* ok, we didn't find anything, lets look for delalloc */ 6788 delalloc_len = count_range_bits(&inode->io_tree, &delalloc_start, 6789 end, len, EXTENT_DELALLOC, 1); 6790 delalloc_end = delalloc_start + delalloc_len; 6791 if (delalloc_end < delalloc_start) 6792 delalloc_end = (u64)-1; 6793 6794 /* 6795 * We didn't find anything useful, return the original results from 6796 * get_extent() 6797 */ 6798 if (delalloc_start > end || delalloc_end <= start) { 6799 em = hole_em; 6800 hole_em = NULL; 6801 goto out; 6802 } 6803 6804 /* 6805 * Adjust the delalloc_start to make sure it doesn't go backwards from 6806 * the start they passed in 6807 */ 6808 delalloc_start = max(start, delalloc_start); 6809 delalloc_len = delalloc_end - delalloc_start; 6810 6811 if (delalloc_len > 0) { 6812 u64 hole_start; 6813 u64 hole_len; 6814 const u64 hole_end = extent_map_end(hole_em); 6815 6816 em = alloc_extent_map(); 6817 if (!em) { 6818 err = -ENOMEM; 6819 goto out; 6820 } 6821 6822 ASSERT(hole_em); 6823 /* 6824 * When btrfs_get_extent can't find anything it returns one 6825 * huge hole 6826 * 6827 * Make sure what it found really fits our range, and adjust to 6828 * make sure it is based on the start from the caller 6829 */ 6830 if (hole_end <= start || hole_em->start > end) { 6831 free_extent_map(hole_em); 6832 hole_em = NULL; 6833 } else { 6834 hole_start = max(hole_em->start, start); 6835 hole_len = hole_end - hole_start; 6836 } 6837 6838 if (hole_em && delalloc_start > hole_start) { 6839 /* 6840 * Our hole starts before our delalloc, so we have to 6841 * return just the parts of the hole that go until the 6842 * delalloc starts 6843 */ 6844 em->len = min(hole_len, delalloc_start - hole_start); 6845 em->start = hole_start; 6846 em->orig_start = hole_start; 6847 /* 6848 * Don't adjust block start at all, it is fixed at 6849 * EXTENT_MAP_HOLE 6850 */ 6851 em->block_start = hole_em->block_start; 6852 em->block_len = hole_len; 6853 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) 6854 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 6855 } else { 6856 /* 6857 * Hole is out of passed range or it starts after 6858 * delalloc range 6859 */ 6860 em->start = delalloc_start; 6861 em->len = delalloc_len; 6862 em->orig_start = delalloc_start; 6863 em->block_start = EXTENT_MAP_DELALLOC; 6864 em->block_len = delalloc_len; 6865 } 6866 } else { 6867 return hole_em; 6868 } 6869 out: 6870 6871 free_extent_map(hole_em); 6872 if (err) { 6873 free_extent_map(em); 6874 return ERR_PTR(err); 6875 } 6876 return em; 6877 } 6878 6879 static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, 6880 const u64 start, 6881 const u64 len, 6882 const u64 orig_start, 6883 const u64 block_start, 6884 const u64 block_len, 6885 const u64 orig_block_len, 6886 const u64 ram_bytes, 6887 const int type) 6888 { 6889 struct extent_map *em = NULL; 6890 int ret; 6891 6892 if (type != BTRFS_ORDERED_NOCOW) { 6893 em = create_io_em(inode, start, len, orig_start, block_start, 6894 block_len, orig_block_len, ram_bytes, 6895 BTRFS_COMPRESS_NONE, /* compress_type */ 6896 type); 6897 if (IS_ERR(em)) 6898 goto out; 6899 } 6900 ret = btrfs_add_ordered_extent_dio(inode, start, block_start, len, 6901 block_len, type); 6902 if (ret) { 6903 if (em) { 6904 free_extent_map(em); 6905 btrfs_drop_extent_cache(inode, start, start + len - 1, 0); 6906 } 6907 em = ERR_PTR(ret); 6908 } 6909 out: 6910 6911 return em; 6912 } 6913 6914 static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode, 6915 u64 start, u64 len) 6916 { 6917 struct btrfs_root *root = inode->root; 6918 struct btrfs_fs_info *fs_info = root->fs_info; 6919 struct extent_map *em; 6920 struct btrfs_key ins; 6921 u64 alloc_hint; 6922 int ret; 6923 6924 alloc_hint = get_extent_allocation_hint(inode, start, len); 6925 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, 6926 0, alloc_hint, &ins, 1, 1); 6927 if (ret) 6928 return ERR_PTR(ret); 6929 6930 em = btrfs_create_dio_extent(inode, start, ins.offset, start, 6931 ins.objectid, ins.offset, ins.offset, 6932 ins.offset, BTRFS_ORDERED_REGULAR); 6933 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 6934 if (IS_ERR(em)) 6935 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 6936 1); 6937 6938 return em; 6939 } 6940 6941 /* 6942 * Check if we can do nocow write into the range [@offset, @offset + @len) 6943 * 6944 * @offset: File offset 6945 * @len: The length to write, will be updated to the nocow writeable 6946 * range 6947 * @orig_start: (optional) Return the original file offset of the file extent 6948 * @orig_len: (optional) Return the original on-disk length of the file extent 6949 * @ram_bytes: (optional) Return the ram_bytes of the file extent 6950 * 6951 * This function will flush ordered extents in the range to ensure proper 6952 * nocow checks for (nowait == false) case. 6953 * 6954 * Return: 6955 * >0 and update @len if we can do nocow write 6956 * 0 if we can't do nocow write 6957 * <0 if error happened 6958 * 6959 * NOTE: This only checks the file extents, caller is responsible to wait for 6960 * any ordered extents. 6961 */ 6962 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 6963 u64 *orig_start, u64 *orig_block_len, 6964 u64 *ram_bytes) 6965 { 6966 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6967 struct btrfs_path *path; 6968 int ret; 6969 struct extent_buffer *leaf; 6970 struct btrfs_root *root = BTRFS_I(inode)->root; 6971 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 6972 struct btrfs_file_extent_item *fi; 6973 struct btrfs_key key; 6974 u64 disk_bytenr; 6975 u64 backref_offset; 6976 u64 extent_end; 6977 u64 num_bytes; 6978 int slot; 6979 int found_type; 6980 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW); 6981 6982 path = btrfs_alloc_path(); 6983 if (!path) 6984 return -ENOMEM; 6985 6986 ret = btrfs_lookup_file_extent(NULL, root, path, 6987 btrfs_ino(BTRFS_I(inode)), offset, 0); 6988 if (ret < 0) 6989 goto out; 6990 6991 slot = path->slots[0]; 6992 if (ret == 1) { 6993 if (slot == 0) { 6994 /* can't find the item, must cow */ 6995 ret = 0; 6996 goto out; 6997 } 6998 slot--; 6999 } 7000 ret = 0; 7001 leaf = path->nodes[0]; 7002 btrfs_item_key_to_cpu(leaf, &key, slot); 7003 if (key.objectid != btrfs_ino(BTRFS_I(inode)) || 7004 key.type != BTRFS_EXTENT_DATA_KEY) { 7005 /* not our file or wrong item type, must cow */ 7006 goto out; 7007 } 7008 7009 if (key.offset > offset) { 7010 /* Wrong offset, must cow */ 7011 goto out; 7012 } 7013 7014 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 7015 found_type = btrfs_file_extent_type(leaf, fi); 7016 if (found_type != BTRFS_FILE_EXTENT_REG && 7017 found_type != BTRFS_FILE_EXTENT_PREALLOC) { 7018 /* not a regular extent, must cow */ 7019 goto out; 7020 } 7021 7022 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG) 7023 goto out; 7024 7025 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 7026 if (extent_end <= offset) 7027 goto out; 7028 7029 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 7030 if (disk_bytenr == 0) 7031 goto out; 7032 7033 if (btrfs_file_extent_compression(leaf, fi) || 7034 btrfs_file_extent_encryption(leaf, fi) || 7035 btrfs_file_extent_other_encoding(leaf, fi)) 7036 goto out; 7037 7038 /* 7039 * Do the same check as in btrfs_cross_ref_exist but without the 7040 * unnecessary search. 7041 */ 7042 if (btrfs_file_extent_generation(leaf, fi) <= 7043 btrfs_root_last_snapshot(&root->root_item)) 7044 goto out; 7045 7046 backref_offset = btrfs_file_extent_offset(leaf, fi); 7047 7048 if (orig_start) { 7049 *orig_start = key.offset - backref_offset; 7050 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 7051 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 7052 } 7053 7054 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 7055 goto out; 7056 7057 num_bytes = min(offset + *len, extent_end) - offset; 7058 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7059 u64 range_end; 7060 7061 range_end = round_up(offset + num_bytes, 7062 root->fs_info->sectorsize) - 1; 7063 ret = test_range_bit(io_tree, offset, range_end, 7064 EXTENT_DELALLOC, 0, NULL); 7065 if (ret) { 7066 ret = -EAGAIN; 7067 goto out; 7068 } 7069 } 7070 7071 btrfs_release_path(path); 7072 7073 /* 7074 * look for other files referencing this extent, if we 7075 * find any we must cow 7076 */ 7077 7078 ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)), 7079 key.offset - backref_offset, disk_bytenr); 7080 if (ret) { 7081 ret = 0; 7082 goto out; 7083 } 7084 7085 /* 7086 * adjust disk_bytenr and num_bytes to cover just the bytes 7087 * in this extent we are about to write. If there 7088 * are any csums in that range we have to cow in order 7089 * to keep the csums correct 7090 */ 7091 disk_bytenr += backref_offset; 7092 disk_bytenr += offset - key.offset; 7093 if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes)) 7094 goto out; 7095 /* 7096 * all of the above have passed, it is safe to overwrite this extent 7097 * without cow 7098 */ 7099 *len = num_bytes; 7100 ret = 1; 7101 out: 7102 btrfs_free_path(path); 7103 return ret; 7104 } 7105 7106 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7107 struct extent_state **cached_state, int writing) 7108 { 7109 struct btrfs_ordered_extent *ordered; 7110 int ret = 0; 7111 7112 while (1) { 7113 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7114 cached_state); 7115 /* 7116 * We're concerned with the entire range that we're going to be 7117 * doing DIO to, so we need to make sure there's no ordered 7118 * extents in this range. 7119 */ 7120 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart, 7121 lockend - lockstart + 1); 7122 7123 /* 7124 * We need to make sure there are no buffered pages in this 7125 * range either, we could have raced between the invalidate in 7126 * generic_file_direct_write and locking the extent. The 7127 * invalidate needs to happen so that reads after a write do not 7128 * get stale data. 7129 */ 7130 if (!ordered && 7131 (!writing || !filemap_range_has_page(inode->i_mapping, 7132 lockstart, lockend))) 7133 break; 7134 7135 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7136 cached_state); 7137 7138 if (ordered) { 7139 /* 7140 * If we are doing a DIO read and the ordered extent we 7141 * found is for a buffered write, we can not wait for it 7142 * to complete and retry, because if we do so we can 7143 * deadlock with concurrent buffered writes on page 7144 * locks. This happens only if our DIO read covers more 7145 * than one extent map, if at this point has already 7146 * created an ordered extent for a previous extent map 7147 * and locked its range in the inode's io tree, and a 7148 * concurrent write against that previous extent map's 7149 * range and this range started (we unlock the ranges 7150 * in the io tree only when the bios complete and 7151 * buffered writes always lock pages before attempting 7152 * to lock range in the io tree). 7153 */ 7154 if (writing || 7155 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) 7156 btrfs_start_ordered_extent(inode, ordered, 1); 7157 else 7158 ret = -ENOTBLK; 7159 btrfs_put_ordered_extent(ordered); 7160 } else { 7161 /* 7162 * We could trigger writeback for this range (and wait 7163 * for it to complete) and then invalidate the pages for 7164 * this range (through invalidate_inode_pages2_range()), 7165 * but that can lead us to a deadlock with a concurrent 7166 * call to readahead (a buffered read or a defrag call 7167 * triggered a readahead) on a page lock due to an 7168 * ordered dio extent we created before but did not have 7169 * yet a corresponding bio submitted (whence it can not 7170 * complete), which makes readahead wait for that 7171 * ordered extent to complete while holding a lock on 7172 * that page. 7173 */ 7174 ret = -ENOTBLK; 7175 } 7176 7177 if (ret) 7178 break; 7179 7180 cond_resched(); 7181 } 7182 7183 return ret; 7184 } 7185 7186 /* The callers of this must take lock_extent() */ 7187 static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, 7188 u64 len, u64 orig_start, u64 block_start, 7189 u64 block_len, u64 orig_block_len, 7190 u64 ram_bytes, int compress_type, 7191 int type) 7192 { 7193 struct extent_map_tree *em_tree; 7194 struct extent_map *em; 7195 int ret; 7196 7197 ASSERT(type == BTRFS_ORDERED_PREALLOC || 7198 type == BTRFS_ORDERED_COMPRESSED || 7199 type == BTRFS_ORDERED_NOCOW || 7200 type == BTRFS_ORDERED_REGULAR); 7201 7202 em_tree = &inode->extent_tree; 7203 em = alloc_extent_map(); 7204 if (!em) 7205 return ERR_PTR(-ENOMEM); 7206 7207 em->start = start; 7208 em->orig_start = orig_start; 7209 em->len = len; 7210 em->block_len = block_len; 7211 em->block_start = block_start; 7212 em->orig_block_len = orig_block_len; 7213 em->ram_bytes = ram_bytes; 7214 em->generation = -1; 7215 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7216 if (type == BTRFS_ORDERED_PREALLOC) { 7217 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7218 } else if (type == BTRFS_ORDERED_COMPRESSED) { 7219 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 7220 em->compress_type = compress_type; 7221 } 7222 7223 do { 7224 btrfs_drop_extent_cache(inode, em->start, 7225 em->start + em->len - 1, 0); 7226 write_lock(&em_tree->lock); 7227 ret = add_extent_mapping(em_tree, em, 1); 7228 write_unlock(&em_tree->lock); 7229 /* 7230 * The caller has taken lock_extent(), who could race with us 7231 * to add em? 7232 */ 7233 } while (ret == -EEXIST); 7234 7235 if (ret) { 7236 free_extent_map(em); 7237 return ERR_PTR(ret); 7238 } 7239 7240 /* em got 2 refs now, callers needs to do free_extent_map once. */ 7241 return em; 7242 } 7243 7244 7245 static int btrfs_get_blocks_direct_read(struct extent_map *em, 7246 struct buffer_head *bh_result, 7247 struct inode *inode, 7248 u64 start, u64 len) 7249 { 7250 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7251 7252 if (em->block_start == EXTENT_MAP_HOLE || 7253 test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7254 return -ENOENT; 7255 7256 len = min(len, em->len - (start - em->start)); 7257 7258 bh_result->b_blocknr = (em->block_start + (start - em->start)) >> 7259 inode->i_blkbits; 7260 bh_result->b_size = len; 7261 bh_result->b_bdev = fs_info->fs_devices->latest_bdev; 7262 set_buffer_mapped(bh_result); 7263 7264 return 0; 7265 } 7266 7267 static int btrfs_get_blocks_direct_write(struct extent_map **map, 7268 struct buffer_head *bh_result, 7269 struct inode *inode, 7270 struct btrfs_dio_data *dio_data, 7271 u64 start, u64 len) 7272 { 7273 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7274 struct extent_map *em = *map; 7275 int ret = 0; 7276 7277 /* 7278 * We don't allocate a new extent in the following cases 7279 * 7280 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7281 * existing extent. 7282 * 2) The extent is marked as PREALLOC. We're good to go here and can 7283 * just use the extent. 7284 * 7285 */ 7286 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7287 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7288 em->block_start != EXTENT_MAP_HOLE)) { 7289 int type; 7290 u64 block_start, orig_start, orig_block_len, ram_bytes; 7291 7292 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7293 type = BTRFS_ORDERED_PREALLOC; 7294 else 7295 type = BTRFS_ORDERED_NOCOW; 7296 len = min(len, em->len - (start - em->start)); 7297 block_start = em->block_start + (start - em->start); 7298 7299 if (can_nocow_extent(inode, start, &len, &orig_start, 7300 &orig_block_len, &ram_bytes) == 1 && 7301 btrfs_inc_nocow_writers(fs_info, block_start)) { 7302 struct extent_map *em2; 7303 7304 em2 = btrfs_create_dio_extent(BTRFS_I(inode), start, len, 7305 orig_start, block_start, 7306 len, orig_block_len, 7307 ram_bytes, type); 7308 btrfs_dec_nocow_writers(fs_info, block_start); 7309 if (type == BTRFS_ORDERED_PREALLOC) { 7310 free_extent_map(em); 7311 *map = em = em2; 7312 } 7313 7314 if (em2 && IS_ERR(em2)) { 7315 ret = PTR_ERR(em2); 7316 goto out; 7317 } 7318 /* 7319 * For inode marked NODATACOW or extent marked PREALLOC, 7320 * use the existing or preallocated extent, so does not 7321 * need to adjust btrfs_space_info's bytes_may_use. 7322 */ 7323 btrfs_free_reserved_data_space_noquota(fs_info, len); 7324 goto skip_cow; 7325 } 7326 } 7327 7328 /* this will cow the extent */ 7329 len = bh_result->b_size; 7330 free_extent_map(em); 7331 *map = em = btrfs_new_extent_direct(BTRFS_I(inode), start, len); 7332 if (IS_ERR(em)) { 7333 ret = PTR_ERR(em); 7334 goto out; 7335 } 7336 7337 len = min(len, em->len - (start - em->start)); 7338 7339 skip_cow: 7340 bh_result->b_blocknr = (em->block_start + (start - em->start)) >> 7341 inode->i_blkbits; 7342 bh_result->b_size = len; 7343 bh_result->b_bdev = fs_info->fs_devices->latest_bdev; 7344 set_buffer_mapped(bh_result); 7345 7346 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7347 set_buffer_new(bh_result); 7348 7349 /* 7350 * Need to update the i_size under the extent lock so buffered 7351 * readers will get the updated i_size when we unlock. 7352 */ 7353 if (!dio_data->overwrite && start + len > i_size_read(inode)) 7354 i_size_write(inode, start + len); 7355 7356 WARN_ON(dio_data->reserve < len); 7357 dio_data->reserve -= len; 7358 dio_data->unsubmitted_oe_range_end = start + len; 7359 current->journal_info = dio_data; 7360 out: 7361 return ret; 7362 } 7363 7364 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, 7365 struct buffer_head *bh_result, int create) 7366 { 7367 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7368 struct extent_map *em; 7369 struct extent_state *cached_state = NULL; 7370 struct btrfs_dio_data *dio_data = NULL; 7371 u64 start = iblock << inode->i_blkbits; 7372 u64 lockstart, lockend; 7373 u64 len = bh_result->b_size; 7374 int ret = 0; 7375 7376 if (!create) 7377 len = min_t(u64, len, fs_info->sectorsize); 7378 7379 lockstart = start; 7380 lockend = start + len - 1; 7381 7382 if (current->journal_info) { 7383 /* 7384 * Need to pull our outstanding extents and set journal_info to NULL so 7385 * that anything that needs to check if there's a transaction doesn't get 7386 * confused. 7387 */ 7388 dio_data = current->journal_info; 7389 current->journal_info = NULL; 7390 } 7391 7392 /* 7393 * If this errors out it's because we couldn't invalidate pagecache for 7394 * this range and we need to fallback to buffered. 7395 */ 7396 if (lock_extent_direct(inode, lockstart, lockend, &cached_state, 7397 create)) { 7398 ret = -ENOTBLK; 7399 goto err; 7400 } 7401 7402 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len); 7403 if (IS_ERR(em)) { 7404 ret = PTR_ERR(em); 7405 goto unlock_err; 7406 } 7407 7408 /* 7409 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7410 * io. INLINE is special, and we could probably kludge it in here, but 7411 * it's still buffered so for safety lets just fall back to the generic 7412 * buffered path. 7413 * 7414 * For COMPRESSED we _have_ to read the entire extent in so we can 7415 * decompress it, so there will be buffering required no matter what we 7416 * do, so go ahead and fallback to buffered. 7417 * 7418 * We return -ENOTBLK because that's what makes DIO go ahead and go back 7419 * to buffered IO. Don't blame me, this is the price we pay for using 7420 * the generic code. 7421 */ 7422 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7423 em->block_start == EXTENT_MAP_INLINE) { 7424 free_extent_map(em); 7425 ret = -ENOTBLK; 7426 goto unlock_err; 7427 } 7428 7429 if (create) { 7430 ret = btrfs_get_blocks_direct_write(&em, bh_result, inode, 7431 dio_data, start, len); 7432 if (ret < 0) 7433 goto unlock_err; 7434 7435 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, 7436 lockend, &cached_state); 7437 } else { 7438 ret = btrfs_get_blocks_direct_read(em, bh_result, inode, 7439 start, len); 7440 /* Can be negative only if we read from a hole */ 7441 if (ret < 0) { 7442 ret = 0; 7443 free_extent_map(em); 7444 goto unlock_err; 7445 } 7446 /* 7447 * We need to unlock only the end area that we aren't using. 7448 * The rest is going to be unlocked by the endio routine. 7449 */ 7450 lockstart = start + bh_result->b_size; 7451 if (lockstart < lockend) { 7452 unlock_extent_cached(&BTRFS_I(inode)->io_tree, 7453 lockstart, lockend, &cached_state); 7454 } else { 7455 free_extent_state(cached_state); 7456 } 7457 } 7458 7459 free_extent_map(em); 7460 7461 return 0; 7462 7463 unlock_err: 7464 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7465 &cached_state); 7466 err: 7467 if (dio_data) 7468 current->journal_info = dio_data; 7469 return ret; 7470 } 7471 7472 static void btrfs_dio_private_put(struct btrfs_dio_private *dip) 7473 { 7474 /* 7475 * This implies a barrier so that stores to dio_bio->bi_status before 7476 * this and loads of dio_bio->bi_status after this are fully ordered. 7477 */ 7478 if (!refcount_dec_and_test(&dip->refs)) 7479 return; 7480 7481 if (bio_op(dip->dio_bio) == REQ_OP_WRITE) { 7482 __endio_write_update_ordered(BTRFS_I(dip->inode), 7483 dip->logical_offset, 7484 dip->bytes, 7485 !dip->dio_bio->bi_status); 7486 } else { 7487 unlock_extent(&BTRFS_I(dip->inode)->io_tree, 7488 dip->logical_offset, 7489 dip->logical_offset + dip->bytes - 1); 7490 } 7491 7492 dio_end_io(dip->dio_bio); 7493 kfree(dip); 7494 } 7495 7496 static blk_status_t submit_dio_repair_bio(struct inode *inode, struct bio *bio, 7497 int mirror_num, 7498 unsigned long bio_flags) 7499 { 7500 struct btrfs_dio_private *dip = bio->bi_private; 7501 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7502 blk_status_t ret; 7503 7504 BUG_ON(bio_op(bio) == REQ_OP_WRITE); 7505 7506 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 7507 if (ret) 7508 return ret; 7509 7510 refcount_inc(&dip->refs); 7511 ret = btrfs_map_bio(fs_info, bio, mirror_num); 7512 if (ret) 7513 refcount_dec(&dip->refs); 7514 return ret; 7515 } 7516 7517 static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, 7518 struct btrfs_io_bio *io_bio, 7519 const bool uptodate) 7520 { 7521 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 7522 const u32 sectorsize = fs_info->sectorsize; 7523 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 7524 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7525 const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); 7526 struct bio_vec bvec; 7527 struct bvec_iter iter; 7528 u64 start = io_bio->logical; 7529 int icsum = 0; 7530 blk_status_t err = BLK_STS_OK; 7531 7532 __bio_for_each_segment(bvec, &io_bio->bio, iter, io_bio->iter) { 7533 unsigned int i, nr_sectors, pgoff; 7534 7535 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 7536 pgoff = bvec.bv_offset; 7537 for (i = 0; i < nr_sectors; i++) { 7538 ASSERT(pgoff < PAGE_SIZE); 7539 if (uptodate && 7540 (!csum || !check_data_csum(inode, io_bio, icsum, 7541 bvec.bv_page, pgoff, 7542 start, sectorsize))) { 7543 clean_io_failure(fs_info, failure_tree, io_tree, 7544 start, bvec.bv_page, 7545 btrfs_ino(BTRFS_I(inode)), 7546 pgoff); 7547 } else { 7548 blk_status_t status; 7549 7550 status = btrfs_submit_read_repair(inode, 7551 &io_bio->bio, 7552 start - io_bio->logical, 7553 bvec.bv_page, pgoff, 7554 start, 7555 start + sectorsize - 1, 7556 io_bio->mirror_num, 7557 submit_dio_repair_bio); 7558 if (status) 7559 err = status; 7560 } 7561 start += sectorsize; 7562 icsum++; 7563 pgoff += sectorsize; 7564 } 7565 } 7566 return err; 7567 } 7568 7569 static void __endio_write_update_ordered(struct btrfs_inode *inode, 7570 const u64 offset, const u64 bytes, 7571 const bool uptodate) 7572 { 7573 struct btrfs_fs_info *fs_info = inode->root->fs_info; 7574 struct btrfs_ordered_extent *ordered = NULL; 7575 struct btrfs_workqueue *wq; 7576 u64 ordered_offset = offset; 7577 u64 ordered_bytes = bytes; 7578 u64 last_offset; 7579 7580 if (btrfs_is_free_space_inode(inode)) 7581 wq = fs_info->endio_freespace_worker; 7582 else 7583 wq = fs_info->endio_write_workers; 7584 7585 while (ordered_offset < offset + bytes) { 7586 last_offset = ordered_offset; 7587 if (btrfs_dec_test_first_ordered_pending(inode, &ordered, 7588 &ordered_offset, 7589 ordered_bytes, 7590 uptodate)) { 7591 btrfs_init_work(&ordered->work, finish_ordered_fn, NULL, 7592 NULL); 7593 btrfs_queue_work(wq, &ordered->work); 7594 } 7595 /* 7596 * If btrfs_dec_test_ordered_pending does not find any ordered 7597 * extent in the range, we can exit. 7598 */ 7599 if (ordered_offset == last_offset) 7600 return; 7601 /* 7602 * Our bio might span multiple ordered extents. In this case 7603 * we keep going until we have accounted the whole dio. 7604 */ 7605 if (ordered_offset < offset + bytes) { 7606 ordered_bytes = offset + bytes - ordered_offset; 7607 ordered = NULL; 7608 } 7609 } 7610 } 7611 7612 static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data, 7613 struct bio *bio, u64 offset) 7614 { 7615 struct inode *inode = private_data; 7616 blk_status_t ret; 7617 ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, offset, 1); 7618 BUG_ON(ret); /* -ENOMEM */ 7619 return 0; 7620 } 7621 7622 static void btrfs_end_dio_bio(struct bio *bio) 7623 { 7624 struct btrfs_dio_private *dip = bio->bi_private; 7625 blk_status_t err = bio->bi_status; 7626 7627 if (err) 7628 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 7629 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", 7630 btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio), 7631 bio->bi_opf, 7632 (unsigned long long)bio->bi_iter.bi_sector, 7633 bio->bi_iter.bi_size, err); 7634 7635 if (bio_op(bio) == REQ_OP_READ) { 7636 err = btrfs_check_read_dio_bio(dip->inode, btrfs_io_bio(bio), 7637 !err); 7638 } 7639 7640 if (err) 7641 dip->dio_bio->bi_status = err; 7642 7643 bio_put(bio); 7644 btrfs_dio_private_put(dip); 7645 } 7646 7647 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, 7648 struct inode *inode, u64 file_offset, int async_submit) 7649 { 7650 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7651 struct btrfs_dio_private *dip = bio->bi_private; 7652 bool write = bio_op(bio) == REQ_OP_WRITE; 7653 blk_status_t ret; 7654 7655 /* Check btrfs_submit_bio_hook() for rules about async submit. */ 7656 if (async_submit) 7657 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers); 7658 7659 if (!write) { 7660 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 7661 if (ret) 7662 goto err; 7663 } 7664 7665 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 7666 goto map; 7667 7668 if (write && async_submit) { 7669 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0, 7670 file_offset, inode, 7671 btrfs_submit_bio_start_direct_io); 7672 goto err; 7673 } else if (write) { 7674 /* 7675 * If we aren't doing async submit, calculate the csum of the 7676 * bio now. 7677 */ 7678 ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, file_offset, 1); 7679 if (ret) 7680 goto err; 7681 } else { 7682 u64 csum_offset; 7683 7684 csum_offset = file_offset - dip->logical_offset; 7685 csum_offset >>= inode->i_sb->s_blocksize_bits; 7686 csum_offset *= btrfs_super_csum_size(fs_info->super_copy); 7687 btrfs_io_bio(bio)->csum = dip->csums + csum_offset; 7688 } 7689 map: 7690 ret = btrfs_map_bio(fs_info, bio, 0); 7691 err: 7692 return ret; 7693 } 7694 7695 /* 7696 * If this succeeds, the btrfs_dio_private is responsible for cleaning up locked 7697 * or ordered extents whether or not we submit any bios. 7698 */ 7699 static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, 7700 struct inode *inode, 7701 loff_t file_offset) 7702 { 7703 const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); 7704 const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); 7705 size_t dip_size; 7706 struct btrfs_dio_private *dip; 7707 7708 dip_size = sizeof(*dip); 7709 if (!write && csum) { 7710 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7711 const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 7712 size_t nblocks; 7713 7714 nblocks = dio_bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits; 7715 dip_size += csum_size * nblocks; 7716 } 7717 7718 dip = kzalloc(dip_size, GFP_NOFS); 7719 if (!dip) 7720 return NULL; 7721 7722 dip->inode = inode; 7723 dip->logical_offset = file_offset; 7724 dip->bytes = dio_bio->bi_iter.bi_size; 7725 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; 7726 dip->dio_bio = dio_bio; 7727 refcount_set(&dip->refs, 1); 7728 7729 if (write) { 7730 struct btrfs_dio_data *dio_data = current->journal_info; 7731 7732 /* 7733 * Setting range start and end to the same value means that 7734 * no cleanup will happen in btrfs_direct_IO 7735 */ 7736 dio_data->unsubmitted_oe_range_end = dip->logical_offset + 7737 dip->bytes; 7738 dio_data->unsubmitted_oe_range_start = 7739 dio_data->unsubmitted_oe_range_end; 7740 } 7741 return dip; 7742 } 7743 7744 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, 7745 loff_t file_offset) 7746 { 7747 const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); 7748 const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); 7749 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7750 const bool raid56 = (btrfs_data_alloc_profile(fs_info) & 7751 BTRFS_BLOCK_GROUP_RAID56_MASK); 7752 struct btrfs_dio_private *dip; 7753 struct bio *bio; 7754 u64 start_sector; 7755 int async_submit = 0; 7756 u64 submit_len; 7757 int clone_offset = 0; 7758 int clone_len; 7759 int ret; 7760 blk_status_t status; 7761 struct btrfs_io_geometry geom; 7762 7763 dip = btrfs_create_dio_private(dio_bio, inode, file_offset); 7764 if (!dip) { 7765 if (!write) { 7766 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, 7767 file_offset + dio_bio->bi_iter.bi_size - 1); 7768 } 7769 dio_bio->bi_status = BLK_STS_RESOURCE; 7770 dio_end_io(dio_bio); 7771 return; 7772 } 7773 7774 if (!write && csum) { 7775 /* 7776 * Load the csums up front to reduce csum tree searches and 7777 * contention when submitting bios. 7778 */ 7779 status = btrfs_lookup_bio_sums(inode, dio_bio, file_offset, 7780 dip->csums); 7781 if (status != BLK_STS_OK) 7782 goto out_err; 7783 } 7784 7785 start_sector = dio_bio->bi_iter.bi_sector; 7786 submit_len = dio_bio->bi_iter.bi_size; 7787 7788 do { 7789 ret = btrfs_get_io_geometry(fs_info, btrfs_op(dio_bio), 7790 start_sector << 9, submit_len, 7791 &geom); 7792 if (ret) { 7793 status = errno_to_blk_status(ret); 7794 goto out_err; 7795 } 7796 ASSERT(geom.len <= INT_MAX); 7797 7798 clone_len = min_t(int, submit_len, geom.len); 7799 7800 /* 7801 * This will never fail as it's passing GPF_NOFS and 7802 * the allocation is backed by btrfs_bioset. 7803 */ 7804 bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len); 7805 bio->bi_private = dip; 7806 bio->bi_end_io = btrfs_end_dio_bio; 7807 btrfs_io_bio(bio)->logical = file_offset; 7808 7809 ASSERT(submit_len >= clone_len); 7810 submit_len -= clone_len; 7811 7812 /* 7813 * Increase the count before we submit the bio so we know 7814 * the end IO handler won't happen before we increase the 7815 * count. Otherwise, the dip might get freed before we're 7816 * done setting it up. 7817 * 7818 * We transfer the initial reference to the last bio, so we 7819 * don't need to increment the reference count for the last one. 7820 */ 7821 if (submit_len > 0) { 7822 refcount_inc(&dip->refs); 7823 /* 7824 * If we are submitting more than one bio, submit them 7825 * all asynchronously. The exception is RAID 5 or 6, as 7826 * asynchronous checksums make it difficult to collect 7827 * full stripe writes. 7828 */ 7829 if (!raid56) 7830 async_submit = 1; 7831 } 7832 7833 status = btrfs_submit_dio_bio(bio, inode, file_offset, 7834 async_submit); 7835 if (status) { 7836 bio_put(bio); 7837 if (submit_len > 0) 7838 refcount_dec(&dip->refs); 7839 goto out_err; 7840 } 7841 7842 clone_offset += clone_len; 7843 start_sector += clone_len >> 9; 7844 file_offset += clone_len; 7845 } while (submit_len > 0); 7846 return; 7847 7848 out_err: 7849 dip->dio_bio->bi_status = status; 7850 btrfs_dio_private_put(dip); 7851 } 7852 7853 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, 7854 const struct iov_iter *iter, loff_t offset) 7855 { 7856 int seg; 7857 int i; 7858 unsigned int blocksize_mask = fs_info->sectorsize - 1; 7859 ssize_t retval = -EINVAL; 7860 7861 if (offset & blocksize_mask) 7862 goto out; 7863 7864 if (iov_iter_alignment(iter) & blocksize_mask) 7865 goto out; 7866 7867 /* If this is a write we don't need to check anymore */ 7868 if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter)) 7869 return 0; 7870 /* 7871 * Check to make sure we don't have duplicate iov_base's in this 7872 * iovec, if so return EINVAL, otherwise we'll get csum errors 7873 * when reading back. 7874 */ 7875 for (seg = 0; seg < iter->nr_segs; seg++) { 7876 for (i = seg + 1; i < iter->nr_segs; i++) { 7877 if (iter->iov[seg].iov_base == iter->iov[i].iov_base) 7878 goto out; 7879 } 7880 } 7881 retval = 0; 7882 out: 7883 return retval; 7884 } 7885 7886 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 7887 { 7888 struct file *file = iocb->ki_filp; 7889 struct inode *inode = file->f_mapping->host; 7890 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7891 struct btrfs_dio_data dio_data = { 0 }; 7892 struct extent_changeset *data_reserved = NULL; 7893 loff_t offset = iocb->ki_pos; 7894 size_t count = 0; 7895 int flags = 0; 7896 bool wakeup = true; 7897 bool relock = false; 7898 ssize_t ret; 7899 7900 if (check_direct_IO(fs_info, iter, offset)) 7901 return 0; 7902 7903 inode_dio_begin(inode); 7904 7905 /* 7906 * The generic stuff only does filemap_write_and_wait_range, which 7907 * isn't enough if we've written compressed pages to this area, so 7908 * we need to flush the dirty pages again to make absolutely sure 7909 * that any outstanding dirty pages are on disk. 7910 */ 7911 count = iov_iter_count(iter); 7912 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 7913 &BTRFS_I(inode)->runtime_flags)) 7914 filemap_fdatawrite_range(inode->i_mapping, offset, 7915 offset + count - 1); 7916 7917 if (iov_iter_rw(iter) == WRITE) { 7918 /* 7919 * If the write DIO is beyond the EOF, we need update 7920 * the isize, but it is protected by i_mutex. So we can 7921 * not unlock the i_mutex at this case. 7922 */ 7923 if (offset + count <= inode->i_size) { 7924 dio_data.overwrite = 1; 7925 inode_unlock(inode); 7926 relock = true; 7927 } 7928 ret = btrfs_delalloc_reserve_space(BTRFS_I(inode), &data_reserved, 7929 offset, count); 7930 if (ret) 7931 goto out; 7932 7933 /* 7934 * We need to know how many extents we reserved so that we can 7935 * do the accounting properly if we go over the number we 7936 * originally calculated. Abuse current->journal_info for this. 7937 */ 7938 dio_data.reserve = round_up(count, 7939 fs_info->sectorsize); 7940 dio_data.unsubmitted_oe_range_start = (u64)offset; 7941 dio_data.unsubmitted_oe_range_end = (u64)offset; 7942 current->journal_info = &dio_data; 7943 down_read(&BTRFS_I(inode)->dio_sem); 7944 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK, 7945 &BTRFS_I(inode)->runtime_flags)) { 7946 inode_dio_end(inode); 7947 flags = DIO_LOCKING | DIO_SKIP_HOLES; 7948 wakeup = false; 7949 } 7950 7951 ret = __blockdev_direct_IO(iocb, inode, 7952 fs_info->fs_devices->latest_bdev, 7953 iter, btrfs_get_blocks_direct, NULL, 7954 btrfs_submit_direct, flags); 7955 if (iov_iter_rw(iter) == WRITE) { 7956 up_read(&BTRFS_I(inode)->dio_sem); 7957 current->journal_info = NULL; 7958 if (ret < 0 && ret != -EIOCBQUEUED) { 7959 if (dio_data.reserve) 7960 btrfs_delalloc_release_space(BTRFS_I(inode), 7961 data_reserved, offset, dio_data.reserve, 7962 true); 7963 /* 7964 * On error we might have left some ordered extents 7965 * without submitting corresponding bios for them, so 7966 * cleanup them up to avoid other tasks getting them 7967 * and waiting for them to complete forever. 7968 */ 7969 if (dio_data.unsubmitted_oe_range_start < 7970 dio_data.unsubmitted_oe_range_end) 7971 __endio_write_update_ordered(BTRFS_I(inode), 7972 dio_data.unsubmitted_oe_range_start, 7973 dio_data.unsubmitted_oe_range_end - 7974 dio_data.unsubmitted_oe_range_start, 7975 false); 7976 } else if (ret >= 0 && (size_t)ret < count) 7977 btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved, 7978 offset, count - (size_t)ret, true); 7979 btrfs_delalloc_release_extents(BTRFS_I(inode), count); 7980 } 7981 out: 7982 if (wakeup) 7983 inode_dio_end(inode); 7984 if (relock) 7985 inode_lock(inode); 7986 7987 extent_changeset_free(data_reserved); 7988 return ret; 7989 } 7990 7991 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 7992 u64 start, u64 len) 7993 { 7994 int ret; 7995 7996 ret = fiemap_prep(inode, fieinfo, start, &len, 0); 7997 if (ret) 7998 return ret; 7999 8000 return extent_fiemap(inode, fieinfo, start, len); 8001 } 8002 8003 int btrfs_readpage(struct file *file, struct page *page) 8004 { 8005 return extent_read_full_page(page, btrfs_get_extent, 0); 8006 } 8007 8008 static int btrfs_writepage(struct page *page, struct writeback_control *wbc) 8009 { 8010 struct inode *inode = page->mapping->host; 8011 int ret; 8012 8013 if (current->flags & PF_MEMALLOC) { 8014 redirty_page_for_writepage(wbc, page); 8015 unlock_page(page); 8016 return 0; 8017 } 8018 8019 /* 8020 * If we are under memory pressure we will call this directly from the 8021 * VM, we need to make sure we have the inode referenced for the ordered 8022 * extent. If not just return like we didn't do anything. 8023 */ 8024 if (!igrab(inode)) { 8025 redirty_page_for_writepage(wbc, page); 8026 return AOP_WRITEPAGE_ACTIVATE; 8027 } 8028 ret = extent_write_full_page(page, wbc); 8029 btrfs_add_delayed_iput(inode); 8030 return ret; 8031 } 8032 8033 static int btrfs_writepages(struct address_space *mapping, 8034 struct writeback_control *wbc) 8035 { 8036 return extent_writepages(mapping, wbc); 8037 } 8038 8039 static void btrfs_readahead(struct readahead_control *rac) 8040 { 8041 extent_readahead(rac); 8042 } 8043 8044 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8045 { 8046 int ret = try_release_extent_mapping(page, gfp_flags); 8047 if (ret == 1) 8048 detach_page_private(page); 8049 return ret; 8050 } 8051 8052 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8053 { 8054 if (PageWriteback(page) || PageDirty(page)) 8055 return 0; 8056 return __btrfs_releasepage(page, gfp_flags); 8057 } 8058 8059 #ifdef CONFIG_MIGRATION 8060 static int btrfs_migratepage(struct address_space *mapping, 8061 struct page *newpage, struct page *page, 8062 enum migrate_mode mode) 8063 { 8064 int ret; 8065 8066 ret = migrate_page_move_mapping(mapping, newpage, page, 0); 8067 if (ret != MIGRATEPAGE_SUCCESS) 8068 return ret; 8069 8070 if (page_has_private(page)) 8071 attach_page_private(newpage, detach_page_private(page)); 8072 8073 if (PagePrivate2(page)) { 8074 ClearPagePrivate2(page); 8075 SetPagePrivate2(newpage); 8076 } 8077 8078 if (mode != MIGRATE_SYNC_NO_COPY) 8079 migrate_page_copy(newpage, page); 8080 else 8081 migrate_page_states(newpage, page); 8082 return MIGRATEPAGE_SUCCESS; 8083 } 8084 #endif 8085 8086 static void btrfs_invalidatepage(struct page *page, unsigned int offset, 8087 unsigned int length) 8088 { 8089 struct inode *inode = page->mapping->host; 8090 struct extent_io_tree *tree; 8091 struct btrfs_ordered_extent *ordered; 8092 struct extent_state *cached_state = NULL; 8093 u64 page_start = page_offset(page); 8094 u64 page_end = page_start + PAGE_SIZE - 1; 8095 u64 start; 8096 u64 end; 8097 int inode_evicting = inode->i_state & I_FREEING; 8098 8099 /* 8100 * we have the page locked, so new writeback can't start, 8101 * and the dirty bit won't be cleared while we are here. 8102 * 8103 * Wait for IO on this page so that we can safely clear 8104 * the PagePrivate2 bit and do ordered accounting 8105 */ 8106 wait_on_page_writeback(page); 8107 8108 tree = &BTRFS_I(inode)->io_tree; 8109 if (offset) { 8110 btrfs_releasepage(page, GFP_NOFS); 8111 return; 8112 } 8113 8114 if (!inode_evicting) 8115 lock_extent_bits(tree, page_start, page_end, &cached_state); 8116 again: 8117 start = page_start; 8118 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start, 8119 page_end - start + 1); 8120 if (ordered) { 8121 end = min(page_end, 8122 ordered->file_offset + ordered->num_bytes - 1); 8123 /* 8124 * IO on this page will never be started, so we need 8125 * to account for any ordered extents now 8126 */ 8127 if (!inode_evicting) 8128 clear_extent_bit(tree, start, end, 8129 EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 8130 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 8131 EXTENT_DEFRAG, 1, 0, &cached_state); 8132 /* 8133 * whoever cleared the private bit is responsible 8134 * for the finish_ordered_io 8135 */ 8136 if (TestClearPagePrivate2(page)) { 8137 struct btrfs_ordered_inode_tree *tree; 8138 u64 new_len; 8139 8140 tree = &BTRFS_I(inode)->ordered_tree; 8141 8142 spin_lock_irq(&tree->lock); 8143 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 8144 new_len = start - ordered->file_offset; 8145 if (new_len < ordered->truncated_len) 8146 ordered->truncated_len = new_len; 8147 spin_unlock_irq(&tree->lock); 8148 8149 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8150 start, 8151 end - start + 1, 1)) 8152 btrfs_finish_ordered_io(ordered); 8153 } 8154 btrfs_put_ordered_extent(ordered); 8155 if (!inode_evicting) { 8156 cached_state = NULL; 8157 lock_extent_bits(tree, start, end, 8158 &cached_state); 8159 } 8160 8161 start = end + 1; 8162 if (start < page_end) 8163 goto again; 8164 } 8165 8166 /* 8167 * Qgroup reserved space handler 8168 * Page here will be either 8169 * 1) Already written to disk or ordered extent already submitted 8170 * Then its QGROUP_RESERVED bit in io_tree is already cleaned. 8171 * Qgroup will be handled by its qgroup_record then. 8172 * btrfs_qgroup_free_data() call will do nothing here. 8173 * 8174 * 2) Not written to disk yet 8175 * Then btrfs_qgroup_free_data() call will clear the QGROUP_RESERVED 8176 * bit of its io_tree, and free the qgroup reserved data space. 8177 * Since the IO will never happen for this page. 8178 */ 8179 btrfs_qgroup_free_data(BTRFS_I(inode), NULL, page_start, PAGE_SIZE); 8180 if (!inode_evicting) { 8181 clear_extent_bit(tree, page_start, page_end, EXTENT_LOCKED | 8182 EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 8183 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1, 8184 &cached_state); 8185 8186 __btrfs_releasepage(page, GFP_NOFS); 8187 } 8188 8189 ClearPageChecked(page); 8190 detach_page_private(page); 8191 } 8192 8193 /* 8194 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8195 * called from a page fault handler when a page is first dirtied. Hence we must 8196 * be careful to check for EOF conditions here. We set the page up correctly 8197 * for a written page which means we get ENOSPC checking when writing into 8198 * holes and correct delalloc and unwritten extent mapping on filesystems that 8199 * support these features. 8200 * 8201 * We are not allowed to take the i_mutex here so we have to play games to 8202 * protect against truncate races as the page could now be beyond EOF. Because 8203 * truncate_setsize() writes the inode size before removing pages, once we have 8204 * the page lock we can determine safely if the page is beyond EOF. If it is not 8205 * beyond EOF, then the page is guaranteed safe against truncation until we 8206 * unlock the page. 8207 */ 8208 vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) 8209 { 8210 struct page *page = vmf->page; 8211 struct inode *inode = file_inode(vmf->vma->vm_file); 8212 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8213 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8214 struct btrfs_ordered_extent *ordered; 8215 struct extent_state *cached_state = NULL; 8216 struct extent_changeset *data_reserved = NULL; 8217 char *kaddr; 8218 unsigned long zero_start; 8219 loff_t size; 8220 vm_fault_t ret; 8221 int ret2; 8222 int reserved = 0; 8223 u64 reserved_space; 8224 u64 page_start; 8225 u64 page_end; 8226 u64 end; 8227 8228 reserved_space = PAGE_SIZE; 8229 8230 sb_start_pagefault(inode->i_sb); 8231 page_start = page_offset(page); 8232 page_end = page_start + PAGE_SIZE - 1; 8233 end = page_end; 8234 8235 /* 8236 * Reserving delalloc space after obtaining the page lock can lead to 8237 * deadlock. For example, if a dirty page is locked by this function 8238 * and the call to btrfs_delalloc_reserve_space() ends up triggering 8239 * dirty page write out, then the btrfs_writepage() function could 8240 * end up waiting indefinitely to get a lock on the page currently 8241 * being processed by btrfs_page_mkwrite() function. 8242 */ 8243 ret2 = btrfs_delalloc_reserve_space(BTRFS_I(inode), &data_reserved, 8244 page_start, reserved_space); 8245 if (!ret2) { 8246 ret2 = file_update_time(vmf->vma->vm_file); 8247 reserved = 1; 8248 } 8249 if (ret2) { 8250 ret = vmf_error(ret2); 8251 if (reserved) 8252 goto out; 8253 goto out_noreserve; 8254 } 8255 8256 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 8257 again: 8258 lock_page(page); 8259 size = i_size_read(inode); 8260 8261 if ((page->mapping != inode->i_mapping) || 8262 (page_start >= size)) { 8263 /* page got truncated out from underneath us */ 8264 goto out_unlock; 8265 } 8266 wait_on_page_writeback(page); 8267 8268 lock_extent_bits(io_tree, page_start, page_end, &cached_state); 8269 set_page_extent_mapped(page); 8270 8271 /* 8272 * we can't set the delalloc bits if there are pending ordered 8273 * extents. Drop our locks and wait for them to finish 8274 */ 8275 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 8276 PAGE_SIZE); 8277 if (ordered) { 8278 unlock_extent_cached(io_tree, page_start, page_end, 8279 &cached_state); 8280 unlock_page(page); 8281 btrfs_start_ordered_extent(inode, ordered, 1); 8282 btrfs_put_ordered_extent(ordered); 8283 goto again; 8284 } 8285 8286 if (page->index == ((size - 1) >> PAGE_SHIFT)) { 8287 reserved_space = round_up(size - page_start, 8288 fs_info->sectorsize); 8289 if (reserved_space < PAGE_SIZE) { 8290 end = page_start + reserved_space - 1; 8291 btrfs_delalloc_release_space(BTRFS_I(inode), 8292 data_reserved, page_start, 8293 PAGE_SIZE - reserved_space, true); 8294 } 8295 } 8296 8297 /* 8298 * page_mkwrite gets called when the page is firstly dirtied after it's 8299 * faulted in, but write(2) could also dirty a page and set delalloc 8300 * bits, thus in this case for space account reason, we still need to 8301 * clear any delalloc bits within this page range since we have to 8302 * reserve data&meta space before lock_page() (see above comments). 8303 */ 8304 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, 8305 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 8306 EXTENT_DEFRAG, 0, 0, &cached_state); 8307 8308 ret2 = btrfs_set_extent_delalloc(BTRFS_I(inode), page_start, end, 0, 8309 &cached_state); 8310 if (ret2) { 8311 unlock_extent_cached(io_tree, page_start, page_end, 8312 &cached_state); 8313 ret = VM_FAULT_SIGBUS; 8314 goto out_unlock; 8315 } 8316 8317 /* page is wholly or partially inside EOF */ 8318 if (page_start + PAGE_SIZE > size) 8319 zero_start = offset_in_page(size); 8320 else 8321 zero_start = PAGE_SIZE; 8322 8323 if (zero_start != PAGE_SIZE) { 8324 kaddr = kmap(page); 8325 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start); 8326 flush_dcache_page(page); 8327 kunmap(page); 8328 } 8329 ClearPageChecked(page); 8330 set_page_dirty(page); 8331 SetPageUptodate(page); 8332 8333 BTRFS_I(inode)->last_trans = fs_info->generation; 8334 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; 8335 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; 8336 8337 unlock_extent_cached(io_tree, page_start, page_end, &cached_state); 8338 8339 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); 8340 sb_end_pagefault(inode->i_sb); 8341 extent_changeset_free(data_reserved); 8342 return VM_FAULT_LOCKED; 8343 8344 out_unlock: 8345 unlock_page(page); 8346 out: 8347 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); 8348 btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved, page_start, 8349 reserved_space, (ret != 0)); 8350 out_noreserve: 8351 sb_end_pagefault(inode->i_sb); 8352 extent_changeset_free(data_reserved); 8353 return ret; 8354 } 8355 8356 static int btrfs_truncate(struct inode *inode, bool skip_writeback) 8357 { 8358 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8359 struct btrfs_root *root = BTRFS_I(inode)->root; 8360 struct btrfs_block_rsv *rsv; 8361 int ret; 8362 struct btrfs_trans_handle *trans; 8363 u64 mask = fs_info->sectorsize - 1; 8364 u64 min_size = btrfs_calc_metadata_size(fs_info, 1); 8365 8366 if (!skip_writeback) { 8367 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), 8368 (u64)-1); 8369 if (ret) 8370 return ret; 8371 } 8372 8373 /* 8374 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of 8375 * things going on here: 8376 * 8377 * 1) We need to reserve space to update our inode. 8378 * 8379 * 2) We need to have something to cache all the space that is going to 8380 * be free'd up by the truncate operation, but also have some slack 8381 * space reserved in case it uses space during the truncate (thank you 8382 * very much snapshotting). 8383 * 8384 * And we need these to be separate. The fact is we can use a lot of 8385 * space doing the truncate, and we have no earthly idea how much space 8386 * we will use, so we need the truncate reservation to be separate so it 8387 * doesn't end up using space reserved for updating the inode. We also 8388 * need to be able to stop the transaction and start a new one, which 8389 * means we need to be able to update the inode several times, and we 8390 * have no idea of knowing how many times that will be, so we can't just 8391 * reserve 1 item for the entirety of the operation, so that has to be 8392 * done separately as well. 8393 * 8394 * So that leaves us with 8395 * 8396 * 1) rsv - for the truncate reservation, which we will steal from the 8397 * transaction reservation. 8398 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for 8399 * updating the inode. 8400 */ 8401 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 8402 if (!rsv) 8403 return -ENOMEM; 8404 rsv->size = min_size; 8405 rsv->failfast = 1; 8406 8407 /* 8408 * 1 for the truncate slack space 8409 * 1 for updating the inode. 8410 */ 8411 trans = btrfs_start_transaction(root, 2); 8412 if (IS_ERR(trans)) { 8413 ret = PTR_ERR(trans); 8414 goto out; 8415 } 8416 8417 /* Migrate the slack space for the truncate to our reserve */ 8418 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv, 8419 min_size, false); 8420 BUG_ON(ret); 8421 8422 /* 8423 * So if we truncate and then write and fsync we normally would just 8424 * write the extents that changed, which is a problem if we need to 8425 * first truncate that entire inode. So set this flag so we write out 8426 * all of the extents in the inode to the sync log so we're completely 8427 * safe. 8428 */ 8429 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 8430 trans->block_rsv = rsv; 8431 8432 while (1) { 8433 ret = btrfs_truncate_inode_items(trans, root, inode, 8434 inode->i_size, 8435 BTRFS_EXTENT_DATA_KEY); 8436 trans->block_rsv = &fs_info->trans_block_rsv; 8437 if (ret != -ENOSPC && ret != -EAGAIN) 8438 break; 8439 8440 ret = btrfs_update_inode(trans, root, inode); 8441 if (ret) 8442 break; 8443 8444 btrfs_end_transaction(trans); 8445 btrfs_btree_balance_dirty(fs_info); 8446 8447 trans = btrfs_start_transaction(root, 2); 8448 if (IS_ERR(trans)) { 8449 ret = PTR_ERR(trans); 8450 trans = NULL; 8451 break; 8452 } 8453 8454 btrfs_block_rsv_release(fs_info, rsv, -1, NULL); 8455 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 8456 rsv, min_size, false); 8457 BUG_ON(ret); /* shouldn't happen */ 8458 trans->block_rsv = rsv; 8459 } 8460 8461 /* 8462 * We can't call btrfs_truncate_block inside a trans handle as we could 8463 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know 8464 * we've truncated everything except the last little bit, and can do 8465 * btrfs_truncate_block and then update the disk_i_size. 8466 */ 8467 if (ret == NEED_TRUNCATE_BLOCK) { 8468 btrfs_end_transaction(trans); 8469 btrfs_btree_balance_dirty(fs_info); 8470 8471 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0); 8472 if (ret) 8473 goto out; 8474 trans = btrfs_start_transaction(root, 1); 8475 if (IS_ERR(trans)) { 8476 ret = PTR_ERR(trans); 8477 goto out; 8478 } 8479 btrfs_inode_safe_disk_i_size_write(inode, 0); 8480 } 8481 8482 if (trans) { 8483 int ret2; 8484 8485 trans->block_rsv = &fs_info->trans_block_rsv; 8486 ret2 = btrfs_update_inode(trans, root, inode); 8487 if (ret2 && !ret) 8488 ret = ret2; 8489 8490 ret2 = btrfs_end_transaction(trans); 8491 if (ret2 && !ret) 8492 ret = ret2; 8493 btrfs_btree_balance_dirty(fs_info); 8494 } 8495 out: 8496 btrfs_free_block_rsv(fs_info, rsv); 8497 8498 return ret; 8499 } 8500 8501 /* 8502 * create a new subvolume directory/inode (helper for the ioctl). 8503 */ 8504 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, 8505 struct btrfs_root *new_root, 8506 struct btrfs_root *parent_root, 8507 u64 new_dirid) 8508 { 8509 struct inode *inode; 8510 int err; 8511 u64 index = 0; 8512 8513 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, 8514 new_dirid, new_dirid, 8515 S_IFDIR | (~current_umask() & S_IRWXUGO), 8516 &index); 8517 if (IS_ERR(inode)) 8518 return PTR_ERR(inode); 8519 inode->i_op = &btrfs_dir_inode_operations; 8520 inode->i_fop = &btrfs_dir_file_operations; 8521 8522 set_nlink(inode, 1); 8523 btrfs_i_size_write(BTRFS_I(inode), 0); 8524 unlock_new_inode(inode); 8525 8526 err = btrfs_subvol_inherit_props(trans, new_root, parent_root); 8527 if (err) 8528 btrfs_err(new_root->fs_info, 8529 "error inheriting subvolume %llu properties: %d", 8530 new_root->root_key.objectid, err); 8531 8532 err = btrfs_update_inode(trans, new_root, inode); 8533 8534 iput(inode); 8535 return err; 8536 } 8537 8538 struct inode *btrfs_alloc_inode(struct super_block *sb) 8539 { 8540 struct btrfs_fs_info *fs_info = btrfs_sb(sb); 8541 struct btrfs_inode *ei; 8542 struct inode *inode; 8543 8544 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL); 8545 if (!ei) 8546 return NULL; 8547 8548 ei->root = NULL; 8549 ei->generation = 0; 8550 ei->last_trans = 0; 8551 ei->last_sub_trans = 0; 8552 ei->logged_trans = 0; 8553 ei->delalloc_bytes = 0; 8554 ei->new_delalloc_bytes = 0; 8555 ei->defrag_bytes = 0; 8556 ei->disk_i_size = 0; 8557 ei->flags = 0; 8558 ei->csum_bytes = 0; 8559 ei->index_cnt = (u64)-1; 8560 ei->dir_index = 0; 8561 ei->last_unlink_trans = 0; 8562 ei->last_reflink_trans = 0; 8563 ei->last_log_commit = 0; 8564 8565 spin_lock_init(&ei->lock); 8566 ei->outstanding_extents = 0; 8567 if (sb->s_magic != BTRFS_TEST_MAGIC) 8568 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, 8569 BTRFS_BLOCK_RSV_DELALLOC); 8570 ei->runtime_flags = 0; 8571 ei->prop_compress = BTRFS_COMPRESS_NONE; 8572 ei->defrag_compress = BTRFS_COMPRESS_NONE; 8573 8574 ei->delayed_node = NULL; 8575 8576 ei->i_otime.tv_sec = 0; 8577 ei->i_otime.tv_nsec = 0; 8578 8579 inode = &ei->vfs_inode; 8580 extent_map_tree_init(&ei->extent_tree); 8581 extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO, inode); 8582 extent_io_tree_init(fs_info, &ei->io_failure_tree, 8583 IO_TREE_INODE_IO_FAILURE, inode); 8584 extent_io_tree_init(fs_info, &ei->file_extent_tree, 8585 IO_TREE_INODE_FILE_EXTENT, inode); 8586 ei->io_tree.track_uptodate = true; 8587 ei->io_failure_tree.track_uptodate = true; 8588 atomic_set(&ei->sync_writers, 0); 8589 mutex_init(&ei->log_mutex); 8590 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 8591 INIT_LIST_HEAD(&ei->delalloc_inodes); 8592 INIT_LIST_HEAD(&ei->delayed_iput); 8593 RB_CLEAR_NODE(&ei->rb_node); 8594 init_rwsem(&ei->dio_sem); 8595 8596 return inode; 8597 } 8598 8599 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 8600 void btrfs_test_destroy_inode(struct inode *inode) 8601 { 8602 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 8603 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8604 } 8605 #endif 8606 8607 void btrfs_free_inode(struct inode *inode) 8608 { 8609 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8610 } 8611 8612 void btrfs_destroy_inode(struct inode *inode) 8613 { 8614 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8615 struct btrfs_ordered_extent *ordered; 8616 struct btrfs_root *root = BTRFS_I(inode)->root; 8617 8618 WARN_ON(!hlist_empty(&inode->i_dentry)); 8619 WARN_ON(inode->i_data.nrpages); 8620 WARN_ON(BTRFS_I(inode)->block_rsv.reserved); 8621 WARN_ON(BTRFS_I(inode)->block_rsv.size); 8622 WARN_ON(BTRFS_I(inode)->outstanding_extents); 8623 WARN_ON(BTRFS_I(inode)->delalloc_bytes); 8624 WARN_ON(BTRFS_I(inode)->new_delalloc_bytes); 8625 WARN_ON(BTRFS_I(inode)->csum_bytes); 8626 WARN_ON(BTRFS_I(inode)->defrag_bytes); 8627 8628 /* 8629 * This can happen where we create an inode, but somebody else also 8630 * created the same inode and we need to destroy the one we already 8631 * created. 8632 */ 8633 if (!root) 8634 return; 8635 8636 while (1) { 8637 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 8638 if (!ordered) 8639 break; 8640 else { 8641 btrfs_err(fs_info, 8642 "found ordered extent %llu %llu on inode cleanup", 8643 ordered->file_offset, ordered->num_bytes); 8644 btrfs_remove_ordered_extent(inode, ordered); 8645 btrfs_put_ordered_extent(ordered); 8646 btrfs_put_ordered_extent(ordered); 8647 } 8648 } 8649 btrfs_qgroup_check_reserved_leak(BTRFS_I(inode)); 8650 inode_tree_del(inode); 8651 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 8652 btrfs_inode_clear_file_extent_range(BTRFS_I(inode), 0, (u64)-1); 8653 btrfs_put_root(BTRFS_I(inode)->root); 8654 } 8655 8656 int btrfs_drop_inode(struct inode *inode) 8657 { 8658 struct btrfs_root *root = BTRFS_I(inode)->root; 8659 8660 if (root == NULL) 8661 return 1; 8662 8663 /* the snap/subvol tree is on deleting */ 8664 if (btrfs_root_refs(&root->root_item) == 0) 8665 return 1; 8666 else 8667 return generic_drop_inode(inode); 8668 } 8669 8670 static void init_once(void *foo) 8671 { 8672 struct btrfs_inode *ei = (struct btrfs_inode *) foo; 8673 8674 inode_init_once(&ei->vfs_inode); 8675 } 8676 8677 void __cold btrfs_destroy_cachep(void) 8678 { 8679 /* 8680 * Make sure all delayed rcu free inodes are flushed before we 8681 * destroy cache. 8682 */ 8683 rcu_barrier(); 8684 kmem_cache_destroy(btrfs_inode_cachep); 8685 kmem_cache_destroy(btrfs_trans_handle_cachep); 8686 kmem_cache_destroy(btrfs_path_cachep); 8687 kmem_cache_destroy(btrfs_free_space_cachep); 8688 kmem_cache_destroy(btrfs_free_space_bitmap_cachep); 8689 } 8690 8691 int __init btrfs_init_cachep(void) 8692 { 8693 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 8694 sizeof(struct btrfs_inode), 0, 8695 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, 8696 init_once); 8697 if (!btrfs_inode_cachep) 8698 goto fail; 8699 8700 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", 8701 sizeof(struct btrfs_trans_handle), 0, 8702 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); 8703 if (!btrfs_trans_handle_cachep) 8704 goto fail; 8705 8706 btrfs_path_cachep = kmem_cache_create("btrfs_path", 8707 sizeof(struct btrfs_path), 0, 8708 SLAB_MEM_SPREAD, NULL); 8709 if (!btrfs_path_cachep) 8710 goto fail; 8711 8712 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space", 8713 sizeof(struct btrfs_free_space), 0, 8714 SLAB_MEM_SPREAD, NULL); 8715 if (!btrfs_free_space_cachep) 8716 goto fail; 8717 8718 btrfs_free_space_bitmap_cachep = kmem_cache_create("btrfs_free_space_bitmap", 8719 PAGE_SIZE, PAGE_SIZE, 8720 SLAB_RED_ZONE, NULL); 8721 if (!btrfs_free_space_bitmap_cachep) 8722 goto fail; 8723 8724 return 0; 8725 fail: 8726 btrfs_destroy_cachep(); 8727 return -ENOMEM; 8728 } 8729 8730 static int btrfs_getattr(const struct path *path, struct kstat *stat, 8731 u32 request_mask, unsigned int flags) 8732 { 8733 u64 delalloc_bytes; 8734 struct inode *inode = d_inode(path->dentry); 8735 u32 blocksize = inode->i_sb->s_blocksize; 8736 u32 bi_flags = BTRFS_I(inode)->flags; 8737 8738 stat->result_mask |= STATX_BTIME; 8739 stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec; 8740 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec; 8741 if (bi_flags & BTRFS_INODE_APPEND) 8742 stat->attributes |= STATX_ATTR_APPEND; 8743 if (bi_flags & BTRFS_INODE_COMPRESS) 8744 stat->attributes |= STATX_ATTR_COMPRESSED; 8745 if (bi_flags & BTRFS_INODE_IMMUTABLE) 8746 stat->attributes |= STATX_ATTR_IMMUTABLE; 8747 if (bi_flags & BTRFS_INODE_NODUMP) 8748 stat->attributes |= STATX_ATTR_NODUMP; 8749 8750 stat->attributes_mask |= (STATX_ATTR_APPEND | 8751 STATX_ATTR_COMPRESSED | 8752 STATX_ATTR_IMMUTABLE | 8753 STATX_ATTR_NODUMP); 8754 8755 generic_fillattr(inode, stat); 8756 stat->dev = BTRFS_I(inode)->root->anon_dev; 8757 8758 spin_lock(&BTRFS_I(inode)->lock); 8759 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; 8760 spin_unlock(&BTRFS_I(inode)->lock); 8761 stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) + 8762 ALIGN(delalloc_bytes, blocksize)) >> 9; 8763 return 0; 8764 } 8765 8766 static int btrfs_rename_exchange(struct inode *old_dir, 8767 struct dentry *old_dentry, 8768 struct inode *new_dir, 8769 struct dentry *new_dentry) 8770 { 8771 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 8772 struct btrfs_trans_handle *trans; 8773 struct btrfs_root *root = BTRFS_I(old_dir)->root; 8774 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 8775 struct inode *new_inode = new_dentry->d_inode; 8776 struct inode *old_inode = old_dentry->d_inode; 8777 struct timespec64 ctime = current_time(old_inode); 8778 struct dentry *parent; 8779 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 8780 u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); 8781 u64 old_idx = 0; 8782 u64 new_idx = 0; 8783 int ret; 8784 bool root_log_pinned = false; 8785 bool dest_log_pinned = false; 8786 struct btrfs_log_ctx ctx_root; 8787 struct btrfs_log_ctx ctx_dest; 8788 bool sync_log_root = false; 8789 bool sync_log_dest = false; 8790 bool commit_transaction = false; 8791 8792 /* we only allow rename subvolume link between subvolumes */ 8793 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 8794 return -EXDEV; 8795 8796 btrfs_init_log_ctx(&ctx_root, old_inode); 8797 btrfs_init_log_ctx(&ctx_dest, new_inode); 8798 8799 /* close the race window with snapshot create/destroy ioctl */ 8800 if (old_ino == BTRFS_FIRST_FREE_OBJECTID || 8801 new_ino == BTRFS_FIRST_FREE_OBJECTID) 8802 down_read(&fs_info->subvol_sem); 8803 8804 /* 8805 * We want to reserve the absolute worst case amount of items. So if 8806 * both inodes are subvols and we need to unlink them then that would 8807 * require 4 item modifications, but if they are both normal inodes it 8808 * would require 5 item modifications, so we'll assume their normal 8809 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items 8810 * should cover the worst case number of items we'll modify. 8811 */ 8812 trans = btrfs_start_transaction(root, 12); 8813 if (IS_ERR(trans)) { 8814 ret = PTR_ERR(trans); 8815 goto out_notrans; 8816 } 8817 8818 if (dest != root) 8819 btrfs_record_root_in_trans(trans, dest); 8820 8821 /* 8822 * We need to find a free sequence number both in the source and 8823 * in the destination directory for the exchange. 8824 */ 8825 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); 8826 if (ret) 8827 goto out_fail; 8828 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); 8829 if (ret) 8830 goto out_fail; 8831 8832 BTRFS_I(old_inode)->dir_index = 0ULL; 8833 BTRFS_I(new_inode)->dir_index = 0ULL; 8834 8835 /* Reference for the source. */ 8836 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8837 /* force full log commit if subvolume involved. */ 8838 btrfs_set_log_full_commit(trans); 8839 } else { 8840 btrfs_pin_log_trans(root); 8841 root_log_pinned = true; 8842 ret = btrfs_insert_inode_ref(trans, dest, 8843 new_dentry->d_name.name, 8844 new_dentry->d_name.len, 8845 old_ino, 8846 btrfs_ino(BTRFS_I(new_dir)), 8847 old_idx); 8848 if (ret) 8849 goto out_fail; 8850 } 8851 8852 /* And now for the dest. */ 8853 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 8854 /* force full log commit if subvolume involved. */ 8855 btrfs_set_log_full_commit(trans); 8856 } else { 8857 btrfs_pin_log_trans(dest); 8858 dest_log_pinned = true; 8859 ret = btrfs_insert_inode_ref(trans, root, 8860 old_dentry->d_name.name, 8861 old_dentry->d_name.len, 8862 new_ino, 8863 btrfs_ino(BTRFS_I(old_dir)), 8864 new_idx); 8865 if (ret) 8866 goto out_fail; 8867 } 8868 8869 /* Update inode version and ctime/mtime. */ 8870 inode_inc_iversion(old_dir); 8871 inode_inc_iversion(new_dir); 8872 inode_inc_iversion(old_inode); 8873 inode_inc_iversion(new_inode); 8874 old_dir->i_ctime = old_dir->i_mtime = ctime; 8875 new_dir->i_ctime = new_dir->i_mtime = ctime; 8876 old_inode->i_ctime = ctime; 8877 new_inode->i_ctime = ctime; 8878 8879 if (old_dentry->d_parent != new_dentry->d_parent) { 8880 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 8881 BTRFS_I(old_inode), 1); 8882 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), 8883 BTRFS_I(new_inode), 1); 8884 } 8885 8886 /* src is a subvolume */ 8887 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8888 ret = btrfs_unlink_subvol(trans, old_dir, old_dentry); 8889 } else { /* src is an inode */ 8890 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 8891 BTRFS_I(old_dentry->d_inode), 8892 old_dentry->d_name.name, 8893 old_dentry->d_name.len); 8894 if (!ret) 8895 ret = btrfs_update_inode(trans, root, old_inode); 8896 } 8897 if (ret) { 8898 btrfs_abort_transaction(trans, ret); 8899 goto out_fail; 8900 } 8901 8902 /* dest is a subvolume */ 8903 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 8904 ret = btrfs_unlink_subvol(trans, new_dir, new_dentry); 8905 } else { /* dest is an inode */ 8906 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 8907 BTRFS_I(new_dentry->d_inode), 8908 new_dentry->d_name.name, 8909 new_dentry->d_name.len); 8910 if (!ret) 8911 ret = btrfs_update_inode(trans, dest, new_inode); 8912 } 8913 if (ret) { 8914 btrfs_abort_transaction(trans, ret); 8915 goto out_fail; 8916 } 8917 8918 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 8919 new_dentry->d_name.name, 8920 new_dentry->d_name.len, 0, old_idx); 8921 if (ret) { 8922 btrfs_abort_transaction(trans, ret); 8923 goto out_fail; 8924 } 8925 8926 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), 8927 old_dentry->d_name.name, 8928 old_dentry->d_name.len, 0, new_idx); 8929 if (ret) { 8930 btrfs_abort_transaction(trans, ret); 8931 goto out_fail; 8932 } 8933 8934 if (old_inode->i_nlink == 1) 8935 BTRFS_I(old_inode)->dir_index = old_idx; 8936 if (new_inode->i_nlink == 1) 8937 BTRFS_I(new_inode)->dir_index = new_idx; 8938 8939 if (root_log_pinned) { 8940 parent = new_dentry->d_parent; 8941 ret = btrfs_log_new_name(trans, BTRFS_I(old_inode), 8942 BTRFS_I(old_dir), parent, 8943 false, &ctx_root); 8944 if (ret == BTRFS_NEED_LOG_SYNC) 8945 sync_log_root = true; 8946 else if (ret == BTRFS_NEED_TRANS_COMMIT) 8947 commit_transaction = true; 8948 ret = 0; 8949 btrfs_end_log_trans(root); 8950 root_log_pinned = false; 8951 } 8952 if (dest_log_pinned) { 8953 if (!commit_transaction) { 8954 parent = old_dentry->d_parent; 8955 ret = btrfs_log_new_name(trans, BTRFS_I(new_inode), 8956 BTRFS_I(new_dir), parent, 8957 false, &ctx_dest); 8958 if (ret == BTRFS_NEED_LOG_SYNC) 8959 sync_log_dest = true; 8960 else if (ret == BTRFS_NEED_TRANS_COMMIT) 8961 commit_transaction = true; 8962 ret = 0; 8963 } 8964 btrfs_end_log_trans(dest); 8965 dest_log_pinned = false; 8966 } 8967 out_fail: 8968 /* 8969 * If we have pinned a log and an error happened, we unpin tasks 8970 * trying to sync the log and force them to fallback to a transaction 8971 * commit if the log currently contains any of the inodes involved in 8972 * this rename operation (to ensure we do not persist a log with an 8973 * inconsistent state for any of these inodes or leading to any 8974 * inconsistencies when replayed). If the transaction was aborted, the 8975 * abortion reason is propagated to userspace when attempting to commit 8976 * the transaction. If the log does not contain any of these inodes, we 8977 * allow the tasks to sync it. 8978 */ 8979 if (ret && (root_log_pinned || dest_log_pinned)) { 8980 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 8981 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 8982 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 8983 (new_inode && 8984 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 8985 btrfs_set_log_full_commit(trans); 8986 8987 if (root_log_pinned) { 8988 btrfs_end_log_trans(root); 8989 root_log_pinned = false; 8990 } 8991 if (dest_log_pinned) { 8992 btrfs_end_log_trans(dest); 8993 dest_log_pinned = false; 8994 } 8995 } 8996 if (!ret && sync_log_root && !commit_transaction) { 8997 ret = btrfs_sync_log(trans, BTRFS_I(old_inode)->root, 8998 &ctx_root); 8999 if (ret) 9000 commit_transaction = true; 9001 } 9002 if (!ret && sync_log_dest && !commit_transaction) { 9003 ret = btrfs_sync_log(trans, BTRFS_I(new_inode)->root, 9004 &ctx_dest); 9005 if (ret) 9006 commit_transaction = true; 9007 } 9008 if (commit_transaction) { 9009 /* 9010 * We may have set commit_transaction when logging the new name 9011 * in the destination root, in which case we left the source 9012 * root context in the list of log contextes. So make sure we 9013 * remove it to avoid invalid memory accesses, since the context 9014 * was allocated in our stack frame. 9015 */ 9016 if (sync_log_root) { 9017 mutex_lock(&root->log_mutex); 9018 list_del_init(&ctx_root.list); 9019 mutex_unlock(&root->log_mutex); 9020 } 9021 ret = btrfs_commit_transaction(trans); 9022 } else { 9023 int ret2; 9024 9025 ret2 = btrfs_end_transaction(trans); 9026 ret = ret ? ret : ret2; 9027 } 9028 out_notrans: 9029 if (new_ino == BTRFS_FIRST_FREE_OBJECTID || 9030 old_ino == BTRFS_FIRST_FREE_OBJECTID) 9031 up_read(&fs_info->subvol_sem); 9032 9033 ASSERT(list_empty(&ctx_root.list)); 9034 ASSERT(list_empty(&ctx_dest.list)); 9035 9036 return ret; 9037 } 9038 9039 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans, 9040 struct btrfs_root *root, 9041 struct inode *dir, 9042 struct dentry *dentry) 9043 { 9044 int ret; 9045 struct inode *inode; 9046 u64 objectid; 9047 u64 index; 9048 9049 ret = btrfs_find_free_ino(root, &objectid); 9050 if (ret) 9051 return ret; 9052 9053 inode = btrfs_new_inode(trans, root, dir, 9054 dentry->d_name.name, 9055 dentry->d_name.len, 9056 btrfs_ino(BTRFS_I(dir)), 9057 objectid, 9058 S_IFCHR | WHITEOUT_MODE, 9059 &index); 9060 9061 if (IS_ERR(inode)) { 9062 ret = PTR_ERR(inode); 9063 return ret; 9064 } 9065 9066 inode->i_op = &btrfs_special_inode_operations; 9067 init_special_inode(inode, inode->i_mode, 9068 WHITEOUT_DEV); 9069 9070 ret = btrfs_init_inode_security(trans, inode, dir, 9071 &dentry->d_name); 9072 if (ret) 9073 goto out; 9074 9075 ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 9076 BTRFS_I(inode), 0, index); 9077 if (ret) 9078 goto out; 9079 9080 ret = btrfs_update_inode(trans, root, inode); 9081 out: 9082 unlock_new_inode(inode); 9083 if (ret) 9084 inode_dec_link_count(inode); 9085 iput(inode); 9086 9087 return ret; 9088 } 9089 9090 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry, 9091 struct inode *new_dir, struct dentry *new_dentry, 9092 unsigned int flags) 9093 { 9094 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9095 struct btrfs_trans_handle *trans; 9096 unsigned int trans_num_items; 9097 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9098 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9099 struct inode *new_inode = d_inode(new_dentry); 9100 struct inode *old_inode = d_inode(old_dentry); 9101 u64 index = 0; 9102 int ret; 9103 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9104 bool log_pinned = false; 9105 struct btrfs_log_ctx ctx; 9106 bool sync_log = false; 9107 bool commit_transaction = false; 9108 9109 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 9110 return -EPERM; 9111 9112 /* we only allow rename subvolume link between subvolumes */ 9113 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9114 return -EXDEV; 9115 9116 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 9117 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) 9118 return -ENOTEMPTY; 9119 9120 if (S_ISDIR(old_inode->i_mode) && new_inode && 9121 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 9122 return -ENOTEMPTY; 9123 9124 9125 /* check for collisions, even if the name isn't there */ 9126 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, 9127 new_dentry->d_name.name, 9128 new_dentry->d_name.len); 9129 9130 if (ret) { 9131 if (ret == -EEXIST) { 9132 /* we shouldn't get 9133 * eexist without a new_inode */ 9134 if (WARN_ON(!new_inode)) { 9135 return ret; 9136 } 9137 } else { 9138 /* maybe -EOVERFLOW */ 9139 return ret; 9140 } 9141 } 9142 ret = 0; 9143 9144 /* 9145 * we're using rename to replace one file with another. Start IO on it 9146 * now so we don't add too much work to the end of the transaction 9147 */ 9148 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 9149 filemap_flush(old_inode->i_mapping); 9150 9151 /* close the racy window with snapshot create/destroy ioctl */ 9152 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9153 down_read(&fs_info->subvol_sem); 9154 /* 9155 * We want to reserve the absolute worst case amount of items. So if 9156 * both inodes are subvols and we need to unlink them then that would 9157 * require 4 item modifications, but if they are both normal inodes it 9158 * would require 5 item modifications, so we'll assume they are normal 9159 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items 9160 * should cover the worst case number of items we'll modify. 9161 * If our rename has the whiteout flag, we need more 5 units for the 9162 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item 9163 * when selinux is enabled). 9164 */ 9165 trans_num_items = 11; 9166 if (flags & RENAME_WHITEOUT) 9167 trans_num_items += 5; 9168 trans = btrfs_start_transaction(root, trans_num_items); 9169 if (IS_ERR(trans)) { 9170 ret = PTR_ERR(trans); 9171 goto out_notrans; 9172 } 9173 9174 if (dest != root) 9175 btrfs_record_root_in_trans(trans, dest); 9176 9177 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); 9178 if (ret) 9179 goto out_fail; 9180 9181 BTRFS_I(old_inode)->dir_index = 0ULL; 9182 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9183 /* force full log commit if subvolume involved. */ 9184 btrfs_set_log_full_commit(trans); 9185 } else { 9186 btrfs_pin_log_trans(root); 9187 log_pinned = true; 9188 ret = btrfs_insert_inode_ref(trans, dest, 9189 new_dentry->d_name.name, 9190 new_dentry->d_name.len, 9191 old_ino, 9192 btrfs_ino(BTRFS_I(new_dir)), index); 9193 if (ret) 9194 goto out_fail; 9195 } 9196 9197 inode_inc_iversion(old_dir); 9198 inode_inc_iversion(new_dir); 9199 inode_inc_iversion(old_inode); 9200 old_dir->i_ctime = old_dir->i_mtime = 9201 new_dir->i_ctime = new_dir->i_mtime = 9202 old_inode->i_ctime = current_time(old_dir); 9203 9204 if (old_dentry->d_parent != new_dentry->d_parent) 9205 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9206 BTRFS_I(old_inode), 1); 9207 9208 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9209 ret = btrfs_unlink_subvol(trans, old_dir, old_dentry); 9210 } else { 9211 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9212 BTRFS_I(d_inode(old_dentry)), 9213 old_dentry->d_name.name, 9214 old_dentry->d_name.len); 9215 if (!ret) 9216 ret = btrfs_update_inode(trans, root, old_inode); 9217 } 9218 if (ret) { 9219 btrfs_abort_transaction(trans, ret); 9220 goto out_fail; 9221 } 9222 9223 if (new_inode) { 9224 inode_inc_iversion(new_inode); 9225 new_inode->i_ctime = current_time(new_inode); 9226 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == 9227 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 9228 ret = btrfs_unlink_subvol(trans, new_dir, new_dentry); 9229 BUG_ON(new_inode->i_nlink == 0); 9230 } else { 9231 ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9232 BTRFS_I(d_inode(new_dentry)), 9233 new_dentry->d_name.name, 9234 new_dentry->d_name.len); 9235 } 9236 if (!ret && new_inode->i_nlink == 0) 9237 ret = btrfs_orphan_add(trans, 9238 BTRFS_I(d_inode(new_dentry))); 9239 if (ret) { 9240 btrfs_abort_transaction(trans, ret); 9241 goto out_fail; 9242 } 9243 } 9244 9245 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9246 new_dentry->d_name.name, 9247 new_dentry->d_name.len, 0, index); 9248 if (ret) { 9249 btrfs_abort_transaction(trans, ret); 9250 goto out_fail; 9251 } 9252 9253 if (old_inode->i_nlink == 1) 9254 BTRFS_I(old_inode)->dir_index = index; 9255 9256 if (log_pinned) { 9257 struct dentry *parent = new_dentry->d_parent; 9258 9259 btrfs_init_log_ctx(&ctx, old_inode); 9260 ret = btrfs_log_new_name(trans, BTRFS_I(old_inode), 9261 BTRFS_I(old_dir), parent, 9262 false, &ctx); 9263 if (ret == BTRFS_NEED_LOG_SYNC) 9264 sync_log = true; 9265 else if (ret == BTRFS_NEED_TRANS_COMMIT) 9266 commit_transaction = true; 9267 ret = 0; 9268 btrfs_end_log_trans(root); 9269 log_pinned = false; 9270 } 9271 9272 if (flags & RENAME_WHITEOUT) { 9273 ret = btrfs_whiteout_for_rename(trans, root, old_dir, 9274 old_dentry); 9275 9276 if (ret) { 9277 btrfs_abort_transaction(trans, ret); 9278 goto out_fail; 9279 } 9280 } 9281 out_fail: 9282 /* 9283 * If we have pinned the log and an error happened, we unpin tasks 9284 * trying to sync the log and force them to fallback to a transaction 9285 * commit if the log currently contains any of the inodes involved in 9286 * this rename operation (to ensure we do not persist a log with an 9287 * inconsistent state for any of these inodes or leading to any 9288 * inconsistencies when replayed). If the transaction was aborted, the 9289 * abortion reason is propagated to userspace when attempting to commit 9290 * the transaction. If the log does not contain any of these inodes, we 9291 * allow the tasks to sync it. 9292 */ 9293 if (ret && log_pinned) { 9294 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9295 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9296 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9297 (new_inode && 9298 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9299 btrfs_set_log_full_commit(trans); 9300 9301 btrfs_end_log_trans(root); 9302 log_pinned = false; 9303 } 9304 if (!ret && sync_log) { 9305 ret = btrfs_sync_log(trans, BTRFS_I(old_inode)->root, &ctx); 9306 if (ret) 9307 commit_transaction = true; 9308 } else if (sync_log) { 9309 mutex_lock(&root->log_mutex); 9310 list_del(&ctx.list); 9311 mutex_unlock(&root->log_mutex); 9312 } 9313 if (commit_transaction) { 9314 ret = btrfs_commit_transaction(trans); 9315 } else { 9316 int ret2; 9317 9318 ret2 = btrfs_end_transaction(trans); 9319 ret = ret ? ret : ret2; 9320 } 9321 out_notrans: 9322 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9323 up_read(&fs_info->subvol_sem); 9324 9325 return ret; 9326 } 9327 9328 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, 9329 struct inode *new_dir, struct dentry *new_dentry, 9330 unsigned int flags) 9331 { 9332 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 9333 return -EINVAL; 9334 9335 if (flags & RENAME_EXCHANGE) 9336 return btrfs_rename_exchange(old_dir, old_dentry, new_dir, 9337 new_dentry); 9338 9339 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags); 9340 } 9341 9342 struct btrfs_delalloc_work { 9343 struct inode *inode; 9344 struct completion completion; 9345 struct list_head list; 9346 struct btrfs_work work; 9347 }; 9348 9349 static void btrfs_run_delalloc_work(struct btrfs_work *work) 9350 { 9351 struct btrfs_delalloc_work *delalloc_work; 9352 struct inode *inode; 9353 9354 delalloc_work = container_of(work, struct btrfs_delalloc_work, 9355 work); 9356 inode = delalloc_work->inode; 9357 filemap_flush(inode->i_mapping); 9358 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9359 &BTRFS_I(inode)->runtime_flags)) 9360 filemap_flush(inode->i_mapping); 9361 9362 iput(inode); 9363 complete(&delalloc_work->completion); 9364 } 9365 9366 static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode) 9367 { 9368 struct btrfs_delalloc_work *work; 9369 9370 work = kmalloc(sizeof(*work), GFP_NOFS); 9371 if (!work) 9372 return NULL; 9373 9374 init_completion(&work->completion); 9375 INIT_LIST_HEAD(&work->list); 9376 work->inode = inode; 9377 btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL, NULL); 9378 9379 return work; 9380 } 9381 9382 /* 9383 * some fairly slow code that needs optimization. This walks the list 9384 * of all the inodes with pending delalloc and forces them to disk. 9385 */ 9386 static int start_delalloc_inodes(struct btrfs_root *root, int nr, bool snapshot) 9387 { 9388 struct btrfs_inode *binode; 9389 struct inode *inode; 9390 struct btrfs_delalloc_work *work, *next; 9391 struct list_head works; 9392 struct list_head splice; 9393 int ret = 0; 9394 9395 INIT_LIST_HEAD(&works); 9396 INIT_LIST_HEAD(&splice); 9397 9398 mutex_lock(&root->delalloc_mutex); 9399 spin_lock(&root->delalloc_lock); 9400 list_splice_init(&root->delalloc_inodes, &splice); 9401 while (!list_empty(&splice)) { 9402 binode = list_entry(splice.next, struct btrfs_inode, 9403 delalloc_inodes); 9404 9405 list_move_tail(&binode->delalloc_inodes, 9406 &root->delalloc_inodes); 9407 inode = igrab(&binode->vfs_inode); 9408 if (!inode) { 9409 cond_resched_lock(&root->delalloc_lock); 9410 continue; 9411 } 9412 spin_unlock(&root->delalloc_lock); 9413 9414 if (snapshot) 9415 set_bit(BTRFS_INODE_SNAPSHOT_FLUSH, 9416 &binode->runtime_flags); 9417 work = btrfs_alloc_delalloc_work(inode); 9418 if (!work) { 9419 iput(inode); 9420 ret = -ENOMEM; 9421 goto out; 9422 } 9423 list_add_tail(&work->list, &works); 9424 btrfs_queue_work(root->fs_info->flush_workers, 9425 &work->work); 9426 ret++; 9427 if (nr != -1 && ret >= nr) 9428 goto out; 9429 cond_resched(); 9430 spin_lock(&root->delalloc_lock); 9431 } 9432 spin_unlock(&root->delalloc_lock); 9433 9434 out: 9435 list_for_each_entry_safe(work, next, &works, list) { 9436 list_del_init(&work->list); 9437 wait_for_completion(&work->completion); 9438 kfree(work); 9439 } 9440 9441 if (!list_empty(&splice)) { 9442 spin_lock(&root->delalloc_lock); 9443 list_splice_tail(&splice, &root->delalloc_inodes); 9444 spin_unlock(&root->delalloc_lock); 9445 } 9446 mutex_unlock(&root->delalloc_mutex); 9447 return ret; 9448 } 9449 9450 int btrfs_start_delalloc_snapshot(struct btrfs_root *root) 9451 { 9452 struct btrfs_fs_info *fs_info = root->fs_info; 9453 int ret; 9454 9455 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 9456 return -EROFS; 9457 9458 ret = start_delalloc_inodes(root, -1, true); 9459 if (ret > 0) 9460 ret = 0; 9461 return ret; 9462 } 9463 9464 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int nr) 9465 { 9466 struct btrfs_root *root; 9467 struct list_head splice; 9468 int ret; 9469 9470 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 9471 return -EROFS; 9472 9473 INIT_LIST_HEAD(&splice); 9474 9475 mutex_lock(&fs_info->delalloc_root_mutex); 9476 spin_lock(&fs_info->delalloc_root_lock); 9477 list_splice_init(&fs_info->delalloc_roots, &splice); 9478 while (!list_empty(&splice) && nr) { 9479 root = list_first_entry(&splice, struct btrfs_root, 9480 delalloc_root); 9481 root = btrfs_grab_root(root); 9482 BUG_ON(!root); 9483 list_move_tail(&root->delalloc_root, 9484 &fs_info->delalloc_roots); 9485 spin_unlock(&fs_info->delalloc_root_lock); 9486 9487 ret = start_delalloc_inodes(root, nr, false); 9488 btrfs_put_root(root); 9489 if (ret < 0) 9490 goto out; 9491 9492 if (nr != -1) { 9493 nr -= ret; 9494 WARN_ON(nr < 0); 9495 } 9496 spin_lock(&fs_info->delalloc_root_lock); 9497 } 9498 spin_unlock(&fs_info->delalloc_root_lock); 9499 9500 ret = 0; 9501 out: 9502 if (!list_empty(&splice)) { 9503 spin_lock(&fs_info->delalloc_root_lock); 9504 list_splice_tail(&splice, &fs_info->delalloc_roots); 9505 spin_unlock(&fs_info->delalloc_root_lock); 9506 } 9507 mutex_unlock(&fs_info->delalloc_root_mutex); 9508 return ret; 9509 } 9510 9511 static int btrfs_symlink(struct inode *dir, struct dentry *dentry, 9512 const char *symname) 9513 { 9514 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 9515 struct btrfs_trans_handle *trans; 9516 struct btrfs_root *root = BTRFS_I(dir)->root; 9517 struct btrfs_path *path; 9518 struct btrfs_key key; 9519 struct inode *inode = NULL; 9520 int err; 9521 u64 objectid; 9522 u64 index = 0; 9523 int name_len; 9524 int datasize; 9525 unsigned long ptr; 9526 struct btrfs_file_extent_item *ei; 9527 struct extent_buffer *leaf; 9528 9529 name_len = strlen(symname); 9530 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 9531 return -ENAMETOOLONG; 9532 9533 /* 9534 * 2 items for inode item and ref 9535 * 2 items for dir items 9536 * 1 item for updating parent inode item 9537 * 1 item for the inline extent item 9538 * 1 item for xattr if selinux is on 9539 */ 9540 trans = btrfs_start_transaction(root, 7); 9541 if (IS_ERR(trans)) 9542 return PTR_ERR(trans); 9543 9544 err = btrfs_find_free_ino(root, &objectid); 9545 if (err) 9546 goto out_unlock; 9547 9548 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 9549 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), 9550 objectid, S_IFLNK|S_IRWXUGO, &index); 9551 if (IS_ERR(inode)) { 9552 err = PTR_ERR(inode); 9553 inode = NULL; 9554 goto out_unlock; 9555 } 9556 9557 /* 9558 * If the active LSM wants to access the inode during 9559 * d_instantiate it needs these. Smack checks to see 9560 * if the filesystem supports xattrs by looking at the 9561 * ops vector. 9562 */ 9563 inode->i_fop = &btrfs_file_operations; 9564 inode->i_op = &btrfs_file_inode_operations; 9565 inode->i_mapping->a_ops = &btrfs_aops; 9566 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 9567 9568 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 9569 if (err) 9570 goto out_unlock; 9571 9572 path = btrfs_alloc_path(); 9573 if (!path) { 9574 err = -ENOMEM; 9575 goto out_unlock; 9576 } 9577 key.objectid = btrfs_ino(BTRFS_I(inode)); 9578 key.offset = 0; 9579 key.type = BTRFS_EXTENT_DATA_KEY; 9580 datasize = btrfs_file_extent_calc_inline_size(name_len); 9581 err = btrfs_insert_empty_item(trans, root, path, &key, 9582 datasize); 9583 if (err) { 9584 btrfs_free_path(path); 9585 goto out_unlock; 9586 } 9587 leaf = path->nodes[0]; 9588 ei = btrfs_item_ptr(leaf, path->slots[0], 9589 struct btrfs_file_extent_item); 9590 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 9591 btrfs_set_file_extent_type(leaf, ei, 9592 BTRFS_FILE_EXTENT_INLINE); 9593 btrfs_set_file_extent_encryption(leaf, ei, 0); 9594 btrfs_set_file_extent_compression(leaf, ei, 0); 9595 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 9596 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 9597 9598 ptr = btrfs_file_extent_inline_start(ei); 9599 write_extent_buffer(leaf, symname, ptr, name_len); 9600 btrfs_mark_buffer_dirty(leaf); 9601 btrfs_free_path(path); 9602 9603 inode->i_op = &btrfs_symlink_inode_operations; 9604 inode_nohighmem(inode); 9605 inode_set_bytes(inode, name_len); 9606 btrfs_i_size_write(BTRFS_I(inode), name_len); 9607 err = btrfs_update_inode(trans, root, inode); 9608 /* 9609 * Last step, add directory indexes for our symlink inode. This is the 9610 * last step to avoid extra cleanup of these indexes if an error happens 9611 * elsewhere above. 9612 */ 9613 if (!err) 9614 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 9615 BTRFS_I(inode), 0, index); 9616 if (err) 9617 goto out_unlock; 9618 9619 d_instantiate_new(dentry, inode); 9620 9621 out_unlock: 9622 btrfs_end_transaction(trans); 9623 if (err && inode) { 9624 inode_dec_link_count(inode); 9625 discard_new_inode(inode); 9626 } 9627 btrfs_btree_balance_dirty(fs_info); 9628 return err; 9629 } 9630 9631 static int insert_prealloc_file_extent(struct btrfs_trans_handle *trans, 9632 struct inode *inode, struct btrfs_key *ins, 9633 u64 file_offset) 9634 { 9635 struct btrfs_file_extent_item stack_fi; 9636 u64 start = ins->objectid; 9637 u64 len = ins->offset; 9638 int ret; 9639 9640 memset(&stack_fi, 0, sizeof(stack_fi)); 9641 9642 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC); 9643 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start); 9644 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len); 9645 btrfs_set_stack_file_extent_num_bytes(&stack_fi, len); 9646 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len); 9647 btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE); 9648 /* Encryption and other encoding is reserved and all 0 */ 9649 9650 ret = btrfs_qgroup_release_data(BTRFS_I(inode), file_offset, len); 9651 if (ret < 0) 9652 return ret; 9653 return insert_reserved_file_extent(trans, BTRFS_I(inode), file_offset, 9654 &stack_fi, ret); 9655 } 9656 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 9657 u64 start, u64 num_bytes, u64 min_size, 9658 loff_t actual_len, u64 *alloc_hint, 9659 struct btrfs_trans_handle *trans) 9660 { 9661 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9662 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 9663 struct extent_map *em; 9664 struct btrfs_root *root = BTRFS_I(inode)->root; 9665 struct btrfs_key ins; 9666 u64 cur_offset = start; 9667 u64 clear_offset = start; 9668 u64 i_size; 9669 u64 cur_bytes; 9670 u64 last_alloc = (u64)-1; 9671 int ret = 0; 9672 bool own_trans = true; 9673 u64 end = start + num_bytes - 1; 9674 9675 if (trans) 9676 own_trans = false; 9677 while (num_bytes > 0) { 9678 if (own_trans) { 9679 trans = btrfs_start_transaction(root, 3); 9680 if (IS_ERR(trans)) { 9681 ret = PTR_ERR(trans); 9682 break; 9683 } 9684 } 9685 9686 cur_bytes = min_t(u64, num_bytes, SZ_256M); 9687 cur_bytes = max(cur_bytes, min_size); 9688 /* 9689 * If we are severely fragmented we could end up with really 9690 * small allocations, so if the allocator is returning small 9691 * chunks lets make its job easier by only searching for those 9692 * sized chunks. 9693 */ 9694 cur_bytes = min(cur_bytes, last_alloc); 9695 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 9696 min_size, 0, *alloc_hint, &ins, 1, 0); 9697 if (ret) { 9698 if (own_trans) 9699 btrfs_end_transaction(trans); 9700 break; 9701 } 9702 9703 /* 9704 * We've reserved this space, and thus converted it from 9705 * ->bytes_may_use to ->bytes_reserved. Any error that happens 9706 * from here on out we will only need to clear our reservation 9707 * for the remaining unreserved area, so advance our 9708 * clear_offset by our extent size. 9709 */ 9710 clear_offset += ins.offset; 9711 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 9712 9713 last_alloc = ins.offset; 9714 ret = insert_prealloc_file_extent(trans, inode, &ins, cur_offset); 9715 if (ret) { 9716 btrfs_free_reserved_extent(fs_info, ins.objectid, 9717 ins.offset, 0); 9718 btrfs_abort_transaction(trans, ret); 9719 if (own_trans) 9720 btrfs_end_transaction(trans); 9721 break; 9722 } 9723 9724 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 9725 cur_offset + ins.offset -1, 0); 9726 9727 em = alloc_extent_map(); 9728 if (!em) { 9729 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 9730 &BTRFS_I(inode)->runtime_flags); 9731 goto next; 9732 } 9733 9734 em->start = cur_offset; 9735 em->orig_start = cur_offset; 9736 em->len = ins.offset; 9737 em->block_start = ins.objectid; 9738 em->block_len = ins.offset; 9739 em->orig_block_len = ins.offset; 9740 em->ram_bytes = ins.offset; 9741 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 9742 em->generation = trans->transid; 9743 9744 while (1) { 9745 write_lock(&em_tree->lock); 9746 ret = add_extent_mapping(em_tree, em, 1); 9747 write_unlock(&em_tree->lock); 9748 if (ret != -EEXIST) 9749 break; 9750 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 9751 cur_offset + ins.offset - 1, 9752 0); 9753 } 9754 free_extent_map(em); 9755 next: 9756 num_bytes -= ins.offset; 9757 cur_offset += ins.offset; 9758 *alloc_hint = ins.objectid + ins.offset; 9759 9760 inode_inc_iversion(inode); 9761 inode->i_ctime = current_time(inode); 9762 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 9763 if (!(mode & FALLOC_FL_KEEP_SIZE) && 9764 (actual_len > inode->i_size) && 9765 (cur_offset > inode->i_size)) { 9766 if (cur_offset > actual_len) 9767 i_size = actual_len; 9768 else 9769 i_size = cur_offset; 9770 i_size_write(inode, i_size); 9771 btrfs_inode_safe_disk_i_size_write(inode, 0); 9772 } 9773 9774 ret = btrfs_update_inode(trans, root, inode); 9775 9776 if (ret) { 9777 btrfs_abort_transaction(trans, ret); 9778 if (own_trans) 9779 btrfs_end_transaction(trans); 9780 break; 9781 } 9782 9783 if (own_trans) 9784 btrfs_end_transaction(trans); 9785 } 9786 if (clear_offset < end) 9787 btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset, 9788 end - clear_offset + 1); 9789 return ret; 9790 } 9791 9792 int btrfs_prealloc_file_range(struct inode *inode, int mode, 9793 u64 start, u64 num_bytes, u64 min_size, 9794 loff_t actual_len, u64 *alloc_hint) 9795 { 9796 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9797 min_size, actual_len, alloc_hint, 9798 NULL); 9799 } 9800 9801 int btrfs_prealloc_file_range_trans(struct inode *inode, 9802 struct btrfs_trans_handle *trans, int mode, 9803 u64 start, u64 num_bytes, u64 min_size, 9804 loff_t actual_len, u64 *alloc_hint) 9805 { 9806 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9807 min_size, actual_len, alloc_hint, trans); 9808 } 9809 9810 static int btrfs_set_page_dirty(struct page *page) 9811 { 9812 return __set_page_dirty_nobuffers(page); 9813 } 9814 9815 static int btrfs_permission(struct inode *inode, int mask) 9816 { 9817 struct btrfs_root *root = BTRFS_I(inode)->root; 9818 umode_t mode = inode->i_mode; 9819 9820 if (mask & MAY_WRITE && 9821 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 9822 if (btrfs_root_readonly(root)) 9823 return -EROFS; 9824 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 9825 return -EACCES; 9826 } 9827 return generic_permission(inode, mask); 9828 } 9829 9830 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 9831 { 9832 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 9833 struct btrfs_trans_handle *trans; 9834 struct btrfs_root *root = BTRFS_I(dir)->root; 9835 struct inode *inode = NULL; 9836 u64 objectid; 9837 u64 index; 9838 int ret = 0; 9839 9840 /* 9841 * 5 units required for adding orphan entry 9842 */ 9843 trans = btrfs_start_transaction(root, 5); 9844 if (IS_ERR(trans)) 9845 return PTR_ERR(trans); 9846 9847 ret = btrfs_find_free_ino(root, &objectid); 9848 if (ret) 9849 goto out; 9850 9851 inode = btrfs_new_inode(trans, root, dir, NULL, 0, 9852 btrfs_ino(BTRFS_I(dir)), objectid, mode, &index); 9853 if (IS_ERR(inode)) { 9854 ret = PTR_ERR(inode); 9855 inode = NULL; 9856 goto out; 9857 } 9858 9859 inode->i_fop = &btrfs_file_operations; 9860 inode->i_op = &btrfs_file_inode_operations; 9861 9862 inode->i_mapping->a_ops = &btrfs_aops; 9863 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 9864 9865 ret = btrfs_init_inode_security(trans, inode, dir, NULL); 9866 if (ret) 9867 goto out; 9868 9869 ret = btrfs_update_inode(trans, root, inode); 9870 if (ret) 9871 goto out; 9872 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 9873 if (ret) 9874 goto out; 9875 9876 /* 9877 * We set number of links to 0 in btrfs_new_inode(), and here we set 9878 * it to 1 because d_tmpfile() will issue a warning if the count is 0, 9879 * through: 9880 * 9881 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 9882 */ 9883 set_nlink(inode, 1); 9884 d_tmpfile(dentry, inode); 9885 unlock_new_inode(inode); 9886 mark_inode_dirty(inode); 9887 out: 9888 btrfs_end_transaction(trans); 9889 if (ret && inode) 9890 discard_new_inode(inode); 9891 btrfs_btree_balance_dirty(fs_info); 9892 return ret; 9893 } 9894 9895 void btrfs_set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end) 9896 { 9897 struct inode *inode = tree->private_data; 9898 unsigned long index = start >> PAGE_SHIFT; 9899 unsigned long end_index = end >> PAGE_SHIFT; 9900 struct page *page; 9901 9902 while (index <= end_index) { 9903 page = find_get_page(inode->i_mapping, index); 9904 ASSERT(page); /* Pages should be in the extent_io_tree */ 9905 set_page_writeback(page); 9906 put_page(page); 9907 index++; 9908 } 9909 } 9910 9911 #ifdef CONFIG_SWAP 9912 /* 9913 * Add an entry indicating a block group or device which is pinned by a 9914 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a 9915 * negative errno on failure. 9916 */ 9917 static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr, 9918 bool is_block_group) 9919 { 9920 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 9921 struct btrfs_swapfile_pin *sp, *entry; 9922 struct rb_node **p; 9923 struct rb_node *parent = NULL; 9924 9925 sp = kmalloc(sizeof(*sp), GFP_NOFS); 9926 if (!sp) 9927 return -ENOMEM; 9928 sp->ptr = ptr; 9929 sp->inode = inode; 9930 sp->is_block_group = is_block_group; 9931 9932 spin_lock(&fs_info->swapfile_pins_lock); 9933 p = &fs_info->swapfile_pins.rb_node; 9934 while (*p) { 9935 parent = *p; 9936 entry = rb_entry(parent, struct btrfs_swapfile_pin, node); 9937 if (sp->ptr < entry->ptr || 9938 (sp->ptr == entry->ptr && sp->inode < entry->inode)) { 9939 p = &(*p)->rb_left; 9940 } else if (sp->ptr > entry->ptr || 9941 (sp->ptr == entry->ptr && sp->inode > entry->inode)) { 9942 p = &(*p)->rb_right; 9943 } else { 9944 spin_unlock(&fs_info->swapfile_pins_lock); 9945 kfree(sp); 9946 return 1; 9947 } 9948 } 9949 rb_link_node(&sp->node, parent, p); 9950 rb_insert_color(&sp->node, &fs_info->swapfile_pins); 9951 spin_unlock(&fs_info->swapfile_pins_lock); 9952 return 0; 9953 } 9954 9955 /* Free all of the entries pinned by this swapfile. */ 9956 static void btrfs_free_swapfile_pins(struct inode *inode) 9957 { 9958 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 9959 struct btrfs_swapfile_pin *sp; 9960 struct rb_node *node, *next; 9961 9962 spin_lock(&fs_info->swapfile_pins_lock); 9963 node = rb_first(&fs_info->swapfile_pins); 9964 while (node) { 9965 next = rb_next(node); 9966 sp = rb_entry(node, struct btrfs_swapfile_pin, node); 9967 if (sp->inode == inode) { 9968 rb_erase(&sp->node, &fs_info->swapfile_pins); 9969 if (sp->is_block_group) 9970 btrfs_put_block_group(sp->ptr); 9971 kfree(sp); 9972 } 9973 node = next; 9974 } 9975 spin_unlock(&fs_info->swapfile_pins_lock); 9976 } 9977 9978 struct btrfs_swap_info { 9979 u64 start; 9980 u64 block_start; 9981 u64 block_len; 9982 u64 lowest_ppage; 9983 u64 highest_ppage; 9984 unsigned long nr_pages; 9985 int nr_extents; 9986 }; 9987 9988 static int btrfs_add_swap_extent(struct swap_info_struct *sis, 9989 struct btrfs_swap_info *bsi) 9990 { 9991 unsigned long nr_pages; 9992 u64 first_ppage, first_ppage_reported, next_ppage; 9993 int ret; 9994 9995 first_ppage = ALIGN(bsi->block_start, PAGE_SIZE) >> PAGE_SHIFT; 9996 next_ppage = ALIGN_DOWN(bsi->block_start + bsi->block_len, 9997 PAGE_SIZE) >> PAGE_SHIFT; 9998 9999 if (first_ppage >= next_ppage) 10000 return 0; 10001 nr_pages = next_ppage - first_ppage; 10002 10003 first_ppage_reported = first_ppage; 10004 if (bsi->start == 0) 10005 first_ppage_reported++; 10006 if (bsi->lowest_ppage > first_ppage_reported) 10007 bsi->lowest_ppage = first_ppage_reported; 10008 if (bsi->highest_ppage < (next_ppage - 1)) 10009 bsi->highest_ppage = next_ppage - 1; 10010 10011 ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage); 10012 if (ret < 0) 10013 return ret; 10014 bsi->nr_extents += ret; 10015 bsi->nr_pages += nr_pages; 10016 return 0; 10017 } 10018 10019 static void btrfs_swap_deactivate(struct file *file) 10020 { 10021 struct inode *inode = file_inode(file); 10022 10023 btrfs_free_swapfile_pins(inode); 10024 atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles); 10025 } 10026 10027 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10028 sector_t *span) 10029 { 10030 struct inode *inode = file_inode(file); 10031 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10032 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 10033 struct extent_state *cached_state = NULL; 10034 struct extent_map *em = NULL; 10035 struct btrfs_device *device = NULL; 10036 struct btrfs_swap_info bsi = { 10037 .lowest_ppage = (sector_t)-1ULL, 10038 }; 10039 int ret = 0; 10040 u64 isize; 10041 u64 start; 10042 10043 /* 10044 * If the swap file was just created, make sure delalloc is done. If the 10045 * file changes again after this, the user is doing something stupid and 10046 * we don't really care. 10047 */ 10048 ret = btrfs_wait_ordered_range(inode, 0, (u64)-1); 10049 if (ret) 10050 return ret; 10051 10052 /* 10053 * The inode is locked, so these flags won't change after we check them. 10054 */ 10055 if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) { 10056 btrfs_warn(fs_info, "swapfile must not be compressed"); 10057 return -EINVAL; 10058 } 10059 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) { 10060 btrfs_warn(fs_info, "swapfile must not be copy-on-write"); 10061 return -EINVAL; 10062 } 10063 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { 10064 btrfs_warn(fs_info, "swapfile must not be checksummed"); 10065 return -EINVAL; 10066 } 10067 10068 /* 10069 * Balance or device remove/replace/resize can move stuff around from 10070 * under us. The EXCL_OP flag makes sure they aren't running/won't run 10071 * concurrently while we are mapping the swap extents, and 10072 * fs_info->swapfile_pins prevents them from running while the swap file 10073 * is active and moving the extents. Note that this also prevents a 10074 * concurrent device add which isn't actually necessary, but it's not 10075 * really worth the trouble to allow it. 10076 */ 10077 if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) { 10078 btrfs_warn(fs_info, 10079 "cannot activate swapfile while exclusive operation is running"); 10080 return -EBUSY; 10081 } 10082 /* 10083 * Snapshots can create extents which require COW even if NODATACOW is 10084 * set. We use this counter to prevent snapshots. We must increment it 10085 * before walking the extents because we don't want a concurrent 10086 * snapshot to run after we've already checked the extents. 10087 */ 10088 atomic_inc(&BTRFS_I(inode)->root->nr_swapfiles); 10089 10090 isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); 10091 10092 lock_extent_bits(io_tree, 0, isize - 1, &cached_state); 10093 start = 0; 10094 while (start < isize) { 10095 u64 logical_block_start, physical_block_start; 10096 struct btrfs_block_group *bg; 10097 u64 len = isize - start; 10098 10099 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len); 10100 if (IS_ERR(em)) { 10101 ret = PTR_ERR(em); 10102 goto out; 10103 } 10104 10105 if (em->block_start == EXTENT_MAP_HOLE) { 10106 btrfs_warn(fs_info, "swapfile must not have holes"); 10107 ret = -EINVAL; 10108 goto out; 10109 } 10110 if (em->block_start == EXTENT_MAP_INLINE) { 10111 /* 10112 * It's unlikely we'll ever actually find ourselves 10113 * here, as a file small enough to fit inline won't be 10114 * big enough to store more than the swap header, but in 10115 * case something changes in the future, let's catch it 10116 * here rather than later. 10117 */ 10118 btrfs_warn(fs_info, "swapfile must not be inline"); 10119 ret = -EINVAL; 10120 goto out; 10121 } 10122 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 10123 btrfs_warn(fs_info, "swapfile must not be compressed"); 10124 ret = -EINVAL; 10125 goto out; 10126 } 10127 10128 logical_block_start = em->block_start + (start - em->start); 10129 len = min(len, em->len - (start - em->start)); 10130 free_extent_map(em); 10131 em = NULL; 10132 10133 ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL); 10134 if (ret < 0) { 10135 goto out; 10136 } else if (ret) { 10137 ret = 0; 10138 } else { 10139 btrfs_warn(fs_info, 10140 "swapfile must not be copy-on-write"); 10141 ret = -EINVAL; 10142 goto out; 10143 } 10144 10145 em = btrfs_get_chunk_map(fs_info, logical_block_start, len); 10146 if (IS_ERR(em)) { 10147 ret = PTR_ERR(em); 10148 goto out; 10149 } 10150 10151 if (em->map_lookup->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { 10152 btrfs_warn(fs_info, 10153 "swapfile must have single data profile"); 10154 ret = -EINVAL; 10155 goto out; 10156 } 10157 10158 if (device == NULL) { 10159 device = em->map_lookup->stripes[0].dev; 10160 ret = btrfs_add_swapfile_pin(inode, device, false); 10161 if (ret == 1) 10162 ret = 0; 10163 else if (ret) 10164 goto out; 10165 } else if (device != em->map_lookup->stripes[0].dev) { 10166 btrfs_warn(fs_info, "swapfile must be on one device"); 10167 ret = -EINVAL; 10168 goto out; 10169 } 10170 10171 physical_block_start = (em->map_lookup->stripes[0].physical + 10172 (logical_block_start - em->start)); 10173 len = min(len, em->len - (logical_block_start - em->start)); 10174 free_extent_map(em); 10175 em = NULL; 10176 10177 bg = btrfs_lookup_block_group(fs_info, logical_block_start); 10178 if (!bg) { 10179 btrfs_warn(fs_info, 10180 "could not find block group containing swapfile"); 10181 ret = -EINVAL; 10182 goto out; 10183 } 10184 10185 ret = btrfs_add_swapfile_pin(inode, bg, true); 10186 if (ret) { 10187 btrfs_put_block_group(bg); 10188 if (ret == 1) 10189 ret = 0; 10190 else 10191 goto out; 10192 } 10193 10194 if (bsi.block_len && 10195 bsi.block_start + bsi.block_len == physical_block_start) { 10196 bsi.block_len += len; 10197 } else { 10198 if (bsi.block_len) { 10199 ret = btrfs_add_swap_extent(sis, &bsi); 10200 if (ret) 10201 goto out; 10202 } 10203 bsi.start = start; 10204 bsi.block_start = physical_block_start; 10205 bsi.block_len = len; 10206 } 10207 10208 start += len; 10209 } 10210 10211 if (bsi.block_len) 10212 ret = btrfs_add_swap_extent(sis, &bsi); 10213 10214 out: 10215 if (!IS_ERR_OR_NULL(em)) 10216 free_extent_map(em); 10217 10218 unlock_extent_cached(io_tree, 0, isize - 1, &cached_state); 10219 10220 if (ret) 10221 btrfs_swap_deactivate(file); 10222 10223 clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags); 10224 10225 if (ret) 10226 return ret; 10227 10228 if (device) 10229 sis->bdev = device->bdev; 10230 *span = bsi.highest_ppage - bsi.lowest_ppage + 1; 10231 sis->max = bsi.nr_pages; 10232 sis->pages = bsi.nr_pages - 1; 10233 sis->highest_bit = bsi.nr_pages - 1; 10234 return bsi.nr_extents; 10235 } 10236 #else 10237 static void btrfs_swap_deactivate(struct file *file) 10238 { 10239 } 10240 10241 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10242 sector_t *span) 10243 { 10244 return -EOPNOTSUPP; 10245 } 10246 #endif 10247 10248 static const struct inode_operations btrfs_dir_inode_operations = { 10249 .getattr = btrfs_getattr, 10250 .lookup = btrfs_lookup, 10251 .create = btrfs_create, 10252 .unlink = btrfs_unlink, 10253 .link = btrfs_link, 10254 .mkdir = btrfs_mkdir, 10255 .rmdir = btrfs_rmdir, 10256 .rename = btrfs_rename2, 10257 .symlink = btrfs_symlink, 10258 .setattr = btrfs_setattr, 10259 .mknod = btrfs_mknod, 10260 .listxattr = btrfs_listxattr, 10261 .permission = btrfs_permission, 10262 .get_acl = btrfs_get_acl, 10263 .set_acl = btrfs_set_acl, 10264 .update_time = btrfs_update_time, 10265 .tmpfile = btrfs_tmpfile, 10266 }; 10267 10268 static const struct file_operations btrfs_dir_file_operations = { 10269 .llseek = generic_file_llseek, 10270 .read = generic_read_dir, 10271 .iterate_shared = btrfs_real_readdir, 10272 .open = btrfs_opendir, 10273 .unlocked_ioctl = btrfs_ioctl, 10274 #ifdef CONFIG_COMPAT 10275 .compat_ioctl = btrfs_compat_ioctl, 10276 #endif 10277 .release = btrfs_release_file, 10278 .fsync = btrfs_sync_file, 10279 }; 10280 10281 static const struct extent_io_ops btrfs_extent_io_ops = { 10282 /* mandatory callbacks */ 10283 .submit_bio_hook = btrfs_submit_bio_hook, 10284 .readpage_end_io_hook = btrfs_readpage_end_io_hook, 10285 }; 10286 10287 /* 10288 * btrfs doesn't support the bmap operation because swapfiles 10289 * use bmap to make a mapping of extents in the file. They assume 10290 * these extents won't change over the life of the file and they 10291 * use the bmap result to do IO directly to the drive. 10292 * 10293 * the btrfs bmap call would return logical addresses that aren't 10294 * suitable for IO and they also will change frequently as COW 10295 * operations happen. So, swapfile + btrfs == corruption. 10296 * 10297 * For now we're avoiding this by dropping bmap. 10298 */ 10299 static const struct address_space_operations btrfs_aops = { 10300 .readpage = btrfs_readpage, 10301 .writepage = btrfs_writepage, 10302 .writepages = btrfs_writepages, 10303 .readahead = btrfs_readahead, 10304 .direct_IO = btrfs_direct_IO, 10305 .invalidatepage = btrfs_invalidatepage, 10306 .releasepage = btrfs_releasepage, 10307 #ifdef CONFIG_MIGRATION 10308 .migratepage = btrfs_migratepage, 10309 #endif 10310 .set_page_dirty = btrfs_set_page_dirty, 10311 .error_remove_page = generic_error_remove_page, 10312 .swap_activate = btrfs_swap_activate, 10313 .swap_deactivate = btrfs_swap_deactivate, 10314 }; 10315 10316 static const struct inode_operations btrfs_file_inode_operations = { 10317 .getattr = btrfs_getattr, 10318 .setattr = btrfs_setattr, 10319 .listxattr = btrfs_listxattr, 10320 .permission = btrfs_permission, 10321 .fiemap = btrfs_fiemap, 10322 .get_acl = btrfs_get_acl, 10323 .set_acl = btrfs_set_acl, 10324 .update_time = btrfs_update_time, 10325 }; 10326 static const struct inode_operations btrfs_special_inode_operations = { 10327 .getattr = btrfs_getattr, 10328 .setattr = btrfs_setattr, 10329 .permission = btrfs_permission, 10330 .listxattr = btrfs_listxattr, 10331 .get_acl = btrfs_get_acl, 10332 .set_acl = btrfs_set_acl, 10333 .update_time = btrfs_update_time, 10334 }; 10335 static const struct inode_operations btrfs_symlink_inode_operations = { 10336 .get_link = page_get_link, 10337 .getattr = btrfs_getattr, 10338 .setattr = btrfs_setattr, 10339 .permission = btrfs_permission, 10340 .listxattr = btrfs_listxattr, 10341 .update_time = btrfs_update_time, 10342 }; 10343 10344 const struct dentry_operations btrfs_dentry_operations = { 10345 .d_delete = btrfs_dentry_delete, 10346 }; 10347