1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 */ 5 6 #include <crypto/hash.h> 7 #include <linux/kernel.h> 8 #include <linux/bio.h> 9 #include <linux/file.h> 10 #include <linux/fs.h> 11 #include <linux/pagemap.h> 12 #include <linux/highmem.h> 13 #include <linux/time.h> 14 #include <linux/init.h> 15 #include <linux/string.h> 16 #include <linux/backing-dev.h> 17 #include <linux/writeback.h> 18 #include <linux/compat.h> 19 #include <linux/xattr.h> 20 #include <linux/posix_acl.h> 21 #include <linux/falloc.h> 22 #include <linux/slab.h> 23 #include <linux/ratelimit.h> 24 #include <linux/btrfs.h> 25 #include <linux/blkdev.h> 26 #include <linux/posix_acl_xattr.h> 27 #include <linux/uio.h> 28 #include <linux/magic.h> 29 #include <linux/iversion.h> 30 #include <linux/swap.h> 31 #include <linux/migrate.h> 32 #include <linux/sched/mm.h> 33 #include <linux/iomap.h> 34 #include <asm/unaligned.h> 35 #include "misc.h" 36 #include "ctree.h" 37 #include "disk-io.h" 38 #include "transaction.h" 39 #include "btrfs_inode.h" 40 #include "print-tree.h" 41 #include "ordered-data.h" 42 #include "xattr.h" 43 #include "tree-log.h" 44 #include "volumes.h" 45 #include "compression.h" 46 #include "locking.h" 47 #include "free-space-cache.h" 48 #include "props.h" 49 #include "qgroup.h" 50 #include "delalloc-space.h" 51 #include "block-group.h" 52 #include "space-info.h" 53 #include "zoned.h" 54 55 struct btrfs_iget_args { 56 u64 ino; 57 struct btrfs_root *root; 58 }; 59 60 struct btrfs_dio_data { 61 u64 reserve; 62 loff_t length; 63 ssize_t submitted; 64 struct extent_changeset *data_reserved; 65 }; 66 67 static const struct inode_operations btrfs_dir_inode_operations; 68 static const struct inode_operations btrfs_symlink_inode_operations; 69 static const struct inode_operations btrfs_special_inode_operations; 70 static const struct inode_operations btrfs_file_inode_operations; 71 static const struct address_space_operations btrfs_aops; 72 static const struct file_operations btrfs_dir_file_operations; 73 74 static struct kmem_cache *btrfs_inode_cachep; 75 struct kmem_cache *btrfs_trans_handle_cachep; 76 struct kmem_cache *btrfs_path_cachep; 77 struct kmem_cache *btrfs_free_space_cachep; 78 struct kmem_cache *btrfs_free_space_bitmap_cachep; 79 80 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 81 static int btrfs_truncate(struct inode *inode, bool skip_writeback); 82 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); 83 static noinline int cow_file_range(struct btrfs_inode *inode, 84 struct page *locked_page, 85 u64 start, u64 end, int *page_started, 86 unsigned long *nr_written, int unlock); 87 static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, 88 u64 len, u64 orig_start, u64 block_start, 89 u64 block_len, u64 orig_block_len, 90 u64 ram_bytes, int compress_type, 91 int type); 92 93 static void __endio_write_update_ordered(struct btrfs_inode *inode, 94 const u64 offset, const u64 bytes, 95 const bool uptodate); 96 97 /* 98 * btrfs_inode_lock - lock inode i_rwsem based on arguments passed 99 * 100 * ilock_flags can have the following bit set: 101 * 102 * BTRFS_ILOCK_SHARED - acquire a shared lock on the inode 103 * BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt 104 * return -EAGAIN 105 */ 106 int btrfs_inode_lock(struct inode *inode, unsigned int ilock_flags) 107 { 108 if (ilock_flags & BTRFS_ILOCK_SHARED) { 109 if (ilock_flags & BTRFS_ILOCK_TRY) { 110 if (!inode_trylock_shared(inode)) 111 return -EAGAIN; 112 else 113 return 0; 114 } 115 inode_lock_shared(inode); 116 } else { 117 if (ilock_flags & BTRFS_ILOCK_TRY) { 118 if (!inode_trylock(inode)) 119 return -EAGAIN; 120 else 121 return 0; 122 } 123 inode_lock(inode); 124 } 125 return 0; 126 } 127 128 /* 129 * btrfs_inode_unlock - unock inode i_rwsem 130 * 131 * ilock_flags should contain the same bits set as passed to btrfs_inode_lock() 132 * to decide whether the lock acquired is shared or exclusive. 133 */ 134 void btrfs_inode_unlock(struct inode *inode, unsigned int ilock_flags) 135 { 136 if (ilock_flags & BTRFS_ILOCK_SHARED) 137 inode_unlock_shared(inode); 138 else 139 inode_unlock(inode); 140 } 141 142 /* 143 * Cleanup all submitted ordered extents in specified range to handle errors 144 * from the btrfs_run_delalloc_range() callback. 145 * 146 * NOTE: caller must ensure that when an error happens, it can not call 147 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING 148 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata 149 * to be released, which we want to happen only when finishing the ordered 150 * extent (btrfs_finish_ordered_io()). 151 */ 152 static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, 153 struct page *locked_page, 154 u64 offset, u64 bytes) 155 { 156 unsigned long index = offset >> PAGE_SHIFT; 157 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT; 158 u64 page_start = page_offset(locked_page); 159 u64 page_end = page_start + PAGE_SIZE - 1; 160 161 struct page *page; 162 163 while (index <= end_index) { 164 page = find_get_page(inode->vfs_inode.i_mapping, index); 165 index++; 166 if (!page) 167 continue; 168 ClearPagePrivate2(page); 169 put_page(page); 170 } 171 172 /* 173 * In case this page belongs to the delalloc range being instantiated 174 * then skip it, since the first page of a range is going to be 175 * properly cleaned up by the caller of run_delalloc_range 176 */ 177 if (page_start >= offset && page_end <= (offset + bytes - 1)) { 178 offset += PAGE_SIZE; 179 bytes -= PAGE_SIZE; 180 } 181 182 return __endio_write_update_ordered(inode, offset, bytes, false); 183 } 184 185 static int btrfs_dirty_inode(struct inode *inode); 186 187 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 188 struct inode *inode, struct inode *dir, 189 const struct qstr *qstr) 190 { 191 int err; 192 193 err = btrfs_init_acl(trans, inode, dir); 194 if (!err) 195 err = btrfs_xattr_security_init(trans, inode, dir, qstr); 196 return err; 197 } 198 199 /* 200 * this does all the hard work for inserting an inline extent into 201 * the btree. The caller should have done a btrfs_drop_extents so that 202 * no overlapping inline items exist in the btree 203 */ 204 static int insert_inline_extent(struct btrfs_trans_handle *trans, 205 struct btrfs_path *path, bool extent_inserted, 206 struct btrfs_root *root, struct inode *inode, 207 u64 start, size_t size, size_t compressed_size, 208 int compress_type, 209 struct page **compressed_pages) 210 { 211 struct extent_buffer *leaf; 212 struct page *page = NULL; 213 char *kaddr; 214 unsigned long ptr; 215 struct btrfs_file_extent_item *ei; 216 int ret; 217 size_t cur_size = size; 218 unsigned long offset; 219 220 ASSERT((compressed_size > 0 && compressed_pages) || 221 (compressed_size == 0 && !compressed_pages)); 222 223 if (compressed_size && compressed_pages) 224 cur_size = compressed_size; 225 226 if (!extent_inserted) { 227 struct btrfs_key key; 228 size_t datasize; 229 230 key.objectid = btrfs_ino(BTRFS_I(inode)); 231 key.offset = start; 232 key.type = BTRFS_EXTENT_DATA_KEY; 233 234 datasize = btrfs_file_extent_calc_inline_size(cur_size); 235 ret = btrfs_insert_empty_item(trans, root, path, &key, 236 datasize); 237 if (ret) 238 goto fail; 239 } 240 leaf = path->nodes[0]; 241 ei = btrfs_item_ptr(leaf, path->slots[0], 242 struct btrfs_file_extent_item); 243 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 244 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 245 btrfs_set_file_extent_encryption(leaf, ei, 0); 246 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 247 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 248 ptr = btrfs_file_extent_inline_start(ei); 249 250 if (compress_type != BTRFS_COMPRESS_NONE) { 251 struct page *cpage; 252 int i = 0; 253 while (compressed_size > 0) { 254 cpage = compressed_pages[i]; 255 cur_size = min_t(unsigned long, compressed_size, 256 PAGE_SIZE); 257 258 kaddr = kmap_atomic(cpage); 259 write_extent_buffer(leaf, kaddr, ptr, cur_size); 260 kunmap_atomic(kaddr); 261 262 i++; 263 ptr += cur_size; 264 compressed_size -= cur_size; 265 } 266 btrfs_set_file_extent_compression(leaf, ei, 267 compress_type); 268 } else { 269 page = find_get_page(inode->i_mapping, 270 start >> PAGE_SHIFT); 271 btrfs_set_file_extent_compression(leaf, ei, 0); 272 kaddr = kmap_atomic(page); 273 offset = offset_in_page(start); 274 write_extent_buffer(leaf, kaddr + offset, ptr, size); 275 kunmap_atomic(kaddr); 276 put_page(page); 277 } 278 btrfs_mark_buffer_dirty(leaf); 279 btrfs_release_path(path); 280 281 /* 282 * We align size to sectorsize for inline extents just for simplicity 283 * sake. 284 */ 285 size = ALIGN(size, root->fs_info->sectorsize); 286 ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), start, size); 287 if (ret) 288 goto fail; 289 290 /* 291 * we're an inline extent, so nobody can 292 * extend the file past i_size without locking 293 * a page we already have locked. 294 * 295 * We must do any isize and inode updates 296 * before we unlock the pages. Otherwise we 297 * could end up racing with unlink. 298 */ 299 BTRFS_I(inode)->disk_i_size = inode->i_size; 300 fail: 301 return ret; 302 } 303 304 305 /* 306 * conditionally insert an inline extent into the file. This 307 * does the checks required to make sure the data is small enough 308 * to fit as an inline extent. 309 */ 310 static noinline int cow_file_range_inline(struct btrfs_inode *inode, u64 start, 311 u64 end, size_t compressed_size, 312 int compress_type, 313 struct page **compressed_pages) 314 { 315 struct btrfs_drop_extents_args drop_args = { 0 }; 316 struct btrfs_root *root = inode->root; 317 struct btrfs_fs_info *fs_info = root->fs_info; 318 struct btrfs_trans_handle *trans; 319 u64 isize = i_size_read(&inode->vfs_inode); 320 u64 actual_end = min(end + 1, isize); 321 u64 inline_len = actual_end - start; 322 u64 aligned_end = ALIGN(end, fs_info->sectorsize); 323 u64 data_len = inline_len; 324 int ret; 325 struct btrfs_path *path; 326 327 if (compressed_size) 328 data_len = compressed_size; 329 330 if (start > 0 || 331 actual_end > fs_info->sectorsize || 332 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 333 (!compressed_size && 334 (actual_end & (fs_info->sectorsize - 1)) == 0) || 335 end + 1 < isize || 336 data_len > fs_info->max_inline) { 337 return 1; 338 } 339 340 path = btrfs_alloc_path(); 341 if (!path) 342 return -ENOMEM; 343 344 trans = btrfs_join_transaction(root); 345 if (IS_ERR(trans)) { 346 btrfs_free_path(path); 347 return PTR_ERR(trans); 348 } 349 trans->block_rsv = &inode->block_rsv; 350 351 drop_args.path = path; 352 drop_args.start = start; 353 drop_args.end = aligned_end; 354 drop_args.drop_cache = true; 355 drop_args.replace_extent = true; 356 357 if (compressed_size && compressed_pages) 358 drop_args.extent_item_size = btrfs_file_extent_calc_inline_size( 359 compressed_size); 360 else 361 drop_args.extent_item_size = btrfs_file_extent_calc_inline_size( 362 inline_len); 363 364 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 365 if (ret) { 366 btrfs_abort_transaction(trans, ret); 367 goto out; 368 } 369 370 if (isize > actual_end) 371 inline_len = min_t(u64, isize, actual_end); 372 ret = insert_inline_extent(trans, path, drop_args.extent_inserted, 373 root, &inode->vfs_inode, start, 374 inline_len, compressed_size, 375 compress_type, compressed_pages); 376 if (ret && ret != -ENOSPC) { 377 btrfs_abort_transaction(trans, ret); 378 goto out; 379 } else if (ret == -ENOSPC) { 380 ret = 1; 381 goto out; 382 } 383 384 btrfs_update_inode_bytes(inode, inline_len, drop_args.bytes_found); 385 ret = btrfs_update_inode(trans, root, inode); 386 if (ret && ret != -ENOSPC) { 387 btrfs_abort_transaction(trans, ret); 388 goto out; 389 } else if (ret == -ENOSPC) { 390 ret = 1; 391 goto out; 392 } 393 394 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); 395 out: 396 /* 397 * Don't forget to free the reserved space, as for inlined extent 398 * it won't count as data extent, free them directly here. 399 * And at reserve time, it's always aligned to page size, so 400 * just free one page here. 401 */ 402 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE); 403 btrfs_free_path(path); 404 btrfs_end_transaction(trans); 405 return ret; 406 } 407 408 struct async_extent { 409 u64 start; 410 u64 ram_size; 411 u64 compressed_size; 412 struct page **pages; 413 unsigned long nr_pages; 414 int compress_type; 415 struct list_head list; 416 }; 417 418 struct async_chunk { 419 struct inode *inode; 420 struct page *locked_page; 421 u64 start; 422 u64 end; 423 unsigned int write_flags; 424 struct list_head extents; 425 struct cgroup_subsys_state *blkcg_css; 426 struct btrfs_work work; 427 atomic_t *pending; 428 }; 429 430 struct async_cow { 431 /* Number of chunks in flight; must be first in the structure */ 432 atomic_t num_chunks; 433 struct async_chunk chunks[]; 434 }; 435 436 static noinline int add_async_extent(struct async_chunk *cow, 437 u64 start, u64 ram_size, 438 u64 compressed_size, 439 struct page **pages, 440 unsigned long nr_pages, 441 int compress_type) 442 { 443 struct async_extent *async_extent; 444 445 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 446 BUG_ON(!async_extent); /* -ENOMEM */ 447 async_extent->start = start; 448 async_extent->ram_size = ram_size; 449 async_extent->compressed_size = compressed_size; 450 async_extent->pages = pages; 451 async_extent->nr_pages = nr_pages; 452 async_extent->compress_type = compress_type; 453 list_add_tail(&async_extent->list, &cow->extents); 454 return 0; 455 } 456 457 /* 458 * Check if the inode has flags compatible with compression 459 */ 460 static inline bool inode_can_compress(struct btrfs_inode *inode) 461 { 462 if (inode->flags & BTRFS_INODE_NODATACOW || 463 inode->flags & BTRFS_INODE_NODATASUM) 464 return false; 465 return true; 466 } 467 468 /* 469 * Check if the inode needs to be submitted to compression, based on mount 470 * options, defragmentation, properties or heuristics. 471 */ 472 static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, 473 u64 end) 474 { 475 struct btrfs_fs_info *fs_info = inode->root->fs_info; 476 477 if (!inode_can_compress(inode)) { 478 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG), 479 KERN_ERR "BTRFS: unexpected compression for ino %llu\n", 480 btrfs_ino(inode)); 481 return 0; 482 } 483 /* force compress */ 484 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 485 return 1; 486 /* defrag ioctl */ 487 if (inode->defrag_compress) 488 return 1; 489 /* bad compression ratios */ 490 if (inode->flags & BTRFS_INODE_NOCOMPRESS) 491 return 0; 492 if (btrfs_test_opt(fs_info, COMPRESS) || 493 inode->flags & BTRFS_INODE_COMPRESS || 494 inode->prop_compress) 495 return btrfs_compress_heuristic(&inode->vfs_inode, start, end); 496 return 0; 497 } 498 499 static inline void inode_should_defrag(struct btrfs_inode *inode, 500 u64 start, u64 end, u64 num_bytes, u64 small_write) 501 { 502 /* If this is a small write inside eof, kick off a defrag */ 503 if (num_bytes < small_write && 504 (start > 0 || end + 1 < inode->disk_i_size)) 505 btrfs_add_inode_defrag(NULL, inode); 506 } 507 508 /* 509 * we create compressed extents in two phases. The first 510 * phase compresses a range of pages that have already been 511 * locked (both pages and state bits are locked). 512 * 513 * This is done inside an ordered work queue, and the compression 514 * is spread across many cpus. The actual IO submission is step 515 * two, and the ordered work queue takes care of making sure that 516 * happens in the same order things were put onto the queue by 517 * writepages and friends. 518 * 519 * If this code finds it can't get good compression, it puts an 520 * entry onto the work queue to write the uncompressed bytes. This 521 * makes sure that both compressed inodes and uncompressed inodes 522 * are written in the same order that the flusher thread sent them 523 * down. 524 */ 525 static noinline int compress_file_range(struct async_chunk *async_chunk) 526 { 527 struct inode *inode = async_chunk->inode; 528 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 529 u64 blocksize = fs_info->sectorsize; 530 u64 start = async_chunk->start; 531 u64 end = async_chunk->end; 532 u64 actual_end; 533 u64 i_size; 534 int ret = 0; 535 struct page **pages = NULL; 536 unsigned long nr_pages; 537 unsigned long total_compressed = 0; 538 unsigned long total_in = 0; 539 int i; 540 int will_compress; 541 int compress_type = fs_info->compress_type; 542 int compressed_extents = 0; 543 int redirty = 0; 544 545 inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1, 546 SZ_16K); 547 548 /* 549 * We need to save i_size before now because it could change in between 550 * us evaluating the size and assigning it. This is because we lock and 551 * unlock the page in truncate and fallocate, and then modify the i_size 552 * later on. 553 * 554 * The barriers are to emulate READ_ONCE, remove that once i_size_read 555 * does that for us. 556 */ 557 barrier(); 558 i_size = i_size_read(inode); 559 barrier(); 560 actual_end = min_t(u64, i_size, end + 1); 561 again: 562 will_compress = 0; 563 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; 564 BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0); 565 nr_pages = min_t(unsigned long, nr_pages, 566 BTRFS_MAX_COMPRESSED / PAGE_SIZE); 567 568 /* 569 * we don't want to send crud past the end of i_size through 570 * compression, that's just a waste of CPU time. So, if the 571 * end of the file is before the start of our current 572 * requested range of bytes, we bail out to the uncompressed 573 * cleanup code that can deal with all of this. 574 * 575 * It isn't really the fastest way to fix things, but this is a 576 * very uncommon corner. 577 */ 578 if (actual_end <= start) 579 goto cleanup_and_bail_uncompressed; 580 581 total_compressed = actual_end - start; 582 583 /* 584 * skip compression for a small file range(<=blocksize) that 585 * isn't an inline extent, since it doesn't save disk space at all. 586 */ 587 if (total_compressed <= blocksize && 588 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 589 goto cleanup_and_bail_uncompressed; 590 591 total_compressed = min_t(unsigned long, total_compressed, 592 BTRFS_MAX_UNCOMPRESSED); 593 total_in = 0; 594 ret = 0; 595 596 /* 597 * we do compression for mount -o compress and when the 598 * inode has not been flagged as nocompress. This flag can 599 * change at any time if we discover bad compression ratios. 600 */ 601 if (inode_need_compress(BTRFS_I(inode), start, end)) { 602 WARN_ON(pages); 603 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 604 if (!pages) { 605 /* just bail out to the uncompressed code */ 606 nr_pages = 0; 607 goto cont; 608 } 609 610 if (BTRFS_I(inode)->defrag_compress) 611 compress_type = BTRFS_I(inode)->defrag_compress; 612 else if (BTRFS_I(inode)->prop_compress) 613 compress_type = BTRFS_I(inode)->prop_compress; 614 615 /* 616 * we need to call clear_page_dirty_for_io on each 617 * page in the range. Otherwise applications with the file 618 * mmap'd can wander in and change the page contents while 619 * we are compressing them. 620 * 621 * If the compression fails for any reason, we set the pages 622 * dirty again later on. 623 * 624 * Note that the remaining part is redirtied, the start pointer 625 * has moved, the end is the original one. 626 */ 627 if (!redirty) { 628 extent_range_clear_dirty_for_io(inode, start, end); 629 redirty = 1; 630 } 631 632 /* Compression level is applied here and only here */ 633 ret = btrfs_compress_pages( 634 compress_type | (fs_info->compress_level << 4), 635 inode->i_mapping, start, 636 pages, 637 &nr_pages, 638 &total_in, 639 &total_compressed); 640 641 if (!ret) { 642 unsigned long offset = offset_in_page(total_compressed); 643 struct page *page = pages[nr_pages - 1]; 644 char *kaddr; 645 646 /* zero the tail end of the last page, we might be 647 * sending it down to disk 648 */ 649 if (offset) { 650 kaddr = kmap_atomic(page); 651 memset(kaddr + offset, 0, 652 PAGE_SIZE - offset); 653 kunmap_atomic(kaddr); 654 } 655 will_compress = 1; 656 } 657 } 658 cont: 659 if (start == 0) { 660 /* lets try to make an inline extent */ 661 if (ret || total_in < actual_end) { 662 /* we didn't compress the entire range, try 663 * to make an uncompressed inline extent. 664 */ 665 ret = cow_file_range_inline(BTRFS_I(inode), start, end, 666 0, BTRFS_COMPRESS_NONE, 667 NULL); 668 } else { 669 /* try making a compressed inline extent */ 670 ret = cow_file_range_inline(BTRFS_I(inode), start, end, 671 total_compressed, 672 compress_type, pages); 673 } 674 if (ret <= 0) { 675 unsigned long clear_flags = EXTENT_DELALLOC | 676 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 677 EXTENT_DO_ACCOUNTING; 678 unsigned long page_error_op; 679 680 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 681 682 /* 683 * inline extent creation worked or returned error, 684 * we don't need to create any more async work items. 685 * Unlock and free up our temp pages. 686 * 687 * We use DO_ACCOUNTING here because we need the 688 * delalloc_release_metadata to be done _after_ we drop 689 * our outstanding extent for clearing delalloc for this 690 * range. 691 */ 692 extent_clear_unlock_delalloc(BTRFS_I(inode), start, end, 693 NULL, 694 clear_flags, 695 PAGE_UNLOCK | 696 PAGE_START_WRITEBACK | 697 page_error_op | 698 PAGE_END_WRITEBACK); 699 700 /* 701 * Ensure we only free the compressed pages if we have 702 * them allocated, as we can still reach here with 703 * inode_need_compress() == false. 704 */ 705 if (pages) { 706 for (i = 0; i < nr_pages; i++) { 707 WARN_ON(pages[i]->mapping); 708 put_page(pages[i]); 709 } 710 kfree(pages); 711 } 712 return 0; 713 } 714 } 715 716 if (will_compress) { 717 /* 718 * we aren't doing an inline extent round the compressed size 719 * up to a block size boundary so the allocator does sane 720 * things 721 */ 722 total_compressed = ALIGN(total_compressed, blocksize); 723 724 /* 725 * one last check to make sure the compression is really a 726 * win, compare the page count read with the blocks on disk, 727 * compression must free at least one sector size 728 */ 729 total_in = ALIGN(total_in, PAGE_SIZE); 730 if (total_compressed + blocksize <= total_in) { 731 compressed_extents++; 732 733 /* 734 * The async work queues will take care of doing actual 735 * allocation on disk for these compressed pages, and 736 * will submit them to the elevator. 737 */ 738 add_async_extent(async_chunk, start, total_in, 739 total_compressed, pages, nr_pages, 740 compress_type); 741 742 if (start + total_in < end) { 743 start += total_in; 744 pages = NULL; 745 cond_resched(); 746 goto again; 747 } 748 return compressed_extents; 749 } 750 } 751 if (pages) { 752 /* 753 * the compression code ran but failed to make things smaller, 754 * free any pages it allocated and our page pointer array 755 */ 756 for (i = 0; i < nr_pages; i++) { 757 WARN_ON(pages[i]->mapping); 758 put_page(pages[i]); 759 } 760 kfree(pages); 761 pages = NULL; 762 total_compressed = 0; 763 nr_pages = 0; 764 765 /* flag the file so we don't compress in the future */ 766 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && 767 !(BTRFS_I(inode)->prop_compress)) { 768 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 769 } 770 } 771 cleanup_and_bail_uncompressed: 772 /* 773 * No compression, but we still need to write the pages in the file 774 * we've been given so far. redirty the locked page if it corresponds 775 * to our extent and set things up for the async work queue to run 776 * cow_file_range to do the normal delalloc dance. 777 */ 778 if (async_chunk->locked_page && 779 (page_offset(async_chunk->locked_page) >= start && 780 page_offset(async_chunk->locked_page)) <= end) { 781 __set_page_dirty_nobuffers(async_chunk->locked_page); 782 /* unlocked later on in the async handlers */ 783 } 784 785 if (redirty) 786 extent_range_redirty_for_io(inode, start, end); 787 add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0, 788 BTRFS_COMPRESS_NONE); 789 compressed_extents++; 790 791 return compressed_extents; 792 } 793 794 static void free_async_extent_pages(struct async_extent *async_extent) 795 { 796 int i; 797 798 if (!async_extent->pages) 799 return; 800 801 for (i = 0; i < async_extent->nr_pages; i++) { 802 WARN_ON(async_extent->pages[i]->mapping); 803 put_page(async_extent->pages[i]); 804 } 805 kfree(async_extent->pages); 806 async_extent->nr_pages = 0; 807 async_extent->pages = NULL; 808 } 809 810 /* 811 * phase two of compressed writeback. This is the ordered portion 812 * of the code, which only gets called in the order the work was 813 * queued. We walk all the async extents created by compress_file_range 814 * and send them down to the disk. 815 */ 816 static noinline void submit_compressed_extents(struct async_chunk *async_chunk) 817 { 818 struct btrfs_inode *inode = BTRFS_I(async_chunk->inode); 819 struct btrfs_fs_info *fs_info = inode->root->fs_info; 820 struct async_extent *async_extent; 821 u64 alloc_hint = 0; 822 struct btrfs_key ins; 823 struct extent_map *em; 824 struct btrfs_root *root = inode->root; 825 struct extent_io_tree *io_tree = &inode->io_tree; 826 int ret = 0; 827 828 again: 829 while (!list_empty(&async_chunk->extents)) { 830 async_extent = list_entry(async_chunk->extents.next, 831 struct async_extent, list); 832 list_del(&async_extent->list); 833 834 retry: 835 lock_extent(io_tree, async_extent->start, 836 async_extent->start + async_extent->ram_size - 1); 837 /* did the compression code fall back to uncompressed IO? */ 838 if (!async_extent->pages) { 839 int page_started = 0; 840 unsigned long nr_written = 0; 841 842 /* allocate blocks */ 843 ret = cow_file_range(inode, async_chunk->locked_page, 844 async_extent->start, 845 async_extent->start + 846 async_extent->ram_size - 1, 847 &page_started, &nr_written, 0); 848 849 /* JDM XXX */ 850 851 /* 852 * if page_started, cow_file_range inserted an 853 * inline extent and took care of all the unlocking 854 * and IO for us. Otherwise, we need to submit 855 * all those pages down to the drive. 856 */ 857 if (!page_started && !ret) 858 extent_write_locked_range(&inode->vfs_inode, 859 async_extent->start, 860 async_extent->start + 861 async_extent->ram_size - 1, 862 WB_SYNC_ALL); 863 else if (ret && async_chunk->locked_page) 864 unlock_page(async_chunk->locked_page); 865 kfree(async_extent); 866 cond_resched(); 867 continue; 868 } 869 870 ret = btrfs_reserve_extent(root, async_extent->ram_size, 871 async_extent->compressed_size, 872 async_extent->compressed_size, 873 0, alloc_hint, &ins, 1, 1); 874 if (ret) { 875 free_async_extent_pages(async_extent); 876 877 if (ret == -ENOSPC) { 878 unlock_extent(io_tree, async_extent->start, 879 async_extent->start + 880 async_extent->ram_size - 1); 881 882 /* 883 * we need to redirty the pages if we decide to 884 * fallback to uncompressed IO, otherwise we 885 * will not submit these pages down to lower 886 * layers. 887 */ 888 extent_range_redirty_for_io(&inode->vfs_inode, 889 async_extent->start, 890 async_extent->start + 891 async_extent->ram_size - 1); 892 893 goto retry; 894 } 895 goto out_free; 896 } 897 /* 898 * here we're doing allocation and writeback of the 899 * compressed pages 900 */ 901 em = create_io_em(inode, async_extent->start, 902 async_extent->ram_size, /* len */ 903 async_extent->start, /* orig_start */ 904 ins.objectid, /* block_start */ 905 ins.offset, /* block_len */ 906 ins.offset, /* orig_block_len */ 907 async_extent->ram_size, /* ram_bytes */ 908 async_extent->compress_type, 909 BTRFS_ORDERED_COMPRESSED); 910 if (IS_ERR(em)) 911 /* ret value is not necessary due to void function */ 912 goto out_free_reserve; 913 free_extent_map(em); 914 915 ret = btrfs_add_ordered_extent_compress(inode, 916 async_extent->start, 917 ins.objectid, 918 async_extent->ram_size, 919 ins.offset, 920 async_extent->compress_type); 921 if (ret) { 922 btrfs_drop_extent_cache(inode, async_extent->start, 923 async_extent->start + 924 async_extent->ram_size - 1, 0); 925 goto out_free_reserve; 926 } 927 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 928 929 /* 930 * clear dirty, set writeback and unlock the pages. 931 */ 932 extent_clear_unlock_delalloc(inode, async_extent->start, 933 async_extent->start + 934 async_extent->ram_size - 1, 935 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 936 PAGE_UNLOCK | PAGE_START_WRITEBACK); 937 if (btrfs_submit_compressed_write(inode, async_extent->start, 938 async_extent->ram_size, 939 ins.objectid, 940 ins.offset, async_extent->pages, 941 async_extent->nr_pages, 942 async_chunk->write_flags, 943 async_chunk->blkcg_css)) { 944 struct page *p = async_extent->pages[0]; 945 const u64 start = async_extent->start; 946 const u64 end = start + async_extent->ram_size - 1; 947 948 p->mapping = inode->vfs_inode.i_mapping; 949 btrfs_writepage_endio_finish_ordered(p, start, end, 0); 950 951 p->mapping = NULL; 952 extent_clear_unlock_delalloc(inode, start, end, NULL, 0, 953 PAGE_END_WRITEBACK | 954 PAGE_SET_ERROR); 955 free_async_extent_pages(async_extent); 956 } 957 alloc_hint = ins.objectid + ins.offset; 958 kfree(async_extent); 959 cond_resched(); 960 } 961 return; 962 out_free_reserve: 963 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 964 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 965 out_free: 966 extent_clear_unlock_delalloc(inode, async_extent->start, 967 async_extent->start + 968 async_extent->ram_size - 1, 969 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 970 EXTENT_DELALLOC_NEW | 971 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 972 PAGE_UNLOCK | PAGE_START_WRITEBACK | 973 PAGE_END_WRITEBACK | PAGE_SET_ERROR); 974 free_async_extent_pages(async_extent); 975 kfree(async_extent); 976 goto again; 977 } 978 979 static u64 get_extent_allocation_hint(struct btrfs_inode *inode, u64 start, 980 u64 num_bytes) 981 { 982 struct extent_map_tree *em_tree = &inode->extent_tree; 983 struct extent_map *em; 984 u64 alloc_hint = 0; 985 986 read_lock(&em_tree->lock); 987 em = search_extent_mapping(em_tree, start, num_bytes); 988 if (em) { 989 /* 990 * if block start isn't an actual block number then find the 991 * first block in this inode and use that as a hint. If that 992 * block is also bogus then just don't worry about it. 993 */ 994 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 995 free_extent_map(em); 996 em = search_extent_mapping(em_tree, 0, 0); 997 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 998 alloc_hint = em->block_start; 999 if (em) 1000 free_extent_map(em); 1001 } else { 1002 alloc_hint = em->block_start; 1003 free_extent_map(em); 1004 } 1005 } 1006 read_unlock(&em_tree->lock); 1007 1008 return alloc_hint; 1009 } 1010 1011 /* 1012 * when extent_io.c finds a delayed allocation range in the file, 1013 * the call backs end up in this code. The basic idea is to 1014 * allocate extents on disk for the range, and create ordered data structs 1015 * in ram to track those extents. 1016 * 1017 * locked_page is the page that writepage had locked already. We use 1018 * it to make sure we don't do extra locks or unlocks. 1019 * 1020 * *page_started is set to one if we unlock locked_page and do everything 1021 * required to start IO on it. It may be clean and already done with 1022 * IO when we return. 1023 */ 1024 static noinline int cow_file_range(struct btrfs_inode *inode, 1025 struct page *locked_page, 1026 u64 start, u64 end, int *page_started, 1027 unsigned long *nr_written, int unlock) 1028 { 1029 struct btrfs_root *root = inode->root; 1030 struct btrfs_fs_info *fs_info = root->fs_info; 1031 u64 alloc_hint = 0; 1032 u64 num_bytes; 1033 unsigned long ram_size; 1034 u64 cur_alloc_size = 0; 1035 u64 min_alloc_size; 1036 u64 blocksize = fs_info->sectorsize; 1037 struct btrfs_key ins; 1038 struct extent_map *em; 1039 unsigned clear_bits; 1040 unsigned long page_ops; 1041 bool extent_reserved = false; 1042 int ret = 0; 1043 1044 if (btrfs_is_free_space_inode(inode)) { 1045 WARN_ON_ONCE(1); 1046 ret = -EINVAL; 1047 goto out_unlock; 1048 } 1049 1050 num_bytes = ALIGN(end - start + 1, blocksize); 1051 num_bytes = max(blocksize, num_bytes); 1052 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); 1053 1054 inode_should_defrag(inode, start, end, num_bytes, SZ_64K); 1055 1056 if (start == 0) { 1057 /* lets try to make an inline extent */ 1058 ret = cow_file_range_inline(inode, start, end, 0, 1059 BTRFS_COMPRESS_NONE, NULL); 1060 if (ret == 0) { 1061 /* 1062 * We use DO_ACCOUNTING here because we need the 1063 * delalloc_release_metadata to be run _after_ we drop 1064 * our outstanding extent for clearing delalloc for this 1065 * range. 1066 */ 1067 extent_clear_unlock_delalloc(inode, start, end, NULL, 1068 EXTENT_LOCKED | EXTENT_DELALLOC | 1069 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 1070 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1071 PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); 1072 *nr_written = *nr_written + 1073 (end - start + PAGE_SIZE) / PAGE_SIZE; 1074 *page_started = 1; 1075 goto out; 1076 } else if (ret < 0) { 1077 goto out_unlock; 1078 } 1079 } 1080 1081 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 1082 btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0); 1083 1084 /* 1085 * Relocation relies on the relocated extents to have exactly the same 1086 * size as the original extents. Normally writeback for relocation data 1087 * extents follows a NOCOW path because relocation preallocates the 1088 * extents. However, due to an operation such as scrub turning a block 1089 * group to RO mode, it may fallback to COW mode, so we must make sure 1090 * an extent allocated during COW has exactly the requested size and can 1091 * not be split into smaller extents, otherwise relocation breaks and 1092 * fails during the stage where it updates the bytenr of file extent 1093 * items. 1094 */ 1095 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 1096 min_alloc_size = num_bytes; 1097 else 1098 min_alloc_size = fs_info->sectorsize; 1099 1100 while (num_bytes > 0) { 1101 cur_alloc_size = num_bytes; 1102 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size, 1103 min_alloc_size, 0, alloc_hint, 1104 &ins, 1, 1); 1105 if (ret < 0) 1106 goto out_unlock; 1107 cur_alloc_size = ins.offset; 1108 extent_reserved = true; 1109 1110 ram_size = ins.offset; 1111 em = create_io_em(inode, start, ins.offset, /* len */ 1112 start, /* orig_start */ 1113 ins.objectid, /* block_start */ 1114 ins.offset, /* block_len */ 1115 ins.offset, /* orig_block_len */ 1116 ram_size, /* ram_bytes */ 1117 BTRFS_COMPRESS_NONE, /* compress_type */ 1118 BTRFS_ORDERED_REGULAR /* type */); 1119 if (IS_ERR(em)) { 1120 ret = PTR_ERR(em); 1121 goto out_reserve; 1122 } 1123 free_extent_map(em); 1124 1125 ret = btrfs_add_ordered_extent(inode, start, ins.objectid, 1126 ram_size, cur_alloc_size, 1127 BTRFS_ORDERED_REGULAR); 1128 if (ret) 1129 goto out_drop_extent_cache; 1130 1131 if (root->root_key.objectid == 1132 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1133 ret = btrfs_reloc_clone_csums(inode, start, 1134 cur_alloc_size); 1135 /* 1136 * Only drop cache here, and process as normal. 1137 * 1138 * We must not allow extent_clear_unlock_delalloc() 1139 * at out_unlock label to free meta of this ordered 1140 * extent, as its meta should be freed by 1141 * btrfs_finish_ordered_io(). 1142 * 1143 * So we must continue until @start is increased to 1144 * skip current ordered extent. 1145 */ 1146 if (ret) 1147 btrfs_drop_extent_cache(inode, start, 1148 start + ram_size - 1, 0); 1149 } 1150 1151 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1152 1153 /* we're not doing compressed IO, don't unlock the first 1154 * page (which the caller expects to stay locked), don't 1155 * clear any dirty bits and don't set any writeback bits 1156 * 1157 * Do set the Private2 bit so we know this page was properly 1158 * setup for writepage 1159 */ 1160 page_ops = unlock ? PAGE_UNLOCK : 0; 1161 page_ops |= PAGE_SET_PRIVATE2; 1162 1163 extent_clear_unlock_delalloc(inode, start, start + ram_size - 1, 1164 locked_page, 1165 EXTENT_LOCKED | EXTENT_DELALLOC, 1166 page_ops); 1167 if (num_bytes < cur_alloc_size) 1168 num_bytes = 0; 1169 else 1170 num_bytes -= cur_alloc_size; 1171 alloc_hint = ins.objectid + ins.offset; 1172 start += cur_alloc_size; 1173 extent_reserved = false; 1174 1175 /* 1176 * btrfs_reloc_clone_csums() error, since start is increased 1177 * extent_clear_unlock_delalloc() at out_unlock label won't 1178 * free metadata of current ordered extent, we're OK to exit. 1179 */ 1180 if (ret) 1181 goto out_unlock; 1182 } 1183 out: 1184 return ret; 1185 1186 out_drop_extent_cache: 1187 btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0); 1188 out_reserve: 1189 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1190 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1191 out_unlock: 1192 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 1193 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; 1194 page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK; 1195 /* 1196 * If we reserved an extent for our delalloc range (or a subrange) and 1197 * failed to create the respective ordered extent, then it means that 1198 * when we reserved the extent we decremented the extent's size from 1199 * the data space_info's bytes_may_use counter and incremented the 1200 * space_info's bytes_reserved counter by the same amount. We must make 1201 * sure extent_clear_unlock_delalloc() does not try to decrement again 1202 * the data space_info's bytes_may_use counter, therefore we do not pass 1203 * it the flag EXTENT_CLEAR_DATA_RESV. 1204 */ 1205 if (extent_reserved) { 1206 extent_clear_unlock_delalloc(inode, start, 1207 start + cur_alloc_size - 1, 1208 locked_page, 1209 clear_bits, 1210 page_ops); 1211 start += cur_alloc_size; 1212 if (start >= end) 1213 goto out; 1214 } 1215 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1216 clear_bits | EXTENT_CLEAR_DATA_RESV, 1217 page_ops); 1218 goto out; 1219 } 1220 1221 /* 1222 * work queue call back to started compression on a file and pages 1223 */ 1224 static noinline void async_cow_start(struct btrfs_work *work) 1225 { 1226 struct async_chunk *async_chunk; 1227 int compressed_extents; 1228 1229 async_chunk = container_of(work, struct async_chunk, work); 1230 1231 compressed_extents = compress_file_range(async_chunk); 1232 if (compressed_extents == 0) { 1233 btrfs_add_delayed_iput(async_chunk->inode); 1234 async_chunk->inode = NULL; 1235 } 1236 } 1237 1238 /* 1239 * work queue call back to submit previously compressed pages 1240 */ 1241 static noinline void async_cow_submit(struct btrfs_work *work) 1242 { 1243 struct async_chunk *async_chunk = container_of(work, struct async_chunk, 1244 work); 1245 struct btrfs_fs_info *fs_info = btrfs_work_owner(work); 1246 unsigned long nr_pages; 1247 1248 nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >> 1249 PAGE_SHIFT; 1250 1251 /* atomic_sub_return implies a barrier */ 1252 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1253 5 * SZ_1M) 1254 cond_wake_up_nomb(&fs_info->async_submit_wait); 1255 1256 /* 1257 * ->inode could be NULL if async_chunk_start has failed to compress, 1258 * in which case we don't have anything to submit, yet we need to 1259 * always adjust ->async_delalloc_pages as its paired with the init 1260 * happening in cow_file_range_async 1261 */ 1262 if (async_chunk->inode) 1263 submit_compressed_extents(async_chunk); 1264 } 1265 1266 static noinline void async_cow_free(struct btrfs_work *work) 1267 { 1268 struct async_chunk *async_chunk; 1269 1270 async_chunk = container_of(work, struct async_chunk, work); 1271 if (async_chunk->inode) 1272 btrfs_add_delayed_iput(async_chunk->inode); 1273 if (async_chunk->blkcg_css) 1274 css_put(async_chunk->blkcg_css); 1275 /* 1276 * Since the pointer to 'pending' is at the beginning of the array of 1277 * async_chunk's, freeing it ensures the whole array has been freed. 1278 */ 1279 if (atomic_dec_and_test(async_chunk->pending)) 1280 kvfree(async_chunk->pending); 1281 } 1282 1283 static int cow_file_range_async(struct btrfs_inode *inode, 1284 struct writeback_control *wbc, 1285 struct page *locked_page, 1286 u64 start, u64 end, int *page_started, 1287 unsigned long *nr_written) 1288 { 1289 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1290 struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc); 1291 struct async_cow *ctx; 1292 struct async_chunk *async_chunk; 1293 unsigned long nr_pages; 1294 u64 cur_end; 1295 u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K); 1296 int i; 1297 bool should_compress; 1298 unsigned nofs_flag; 1299 const unsigned int write_flags = wbc_to_write_flags(wbc); 1300 1301 unlock_extent(&inode->io_tree, start, end); 1302 1303 if (inode->flags & BTRFS_INODE_NOCOMPRESS && 1304 !btrfs_test_opt(fs_info, FORCE_COMPRESS)) { 1305 num_chunks = 1; 1306 should_compress = false; 1307 } else { 1308 should_compress = true; 1309 } 1310 1311 nofs_flag = memalloc_nofs_save(); 1312 ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL); 1313 memalloc_nofs_restore(nofs_flag); 1314 1315 if (!ctx) { 1316 unsigned clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | 1317 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 1318 EXTENT_DO_ACCOUNTING; 1319 unsigned long page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | 1320 PAGE_END_WRITEBACK | PAGE_SET_ERROR; 1321 1322 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1323 clear_bits, page_ops); 1324 return -ENOMEM; 1325 } 1326 1327 async_chunk = ctx->chunks; 1328 atomic_set(&ctx->num_chunks, num_chunks); 1329 1330 for (i = 0; i < num_chunks; i++) { 1331 if (should_compress) 1332 cur_end = min(end, start + SZ_512K - 1); 1333 else 1334 cur_end = end; 1335 1336 /* 1337 * igrab is called higher up in the call chain, take only the 1338 * lightweight reference for the callback lifetime 1339 */ 1340 ihold(&inode->vfs_inode); 1341 async_chunk[i].pending = &ctx->num_chunks; 1342 async_chunk[i].inode = &inode->vfs_inode; 1343 async_chunk[i].start = start; 1344 async_chunk[i].end = cur_end; 1345 async_chunk[i].write_flags = write_flags; 1346 INIT_LIST_HEAD(&async_chunk[i].extents); 1347 1348 /* 1349 * The locked_page comes all the way from writepage and its 1350 * the original page we were actually given. As we spread 1351 * this large delalloc region across multiple async_chunk 1352 * structs, only the first struct needs a pointer to locked_page 1353 * 1354 * This way we don't need racey decisions about who is supposed 1355 * to unlock it. 1356 */ 1357 if (locked_page) { 1358 /* 1359 * Depending on the compressibility, the pages might or 1360 * might not go through async. We want all of them to 1361 * be accounted against wbc once. Let's do it here 1362 * before the paths diverge. wbc accounting is used 1363 * only for foreign writeback detection and doesn't 1364 * need full accuracy. Just account the whole thing 1365 * against the first page. 1366 */ 1367 wbc_account_cgroup_owner(wbc, locked_page, 1368 cur_end - start); 1369 async_chunk[i].locked_page = locked_page; 1370 locked_page = NULL; 1371 } else { 1372 async_chunk[i].locked_page = NULL; 1373 } 1374 1375 if (blkcg_css != blkcg_root_css) { 1376 css_get(blkcg_css); 1377 async_chunk[i].blkcg_css = blkcg_css; 1378 } else { 1379 async_chunk[i].blkcg_css = NULL; 1380 } 1381 1382 btrfs_init_work(&async_chunk[i].work, async_cow_start, 1383 async_cow_submit, async_cow_free); 1384 1385 nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE); 1386 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1387 1388 btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work); 1389 1390 *nr_written += nr_pages; 1391 start = cur_end + 1; 1392 } 1393 *page_started = 1; 1394 return 0; 1395 } 1396 1397 static noinline int run_delalloc_zoned(struct btrfs_inode *inode, 1398 struct page *locked_page, u64 start, 1399 u64 end, int *page_started, 1400 unsigned long *nr_written) 1401 { 1402 int ret; 1403 1404 ret = cow_file_range(inode, locked_page, start, end, page_started, 1405 nr_written, 0); 1406 if (ret) 1407 return ret; 1408 1409 if (*page_started) 1410 return 0; 1411 1412 __set_page_dirty_nobuffers(locked_page); 1413 account_page_redirty(locked_page); 1414 extent_write_locked_range(&inode->vfs_inode, start, end, WB_SYNC_ALL); 1415 *page_started = 1; 1416 1417 return 0; 1418 } 1419 1420 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, 1421 u64 bytenr, u64 num_bytes) 1422 { 1423 int ret; 1424 struct btrfs_ordered_sum *sums; 1425 LIST_HEAD(list); 1426 1427 ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr, 1428 bytenr + num_bytes - 1, &list, 0); 1429 if (ret == 0 && list_empty(&list)) 1430 return 0; 1431 1432 while (!list_empty(&list)) { 1433 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1434 list_del(&sums->list); 1435 kfree(sums); 1436 } 1437 if (ret < 0) 1438 return ret; 1439 return 1; 1440 } 1441 1442 static int fallback_to_cow(struct btrfs_inode *inode, struct page *locked_page, 1443 const u64 start, const u64 end, 1444 int *page_started, unsigned long *nr_written) 1445 { 1446 const bool is_space_ino = btrfs_is_free_space_inode(inode); 1447 const bool is_reloc_ino = (inode->root->root_key.objectid == 1448 BTRFS_DATA_RELOC_TREE_OBJECTID); 1449 const u64 range_bytes = end + 1 - start; 1450 struct extent_io_tree *io_tree = &inode->io_tree; 1451 u64 range_start = start; 1452 u64 count; 1453 1454 /* 1455 * If EXTENT_NORESERVE is set it means that when the buffered write was 1456 * made we had not enough available data space and therefore we did not 1457 * reserve data space for it, since we though we could do NOCOW for the 1458 * respective file range (either there is prealloc extent or the inode 1459 * has the NOCOW bit set). 1460 * 1461 * However when we need to fallback to COW mode (because for example the 1462 * block group for the corresponding extent was turned to RO mode by a 1463 * scrub or relocation) we need to do the following: 1464 * 1465 * 1) We increment the bytes_may_use counter of the data space info. 1466 * If COW succeeds, it allocates a new data extent and after doing 1467 * that it decrements the space info's bytes_may_use counter and 1468 * increments its bytes_reserved counter by the same amount (we do 1469 * this at btrfs_add_reserved_bytes()). So we need to increment the 1470 * bytes_may_use counter to compensate (when space is reserved at 1471 * buffered write time, the bytes_may_use counter is incremented); 1472 * 1473 * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so 1474 * that if the COW path fails for any reason, it decrements (through 1475 * extent_clear_unlock_delalloc()) the bytes_may_use counter of the 1476 * data space info, which we incremented in the step above. 1477 * 1478 * If we need to fallback to cow and the inode corresponds to a free 1479 * space cache inode or an inode of the data relocation tree, we must 1480 * also increment bytes_may_use of the data space_info for the same 1481 * reason. Space caches and relocated data extents always get a prealloc 1482 * extent for them, however scrub or balance may have set the block 1483 * group that contains that extent to RO mode and therefore force COW 1484 * when starting writeback. 1485 */ 1486 count = count_range_bits(io_tree, &range_start, end, range_bytes, 1487 EXTENT_NORESERVE, 0); 1488 if (count > 0 || is_space_ino || is_reloc_ino) { 1489 u64 bytes = count; 1490 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1491 struct btrfs_space_info *sinfo = fs_info->data_sinfo; 1492 1493 if (is_space_ino || is_reloc_ino) 1494 bytes = range_bytes; 1495 1496 spin_lock(&sinfo->lock); 1497 btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes); 1498 spin_unlock(&sinfo->lock); 1499 1500 if (count > 0) 1501 clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, 1502 0, 0, NULL); 1503 } 1504 1505 return cow_file_range(inode, locked_page, start, end, page_started, 1506 nr_written, 1); 1507 } 1508 1509 /* 1510 * when nowcow writeback call back. This checks for snapshots or COW copies 1511 * of the extents that exist in the file, and COWs the file as required. 1512 * 1513 * If no cow copies or snapshots exist, we write directly to the existing 1514 * blocks on disk 1515 */ 1516 static noinline int run_delalloc_nocow(struct btrfs_inode *inode, 1517 struct page *locked_page, 1518 const u64 start, const u64 end, 1519 int *page_started, int force, 1520 unsigned long *nr_written) 1521 { 1522 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1523 struct btrfs_root *root = inode->root; 1524 struct btrfs_path *path; 1525 u64 cow_start = (u64)-1; 1526 u64 cur_offset = start; 1527 int ret; 1528 bool check_prev = true; 1529 const bool freespace_inode = btrfs_is_free_space_inode(inode); 1530 u64 ino = btrfs_ino(inode); 1531 bool nocow = false; 1532 u64 disk_bytenr = 0; 1533 1534 path = btrfs_alloc_path(); 1535 if (!path) { 1536 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1537 EXTENT_LOCKED | EXTENT_DELALLOC | 1538 EXTENT_DO_ACCOUNTING | 1539 EXTENT_DEFRAG, PAGE_UNLOCK | 1540 PAGE_START_WRITEBACK | 1541 PAGE_END_WRITEBACK); 1542 return -ENOMEM; 1543 } 1544 1545 while (1) { 1546 struct btrfs_key found_key; 1547 struct btrfs_file_extent_item *fi; 1548 struct extent_buffer *leaf; 1549 u64 extent_end; 1550 u64 extent_offset; 1551 u64 num_bytes = 0; 1552 u64 disk_num_bytes; 1553 u64 ram_bytes; 1554 int extent_type; 1555 1556 nocow = false; 1557 1558 ret = btrfs_lookup_file_extent(NULL, root, path, ino, 1559 cur_offset, 0); 1560 if (ret < 0) 1561 goto error; 1562 1563 /* 1564 * If there is no extent for our range when doing the initial 1565 * search, then go back to the previous slot as it will be the 1566 * one containing the search offset 1567 */ 1568 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1569 leaf = path->nodes[0]; 1570 btrfs_item_key_to_cpu(leaf, &found_key, 1571 path->slots[0] - 1); 1572 if (found_key.objectid == ino && 1573 found_key.type == BTRFS_EXTENT_DATA_KEY) 1574 path->slots[0]--; 1575 } 1576 check_prev = false; 1577 next_slot: 1578 /* Go to next leaf if we have exhausted the current one */ 1579 leaf = path->nodes[0]; 1580 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1581 ret = btrfs_next_leaf(root, path); 1582 if (ret < 0) { 1583 if (cow_start != (u64)-1) 1584 cur_offset = cow_start; 1585 goto error; 1586 } 1587 if (ret > 0) 1588 break; 1589 leaf = path->nodes[0]; 1590 } 1591 1592 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1593 1594 /* Didn't find anything for our INO */ 1595 if (found_key.objectid > ino) 1596 break; 1597 /* 1598 * Keep searching until we find an EXTENT_ITEM or there are no 1599 * more extents for this inode 1600 */ 1601 if (WARN_ON_ONCE(found_key.objectid < ino) || 1602 found_key.type < BTRFS_EXTENT_DATA_KEY) { 1603 path->slots[0]++; 1604 goto next_slot; 1605 } 1606 1607 /* Found key is not EXTENT_DATA_KEY or starts after req range */ 1608 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 1609 found_key.offset > end) 1610 break; 1611 1612 /* 1613 * If the found extent starts after requested offset, then 1614 * adjust extent_end to be right before this extent begins 1615 */ 1616 if (found_key.offset > cur_offset) { 1617 extent_end = found_key.offset; 1618 extent_type = 0; 1619 goto out_check; 1620 } 1621 1622 /* 1623 * Found extent which begins before our range and potentially 1624 * intersect it 1625 */ 1626 fi = btrfs_item_ptr(leaf, path->slots[0], 1627 struct btrfs_file_extent_item); 1628 extent_type = btrfs_file_extent_type(leaf, fi); 1629 1630 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1631 if (extent_type == BTRFS_FILE_EXTENT_REG || 1632 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1633 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1634 extent_offset = btrfs_file_extent_offset(leaf, fi); 1635 extent_end = found_key.offset + 1636 btrfs_file_extent_num_bytes(leaf, fi); 1637 disk_num_bytes = 1638 btrfs_file_extent_disk_num_bytes(leaf, fi); 1639 /* 1640 * If the extent we got ends before our current offset, 1641 * skip to the next extent. 1642 */ 1643 if (extent_end <= cur_offset) { 1644 path->slots[0]++; 1645 goto next_slot; 1646 } 1647 /* Skip holes */ 1648 if (disk_bytenr == 0) 1649 goto out_check; 1650 /* Skip compressed/encrypted/encoded extents */ 1651 if (btrfs_file_extent_compression(leaf, fi) || 1652 btrfs_file_extent_encryption(leaf, fi) || 1653 btrfs_file_extent_other_encoding(leaf, fi)) 1654 goto out_check; 1655 /* 1656 * If extent is created before the last volume's snapshot 1657 * this implies the extent is shared, hence we can't do 1658 * nocow. This is the same check as in 1659 * btrfs_cross_ref_exist but without calling 1660 * btrfs_search_slot. 1661 */ 1662 if (!freespace_inode && 1663 btrfs_file_extent_generation(leaf, fi) <= 1664 btrfs_root_last_snapshot(&root->root_item)) 1665 goto out_check; 1666 if (extent_type == BTRFS_FILE_EXTENT_REG && !force) 1667 goto out_check; 1668 1669 /* 1670 * The following checks can be expensive, as they need to 1671 * take other locks and do btree or rbtree searches, so 1672 * release the path to avoid blocking other tasks for too 1673 * long. 1674 */ 1675 btrfs_release_path(path); 1676 1677 /* If extent is RO, we must COW it */ 1678 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 1679 goto out_check; 1680 ret = btrfs_cross_ref_exist(root, ino, 1681 found_key.offset - 1682 extent_offset, disk_bytenr, false); 1683 if (ret) { 1684 /* 1685 * ret could be -EIO if the above fails to read 1686 * metadata. 1687 */ 1688 if (ret < 0) { 1689 if (cow_start != (u64)-1) 1690 cur_offset = cow_start; 1691 goto error; 1692 } 1693 1694 WARN_ON_ONCE(freespace_inode); 1695 goto out_check; 1696 } 1697 disk_bytenr += extent_offset; 1698 disk_bytenr += cur_offset - found_key.offset; 1699 num_bytes = min(end + 1, extent_end) - cur_offset; 1700 /* 1701 * If there are pending snapshots for this root, we 1702 * fall into common COW way 1703 */ 1704 if (!freespace_inode && atomic_read(&root->snapshot_force_cow)) 1705 goto out_check; 1706 /* 1707 * force cow if csum exists in the range. 1708 * this ensure that csum for a given extent are 1709 * either valid or do not exist. 1710 */ 1711 ret = csum_exist_in_range(fs_info, disk_bytenr, 1712 num_bytes); 1713 if (ret) { 1714 /* 1715 * ret could be -EIO if the above fails to read 1716 * metadata. 1717 */ 1718 if (ret < 0) { 1719 if (cow_start != (u64)-1) 1720 cur_offset = cow_start; 1721 goto error; 1722 } 1723 WARN_ON_ONCE(freespace_inode); 1724 goto out_check; 1725 } 1726 if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) 1727 goto out_check; 1728 nocow = true; 1729 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 1730 extent_end = found_key.offset + ram_bytes; 1731 extent_end = ALIGN(extent_end, fs_info->sectorsize); 1732 /* Skip extents outside of our requested range */ 1733 if (extent_end <= start) { 1734 path->slots[0]++; 1735 goto next_slot; 1736 } 1737 } else { 1738 /* If this triggers then we have a memory corruption */ 1739 BUG(); 1740 } 1741 out_check: 1742 /* 1743 * If nocow is false then record the beginning of the range 1744 * that needs to be COWed 1745 */ 1746 if (!nocow) { 1747 if (cow_start == (u64)-1) 1748 cow_start = cur_offset; 1749 cur_offset = extent_end; 1750 if (cur_offset > end) 1751 break; 1752 if (!path->nodes[0]) 1753 continue; 1754 path->slots[0]++; 1755 goto next_slot; 1756 } 1757 1758 /* 1759 * COW range from cow_start to found_key.offset - 1. As the key 1760 * will contain the beginning of the first extent that can be 1761 * NOCOW, following one which needs to be COW'ed 1762 */ 1763 if (cow_start != (u64)-1) { 1764 ret = fallback_to_cow(inode, locked_page, 1765 cow_start, found_key.offset - 1, 1766 page_started, nr_written); 1767 if (ret) 1768 goto error; 1769 cow_start = (u64)-1; 1770 } 1771 1772 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1773 u64 orig_start = found_key.offset - extent_offset; 1774 struct extent_map *em; 1775 1776 em = create_io_em(inode, cur_offset, num_bytes, 1777 orig_start, 1778 disk_bytenr, /* block_start */ 1779 num_bytes, /* block_len */ 1780 disk_num_bytes, /* orig_block_len */ 1781 ram_bytes, BTRFS_COMPRESS_NONE, 1782 BTRFS_ORDERED_PREALLOC); 1783 if (IS_ERR(em)) { 1784 ret = PTR_ERR(em); 1785 goto error; 1786 } 1787 free_extent_map(em); 1788 ret = btrfs_add_ordered_extent(inode, cur_offset, 1789 disk_bytenr, num_bytes, 1790 num_bytes, 1791 BTRFS_ORDERED_PREALLOC); 1792 if (ret) { 1793 btrfs_drop_extent_cache(inode, cur_offset, 1794 cur_offset + num_bytes - 1, 1795 0); 1796 goto error; 1797 } 1798 } else { 1799 ret = btrfs_add_ordered_extent(inode, cur_offset, 1800 disk_bytenr, num_bytes, 1801 num_bytes, 1802 BTRFS_ORDERED_NOCOW); 1803 if (ret) 1804 goto error; 1805 } 1806 1807 if (nocow) 1808 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1809 nocow = false; 1810 1811 if (root->root_key.objectid == 1812 BTRFS_DATA_RELOC_TREE_OBJECTID) 1813 /* 1814 * Error handled later, as we must prevent 1815 * extent_clear_unlock_delalloc() in error handler 1816 * from freeing metadata of created ordered extent. 1817 */ 1818 ret = btrfs_reloc_clone_csums(inode, cur_offset, 1819 num_bytes); 1820 1821 extent_clear_unlock_delalloc(inode, cur_offset, 1822 cur_offset + num_bytes - 1, 1823 locked_page, EXTENT_LOCKED | 1824 EXTENT_DELALLOC | 1825 EXTENT_CLEAR_DATA_RESV, 1826 PAGE_UNLOCK | PAGE_SET_PRIVATE2); 1827 1828 cur_offset = extent_end; 1829 1830 /* 1831 * btrfs_reloc_clone_csums() error, now we're OK to call error 1832 * handler, as metadata for created ordered extent will only 1833 * be freed by btrfs_finish_ordered_io(). 1834 */ 1835 if (ret) 1836 goto error; 1837 if (cur_offset > end) 1838 break; 1839 } 1840 btrfs_release_path(path); 1841 1842 if (cur_offset <= end && cow_start == (u64)-1) 1843 cow_start = cur_offset; 1844 1845 if (cow_start != (u64)-1) { 1846 cur_offset = end; 1847 ret = fallback_to_cow(inode, locked_page, cow_start, end, 1848 page_started, nr_written); 1849 if (ret) 1850 goto error; 1851 } 1852 1853 error: 1854 if (nocow) 1855 btrfs_dec_nocow_writers(fs_info, disk_bytenr); 1856 1857 if (ret && cur_offset < end) 1858 extent_clear_unlock_delalloc(inode, cur_offset, end, 1859 locked_page, EXTENT_LOCKED | 1860 EXTENT_DELALLOC | EXTENT_DEFRAG | 1861 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1862 PAGE_START_WRITEBACK | 1863 PAGE_END_WRITEBACK); 1864 btrfs_free_path(path); 1865 return ret; 1866 } 1867 1868 static inline int need_force_cow(struct btrfs_inode *inode, u64 start, u64 end) 1869 { 1870 1871 if (!(inode->flags & BTRFS_INODE_NODATACOW) && 1872 !(inode->flags & BTRFS_INODE_PREALLOC)) 1873 return 0; 1874 1875 /* 1876 * @defrag_bytes is a hint value, no spinlock held here, 1877 * if is not zero, it means the file is defragging. 1878 * Force cow if given extent needs to be defragged. 1879 */ 1880 if (inode->defrag_bytes && 1881 test_range_bit(&inode->io_tree, start, end, EXTENT_DEFRAG, 0, NULL)) 1882 return 1; 1883 1884 return 0; 1885 } 1886 1887 /* 1888 * Function to process delayed allocation (create CoW) for ranges which are 1889 * being touched for the first time. 1890 */ 1891 int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct page *locked_page, 1892 u64 start, u64 end, int *page_started, unsigned long *nr_written, 1893 struct writeback_control *wbc) 1894 { 1895 int ret; 1896 int force_cow = need_force_cow(inode, start, end); 1897 const bool zoned = btrfs_is_zoned(inode->root->fs_info); 1898 1899 if (inode->flags & BTRFS_INODE_NODATACOW && !force_cow) { 1900 ASSERT(!zoned); 1901 ret = run_delalloc_nocow(inode, locked_page, start, end, 1902 page_started, 1, nr_written); 1903 } else if (inode->flags & BTRFS_INODE_PREALLOC && !force_cow) { 1904 ASSERT(!zoned); 1905 ret = run_delalloc_nocow(inode, locked_page, start, end, 1906 page_started, 0, nr_written); 1907 } else if (!inode_can_compress(inode) || 1908 !inode_need_compress(inode, start, end)) { 1909 if (zoned) 1910 ret = run_delalloc_zoned(inode, locked_page, start, end, 1911 page_started, nr_written); 1912 else 1913 ret = cow_file_range(inode, locked_page, start, end, 1914 page_started, nr_written, 1); 1915 } else { 1916 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags); 1917 ret = cow_file_range_async(inode, wbc, locked_page, start, end, 1918 page_started, nr_written); 1919 } 1920 if (ret) 1921 btrfs_cleanup_ordered_extents(inode, locked_page, start, 1922 end - start + 1); 1923 return ret; 1924 } 1925 1926 void btrfs_split_delalloc_extent(struct inode *inode, 1927 struct extent_state *orig, u64 split) 1928 { 1929 u64 size; 1930 1931 /* not delalloc, ignore it */ 1932 if (!(orig->state & EXTENT_DELALLOC)) 1933 return; 1934 1935 size = orig->end - orig->start + 1; 1936 if (size > BTRFS_MAX_EXTENT_SIZE) { 1937 u32 num_extents; 1938 u64 new_size; 1939 1940 /* 1941 * See the explanation in btrfs_merge_delalloc_extent, the same 1942 * applies here, just in reverse. 1943 */ 1944 new_size = orig->end - split + 1; 1945 num_extents = count_max_extents(new_size); 1946 new_size = split - orig->start; 1947 num_extents += count_max_extents(new_size); 1948 if (count_max_extents(size) >= num_extents) 1949 return; 1950 } 1951 1952 spin_lock(&BTRFS_I(inode)->lock); 1953 btrfs_mod_outstanding_extents(BTRFS_I(inode), 1); 1954 spin_unlock(&BTRFS_I(inode)->lock); 1955 } 1956 1957 /* 1958 * Handle merged delayed allocation extents so we can keep track of new extents 1959 * that are just merged onto old extents, such as when we are doing sequential 1960 * writes, so we can properly account for the metadata space we'll need. 1961 */ 1962 void btrfs_merge_delalloc_extent(struct inode *inode, struct extent_state *new, 1963 struct extent_state *other) 1964 { 1965 u64 new_size, old_size; 1966 u32 num_extents; 1967 1968 /* not delalloc, ignore it */ 1969 if (!(other->state & EXTENT_DELALLOC)) 1970 return; 1971 1972 if (new->start > other->start) 1973 new_size = new->end - other->start + 1; 1974 else 1975 new_size = other->end - new->start + 1; 1976 1977 /* we're not bigger than the max, unreserve the space and go */ 1978 if (new_size <= BTRFS_MAX_EXTENT_SIZE) { 1979 spin_lock(&BTRFS_I(inode)->lock); 1980 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 1981 spin_unlock(&BTRFS_I(inode)->lock); 1982 return; 1983 } 1984 1985 /* 1986 * We have to add up either side to figure out how many extents were 1987 * accounted for before we merged into one big extent. If the number of 1988 * extents we accounted for is <= the amount we need for the new range 1989 * then we can return, otherwise drop. Think of it like this 1990 * 1991 * [ 4k][MAX_SIZE] 1992 * 1993 * So we've grown the extent by a MAX_SIZE extent, this would mean we 1994 * need 2 outstanding extents, on one side we have 1 and the other side 1995 * we have 1 so they are == and we can return. But in this case 1996 * 1997 * [MAX_SIZE+4k][MAX_SIZE+4k] 1998 * 1999 * Each range on their own accounts for 2 extents, but merged together 2000 * they are only 3 extents worth of accounting, so we need to drop in 2001 * this case. 2002 */ 2003 old_size = other->end - other->start + 1; 2004 num_extents = count_max_extents(old_size); 2005 old_size = new->end - new->start + 1; 2006 num_extents += count_max_extents(old_size); 2007 if (count_max_extents(new_size) >= num_extents) 2008 return; 2009 2010 spin_lock(&BTRFS_I(inode)->lock); 2011 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1); 2012 spin_unlock(&BTRFS_I(inode)->lock); 2013 } 2014 2015 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 2016 struct inode *inode) 2017 { 2018 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2019 2020 spin_lock(&root->delalloc_lock); 2021 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 2022 list_add_tail(&BTRFS_I(inode)->delalloc_inodes, 2023 &root->delalloc_inodes); 2024 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2025 &BTRFS_I(inode)->runtime_flags); 2026 root->nr_delalloc_inodes++; 2027 if (root->nr_delalloc_inodes == 1) { 2028 spin_lock(&fs_info->delalloc_root_lock); 2029 BUG_ON(!list_empty(&root->delalloc_root)); 2030 list_add_tail(&root->delalloc_root, 2031 &fs_info->delalloc_roots); 2032 spin_unlock(&fs_info->delalloc_root_lock); 2033 } 2034 } 2035 spin_unlock(&root->delalloc_lock); 2036 } 2037 2038 2039 void __btrfs_del_delalloc_inode(struct btrfs_root *root, 2040 struct btrfs_inode *inode) 2041 { 2042 struct btrfs_fs_info *fs_info = root->fs_info; 2043 2044 if (!list_empty(&inode->delalloc_inodes)) { 2045 list_del_init(&inode->delalloc_inodes); 2046 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2047 &inode->runtime_flags); 2048 root->nr_delalloc_inodes--; 2049 if (!root->nr_delalloc_inodes) { 2050 ASSERT(list_empty(&root->delalloc_inodes)); 2051 spin_lock(&fs_info->delalloc_root_lock); 2052 BUG_ON(list_empty(&root->delalloc_root)); 2053 list_del_init(&root->delalloc_root); 2054 spin_unlock(&fs_info->delalloc_root_lock); 2055 } 2056 } 2057 } 2058 2059 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 2060 struct btrfs_inode *inode) 2061 { 2062 spin_lock(&root->delalloc_lock); 2063 __btrfs_del_delalloc_inode(root, inode); 2064 spin_unlock(&root->delalloc_lock); 2065 } 2066 2067 /* 2068 * Properly track delayed allocation bytes in the inode and to maintain the 2069 * list of inodes that have pending delalloc work to be done. 2070 */ 2071 void btrfs_set_delalloc_extent(struct inode *inode, struct extent_state *state, 2072 unsigned *bits) 2073 { 2074 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2075 2076 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC)) 2077 WARN_ON(1); 2078 /* 2079 * set_bit and clear bit hooks normally require _irqsave/restore 2080 * but in this case, we are only testing for the DELALLOC 2081 * bit, which is only set or cleared with irqs on 2082 */ 2083 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 2084 struct btrfs_root *root = BTRFS_I(inode)->root; 2085 u64 len = state->end + 1 - state->start; 2086 u32 num_extents = count_max_extents(len); 2087 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode)); 2088 2089 spin_lock(&BTRFS_I(inode)->lock); 2090 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents); 2091 spin_unlock(&BTRFS_I(inode)->lock); 2092 2093 /* For sanity tests */ 2094 if (btrfs_is_testing(fs_info)) 2095 return; 2096 2097 percpu_counter_add_batch(&fs_info->delalloc_bytes, len, 2098 fs_info->delalloc_batch); 2099 spin_lock(&BTRFS_I(inode)->lock); 2100 BTRFS_I(inode)->delalloc_bytes += len; 2101 if (*bits & EXTENT_DEFRAG) 2102 BTRFS_I(inode)->defrag_bytes += len; 2103 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2104 &BTRFS_I(inode)->runtime_flags)) 2105 btrfs_add_delalloc_inodes(root, inode); 2106 spin_unlock(&BTRFS_I(inode)->lock); 2107 } 2108 2109 if (!(state->state & EXTENT_DELALLOC_NEW) && 2110 (*bits & EXTENT_DELALLOC_NEW)) { 2111 spin_lock(&BTRFS_I(inode)->lock); 2112 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 - 2113 state->start; 2114 spin_unlock(&BTRFS_I(inode)->lock); 2115 } 2116 } 2117 2118 /* 2119 * Once a range is no longer delalloc this function ensures that proper 2120 * accounting happens. 2121 */ 2122 void btrfs_clear_delalloc_extent(struct inode *vfs_inode, 2123 struct extent_state *state, unsigned *bits) 2124 { 2125 struct btrfs_inode *inode = BTRFS_I(vfs_inode); 2126 struct btrfs_fs_info *fs_info = btrfs_sb(vfs_inode->i_sb); 2127 u64 len = state->end + 1 - state->start; 2128 u32 num_extents = count_max_extents(len); 2129 2130 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) { 2131 spin_lock(&inode->lock); 2132 inode->defrag_bytes -= len; 2133 spin_unlock(&inode->lock); 2134 } 2135 2136 /* 2137 * set_bit and clear bit hooks normally require _irqsave/restore 2138 * but in this case, we are only testing for the DELALLOC 2139 * bit, which is only set or cleared with irqs on 2140 */ 2141 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 2142 struct btrfs_root *root = inode->root; 2143 bool do_list = !btrfs_is_free_space_inode(inode); 2144 2145 spin_lock(&inode->lock); 2146 btrfs_mod_outstanding_extents(inode, -num_extents); 2147 spin_unlock(&inode->lock); 2148 2149 /* 2150 * We don't reserve metadata space for space cache inodes so we 2151 * don't need to call delalloc_release_metadata if there is an 2152 * error. 2153 */ 2154 if (*bits & EXTENT_CLEAR_META_RESV && 2155 root != fs_info->tree_root) 2156 btrfs_delalloc_release_metadata(inode, len, false); 2157 2158 /* For sanity tests. */ 2159 if (btrfs_is_testing(fs_info)) 2160 return; 2161 2162 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID && 2163 do_list && !(state->state & EXTENT_NORESERVE) && 2164 (*bits & EXTENT_CLEAR_DATA_RESV)) 2165 btrfs_free_reserved_data_space_noquota(fs_info, len); 2166 2167 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, 2168 fs_info->delalloc_batch); 2169 spin_lock(&inode->lock); 2170 inode->delalloc_bytes -= len; 2171 if (do_list && inode->delalloc_bytes == 0 && 2172 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2173 &inode->runtime_flags)) 2174 btrfs_del_delalloc_inode(root, inode); 2175 spin_unlock(&inode->lock); 2176 } 2177 2178 if ((state->state & EXTENT_DELALLOC_NEW) && 2179 (*bits & EXTENT_DELALLOC_NEW)) { 2180 spin_lock(&inode->lock); 2181 ASSERT(inode->new_delalloc_bytes >= len); 2182 inode->new_delalloc_bytes -= len; 2183 if (*bits & EXTENT_ADD_INODE_BYTES) 2184 inode_add_bytes(&inode->vfs_inode, len); 2185 spin_unlock(&inode->lock); 2186 } 2187 } 2188 2189 /* 2190 * btrfs_bio_fits_in_stripe - Checks whether the size of the given bio will fit 2191 * in a chunk's stripe. This function ensures that bios do not span a 2192 * stripe/chunk 2193 * 2194 * @page - The page we are about to add to the bio 2195 * @size - size we want to add to the bio 2196 * @bio - bio we want to ensure is smaller than a stripe 2197 * @bio_flags - flags of the bio 2198 * 2199 * return 1 if page cannot be added to the bio 2200 * return 0 if page can be added to the bio 2201 * return error otherwise 2202 */ 2203 int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, 2204 unsigned long bio_flags) 2205 { 2206 struct inode *inode = page->mapping->host; 2207 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2208 u64 logical = bio->bi_iter.bi_sector << 9; 2209 struct extent_map *em; 2210 u64 length = 0; 2211 u64 map_length; 2212 int ret = 0; 2213 struct btrfs_io_geometry geom; 2214 2215 if (bio_flags & EXTENT_BIO_COMPRESSED) 2216 return 0; 2217 2218 length = bio->bi_iter.bi_size; 2219 map_length = length; 2220 em = btrfs_get_chunk_map(fs_info, logical, map_length); 2221 if (IS_ERR(em)) 2222 return PTR_ERR(em); 2223 ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio), logical, 2224 map_length, &geom); 2225 if (ret < 0) 2226 goto out; 2227 2228 if (geom.len < length + size) 2229 ret = 1; 2230 out: 2231 free_extent_map(em); 2232 return ret; 2233 } 2234 2235 /* 2236 * in order to insert checksums into the metadata in large chunks, 2237 * we wait until bio submission time. All the pages in the bio are 2238 * checksummed and sums are attached onto the ordered extent record. 2239 * 2240 * At IO completion time the cums attached on the ordered extent record 2241 * are inserted into the btree 2242 */ 2243 static blk_status_t btrfs_submit_bio_start(struct inode *inode, struct bio *bio, 2244 u64 dio_file_offset) 2245 { 2246 return btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0); 2247 } 2248 2249 bool btrfs_bio_fits_in_ordered_extent(struct page *page, struct bio *bio, 2250 unsigned int size) 2251 { 2252 struct btrfs_inode *inode = BTRFS_I(page->mapping->host); 2253 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2254 struct btrfs_ordered_extent *ordered; 2255 u64 len = bio->bi_iter.bi_size + size; 2256 bool ret = true; 2257 2258 ASSERT(btrfs_is_zoned(fs_info)); 2259 ASSERT(fs_info->max_zone_append_size > 0); 2260 ASSERT(bio_op(bio) == REQ_OP_ZONE_APPEND); 2261 2262 /* Ordered extent not yet created, so we're good */ 2263 ordered = btrfs_lookup_ordered_extent(inode, page_offset(page)); 2264 if (!ordered) 2265 return ret; 2266 2267 if ((bio->bi_iter.bi_sector << SECTOR_SHIFT) + len > 2268 ordered->disk_bytenr + ordered->disk_num_bytes) 2269 ret = false; 2270 2271 btrfs_put_ordered_extent(ordered); 2272 2273 return ret; 2274 } 2275 2276 static blk_status_t extract_ordered_extent(struct btrfs_inode *inode, 2277 struct bio *bio, loff_t file_offset) 2278 { 2279 struct btrfs_ordered_extent *ordered; 2280 struct extent_map *em = NULL, *em_new = NULL; 2281 struct extent_map_tree *em_tree = &inode->extent_tree; 2282 u64 start = (u64)bio->bi_iter.bi_sector << SECTOR_SHIFT; 2283 u64 len = bio->bi_iter.bi_size; 2284 u64 end = start + len; 2285 u64 ordered_end; 2286 u64 pre, post; 2287 int ret = 0; 2288 2289 ordered = btrfs_lookup_ordered_extent(inode, file_offset); 2290 if (WARN_ON_ONCE(!ordered)) 2291 return BLK_STS_IOERR; 2292 2293 /* No need to split */ 2294 if (ordered->disk_num_bytes == len) 2295 goto out; 2296 2297 /* We cannot split once end_bio'd ordered extent */ 2298 if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes)) { 2299 ret = -EINVAL; 2300 goto out; 2301 } 2302 2303 /* We cannot split a compressed ordered extent */ 2304 if (WARN_ON_ONCE(ordered->disk_num_bytes != ordered->num_bytes)) { 2305 ret = -EINVAL; 2306 goto out; 2307 } 2308 2309 ordered_end = ordered->disk_bytenr + ordered->disk_num_bytes; 2310 /* bio must be in one ordered extent */ 2311 if (WARN_ON_ONCE(start < ordered->disk_bytenr || end > ordered_end)) { 2312 ret = -EINVAL; 2313 goto out; 2314 } 2315 2316 /* Checksum list should be empty */ 2317 if (WARN_ON_ONCE(!list_empty(&ordered->list))) { 2318 ret = -EINVAL; 2319 goto out; 2320 } 2321 2322 pre = start - ordered->disk_bytenr; 2323 post = ordered_end - end; 2324 2325 ret = btrfs_split_ordered_extent(ordered, pre, post); 2326 if (ret) 2327 goto out; 2328 2329 read_lock(&em_tree->lock); 2330 em = lookup_extent_mapping(em_tree, ordered->file_offset, len); 2331 if (!em) { 2332 read_unlock(&em_tree->lock); 2333 ret = -EIO; 2334 goto out; 2335 } 2336 read_unlock(&em_tree->lock); 2337 2338 ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)); 2339 /* 2340 * We cannot reuse em_new here but have to create a new one, as 2341 * unpin_extent_cache() expects the start of the extent map to be the 2342 * logical offset of the file, which does not hold true anymore after 2343 * splitting. 2344 */ 2345 em_new = create_io_em(inode, em->start + pre, len, 2346 em->start + pre, em->block_start + pre, len, 2347 len, len, BTRFS_COMPRESS_NONE, 2348 BTRFS_ORDERED_REGULAR); 2349 if (IS_ERR(em_new)) { 2350 ret = PTR_ERR(em_new); 2351 goto out; 2352 } 2353 free_extent_map(em_new); 2354 2355 out: 2356 free_extent_map(em); 2357 btrfs_put_ordered_extent(ordered); 2358 2359 return errno_to_blk_status(ret); 2360 } 2361 2362 /* 2363 * extent_io.c submission hook. This does the right thing for csum calculation 2364 * on write, or reading the csums from the tree before a read. 2365 * 2366 * Rules about async/sync submit, 2367 * a) read: sync submit 2368 * 2369 * b) write without checksum: sync submit 2370 * 2371 * c) write with checksum: 2372 * c-1) if bio is issued by fsync: sync submit 2373 * (sync_writers != 0) 2374 * 2375 * c-2) if root is reloc root: sync submit 2376 * (only in case of buffered IO) 2377 * 2378 * c-3) otherwise: async submit 2379 */ 2380 blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio, 2381 int mirror_num, unsigned long bio_flags) 2382 2383 { 2384 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2385 struct btrfs_root *root = BTRFS_I(inode)->root; 2386 enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA; 2387 blk_status_t ret = 0; 2388 int skip_sum; 2389 int async = !atomic_read(&BTRFS_I(inode)->sync_writers); 2390 2391 skip_sum = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) || 2392 !fs_info->csum_root; 2393 2394 if (btrfs_is_free_space_inode(BTRFS_I(inode))) 2395 metadata = BTRFS_WQ_ENDIO_FREE_SPACE; 2396 2397 if (bio_op(bio) == REQ_OP_ZONE_APPEND) { 2398 struct page *page = bio_first_bvec_all(bio)->bv_page; 2399 loff_t file_offset = page_offset(page); 2400 2401 ret = extract_ordered_extent(BTRFS_I(inode), bio, file_offset); 2402 if (ret) 2403 goto out; 2404 } 2405 2406 if (btrfs_op(bio) != BTRFS_MAP_WRITE) { 2407 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); 2408 if (ret) 2409 goto out; 2410 2411 if (bio_flags & EXTENT_BIO_COMPRESSED) { 2412 ret = btrfs_submit_compressed_read(inode, bio, 2413 mirror_num, 2414 bio_flags); 2415 goto out; 2416 } else { 2417 /* 2418 * Lookup bio sums does extra checks around whether we 2419 * need to csum or not, which is why we ignore skip_sum 2420 * here. 2421 */ 2422 ret = btrfs_lookup_bio_sums(inode, bio, NULL); 2423 if (ret) 2424 goto out; 2425 } 2426 goto mapit; 2427 } else if (async && !skip_sum) { 2428 /* csum items have already been cloned */ 2429 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 2430 goto mapit; 2431 /* we're doing a write, do the async checksumming */ 2432 ret = btrfs_wq_submit_bio(inode, bio, mirror_num, bio_flags, 2433 0, btrfs_submit_bio_start); 2434 goto out; 2435 } else if (!skip_sum) { 2436 ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0); 2437 if (ret) 2438 goto out; 2439 } 2440 2441 mapit: 2442 ret = btrfs_map_bio(fs_info, bio, mirror_num); 2443 2444 out: 2445 if (ret) { 2446 bio->bi_status = ret; 2447 bio_endio(bio); 2448 } 2449 return ret; 2450 } 2451 2452 /* 2453 * given a list of ordered sums record them in the inode. This happens 2454 * at IO completion time based on sums calculated at bio submission time. 2455 */ 2456 static int add_pending_csums(struct btrfs_trans_handle *trans, 2457 struct list_head *list) 2458 { 2459 struct btrfs_ordered_sum *sum; 2460 int ret; 2461 2462 list_for_each_entry(sum, list, list) { 2463 trans->adding_csums = true; 2464 ret = btrfs_csum_file_blocks(trans, trans->fs_info->csum_root, sum); 2465 trans->adding_csums = false; 2466 if (ret) 2467 return ret; 2468 } 2469 return 0; 2470 } 2471 2472 static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode, 2473 const u64 start, 2474 const u64 len, 2475 struct extent_state **cached_state) 2476 { 2477 u64 search_start = start; 2478 const u64 end = start + len - 1; 2479 2480 while (search_start < end) { 2481 const u64 search_len = end - search_start + 1; 2482 struct extent_map *em; 2483 u64 em_len; 2484 int ret = 0; 2485 2486 em = btrfs_get_extent(inode, NULL, 0, search_start, search_len); 2487 if (IS_ERR(em)) 2488 return PTR_ERR(em); 2489 2490 if (em->block_start != EXTENT_MAP_HOLE) 2491 goto next; 2492 2493 em_len = em->len; 2494 if (em->start < search_start) 2495 em_len -= search_start - em->start; 2496 if (em_len > search_len) 2497 em_len = search_len; 2498 2499 ret = set_extent_bit(&inode->io_tree, search_start, 2500 search_start + em_len - 1, 2501 EXTENT_DELALLOC_NEW, 0, NULL, cached_state, 2502 GFP_NOFS, NULL); 2503 next: 2504 search_start = extent_map_end(em); 2505 free_extent_map(em); 2506 if (ret) 2507 return ret; 2508 } 2509 return 0; 2510 } 2511 2512 int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, 2513 unsigned int extra_bits, 2514 struct extent_state **cached_state) 2515 { 2516 WARN_ON(PAGE_ALIGNED(end)); 2517 2518 if (start >= i_size_read(&inode->vfs_inode) && 2519 !(inode->flags & BTRFS_INODE_PREALLOC)) { 2520 /* 2521 * There can't be any extents following eof in this case so just 2522 * set the delalloc new bit for the range directly. 2523 */ 2524 extra_bits |= EXTENT_DELALLOC_NEW; 2525 } else { 2526 int ret; 2527 2528 ret = btrfs_find_new_delalloc_bytes(inode, start, 2529 end + 1 - start, 2530 cached_state); 2531 if (ret) 2532 return ret; 2533 } 2534 2535 return set_extent_delalloc(&inode->io_tree, start, end, extra_bits, 2536 cached_state); 2537 } 2538 2539 /* see btrfs_writepage_start_hook for details on why this is required */ 2540 struct btrfs_writepage_fixup { 2541 struct page *page; 2542 struct inode *inode; 2543 struct btrfs_work work; 2544 }; 2545 2546 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2547 { 2548 struct btrfs_writepage_fixup *fixup; 2549 struct btrfs_ordered_extent *ordered; 2550 struct extent_state *cached_state = NULL; 2551 struct extent_changeset *data_reserved = NULL; 2552 struct page *page; 2553 struct btrfs_inode *inode; 2554 u64 page_start; 2555 u64 page_end; 2556 int ret = 0; 2557 bool free_delalloc_space = true; 2558 2559 fixup = container_of(work, struct btrfs_writepage_fixup, work); 2560 page = fixup->page; 2561 inode = BTRFS_I(fixup->inode); 2562 page_start = page_offset(page); 2563 page_end = page_offset(page) + PAGE_SIZE - 1; 2564 2565 /* 2566 * This is similar to page_mkwrite, we need to reserve the space before 2567 * we take the page lock. 2568 */ 2569 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 2570 PAGE_SIZE); 2571 again: 2572 lock_page(page); 2573 2574 /* 2575 * Before we queued this fixup, we took a reference on the page. 2576 * page->mapping may go NULL, but it shouldn't be moved to a different 2577 * address space. 2578 */ 2579 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 2580 /* 2581 * Unfortunately this is a little tricky, either 2582 * 2583 * 1) We got here and our page had already been dealt with and 2584 * we reserved our space, thus ret == 0, so we need to just 2585 * drop our space reservation and bail. This can happen the 2586 * first time we come into the fixup worker, or could happen 2587 * while waiting for the ordered extent. 2588 * 2) Our page was already dealt with, but we happened to get an 2589 * ENOSPC above from the btrfs_delalloc_reserve_space. In 2590 * this case we obviously don't have anything to release, but 2591 * because the page was already dealt with we don't want to 2592 * mark the page with an error, so make sure we're resetting 2593 * ret to 0. This is why we have this check _before_ the ret 2594 * check, because we do not want to have a surprise ENOSPC 2595 * when the page was already properly dealt with. 2596 */ 2597 if (!ret) { 2598 btrfs_delalloc_release_extents(inode, PAGE_SIZE); 2599 btrfs_delalloc_release_space(inode, data_reserved, 2600 page_start, PAGE_SIZE, 2601 true); 2602 } 2603 ret = 0; 2604 goto out_page; 2605 } 2606 2607 /* 2608 * We can't mess with the page state unless it is locked, so now that 2609 * it is locked bail if we failed to make our space reservation. 2610 */ 2611 if (ret) 2612 goto out_page; 2613 2614 lock_extent_bits(&inode->io_tree, page_start, page_end, &cached_state); 2615 2616 /* already ordered? We're done */ 2617 if (PagePrivate2(page)) 2618 goto out_reserved; 2619 2620 ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); 2621 if (ordered) { 2622 unlock_extent_cached(&inode->io_tree, page_start, page_end, 2623 &cached_state); 2624 unlock_page(page); 2625 btrfs_start_ordered_extent(ordered, 1); 2626 btrfs_put_ordered_extent(ordered); 2627 goto again; 2628 } 2629 2630 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, 2631 &cached_state); 2632 if (ret) 2633 goto out_reserved; 2634 2635 /* 2636 * Everything went as planned, we're now the owner of a dirty page with 2637 * delayed allocation bits set and space reserved for our COW 2638 * destination. 2639 * 2640 * The page was dirty when we started, nothing should have cleaned it. 2641 */ 2642 BUG_ON(!PageDirty(page)); 2643 free_delalloc_space = false; 2644 out_reserved: 2645 btrfs_delalloc_release_extents(inode, PAGE_SIZE); 2646 if (free_delalloc_space) 2647 btrfs_delalloc_release_space(inode, data_reserved, page_start, 2648 PAGE_SIZE, true); 2649 unlock_extent_cached(&inode->io_tree, page_start, page_end, 2650 &cached_state); 2651 out_page: 2652 if (ret) { 2653 /* 2654 * We hit ENOSPC or other errors. Update the mapping and page 2655 * to reflect the errors and clean the page. 2656 */ 2657 mapping_set_error(page->mapping, ret); 2658 end_extent_writepage(page, ret, page_start, page_end); 2659 clear_page_dirty_for_io(page); 2660 SetPageError(page); 2661 } 2662 ClearPageChecked(page); 2663 unlock_page(page); 2664 put_page(page); 2665 kfree(fixup); 2666 extent_changeset_free(data_reserved); 2667 /* 2668 * As a precaution, do a delayed iput in case it would be the last iput 2669 * that could need flushing space. Recursing back to fixup worker would 2670 * deadlock. 2671 */ 2672 btrfs_add_delayed_iput(&inode->vfs_inode); 2673 } 2674 2675 /* 2676 * There are a few paths in the higher layers of the kernel that directly 2677 * set the page dirty bit without asking the filesystem if it is a 2678 * good idea. This causes problems because we want to make sure COW 2679 * properly happens and the data=ordered rules are followed. 2680 * 2681 * In our case any range that doesn't have the ORDERED bit set 2682 * hasn't been properly setup for IO. We kick off an async process 2683 * to fix it up. The async helper will wait for ordered extents, set 2684 * the delalloc bit and make it safe to write the page. 2685 */ 2686 int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end) 2687 { 2688 struct inode *inode = page->mapping->host; 2689 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2690 struct btrfs_writepage_fixup *fixup; 2691 2692 /* this page is properly in the ordered list */ 2693 if (TestClearPagePrivate2(page)) 2694 return 0; 2695 2696 /* 2697 * PageChecked is set below when we create a fixup worker for this page, 2698 * don't try to create another one if we're already PageChecked() 2699 * 2700 * The extent_io writepage code will redirty the page if we send back 2701 * EAGAIN. 2702 */ 2703 if (PageChecked(page)) 2704 return -EAGAIN; 2705 2706 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2707 if (!fixup) 2708 return -EAGAIN; 2709 2710 /* 2711 * We are already holding a reference to this inode from 2712 * write_cache_pages. We need to hold it because the space reservation 2713 * takes place outside of the page lock, and we can't trust 2714 * page->mapping outside of the page lock. 2715 */ 2716 ihold(inode); 2717 SetPageChecked(page); 2718 get_page(page); 2719 btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL, NULL); 2720 fixup->page = page; 2721 fixup->inode = inode; 2722 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2723 2724 return -EAGAIN; 2725 } 2726 2727 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2728 struct btrfs_inode *inode, u64 file_pos, 2729 struct btrfs_file_extent_item *stack_fi, 2730 const bool update_inode_bytes, 2731 u64 qgroup_reserved) 2732 { 2733 struct btrfs_root *root = inode->root; 2734 const u64 sectorsize = root->fs_info->sectorsize; 2735 struct btrfs_path *path; 2736 struct extent_buffer *leaf; 2737 struct btrfs_key ins; 2738 u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi); 2739 u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi); 2740 u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi); 2741 u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi); 2742 struct btrfs_drop_extents_args drop_args = { 0 }; 2743 int ret; 2744 2745 path = btrfs_alloc_path(); 2746 if (!path) 2747 return -ENOMEM; 2748 2749 /* 2750 * we may be replacing one extent in the tree with another. 2751 * The new extent is pinned in the extent map, and we don't want 2752 * to drop it from the cache until it is completely in the btree. 2753 * 2754 * So, tell btrfs_drop_extents to leave this extent in the cache. 2755 * the caller is expected to unpin it and allow it to be merged 2756 * with the others. 2757 */ 2758 drop_args.path = path; 2759 drop_args.start = file_pos; 2760 drop_args.end = file_pos + num_bytes; 2761 drop_args.replace_extent = true; 2762 drop_args.extent_item_size = sizeof(*stack_fi); 2763 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 2764 if (ret) 2765 goto out; 2766 2767 if (!drop_args.extent_inserted) { 2768 ins.objectid = btrfs_ino(inode); 2769 ins.offset = file_pos; 2770 ins.type = BTRFS_EXTENT_DATA_KEY; 2771 2772 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2773 sizeof(*stack_fi)); 2774 if (ret) 2775 goto out; 2776 } 2777 leaf = path->nodes[0]; 2778 btrfs_set_stack_file_extent_generation(stack_fi, trans->transid); 2779 write_extent_buffer(leaf, stack_fi, 2780 btrfs_item_ptr_offset(leaf, path->slots[0]), 2781 sizeof(struct btrfs_file_extent_item)); 2782 2783 btrfs_mark_buffer_dirty(leaf); 2784 btrfs_release_path(path); 2785 2786 /* 2787 * If we dropped an inline extent here, we know the range where it is 2788 * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the 2789 * number of bytes only for that range contaning the inline extent. 2790 * The remaining of the range will be processed when clearning the 2791 * EXTENT_DELALLOC_BIT bit through the ordered extent completion. 2792 */ 2793 if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) { 2794 u64 inline_size = round_down(drop_args.bytes_found, sectorsize); 2795 2796 inline_size = drop_args.bytes_found - inline_size; 2797 btrfs_update_inode_bytes(inode, sectorsize, inline_size); 2798 drop_args.bytes_found -= inline_size; 2799 num_bytes -= sectorsize; 2800 } 2801 2802 if (update_inode_bytes) 2803 btrfs_update_inode_bytes(inode, num_bytes, drop_args.bytes_found); 2804 2805 ins.objectid = disk_bytenr; 2806 ins.offset = disk_num_bytes; 2807 ins.type = BTRFS_EXTENT_ITEM_KEY; 2808 2809 ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes); 2810 if (ret) 2811 goto out; 2812 2813 ret = btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode), 2814 file_pos, qgroup_reserved, &ins); 2815 out: 2816 btrfs_free_path(path); 2817 2818 return ret; 2819 } 2820 2821 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 2822 u64 start, u64 len) 2823 { 2824 struct btrfs_block_group *cache; 2825 2826 cache = btrfs_lookup_block_group(fs_info, start); 2827 ASSERT(cache); 2828 2829 spin_lock(&cache->lock); 2830 cache->delalloc_bytes -= len; 2831 spin_unlock(&cache->lock); 2832 2833 btrfs_put_block_group(cache); 2834 } 2835 2836 static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans, 2837 struct btrfs_ordered_extent *oe) 2838 { 2839 struct btrfs_file_extent_item stack_fi; 2840 u64 logical_len; 2841 bool update_inode_bytes; 2842 2843 memset(&stack_fi, 0, sizeof(stack_fi)); 2844 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG); 2845 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr); 2846 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, 2847 oe->disk_num_bytes); 2848 if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags)) 2849 logical_len = oe->truncated_len; 2850 else 2851 logical_len = oe->num_bytes; 2852 btrfs_set_stack_file_extent_num_bytes(&stack_fi, logical_len); 2853 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, logical_len); 2854 btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type); 2855 /* Encryption and other encoding is reserved and all 0 */ 2856 2857 /* 2858 * For delalloc, when completing an ordered extent we update the inode's 2859 * bytes when clearing the range in the inode's io tree, so pass false 2860 * as the argument 'update_inode_bytes' to insert_reserved_file_extent(), 2861 * except if the ordered extent was truncated. 2862 */ 2863 update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) || 2864 test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags); 2865 2866 return insert_reserved_file_extent(trans, BTRFS_I(oe->inode), 2867 oe->file_offset, &stack_fi, 2868 update_inode_bytes, oe->qgroup_rsv); 2869 } 2870 2871 /* 2872 * As ordered data IO finishes, this gets called so we can finish 2873 * an ordered extent if the range of bytes in the file it covers are 2874 * fully written. 2875 */ 2876 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 2877 { 2878 struct btrfs_inode *inode = BTRFS_I(ordered_extent->inode); 2879 struct btrfs_root *root = inode->root; 2880 struct btrfs_fs_info *fs_info = root->fs_info; 2881 struct btrfs_trans_handle *trans = NULL; 2882 struct extent_io_tree *io_tree = &inode->io_tree; 2883 struct extent_state *cached_state = NULL; 2884 u64 start, end; 2885 int compress_type = 0; 2886 int ret = 0; 2887 u64 logical_len = ordered_extent->num_bytes; 2888 bool freespace_inode; 2889 bool truncated = false; 2890 bool clear_reserved_extent = true; 2891 unsigned int clear_bits = EXTENT_DEFRAG; 2892 2893 start = ordered_extent->file_offset; 2894 end = start + ordered_extent->num_bytes - 1; 2895 2896 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 2897 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && 2898 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags)) 2899 clear_bits |= EXTENT_DELALLOC_NEW; 2900 2901 freespace_inode = btrfs_is_free_space_inode(inode); 2902 2903 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 2904 ret = -EIO; 2905 goto out; 2906 } 2907 2908 if (ordered_extent->disk) 2909 btrfs_rewrite_logical_zoned(ordered_extent); 2910 2911 btrfs_free_io_failure_record(inode, start, end); 2912 2913 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 2914 truncated = true; 2915 logical_len = ordered_extent->truncated_len; 2916 /* Truncated the entire extent, don't bother adding */ 2917 if (!logical_len) 2918 goto out; 2919 } 2920 2921 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 2922 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 2923 2924 btrfs_inode_safe_disk_i_size_write(inode, 0); 2925 if (freespace_inode) 2926 trans = btrfs_join_transaction_spacecache(root); 2927 else 2928 trans = btrfs_join_transaction(root); 2929 if (IS_ERR(trans)) { 2930 ret = PTR_ERR(trans); 2931 trans = NULL; 2932 goto out; 2933 } 2934 trans->block_rsv = &inode->block_rsv; 2935 ret = btrfs_update_inode_fallback(trans, root, inode); 2936 if (ret) /* -ENOMEM or corruption */ 2937 btrfs_abort_transaction(trans, ret); 2938 goto out; 2939 } 2940 2941 clear_bits |= EXTENT_LOCKED; 2942 lock_extent_bits(io_tree, start, end, &cached_state); 2943 2944 if (freespace_inode) 2945 trans = btrfs_join_transaction_spacecache(root); 2946 else 2947 trans = btrfs_join_transaction(root); 2948 if (IS_ERR(trans)) { 2949 ret = PTR_ERR(trans); 2950 trans = NULL; 2951 goto out; 2952 } 2953 2954 trans->block_rsv = &inode->block_rsv; 2955 2956 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 2957 compress_type = ordered_extent->compress_type; 2958 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 2959 BUG_ON(compress_type); 2960 ret = btrfs_mark_extent_written(trans, inode, 2961 ordered_extent->file_offset, 2962 ordered_extent->file_offset + 2963 logical_len); 2964 } else { 2965 BUG_ON(root == fs_info->tree_root); 2966 ret = insert_ordered_extent_file_extent(trans, ordered_extent); 2967 if (!ret) { 2968 clear_reserved_extent = false; 2969 btrfs_release_delalloc_bytes(fs_info, 2970 ordered_extent->disk_bytenr, 2971 ordered_extent->disk_num_bytes); 2972 } 2973 } 2974 unpin_extent_cache(&inode->extent_tree, ordered_extent->file_offset, 2975 ordered_extent->num_bytes, trans->transid); 2976 if (ret < 0) { 2977 btrfs_abort_transaction(trans, ret); 2978 goto out; 2979 } 2980 2981 ret = add_pending_csums(trans, &ordered_extent->list); 2982 if (ret) { 2983 btrfs_abort_transaction(trans, ret); 2984 goto out; 2985 } 2986 2987 /* 2988 * If this is a new delalloc range, clear its new delalloc flag to 2989 * update the inode's number of bytes. This needs to be done first 2990 * before updating the inode item. 2991 */ 2992 if ((clear_bits & EXTENT_DELALLOC_NEW) && 2993 !test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) 2994 clear_extent_bit(&inode->io_tree, start, end, 2995 EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES, 2996 0, 0, &cached_state); 2997 2998 btrfs_inode_safe_disk_i_size_write(inode, 0); 2999 ret = btrfs_update_inode_fallback(trans, root, inode); 3000 if (ret) { /* -ENOMEM or corruption */ 3001 btrfs_abort_transaction(trans, ret); 3002 goto out; 3003 } 3004 ret = 0; 3005 out: 3006 clear_extent_bit(&inode->io_tree, start, end, clear_bits, 3007 (clear_bits & EXTENT_LOCKED) ? 1 : 0, 0, 3008 &cached_state); 3009 3010 if (trans) 3011 btrfs_end_transaction(trans); 3012 3013 if (ret || truncated) { 3014 u64 unwritten_start = start; 3015 3016 if (truncated) 3017 unwritten_start += logical_len; 3018 clear_extent_uptodate(io_tree, unwritten_start, end, NULL); 3019 3020 /* Drop the cache for the part of the extent we didn't write. */ 3021 btrfs_drop_extent_cache(inode, unwritten_start, end, 0); 3022 3023 /* 3024 * If the ordered extent had an IOERR or something else went 3025 * wrong we need to return the space for this ordered extent 3026 * back to the allocator. We only free the extent in the 3027 * truncated case if we didn't write out the extent at all. 3028 * 3029 * If we made it past insert_reserved_file_extent before we 3030 * errored out then we don't need to do this as the accounting 3031 * has already been done. 3032 */ 3033 if ((ret || !logical_len) && 3034 clear_reserved_extent && 3035 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3036 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3037 /* 3038 * Discard the range before returning it back to the 3039 * free space pool 3040 */ 3041 if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC)) 3042 btrfs_discard_extent(fs_info, 3043 ordered_extent->disk_bytenr, 3044 ordered_extent->disk_num_bytes, 3045 NULL); 3046 btrfs_free_reserved_extent(fs_info, 3047 ordered_extent->disk_bytenr, 3048 ordered_extent->disk_num_bytes, 1); 3049 } 3050 } 3051 3052 /* 3053 * This needs to be done to make sure anybody waiting knows we are done 3054 * updating everything for this ordered extent. 3055 */ 3056 btrfs_remove_ordered_extent(inode, ordered_extent); 3057 3058 /* once for us */ 3059 btrfs_put_ordered_extent(ordered_extent); 3060 /* once for the tree */ 3061 btrfs_put_ordered_extent(ordered_extent); 3062 3063 return ret; 3064 } 3065 3066 static void finish_ordered_fn(struct btrfs_work *work) 3067 { 3068 struct btrfs_ordered_extent *ordered_extent; 3069 ordered_extent = container_of(work, struct btrfs_ordered_extent, work); 3070 btrfs_finish_ordered_io(ordered_extent); 3071 } 3072 3073 void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, 3074 u64 end, int uptodate) 3075 { 3076 struct btrfs_inode *inode = BTRFS_I(page->mapping->host); 3077 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3078 struct btrfs_ordered_extent *ordered_extent = NULL; 3079 struct btrfs_workqueue *wq; 3080 3081 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); 3082 3083 ClearPagePrivate2(page); 3084 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, 3085 end - start + 1, uptodate)) 3086 return; 3087 3088 if (btrfs_is_free_space_inode(inode)) 3089 wq = fs_info->endio_freespace_worker; 3090 else 3091 wq = fs_info->endio_write_workers; 3092 3093 btrfs_init_work(&ordered_extent->work, finish_ordered_fn, NULL, NULL); 3094 btrfs_queue_work(wq, &ordered_extent->work); 3095 } 3096 3097 /* 3098 * check_data_csum - verify checksum of one sector of uncompressed data 3099 * @inode: inode 3100 * @io_bio: btrfs_io_bio which contains the csum 3101 * @bio_offset: offset to the beginning of the bio (in bytes) 3102 * @page: page where is the data to be verified 3103 * @pgoff: offset inside the page 3104 * 3105 * The length of such check is always one sector size. 3106 */ 3107 static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, 3108 u32 bio_offset, struct page *page, u32 pgoff) 3109 { 3110 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3111 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 3112 char *kaddr; 3113 u32 len = fs_info->sectorsize; 3114 const u32 csum_size = fs_info->csum_size; 3115 unsigned int offset_sectors; 3116 u8 *csum_expected; 3117 u8 csum[BTRFS_CSUM_SIZE]; 3118 3119 ASSERT(pgoff + len <= PAGE_SIZE); 3120 3121 offset_sectors = bio_offset >> fs_info->sectorsize_bits; 3122 csum_expected = ((u8 *)io_bio->csum) + offset_sectors * csum_size; 3123 3124 kaddr = kmap_atomic(page); 3125 shash->tfm = fs_info->csum_shash; 3126 3127 crypto_shash_digest(shash, kaddr + pgoff, len, csum); 3128 3129 if (memcmp(csum, csum_expected, csum_size)) 3130 goto zeroit; 3131 3132 kunmap_atomic(kaddr); 3133 return 0; 3134 zeroit: 3135 btrfs_print_data_csum_error(BTRFS_I(inode), page_offset(page) + pgoff, 3136 csum, csum_expected, io_bio->mirror_num); 3137 if (io_bio->device) 3138 btrfs_dev_stat_inc_and_print(io_bio->device, 3139 BTRFS_DEV_STAT_CORRUPTION_ERRS); 3140 memset(kaddr + pgoff, 1, len); 3141 flush_dcache_page(page); 3142 kunmap_atomic(kaddr); 3143 return -EIO; 3144 } 3145 3146 /* 3147 * When reads are done, we need to check csums to verify the data is correct. 3148 * if there's a match, we allow the bio to finish. If not, the code in 3149 * extent_io.c will try to find good copies for us. 3150 * 3151 * @bio_offset: offset to the beginning of the bio (in bytes) 3152 * @start: file offset of the range start 3153 * @end: file offset of the range end (inclusive) 3154 * @mirror: mirror number 3155 */ 3156 int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset, 3157 struct page *page, u64 start, u64 end, int mirror) 3158 { 3159 struct inode *inode = page->mapping->host; 3160 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 3161 struct btrfs_root *root = BTRFS_I(inode)->root; 3162 const u32 sectorsize = root->fs_info->sectorsize; 3163 u32 pg_off; 3164 3165 if (PageChecked(page)) { 3166 ClearPageChecked(page); 3167 return 0; 3168 } 3169 3170 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 3171 return 0; 3172 3173 if (!root->fs_info->csum_root) 3174 return 0; 3175 3176 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID && 3177 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) { 3178 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM); 3179 return 0; 3180 } 3181 3182 ASSERT(page_offset(page) <= start && 3183 end <= page_offset(page) + PAGE_SIZE - 1); 3184 for (pg_off = offset_in_page(start); 3185 pg_off < offset_in_page(end); 3186 pg_off += sectorsize, bio_offset += sectorsize) { 3187 int ret; 3188 3189 ret = check_data_csum(inode, io_bio, bio_offset, page, pg_off); 3190 if (ret < 0) 3191 return -EIO; 3192 } 3193 return 0; 3194 } 3195 3196 /* 3197 * btrfs_add_delayed_iput - perform a delayed iput on @inode 3198 * 3199 * @inode: The inode we want to perform iput on 3200 * 3201 * This function uses the generic vfs_inode::i_count to track whether we should 3202 * just decrement it (in case it's > 1) or if this is the last iput then link 3203 * the inode to the delayed iput machinery. Delayed iputs are processed at 3204 * transaction commit time/superblock commit/cleaner kthread. 3205 */ 3206 void btrfs_add_delayed_iput(struct inode *inode) 3207 { 3208 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3209 struct btrfs_inode *binode = BTRFS_I(inode); 3210 3211 if (atomic_add_unless(&inode->i_count, -1, 1)) 3212 return; 3213 3214 atomic_inc(&fs_info->nr_delayed_iputs); 3215 spin_lock(&fs_info->delayed_iput_lock); 3216 ASSERT(list_empty(&binode->delayed_iput)); 3217 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs); 3218 spin_unlock(&fs_info->delayed_iput_lock); 3219 if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags)) 3220 wake_up_process(fs_info->cleaner_kthread); 3221 } 3222 3223 static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info, 3224 struct btrfs_inode *inode) 3225 { 3226 list_del_init(&inode->delayed_iput); 3227 spin_unlock(&fs_info->delayed_iput_lock); 3228 iput(&inode->vfs_inode); 3229 if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) 3230 wake_up(&fs_info->delayed_iputs_wait); 3231 spin_lock(&fs_info->delayed_iput_lock); 3232 } 3233 3234 static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info, 3235 struct btrfs_inode *inode) 3236 { 3237 if (!list_empty(&inode->delayed_iput)) { 3238 spin_lock(&fs_info->delayed_iput_lock); 3239 if (!list_empty(&inode->delayed_iput)) 3240 run_delayed_iput_locked(fs_info, inode); 3241 spin_unlock(&fs_info->delayed_iput_lock); 3242 } 3243 } 3244 3245 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 3246 { 3247 3248 spin_lock(&fs_info->delayed_iput_lock); 3249 while (!list_empty(&fs_info->delayed_iputs)) { 3250 struct btrfs_inode *inode; 3251 3252 inode = list_first_entry(&fs_info->delayed_iputs, 3253 struct btrfs_inode, delayed_iput); 3254 run_delayed_iput_locked(fs_info, inode); 3255 } 3256 spin_unlock(&fs_info->delayed_iput_lock); 3257 } 3258 3259 /** 3260 * Wait for flushing all delayed iputs 3261 * 3262 * @fs_info: the filesystem 3263 * 3264 * This will wait on any delayed iputs that are currently running with KILLABLE 3265 * set. Once they are all done running we will return, unless we are killed in 3266 * which case we return EINTR. This helps in user operations like fallocate etc 3267 * that might get blocked on the iputs. 3268 * 3269 * Return EINTR if we were killed, 0 if nothing's pending 3270 */ 3271 int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info) 3272 { 3273 int ret = wait_event_killable(fs_info->delayed_iputs_wait, 3274 atomic_read(&fs_info->nr_delayed_iputs) == 0); 3275 if (ret) 3276 return -EINTR; 3277 return 0; 3278 } 3279 3280 /* 3281 * This creates an orphan entry for the given inode in case something goes wrong 3282 * in the middle of an unlink. 3283 */ 3284 int btrfs_orphan_add(struct btrfs_trans_handle *trans, 3285 struct btrfs_inode *inode) 3286 { 3287 int ret; 3288 3289 ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode)); 3290 if (ret && ret != -EEXIST) { 3291 btrfs_abort_transaction(trans, ret); 3292 return ret; 3293 } 3294 3295 return 0; 3296 } 3297 3298 /* 3299 * We have done the delete so we can go ahead and remove the orphan item for 3300 * this particular inode. 3301 */ 3302 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3303 struct btrfs_inode *inode) 3304 { 3305 return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode)); 3306 } 3307 3308 /* 3309 * this cleans up any orphans that may be left on the list from the last use 3310 * of this root. 3311 */ 3312 int btrfs_orphan_cleanup(struct btrfs_root *root) 3313 { 3314 struct btrfs_fs_info *fs_info = root->fs_info; 3315 struct btrfs_path *path; 3316 struct extent_buffer *leaf; 3317 struct btrfs_key key, found_key; 3318 struct btrfs_trans_handle *trans; 3319 struct inode *inode; 3320 u64 last_objectid = 0; 3321 int ret = 0, nr_unlink = 0; 3322 3323 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED)) 3324 return 0; 3325 3326 path = btrfs_alloc_path(); 3327 if (!path) { 3328 ret = -ENOMEM; 3329 goto out; 3330 } 3331 path->reada = READA_BACK; 3332 3333 key.objectid = BTRFS_ORPHAN_OBJECTID; 3334 key.type = BTRFS_ORPHAN_ITEM_KEY; 3335 key.offset = (u64)-1; 3336 3337 while (1) { 3338 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3339 if (ret < 0) 3340 goto out; 3341 3342 /* 3343 * if ret == 0 means we found what we were searching for, which 3344 * is weird, but possible, so only screw with path if we didn't 3345 * find the key and see if we have stuff that matches 3346 */ 3347 if (ret > 0) { 3348 ret = 0; 3349 if (path->slots[0] == 0) 3350 break; 3351 path->slots[0]--; 3352 } 3353 3354 /* pull out the item */ 3355 leaf = path->nodes[0]; 3356 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3357 3358 /* make sure the item matches what we want */ 3359 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3360 break; 3361 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3362 break; 3363 3364 /* release the path since we're done with it */ 3365 btrfs_release_path(path); 3366 3367 /* 3368 * this is where we are basically btrfs_lookup, without the 3369 * crossing root thing. we store the inode number in the 3370 * offset of the orphan item. 3371 */ 3372 3373 if (found_key.offset == last_objectid) { 3374 btrfs_err(fs_info, 3375 "Error removing orphan entry, stopping orphan cleanup"); 3376 ret = -EINVAL; 3377 goto out; 3378 } 3379 3380 last_objectid = found_key.offset; 3381 3382 found_key.objectid = found_key.offset; 3383 found_key.type = BTRFS_INODE_ITEM_KEY; 3384 found_key.offset = 0; 3385 inode = btrfs_iget(fs_info->sb, last_objectid, root); 3386 ret = PTR_ERR_OR_ZERO(inode); 3387 if (ret && ret != -ENOENT) 3388 goto out; 3389 3390 if (ret == -ENOENT && root == fs_info->tree_root) { 3391 struct btrfs_root *dead_root; 3392 int is_dead_root = 0; 3393 3394 /* 3395 * this is an orphan in the tree root. Currently these 3396 * could come from 2 sources: 3397 * a) a snapshot deletion in progress 3398 * b) a free space cache inode 3399 * We need to distinguish those two, as the snapshot 3400 * orphan must not get deleted. 3401 * find_dead_roots already ran before us, so if this 3402 * is a snapshot deletion, we should find the root 3403 * in the fs_roots radix tree. 3404 */ 3405 3406 spin_lock(&fs_info->fs_roots_radix_lock); 3407 dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, 3408 (unsigned long)found_key.objectid); 3409 if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) 3410 is_dead_root = 1; 3411 spin_unlock(&fs_info->fs_roots_radix_lock); 3412 3413 if (is_dead_root) { 3414 /* prevent this orphan from being found again */ 3415 key.offset = found_key.objectid - 1; 3416 continue; 3417 } 3418 3419 } 3420 3421 /* 3422 * If we have an inode with links, there are a couple of 3423 * possibilities. Old kernels (before v3.12) used to create an 3424 * orphan item for truncate indicating that there were possibly 3425 * extent items past i_size that needed to be deleted. In v3.12, 3426 * truncate was changed to update i_size in sync with the extent 3427 * items, but the (useless) orphan item was still created. Since 3428 * v4.18, we don't create the orphan item for truncate at all. 3429 * 3430 * So, this item could mean that we need to do a truncate, but 3431 * only if this filesystem was last used on a pre-v3.12 kernel 3432 * and was not cleanly unmounted. The odds of that are quite 3433 * slim, and it's a pain to do the truncate now, so just delete 3434 * the orphan item. 3435 * 3436 * It's also possible that this orphan item was supposed to be 3437 * deleted but wasn't. The inode number may have been reused, 3438 * but either way, we can delete the orphan item. 3439 */ 3440 if (ret == -ENOENT || inode->i_nlink) { 3441 if (!ret) 3442 iput(inode); 3443 trans = btrfs_start_transaction(root, 1); 3444 if (IS_ERR(trans)) { 3445 ret = PTR_ERR(trans); 3446 goto out; 3447 } 3448 btrfs_debug(fs_info, "auto deleting %Lu", 3449 found_key.objectid); 3450 ret = btrfs_del_orphan_item(trans, root, 3451 found_key.objectid); 3452 btrfs_end_transaction(trans); 3453 if (ret) 3454 goto out; 3455 continue; 3456 } 3457 3458 nr_unlink++; 3459 3460 /* this will do delete_inode and everything for us */ 3461 iput(inode); 3462 } 3463 /* release the path since we're done with it */ 3464 btrfs_release_path(path); 3465 3466 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE; 3467 3468 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3469 trans = btrfs_join_transaction(root); 3470 if (!IS_ERR(trans)) 3471 btrfs_end_transaction(trans); 3472 } 3473 3474 if (nr_unlink) 3475 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3476 3477 out: 3478 if (ret) 3479 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3480 btrfs_free_path(path); 3481 return ret; 3482 } 3483 3484 /* 3485 * very simple check to peek ahead in the leaf looking for xattrs. If we 3486 * don't find any xattrs, we know there can't be any acls. 3487 * 3488 * slot is the slot the inode is in, objectid is the objectid of the inode 3489 */ 3490 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3491 int slot, u64 objectid, 3492 int *first_xattr_slot) 3493 { 3494 u32 nritems = btrfs_header_nritems(leaf); 3495 struct btrfs_key found_key; 3496 static u64 xattr_access = 0; 3497 static u64 xattr_default = 0; 3498 int scanned = 0; 3499 3500 if (!xattr_access) { 3501 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3502 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3503 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3504 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3505 } 3506 3507 slot++; 3508 *first_xattr_slot = -1; 3509 while (slot < nritems) { 3510 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3511 3512 /* we found a different objectid, there must not be acls */ 3513 if (found_key.objectid != objectid) 3514 return 0; 3515 3516 /* we found an xattr, assume we've got an acl */ 3517 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3518 if (*first_xattr_slot == -1) 3519 *first_xattr_slot = slot; 3520 if (found_key.offset == xattr_access || 3521 found_key.offset == xattr_default) 3522 return 1; 3523 } 3524 3525 /* 3526 * we found a key greater than an xattr key, there can't 3527 * be any acls later on 3528 */ 3529 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3530 return 0; 3531 3532 slot++; 3533 scanned++; 3534 3535 /* 3536 * it goes inode, inode backrefs, xattrs, extents, 3537 * so if there are a ton of hard links to an inode there can 3538 * be a lot of backrefs. Don't waste time searching too hard, 3539 * this is just an optimization 3540 */ 3541 if (scanned >= 8) 3542 break; 3543 } 3544 /* we hit the end of the leaf before we found an xattr or 3545 * something larger than an xattr. We have to assume the inode 3546 * has acls 3547 */ 3548 if (*first_xattr_slot == -1) 3549 *first_xattr_slot = slot; 3550 return 1; 3551 } 3552 3553 /* 3554 * read an inode from the btree into the in-memory inode 3555 */ 3556 static int btrfs_read_locked_inode(struct inode *inode, 3557 struct btrfs_path *in_path) 3558 { 3559 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3560 struct btrfs_path *path = in_path; 3561 struct extent_buffer *leaf; 3562 struct btrfs_inode_item *inode_item; 3563 struct btrfs_root *root = BTRFS_I(inode)->root; 3564 struct btrfs_key location; 3565 unsigned long ptr; 3566 int maybe_acls; 3567 u32 rdev; 3568 int ret; 3569 bool filled = false; 3570 int first_xattr_slot; 3571 3572 ret = btrfs_fill_inode(inode, &rdev); 3573 if (!ret) 3574 filled = true; 3575 3576 if (!path) { 3577 path = btrfs_alloc_path(); 3578 if (!path) 3579 return -ENOMEM; 3580 } 3581 3582 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3583 3584 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3585 if (ret) { 3586 if (path != in_path) 3587 btrfs_free_path(path); 3588 return ret; 3589 } 3590 3591 leaf = path->nodes[0]; 3592 3593 if (filled) 3594 goto cache_index; 3595 3596 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3597 struct btrfs_inode_item); 3598 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3599 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3600 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3601 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3602 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item)); 3603 btrfs_inode_set_file_extent_range(BTRFS_I(inode), 0, 3604 round_up(i_size_read(inode), fs_info->sectorsize)); 3605 3606 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime); 3607 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime); 3608 3609 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime); 3610 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime); 3611 3612 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime); 3613 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime); 3614 3615 BTRFS_I(inode)->i_otime.tv_sec = 3616 btrfs_timespec_sec(leaf, &inode_item->otime); 3617 BTRFS_I(inode)->i_otime.tv_nsec = 3618 btrfs_timespec_nsec(leaf, &inode_item->otime); 3619 3620 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3621 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3622 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3623 3624 inode_set_iversion_queried(inode, 3625 btrfs_inode_sequence(leaf, inode_item)); 3626 inode->i_generation = BTRFS_I(inode)->generation; 3627 inode->i_rdev = 0; 3628 rdev = btrfs_inode_rdev(leaf, inode_item); 3629 3630 BTRFS_I(inode)->index_cnt = (u64)-1; 3631 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item); 3632 3633 cache_index: 3634 /* 3635 * If we were modified in the current generation and evicted from memory 3636 * and then re-read we need to do a full sync since we don't have any 3637 * idea about which extents were modified before we were evicted from 3638 * cache. 3639 * 3640 * This is required for both inode re-read from disk and delayed inode 3641 * in delayed_nodes_tree. 3642 */ 3643 if (BTRFS_I(inode)->last_trans == fs_info->generation) 3644 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3645 &BTRFS_I(inode)->runtime_flags); 3646 3647 /* 3648 * We don't persist the id of the transaction where an unlink operation 3649 * against the inode was last made. So here we assume the inode might 3650 * have been evicted, and therefore the exact value of last_unlink_trans 3651 * lost, and set it to last_trans to avoid metadata inconsistencies 3652 * between the inode and its parent if the inode is fsync'ed and the log 3653 * replayed. For example, in the scenario: 3654 * 3655 * touch mydir/foo 3656 * ln mydir/foo mydir/bar 3657 * sync 3658 * unlink mydir/bar 3659 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 3660 * xfs_io -c fsync mydir/foo 3661 * <power failure> 3662 * mount fs, triggers fsync log replay 3663 * 3664 * We must make sure that when we fsync our inode foo we also log its 3665 * parent inode, otherwise after log replay the parent still has the 3666 * dentry with the "bar" name but our inode foo has a link count of 1 3667 * and doesn't have an inode ref with the name "bar" anymore. 3668 * 3669 * Setting last_unlink_trans to last_trans is a pessimistic approach, 3670 * but it guarantees correctness at the expense of occasional full 3671 * transaction commits on fsync if our inode is a directory, or if our 3672 * inode is not a directory, logging its parent unnecessarily. 3673 */ 3674 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans; 3675 3676 /* 3677 * Same logic as for last_unlink_trans. We don't persist the generation 3678 * of the last transaction where this inode was used for a reflink 3679 * operation, so after eviction and reloading the inode we must be 3680 * pessimistic and assume the last transaction that modified the inode. 3681 */ 3682 BTRFS_I(inode)->last_reflink_trans = BTRFS_I(inode)->last_trans; 3683 3684 path->slots[0]++; 3685 if (inode->i_nlink != 1 || 3686 path->slots[0] >= btrfs_header_nritems(leaf)) 3687 goto cache_acl; 3688 3689 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3690 if (location.objectid != btrfs_ino(BTRFS_I(inode))) 3691 goto cache_acl; 3692 3693 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3694 if (location.type == BTRFS_INODE_REF_KEY) { 3695 struct btrfs_inode_ref *ref; 3696 3697 ref = (struct btrfs_inode_ref *)ptr; 3698 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3699 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3700 struct btrfs_inode_extref *extref; 3701 3702 extref = (struct btrfs_inode_extref *)ptr; 3703 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3704 extref); 3705 } 3706 cache_acl: 3707 /* 3708 * try to precache a NULL acl entry for files that don't have 3709 * any xattrs or acls 3710 */ 3711 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3712 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot); 3713 if (first_xattr_slot != -1) { 3714 path->slots[0] = first_xattr_slot; 3715 ret = btrfs_load_inode_props(inode, path); 3716 if (ret) 3717 btrfs_err(fs_info, 3718 "error loading props for ino %llu (root %llu): %d", 3719 btrfs_ino(BTRFS_I(inode)), 3720 root->root_key.objectid, ret); 3721 } 3722 if (path != in_path) 3723 btrfs_free_path(path); 3724 3725 if (!maybe_acls) 3726 cache_no_acl(inode); 3727 3728 switch (inode->i_mode & S_IFMT) { 3729 case S_IFREG: 3730 inode->i_mapping->a_ops = &btrfs_aops; 3731 inode->i_fop = &btrfs_file_operations; 3732 inode->i_op = &btrfs_file_inode_operations; 3733 break; 3734 case S_IFDIR: 3735 inode->i_fop = &btrfs_dir_file_operations; 3736 inode->i_op = &btrfs_dir_inode_operations; 3737 break; 3738 case S_IFLNK: 3739 inode->i_op = &btrfs_symlink_inode_operations; 3740 inode_nohighmem(inode); 3741 inode->i_mapping->a_ops = &btrfs_aops; 3742 break; 3743 default: 3744 inode->i_op = &btrfs_special_inode_operations; 3745 init_special_inode(inode, inode->i_mode, rdev); 3746 break; 3747 } 3748 3749 btrfs_sync_inode_flags_to_i_flags(inode); 3750 return 0; 3751 } 3752 3753 /* 3754 * given a leaf and an inode, copy the inode fields into the leaf 3755 */ 3756 static void fill_inode_item(struct btrfs_trans_handle *trans, 3757 struct extent_buffer *leaf, 3758 struct btrfs_inode_item *item, 3759 struct inode *inode) 3760 { 3761 struct btrfs_map_token token; 3762 3763 btrfs_init_map_token(&token, leaf); 3764 3765 btrfs_set_token_inode_uid(&token, item, i_uid_read(inode)); 3766 btrfs_set_token_inode_gid(&token, item, i_gid_read(inode)); 3767 btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size); 3768 btrfs_set_token_inode_mode(&token, item, inode->i_mode); 3769 btrfs_set_token_inode_nlink(&token, item, inode->i_nlink); 3770 3771 btrfs_set_token_timespec_sec(&token, &item->atime, 3772 inode->i_atime.tv_sec); 3773 btrfs_set_token_timespec_nsec(&token, &item->atime, 3774 inode->i_atime.tv_nsec); 3775 3776 btrfs_set_token_timespec_sec(&token, &item->mtime, 3777 inode->i_mtime.tv_sec); 3778 btrfs_set_token_timespec_nsec(&token, &item->mtime, 3779 inode->i_mtime.tv_nsec); 3780 3781 btrfs_set_token_timespec_sec(&token, &item->ctime, 3782 inode->i_ctime.tv_sec); 3783 btrfs_set_token_timespec_nsec(&token, &item->ctime, 3784 inode->i_ctime.tv_nsec); 3785 3786 btrfs_set_token_timespec_sec(&token, &item->otime, 3787 BTRFS_I(inode)->i_otime.tv_sec); 3788 btrfs_set_token_timespec_nsec(&token, &item->otime, 3789 BTRFS_I(inode)->i_otime.tv_nsec); 3790 3791 btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode)); 3792 btrfs_set_token_inode_generation(&token, item, 3793 BTRFS_I(inode)->generation); 3794 btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode)); 3795 btrfs_set_token_inode_transid(&token, item, trans->transid); 3796 btrfs_set_token_inode_rdev(&token, item, inode->i_rdev); 3797 btrfs_set_token_inode_flags(&token, item, BTRFS_I(inode)->flags); 3798 btrfs_set_token_inode_block_group(&token, item, 0); 3799 } 3800 3801 /* 3802 * copy everything in the in-memory inode into the btree. 3803 */ 3804 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 3805 struct btrfs_root *root, 3806 struct btrfs_inode *inode) 3807 { 3808 struct btrfs_inode_item *inode_item; 3809 struct btrfs_path *path; 3810 struct extent_buffer *leaf; 3811 int ret; 3812 3813 path = btrfs_alloc_path(); 3814 if (!path) 3815 return -ENOMEM; 3816 3817 ret = btrfs_lookup_inode(trans, root, path, &inode->location, 1); 3818 if (ret) { 3819 if (ret > 0) 3820 ret = -ENOENT; 3821 goto failed; 3822 } 3823 3824 leaf = path->nodes[0]; 3825 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3826 struct btrfs_inode_item); 3827 3828 fill_inode_item(trans, leaf, inode_item, &inode->vfs_inode); 3829 btrfs_mark_buffer_dirty(leaf); 3830 btrfs_set_inode_last_trans(trans, inode); 3831 ret = 0; 3832 failed: 3833 btrfs_free_path(path); 3834 return ret; 3835 } 3836 3837 /* 3838 * copy everything in the in-memory inode into the btree. 3839 */ 3840 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 3841 struct btrfs_root *root, 3842 struct btrfs_inode *inode) 3843 { 3844 struct btrfs_fs_info *fs_info = root->fs_info; 3845 int ret; 3846 3847 /* 3848 * If the inode is a free space inode, we can deadlock during commit 3849 * if we put it into the delayed code. 3850 * 3851 * The data relocation inode should also be directly updated 3852 * without delay 3853 */ 3854 if (!btrfs_is_free_space_inode(inode) 3855 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 3856 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 3857 btrfs_update_root_times(trans, root); 3858 3859 ret = btrfs_delayed_update_inode(trans, root, inode); 3860 if (!ret) 3861 btrfs_set_inode_last_trans(trans, inode); 3862 return ret; 3863 } 3864 3865 return btrfs_update_inode_item(trans, root, inode); 3866 } 3867 3868 int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 3869 struct btrfs_root *root, struct btrfs_inode *inode) 3870 { 3871 int ret; 3872 3873 ret = btrfs_update_inode(trans, root, inode); 3874 if (ret == -ENOSPC) 3875 return btrfs_update_inode_item(trans, root, inode); 3876 return ret; 3877 } 3878 3879 /* 3880 * unlink helper that gets used here in inode.c and in the tree logging 3881 * recovery code. It remove a link in a directory with a given name, and 3882 * also drops the back refs in the inode to the directory 3883 */ 3884 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 3885 struct btrfs_root *root, 3886 struct btrfs_inode *dir, 3887 struct btrfs_inode *inode, 3888 const char *name, int name_len) 3889 { 3890 struct btrfs_fs_info *fs_info = root->fs_info; 3891 struct btrfs_path *path; 3892 int ret = 0; 3893 struct btrfs_dir_item *di; 3894 u64 index; 3895 u64 ino = btrfs_ino(inode); 3896 u64 dir_ino = btrfs_ino(dir); 3897 3898 path = btrfs_alloc_path(); 3899 if (!path) { 3900 ret = -ENOMEM; 3901 goto out; 3902 } 3903 3904 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 3905 name, name_len, -1); 3906 if (IS_ERR_OR_NULL(di)) { 3907 ret = di ? PTR_ERR(di) : -ENOENT; 3908 goto err; 3909 } 3910 ret = btrfs_delete_one_dir_name(trans, root, path, di); 3911 if (ret) 3912 goto err; 3913 btrfs_release_path(path); 3914 3915 /* 3916 * If we don't have dir index, we have to get it by looking up 3917 * the inode ref, since we get the inode ref, remove it directly, 3918 * it is unnecessary to do delayed deletion. 3919 * 3920 * But if we have dir index, needn't search inode ref to get it. 3921 * Since the inode ref is close to the inode item, it is better 3922 * that we delay to delete it, and just do this deletion when 3923 * we update the inode item. 3924 */ 3925 if (inode->dir_index) { 3926 ret = btrfs_delayed_delete_inode_ref(inode); 3927 if (!ret) { 3928 index = inode->dir_index; 3929 goto skip_backref; 3930 } 3931 } 3932 3933 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino, 3934 dir_ino, &index); 3935 if (ret) { 3936 btrfs_info(fs_info, 3937 "failed to delete reference to %.*s, inode %llu parent %llu", 3938 name_len, name, ino, dir_ino); 3939 btrfs_abort_transaction(trans, ret); 3940 goto err; 3941 } 3942 skip_backref: 3943 ret = btrfs_delete_delayed_dir_index(trans, dir, index); 3944 if (ret) { 3945 btrfs_abort_transaction(trans, ret); 3946 goto err; 3947 } 3948 3949 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode, 3950 dir_ino); 3951 if (ret != 0 && ret != -ENOENT) { 3952 btrfs_abort_transaction(trans, ret); 3953 goto err; 3954 } 3955 3956 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir, 3957 index); 3958 if (ret == -ENOENT) 3959 ret = 0; 3960 else if (ret) 3961 btrfs_abort_transaction(trans, ret); 3962 3963 /* 3964 * If we have a pending delayed iput we could end up with the final iput 3965 * being run in btrfs-cleaner context. If we have enough of these built 3966 * up we can end up burning a lot of time in btrfs-cleaner without any 3967 * way to throttle the unlinks. Since we're currently holding a ref on 3968 * the inode we can run the delayed iput here without any issues as the 3969 * final iput won't be done until after we drop the ref we're currently 3970 * holding. 3971 */ 3972 btrfs_run_delayed_iput(fs_info, inode); 3973 err: 3974 btrfs_free_path(path); 3975 if (ret) 3976 goto out; 3977 3978 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2); 3979 inode_inc_iversion(&inode->vfs_inode); 3980 inode_inc_iversion(&dir->vfs_inode); 3981 inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime = 3982 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode); 3983 ret = btrfs_update_inode(trans, root, dir); 3984 out: 3985 return ret; 3986 } 3987 3988 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 3989 struct btrfs_root *root, 3990 struct btrfs_inode *dir, struct btrfs_inode *inode, 3991 const char *name, int name_len) 3992 { 3993 int ret; 3994 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len); 3995 if (!ret) { 3996 drop_nlink(&inode->vfs_inode); 3997 ret = btrfs_update_inode(trans, root, inode); 3998 } 3999 return ret; 4000 } 4001 4002 /* 4003 * helper to start transaction for unlink and rmdir. 4004 * 4005 * unlink and rmdir are special in btrfs, they do not always free space, so 4006 * if we cannot make our reservations the normal way try and see if there is 4007 * plenty of slack room in the global reserve to migrate, otherwise we cannot 4008 * allow the unlink to occur. 4009 */ 4010 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) 4011 { 4012 struct btrfs_root *root = BTRFS_I(dir)->root; 4013 4014 /* 4015 * 1 for the possible orphan item 4016 * 1 for the dir item 4017 * 1 for the dir index 4018 * 1 for the inode ref 4019 * 1 for the inode 4020 */ 4021 return btrfs_start_transaction_fallback_global_rsv(root, 5); 4022 } 4023 4024 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 4025 { 4026 struct btrfs_root *root = BTRFS_I(dir)->root; 4027 struct btrfs_trans_handle *trans; 4028 struct inode *inode = d_inode(dentry); 4029 int ret; 4030 4031 trans = __unlink_start_trans(dir); 4032 if (IS_ERR(trans)) 4033 return PTR_ERR(trans); 4034 4035 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4036 0); 4037 4038 ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4039 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4040 dentry->d_name.len); 4041 if (ret) 4042 goto out; 4043 4044 if (inode->i_nlink == 0) { 4045 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 4046 if (ret) 4047 goto out; 4048 } 4049 4050 out: 4051 btrfs_end_transaction(trans); 4052 btrfs_btree_balance_dirty(root->fs_info); 4053 return ret; 4054 } 4055 4056 static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 4057 struct inode *dir, struct dentry *dentry) 4058 { 4059 struct btrfs_root *root = BTRFS_I(dir)->root; 4060 struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); 4061 struct btrfs_path *path; 4062 struct extent_buffer *leaf; 4063 struct btrfs_dir_item *di; 4064 struct btrfs_key key; 4065 const char *name = dentry->d_name.name; 4066 int name_len = dentry->d_name.len; 4067 u64 index; 4068 int ret; 4069 u64 objectid; 4070 u64 dir_ino = btrfs_ino(BTRFS_I(dir)); 4071 4072 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) { 4073 objectid = inode->root->root_key.objectid; 4074 } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 4075 objectid = inode->location.objectid; 4076 } else { 4077 WARN_ON(1); 4078 return -EINVAL; 4079 } 4080 4081 path = btrfs_alloc_path(); 4082 if (!path) 4083 return -ENOMEM; 4084 4085 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4086 name, name_len, -1); 4087 if (IS_ERR_OR_NULL(di)) { 4088 ret = di ? PTR_ERR(di) : -ENOENT; 4089 goto out; 4090 } 4091 4092 leaf = path->nodes[0]; 4093 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4094 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 4095 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4096 if (ret) { 4097 btrfs_abort_transaction(trans, ret); 4098 goto out; 4099 } 4100 btrfs_release_path(path); 4101 4102 /* 4103 * This is a placeholder inode for a subvolume we didn't have a 4104 * reference to at the time of the snapshot creation. In the meantime 4105 * we could have renamed the real subvol link into our snapshot, so 4106 * depending on btrfs_del_root_ref to return -ENOENT here is incorret. 4107 * Instead simply lookup the dir_index_item for this entry so we can 4108 * remove it. Otherwise we know we have a ref to the root and we can 4109 * call btrfs_del_root_ref, and it _shouldn't_ fail. 4110 */ 4111 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 4112 di = btrfs_search_dir_index_item(root, path, dir_ino, 4113 name, name_len); 4114 if (IS_ERR_OR_NULL(di)) { 4115 if (!di) 4116 ret = -ENOENT; 4117 else 4118 ret = PTR_ERR(di); 4119 btrfs_abort_transaction(trans, ret); 4120 goto out; 4121 } 4122 4123 leaf = path->nodes[0]; 4124 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4125 index = key.offset; 4126 btrfs_release_path(path); 4127 } else { 4128 ret = btrfs_del_root_ref(trans, objectid, 4129 root->root_key.objectid, dir_ino, 4130 &index, name, name_len); 4131 if (ret) { 4132 btrfs_abort_transaction(trans, ret); 4133 goto out; 4134 } 4135 } 4136 4137 ret = btrfs_delete_delayed_dir_index(trans, BTRFS_I(dir), index); 4138 if (ret) { 4139 btrfs_abort_transaction(trans, ret); 4140 goto out; 4141 } 4142 4143 btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2); 4144 inode_inc_iversion(dir); 4145 dir->i_mtime = dir->i_ctime = current_time(dir); 4146 ret = btrfs_update_inode_fallback(trans, root, BTRFS_I(dir)); 4147 if (ret) 4148 btrfs_abort_transaction(trans, ret); 4149 out: 4150 btrfs_free_path(path); 4151 return ret; 4152 } 4153 4154 /* 4155 * Helper to check if the subvolume references other subvolumes or if it's 4156 * default. 4157 */ 4158 static noinline int may_destroy_subvol(struct btrfs_root *root) 4159 { 4160 struct btrfs_fs_info *fs_info = root->fs_info; 4161 struct btrfs_path *path; 4162 struct btrfs_dir_item *di; 4163 struct btrfs_key key; 4164 u64 dir_id; 4165 int ret; 4166 4167 path = btrfs_alloc_path(); 4168 if (!path) 4169 return -ENOMEM; 4170 4171 /* Make sure this root isn't set as the default subvol */ 4172 dir_id = btrfs_super_root_dir(fs_info->super_copy); 4173 di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path, 4174 dir_id, "default", 7, 0); 4175 if (di && !IS_ERR(di)) { 4176 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key); 4177 if (key.objectid == root->root_key.objectid) { 4178 ret = -EPERM; 4179 btrfs_err(fs_info, 4180 "deleting default subvolume %llu is not allowed", 4181 key.objectid); 4182 goto out; 4183 } 4184 btrfs_release_path(path); 4185 } 4186 4187 key.objectid = root->root_key.objectid; 4188 key.type = BTRFS_ROOT_REF_KEY; 4189 key.offset = (u64)-1; 4190 4191 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 4192 if (ret < 0) 4193 goto out; 4194 BUG_ON(ret == 0); 4195 4196 ret = 0; 4197 if (path->slots[0] > 0) { 4198 path->slots[0]--; 4199 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); 4200 if (key.objectid == root->root_key.objectid && 4201 key.type == BTRFS_ROOT_REF_KEY) 4202 ret = -ENOTEMPTY; 4203 } 4204 out: 4205 btrfs_free_path(path); 4206 return ret; 4207 } 4208 4209 /* Delete all dentries for inodes belonging to the root */ 4210 static void btrfs_prune_dentries(struct btrfs_root *root) 4211 { 4212 struct btrfs_fs_info *fs_info = root->fs_info; 4213 struct rb_node *node; 4214 struct rb_node *prev; 4215 struct btrfs_inode *entry; 4216 struct inode *inode; 4217 u64 objectid = 0; 4218 4219 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 4220 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 4221 4222 spin_lock(&root->inode_lock); 4223 again: 4224 node = root->inode_tree.rb_node; 4225 prev = NULL; 4226 while (node) { 4227 prev = node; 4228 entry = rb_entry(node, struct btrfs_inode, rb_node); 4229 4230 if (objectid < btrfs_ino(entry)) 4231 node = node->rb_left; 4232 else if (objectid > btrfs_ino(entry)) 4233 node = node->rb_right; 4234 else 4235 break; 4236 } 4237 if (!node) { 4238 while (prev) { 4239 entry = rb_entry(prev, struct btrfs_inode, rb_node); 4240 if (objectid <= btrfs_ino(entry)) { 4241 node = prev; 4242 break; 4243 } 4244 prev = rb_next(prev); 4245 } 4246 } 4247 while (node) { 4248 entry = rb_entry(node, struct btrfs_inode, rb_node); 4249 objectid = btrfs_ino(entry) + 1; 4250 inode = igrab(&entry->vfs_inode); 4251 if (inode) { 4252 spin_unlock(&root->inode_lock); 4253 if (atomic_read(&inode->i_count) > 1) 4254 d_prune_aliases(inode); 4255 /* 4256 * btrfs_drop_inode will have it removed from the inode 4257 * cache when its usage count hits zero. 4258 */ 4259 iput(inode); 4260 cond_resched(); 4261 spin_lock(&root->inode_lock); 4262 goto again; 4263 } 4264 4265 if (cond_resched_lock(&root->inode_lock)) 4266 goto again; 4267 4268 node = rb_next(node); 4269 } 4270 spin_unlock(&root->inode_lock); 4271 } 4272 4273 int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) 4274 { 4275 struct btrfs_fs_info *fs_info = btrfs_sb(dentry->d_sb); 4276 struct btrfs_root *root = BTRFS_I(dir)->root; 4277 struct inode *inode = d_inode(dentry); 4278 struct btrfs_root *dest = BTRFS_I(inode)->root; 4279 struct btrfs_trans_handle *trans; 4280 struct btrfs_block_rsv block_rsv; 4281 u64 root_flags; 4282 int ret; 4283 4284 /* 4285 * Don't allow to delete a subvolume with send in progress. This is 4286 * inside the inode lock so the error handling that has to drop the bit 4287 * again is not run concurrently. 4288 */ 4289 spin_lock(&dest->root_item_lock); 4290 if (dest->send_in_progress) { 4291 spin_unlock(&dest->root_item_lock); 4292 btrfs_warn(fs_info, 4293 "attempt to delete subvolume %llu during send", 4294 dest->root_key.objectid); 4295 return -EPERM; 4296 } 4297 root_flags = btrfs_root_flags(&dest->root_item); 4298 btrfs_set_root_flags(&dest->root_item, 4299 root_flags | BTRFS_ROOT_SUBVOL_DEAD); 4300 spin_unlock(&dest->root_item_lock); 4301 4302 down_write(&fs_info->subvol_sem); 4303 4304 ret = may_destroy_subvol(dest); 4305 if (ret) 4306 goto out_up_write; 4307 4308 btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP); 4309 /* 4310 * One for dir inode, 4311 * two for dir entries, 4312 * two for root ref/backref. 4313 */ 4314 ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true); 4315 if (ret) 4316 goto out_up_write; 4317 4318 trans = btrfs_start_transaction(root, 0); 4319 if (IS_ERR(trans)) { 4320 ret = PTR_ERR(trans); 4321 goto out_release; 4322 } 4323 trans->block_rsv = &block_rsv; 4324 trans->bytes_reserved = block_rsv.size; 4325 4326 btrfs_record_snapshot_destroy(trans, BTRFS_I(dir)); 4327 4328 ret = btrfs_unlink_subvol(trans, dir, dentry); 4329 if (ret) { 4330 btrfs_abort_transaction(trans, ret); 4331 goto out_end_trans; 4332 } 4333 4334 btrfs_record_root_in_trans(trans, dest); 4335 4336 memset(&dest->root_item.drop_progress, 0, 4337 sizeof(dest->root_item.drop_progress)); 4338 btrfs_set_root_drop_level(&dest->root_item, 0); 4339 btrfs_set_root_refs(&dest->root_item, 0); 4340 4341 if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) { 4342 ret = btrfs_insert_orphan_item(trans, 4343 fs_info->tree_root, 4344 dest->root_key.objectid); 4345 if (ret) { 4346 btrfs_abort_transaction(trans, ret); 4347 goto out_end_trans; 4348 } 4349 } 4350 4351 ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid, 4352 BTRFS_UUID_KEY_SUBVOL, 4353 dest->root_key.objectid); 4354 if (ret && ret != -ENOENT) { 4355 btrfs_abort_transaction(trans, ret); 4356 goto out_end_trans; 4357 } 4358 if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) { 4359 ret = btrfs_uuid_tree_remove(trans, 4360 dest->root_item.received_uuid, 4361 BTRFS_UUID_KEY_RECEIVED_SUBVOL, 4362 dest->root_key.objectid); 4363 if (ret && ret != -ENOENT) { 4364 btrfs_abort_transaction(trans, ret); 4365 goto out_end_trans; 4366 } 4367 } 4368 4369 free_anon_bdev(dest->anon_dev); 4370 dest->anon_dev = 0; 4371 out_end_trans: 4372 trans->block_rsv = NULL; 4373 trans->bytes_reserved = 0; 4374 ret = btrfs_end_transaction(trans); 4375 inode->i_flags |= S_DEAD; 4376 out_release: 4377 btrfs_subvolume_release_metadata(root, &block_rsv); 4378 out_up_write: 4379 up_write(&fs_info->subvol_sem); 4380 if (ret) { 4381 spin_lock(&dest->root_item_lock); 4382 root_flags = btrfs_root_flags(&dest->root_item); 4383 btrfs_set_root_flags(&dest->root_item, 4384 root_flags & ~BTRFS_ROOT_SUBVOL_DEAD); 4385 spin_unlock(&dest->root_item_lock); 4386 } else { 4387 d_invalidate(dentry); 4388 btrfs_prune_dentries(dest); 4389 ASSERT(dest->send_in_progress == 0); 4390 } 4391 4392 return ret; 4393 } 4394 4395 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4396 { 4397 struct inode *inode = d_inode(dentry); 4398 int err = 0; 4399 struct btrfs_root *root = BTRFS_I(dir)->root; 4400 struct btrfs_trans_handle *trans; 4401 u64 last_unlink_trans; 4402 4403 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4404 return -ENOTEMPTY; 4405 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) 4406 return btrfs_delete_subvolume(dir, dentry); 4407 4408 trans = __unlink_start_trans(dir); 4409 if (IS_ERR(trans)) 4410 return PTR_ERR(trans); 4411 4412 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4413 err = btrfs_unlink_subvol(trans, dir, dentry); 4414 goto out; 4415 } 4416 4417 err = btrfs_orphan_add(trans, BTRFS_I(inode)); 4418 if (err) 4419 goto out; 4420 4421 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; 4422 4423 /* now the directory is empty */ 4424 err = btrfs_unlink_inode(trans, root, BTRFS_I(dir), 4425 BTRFS_I(d_inode(dentry)), dentry->d_name.name, 4426 dentry->d_name.len); 4427 if (!err) { 4428 btrfs_i_size_write(BTRFS_I(inode), 0); 4429 /* 4430 * Propagate the last_unlink_trans value of the deleted dir to 4431 * its parent directory. This is to prevent an unrecoverable 4432 * log tree in the case we do something like this: 4433 * 1) create dir foo 4434 * 2) create snapshot under dir foo 4435 * 3) delete the snapshot 4436 * 4) rmdir foo 4437 * 5) mkdir foo 4438 * 6) fsync foo or some file inside foo 4439 */ 4440 if (last_unlink_trans >= trans->transid) 4441 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; 4442 } 4443 out: 4444 btrfs_end_transaction(trans); 4445 btrfs_btree_balance_dirty(root->fs_info); 4446 4447 return err; 4448 } 4449 4450 /* 4451 * Return this if we need to call truncate_block for the last bit of the 4452 * truncate. 4453 */ 4454 #define NEED_TRUNCATE_BLOCK 1 4455 4456 /* 4457 * this can truncate away extent items, csum items and directory items. 4458 * It starts at a high offset and removes keys until it can't find 4459 * any higher than new_size 4460 * 4461 * csum items that cross the new i_size are truncated to the new size 4462 * as well. 4463 * 4464 * min_type is the minimum key type to truncate down to. If set to 0, this 4465 * will kill all the items on this inode, including the INODE_ITEM_KEY. 4466 */ 4467 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, 4468 struct btrfs_root *root, 4469 struct btrfs_inode *inode, 4470 u64 new_size, u32 min_type) 4471 { 4472 struct btrfs_fs_info *fs_info = root->fs_info; 4473 struct btrfs_path *path; 4474 struct extent_buffer *leaf; 4475 struct btrfs_file_extent_item *fi; 4476 struct btrfs_key key; 4477 struct btrfs_key found_key; 4478 u64 extent_start = 0; 4479 u64 extent_num_bytes = 0; 4480 u64 extent_offset = 0; 4481 u64 item_end = 0; 4482 u64 last_size = new_size; 4483 u32 found_type = (u8)-1; 4484 int found_extent; 4485 int del_item; 4486 int pending_del_nr = 0; 4487 int pending_del_slot = 0; 4488 int extent_type = -1; 4489 int ret; 4490 u64 ino = btrfs_ino(inode); 4491 u64 bytes_deleted = 0; 4492 bool be_nice = false; 4493 bool should_throttle = false; 4494 const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize); 4495 struct extent_state *cached_state = NULL; 4496 4497 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); 4498 4499 /* 4500 * For non-free space inodes and non-shareable roots, we want to back 4501 * off from time to time. This means all inodes in subvolume roots, 4502 * reloc roots, and data reloc roots. 4503 */ 4504 if (!btrfs_is_free_space_inode(inode) && 4505 test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) 4506 be_nice = true; 4507 4508 path = btrfs_alloc_path(); 4509 if (!path) 4510 return -ENOMEM; 4511 path->reada = READA_BACK; 4512 4513 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4514 lock_extent_bits(&inode->io_tree, lock_start, (u64)-1, 4515 &cached_state); 4516 4517 /* 4518 * We want to drop from the next block forward in case this 4519 * new size is not block aligned since we will be keeping the 4520 * last block of the extent just the way it is. 4521 */ 4522 btrfs_drop_extent_cache(inode, ALIGN(new_size, 4523 fs_info->sectorsize), 4524 (u64)-1, 0); 4525 } 4526 4527 /* 4528 * This function is also used to drop the items in the log tree before 4529 * we relog the inode, so if root != BTRFS_I(inode)->root, it means 4530 * it is used to drop the logged items. So we shouldn't kill the delayed 4531 * items. 4532 */ 4533 if (min_type == 0 && root == inode->root) 4534 btrfs_kill_delayed_inode_items(inode); 4535 4536 key.objectid = ino; 4537 key.offset = (u64)-1; 4538 key.type = (u8)-1; 4539 4540 search_again: 4541 /* 4542 * with a 16K leaf size and 128MB extents, you can actually queue 4543 * up a huge file in a single leaf. Most of the time that 4544 * bytes_deleted is > 0, it will be huge by the time we get here 4545 */ 4546 if (be_nice && bytes_deleted > SZ_32M && 4547 btrfs_should_end_transaction(trans)) { 4548 ret = -EAGAIN; 4549 goto out; 4550 } 4551 4552 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 4553 if (ret < 0) 4554 goto out; 4555 4556 if (ret > 0) { 4557 ret = 0; 4558 /* there are no items in the tree for us to truncate, we're 4559 * done 4560 */ 4561 if (path->slots[0] == 0) 4562 goto out; 4563 path->slots[0]--; 4564 } 4565 4566 while (1) { 4567 u64 clear_start = 0, clear_len = 0; 4568 4569 fi = NULL; 4570 leaf = path->nodes[0]; 4571 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4572 found_type = found_key.type; 4573 4574 if (found_key.objectid != ino) 4575 break; 4576 4577 if (found_type < min_type) 4578 break; 4579 4580 item_end = found_key.offset; 4581 if (found_type == BTRFS_EXTENT_DATA_KEY) { 4582 fi = btrfs_item_ptr(leaf, path->slots[0], 4583 struct btrfs_file_extent_item); 4584 extent_type = btrfs_file_extent_type(leaf, fi); 4585 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4586 item_end += 4587 btrfs_file_extent_num_bytes(leaf, fi); 4588 4589 trace_btrfs_truncate_show_fi_regular( 4590 inode, leaf, fi, found_key.offset); 4591 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4592 item_end += btrfs_file_extent_ram_bytes(leaf, 4593 fi); 4594 4595 trace_btrfs_truncate_show_fi_inline( 4596 inode, leaf, fi, path->slots[0], 4597 found_key.offset); 4598 } 4599 item_end--; 4600 } 4601 if (found_type > min_type) { 4602 del_item = 1; 4603 } else { 4604 if (item_end < new_size) 4605 break; 4606 if (found_key.offset >= new_size) 4607 del_item = 1; 4608 else 4609 del_item = 0; 4610 } 4611 found_extent = 0; 4612 /* FIXME, shrink the extent if the ref count is only 1 */ 4613 if (found_type != BTRFS_EXTENT_DATA_KEY) 4614 goto delete; 4615 4616 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4617 u64 num_dec; 4618 4619 clear_start = found_key.offset; 4620 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); 4621 if (!del_item) { 4622 u64 orig_num_bytes = 4623 btrfs_file_extent_num_bytes(leaf, fi); 4624 extent_num_bytes = ALIGN(new_size - 4625 found_key.offset, 4626 fs_info->sectorsize); 4627 clear_start = ALIGN(new_size, fs_info->sectorsize); 4628 btrfs_set_file_extent_num_bytes(leaf, fi, 4629 extent_num_bytes); 4630 num_dec = (orig_num_bytes - 4631 extent_num_bytes); 4632 if (test_bit(BTRFS_ROOT_SHAREABLE, 4633 &root->state) && 4634 extent_start != 0) 4635 inode_sub_bytes(&inode->vfs_inode, 4636 num_dec); 4637 btrfs_mark_buffer_dirty(leaf); 4638 } else { 4639 extent_num_bytes = 4640 btrfs_file_extent_disk_num_bytes(leaf, 4641 fi); 4642 extent_offset = found_key.offset - 4643 btrfs_file_extent_offset(leaf, fi); 4644 4645 /* FIXME blocksize != 4096 */ 4646 num_dec = btrfs_file_extent_num_bytes(leaf, fi); 4647 if (extent_start != 0) { 4648 found_extent = 1; 4649 if (test_bit(BTRFS_ROOT_SHAREABLE, 4650 &root->state)) 4651 inode_sub_bytes(&inode->vfs_inode, 4652 num_dec); 4653 } 4654 } 4655 clear_len = num_dec; 4656 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4657 /* 4658 * we can't truncate inline items that have had 4659 * special encodings 4660 */ 4661 if (!del_item && 4662 btrfs_file_extent_encryption(leaf, fi) == 0 && 4663 btrfs_file_extent_other_encoding(leaf, fi) == 0 && 4664 btrfs_file_extent_compression(leaf, fi) == 0) { 4665 u32 size = (u32)(new_size - found_key.offset); 4666 4667 btrfs_set_file_extent_ram_bytes(leaf, fi, size); 4668 size = btrfs_file_extent_calc_inline_size(size); 4669 btrfs_truncate_item(path, size, 1); 4670 } else if (!del_item) { 4671 /* 4672 * We have to bail so the last_size is set to 4673 * just before this extent. 4674 */ 4675 ret = NEED_TRUNCATE_BLOCK; 4676 break; 4677 } else { 4678 /* 4679 * Inline extents are special, we just treat 4680 * them as a full sector worth in the file 4681 * extent tree just for simplicity sake. 4682 */ 4683 clear_len = fs_info->sectorsize; 4684 } 4685 4686 if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) 4687 inode_sub_bytes(&inode->vfs_inode, 4688 item_end + 1 - new_size); 4689 } 4690 delete: 4691 /* 4692 * We use btrfs_truncate_inode_items() to clean up log trees for 4693 * multiple fsyncs, and in this case we don't want to clear the 4694 * file extent range because it's just the log. 4695 */ 4696 if (root == inode->root) { 4697 ret = btrfs_inode_clear_file_extent_range(inode, 4698 clear_start, clear_len); 4699 if (ret) { 4700 btrfs_abort_transaction(trans, ret); 4701 break; 4702 } 4703 } 4704 4705 if (del_item) 4706 last_size = found_key.offset; 4707 else 4708 last_size = new_size; 4709 if (del_item) { 4710 if (!pending_del_nr) { 4711 /* no pending yet, add ourselves */ 4712 pending_del_slot = path->slots[0]; 4713 pending_del_nr = 1; 4714 } else if (pending_del_nr && 4715 path->slots[0] + 1 == pending_del_slot) { 4716 /* hop on the pending chunk */ 4717 pending_del_nr++; 4718 pending_del_slot = path->slots[0]; 4719 } else { 4720 BUG(); 4721 } 4722 } else { 4723 break; 4724 } 4725 should_throttle = false; 4726 4727 if (found_extent && 4728 root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4729 struct btrfs_ref ref = { 0 }; 4730 4731 bytes_deleted += extent_num_bytes; 4732 4733 btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF, 4734 extent_start, extent_num_bytes, 0); 4735 ref.real_root = root->root_key.objectid; 4736 btrfs_init_data_ref(&ref, btrfs_header_owner(leaf), 4737 ino, extent_offset); 4738 ret = btrfs_free_extent(trans, &ref); 4739 if (ret) { 4740 btrfs_abort_transaction(trans, ret); 4741 break; 4742 } 4743 if (be_nice) { 4744 if (btrfs_should_throttle_delayed_refs(trans)) 4745 should_throttle = true; 4746 } 4747 } 4748 4749 if (found_type == BTRFS_INODE_ITEM_KEY) 4750 break; 4751 4752 if (path->slots[0] == 0 || 4753 path->slots[0] != pending_del_slot || 4754 should_throttle) { 4755 if (pending_del_nr) { 4756 ret = btrfs_del_items(trans, root, path, 4757 pending_del_slot, 4758 pending_del_nr); 4759 if (ret) { 4760 btrfs_abort_transaction(trans, ret); 4761 break; 4762 } 4763 pending_del_nr = 0; 4764 } 4765 btrfs_release_path(path); 4766 4767 /* 4768 * We can generate a lot of delayed refs, so we need to 4769 * throttle every once and a while and make sure we're 4770 * adding enough space to keep up with the work we are 4771 * generating. Since we hold a transaction here we 4772 * can't flush, and we don't want to FLUSH_LIMIT because 4773 * we could have generated too many delayed refs to 4774 * actually allocate, so just bail if we're short and 4775 * let the normal reservation dance happen higher up. 4776 */ 4777 if (should_throttle) { 4778 ret = btrfs_delayed_refs_rsv_refill(fs_info, 4779 BTRFS_RESERVE_NO_FLUSH); 4780 if (ret) { 4781 ret = -EAGAIN; 4782 break; 4783 } 4784 } 4785 goto search_again; 4786 } else { 4787 path->slots[0]--; 4788 } 4789 } 4790 out: 4791 if (ret >= 0 && pending_del_nr) { 4792 int err; 4793 4794 err = btrfs_del_items(trans, root, path, pending_del_slot, 4795 pending_del_nr); 4796 if (err) { 4797 btrfs_abort_transaction(trans, err); 4798 ret = err; 4799 } 4800 } 4801 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { 4802 ASSERT(last_size >= new_size); 4803 if (!ret && last_size > new_size) 4804 last_size = new_size; 4805 btrfs_inode_safe_disk_i_size_write(inode, last_size); 4806 unlock_extent_cached(&inode->io_tree, lock_start, (u64)-1, 4807 &cached_state); 4808 } 4809 4810 btrfs_free_path(path); 4811 return ret; 4812 } 4813 4814 /* 4815 * btrfs_truncate_block - read, zero a chunk and write a block 4816 * @inode - inode that we're zeroing 4817 * @from - the offset to start zeroing 4818 * @len - the length to zero, 0 to zero the entire range respective to the 4819 * offset 4820 * @front - zero up to the offset instead of from the offset on 4821 * 4822 * This will find the block for the "from" offset and cow the block and zero the 4823 * part we want to zero. This is used with truncate and hole punching. 4824 */ 4825 int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len, 4826 int front) 4827 { 4828 struct btrfs_fs_info *fs_info = inode->root->fs_info; 4829 struct address_space *mapping = inode->vfs_inode.i_mapping; 4830 struct extent_io_tree *io_tree = &inode->io_tree; 4831 struct btrfs_ordered_extent *ordered; 4832 struct extent_state *cached_state = NULL; 4833 struct extent_changeset *data_reserved = NULL; 4834 char *kaddr; 4835 bool only_release_metadata = false; 4836 u32 blocksize = fs_info->sectorsize; 4837 pgoff_t index = from >> PAGE_SHIFT; 4838 unsigned offset = from & (blocksize - 1); 4839 struct page *page; 4840 gfp_t mask = btrfs_alloc_write_mask(mapping); 4841 size_t write_bytes = blocksize; 4842 int ret = 0; 4843 u64 block_start; 4844 u64 block_end; 4845 4846 if (IS_ALIGNED(offset, blocksize) && 4847 (!len || IS_ALIGNED(len, blocksize))) 4848 goto out; 4849 4850 block_start = round_down(from, blocksize); 4851 block_end = block_start + blocksize - 1; 4852 4853 ret = btrfs_check_data_free_space(inode, &data_reserved, block_start, 4854 blocksize); 4855 if (ret < 0) { 4856 if (btrfs_check_nocow_lock(inode, block_start, &write_bytes) > 0) { 4857 /* For nocow case, no need to reserve data space */ 4858 only_release_metadata = true; 4859 } else { 4860 goto out; 4861 } 4862 } 4863 ret = btrfs_delalloc_reserve_metadata(inode, blocksize); 4864 if (ret < 0) { 4865 if (!only_release_metadata) 4866 btrfs_free_reserved_data_space(inode, data_reserved, 4867 block_start, blocksize); 4868 goto out; 4869 } 4870 again: 4871 page = find_or_create_page(mapping, index, mask); 4872 if (!page) { 4873 btrfs_delalloc_release_space(inode, data_reserved, block_start, 4874 blocksize, true); 4875 btrfs_delalloc_release_extents(inode, blocksize); 4876 ret = -ENOMEM; 4877 goto out; 4878 } 4879 ret = set_page_extent_mapped(page); 4880 if (ret < 0) 4881 goto out_unlock; 4882 4883 if (!PageUptodate(page)) { 4884 ret = btrfs_readpage(NULL, page); 4885 lock_page(page); 4886 if (page->mapping != mapping) { 4887 unlock_page(page); 4888 put_page(page); 4889 goto again; 4890 } 4891 if (!PageUptodate(page)) { 4892 ret = -EIO; 4893 goto out_unlock; 4894 } 4895 } 4896 wait_on_page_writeback(page); 4897 4898 lock_extent_bits(io_tree, block_start, block_end, &cached_state); 4899 4900 ordered = btrfs_lookup_ordered_extent(inode, block_start); 4901 if (ordered) { 4902 unlock_extent_cached(io_tree, block_start, block_end, 4903 &cached_state); 4904 unlock_page(page); 4905 put_page(page); 4906 btrfs_start_ordered_extent(ordered, 1); 4907 btrfs_put_ordered_extent(ordered); 4908 goto again; 4909 } 4910 4911 clear_extent_bit(&inode->io_tree, block_start, block_end, 4912 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4913 0, 0, &cached_state); 4914 4915 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, 4916 &cached_state); 4917 if (ret) { 4918 unlock_extent_cached(io_tree, block_start, block_end, 4919 &cached_state); 4920 goto out_unlock; 4921 } 4922 4923 if (offset != blocksize) { 4924 if (!len) 4925 len = blocksize - offset; 4926 kaddr = kmap(page); 4927 if (front) 4928 memset(kaddr + (block_start - page_offset(page)), 4929 0, offset); 4930 else 4931 memset(kaddr + (block_start - page_offset(page)) + offset, 4932 0, len); 4933 flush_dcache_page(page); 4934 kunmap(page); 4935 } 4936 ClearPageChecked(page); 4937 set_page_dirty(page); 4938 unlock_extent_cached(io_tree, block_start, block_end, &cached_state); 4939 4940 if (only_release_metadata) 4941 set_extent_bit(&inode->io_tree, block_start, block_end, 4942 EXTENT_NORESERVE, 0, NULL, NULL, GFP_NOFS, NULL); 4943 4944 out_unlock: 4945 if (ret) { 4946 if (only_release_metadata) 4947 btrfs_delalloc_release_metadata(inode, blocksize, true); 4948 else 4949 btrfs_delalloc_release_space(inode, data_reserved, 4950 block_start, blocksize, true); 4951 } 4952 btrfs_delalloc_release_extents(inode, blocksize); 4953 unlock_page(page); 4954 put_page(page); 4955 out: 4956 if (only_release_metadata) 4957 btrfs_check_nocow_unlock(inode); 4958 extent_changeset_free(data_reserved); 4959 return ret; 4960 } 4961 4962 static int maybe_insert_hole(struct btrfs_root *root, struct btrfs_inode *inode, 4963 u64 offset, u64 len) 4964 { 4965 struct btrfs_fs_info *fs_info = root->fs_info; 4966 struct btrfs_trans_handle *trans; 4967 struct btrfs_drop_extents_args drop_args = { 0 }; 4968 int ret; 4969 4970 /* 4971 * Still need to make sure the inode looks like it's been updated so 4972 * that any holes get logged if we fsync. 4973 */ 4974 if (btrfs_fs_incompat(fs_info, NO_HOLES)) { 4975 inode->last_trans = fs_info->generation; 4976 inode->last_sub_trans = root->log_transid; 4977 inode->last_log_commit = root->last_log_commit; 4978 return 0; 4979 } 4980 4981 /* 4982 * 1 - for the one we're dropping 4983 * 1 - for the one we're adding 4984 * 1 - for updating the inode. 4985 */ 4986 trans = btrfs_start_transaction(root, 3); 4987 if (IS_ERR(trans)) 4988 return PTR_ERR(trans); 4989 4990 drop_args.start = offset; 4991 drop_args.end = offset + len; 4992 drop_args.drop_cache = true; 4993 4994 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 4995 if (ret) { 4996 btrfs_abort_transaction(trans, ret); 4997 btrfs_end_transaction(trans); 4998 return ret; 4999 } 5000 5001 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), 5002 offset, 0, 0, len, 0, len, 0, 0, 0); 5003 if (ret) { 5004 btrfs_abort_transaction(trans, ret); 5005 } else { 5006 btrfs_update_inode_bytes(inode, 0, drop_args.bytes_found); 5007 btrfs_update_inode(trans, root, inode); 5008 } 5009 btrfs_end_transaction(trans); 5010 return ret; 5011 } 5012 5013 /* 5014 * This function puts in dummy file extents for the area we're creating a hole 5015 * for. So if we are truncating this file to a larger size we need to insert 5016 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 5017 * the range between oldsize and size 5018 */ 5019 int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size) 5020 { 5021 struct btrfs_root *root = inode->root; 5022 struct btrfs_fs_info *fs_info = root->fs_info; 5023 struct extent_io_tree *io_tree = &inode->io_tree; 5024 struct extent_map *em = NULL; 5025 struct extent_state *cached_state = NULL; 5026 struct extent_map_tree *em_tree = &inode->extent_tree; 5027 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 5028 u64 block_end = ALIGN(size, fs_info->sectorsize); 5029 u64 last_byte; 5030 u64 cur_offset; 5031 u64 hole_size; 5032 int err = 0; 5033 5034 /* 5035 * If our size started in the middle of a block we need to zero out the 5036 * rest of the block before we expand the i_size, otherwise we could 5037 * expose stale data. 5038 */ 5039 err = btrfs_truncate_block(inode, oldsize, 0, 0); 5040 if (err) 5041 return err; 5042 5043 if (size <= hole_start) 5044 return 0; 5045 5046 btrfs_lock_and_flush_ordered_range(inode, hole_start, block_end - 1, 5047 &cached_state); 5048 cur_offset = hole_start; 5049 while (1) { 5050 em = btrfs_get_extent(inode, NULL, 0, cur_offset, 5051 block_end - cur_offset); 5052 if (IS_ERR(em)) { 5053 err = PTR_ERR(em); 5054 em = NULL; 5055 break; 5056 } 5057 last_byte = min(extent_map_end(em), block_end); 5058 last_byte = ALIGN(last_byte, fs_info->sectorsize); 5059 hole_size = last_byte - cur_offset; 5060 5061 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 5062 struct extent_map *hole_em; 5063 5064 err = maybe_insert_hole(root, inode, cur_offset, 5065 hole_size); 5066 if (err) 5067 break; 5068 5069 err = btrfs_inode_set_file_extent_range(inode, 5070 cur_offset, hole_size); 5071 if (err) 5072 break; 5073 5074 btrfs_drop_extent_cache(inode, cur_offset, 5075 cur_offset + hole_size - 1, 0); 5076 hole_em = alloc_extent_map(); 5077 if (!hole_em) { 5078 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 5079 &inode->runtime_flags); 5080 goto next; 5081 } 5082 hole_em->start = cur_offset; 5083 hole_em->len = hole_size; 5084 hole_em->orig_start = cur_offset; 5085 5086 hole_em->block_start = EXTENT_MAP_HOLE; 5087 hole_em->block_len = 0; 5088 hole_em->orig_block_len = 0; 5089 hole_em->ram_bytes = hole_size; 5090 hole_em->compress_type = BTRFS_COMPRESS_NONE; 5091 hole_em->generation = fs_info->generation; 5092 5093 while (1) { 5094 write_lock(&em_tree->lock); 5095 err = add_extent_mapping(em_tree, hole_em, 1); 5096 write_unlock(&em_tree->lock); 5097 if (err != -EEXIST) 5098 break; 5099 btrfs_drop_extent_cache(inode, cur_offset, 5100 cur_offset + 5101 hole_size - 1, 0); 5102 } 5103 free_extent_map(hole_em); 5104 } else { 5105 err = btrfs_inode_set_file_extent_range(inode, 5106 cur_offset, hole_size); 5107 if (err) 5108 break; 5109 } 5110 next: 5111 free_extent_map(em); 5112 em = NULL; 5113 cur_offset = last_byte; 5114 if (cur_offset >= block_end) 5115 break; 5116 } 5117 free_extent_map(em); 5118 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state); 5119 return err; 5120 } 5121 5122 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 5123 { 5124 struct btrfs_root *root = BTRFS_I(inode)->root; 5125 struct btrfs_trans_handle *trans; 5126 loff_t oldsize = i_size_read(inode); 5127 loff_t newsize = attr->ia_size; 5128 int mask = attr->ia_valid; 5129 int ret; 5130 5131 /* 5132 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 5133 * special case where we need to update the times despite not having 5134 * these flags set. For all other operations the VFS set these flags 5135 * explicitly if it wants a timestamp update. 5136 */ 5137 if (newsize != oldsize) { 5138 inode_inc_iversion(inode); 5139 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) 5140 inode->i_ctime = inode->i_mtime = 5141 current_time(inode); 5142 } 5143 5144 if (newsize > oldsize) { 5145 /* 5146 * Don't do an expanding truncate while snapshotting is ongoing. 5147 * This is to ensure the snapshot captures a fully consistent 5148 * state of this file - if the snapshot captures this expanding 5149 * truncation, it must capture all writes that happened before 5150 * this truncation. 5151 */ 5152 btrfs_drew_write_lock(&root->snapshot_lock); 5153 ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, newsize); 5154 if (ret) { 5155 btrfs_drew_write_unlock(&root->snapshot_lock); 5156 return ret; 5157 } 5158 5159 trans = btrfs_start_transaction(root, 1); 5160 if (IS_ERR(trans)) { 5161 btrfs_drew_write_unlock(&root->snapshot_lock); 5162 return PTR_ERR(trans); 5163 } 5164 5165 i_size_write(inode, newsize); 5166 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); 5167 pagecache_isize_extended(inode, oldsize, newsize); 5168 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 5169 btrfs_drew_write_unlock(&root->snapshot_lock); 5170 btrfs_end_transaction(trans); 5171 } else { 5172 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5173 5174 if (btrfs_is_zoned(fs_info)) { 5175 ret = btrfs_wait_ordered_range(inode, 5176 ALIGN(newsize, fs_info->sectorsize), 5177 (u64)-1); 5178 if (ret) 5179 return ret; 5180 } 5181 5182 /* 5183 * We're truncating a file that used to have good data down to 5184 * zero. Make sure any new writes to the file get on disk 5185 * on close. 5186 */ 5187 if (newsize == 0) 5188 set_bit(BTRFS_INODE_FLUSH_ON_CLOSE, 5189 &BTRFS_I(inode)->runtime_flags); 5190 5191 truncate_setsize(inode, newsize); 5192 5193 inode_dio_wait(inode); 5194 5195 ret = btrfs_truncate(inode, newsize == oldsize); 5196 if (ret && inode->i_nlink) { 5197 int err; 5198 5199 /* 5200 * Truncate failed, so fix up the in-memory size. We 5201 * adjusted disk_i_size down as we removed extents, so 5202 * wait for disk_i_size to be stable and then update the 5203 * in-memory size to match. 5204 */ 5205 err = btrfs_wait_ordered_range(inode, 0, (u64)-1); 5206 if (err) 5207 return err; 5208 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 5209 } 5210 } 5211 5212 return ret; 5213 } 5214 5215 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) 5216 { 5217 struct inode *inode = d_inode(dentry); 5218 struct btrfs_root *root = BTRFS_I(inode)->root; 5219 int err; 5220 5221 if (btrfs_root_readonly(root)) 5222 return -EROFS; 5223 5224 err = setattr_prepare(dentry, attr); 5225 if (err) 5226 return err; 5227 5228 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 5229 err = btrfs_setsize(inode, attr); 5230 if (err) 5231 return err; 5232 } 5233 5234 if (attr->ia_valid) { 5235 setattr_copy(inode, attr); 5236 inode_inc_iversion(inode); 5237 err = btrfs_dirty_inode(inode); 5238 5239 if (!err && attr->ia_valid & ATTR_MODE) 5240 err = posix_acl_chmod(inode, inode->i_mode); 5241 } 5242 5243 return err; 5244 } 5245 5246 /* 5247 * While truncating the inode pages during eviction, we get the VFS calling 5248 * btrfs_invalidatepage() against each page of the inode. This is slow because 5249 * the calls to btrfs_invalidatepage() result in a huge amount of calls to 5250 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting 5251 * extent_state structures over and over, wasting lots of time. 5252 * 5253 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all 5254 * those expensive operations on a per page basis and do only the ordered io 5255 * finishing, while we release here the extent_map and extent_state structures, 5256 * without the excessive merging and splitting. 5257 */ 5258 static void evict_inode_truncate_pages(struct inode *inode) 5259 { 5260 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 5261 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; 5262 struct rb_node *node; 5263 5264 ASSERT(inode->i_state & I_FREEING); 5265 truncate_inode_pages_final(&inode->i_data); 5266 5267 write_lock(&map_tree->lock); 5268 while (!RB_EMPTY_ROOT(&map_tree->map.rb_root)) { 5269 struct extent_map *em; 5270 5271 node = rb_first_cached(&map_tree->map); 5272 em = rb_entry(node, struct extent_map, rb_node); 5273 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 5274 clear_bit(EXTENT_FLAG_LOGGING, &em->flags); 5275 remove_extent_mapping(map_tree, em); 5276 free_extent_map(em); 5277 if (need_resched()) { 5278 write_unlock(&map_tree->lock); 5279 cond_resched(); 5280 write_lock(&map_tree->lock); 5281 } 5282 } 5283 write_unlock(&map_tree->lock); 5284 5285 /* 5286 * Keep looping until we have no more ranges in the io tree. 5287 * We can have ongoing bios started by readahead that have 5288 * their endio callback (extent_io.c:end_bio_extent_readpage) 5289 * still in progress (unlocked the pages in the bio but did not yet 5290 * unlocked the ranges in the io tree). Therefore this means some 5291 * ranges can still be locked and eviction started because before 5292 * submitting those bios, which are executed by a separate task (work 5293 * queue kthread), inode references (inode->i_count) were not taken 5294 * (which would be dropped in the end io callback of each bio). 5295 * Therefore here we effectively end up waiting for those bios and 5296 * anyone else holding locked ranges without having bumped the inode's 5297 * reference count - if we don't do it, when they access the inode's 5298 * io_tree to unlock a range it may be too late, leading to an 5299 * use-after-free issue. 5300 */ 5301 spin_lock(&io_tree->lock); 5302 while (!RB_EMPTY_ROOT(&io_tree->state)) { 5303 struct extent_state *state; 5304 struct extent_state *cached_state = NULL; 5305 u64 start; 5306 u64 end; 5307 unsigned state_flags; 5308 5309 node = rb_first(&io_tree->state); 5310 state = rb_entry(node, struct extent_state, rb_node); 5311 start = state->start; 5312 end = state->end; 5313 state_flags = state->state; 5314 spin_unlock(&io_tree->lock); 5315 5316 lock_extent_bits(io_tree, start, end, &cached_state); 5317 5318 /* 5319 * If still has DELALLOC flag, the extent didn't reach disk, 5320 * and its reserved space won't be freed by delayed_ref. 5321 * So we need to free its reserved space here. 5322 * (Refer to comment in btrfs_invalidatepage, case 2) 5323 * 5324 * Note, end is the bytenr of last byte, so we need + 1 here. 5325 */ 5326 if (state_flags & EXTENT_DELALLOC) 5327 btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start, 5328 end - start + 1); 5329 5330 clear_extent_bit(io_tree, start, end, 5331 EXTENT_LOCKED | EXTENT_DELALLOC | 5332 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1, 5333 &cached_state); 5334 5335 cond_resched(); 5336 spin_lock(&io_tree->lock); 5337 } 5338 spin_unlock(&io_tree->lock); 5339 } 5340 5341 static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root, 5342 struct btrfs_block_rsv *rsv) 5343 { 5344 struct btrfs_fs_info *fs_info = root->fs_info; 5345 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; 5346 struct btrfs_trans_handle *trans; 5347 u64 delayed_refs_extra = btrfs_calc_insert_metadata_size(fs_info, 1); 5348 int ret; 5349 5350 /* 5351 * Eviction should be taking place at some place safe because of our 5352 * delayed iputs. However the normal flushing code will run delayed 5353 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock. 5354 * 5355 * We reserve the delayed_refs_extra here again because we can't use 5356 * btrfs_start_transaction(root, 0) for the same deadlocky reason as 5357 * above. We reserve our extra bit here because we generate a ton of 5358 * delayed refs activity by truncating. 5359 * 5360 * If we cannot make our reservation we'll attempt to steal from the 5361 * global reserve, because we really want to be able to free up space. 5362 */ 5363 ret = btrfs_block_rsv_refill(root, rsv, rsv->size + delayed_refs_extra, 5364 BTRFS_RESERVE_FLUSH_EVICT); 5365 if (ret) { 5366 /* 5367 * Try to steal from the global reserve if there is space for 5368 * it. 5369 */ 5370 if (btrfs_check_space_for_delayed_refs(fs_info) || 5371 btrfs_block_rsv_migrate(global_rsv, rsv, rsv->size, 0)) { 5372 btrfs_warn(fs_info, 5373 "could not allocate space for delete; will truncate on mount"); 5374 return ERR_PTR(-ENOSPC); 5375 } 5376 delayed_refs_extra = 0; 5377 } 5378 5379 trans = btrfs_join_transaction(root); 5380 if (IS_ERR(trans)) 5381 return trans; 5382 5383 if (delayed_refs_extra) { 5384 trans->block_rsv = &fs_info->trans_block_rsv; 5385 trans->bytes_reserved = delayed_refs_extra; 5386 btrfs_block_rsv_migrate(rsv, trans->block_rsv, 5387 delayed_refs_extra, 1); 5388 } 5389 return trans; 5390 } 5391 5392 void btrfs_evict_inode(struct inode *inode) 5393 { 5394 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5395 struct btrfs_trans_handle *trans; 5396 struct btrfs_root *root = BTRFS_I(inode)->root; 5397 struct btrfs_block_rsv *rsv; 5398 int ret; 5399 5400 trace_btrfs_inode_evict(inode); 5401 5402 if (!root) { 5403 clear_inode(inode); 5404 return; 5405 } 5406 5407 evict_inode_truncate_pages(inode); 5408 5409 if (inode->i_nlink && 5410 ((btrfs_root_refs(&root->root_item) != 0 && 5411 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 5412 btrfs_is_free_space_inode(BTRFS_I(inode)))) 5413 goto no_delete; 5414 5415 if (is_bad_inode(inode)) 5416 goto no_delete; 5417 5418 btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1); 5419 5420 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) 5421 goto no_delete; 5422 5423 if (inode->i_nlink > 0) { 5424 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5425 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 5426 goto no_delete; 5427 } 5428 5429 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); 5430 if (ret) 5431 goto no_delete; 5432 5433 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 5434 if (!rsv) 5435 goto no_delete; 5436 rsv->size = btrfs_calc_metadata_size(fs_info, 1); 5437 rsv->failfast = 1; 5438 5439 btrfs_i_size_write(BTRFS_I(inode), 0); 5440 5441 while (1) { 5442 trans = evict_refill_and_join(root, rsv); 5443 if (IS_ERR(trans)) 5444 goto free_rsv; 5445 5446 trans->block_rsv = rsv; 5447 5448 ret = btrfs_truncate_inode_items(trans, root, BTRFS_I(inode), 5449 0, 0); 5450 trans->block_rsv = &fs_info->trans_block_rsv; 5451 btrfs_end_transaction(trans); 5452 btrfs_btree_balance_dirty(fs_info); 5453 if (ret && ret != -ENOSPC && ret != -EAGAIN) 5454 goto free_rsv; 5455 else if (!ret) 5456 break; 5457 } 5458 5459 /* 5460 * Errors here aren't a big deal, it just means we leave orphan items in 5461 * the tree. They will be cleaned up on the next mount. If the inode 5462 * number gets reused, cleanup deletes the orphan item without doing 5463 * anything, and unlink reuses the existing orphan item. 5464 * 5465 * If it turns out that we are dropping too many of these, we might want 5466 * to add a mechanism for retrying these after a commit. 5467 */ 5468 trans = evict_refill_and_join(root, rsv); 5469 if (!IS_ERR(trans)) { 5470 trans->block_rsv = rsv; 5471 btrfs_orphan_del(trans, BTRFS_I(inode)); 5472 trans->block_rsv = &fs_info->trans_block_rsv; 5473 btrfs_end_transaction(trans); 5474 } 5475 5476 free_rsv: 5477 btrfs_free_block_rsv(fs_info, rsv); 5478 no_delete: 5479 /* 5480 * If we didn't successfully delete, the orphan item will still be in 5481 * the tree and we'll retry on the next mount. Again, we might also want 5482 * to retry these periodically in the future. 5483 */ 5484 btrfs_remove_delayed_node(BTRFS_I(inode)); 5485 clear_inode(inode); 5486 } 5487 5488 /* 5489 * Return the key found in the dir entry in the location pointer, fill @type 5490 * with BTRFS_FT_*, and return 0. 5491 * 5492 * If no dir entries were found, returns -ENOENT. 5493 * If found a corrupted location in dir entry, returns -EUCLEAN. 5494 */ 5495 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry, 5496 struct btrfs_key *location, u8 *type) 5497 { 5498 const char *name = dentry->d_name.name; 5499 int namelen = dentry->d_name.len; 5500 struct btrfs_dir_item *di; 5501 struct btrfs_path *path; 5502 struct btrfs_root *root = BTRFS_I(dir)->root; 5503 int ret = 0; 5504 5505 path = btrfs_alloc_path(); 5506 if (!path) 5507 return -ENOMEM; 5508 5509 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)), 5510 name, namelen, 0); 5511 if (IS_ERR_OR_NULL(di)) { 5512 ret = di ? PTR_ERR(di) : -ENOENT; 5513 goto out; 5514 } 5515 5516 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5517 if (location->type != BTRFS_INODE_ITEM_KEY && 5518 location->type != BTRFS_ROOT_ITEM_KEY) { 5519 ret = -EUCLEAN; 5520 btrfs_warn(root->fs_info, 5521 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))", 5522 __func__, name, btrfs_ino(BTRFS_I(dir)), 5523 location->objectid, location->type, location->offset); 5524 } 5525 if (!ret) 5526 *type = btrfs_dir_type(path->nodes[0], di); 5527 out: 5528 btrfs_free_path(path); 5529 return ret; 5530 } 5531 5532 /* 5533 * when we hit a tree root in a directory, the btrfs part of the inode 5534 * needs to be changed to reflect the root directory of the tree root. This 5535 * is kind of like crossing a mount point. 5536 */ 5537 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5538 struct inode *dir, 5539 struct dentry *dentry, 5540 struct btrfs_key *location, 5541 struct btrfs_root **sub_root) 5542 { 5543 struct btrfs_path *path; 5544 struct btrfs_root *new_root; 5545 struct btrfs_root_ref *ref; 5546 struct extent_buffer *leaf; 5547 struct btrfs_key key; 5548 int ret; 5549 int err = 0; 5550 5551 path = btrfs_alloc_path(); 5552 if (!path) { 5553 err = -ENOMEM; 5554 goto out; 5555 } 5556 5557 err = -ENOENT; 5558 key.objectid = BTRFS_I(dir)->root->root_key.objectid; 5559 key.type = BTRFS_ROOT_REF_KEY; 5560 key.offset = location->objectid; 5561 5562 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5563 if (ret) { 5564 if (ret < 0) 5565 err = ret; 5566 goto out; 5567 } 5568 5569 leaf = path->nodes[0]; 5570 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5571 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) || 5572 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len) 5573 goto out; 5574 5575 ret = memcmp_extent_buffer(leaf, dentry->d_name.name, 5576 (unsigned long)(ref + 1), 5577 dentry->d_name.len); 5578 if (ret) 5579 goto out; 5580 5581 btrfs_release_path(path); 5582 5583 new_root = btrfs_get_fs_root(fs_info, location->objectid, true); 5584 if (IS_ERR(new_root)) { 5585 err = PTR_ERR(new_root); 5586 goto out; 5587 } 5588 5589 *sub_root = new_root; 5590 location->objectid = btrfs_root_dirid(&new_root->root_item); 5591 location->type = BTRFS_INODE_ITEM_KEY; 5592 location->offset = 0; 5593 err = 0; 5594 out: 5595 btrfs_free_path(path); 5596 return err; 5597 } 5598 5599 static void inode_tree_add(struct inode *inode) 5600 { 5601 struct btrfs_root *root = BTRFS_I(inode)->root; 5602 struct btrfs_inode *entry; 5603 struct rb_node **p; 5604 struct rb_node *parent; 5605 struct rb_node *new = &BTRFS_I(inode)->rb_node; 5606 u64 ino = btrfs_ino(BTRFS_I(inode)); 5607 5608 if (inode_unhashed(inode)) 5609 return; 5610 parent = NULL; 5611 spin_lock(&root->inode_lock); 5612 p = &root->inode_tree.rb_node; 5613 while (*p) { 5614 parent = *p; 5615 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5616 5617 if (ino < btrfs_ino(entry)) 5618 p = &parent->rb_left; 5619 else if (ino > btrfs_ino(entry)) 5620 p = &parent->rb_right; 5621 else { 5622 WARN_ON(!(entry->vfs_inode.i_state & 5623 (I_WILL_FREE | I_FREEING))); 5624 rb_replace_node(parent, new, &root->inode_tree); 5625 RB_CLEAR_NODE(parent); 5626 spin_unlock(&root->inode_lock); 5627 return; 5628 } 5629 } 5630 rb_link_node(new, parent, p); 5631 rb_insert_color(new, &root->inode_tree); 5632 spin_unlock(&root->inode_lock); 5633 } 5634 5635 static void inode_tree_del(struct btrfs_inode *inode) 5636 { 5637 struct btrfs_root *root = inode->root; 5638 int empty = 0; 5639 5640 spin_lock(&root->inode_lock); 5641 if (!RB_EMPTY_NODE(&inode->rb_node)) { 5642 rb_erase(&inode->rb_node, &root->inode_tree); 5643 RB_CLEAR_NODE(&inode->rb_node); 5644 empty = RB_EMPTY_ROOT(&root->inode_tree); 5645 } 5646 spin_unlock(&root->inode_lock); 5647 5648 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5649 spin_lock(&root->inode_lock); 5650 empty = RB_EMPTY_ROOT(&root->inode_tree); 5651 spin_unlock(&root->inode_lock); 5652 if (empty) 5653 btrfs_add_dead_root(root); 5654 } 5655 } 5656 5657 5658 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5659 { 5660 struct btrfs_iget_args *args = p; 5661 5662 inode->i_ino = args->ino; 5663 BTRFS_I(inode)->location.objectid = args->ino; 5664 BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY; 5665 BTRFS_I(inode)->location.offset = 0; 5666 BTRFS_I(inode)->root = btrfs_grab_root(args->root); 5667 BUG_ON(args->root && !BTRFS_I(inode)->root); 5668 return 0; 5669 } 5670 5671 static int btrfs_find_actor(struct inode *inode, void *opaque) 5672 { 5673 struct btrfs_iget_args *args = opaque; 5674 5675 return args->ino == BTRFS_I(inode)->location.objectid && 5676 args->root == BTRFS_I(inode)->root; 5677 } 5678 5679 static struct inode *btrfs_iget_locked(struct super_block *s, u64 ino, 5680 struct btrfs_root *root) 5681 { 5682 struct inode *inode; 5683 struct btrfs_iget_args args; 5684 unsigned long hashval = btrfs_inode_hash(ino, root); 5685 5686 args.ino = ino; 5687 args.root = root; 5688 5689 inode = iget5_locked(s, hashval, btrfs_find_actor, 5690 btrfs_init_locked_inode, 5691 (void *)&args); 5692 return inode; 5693 } 5694 5695 /* 5696 * Get an inode object given its inode number and corresponding root. 5697 * Path can be preallocated to prevent recursing back to iget through 5698 * allocator. NULL is also valid but may require an additional allocation 5699 * later. 5700 */ 5701 struct inode *btrfs_iget_path(struct super_block *s, u64 ino, 5702 struct btrfs_root *root, struct btrfs_path *path) 5703 { 5704 struct inode *inode; 5705 5706 inode = btrfs_iget_locked(s, ino, root); 5707 if (!inode) 5708 return ERR_PTR(-ENOMEM); 5709 5710 if (inode->i_state & I_NEW) { 5711 int ret; 5712 5713 ret = btrfs_read_locked_inode(inode, path); 5714 if (!ret) { 5715 inode_tree_add(inode); 5716 unlock_new_inode(inode); 5717 } else { 5718 iget_failed(inode); 5719 /* 5720 * ret > 0 can come from btrfs_search_slot called by 5721 * btrfs_read_locked_inode, this means the inode item 5722 * was not found. 5723 */ 5724 if (ret > 0) 5725 ret = -ENOENT; 5726 inode = ERR_PTR(ret); 5727 } 5728 } 5729 5730 return inode; 5731 } 5732 5733 struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root) 5734 { 5735 return btrfs_iget_path(s, ino, root, NULL); 5736 } 5737 5738 static struct inode *new_simple_dir(struct super_block *s, 5739 struct btrfs_key *key, 5740 struct btrfs_root *root) 5741 { 5742 struct inode *inode = new_inode(s); 5743 5744 if (!inode) 5745 return ERR_PTR(-ENOMEM); 5746 5747 BTRFS_I(inode)->root = btrfs_grab_root(root); 5748 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5749 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5750 5751 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5752 /* 5753 * We only need lookup, the rest is read-only and there's no inode 5754 * associated with the dentry 5755 */ 5756 inode->i_op = &simple_dir_inode_operations; 5757 inode->i_opflags &= ~IOP_XATTR; 5758 inode->i_fop = &simple_dir_operations; 5759 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5760 inode->i_mtime = current_time(inode); 5761 inode->i_atime = inode->i_mtime; 5762 inode->i_ctime = inode->i_mtime; 5763 BTRFS_I(inode)->i_otime = inode->i_mtime; 5764 5765 return inode; 5766 } 5767 5768 static inline u8 btrfs_inode_type(struct inode *inode) 5769 { 5770 /* 5771 * Compile-time asserts that generic FT_* types still match 5772 * BTRFS_FT_* types 5773 */ 5774 BUILD_BUG_ON(BTRFS_FT_UNKNOWN != FT_UNKNOWN); 5775 BUILD_BUG_ON(BTRFS_FT_REG_FILE != FT_REG_FILE); 5776 BUILD_BUG_ON(BTRFS_FT_DIR != FT_DIR); 5777 BUILD_BUG_ON(BTRFS_FT_CHRDEV != FT_CHRDEV); 5778 BUILD_BUG_ON(BTRFS_FT_BLKDEV != FT_BLKDEV); 5779 BUILD_BUG_ON(BTRFS_FT_FIFO != FT_FIFO); 5780 BUILD_BUG_ON(BTRFS_FT_SOCK != FT_SOCK); 5781 BUILD_BUG_ON(BTRFS_FT_SYMLINK != FT_SYMLINK); 5782 5783 return fs_umode_to_ftype(inode->i_mode); 5784 } 5785 5786 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5787 { 5788 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 5789 struct inode *inode; 5790 struct btrfs_root *root = BTRFS_I(dir)->root; 5791 struct btrfs_root *sub_root = root; 5792 struct btrfs_key location; 5793 u8 di_type = 0; 5794 int ret = 0; 5795 5796 if (dentry->d_name.len > BTRFS_NAME_LEN) 5797 return ERR_PTR(-ENAMETOOLONG); 5798 5799 ret = btrfs_inode_by_name(dir, dentry, &location, &di_type); 5800 if (ret < 0) 5801 return ERR_PTR(ret); 5802 5803 if (location.type == BTRFS_INODE_ITEM_KEY) { 5804 inode = btrfs_iget(dir->i_sb, location.objectid, root); 5805 if (IS_ERR(inode)) 5806 return inode; 5807 5808 /* Do extra check against inode mode with di_type */ 5809 if (btrfs_inode_type(inode) != di_type) { 5810 btrfs_crit(fs_info, 5811 "inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u", 5812 inode->i_mode, btrfs_inode_type(inode), 5813 di_type); 5814 iput(inode); 5815 return ERR_PTR(-EUCLEAN); 5816 } 5817 return inode; 5818 } 5819 5820 ret = fixup_tree_root_location(fs_info, dir, dentry, 5821 &location, &sub_root); 5822 if (ret < 0) { 5823 if (ret != -ENOENT) 5824 inode = ERR_PTR(ret); 5825 else 5826 inode = new_simple_dir(dir->i_sb, &location, sub_root); 5827 } else { 5828 inode = btrfs_iget(dir->i_sb, location.objectid, sub_root); 5829 } 5830 if (root != sub_root) 5831 btrfs_put_root(sub_root); 5832 5833 if (!IS_ERR(inode) && root != sub_root) { 5834 down_read(&fs_info->cleanup_work_sem); 5835 if (!sb_rdonly(inode->i_sb)) 5836 ret = btrfs_orphan_cleanup(sub_root); 5837 up_read(&fs_info->cleanup_work_sem); 5838 if (ret) { 5839 iput(inode); 5840 inode = ERR_PTR(ret); 5841 } 5842 } 5843 5844 return inode; 5845 } 5846 5847 static int btrfs_dentry_delete(const struct dentry *dentry) 5848 { 5849 struct btrfs_root *root; 5850 struct inode *inode = d_inode(dentry); 5851 5852 if (!inode && !IS_ROOT(dentry)) 5853 inode = d_inode(dentry->d_parent); 5854 5855 if (inode) { 5856 root = BTRFS_I(inode)->root; 5857 if (btrfs_root_refs(&root->root_item) == 0) 5858 return 1; 5859 5860 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5861 return 1; 5862 } 5863 return 0; 5864 } 5865 5866 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5867 unsigned int flags) 5868 { 5869 struct inode *inode = btrfs_lookup_dentry(dir, dentry); 5870 5871 if (inode == ERR_PTR(-ENOENT)) 5872 inode = NULL; 5873 return d_splice_alias(inode, dentry); 5874 } 5875 5876 /* 5877 * All this infrastructure exists because dir_emit can fault, and we are holding 5878 * the tree lock when doing readdir. For now just allocate a buffer and copy 5879 * our information into that, and then dir_emit from the buffer. This is 5880 * similar to what NFS does, only we don't keep the buffer around in pagecache 5881 * because I'm afraid I'll mess that up. Long term we need to make filldir do 5882 * copy_to_user_inatomic so we don't have to worry about page faulting under the 5883 * tree lock. 5884 */ 5885 static int btrfs_opendir(struct inode *inode, struct file *file) 5886 { 5887 struct btrfs_file_private *private; 5888 5889 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL); 5890 if (!private) 5891 return -ENOMEM; 5892 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 5893 if (!private->filldir_buf) { 5894 kfree(private); 5895 return -ENOMEM; 5896 } 5897 file->private_data = private; 5898 return 0; 5899 } 5900 5901 struct dir_entry { 5902 u64 ino; 5903 u64 offset; 5904 unsigned type; 5905 int name_len; 5906 }; 5907 5908 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) 5909 { 5910 while (entries--) { 5911 struct dir_entry *entry = addr; 5912 char *name = (char *)(entry + 1); 5913 5914 ctx->pos = get_unaligned(&entry->offset); 5915 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len), 5916 get_unaligned(&entry->ino), 5917 get_unaligned(&entry->type))) 5918 return 1; 5919 addr += sizeof(struct dir_entry) + 5920 get_unaligned(&entry->name_len); 5921 ctx->pos++; 5922 } 5923 return 0; 5924 } 5925 5926 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5927 { 5928 struct inode *inode = file_inode(file); 5929 struct btrfs_root *root = BTRFS_I(inode)->root; 5930 struct btrfs_file_private *private = file->private_data; 5931 struct btrfs_dir_item *di; 5932 struct btrfs_key key; 5933 struct btrfs_key found_key; 5934 struct btrfs_path *path; 5935 void *addr; 5936 struct list_head ins_list; 5937 struct list_head del_list; 5938 int ret; 5939 struct extent_buffer *leaf; 5940 int slot; 5941 char *name_ptr; 5942 int name_len; 5943 int entries = 0; 5944 int total_len = 0; 5945 bool put = false; 5946 struct btrfs_key location; 5947 5948 if (!dir_emit_dots(file, ctx)) 5949 return 0; 5950 5951 path = btrfs_alloc_path(); 5952 if (!path) 5953 return -ENOMEM; 5954 5955 addr = private->filldir_buf; 5956 path->reada = READA_FORWARD; 5957 5958 INIT_LIST_HEAD(&ins_list); 5959 INIT_LIST_HEAD(&del_list); 5960 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list); 5961 5962 again: 5963 key.type = BTRFS_DIR_INDEX_KEY; 5964 key.offset = ctx->pos; 5965 key.objectid = btrfs_ino(BTRFS_I(inode)); 5966 5967 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5968 if (ret < 0) 5969 goto err; 5970 5971 while (1) { 5972 struct dir_entry *entry; 5973 5974 leaf = path->nodes[0]; 5975 slot = path->slots[0]; 5976 if (slot >= btrfs_header_nritems(leaf)) { 5977 ret = btrfs_next_leaf(root, path); 5978 if (ret < 0) 5979 goto err; 5980 else if (ret > 0) 5981 break; 5982 continue; 5983 } 5984 5985 btrfs_item_key_to_cpu(leaf, &found_key, slot); 5986 5987 if (found_key.objectid != key.objectid) 5988 break; 5989 if (found_key.type != BTRFS_DIR_INDEX_KEY) 5990 break; 5991 if (found_key.offset < ctx->pos) 5992 goto next; 5993 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 5994 goto next; 5995 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); 5996 name_len = btrfs_dir_name_len(leaf, di); 5997 if ((total_len + sizeof(struct dir_entry) + name_len) >= 5998 PAGE_SIZE) { 5999 btrfs_release_path(path); 6000 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6001 if (ret) 6002 goto nopos; 6003 addr = private->filldir_buf; 6004 entries = 0; 6005 total_len = 0; 6006 goto again; 6007 } 6008 6009 entry = addr; 6010 put_unaligned(name_len, &entry->name_len); 6011 name_ptr = (char *)(entry + 1); 6012 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1), 6013 name_len); 6014 put_unaligned(fs_ftype_to_dtype(btrfs_dir_type(leaf, di)), 6015 &entry->type); 6016 btrfs_dir_item_key_to_cpu(leaf, di, &location); 6017 put_unaligned(location.objectid, &entry->ino); 6018 put_unaligned(found_key.offset, &entry->offset); 6019 entries++; 6020 addr += sizeof(struct dir_entry) + name_len; 6021 total_len += sizeof(struct dir_entry) + name_len; 6022 next: 6023 path->slots[0]++; 6024 } 6025 btrfs_release_path(path); 6026 6027 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6028 if (ret) 6029 goto nopos; 6030 6031 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 6032 if (ret) 6033 goto nopos; 6034 6035 /* 6036 * Stop new entries from being returned after we return the last 6037 * entry. 6038 * 6039 * New directory entries are assigned a strictly increasing 6040 * offset. This means that new entries created during readdir 6041 * are *guaranteed* to be seen in the future by that readdir. 6042 * This has broken buggy programs which operate on names as 6043 * they're returned by readdir. Until we re-use freed offsets 6044 * we have this hack to stop new entries from being returned 6045 * under the assumption that they'll never reach this huge 6046 * offset. 6047 * 6048 * This is being careful not to overflow 32bit loff_t unless the 6049 * last entry requires it because doing so has broken 32bit apps 6050 * in the past. 6051 */ 6052 if (ctx->pos >= INT_MAX) 6053 ctx->pos = LLONG_MAX; 6054 else 6055 ctx->pos = INT_MAX; 6056 nopos: 6057 ret = 0; 6058 err: 6059 if (put) 6060 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list); 6061 btrfs_free_path(path); 6062 return ret; 6063 } 6064 6065 /* 6066 * This is somewhat expensive, updating the tree every time the 6067 * inode changes. But, it is most likely to find the inode in cache. 6068 * FIXME, needs more benchmarking...there are no reasons other than performance 6069 * to keep or drop this code. 6070 */ 6071 static int btrfs_dirty_inode(struct inode *inode) 6072 { 6073 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6074 struct btrfs_root *root = BTRFS_I(inode)->root; 6075 struct btrfs_trans_handle *trans; 6076 int ret; 6077 6078 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 6079 return 0; 6080 6081 trans = btrfs_join_transaction(root); 6082 if (IS_ERR(trans)) 6083 return PTR_ERR(trans); 6084 6085 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 6086 if (ret && ret == -ENOSPC) { 6087 /* whoops, lets try again with the full transaction */ 6088 btrfs_end_transaction(trans); 6089 trans = btrfs_start_transaction(root, 1); 6090 if (IS_ERR(trans)) 6091 return PTR_ERR(trans); 6092 6093 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 6094 } 6095 btrfs_end_transaction(trans); 6096 if (BTRFS_I(inode)->delayed_node) 6097 btrfs_balance_delayed_items(fs_info); 6098 6099 return ret; 6100 } 6101 6102 /* 6103 * This is a copy of file_update_time. We need this so we can return error on 6104 * ENOSPC for updating the inode in the case of file write and mmap writes. 6105 */ 6106 static int btrfs_update_time(struct inode *inode, struct timespec64 *now, 6107 int flags) 6108 { 6109 struct btrfs_root *root = BTRFS_I(inode)->root; 6110 bool dirty = flags & ~S_VERSION; 6111 6112 if (btrfs_root_readonly(root)) 6113 return -EROFS; 6114 6115 if (flags & S_VERSION) 6116 dirty |= inode_maybe_inc_iversion(inode, dirty); 6117 if (flags & S_CTIME) 6118 inode->i_ctime = *now; 6119 if (flags & S_MTIME) 6120 inode->i_mtime = *now; 6121 if (flags & S_ATIME) 6122 inode->i_atime = *now; 6123 return dirty ? btrfs_dirty_inode(inode) : 0; 6124 } 6125 6126 /* 6127 * find the highest existing sequence number in a directory 6128 * and then set the in-memory index_cnt variable to reflect 6129 * free sequence numbers 6130 */ 6131 static int btrfs_set_inode_index_count(struct btrfs_inode *inode) 6132 { 6133 struct btrfs_root *root = inode->root; 6134 struct btrfs_key key, found_key; 6135 struct btrfs_path *path; 6136 struct extent_buffer *leaf; 6137 int ret; 6138 6139 key.objectid = btrfs_ino(inode); 6140 key.type = BTRFS_DIR_INDEX_KEY; 6141 key.offset = (u64)-1; 6142 6143 path = btrfs_alloc_path(); 6144 if (!path) 6145 return -ENOMEM; 6146 6147 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 6148 if (ret < 0) 6149 goto out; 6150 /* FIXME: we should be able to handle this */ 6151 if (ret == 0) 6152 goto out; 6153 ret = 0; 6154 6155 /* 6156 * MAGIC NUMBER EXPLANATION: 6157 * since we search a directory based on f_pos we have to start at 2 6158 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody 6159 * else has to start at 2 6160 */ 6161 if (path->slots[0] == 0) { 6162 inode->index_cnt = 2; 6163 goto out; 6164 } 6165 6166 path->slots[0]--; 6167 6168 leaf = path->nodes[0]; 6169 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6170 6171 if (found_key.objectid != btrfs_ino(inode) || 6172 found_key.type != BTRFS_DIR_INDEX_KEY) { 6173 inode->index_cnt = 2; 6174 goto out; 6175 } 6176 6177 inode->index_cnt = found_key.offset + 1; 6178 out: 6179 btrfs_free_path(path); 6180 return ret; 6181 } 6182 6183 /* 6184 * helper to find a free sequence number in a given directory. This current 6185 * code is very simple, later versions will do smarter things in the btree 6186 */ 6187 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) 6188 { 6189 int ret = 0; 6190 6191 if (dir->index_cnt == (u64)-1) { 6192 ret = btrfs_inode_delayed_dir_index_count(dir); 6193 if (ret) { 6194 ret = btrfs_set_inode_index_count(dir); 6195 if (ret) 6196 return ret; 6197 } 6198 } 6199 6200 *index = dir->index_cnt; 6201 dir->index_cnt++; 6202 6203 return ret; 6204 } 6205 6206 static int btrfs_insert_inode_locked(struct inode *inode) 6207 { 6208 struct btrfs_iget_args args; 6209 6210 args.ino = BTRFS_I(inode)->location.objectid; 6211 args.root = BTRFS_I(inode)->root; 6212 6213 return insert_inode_locked4(inode, 6214 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 6215 btrfs_find_actor, &args); 6216 } 6217 6218 /* 6219 * Inherit flags from the parent inode. 6220 * 6221 * Currently only the compression flags and the cow flags are inherited. 6222 */ 6223 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir) 6224 { 6225 unsigned int flags; 6226 6227 if (!dir) 6228 return; 6229 6230 flags = BTRFS_I(dir)->flags; 6231 6232 if (flags & BTRFS_INODE_NOCOMPRESS) { 6233 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS; 6234 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 6235 } else if (flags & BTRFS_INODE_COMPRESS) { 6236 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS; 6237 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS; 6238 } 6239 6240 if (flags & BTRFS_INODE_NODATACOW) { 6241 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW; 6242 if (S_ISREG(inode->i_mode)) 6243 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6244 } 6245 6246 btrfs_sync_inode_flags_to_i_flags(inode); 6247 } 6248 6249 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, 6250 struct btrfs_root *root, 6251 struct inode *dir, 6252 const char *name, int name_len, 6253 u64 ref_objectid, u64 objectid, 6254 umode_t mode, u64 *index) 6255 { 6256 struct btrfs_fs_info *fs_info = root->fs_info; 6257 struct inode *inode; 6258 struct btrfs_inode_item *inode_item; 6259 struct btrfs_key *location; 6260 struct btrfs_path *path; 6261 struct btrfs_inode_ref *ref; 6262 struct btrfs_key key[2]; 6263 u32 sizes[2]; 6264 int nitems = name ? 2 : 1; 6265 unsigned long ptr; 6266 unsigned int nofs_flag; 6267 int ret; 6268 6269 path = btrfs_alloc_path(); 6270 if (!path) 6271 return ERR_PTR(-ENOMEM); 6272 6273 nofs_flag = memalloc_nofs_save(); 6274 inode = new_inode(fs_info->sb); 6275 memalloc_nofs_restore(nofs_flag); 6276 if (!inode) { 6277 btrfs_free_path(path); 6278 return ERR_PTR(-ENOMEM); 6279 } 6280 6281 /* 6282 * O_TMPFILE, set link count to 0, so that after this point, 6283 * we fill in an inode item with the correct link count. 6284 */ 6285 if (!name) 6286 set_nlink(inode, 0); 6287 6288 /* 6289 * we have to initialize this early, so we can reclaim the inode 6290 * number if we fail afterwards in this function. 6291 */ 6292 inode->i_ino = objectid; 6293 6294 if (dir && name) { 6295 trace_btrfs_inode_request(dir); 6296 6297 ret = btrfs_set_inode_index(BTRFS_I(dir), index); 6298 if (ret) { 6299 btrfs_free_path(path); 6300 iput(inode); 6301 return ERR_PTR(ret); 6302 } 6303 } else if (dir) { 6304 *index = 0; 6305 } 6306 /* 6307 * index_cnt is ignored for everything but a dir, 6308 * btrfs_set_inode_index_count has an explanation for the magic 6309 * number 6310 */ 6311 BTRFS_I(inode)->index_cnt = 2; 6312 BTRFS_I(inode)->dir_index = *index; 6313 BTRFS_I(inode)->root = btrfs_grab_root(root); 6314 BTRFS_I(inode)->generation = trans->transid; 6315 inode->i_generation = BTRFS_I(inode)->generation; 6316 6317 /* 6318 * We could have gotten an inode number from somebody who was fsynced 6319 * and then removed in this same transaction, so let's just set full 6320 * sync since it will be a full sync anyway and this will blow away the 6321 * old info in the log. 6322 */ 6323 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 6324 6325 key[0].objectid = objectid; 6326 key[0].type = BTRFS_INODE_ITEM_KEY; 6327 key[0].offset = 0; 6328 6329 sizes[0] = sizeof(struct btrfs_inode_item); 6330 6331 if (name) { 6332 /* 6333 * Start new inodes with an inode_ref. This is slightly more 6334 * efficient for small numbers of hard links since they will 6335 * be packed into one item. Extended refs will kick in if we 6336 * add more hard links than can fit in the ref item. 6337 */ 6338 key[1].objectid = objectid; 6339 key[1].type = BTRFS_INODE_REF_KEY; 6340 key[1].offset = ref_objectid; 6341 6342 sizes[1] = name_len + sizeof(*ref); 6343 } 6344 6345 location = &BTRFS_I(inode)->location; 6346 location->objectid = objectid; 6347 location->offset = 0; 6348 location->type = BTRFS_INODE_ITEM_KEY; 6349 6350 ret = btrfs_insert_inode_locked(inode); 6351 if (ret < 0) { 6352 iput(inode); 6353 goto fail; 6354 } 6355 6356 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems); 6357 if (ret != 0) 6358 goto fail_unlock; 6359 6360 inode_init_owner(inode, dir, mode); 6361 inode_set_bytes(inode, 0); 6362 6363 inode->i_mtime = current_time(inode); 6364 inode->i_atime = inode->i_mtime; 6365 inode->i_ctime = inode->i_mtime; 6366 BTRFS_I(inode)->i_otime = inode->i_mtime; 6367 6368 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6369 struct btrfs_inode_item); 6370 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6371 sizeof(*inode_item)); 6372 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6373 6374 if (name) { 6375 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6376 struct btrfs_inode_ref); 6377 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len); 6378 btrfs_set_inode_ref_index(path->nodes[0], ref, *index); 6379 ptr = (unsigned long)(ref + 1); 6380 write_extent_buffer(path->nodes[0], name, ptr, name_len); 6381 } 6382 6383 btrfs_mark_buffer_dirty(path->nodes[0]); 6384 btrfs_free_path(path); 6385 6386 btrfs_inherit_iflags(inode, dir); 6387 6388 if (S_ISREG(mode)) { 6389 if (btrfs_test_opt(fs_info, NODATASUM)) 6390 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6391 if (btrfs_test_opt(fs_info, NODATACOW)) 6392 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6393 BTRFS_INODE_NODATASUM; 6394 } 6395 6396 inode_tree_add(inode); 6397 6398 trace_btrfs_inode_new(inode); 6399 btrfs_set_inode_last_trans(trans, BTRFS_I(inode)); 6400 6401 btrfs_update_root_times(trans, root); 6402 6403 ret = btrfs_inode_inherit_props(trans, inode, dir); 6404 if (ret) 6405 btrfs_err(fs_info, 6406 "error inheriting props for ino %llu (root %llu): %d", 6407 btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret); 6408 6409 return inode; 6410 6411 fail_unlock: 6412 discard_new_inode(inode); 6413 fail: 6414 if (dir && name) 6415 BTRFS_I(dir)->index_cnt--; 6416 btrfs_free_path(path); 6417 return ERR_PTR(ret); 6418 } 6419 6420 /* 6421 * utility function to add 'inode' into 'parent_inode' with 6422 * a give name and a given sequence number. 6423 * if 'add_backref' is true, also insert a backref from the 6424 * inode to the parent directory. 6425 */ 6426 int btrfs_add_link(struct btrfs_trans_handle *trans, 6427 struct btrfs_inode *parent_inode, struct btrfs_inode *inode, 6428 const char *name, int name_len, int add_backref, u64 index) 6429 { 6430 int ret = 0; 6431 struct btrfs_key key; 6432 struct btrfs_root *root = parent_inode->root; 6433 u64 ino = btrfs_ino(inode); 6434 u64 parent_ino = btrfs_ino(parent_inode); 6435 6436 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6437 memcpy(&key, &inode->root->root_key, sizeof(key)); 6438 } else { 6439 key.objectid = ino; 6440 key.type = BTRFS_INODE_ITEM_KEY; 6441 key.offset = 0; 6442 } 6443 6444 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6445 ret = btrfs_add_root_ref(trans, key.objectid, 6446 root->root_key.objectid, parent_ino, 6447 index, name, name_len); 6448 } else if (add_backref) { 6449 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino, 6450 parent_ino, index); 6451 } 6452 6453 /* Nothing to clean up yet */ 6454 if (ret) 6455 return ret; 6456 6457 ret = btrfs_insert_dir_item(trans, name, name_len, parent_inode, &key, 6458 btrfs_inode_type(&inode->vfs_inode), index); 6459 if (ret == -EEXIST || ret == -EOVERFLOW) 6460 goto fail_dir_item; 6461 else if (ret) { 6462 btrfs_abort_transaction(trans, ret); 6463 return ret; 6464 } 6465 6466 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + 6467 name_len * 2); 6468 inode_inc_iversion(&parent_inode->vfs_inode); 6469 /* 6470 * If we are replaying a log tree, we do not want to update the mtime 6471 * and ctime of the parent directory with the current time, since the 6472 * log replay procedure is responsible for setting them to their correct 6473 * values (the ones it had when the fsync was done). 6474 */ 6475 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags)) { 6476 struct timespec64 now = current_time(&parent_inode->vfs_inode); 6477 6478 parent_inode->vfs_inode.i_mtime = now; 6479 parent_inode->vfs_inode.i_ctime = now; 6480 } 6481 ret = btrfs_update_inode(trans, root, parent_inode); 6482 if (ret) 6483 btrfs_abort_transaction(trans, ret); 6484 return ret; 6485 6486 fail_dir_item: 6487 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6488 u64 local_index; 6489 int err; 6490 err = btrfs_del_root_ref(trans, key.objectid, 6491 root->root_key.objectid, parent_ino, 6492 &local_index, name, name_len); 6493 if (err) 6494 btrfs_abort_transaction(trans, err); 6495 } else if (add_backref) { 6496 u64 local_index; 6497 int err; 6498 6499 err = btrfs_del_inode_ref(trans, root, name, name_len, 6500 ino, parent_ino, &local_index); 6501 if (err) 6502 btrfs_abort_transaction(trans, err); 6503 } 6504 6505 /* Return the original error code */ 6506 return ret; 6507 } 6508 6509 static int btrfs_add_nondir(struct btrfs_trans_handle *trans, 6510 struct btrfs_inode *dir, struct dentry *dentry, 6511 struct btrfs_inode *inode, int backref, u64 index) 6512 { 6513 int err = btrfs_add_link(trans, dir, inode, 6514 dentry->d_name.name, dentry->d_name.len, 6515 backref, index); 6516 if (err > 0) 6517 err = -EEXIST; 6518 return err; 6519 } 6520 6521 static int btrfs_mknod(struct inode *dir, struct dentry *dentry, 6522 umode_t mode, dev_t rdev) 6523 { 6524 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6525 struct btrfs_trans_handle *trans; 6526 struct btrfs_root *root = BTRFS_I(dir)->root; 6527 struct inode *inode = NULL; 6528 int err; 6529 u64 objectid; 6530 u64 index = 0; 6531 6532 /* 6533 * 2 for inode item and ref 6534 * 2 for dir items 6535 * 1 for xattr if selinux is on 6536 */ 6537 trans = btrfs_start_transaction(root, 5); 6538 if (IS_ERR(trans)) 6539 return PTR_ERR(trans); 6540 6541 err = btrfs_get_free_objectid(root, &objectid); 6542 if (err) 6543 goto out_unlock; 6544 6545 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6546 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6547 mode, &index); 6548 if (IS_ERR(inode)) { 6549 err = PTR_ERR(inode); 6550 inode = NULL; 6551 goto out_unlock; 6552 } 6553 6554 /* 6555 * If the active LSM wants to access the inode during 6556 * d_instantiate it needs these. Smack checks to see 6557 * if the filesystem supports xattrs by looking at the 6558 * ops vector. 6559 */ 6560 inode->i_op = &btrfs_special_inode_operations; 6561 init_special_inode(inode, inode->i_mode, rdev); 6562 6563 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6564 if (err) 6565 goto out_unlock; 6566 6567 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6568 0, index); 6569 if (err) 6570 goto out_unlock; 6571 6572 btrfs_update_inode(trans, root, BTRFS_I(inode)); 6573 d_instantiate_new(dentry, inode); 6574 6575 out_unlock: 6576 btrfs_end_transaction(trans); 6577 btrfs_btree_balance_dirty(fs_info); 6578 if (err && inode) { 6579 inode_dec_link_count(inode); 6580 discard_new_inode(inode); 6581 } 6582 return err; 6583 } 6584 6585 static int btrfs_create(struct inode *dir, struct dentry *dentry, 6586 umode_t mode, bool excl) 6587 { 6588 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6589 struct btrfs_trans_handle *trans; 6590 struct btrfs_root *root = BTRFS_I(dir)->root; 6591 struct inode *inode = NULL; 6592 int err; 6593 u64 objectid; 6594 u64 index = 0; 6595 6596 /* 6597 * 2 for inode item and ref 6598 * 2 for dir items 6599 * 1 for xattr if selinux is on 6600 */ 6601 trans = btrfs_start_transaction(root, 5); 6602 if (IS_ERR(trans)) 6603 return PTR_ERR(trans); 6604 6605 err = btrfs_get_free_objectid(root, &objectid); 6606 if (err) 6607 goto out_unlock; 6608 6609 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6610 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6611 mode, &index); 6612 if (IS_ERR(inode)) { 6613 err = PTR_ERR(inode); 6614 inode = NULL; 6615 goto out_unlock; 6616 } 6617 /* 6618 * If the active LSM wants to access the inode during 6619 * d_instantiate it needs these. Smack checks to see 6620 * if the filesystem supports xattrs by looking at the 6621 * ops vector. 6622 */ 6623 inode->i_fop = &btrfs_file_operations; 6624 inode->i_op = &btrfs_file_inode_operations; 6625 inode->i_mapping->a_ops = &btrfs_aops; 6626 6627 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6628 if (err) 6629 goto out_unlock; 6630 6631 err = btrfs_update_inode(trans, root, BTRFS_I(inode)); 6632 if (err) 6633 goto out_unlock; 6634 6635 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6636 0, index); 6637 if (err) 6638 goto out_unlock; 6639 6640 d_instantiate_new(dentry, inode); 6641 6642 out_unlock: 6643 btrfs_end_transaction(trans); 6644 if (err && inode) { 6645 inode_dec_link_count(inode); 6646 discard_new_inode(inode); 6647 } 6648 btrfs_btree_balance_dirty(fs_info); 6649 return err; 6650 } 6651 6652 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6653 struct dentry *dentry) 6654 { 6655 struct btrfs_trans_handle *trans = NULL; 6656 struct btrfs_root *root = BTRFS_I(dir)->root; 6657 struct inode *inode = d_inode(old_dentry); 6658 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6659 u64 index; 6660 int err; 6661 int drop_inode = 0; 6662 6663 /* do not allow sys_link's with other subvols of the same device */ 6664 if (root->root_key.objectid != BTRFS_I(inode)->root->root_key.objectid) 6665 return -EXDEV; 6666 6667 if (inode->i_nlink >= BTRFS_LINK_MAX) 6668 return -EMLINK; 6669 6670 err = btrfs_set_inode_index(BTRFS_I(dir), &index); 6671 if (err) 6672 goto fail; 6673 6674 /* 6675 * 2 items for inode and inode ref 6676 * 2 items for dir items 6677 * 1 item for parent inode 6678 * 1 item for orphan item deletion if O_TMPFILE 6679 */ 6680 trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6); 6681 if (IS_ERR(trans)) { 6682 err = PTR_ERR(trans); 6683 trans = NULL; 6684 goto fail; 6685 } 6686 6687 /* There are several dir indexes for this inode, clear the cache. */ 6688 BTRFS_I(inode)->dir_index = 0ULL; 6689 inc_nlink(inode); 6690 inode_inc_iversion(inode); 6691 inode->i_ctime = current_time(inode); 6692 ihold(inode); 6693 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6694 6695 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode), 6696 1, index); 6697 6698 if (err) { 6699 drop_inode = 1; 6700 } else { 6701 struct dentry *parent = dentry->d_parent; 6702 6703 err = btrfs_update_inode(trans, root, BTRFS_I(inode)); 6704 if (err) 6705 goto fail; 6706 if (inode->i_nlink == 1) { 6707 /* 6708 * If new hard link count is 1, it's a file created 6709 * with open(2) O_TMPFILE flag. 6710 */ 6711 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 6712 if (err) 6713 goto fail; 6714 } 6715 d_instantiate(dentry, inode); 6716 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent); 6717 } 6718 6719 fail: 6720 if (trans) 6721 btrfs_end_transaction(trans); 6722 if (drop_inode) { 6723 inode_dec_link_count(inode); 6724 iput(inode); 6725 } 6726 btrfs_btree_balance_dirty(fs_info); 6727 return err; 6728 } 6729 6730 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 6731 { 6732 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6733 struct inode *inode = NULL; 6734 struct btrfs_trans_handle *trans; 6735 struct btrfs_root *root = BTRFS_I(dir)->root; 6736 int err = 0; 6737 u64 objectid = 0; 6738 u64 index = 0; 6739 6740 /* 6741 * 2 items for inode and ref 6742 * 2 items for dir items 6743 * 1 for xattr if selinux is on 6744 */ 6745 trans = btrfs_start_transaction(root, 5); 6746 if (IS_ERR(trans)) 6747 return PTR_ERR(trans); 6748 6749 err = btrfs_get_free_objectid(root, &objectid); 6750 if (err) 6751 goto out_fail; 6752 6753 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6754 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid, 6755 S_IFDIR | mode, &index); 6756 if (IS_ERR(inode)) { 6757 err = PTR_ERR(inode); 6758 inode = NULL; 6759 goto out_fail; 6760 } 6761 6762 /* these must be set before we unlock the inode */ 6763 inode->i_op = &btrfs_dir_inode_operations; 6764 inode->i_fop = &btrfs_dir_file_operations; 6765 6766 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6767 if (err) 6768 goto out_fail; 6769 6770 btrfs_i_size_write(BTRFS_I(inode), 0); 6771 err = btrfs_update_inode(trans, root, BTRFS_I(inode)); 6772 if (err) 6773 goto out_fail; 6774 6775 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), 6776 dentry->d_name.name, 6777 dentry->d_name.len, 0, index); 6778 if (err) 6779 goto out_fail; 6780 6781 d_instantiate_new(dentry, inode); 6782 6783 out_fail: 6784 btrfs_end_transaction(trans); 6785 if (err && inode) { 6786 inode_dec_link_count(inode); 6787 discard_new_inode(inode); 6788 } 6789 btrfs_btree_balance_dirty(fs_info); 6790 return err; 6791 } 6792 6793 static noinline int uncompress_inline(struct btrfs_path *path, 6794 struct page *page, 6795 size_t pg_offset, u64 extent_offset, 6796 struct btrfs_file_extent_item *item) 6797 { 6798 int ret; 6799 struct extent_buffer *leaf = path->nodes[0]; 6800 char *tmp; 6801 size_t max_size; 6802 unsigned long inline_size; 6803 unsigned long ptr; 6804 int compress_type; 6805 6806 WARN_ON(pg_offset != 0); 6807 compress_type = btrfs_file_extent_compression(leaf, item); 6808 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6809 inline_size = btrfs_file_extent_inline_item_len(leaf, 6810 btrfs_item_nr(path->slots[0])); 6811 tmp = kmalloc(inline_size, GFP_NOFS); 6812 if (!tmp) 6813 return -ENOMEM; 6814 ptr = btrfs_file_extent_inline_start(item); 6815 6816 read_extent_buffer(leaf, tmp, ptr, inline_size); 6817 6818 max_size = min_t(unsigned long, PAGE_SIZE, max_size); 6819 ret = btrfs_decompress(compress_type, tmp, page, 6820 extent_offset, inline_size, max_size); 6821 6822 /* 6823 * decompression code contains a memset to fill in any space between the end 6824 * of the uncompressed data and the end of max_size in case the decompressed 6825 * data ends up shorter than ram_bytes. That doesn't cover the hole between 6826 * the end of an inline extent and the beginning of the next block, so we 6827 * cover that region here. 6828 */ 6829 6830 if (max_size + pg_offset < PAGE_SIZE) { 6831 char *map = kmap(page); 6832 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset); 6833 kunmap(page); 6834 } 6835 kfree(tmp); 6836 return ret; 6837 } 6838 6839 /** 6840 * btrfs_get_extent - Lookup the first extent overlapping a range in a file. 6841 * @inode: file to search in 6842 * @page: page to read extent data into if the extent is inline 6843 * @pg_offset: offset into @page to copy to 6844 * @start: file offset 6845 * @len: length of range starting at @start 6846 * 6847 * This returns the first &struct extent_map which overlaps with the given 6848 * range, reading it from the B-tree and caching it if necessary. Note that 6849 * there may be more extents which overlap the given range after the returned 6850 * extent_map. 6851 * 6852 * If @page is not NULL and the extent is inline, this also reads the extent 6853 * data directly into the page and marks the extent up to date in the io_tree. 6854 * 6855 * Return: ERR_PTR on error, non-NULL extent_map on success. 6856 */ 6857 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, 6858 struct page *page, size_t pg_offset, 6859 u64 start, u64 len) 6860 { 6861 struct btrfs_fs_info *fs_info = inode->root->fs_info; 6862 int ret = 0; 6863 u64 extent_start = 0; 6864 u64 extent_end = 0; 6865 u64 objectid = btrfs_ino(inode); 6866 int extent_type = -1; 6867 struct btrfs_path *path = NULL; 6868 struct btrfs_root *root = inode->root; 6869 struct btrfs_file_extent_item *item; 6870 struct extent_buffer *leaf; 6871 struct btrfs_key found_key; 6872 struct extent_map *em = NULL; 6873 struct extent_map_tree *em_tree = &inode->extent_tree; 6874 struct extent_io_tree *io_tree = &inode->io_tree; 6875 6876 read_lock(&em_tree->lock); 6877 em = lookup_extent_mapping(em_tree, start, len); 6878 read_unlock(&em_tree->lock); 6879 6880 if (em) { 6881 if (em->start > start || em->start + em->len <= start) 6882 free_extent_map(em); 6883 else if (em->block_start == EXTENT_MAP_INLINE && page) 6884 free_extent_map(em); 6885 else 6886 goto out; 6887 } 6888 em = alloc_extent_map(); 6889 if (!em) { 6890 ret = -ENOMEM; 6891 goto out; 6892 } 6893 em->start = EXTENT_MAP_HOLE; 6894 em->orig_start = EXTENT_MAP_HOLE; 6895 em->len = (u64)-1; 6896 em->block_len = (u64)-1; 6897 6898 path = btrfs_alloc_path(); 6899 if (!path) { 6900 ret = -ENOMEM; 6901 goto out; 6902 } 6903 6904 /* Chances are we'll be called again, so go ahead and do readahead */ 6905 path->reada = READA_FORWARD; 6906 6907 /* 6908 * The same explanation in load_free_space_cache applies here as well, 6909 * we only read when we're loading the free space cache, and at that 6910 * point the commit_root has everything we need. 6911 */ 6912 if (btrfs_is_free_space_inode(inode)) { 6913 path->search_commit_root = 1; 6914 path->skip_locking = 1; 6915 } 6916 6917 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); 6918 if (ret < 0) { 6919 goto out; 6920 } else if (ret > 0) { 6921 if (path->slots[0] == 0) 6922 goto not_found; 6923 path->slots[0]--; 6924 ret = 0; 6925 } 6926 6927 leaf = path->nodes[0]; 6928 item = btrfs_item_ptr(leaf, path->slots[0], 6929 struct btrfs_file_extent_item); 6930 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6931 if (found_key.objectid != objectid || 6932 found_key.type != BTRFS_EXTENT_DATA_KEY) { 6933 /* 6934 * If we backup past the first extent we want to move forward 6935 * and see if there is an extent in front of us, otherwise we'll 6936 * say there is a hole for our whole search range which can 6937 * cause problems. 6938 */ 6939 extent_end = start; 6940 goto next; 6941 } 6942 6943 extent_type = btrfs_file_extent_type(leaf, item); 6944 extent_start = found_key.offset; 6945 extent_end = btrfs_file_extent_end(path); 6946 if (extent_type == BTRFS_FILE_EXTENT_REG || 6947 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 6948 /* Only regular file could have regular/prealloc extent */ 6949 if (!S_ISREG(inode->vfs_inode.i_mode)) { 6950 ret = -EUCLEAN; 6951 btrfs_crit(fs_info, 6952 "regular/prealloc extent found for non-regular inode %llu", 6953 btrfs_ino(inode)); 6954 goto out; 6955 } 6956 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, 6957 extent_start); 6958 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 6959 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, 6960 path->slots[0], 6961 extent_start); 6962 } 6963 next: 6964 if (start >= extent_end) { 6965 path->slots[0]++; 6966 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 6967 ret = btrfs_next_leaf(root, path); 6968 if (ret < 0) 6969 goto out; 6970 else if (ret > 0) 6971 goto not_found; 6972 6973 leaf = path->nodes[0]; 6974 } 6975 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6976 if (found_key.objectid != objectid || 6977 found_key.type != BTRFS_EXTENT_DATA_KEY) 6978 goto not_found; 6979 if (start + len <= found_key.offset) 6980 goto not_found; 6981 if (start > found_key.offset) 6982 goto next; 6983 6984 /* New extent overlaps with existing one */ 6985 em->start = start; 6986 em->orig_start = start; 6987 em->len = found_key.offset - start; 6988 em->block_start = EXTENT_MAP_HOLE; 6989 goto insert; 6990 } 6991 6992 btrfs_extent_item_to_extent_map(inode, path, item, !page, em); 6993 6994 if (extent_type == BTRFS_FILE_EXTENT_REG || 6995 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 6996 goto insert; 6997 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 6998 unsigned long ptr; 6999 char *map; 7000 size_t size; 7001 size_t extent_offset; 7002 size_t copy_size; 7003 7004 if (!page) 7005 goto out; 7006 7007 size = btrfs_file_extent_ram_bytes(leaf, item); 7008 extent_offset = page_offset(page) + pg_offset - extent_start; 7009 copy_size = min_t(u64, PAGE_SIZE - pg_offset, 7010 size - extent_offset); 7011 em->start = extent_start + extent_offset; 7012 em->len = ALIGN(copy_size, fs_info->sectorsize); 7013 em->orig_block_len = em->len; 7014 em->orig_start = em->start; 7015 ptr = btrfs_file_extent_inline_start(item) + extent_offset; 7016 7017 if (!PageUptodate(page)) { 7018 if (btrfs_file_extent_compression(leaf, item) != 7019 BTRFS_COMPRESS_NONE) { 7020 ret = uncompress_inline(path, page, pg_offset, 7021 extent_offset, item); 7022 if (ret) 7023 goto out; 7024 } else { 7025 map = kmap(page); 7026 read_extent_buffer(leaf, map + pg_offset, ptr, 7027 copy_size); 7028 if (pg_offset + copy_size < PAGE_SIZE) { 7029 memset(map + pg_offset + copy_size, 0, 7030 PAGE_SIZE - pg_offset - 7031 copy_size); 7032 } 7033 kunmap(page); 7034 } 7035 flush_dcache_page(page); 7036 } 7037 set_extent_uptodate(io_tree, em->start, 7038 extent_map_end(em) - 1, NULL, GFP_NOFS); 7039 goto insert; 7040 } 7041 not_found: 7042 em->start = start; 7043 em->orig_start = start; 7044 em->len = len; 7045 em->block_start = EXTENT_MAP_HOLE; 7046 insert: 7047 ret = 0; 7048 btrfs_release_path(path); 7049 if (em->start > start || extent_map_end(em) <= start) { 7050 btrfs_err(fs_info, 7051 "bad extent! em: [%llu %llu] passed [%llu %llu]", 7052 em->start, em->len, start, len); 7053 ret = -EIO; 7054 goto out; 7055 } 7056 7057 write_lock(&em_tree->lock); 7058 ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len); 7059 write_unlock(&em_tree->lock); 7060 out: 7061 btrfs_free_path(path); 7062 7063 trace_btrfs_get_extent(root, inode, em); 7064 7065 if (ret) { 7066 free_extent_map(em); 7067 return ERR_PTR(ret); 7068 } 7069 return em; 7070 } 7071 7072 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, 7073 u64 start, u64 len) 7074 { 7075 struct extent_map *em; 7076 struct extent_map *hole_em = NULL; 7077 u64 delalloc_start = start; 7078 u64 end; 7079 u64 delalloc_len; 7080 u64 delalloc_end; 7081 int err = 0; 7082 7083 em = btrfs_get_extent(inode, NULL, 0, start, len); 7084 if (IS_ERR(em)) 7085 return em; 7086 /* 7087 * If our em maps to: 7088 * - a hole or 7089 * - a pre-alloc extent, 7090 * there might actually be delalloc bytes behind it. 7091 */ 7092 if (em->block_start != EXTENT_MAP_HOLE && 7093 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7094 return em; 7095 else 7096 hole_em = em; 7097 7098 /* check to see if we've wrapped (len == -1 or similar) */ 7099 end = start + len; 7100 if (end < start) 7101 end = (u64)-1; 7102 else 7103 end -= 1; 7104 7105 em = NULL; 7106 7107 /* ok, we didn't find anything, lets look for delalloc */ 7108 delalloc_len = count_range_bits(&inode->io_tree, &delalloc_start, 7109 end, len, EXTENT_DELALLOC, 1); 7110 delalloc_end = delalloc_start + delalloc_len; 7111 if (delalloc_end < delalloc_start) 7112 delalloc_end = (u64)-1; 7113 7114 /* 7115 * We didn't find anything useful, return the original results from 7116 * get_extent() 7117 */ 7118 if (delalloc_start > end || delalloc_end <= start) { 7119 em = hole_em; 7120 hole_em = NULL; 7121 goto out; 7122 } 7123 7124 /* 7125 * Adjust the delalloc_start to make sure it doesn't go backwards from 7126 * the start they passed in 7127 */ 7128 delalloc_start = max(start, delalloc_start); 7129 delalloc_len = delalloc_end - delalloc_start; 7130 7131 if (delalloc_len > 0) { 7132 u64 hole_start; 7133 u64 hole_len; 7134 const u64 hole_end = extent_map_end(hole_em); 7135 7136 em = alloc_extent_map(); 7137 if (!em) { 7138 err = -ENOMEM; 7139 goto out; 7140 } 7141 7142 ASSERT(hole_em); 7143 /* 7144 * When btrfs_get_extent can't find anything it returns one 7145 * huge hole 7146 * 7147 * Make sure what it found really fits our range, and adjust to 7148 * make sure it is based on the start from the caller 7149 */ 7150 if (hole_end <= start || hole_em->start > end) { 7151 free_extent_map(hole_em); 7152 hole_em = NULL; 7153 } else { 7154 hole_start = max(hole_em->start, start); 7155 hole_len = hole_end - hole_start; 7156 } 7157 7158 if (hole_em && delalloc_start > hole_start) { 7159 /* 7160 * Our hole starts before our delalloc, so we have to 7161 * return just the parts of the hole that go until the 7162 * delalloc starts 7163 */ 7164 em->len = min(hole_len, delalloc_start - hole_start); 7165 em->start = hole_start; 7166 em->orig_start = hole_start; 7167 /* 7168 * Don't adjust block start at all, it is fixed at 7169 * EXTENT_MAP_HOLE 7170 */ 7171 em->block_start = hole_em->block_start; 7172 em->block_len = hole_len; 7173 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) 7174 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 7175 } else { 7176 /* 7177 * Hole is out of passed range or it starts after 7178 * delalloc range 7179 */ 7180 em->start = delalloc_start; 7181 em->len = delalloc_len; 7182 em->orig_start = delalloc_start; 7183 em->block_start = EXTENT_MAP_DELALLOC; 7184 em->block_len = delalloc_len; 7185 } 7186 } else { 7187 return hole_em; 7188 } 7189 out: 7190 7191 free_extent_map(hole_em); 7192 if (err) { 7193 free_extent_map(em); 7194 return ERR_PTR(err); 7195 } 7196 return em; 7197 } 7198 7199 static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, 7200 const u64 start, 7201 const u64 len, 7202 const u64 orig_start, 7203 const u64 block_start, 7204 const u64 block_len, 7205 const u64 orig_block_len, 7206 const u64 ram_bytes, 7207 const int type) 7208 { 7209 struct extent_map *em = NULL; 7210 int ret; 7211 7212 if (type != BTRFS_ORDERED_NOCOW) { 7213 em = create_io_em(inode, start, len, orig_start, block_start, 7214 block_len, orig_block_len, ram_bytes, 7215 BTRFS_COMPRESS_NONE, /* compress_type */ 7216 type); 7217 if (IS_ERR(em)) 7218 goto out; 7219 } 7220 ret = btrfs_add_ordered_extent_dio(inode, start, block_start, len, 7221 block_len, type); 7222 if (ret) { 7223 if (em) { 7224 free_extent_map(em); 7225 btrfs_drop_extent_cache(inode, start, start + len - 1, 0); 7226 } 7227 em = ERR_PTR(ret); 7228 } 7229 out: 7230 7231 return em; 7232 } 7233 7234 static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode, 7235 u64 start, u64 len) 7236 { 7237 struct btrfs_root *root = inode->root; 7238 struct btrfs_fs_info *fs_info = root->fs_info; 7239 struct extent_map *em; 7240 struct btrfs_key ins; 7241 u64 alloc_hint; 7242 int ret; 7243 7244 alloc_hint = get_extent_allocation_hint(inode, start, len); 7245 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, 7246 0, alloc_hint, &ins, 1, 1); 7247 if (ret) 7248 return ERR_PTR(ret); 7249 7250 em = btrfs_create_dio_extent(inode, start, ins.offset, start, 7251 ins.objectid, ins.offset, ins.offset, 7252 ins.offset, BTRFS_ORDERED_REGULAR); 7253 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 7254 if (IS_ERR(em)) 7255 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 7256 1); 7257 7258 return em; 7259 } 7260 7261 /* 7262 * Check if we can do nocow write into the range [@offset, @offset + @len) 7263 * 7264 * @offset: File offset 7265 * @len: The length to write, will be updated to the nocow writeable 7266 * range 7267 * @orig_start: (optional) Return the original file offset of the file extent 7268 * @orig_len: (optional) Return the original on-disk length of the file extent 7269 * @ram_bytes: (optional) Return the ram_bytes of the file extent 7270 * @strict: if true, omit optimizations that might force us into unnecessary 7271 * cow. e.g., don't trust generation number. 7272 * 7273 * Return: 7274 * >0 and update @len if we can do nocow write 7275 * 0 if we can't do nocow write 7276 * <0 if error happened 7277 * 7278 * NOTE: This only checks the file extents, caller is responsible to wait for 7279 * any ordered extents. 7280 */ 7281 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 7282 u64 *orig_start, u64 *orig_block_len, 7283 u64 *ram_bytes, bool strict) 7284 { 7285 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7286 struct btrfs_path *path; 7287 int ret; 7288 struct extent_buffer *leaf; 7289 struct btrfs_root *root = BTRFS_I(inode)->root; 7290 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7291 struct btrfs_file_extent_item *fi; 7292 struct btrfs_key key; 7293 u64 disk_bytenr; 7294 u64 backref_offset; 7295 u64 extent_end; 7296 u64 num_bytes; 7297 int slot; 7298 int found_type; 7299 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW); 7300 7301 path = btrfs_alloc_path(); 7302 if (!path) 7303 return -ENOMEM; 7304 7305 ret = btrfs_lookup_file_extent(NULL, root, path, 7306 btrfs_ino(BTRFS_I(inode)), offset, 0); 7307 if (ret < 0) 7308 goto out; 7309 7310 slot = path->slots[0]; 7311 if (ret == 1) { 7312 if (slot == 0) { 7313 /* can't find the item, must cow */ 7314 ret = 0; 7315 goto out; 7316 } 7317 slot--; 7318 } 7319 ret = 0; 7320 leaf = path->nodes[0]; 7321 btrfs_item_key_to_cpu(leaf, &key, slot); 7322 if (key.objectid != btrfs_ino(BTRFS_I(inode)) || 7323 key.type != BTRFS_EXTENT_DATA_KEY) { 7324 /* not our file or wrong item type, must cow */ 7325 goto out; 7326 } 7327 7328 if (key.offset > offset) { 7329 /* Wrong offset, must cow */ 7330 goto out; 7331 } 7332 7333 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 7334 found_type = btrfs_file_extent_type(leaf, fi); 7335 if (found_type != BTRFS_FILE_EXTENT_REG && 7336 found_type != BTRFS_FILE_EXTENT_PREALLOC) { 7337 /* not a regular extent, must cow */ 7338 goto out; 7339 } 7340 7341 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG) 7342 goto out; 7343 7344 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 7345 if (extent_end <= offset) 7346 goto out; 7347 7348 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 7349 if (disk_bytenr == 0) 7350 goto out; 7351 7352 if (btrfs_file_extent_compression(leaf, fi) || 7353 btrfs_file_extent_encryption(leaf, fi) || 7354 btrfs_file_extent_other_encoding(leaf, fi)) 7355 goto out; 7356 7357 /* 7358 * Do the same check as in btrfs_cross_ref_exist but without the 7359 * unnecessary search. 7360 */ 7361 if (!strict && 7362 (btrfs_file_extent_generation(leaf, fi) <= 7363 btrfs_root_last_snapshot(&root->root_item))) 7364 goto out; 7365 7366 backref_offset = btrfs_file_extent_offset(leaf, fi); 7367 7368 if (orig_start) { 7369 *orig_start = key.offset - backref_offset; 7370 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 7371 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 7372 } 7373 7374 if (btrfs_extent_readonly(fs_info, disk_bytenr)) 7375 goto out; 7376 7377 num_bytes = min(offset + *len, extent_end) - offset; 7378 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7379 u64 range_end; 7380 7381 range_end = round_up(offset + num_bytes, 7382 root->fs_info->sectorsize) - 1; 7383 ret = test_range_bit(io_tree, offset, range_end, 7384 EXTENT_DELALLOC, 0, NULL); 7385 if (ret) { 7386 ret = -EAGAIN; 7387 goto out; 7388 } 7389 } 7390 7391 btrfs_release_path(path); 7392 7393 /* 7394 * look for other files referencing this extent, if we 7395 * find any we must cow 7396 */ 7397 7398 ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)), 7399 key.offset - backref_offset, disk_bytenr, 7400 strict); 7401 if (ret) { 7402 ret = 0; 7403 goto out; 7404 } 7405 7406 /* 7407 * adjust disk_bytenr and num_bytes to cover just the bytes 7408 * in this extent we are about to write. If there 7409 * are any csums in that range we have to cow in order 7410 * to keep the csums correct 7411 */ 7412 disk_bytenr += backref_offset; 7413 disk_bytenr += offset - key.offset; 7414 if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes)) 7415 goto out; 7416 /* 7417 * all of the above have passed, it is safe to overwrite this extent 7418 * without cow 7419 */ 7420 *len = num_bytes; 7421 ret = 1; 7422 out: 7423 btrfs_free_path(path); 7424 return ret; 7425 } 7426 7427 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7428 struct extent_state **cached_state, bool writing) 7429 { 7430 struct btrfs_ordered_extent *ordered; 7431 int ret = 0; 7432 7433 while (1) { 7434 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7435 cached_state); 7436 /* 7437 * We're concerned with the entire range that we're going to be 7438 * doing DIO to, so we need to make sure there's no ordered 7439 * extents in this range. 7440 */ 7441 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart, 7442 lockend - lockstart + 1); 7443 7444 /* 7445 * We need to make sure there are no buffered pages in this 7446 * range either, we could have raced between the invalidate in 7447 * generic_file_direct_write and locking the extent. The 7448 * invalidate needs to happen so that reads after a write do not 7449 * get stale data. 7450 */ 7451 if (!ordered && 7452 (!writing || !filemap_range_has_page(inode->i_mapping, 7453 lockstart, lockend))) 7454 break; 7455 7456 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7457 cached_state); 7458 7459 if (ordered) { 7460 /* 7461 * If we are doing a DIO read and the ordered extent we 7462 * found is for a buffered write, we can not wait for it 7463 * to complete and retry, because if we do so we can 7464 * deadlock with concurrent buffered writes on page 7465 * locks. This happens only if our DIO read covers more 7466 * than one extent map, if at this point has already 7467 * created an ordered extent for a previous extent map 7468 * and locked its range in the inode's io tree, and a 7469 * concurrent write against that previous extent map's 7470 * range and this range started (we unlock the ranges 7471 * in the io tree only when the bios complete and 7472 * buffered writes always lock pages before attempting 7473 * to lock range in the io tree). 7474 */ 7475 if (writing || 7476 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) 7477 btrfs_start_ordered_extent(ordered, 1); 7478 else 7479 ret = -ENOTBLK; 7480 btrfs_put_ordered_extent(ordered); 7481 } else { 7482 /* 7483 * We could trigger writeback for this range (and wait 7484 * for it to complete) and then invalidate the pages for 7485 * this range (through invalidate_inode_pages2_range()), 7486 * but that can lead us to a deadlock with a concurrent 7487 * call to readahead (a buffered read or a defrag call 7488 * triggered a readahead) on a page lock due to an 7489 * ordered dio extent we created before but did not have 7490 * yet a corresponding bio submitted (whence it can not 7491 * complete), which makes readahead wait for that 7492 * ordered extent to complete while holding a lock on 7493 * that page. 7494 */ 7495 ret = -ENOTBLK; 7496 } 7497 7498 if (ret) 7499 break; 7500 7501 cond_resched(); 7502 } 7503 7504 return ret; 7505 } 7506 7507 /* The callers of this must take lock_extent() */ 7508 static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, 7509 u64 len, u64 orig_start, u64 block_start, 7510 u64 block_len, u64 orig_block_len, 7511 u64 ram_bytes, int compress_type, 7512 int type) 7513 { 7514 struct extent_map_tree *em_tree; 7515 struct extent_map *em; 7516 int ret; 7517 7518 ASSERT(type == BTRFS_ORDERED_PREALLOC || 7519 type == BTRFS_ORDERED_COMPRESSED || 7520 type == BTRFS_ORDERED_NOCOW || 7521 type == BTRFS_ORDERED_REGULAR); 7522 7523 em_tree = &inode->extent_tree; 7524 em = alloc_extent_map(); 7525 if (!em) 7526 return ERR_PTR(-ENOMEM); 7527 7528 em->start = start; 7529 em->orig_start = orig_start; 7530 em->len = len; 7531 em->block_len = block_len; 7532 em->block_start = block_start; 7533 em->orig_block_len = orig_block_len; 7534 em->ram_bytes = ram_bytes; 7535 em->generation = -1; 7536 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7537 if (type == BTRFS_ORDERED_PREALLOC) { 7538 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7539 } else if (type == BTRFS_ORDERED_COMPRESSED) { 7540 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 7541 em->compress_type = compress_type; 7542 } 7543 7544 do { 7545 btrfs_drop_extent_cache(inode, em->start, 7546 em->start + em->len - 1, 0); 7547 write_lock(&em_tree->lock); 7548 ret = add_extent_mapping(em_tree, em, 1); 7549 write_unlock(&em_tree->lock); 7550 /* 7551 * The caller has taken lock_extent(), who could race with us 7552 * to add em? 7553 */ 7554 } while (ret == -EEXIST); 7555 7556 if (ret) { 7557 free_extent_map(em); 7558 return ERR_PTR(ret); 7559 } 7560 7561 /* em got 2 refs now, callers needs to do free_extent_map once. */ 7562 return em; 7563 } 7564 7565 7566 static int btrfs_get_blocks_direct_write(struct extent_map **map, 7567 struct inode *inode, 7568 struct btrfs_dio_data *dio_data, 7569 u64 start, u64 len) 7570 { 7571 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7572 struct extent_map *em = *map; 7573 int ret = 0; 7574 7575 /* 7576 * We don't allocate a new extent in the following cases 7577 * 7578 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7579 * existing extent. 7580 * 2) The extent is marked as PREALLOC. We're good to go here and can 7581 * just use the extent. 7582 * 7583 */ 7584 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7585 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7586 em->block_start != EXTENT_MAP_HOLE)) { 7587 int type; 7588 u64 block_start, orig_start, orig_block_len, ram_bytes; 7589 7590 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7591 type = BTRFS_ORDERED_PREALLOC; 7592 else 7593 type = BTRFS_ORDERED_NOCOW; 7594 len = min(len, em->len - (start - em->start)); 7595 block_start = em->block_start + (start - em->start); 7596 7597 if (can_nocow_extent(inode, start, &len, &orig_start, 7598 &orig_block_len, &ram_bytes, false) == 1 && 7599 btrfs_inc_nocow_writers(fs_info, block_start)) { 7600 struct extent_map *em2; 7601 7602 em2 = btrfs_create_dio_extent(BTRFS_I(inode), start, len, 7603 orig_start, block_start, 7604 len, orig_block_len, 7605 ram_bytes, type); 7606 btrfs_dec_nocow_writers(fs_info, block_start); 7607 if (type == BTRFS_ORDERED_PREALLOC) { 7608 free_extent_map(em); 7609 *map = em = em2; 7610 } 7611 7612 if (em2 && IS_ERR(em2)) { 7613 ret = PTR_ERR(em2); 7614 goto out; 7615 } 7616 /* 7617 * For inode marked NODATACOW or extent marked PREALLOC, 7618 * use the existing or preallocated extent, so does not 7619 * need to adjust btrfs_space_info's bytes_may_use. 7620 */ 7621 btrfs_free_reserved_data_space_noquota(fs_info, len); 7622 goto skip_cow; 7623 } 7624 } 7625 7626 /* this will cow the extent */ 7627 free_extent_map(em); 7628 *map = em = btrfs_new_extent_direct(BTRFS_I(inode), start, len); 7629 if (IS_ERR(em)) { 7630 ret = PTR_ERR(em); 7631 goto out; 7632 } 7633 7634 len = min(len, em->len - (start - em->start)); 7635 7636 skip_cow: 7637 /* 7638 * Need to update the i_size under the extent lock so buffered 7639 * readers will get the updated i_size when we unlock. 7640 */ 7641 if (start + len > i_size_read(inode)) 7642 i_size_write(inode, start + len); 7643 7644 dio_data->reserve -= len; 7645 out: 7646 return ret; 7647 } 7648 7649 static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, 7650 loff_t length, unsigned int flags, struct iomap *iomap, 7651 struct iomap *srcmap) 7652 { 7653 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7654 struct extent_map *em; 7655 struct extent_state *cached_state = NULL; 7656 struct btrfs_dio_data *dio_data = NULL; 7657 u64 lockstart, lockend; 7658 const bool write = !!(flags & IOMAP_WRITE); 7659 int ret = 0; 7660 u64 len = length; 7661 bool unlock_extents = false; 7662 7663 if (!write) 7664 len = min_t(u64, len, fs_info->sectorsize); 7665 7666 lockstart = start; 7667 lockend = start + len - 1; 7668 7669 /* 7670 * The generic stuff only does filemap_write_and_wait_range, which 7671 * isn't enough if we've written compressed pages to this area, so we 7672 * need to flush the dirty pages again to make absolutely sure that any 7673 * outstanding dirty pages are on disk. 7674 */ 7675 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 7676 &BTRFS_I(inode)->runtime_flags)) { 7677 ret = filemap_fdatawrite_range(inode->i_mapping, start, 7678 start + length - 1); 7679 if (ret) 7680 return ret; 7681 } 7682 7683 dio_data = kzalloc(sizeof(*dio_data), GFP_NOFS); 7684 if (!dio_data) 7685 return -ENOMEM; 7686 7687 dio_data->length = length; 7688 if (write) { 7689 dio_data->reserve = round_up(length, fs_info->sectorsize); 7690 ret = btrfs_delalloc_reserve_space(BTRFS_I(inode), 7691 &dio_data->data_reserved, 7692 start, dio_data->reserve); 7693 if (ret) { 7694 extent_changeset_free(dio_data->data_reserved); 7695 kfree(dio_data); 7696 return ret; 7697 } 7698 } 7699 iomap->private = dio_data; 7700 7701 7702 /* 7703 * If this errors out it's because we couldn't invalidate pagecache for 7704 * this range and we need to fallback to buffered. 7705 */ 7706 if (lock_extent_direct(inode, lockstart, lockend, &cached_state, write)) { 7707 ret = -ENOTBLK; 7708 goto err; 7709 } 7710 7711 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len); 7712 if (IS_ERR(em)) { 7713 ret = PTR_ERR(em); 7714 goto unlock_err; 7715 } 7716 7717 /* 7718 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7719 * io. INLINE is special, and we could probably kludge it in here, but 7720 * it's still buffered so for safety lets just fall back to the generic 7721 * buffered path. 7722 * 7723 * For COMPRESSED we _have_ to read the entire extent in so we can 7724 * decompress it, so there will be buffering required no matter what we 7725 * do, so go ahead and fallback to buffered. 7726 * 7727 * We return -ENOTBLK because that's what makes DIO go ahead and go back 7728 * to buffered IO. Don't blame me, this is the price we pay for using 7729 * the generic code. 7730 */ 7731 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7732 em->block_start == EXTENT_MAP_INLINE) { 7733 free_extent_map(em); 7734 ret = -ENOTBLK; 7735 goto unlock_err; 7736 } 7737 7738 len = min(len, em->len - (start - em->start)); 7739 if (write) { 7740 ret = btrfs_get_blocks_direct_write(&em, inode, dio_data, 7741 start, len); 7742 if (ret < 0) 7743 goto unlock_err; 7744 unlock_extents = true; 7745 /* Recalc len in case the new em is smaller than requested */ 7746 len = min(len, em->len - (start - em->start)); 7747 } else { 7748 /* 7749 * We need to unlock only the end area that we aren't using. 7750 * The rest is going to be unlocked by the endio routine. 7751 */ 7752 lockstart = start + len; 7753 if (lockstart < lockend) 7754 unlock_extents = true; 7755 } 7756 7757 if (unlock_extents) 7758 unlock_extent_cached(&BTRFS_I(inode)->io_tree, 7759 lockstart, lockend, &cached_state); 7760 else 7761 free_extent_state(cached_state); 7762 7763 /* 7764 * Translate extent map information to iomap. 7765 * We trim the extents (and move the addr) even though iomap code does 7766 * that, since we have locked only the parts we are performing I/O in. 7767 */ 7768 if ((em->block_start == EXTENT_MAP_HOLE) || 7769 (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) && !write)) { 7770 iomap->addr = IOMAP_NULL_ADDR; 7771 iomap->type = IOMAP_HOLE; 7772 } else { 7773 iomap->addr = em->block_start + (start - em->start); 7774 iomap->type = IOMAP_MAPPED; 7775 } 7776 iomap->offset = start; 7777 iomap->bdev = fs_info->fs_devices->latest_bdev; 7778 iomap->length = len; 7779 7780 if (write && btrfs_use_zone_append(BTRFS_I(inode), em)) 7781 iomap->flags |= IOMAP_F_ZONE_APPEND; 7782 7783 free_extent_map(em); 7784 7785 return 0; 7786 7787 unlock_err: 7788 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7789 &cached_state); 7790 err: 7791 if (dio_data) { 7792 btrfs_delalloc_release_space(BTRFS_I(inode), 7793 dio_data->data_reserved, start, 7794 dio_data->reserve, true); 7795 btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->reserve); 7796 extent_changeset_free(dio_data->data_reserved); 7797 kfree(dio_data); 7798 } 7799 return ret; 7800 } 7801 7802 static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, 7803 ssize_t written, unsigned int flags, struct iomap *iomap) 7804 { 7805 int ret = 0; 7806 struct btrfs_dio_data *dio_data = iomap->private; 7807 size_t submitted = dio_data->submitted; 7808 const bool write = !!(flags & IOMAP_WRITE); 7809 7810 if (!write && (iomap->type == IOMAP_HOLE)) { 7811 /* If reading from a hole, unlock and return */ 7812 unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1); 7813 goto out; 7814 } 7815 7816 if (submitted < length) { 7817 pos += submitted; 7818 length -= submitted; 7819 if (write) 7820 __endio_write_update_ordered(BTRFS_I(inode), pos, 7821 length, false); 7822 else 7823 unlock_extent(&BTRFS_I(inode)->io_tree, pos, 7824 pos + length - 1); 7825 ret = -ENOTBLK; 7826 } 7827 7828 if (write) { 7829 if (dio_data->reserve) 7830 btrfs_delalloc_release_space(BTRFS_I(inode), 7831 dio_data->data_reserved, pos, 7832 dio_data->reserve, true); 7833 btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->length); 7834 extent_changeset_free(dio_data->data_reserved); 7835 } 7836 out: 7837 kfree(dio_data); 7838 iomap->private = NULL; 7839 7840 return ret; 7841 } 7842 7843 static void btrfs_dio_private_put(struct btrfs_dio_private *dip) 7844 { 7845 /* 7846 * This implies a barrier so that stores to dio_bio->bi_status before 7847 * this and loads of dio_bio->bi_status after this are fully ordered. 7848 */ 7849 if (!refcount_dec_and_test(&dip->refs)) 7850 return; 7851 7852 if (btrfs_op(dip->dio_bio) == BTRFS_MAP_WRITE) { 7853 __endio_write_update_ordered(BTRFS_I(dip->inode), 7854 dip->logical_offset, 7855 dip->bytes, 7856 !dip->dio_bio->bi_status); 7857 } else { 7858 unlock_extent(&BTRFS_I(dip->inode)->io_tree, 7859 dip->logical_offset, 7860 dip->logical_offset + dip->bytes - 1); 7861 } 7862 7863 bio_endio(dip->dio_bio); 7864 kfree(dip); 7865 } 7866 7867 static blk_status_t submit_dio_repair_bio(struct inode *inode, struct bio *bio, 7868 int mirror_num, 7869 unsigned long bio_flags) 7870 { 7871 struct btrfs_dio_private *dip = bio->bi_private; 7872 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7873 blk_status_t ret; 7874 7875 BUG_ON(bio_op(bio) == REQ_OP_WRITE); 7876 7877 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 7878 if (ret) 7879 return ret; 7880 7881 refcount_inc(&dip->refs); 7882 ret = btrfs_map_bio(fs_info, bio, mirror_num); 7883 if (ret) 7884 refcount_dec(&dip->refs); 7885 return ret; 7886 } 7887 7888 static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, 7889 struct btrfs_io_bio *io_bio, 7890 const bool uptodate) 7891 { 7892 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 7893 const u32 sectorsize = fs_info->sectorsize; 7894 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 7895 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7896 const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); 7897 struct bio_vec bvec; 7898 struct bvec_iter iter; 7899 u64 start = io_bio->logical; 7900 u32 bio_offset = 0; 7901 blk_status_t err = BLK_STS_OK; 7902 7903 __bio_for_each_segment(bvec, &io_bio->bio, iter, io_bio->iter) { 7904 unsigned int i, nr_sectors, pgoff; 7905 7906 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); 7907 pgoff = bvec.bv_offset; 7908 for (i = 0; i < nr_sectors; i++) { 7909 ASSERT(pgoff < PAGE_SIZE); 7910 if (uptodate && 7911 (!csum || !check_data_csum(inode, io_bio, 7912 bio_offset, bvec.bv_page, pgoff))) { 7913 clean_io_failure(fs_info, failure_tree, io_tree, 7914 start, bvec.bv_page, 7915 btrfs_ino(BTRFS_I(inode)), 7916 pgoff); 7917 } else { 7918 blk_status_t status; 7919 7920 ASSERT((start - io_bio->logical) < UINT_MAX); 7921 status = btrfs_submit_read_repair(inode, 7922 &io_bio->bio, 7923 start - io_bio->logical, 7924 bvec.bv_page, pgoff, 7925 start, 7926 start + sectorsize - 1, 7927 io_bio->mirror_num, 7928 submit_dio_repair_bio); 7929 if (status) 7930 err = status; 7931 } 7932 start += sectorsize; 7933 ASSERT(bio_offset + sectorsize > bio_offset); 7934 bio_offset += sectorsize; 7935 pgoff += sectorsize; 7936 } 7937 } 7938 return err; 7939 } 7940 7941 static void __endio_write_update_ordered(struct btrfs_inode *inode, 7942 const u64 offset, const u64 bytes, 7943 const bool uptodate) 7944 { 7945 struct btrfs_fs_info *fs_info = inode->root->fs_info; 7946 struct btrfs_ordered_extent *ordered = NULL; 7947 struct btrfs_workqueue *wq; 7948 u64 ordered_offset = offset; 7949 u64 ordered_bytes = bytes; 7950 u64 last_offset; 7951 7952 if (btrfs_is_free_space_inode(inode)) 7953 wq = fs_info->endio_freespace_worker; 7954 else 7955 wq = fs_info->endio_write_workers; 7956 7957 while (ordered_offset < offset + bytes) { 7958 last_offset = ordered_offset; 7959 if (btrfs_dec_test_first_ordered_pending(inode, &ordered, 7960 &ordered_offset, 7961 ordered_bytes, 7962 uptodate)) { 7963 btrfs_init_work(&ordered->work, finish_ordered_fn, NULL, 7964 NULL); 7965 btrfs_queue_work(wq, &ordered->work); 7966 } 7967 7968 /* No ordered extent found in the range, exit */ 7969 if (ordered_offset == last_offset) 7970 return; 7971 /* 7972 * Our bio might span multiple ordered extents. In this case 7973 * we keep going until we have accounted the whole dio. 7974 */ 7975 if (ordered_offset < offset + bytes) { 7976 ordered_bytes = offset + bytes - ordered_offset; 7977 ordered = NULL; 7978 } 7979 } 7980 } 7981 7982 static blk_status_t btrfs_submit_bio_start_direct_io(struct inode *inode, 7983 struct bio *bio, 7984 u64 dio_file_offset) 7985 { 7986 return btrfs_csum_one_bio(BTRFS_I(inode), bio, dio_file_offset, 1); 7987 } 7988 7989 static void btrfs_end_dio_bio(struct bio *bio) 7990 { 7991 struct btrfs_dio_private *dip = bio->bi_private; 7992 blk_status_t err = bio->bi_status; 7993 7994 if (err) 7995 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 7996 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", 7997 btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio), 7998 bio->bi_opf, bio->bi_iter.bi_sector, 7999 bio->bi_iter.bi_size, err); 8000 8001 if (bio_op(bio) == REQ_OP_READ) { 8002 err = btrfs_check_read_dio_bio(dip->inode, btrfs_io_bio(bio), 8003 !err); 8004 } 8005 8006 if (err) 8007 dip->dio_bio->bi_status = err; 8008 8009 btrfs_record_physical_zoned(dip->inode, dip->logical_offset, bio); 8010 8011 bio_put(bio); 8012 btrfs_dio_private_put(dip); 8013 } 8014 8015 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, 8016 struct inode *inode, u64 file_offset, int async_submit) 8017 { 8018 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8019 struct btrfs_dio_private *dip = bio->bi_private; 8020 bool write = btrfs_op(bio) == BTRFS_MAP_WRITE; 8021 blk_status_t ret; 8022 8023 /* Check btrfs_submit_bio_hook() for rules about async submit. */ 8024 if (async_submit) 8025 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers); 8026 8027 if (!write) { 8028 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); 8029 if (ret) 8030 goto err; 8031 } 8032 8033 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 8034 goto map; 8035 8036 if (write && async_submit) { 8037 ret = btrfs_wq_submit_bio(inode, bio, 0, 0, file_offset, 8038 btrfs_submit_bio_start_direct_io); 8039 goto err; 8040 } else if (write) { 8041 /* 8042 * If we aren't doing async submit, calculate the csum of the 8043 * bio now. 8044 */ 8045 ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, file_offset, 1); 8046 if (ret) 8047 goto err; 8048 } else { 8049 u64 csum_offset; 8050 8051 csum_offset = file_offset - dip->logical_offset; 8052 csum_offset >>= fs_info->sectorsize_bits; 8053 csum_offset *= fs_info->csum_size; 8054 btrfs_io_bio(bio)->csum = dip->csums + csum_offset; 8055 } 8056 map: 8057 ret = btrfs_map_bio(fs_info, bio, 0); 8058 err: 8059 return ret; 8060 } 8061 8062 /* 8063 * If this succeeds, the btrfs_dio_private is responsible for cleaning up locked 8064 * or ordered extents whether or not we submit any bios. 8065 */ 8066 static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, 8067 struct inode *inode, 8068 loff_t file_offset) 8069 { 8070 const bool write = (btrfs_op(dio_bio) == BTRFS_MAP_WRITE); 8071 const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); 8072 size_t dip_size; 8073 struct btrfs_dio_private *dip; 8074 8075 dip_size = sizeof(*dip); 8076 if (!write && csum) { 8077 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8078 size_t nblocks; 8079 8080 nblocks = dio_bio->bi_iter.bi_size >> fs_info->sectorsize_bits; 8081 dip_size += fs_info->csum_size * nblocks; 8082 } 8083 8084 dip = kzalloc(dip_size, GFP_NOFS); 8085 if (!dip) 8086 return NULL; 8087 8088 dip->inode = inode; 8089 dip->logical_offset = file_offset; 8090 dip->bytes = dio_bio->bi_iter.bi_size; 8091 dip->disk_bytenr = dio_bio->bi_iter.bi_sector << 9; 8092 dip->dio_bio = dio_bio; 8093 refcount_set(&dip->refs, 1); 8094 return dip; 8095 } 8096 8097 static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, 8098 struct bio *dio_bio, loff_t file_offset) 8099 { 8100 const bool write = (btrfs_op(dio_bio) == BTRFS_MAP_WRITE); 8101 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8102 const bool raid56 = (btrfs_data_alloc_profile(fs_info) & 8103 BTRFS_BLOCK_GROUP_RAID56_MASK); 8104 struct btrfs_dio_private *dip; 8105 struct bio *bio; 8106 u64 start_sector; 8107 int async_submit = 0; 8108 u64 submit_len; 8109 int clone_offset = 0; 8110 int clone_len; 8111 u64 logical; 8112 int ret; 8113 blk_status_t status; 8114 struct btrfs_io_geometry geom; 8115 struct btrfs_dio_data *dio_data = iomap->private; 8116 struct extent_map *em = NULL; 8117 8118 dip = btrfs_create_dio_private(dio_bio, inode, file_offset); 8119 if (!dip) { 8120 if (!write) { 8121 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, 8122 file_offset + dio_bio->bi_iter.bi_size - 1); 8123 } 8124 dio_bio->bi_status = BLK_STS_RESOURCE; 8125 bio_endio(dio_bio); 8126 return BLK_QC_T_NONE; 8127 } 8128 8129 if (!write) { 8130 /* 8131 * Load the csums up front to reduce csum tree searches and 8132 * contention when submitting bios. 8133 * 8134 * If we have csums disabled this will do nothing. 8135 */ 8136 status = btrfs_lookup_bio_sums(inode, dio_bio, dip->csums); 8137 if (status != BLK_STS_OK) 8138 goto out_err; 8139 } 8140 8141 start_sector = dio_bio->bi_iter.bi_sector; 8142 submit_len = dio_bio->bi_iter.bi_size; 8143 8144 do { 8145 logical = start_sector << 9; 8146 em = btrfs_get_chunk_map(fs_info, logical, submit_len); 8147 if (IS_ERR(em)) { 8148 status = errno_to_blk_status(PTR_ERR(em)); 8149 em = NULL; 8150 goto out_err_em; 8151 } 8152 ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(dio_bio), 8153 logical, submit_len, &geom); 8154 if (ret) { 8155 status = errno_to_blk_status(ret); 8156 goto out_err_em; 8157 } 8158 ASSERT(geom.len <= INT_MAX); 8159 8160 clone_len = min_t(int, submit_len, geom.len); 8161 8162 /* 8163 * This will never fail as it's passing GPF_NOFS and 8164 * the allocation is backed by btrfs_bioset. 8165 */ 8166 bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len); 8167 bio->bi_private = dip; 8168 bio->bi_end_io = btrfs_end_dio_bio; 8169 btrfs_io_bio(bio)->logical = file_offset; 8170 8171 WARN_ON_ONCE(write && btrfs_is_zoned(fs_info) && 8172 fs_info->max_zone_append_size && 8173 bio_op(bio) != REQ_OP_ZONE_APPEND); 8174 8175 if (bio_op(bio) == REQ_OP_ZONE_APPEND) { 8176 status = extract_ordered_extent(BTRFS_I(inode), bio, 8177 file_offset); 8178 if (status) { 8179 bio_put(bio); 8180 goto out_err; 8181 } 8182 } 8183 8184 ASSERT(submit_len >= clone_len); 8185 submit_len -= clone_len; 8186 8187 /* 8188 * Increase the count before we submit the bio so we know 8189 * the end IO handler won't happen before we increase the 8190 * count. Otherwise, the dip might get freed before we're 8191 * done setting it up. 8192 * 8193 * We transfer the initial reference to the last bio, so we 8194 * don't need to increment the reference count for the last one. 8195 */ 8196 if (submit_len > 0) { 8197 refcount_inc(&dip->refs); 8198 /* 8199 * If we are submitting more than one bio, submit them 8200 * all asynchronously. The exception is RAID 5 or 6, as 8201 * asynchronous checksums make it difficult to collect 8202 * full stripe writes. 8203 */ 8204 if (!raid56) 8205 async_submit = 1; 8206 } 8207 8208 status = btrfs_submit_dio_bio(bio, inode, file_offset, 8209 async_submit); 8210 if (status) { 8211 bio_put(bio); 8212 if (submit_len > 0) 8213 refcount_dec(&dip->refs); 8214 goto out_err_em; 8215 } 8216 8217 dio_data->submitted += clone_len; 8218 clone_offset += clone_len; 8219 start_sector += clone_len >> 9; 8220 file_offset += clone_len; 8221 8222 free_extent_map(em); 8223 } while (submit_len > 0); 8224 return BLK_QC_T_NONE; 8225 8226 out_err_em: 8227 free_extent_map(em); 8228 out_err: 8229 dip->dio_bio->bi_status = status; 8230 btrfs_dio_private_put(dip); 8231 8232 return BLK_QC_T_NONE; 8233 } 8234 8235 const struct iomap_ops btrfs_dio_iomap_ops = { 8236 .iomap_begin = btrfs_dio_iomap_begin, 8237 .iomap_end = btrfs_dio_iomap_end, 8238 }; 8239 8240 const struct iomap_dio_ops btrfs_dio_ops = { 8241 .submit_io = btrfs_submit_direct, 8242 }; 8243 8244 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 8245 u64 start, u64 len) 8246 { 8247 int ret; 8248 8249 ret = fiemap_prep(inode, fieinfo, start, &len, 0); 8250 if (ret) 8251 return ret; 8252 8253 return extent_fiemap(BTRFS_I(inode), fieinfo, start, len); 8254 } 8255 8256 int btrfs_readpage(struct file *file, struct page *page) 8257 { 8258 struct btrfs_inode *inode = BTRFS_I(page->mapping->host); 8259 u64 start = page_offset(page); 8260 u64 end = start + PAGE_SIZE - 1; 8261 unsigned long bio_flags = 0; 8262 struct bio *bio = NULL; 8263 int ret; 8264 8265 btrfs_lock_and_flush_ordered_range(inode, start, end, NULL); 8266 8267 ret = btrfs_do_readpage(page, NULL, &bio, &bio_flags, 0, NULL); 8268 if (bio) 8269 ret = submit_one_bio(bio, 0, bio_flags); 8270 return ret; 8271 } 8272 8273 static int btrfs_writepage(struct page *page, struct writeback_control *wbc) 8274 { 8275 struct inode *inode = page->mapping->host; 8276 int ret; 8277 8278 if (current->flags & PF_MEMALLOC) { 8279 redirty_page_for_writepage(wbc, page); 8280 unlock_page(page); 8281 return 0; 8282 } 8283 8284 /* 8285 * If we are under memory pressure we will call this directly from the 8286 * VM, we need to make sure we have the inode referenced for the ordered 8287 * extent. If not just return like we didn't do anything. 8288 */ 8289 if (!igrab(inode)) { 8290 redirty_page_for_writepage(wbc, page); 8291 return AOP_WRITEPAGE_ACTIVATE; 8292 } 8293 ret = extent_write_full_page(page, wbc); 8294 btrfs_add_delayed_iput(inode); 8295 return ret; 8296 } 8297 8298 static int btrfs_writepages(struct address_space *mapping, 8299 struct writeback_control *wbc) 8300 { 8301 return extent_writepages(mapping, wbc); 8302 } 8303 8304 static void btrfs_readahead(struct readahead_control *rac) 8305 { 8306 extent_readahead(rac); 8307 } 8308 8309 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8310 { 8311 int ret = try_release_extent_mapping(page, gfp_flags); 8312 if (ret == 1) 8313 clear_page_extent_mapped(page); 8314 return ret; 8315 } 8316 8317 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8318 { 8319 if (PageWriteback(page) || PageDirty(page)) 8320 return 0; 8321 return __btrfs_releasepage(page, gfp_flags); 8322 } 8323 8324 #ifdef CONFIG_MIGRATION 8325 static int btrfs_migratepage(struct address_space *mapping, 8326 struct page *newpage, struct page *page, 8327 enum migrate_mode mode) 8328 { 8329 int ret; 8330 8331 ret = migrate_page_move_mapping(mapping, newpage, page, 0); 8332 if (ret != MIGRATEPAGE_SUCCESS) 8333 return ret; 8334 8335 if (page_has_private(page)) 8336 attach_page_private(newpage, detach_page_private(page)); 8337 8338 if (PagePrivate2(page)) { 8339 ClearPagePrivate2(page); 8340 SetPagePrivate2(newpage); 8341 } 8342 8343 if (mode != MIGRATE_SYNC_NO_COPY) 8344 migrate_page_copy(newpage, page); 8345 else 8346 migrate_page_states(newpage, page); 8347 return MIGRATEPAGE_SUCCESS; 8348 } 8349 #endif 8350 8351 static void btrfs_invalidatepage(struct page *page, unsigned int offset, 8352 unsigned int length) 8353 { 8354 struct btrfs_inode *inode = BTRFS_I(page->mapping->host); 8355 struct extent_io_tree *tree = &inode->io_tree; 8356 struct btrfs_ordered_extent *ordered; 8357 struct extent_state *cached_state = NULL; 8358 u64 page_start = page_offset(page); 8359 u64 page_end = page_start + PAGE_SIZE - 1; 8360 u64 start; 8361 u64 end; 8362 int inode_evicting = inode->vfs_inode.i_state & I_FREEING; 8363 bool found_ordered = false; 8364 bool completed_ordered = false; 8365 8366 /* 8367 * we have the page locked, so new writeback can't start, 8368 * and the dirty bit won't be cleared while we are here. 8369 * 8370 * Wait for IO on this page so that we can safely clear 8371 * the PagePrivate2 bit and do ordered accounting 8372 */ 8373 wait_on_page_writeback(page); 8374 8375 if (offset) { 8376 btrfs_releasepage(page, GFP_NOFS); 8377 return; 8378 } 8379 8380 if (!inode_evicting) 8381 lock_extent_bits(tree, page_start, page_end, &cached_state); 8382 8383 start = page_start; 8384 again: 8385 ordered = btrfs_lookup_ordered_range(inode, start, page_end - start + 1); 8386 if (ordered) { 8387 found_ordered = true; 8388 end = min(page_end, 8389 ordered->file_offset + ordered->num_bytes - 1); 8390 /* 8391 * IO on this page will never be started, so we need to account 8392 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW 8393 * here, must leave that up for the ordered extent completion. 8394 */ 8395 if (!inode_evicting) 8396 clear_extent_bit(tree, start, end, 8397 EXTENT_DELALLOC | 8398 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 8399 EXTENT_DEFRAG, 1, 0, &cached_state); 8400 /* 8401 * whoever cleared the private bit is responsible 8402 * for the finish_ordered_io 8403 */ 8404 if (TestClearPagePrivate2(page)) { 8405 struct btrfs_ordered_inode_tree *tree; 8406 u64 new_len; 8407 8408 tree = &inode->ordered_tree; 8409 8410 spin_lock_irq(&tree->lock); 8411 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 8412 new_len = start - ordered->file_offset; 8413 if (new_len < ordered->truncated_len) 8414 ordered->truncated_len = new_len; 8415 spin_unlock_irq(&tree->lock); 8416 8417 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8418 start, 8419 end - start + 1, 1)) { 8420 btrfs_finish_ordered_io(ordered); 8421 completed_ordered = true; 8422 } 8423 } 8424 btrfs_put_ordered_extent(ordered); 8425 if (!inode_evicting) { 8426 cached_state = NULL; 8427 lock_extent_bits(tree, start, end, 8428 &cached_state); 8429 } 8430 8431 start = end + 1; 8432 if (start < page_end) 8433 goto again; 8434 } 8435 8436 /* 8437 * Qgroup reserved space handler 8438 * Page here will be either 8439 * 1) Already written to disk or ordered extent already submitted 8440 * Then its QGROUP_RESERVED bit in io_tree is already cleaned. 8441 * Qgroup will be handled by its qgroup_record then. 8442 * btrfs_qgroup_free_data() call will do nothing here. 8443 * 8444 * 2) Not written to disk yet 8445 * Then btrfs_qgroup_free_data() call will clear the QGROUP_RESERVED 8446 * bit of its io_tree, and free the qgroup reserved data space. 8447 * Since the IO will never happen for this page. 8448 */ 8449 btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE); 8450 if (!inode_evicting) { 8451 bool delete = true; 8452 8453 /* 8454 * If there's an ordered extent for this range and we have not 8455 * finished it ourselves, we must leave EXTENT_DELALLOC_NEW set 8456 * in the range for the ordered extent completion. We must also 8457 * not delete the range, otherwise we would lose that bit (and 8458 * any other bits set in the range). Make sure EXTENT_UPTODATE 8459 * is cleared if we don't delete, otherwise it can lead to 8460 * corruptions if the i_size is extented later. 8461 */ 8462 if (found_ordered && !completed_ordered) 8463 delete = false; 8464 clear_extent_bit(tree, page_start, page_end, EXTENT_LOCKED | 8465 EXTENT_DELALLOC | EXTENT_UPTODATE | 8466 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 8467 delete, &cached_state); 8468 8469 __btrfs_releasepage(page, GFP_NOFS); 8470 } 8471 8472 ClearPageChecked(page); 8473 clear_page_extent_mapped(page); 8474 } 8475 8476 /* 8477 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8478 * called from a page fault handler when a page is first dirtied. Hence we must 8479 * be careful to check for EOF conditions here. We set the page up correctly 8480 * for a written page which means we get ENOSPC checking when writing into 8481 * holes and correct delalloc and unwritten extent mapping on filesystems that 8482 * support these features. 8483 * 8484 * We are not allowed to take the i_mutex here so we have to play games to 8485 * protect against truncate races as the page could now be beyond EOF. Because 8486 * truncate_setsize() writes the inode size before removing pages, once we have 8487 * the page lock we can determine safely if the page is beyond EOF. If it is not 8488 * beyond EOF, then the page is guaranteed safe against truncation until we 8489 * unlock the page. 8490 */ 8491 vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) 8492 { 8493 struct page *page = vmf->page; 8494 struct inode *inode = file_inode(vmf->vma->vm_file); 8495 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8496 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8497 struct btrfs_ordered_extent *ordered; 8498 struct extent_state *cached_state = NULL; 8499 struct extent_changeset *data_reserved = NULL; 8500 char *kaddr; 8501 unsigned long zero_start; 8502 loff_t size; 8503 vm_fault_t ret; 8504 int ret2; 8505 int reserved = 0; 8506 u64 reserved_space; 8507 u64 page_start; 8508 u64 page_end; 8509 u64 end; 8510 8511 reserved_space = PAGE_SIZE; 8512 8513 sb_start_pagefault(inode->i_sb); 8514 page_start = page_offset(page); 8515 page_end = page_start + PAGE_SIZE - 1; 8516 end = page_end; 8517 8518 /* 8519 * Reserving delalloc space after obtaining the page lock can lead to 8520 * deadlock. For example, if a dirty page is locked by this function 8521 * and the call to btrfs_delalloc_reserve_space() ends up triggering 8522 * dirty page write out, then the btrfs_writepage() function could 8523 * end up waiting indefinitely to get a lock on the page currently 8524 * being processed by btrfs_page_mkwrite() function. 8525 */ 8526 ret2 = btrfs_delalloc_reserve_space(BTRFS_I(inode), &data_reserved, 8527 page_start, reserved_space); 8528 if (!ret2) { 8529 ret2 = file_update_time(vmf->vma->vm_file); 8530 reserved = 1; 8531 } 8532 if (ret2) { 8533 ret = vmf_error(ret2); 8534 if (reserved) 8535 goto out; 8536 goto out_noreserve; 8537 } 8538 8539 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 8540 again: 8541 lock_page(page); 8542 size = i_size_read(inode); 8543 8544 if ((page->mapping != inode->i_mapping) || 8545 (page_start >= size)) { 8546 /* page got truncated out from underneath us */ 8547 goto out_unlock; 8548 } 8549 wait_on_page_writeback(page); 8550 8551 lock_extent_bits(io_tree, page_start, page_end, &cached_state); 8552 ret2 = set_page_extent_mapped(page); 8553 if (ret2 < 0) { 8554 ret = vmf_error(ret2); 8555 unlock_extent_cached(io_tree, page_start, page_end, &cached_state); 8556 goto out_unlock; 8557 } 8558 8559 /* 8560 * we can't set the delalloc bits if there are pending ordered 8561 * extents. Drop our locks and wait for them to finish 8562 */ 8563 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 8564 PAGE_SIZE); 8565 if (ordered) { 8566 unlock_extent_cached(io_tree, page_start, page_end, 8567 &cached_state); 8568 unlock_page(page); 8569 btrfs_start_ordered_extent(ordered, 1); 8570 btrfs_put_ordered_extent(ordered); 8571 goto again; 8572 } 8573 8574 if (page->index == ((size - 1) >> PAGE_SHIFT)) { 8575 reserved_space = round_up(size - page_start, 8576 fs_info->sectorsize); 8577 if (reserved_space < PAGE_SIZE) { 8578 end = page_start + reserved_space - 1; 8579 btrfs_delalloc_release_space(BTRFS_I(inode), 8580 data_reserved, page_start, 8581 PAGE_SIZE - reserved_space, true); 8582 } 8583 } 8584 8585 /* 8586 * page_mkwrite gets called when the page is firstly dirtied after it's 8587 * faulted in, but write(2) could also dirty a page and set delalloc 8588 * bits, thus in this case for space account reason, we still need to 8589 * clear any delalloc bits within this page range since we have to 8590 * reserve data&meta space before lock_page() (see above comments). 8591 */ 8592 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, 8593 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 8594 EXTENT_DEFRAG, 0, 0, &cached_state); 8595 8596 ret2 = btrfs_set_extent_delalloc(BTRFS_I(inode), page_start, end, 0, 8597 &cached_state); 8598 if (ret2) { 8599 unlock_extent_cached(io_tree, page_start, page_end, 8600 &cached_state); 8601 ret = VM_FAULT_SIGBUS; 8602 goto out_unlock; 8603 } 8604 8605 /* page is wholly or partially inside EOF */ 8606 if (page_start + PAGE_SIZE > size) 8607 zero_start = offset_in_page(size); 8608 else 8609 zero_start = PAGE_SIZE; 8610 8611 if (zero_start != PAGE_SIZE) { 8612 kaddr = kmap(page); 8613 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start); 8614 flush_dcache_page(page); 8615 kunmap(page); 8616 } 8617 ClearPageChecked(page); 8618 set_page_dirty(page); 8619 SetPageUptodate(page); 8620 8621 BTRFS_I(inode)->last_trans = fs_info->generation; 8622 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; 8623 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; 8624 8625 unlock_extent_cached(io_tree, page_start, page_end, &cached_state); 8626 8627 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); 8628 sb_end_pagefault(inode->i_sb); 8629 extent_changeset_free(data_reserved); 8630 return VM_FAULT_LOCKED; 8631 8632 out_unlock: 8633 unlock_page(page); 8634 out: 8635 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); 8636 btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved, page_start, 8637 reserved_space, (ret != 0)); 8638 out_noreserve: 8639 sb_end_pagefault(inode->i_sb); 8640 extent_changeset_free(data_reserved); 8641 return ret; 8642 } 8643 8644 static int btrfs_truncate(struct inode *inode, bool skip_writeback) 8645 { 8646 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8647 struct btrfs_root *root = BTRFS_I(inode)->root; 8648 struct btrfs_block_rsv *rsv; 8649 int ret; 8650 struct btrfs_trans_handle *trans; 8651 u64 mask = fs_info->sectorsize - 1; 8652 u64 min_size = btrfs_calc_metadata_size(fs_info, 1); 8653 8654 if (!skip_writeback) { 8655 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), 8656 (u64)-1); 8657 if (ret) 8658 return ret; 8659 } 8660 8661 /* 8662 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of 8663 * things going on here: 8664 * 8665 * 1) We need to reserve space to update our inode. 8666 * 8667 * 2) We need to have something to cache all the space that is going to 8668 * be free'd up by the truncate operation, but also have some slack 8669 * space reserved in case it uses space during the truncate (thank you 8670 * very much snapshotting). 8671 * 8672 * And we need these to be separate. The fact is we can use a lot of 8673 * space doing the truncate, and we have no earthly idea how much space 8674 * we will use, so we need the truncate reservation to be separate so it 8675 * doesn't end up using space reserved for updating the inode. We also 8676 * need to be able to stop the transaction and start a new one, which 8677 * means we need to be able to update the inode several times, and we 8678 * have no idea of knowing how many times that will be, so we can't just 8679 * reserve 1 item for the entirety of the operation, so that has to be 8680 * done separately as well. 8681 * 8682 * So that leaves us with 8683 * 8684 * 1) rsv - for the truncate reservation, which we will steal from the 8685 * transaction reservation. 8686 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for 8687 * updating the inode. 8688 */ 8689 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 8690 if (!rsv) 8691 return -ENOMEM; 8692 rsv->size = min_size; 8693 rsv->failfast = 1; 8694 8695 /* 8696 * 1 for the truncate slack space 8697 * 1 for updating the inode. 8698 */ 8699 trans = btrfs_start_transaction(root, 2); 8700 if (IS_ERR(trans)) { 8701 ret = PTR_ERR(trans); 8702 goto out; 8703 } 8704 8705 /* Migrate the slack space for the truncate to our reserve */ 8706 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv, 8707 min_size, false); 8708 BUG_ON(ret); 8709 8710 /* 8711 * So if we truncate and then write and fsync we normally would just 8712 * write the extents that changed, which is a problem if we need to 8713 * first truncate that entire inode. So set this flag so we write out 8714 * all of the extents in the inode to the sync log so we're completely 8715 * safe. 8716 */ 8717 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 8718 trans->block_rsv = rsv; 8719 8720 while (1) { 8721 ret = btrfs_truncate_inode_items(trans, root, BTRFS_I(inode), 8722 inode->i_size, 8723 BTRFS_EXTENT_DATA_KEY); 8724 trans->block_rsv = &fs_info->trans_block_rsv; 8725 if (ret != -ENOSPC && ret != -EAGAIN) 8726 break; 8727 8728 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 8729 if (ret) 8730 break; 8731 8732 btrfs_end_transaction(trans); 8733 btrfs_btree_balance_dirty(fs_info); 8734 8735 trans = btrfs_start_transaction(root, 2); 8736 if (IS_ERR(trans)) { 8737 ret = PTR_ERR(trans); 8738 trans = NULL; 8739 break; 8740 } 8741 8742 btrfs_block_rsv_release(fs_info, rsv, -1, NULL); 8743 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 8744 rsv, min_size, false); 8745 BUG_ON(ret); /* shouldn't happen */ 8746 trans->block_rsv = rsv; 8747 } 8748 8749 /* 8750 * We can't call btrfs_truncate_block inside a trans handle as we could 8751 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know 8752 * we've truncated everything except the last little bit, and can do 8753 * btrfs_truncate_block and then update the disk_i_size. 8754 */ 8755 if (ret == NEED_TRUNCATE_BLOCK) { 8756 btrfs_end_transaction(trans); 8757 btrfs_btree_balance_dirty(fs_info); 8758 8759 ret = btrfs_truncate_block(BTRFS_I(inode), inode->i_size, 0, 0); 8760 if (ret) 8761 goto out; 8762 trans = btrfs_start_transaction(root, 1); 8763 if (IS_ERR(trans)) { 8764 ret = PTR_ERR(trans); 8765 goto out; 8766 } 8767 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); 8768 } 8769 8770 if (trans) { 8771 int ret2; 8772 8773 trans->block_rsv = &fs_info->trans_block_rsv; 8774 ret2 = btrfs_update_inode(trans, root, BTRFS_I(inode)); 8775 if (ret2 && !ret) 8776 ret = ret2; 8777 8778 ret2 = btrfs_end_transaction(trans); 8779 if (ret2 && !ret) 8780 ret = ret2; 8781 btrfs_btree_balance_dirty(fs_info); 8782 } 8783 out: 8784 btrfs_free_block_rsv(fs_info, rsv); 8785 8786 return ret; 8787 } 8788 8789 /* 8790 * create a new subvolume directory/inode (helper for the ioctl). 8791 */ 8792 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, 8793 struct btrfs_root *new_root, 8794 struct btrfs_root *parent_root) 8795 { 8796 struct inode *inode; 8797 int err; 8798 u64 index = 0; 8799 u64 ino; 8800 8801 err = btrfs_get_free_objectid(new_root, &ino); 8802 if (err < 0) 8803 return err; 8804 8805 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, ino, ino, 8806 S_IFDIR | (~current_umask() & S_IRWXUGO), 8807 &index); 8808 if (IS_ERR(inode)) 8809 return PTR_ERR(inode); 8810 inode->i_op = &btrfs_dir_inode_operations; 8811 inode->i_fop = &btrfs_dir_file_operations; 8812 8813 set_nlink(inode, 1); 8814 btrfs_i_size_write(BTRFS_I(inode), 0); 8815 unlock_new_inode(inode); 8816 8817 err = btrfs_subvol_inherit_props(trans, new_root, parent_root); 8818 if (err) 8819 btrfs_err(new_root->fs_info, 8820 "error inheriting subvolume %llu properties: %d", 8821 new_root->root_key.objectid, err); 8822 8823 err = btrfs_update_inode(trans, new_root, BTRFS_I(inode)); 8824 8825 iput(inode); 8826 return err; 8827 } 8828 8829 struct inode *btrfs_alloc_inode(struct super_block *sb) 8830 { 8831 struct btrfs_fs_info *fs_info = btrfs_sb(sb); 8832 struct btrfs_inode *ei; 8833 struct inode *inode; 8834 8835 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL); 8836 if (!ei) 8837 return NULL; 8838 8839 ei->root = NULL; 8840 ei->generation = 0; 8841 ei->last_trans = 0; 8842 ei->last_sub_trans = 0; 8843 ei->logged_trans = 0; 8844 ei->delalloc_bytes = 0; 8845 ei->new_delalloc_bytes = 0; 8846 ei->defrag_bytes = 0; 8847 ei->disk_i_size = 0; 8848 ei->flags = 0; 8849 ei->csum_bytes = 0; 8850 ei->index_cnt = (u64)-1; 8851 ei->dir_index = 0; 8852 ei->last_unlink_trans = 0; 8853 ei->last_reflink_trans = 0; 8854 ei->last_log_commit = 0; 8855 8856 spin_lock_init(&ei->lock); 8857 ei->outstanding_extents = 0; 8858 if (sb->s_magic != BTRFS_TEST_MAGIC) 8859 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, 8860 BTRFS_BLOCK_RSV_DELALLOC); 8861 ei->runtime_flags = 0; 8862 ei->prop_compress = BTRFS_COMPRESS_NONE; 8863 ei->defrag_compress = BTRFS_COMPRESS_NONE; 8864 8865 ei->delayed_node = NULL; 8866 8867 ei->i_otime.tv_sec = 0; 8868 ei->i_otime.tv_nsec = 0; 8869 8870 inode = &ei->vfs_inode; 8871 extent_map_tree_init(&ei->extent_tree); 8872 extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO, inode); 8873 extent_io_tree_init(fs_info, &ei->io_failure_tree, 8874 IO_TREE_INODE_IO_FAILURE, inode); 8875 extent_io_tree_init(fs_info, &ei->file_extent_tree, 8876 IO_TREE_INODE_FILE_EXTENT, inode); 8877 ei->io_tree.track_uptodate = true; 8878 ei->io_failure_tree.track_uptodate = true; 8879 atomic_set(&ei->sync_writers, 0); 8880 mutex_init(&ei->log_mutex); 8881 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 8882 INIT_LIST_HEAD(&ei->delalloc_inodes); 8883 INIT_LIST_HEAD(&ei->delayed_iput); 8884 RB_CLEAR_NODE(&ei->rb_node); 8885 8886 return inode; 8887 } 8888 8889 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 8890 void btrfs_test_destroy_inode(struct inode *inode) 8891 { 8892 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); 8893 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8894 } 8895 #endif 8896 8897 void btrfs_free_inode(struct inode *inode) 8898 { 8899 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8900 } 8901 8902 void btrfs_destroy_inode(struct inode *vfs_inode) 8903 { 8904 struct btrfs_ordered_extent *ordered; 8905 struct btrfs_inode *inode = BTRFS_I(vfs_inode); 8906 struct btrfs_root *root = inode->root; 8907 8908 WARN_ON(!hlist_empty(&vfs_inode->i_dentry)); 8909 WARN_ON(vfs_inode->i_data.nrpages); 8910 WARN_ON(inode->block_rsv.reserved); 8911 WARN_ON(inode->block_rsv.size); 8912 WARN_ON(inode->outstanding_extents); 8913 WARN_ON(inode->delalloc_bytes); 8914 WARN_ON(inode->new_delalloc_bytes); 8915 WARN_ON(inode->csum_bytes); 8916 WARN_ON(inode->defrag_bytes); 8917 8918 /* 8919 * This can happen where we create an inode, but somebody else also 8920 * created the same inode and we need to destroy the one we already 8921 * created. 8922 */ 8923 if (!root) 8924 return; 8925 8926 while (1) { 8927 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 8928 if (!ordered) 8929 break; 8930 else { 8931 btrfs_err(root->fs_info, 8932 "found ordered extent %llu %llu on inode cleanup", 8933 ordered->file_offset, ordered->num_bytes); 8934 btrfs_remove_ordered_extent(inode, ordered); 8935 btrfs_put_ordered_extent(ordered); 8936 btrfs_put_ordered_extent(ordered); 8937 } 8938 } 8939 btrfs_qgroup_check_reserved_leak(inode); 8940 inode_tree_del(inode); 8941 btrfs_drop_extent_cache(inode, 0, (u64)-1, 0); 8942 btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1); 8943 btrfs_put_root(inode->root); 8944 } 8945 8946 int btrfs_drop_inode(struct inode *inode) 8947 { 8948 struct btrfs_root *root = BTRFS_I(inode)->root; 8949 8950 if (root == NULL) 8951 return 1; 8952 8953 /* the snap/subvol tree is on deleting */ 8954 if (btrfs_root_refs(&root->root_item) == 0) 8955 return 1; 8956 else 8957 return generic_drop_inode(inode); 8958 } 8959 8960 static void init_once(void *foo) 8961 { 8962 struct btrfs_inode *ei = (struct btrfs_inode *) foo; 8963 8964 inode_init_once(&ei->vfs_inode); 8965 } 8966 8967 void __cold btrfs_destroy_cachep(void) 8968 { 8969 /* 8970 * Make sure all delayed rcu free inodes are flushed before we 8971 * destroy cache. 8972 */ 8973 rcu_barrier(); 8974 kmem_cache_destroy(btrfs_inode_cachep); 8975 kmem_cache_destroy(btrfs_trans_handle_cachep); 8976 kmem_cache_destroy(btrfs_path_cachep); 8977 kmem_cache_destroy(btrfs_free_space_cachep); 8978 kmem_cache_destroy(btrfs_free_space_bitmap_cachep); 8979 } 8980 8981 int __init btrfs_init_cachep(void) 8982 { 8983 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 8984 sizeof(struct btrfs_inode), 0, 8985 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, 8986 init_once); 8987 if (!btrfs_inode_cachep) 8988 goto fail; 8989 8990 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", 8991 sizeof(struct btrfs_trans_handle), 0, 8992 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); 8993 if (!btrfs_trans_handle_cachep) 8994 goto fail; 8995 8996 btrfs_path_cachep = kmem_cache_create("btrfs_path", 8997 sizeof(struct btrfs_path), 0, 8998 SLAB_MEM_SPREAD, NULL); 8999 if (!btrfs_path_cachep) 9000 goto fail; 9001 9002 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space", 9003 sizeof(struct btrfs_free_space), 0, 9004 SLAB_MEM_SPREAD, NULL); 9005 if (!btrfs_free_space_cachep) 9006 goto fail; 9007 9008 btrfs_free_space_bitmap_cachep = kmem_cache_create("btrfs_free_space_bitmap", 9009 PAGE_SIZE, PAGE_SIZE, 9010 SLAB_RED_ZONE, NULL); 9011 if (!btrfs_free_space_bitmap_cachep) 9012 goto fail; 9013 9014 return 0; 9015 fail: 9016 btrfs_destroy_cachep(); 9017 return -ENOMEM; 9018 } 9019 9020 static int btrfs_getattr(const struct path *path, struct kstat *stat, 9021 u32 request_mask, unsigned int flags) 9022 { 9023 u64 delalloc_bytes; 9024 u64 inode_bytes; 9025 struct inode *inode = d_inode(path->dentry); 9026 u32 blocksize = inode->i_sb->s_blocksize; 9027 u32 bi_flags = BTRFS_I(inode)->flags; 9028 9029 stat->result_mask |= STATX_BTIME; 9030 stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec; 9031 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec; 9032 if (bi_flags & BTRFS_INODE_APPEND) 9033 stat->attributes |= STATX_ATTR_APPEND; 9034 if (bi_flags & BTRFS_INODE_COMPRESS) 9035 stat->attributes |= STATX_ATTR_COMPRESSED; 9036 if (bi_flags & BTRFS_INODE_IMMUTABLE) 9037 stat->attributes |= STATX_ATTR_IMMUTABLE; 9038 if (bi_flags & BTRFS_INODE_NODUMP) 9039 stat->attributes |= STATX_ATTR_NODUMP; 9040 9041 stat->attributes_mask |= (STATX_ATTR_APPEND | 9042 STATX_ATTR_COMPRESSED | 9043 STATX_ATTR_IMMUTABLE | 9044 STATX_ATTR_NODUMP); 9045 9046 generic_fillattr(inode, stat); 9047 stat->dev = BTRFS_I(inode)->root->anon_dev; 9048 9049 spin_lock(&BTRFS_I(inode)->lock); 9050 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; 9051 inode_bytes = inode_get_bytes(inode); 9052 spin_unlock(&BTRFS_I(inode)->lock); 9053 stat->blocks = (ALIGN(inode_bytes, blocksize) + 9054 ALIGN(delalloc_bytes, blocksize)) >> 9; 9055 return 0; 9056 } 9057 9058 static int btrfs_rename_exchange(struct inode *old_dir, 9059 struct dentry *old_dentry, 9060 struct inode *new_dir, 9061 struct dentry *new_dentry) 9062 { 9063 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9064 struct btrfs_trans_handle *trans; 9065 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9066 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9067 struct inode *new_inode = new_dentry->d_inode; 9068 struct inode *old_inode = old_dentry->d_inode; 9069 struct timespec64 ctime = current_time(old_inode); 9070 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9071 u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); 9072 u64 old_idx = 0; 9073 u64 new_idx = 0; 9074 int ret; 9075 int ret2; 9076 bool root_log_pinned = false; 9077 bool dest_log_pinned = false; 9078 9079 /* we only allow rename subvolume link between subvolumes */ 9080 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9081 return -EXDEV; 9082 9083 /* close the race window with snapshot create/destroy ioctl */ 9084 if (old_ino == BTRFS_FIRST_FREE_OBJECTID || 9085 new_ino == BTRFS_FIRST_FREE_OBJECTID) 9086 down_read(&fs_info->subvol_sem); 9087 9088 /* 9089 * We want to reserve the absolute worst case amount of items. So if 9090 * both inodes are subvols and we need to unlink them then that would 9091 * require 4 item modifications, but if they are both normal inodes it 9092 * would require 5 item modifications, so we'll assume their normal 9093 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items 9094 * should cover the worst case number of items we'll modify. 9095 */ 9096 trans = btrfs_start_transaction(root, 12); 9097 if (IS_ERR(trans)) { 9098 ret = PTR_ERR(trans); 9099 goto out_notrans; 9100 } 9101 9102 if (dest != root) 9103 btrfs_record_root_in_trans(trans, dest); 9104 9105 /* 9106 * We need to find a free sequence number both in the source and 9107 * in the destination directory for the exchange. 9108 */ 9109 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); 9110 if (ret) 9111 goto out_fail; 9112 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); 9113 if (ret) 9114 goto out_fail; 9115 9116 BTRFS_I(old_inode)->dir_index = 0ULL; 9117 BTRFS_I(new_inode)->dir_index = 0ULL; 9118 9119 /* Reference for the source. */ 9120 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9121 /* force full log commit if subvolume involved. */ 9122 btrfs_set_log_full_commit(trans); 9123 } else { 9124 btrfs_pin_log_trans(root); 9125 root_log_pinned = true; 9126 ret = btrfs_insert_inode_ref(trans, dest, 9127 new_dentry->d_name.name, 9128 new_dentry->d_name.len, 9129 old_ino, 9130 btrfs_ino(BTRFS_I(new_dir)), 9131 old_idx); 9132 if (ret) 9133 goto out_fail; 9134 } 9135 9136 /* And now for the dest. */ 9137 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9138 /* force full log commit if subvolume involved. */ 9139 btrfs_set_log_full_commit(trans); 9140 } else { 9141 btrfs_pin_log_trans(dest); 9142 dest_log_pinned = true; 9143 ret = btrfs_insert_inode_ref(trans, root, 9144 old_dentry->d_name.name, 9145 old_dentry->d_name.len, 9146 new_ino, 9147 btrfs_ino(BTRFS_I(old_dir)), 9148 new_idx); 9149 if (ret) 9150 goto out_fail; 9151 } 9152 9153 /* Update inode version and ctime/mtime. */ 9154 inode_inc_iversion(old_dir); 9155 inode_inc_iversion(new_dir); 9156 inode_inc_iversion(old_inode); 9157 inode_inc_iversion(new_inode); 9158 old_dir->i_ctime = old_dir->i_mtime = ctime; 9159 new_dir->i_ctime = new_dir->i_mtime = ctime; 9160 old_inode->i_ctime = ctime; 9161 new_inode->i_ctime = ctime; 9162 9163 if (old_dentry->d_parent != new_dentry->d_parent) { 9164 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9165 BTRFS_I(old_inode), 1); 9166 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), 9167 BTRFS_I(new_inode), 1); 9168 } 9169 9170 /* src is a subvolume */ 9171 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 9172 ret = btrfs_unlink_subvol(trans, old_dir, old_dentry); 9173 } else { /* src is an inode */ 9174 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9175 BTRFS_I(old_dentry->d_inode), 9176 old_dentry->d_name.name, 9177 old_dentry->d_name.len); 9178 if (!ret) 9179 ret = btrfs_update_inode(trans, root, BTRFS_I(old_inode)); 9180 } 9181 if (ret) { 9182 btrfs_abort_transaction(trans, ret); 9183 goto out_fail; 9184 } 9185 9186 /* dest is a subvolume */ 9187 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 9188 ret = btrfs_unlink_subvol(trans, new_dir, new_dentry); 9189 } else { /* dest is an inode */ 9190 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9191 BTRFS_I(new_dentry->d_inode), 9192 new_dentry->d_name.name, 9193 new_dentry->d_name.len); 9194 if (!ret) 9195 ret = btrfs_update_inode(trans, dest, BTRFS_I(new_inode)); 9196 } 9197 if (ret) { 9198 btrfs_abort_transaction(trans, ret); 9199 goto out_fail; 9200 } 9201 9202 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9203 new_dentry->d_name.name, 9204 new_dentry->d_name.len, 0, old_idx); 9205 if (ret) { 9206 btrfs_abort_transaction(trans, ret); 9207 goto out_fail; 9208 } 9209 9210 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), 9211 old_dentry->d_name.name, 9212 old_dentry->d_name.len, 0, new_idx); 9213 if (ret) { 9214 btrfs_abort_transaction(trans, ret); 9215 goto out_fail; 9216 } 9217 9218 if (old_inode->i_nlink == 1) 9219 BTRFS_I(old_inode)->dir_index = old_idx; 9220 if (new_inode->i_nlink == 1) 9221 BTRFS_I(new_inode)->dir_index = new_idx; 9222 9223 if (root_log_pinned) { 9224 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9225 new_dentry->d_parent); 9226 btrfs_end_log_trans(root); 9227 root_log_pinned = false; 9228 } 9229 if (dest_log_pinned) { 9230 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir), 9231 old_dentry->d_parent); 9232 btrfs_end_log_trans(dest); 9233 dest_log_pinned = false; 9234 } 9235 out_fail: 9236 /* 9237 * If we have pinned a log and an error happened, we unpin tasks 9238 * trying to sync the log and force them to fallback to a transaction 9239 * commit if the log currently contains any of the inodes involved in 9240 * this rename operation (to ensure we do not persist a log with an 9241 * inconsistent state for any of these inodes or leading to any 9242 * inconsistencies when replayed). If the transaction was aborted, the 9243 * abortion reason is propagated to userspace when attempting to commit 9244 * the transaction. If the log does not contain any of these inodes, we 9245 * allow the tasks to sync it. 9246 */ 9247 if (ret && (root_log_pinned || dest_log_pinned)) { 9248 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9249 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9250 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9251 (new_inode && 9252 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9253 btrfs_set_log_full_commit(trans); 9254 9255 if (root_log_pinned) { 9256 btrfs_end_log_trans(root); 9257 root_log_pinned = false; 9258 } 9259 if (dest_log_pinned) { 9260 btrfs_end_log_trans(dest); 9261 dest_log_pinned = false; 9262 } 9263 } 9264 ret2 = btrfs_end_transaction(trans); 9265 ret = ret ? ret : ret2; 9266 out_notrans: 9267 if (new_ino == BTRFS_FIRST_FREE_OBJECTID || 9268 old_ino == BTRFS_FIRST_FREE_OBJECTID) 9269 up_read(&fs_info->subvol_sem); 9270 9271 return ret; 9272 } 9273 9274 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans, 9275 struct btrfs_root *root, 9276 struct inode *dir, 9277 struct dentry *dentry) 9278 { 9279 int ret; 9280 struct inode *inode; 9281 u64 objectid; 9282 u64 index; 9283 9284 ret = btrfs_get_free_objectid(root, &objectid); 9285 if (ret) 9286 return ret; 9287 9288 inode = btrfs_new_inode(trans, root, dir, 9289 dentry->d_name.name, 9290 dentry->d_name.len, 9291 btrfs_ino(BTRFS_I(dir)), 9292 objectid, 9293 S_IFCHR | WHITEOUT_MODE, 9294 &index); 9295 9296 if (IS_ERR(inode)) { 9297 ret = PTR_ERR(inode); 9298 return ret; 9299 } 9300 9301 inode->i_op = &btrfs_special_inode_operations; 9302 init_special_inode(inode, inode->i_mode, 9303 WHITEOUT_DEV); 9304 9305 ret = btrfs_init_inode_security(trans, inode, dir, 9306 &dentry->d_name); 9307 if (ret) 9308 goto out; 9309 9310 ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 9311 BTRFS_I(inode), 0, index); 9312 if (ret) 9313 goto out; 9314 9315 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 9316 out: 9317 unlock_new_inode(inode); 9318 if (ret) 9319 inode_dec_link_count(inode); 9320 iput(inode); 9321 9322 return ret; 9323 } 9324 9325 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry, 9326 struct inode *new_dir, struct dentry *new_dentry, 9327 unsigned int flags) 9328 { 9329 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 9330 struct btrfs_trans_handle *trans; 9331 unsigned int trans_num_items; 9332 struct btrfs_root *root = BTRFS_I(old_dir)->root; 9333 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 9334 struct inode *new_inode = d_inode(new_dentry); 9335 struct inode *old_inode = d_inode(old_dentry); 9336 u64 index = 0; 9337 int ret; 9338 int ret2; 9339 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 9340 bool log_pinned = false; 9341 9342 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 9343 return -EPERM; 9344 9345 /* we only allow rename subvolume link between subvolumes */ 9346 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 9347 return -EXDEV; 9348 9349 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 9350 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) 9351 return -ENOTEMPTY; 9352 9353 if (S_ISDIR(old_inode->i_mode) && new_inode && 9354 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 9355 return -ENOTEMPTY; 9356 9357 9358 /* check for collisions, even if the name isn't there */ 9359 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, 9360 new_dentry->d_name.name, 9361 new_dentry->d_name.len); 9362 9363 if (ret) { 9364 if (ret == -EEXIST) { 9365 /* we shouldn't get 9366 * eexist without a new_inode */ 9367 if (WARN_ON(!new_inode)) { 9368 return ret; 9369 } 9370 } else { 9371 /* maybe -EOVERFLOW */ 9372 return ret; 9373 } 9374 } 9375 ret = 0; 9376 9377 /* 9378 * we're using rename to replace one file with another. Start IO on it 9379 * now so we don't add too much work to the end of the transaction 9380 */ 9381 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 9382 filemap_flush(old_inode->i_mapping); 9383 9384 /* close the racy window with snapshot create/destroy ioctl */ 9385 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9386 down_read(&fs_info->subvol_sem); 9387 /* 9388 * We want to reserve the absolute worst case amount of items. So if 9389 * both inodes are subvols and we need to unlink them then that would 9390 * require 4 item modifications, but if they are both normal inodes it 9391 * would require 5 item modifications, so we'll assume they are normal 9392 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items 9393 * should cover the worst case number of items we'll modify. 9394 * If our rename has the whiteout flag, we need more 5 units for the 9395 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item 9396 * when selinux is enabled). 9397 */ 9398 trans_num_items = 11; 9399 if (flags & RENAME_WHITEOUT) 9400 trans_num_items += 5; 9401 trans = btrfs_start_transaction(root, trans_num_items); 9402 if (IS_ERR(trans)) { 9403 ret = PTR_ERR(trans); 9404 goto out_notrans; 9405 } 9406 9407 if (dest != root) 9408 btrfs_record_root_in_trans(trans, dest); 9409 9410 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); 9411 if (ret) 9412 goto out_fail; 9413 9414 BTRFS_I(old_inode)->dir_index = 0ULL; 9415 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9416 /* force full log commit if subvolume involved. */ 9417 btrfs_set_log_full_commit(trans); 9418 } else { 9419 btrfs_pin_log_trans(root); 9420 log_pinned = true; 9421 ret = btrfs_insert_inode_ref(trans, dest, 9422 new_dentry->d_name.name, 9423 new_dentry->d_name.len, 9424 old_ino, 9425 btrfs_ino(BTRFS_I(new_dir)), index); 9426 if (ret) 9427 goto out_fail; 9428 } 9429 9430 inode_inc_iversion(old_dir); 9431 inode_inc_iversion(new_dir); 9432 inode_inc_iversion(old_inode); 9433 old_dir->i_ctime = old_dir->i_mtime = 9434 new_dir->i_ctime = new_dir->i_mtime = 9435 old_inode->i_ctime = current_time(old_dir); 9436 9437 if (old_dentry->d_parent != new_dentry->d_parent) 9438 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9439 BTRFS_I(old_inode), 1); 9440 9441 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9442 ret = btrfs_unlink_subvol(trans, old_dir, old_dentry); 9443 } else { 9444 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir), 9445 BTRFS_I(d_inode(old_dentry)), 9446 old_dentry->d_name.name, 9447 old_dentry->d_name.len); 9448 if (!ret) 9449 ret = btrfs_update_inode(trans, root, BTRFS_I(old_inode)); 9450 } 9451 if (ret) { 9452 btrfs_abort_transaction(trans, ret); 9453 goto out_fail; 9454 } 9455 9456 if (new_inode) { 9457 inode_inc_iversion(new_inode); 9458 new_inode->i_ctime = current_time(new_inode); 9459 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == 9460 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 9461 ret = btrfs_unlink_subvol(trans, new_dir, new_dentry); 9462 BUG_ON(new_inode->i_nlink == 0); 9463 } else { 9464 ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir), 9465 BTRFS_I(d_inode(new_dentry)), 9466 new_dentry->d_name.name, 9467 new_dentry->d_name.len); 9468 } 9469 if (!ret && new_inode->i_nlink == 0) 9470 ret = btrfs_orphan_add(trans, 9471 BTRFS_I(d_inode(new_dentry))); 9472 if (ret) { 9473 btrfs_abort_transaction(trans, ret); 9474 goto out_fail; 9475 } 9476 } 9477 9478 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9479 new_dentry->d_name.name, 9480 new_dentry->d_name.len, 0, index); 9481 if (ret) { 9482 btrfs_abort_transaction(trans, ret); 9483 goto out_fail; 9484 } 9485 9486 if (old_inode->i_nlink == 1) 9487 BTRFS_I(old_inode)->dir_index = index; 9488 9489 if (log_pinned) { 9490 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir), 9491 new_dentry->d_parent); 9492 btrfs_end_log_trans(root); 9493 log_pinned = false; 9494 } 9495 9496 if (flags & RENAME_WHITEOUT) { 9497 ret = btrfs_whiteout_for_rename(trans, root, old_dir, 9498 old_dentry); 9499 9500 if (ret) { 9501 btrfs_abort_transaction(trans, ret); 9502 goto out_fail; 9503 } 9504 } 9505 out_fail: 9506 /* 9507 * If we have pinned the log and an error happened, we unpin tasks 9508 * trying to sync the log and force them to fallback to a transaction 9509 * commit if the log currently contains any of the inodes involved in 9510 * this rename operation (to ensure we do not persist a log with an 9511 * inconsistent state for any of these inodes or leading to any 9512 * inconsistencies when replayed). If the transaction was aborted, the 9513 * abortion reason is propagated to userspace when attempting to commit 9514 * the transaction. If the log does not contain any of these inodes, we 9515 * allow the tasks to sync it. 9516 */ 9517 if (ret && log_pinned) { 9518 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) || 9519 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) || 9520 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) || 9521 (new_inode && 9522 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation))) 9523 btrfs_set_log_full_commit(trans); 9524 9525 btrfs_end_log_trans(root); 9526 log_pinned = false; 9527 } 9528 ret2 = btrfs_end_transaction(trans); 9529 ret = ret ? ret : ret2; 9530 out_notrans: 9531 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9532 up_read(&fs_info->subvol_sem); 9533 9534 return ret; 9535 } 9536 9537 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, 9538 struct inode *new_dir, struct dentry *new_dentry, 9539 unsigned int flags) 9540 { 9541 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 9542 return -EINVAL; 9543 9544 if (flags & RENAME_EXCHANGE) 9545 return btrfs_rename_exchange(old_dir, old_dentry, new_dir, 9546 new_dentry); 9547 9548 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags); 9549 } 9550 9551 struct btrfs_delalloc_work { 9552 struct inode *inode; 9553 struct completion completion; 9554 struct list_head list; 9555 struct btrfs_work work; 9556 }; 9557 9558 static void btrfs_run_delalloc_work(struct btrfs_work *work) 9559 { 9560 struct btrfs_delalloc_work *delalloc_work; 9561 struct inode *inode; 9562 9563 delalloc_work = container_of(work, struct btrfs_delalloc_work, 9564 work); 9565 inode = delalloc_work->inode; 9566 filemap_flush(inode->i_mapping); 9567 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9568 &BTRFS_I(inode)->runtime_flags)) 9569 filemap_flush(inode->i_mapping); 9570 9571 iput(inode); 9572 complete(&delalloc_work->completion); 9573 } 9574 9575 static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode) 9576 { 9577 struct btrfs_delalloc_work *work; 9578 9579 work = kmalloc(sizeof(*work), GFP_NOFS); 9580 if (!work) 9581 return NULL; 9582 9583 init_completion(&work->completion); 9584 INIT_LIST_HEAD(&work->list); 9585 work->inode = inode; 9586 btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL, NULL); 9587 9588 return work; 9589 } 9590 9591 /* 9592 * some fairly slow code that needs optimization. This walks the list 9593 * of all the inodes with pending delalloc and forces them to disk. 9594 */ 9595 static int start_delalloc_inodes(struct btrfs_root *root, 9596 struct writeback_control *wbc, bool snapshot, 9597 bool in_reclaim_context) 9598 { 9599 struct btrfs_inode *binode; 9600 struct inode *inode; 9601 struct btrfs_delalloc_work *work, *next; 9602 struct list_head works; 9603 struct list_head splice; 9604 int ret = 0; 9605 bool full_flush = wbc->nr_to_write == LONG_MAX; 9606 9607 INIT_LIST_HEAD(&works); 9608 INIT_LIST_HEAD(&splice); 9609 9610 mutex_lock(&root->delalloc_mutex); 9611 spin_lock(&root->delalloc_lock); 9612 list_splice_init(&root->delalloc_inodes, &splice); 9613 while (!list_empty(&splice)) { 9614 binode = list_entry(splice.next, struct btrfs_inode, 9615 delalloc_inodes); 9616 9617 list_move_tail(&binode->delalloc_inodes, 9618 &root->delalloc_inodes); 9619 9620 if (in_reclaim_context && 9621 test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &binode->runtime_flags)) 9622 continue; 9623 9624 inode = igrab(&binode->vfs_inode); 9625 if (!inode) { 9626 cond_resched_lock(&root->delalloc_lock); 9627 continue; 9628 } 9629 spin_unlock(&root->delalloc_lock); 9630 9631 if (snapshot) 9632 set_bit(BTRFS_INODE_SNAPSHOT_FLUSH, 9633 &binode->runtime_flags); 9634 if (full_flush) { 9635 work = btrfs_alloc_delalloc_work(inode); 9636 if (!work) { 9637 iput(inode); 9638 ret = -ENOMEM; 9639 goto out; 9640 } 9641 list_add_tail(&work->list, &works); 9642 btrfs_queue_work(root->fs_info->flush_workers, 9643 &work->work); 9644 } else { 9645 ret = sync_inode(inode, wbc); 9646 if (!ret && 9647 test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9648 &BTRFS_I(inode)->runtime_flags)) 9649 ret = sync_inode(inode, wbc); 9650 btrfs_add_delayed_iput(inode); 9651 if (ret || wbc->nr_to_write <= 0) 9652 goto out; 9653 } 9654 cond_resched(); 9655 spin_lock(&root->delalloc_lock); 9656 } 9657 spin_unlock(&root->delalloc_lock); 9658 9659 out: 9660 list_for_each_entry_safe(work, next, &works, list) { 9661 list_del_init(&work->list); 9662 wait_for_completion(&work->completion); 9663 kfree(work); 9664 } 9665 9666 if (!list_empty(&splice)) { 9667 spin_lock(&root->delalloc_lock); 9668 list_splice_tail(&splice, &root->delalloc_inodes); 9669 spin_unlock(&root->delalloc_lock); 9670 } 9671 mutex_unlock(&root->delalloc_mutex); 9672 return ret; 9673 } 9674 9675 int btrfs_start_delalloc_snapshot(struct btrfs_root *root) 9676 { 9677 struct writeback_control wbc = { 9678 .nr_to_write = LONG_MAX, 9679 .sync_mode = WB_SYNC_NONE, 9680 .range_start = 0, 9681 .range_end = LLONG_MAX, 9682 }; 9683 struct btrfs_fs_info *fs_info = root->fs_info; 9684 9685 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 9686 return -EROFS; 9687 9688 return start_delalloc_inodes(root, &wbc, true, false); 9689 } 9690 9691 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, 9692 bool in_reclaim_context) 9693 { 9694 struct writeback_control wbc = { 9695 .nr_to_write = nr, 9696 .sync_mode = WB_SYNC_NONE, 9697 .range_start = 0, 9698 .range_end = LLONG_MAX, 9699 }; 9700 struct btrfs_root *root; 9701 struct list_head splice; 9702 int ret; 9703 9704 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 9705 return -EROFS; 9706 9707 INIT_LIST_HEAD(&splice); 9708 9709 mutex_lock(&fs_info->delalloc_root_mutex); 9710 spin_lock(&fs_info->delalloc_root_lock); 9711 list_splice_init(&fs_info->delalloc_roots, &splice); 9712 while (!list_empty(&splice)) { 9713 /* 9714 * Reset nr_to_write here so we know that we're doing a full 9715 * flush. 9716 */ 9717 if (nr == LONG_MAX) 9718 wbc.nr_to_write = LONG_MAX; 9719 9720 root = list_first_entry(&splice, struct btrfs_root, 9721 delalloc_root); 9722 root = btrfs_grab_root(root); 9723 BUG_ON(!root); 9724 list_move_tail(&root->delalloc_root, 9725 &fs_info->delalloc_roots); 9726 spin_unlock(&fs_info->delalloc_root_lock); 9727 9728 ret = start_delalloc_inodes(root, &wbc, false, in_reclaim_context); 9729 btrfs_put_root(root); 9730 if (ret < 0 || wbc.nr_to_write <= 0) 9731 goto out; 9732 spin_lock(&fs_info->delalloc_root_lock); 9733 } 9734 spin_unlock(&fs_info->delalloc_root_lock); 9735 9736 ret = 0; 9737 out: 9738 if (!list_empty(&splice)) { 9739 spin_lock(&fs_info->delalloc_root_lock); 9740 list_splice_tail(&splice, &fs_info->delalloc_roots); 9741 spin_unlock(&fs_info->delalloc_root_lock); 9742 } 9743 mutex_unlock(&fs_info->delalloc_root_mutex); 9744 return ret; 9745 } 9746 9747 static int btrfs_symlink(struct inode *dir, struct dentry *dentry, 9748 const char *symname) 9749 { 9750 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 9751 struct btrfs_trans_handle *trans; 9752 struct btrfs_root *root = BTRFS_I(dir)->root; 9753 struct btrfs_path *path; 9754 struct btrfs_key key; 9755 struct inode *inode = NULL; 9756 int err; 9757 u64 objectid; 9758 u64 index = 0; 9759 int name_len; 9760 int datasize; 9761 unsigned long ptr; 9762 struct btrfs_file_extent_item *ei; 9763 struct extent_buffer *leaf; 9764 9765 name_len = strlen(symname); 9766 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 9767 return -ENAMETOOLONG; 9768 9769 /* 9770 * 2 items for inode item and ref 9771 * 2 items for dir items 9772 * 1 item for updating parent inode item 9773 * 1 item for the inline extent item 9774 * 1 item for xattr if selinux is on 9775 */ 9776 trans = btrfs_start_transaction(root, 7); 9777 if (IS_ERR(trans)) 9778 return PTR_ERR(trans); 9779 9780 err = btrfs_get_free_objectid(root, &objectid); 9781 if (err) 9782 goto out_unlock; 9783 9784 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 9785 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), 9786 objectid, S_IFLNK|S_IRWXUGO, &index); 9787 if (IS_ERR(inode)) { 9788 err = PTR_ERR(inode); 9789 inode = NULL; 9790 goto out_unlock; 9791 } 9792 9793 /* 9794 * If the active LSM wants to access the inode during 9795 * d_instantiate it needs these. Smack checks to see 9796 * if the filesystem supports xattrs by looking at the 9797 * ops vector. 9798 */ 9799 inode->i_fop = &btrfs_file_operations; 9800 inode->i_op = &btrfs_file_inode_operations; 9801 inode->i_mapping->a_ops = &btrfs_aops; 9802 9803 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 9804 if (err) 9805 goto out_unlock; 9806 9807 path = btrfs_alloc_path(); 9808 if (!path) { 9809 err = -ENOMEM; 9810 goto out_unlock; 9811 } 9812 key.objectid = btrfs_ino(BTRFS_I(inode)); 9813 key.offset = 0; 9814 key.type = BTRFS_EXTENT_DATA_KEY; 9815 datasize = btrfs_file_extent_calc_inline_size(name_len); 9816 err = btrfs_insert_empty_item(trans, root, path, &key, 9817 datasize); 9818 if (err) { 9819 btrfs_free_path(path); 9820 goto out_unlock; 9821 } 9822 leaf = path->nodes[0]; 9823 ei = btrfs_item_ptr(leaf, path->slots[0], 9824 struct btrfs_file_extent_item); 9825 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 9826 btrfs_set_file_extent_type(leaf, ei, 9827 BTRFS_FILE_EXTENT_INLINE); 9828 btrfs_set_file_extent_encryption(leaf, ei, 0); 9829 btrfs_set_file_extent_compression(leaf, ei, 0); 9830 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 9831 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 9832 9833 ptr = btrfs_file_extent_inline_start(ei); 9834 write_extent_buffer(leaf, symname, ptr, name_len); 9835 btrfs_mark_buffer_dirty(leaf); 9836 btrfs_free_path(path); 9837 9838 inode->i_op = &btrfs_symlink_inode_operations; 9839 inode_nohighmem(inode); 9840 inode_set_bytes(inode, name_len); 9841 btrfs_i_size_write(BTRFS_I(inode), name_len); 9842 err = btrfs_update_inode(trans, root, BTRFS_I(inode)); 9843 /* 9844 * Last step, add directory indexes for our symlink inode. This is the 9845 * last step to avoid extra cleanup of these indexes if an error happens 9846 * elsewhere above. 9847 */ 9848 if (!err) 9849 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, 9850 BTRFS_I(inode), 0, index); 9851 if (err) 9852 goto out_unlock; 9853 9854 d_instantiate_new(dentry, inode); 9855 9856 out_unlock: 9857 btrfs_end_transaction(trans); 9858 if (err && inode) { 9859 inode_dec_link_count(inode); 9860 discard_new_inode(inode); 9861 } 9862 btrfs_btree_balance_dirty(fs_info); 9863 return err; 9864 } 9865 9866 static struct btrfs_trans_handle *insert_prealloc_file_extent( 9867 struct btrfs_trans_handle *trans_in, 9868 struct btrfs_inode *inode, 9869 struct btrfs_key *ins, 9870 u64 file_offset) 9871 { 9872 struct btrfs_file_extent_item stack_fi; 9873 struct btrfs_replace_extent_info extent_info; 9874 struct btrfs_trans_handle *trans = trans_in; 9875 struct btrfs_path *path; 9876 u64 start = ins->objectid; 9877 u64 len = ins->offset; 9878 int ret; 9879 9880 memset(&stack_fi, 0, sizeof(stack_fi)); 9881 9882 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC); 9883 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start); 9884 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len); 9885 btrfs_set_stack_file_extent_num_bytes(&stack_fi, len); 9886 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len); 9887 btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE); 9888 /* Encryption and other encoding is reserved and all 0 */ 9889 9890 ret = btrfs_qgroup_release_data(inode, file_offset, len); 9891 if (ret < 0) 9892 return ERR_PTR(ret); 9893 9894 if (trans) { 9895 ret = insert_reserved_file_extent(trans, inode, 9896 file_offset, &stack_fi, 9897 true, ret); 9898 if (ret) 9899 return ERR_PTR(ret); 9900 return trans; 9901 } 9902 9903 extent_info.disk_offset = start; 9904 extent_info.disk_len = len; 9905 extent_info.data_offset = 0; 9906 extent_info.data_len = len; 9907 extent_info.file_offset = file_offset; 9908 extent_info.extent_buf = (char *)&stack_fi; 9909 extent_info.is_new_extent = true; 9910 extent_info.qgroup_reserved = ret; 9911 extent_info.insertions = 0; 9912 9913 path = btrfs_alloc_path(); 9914 if (!path) 9915 return ERR_PTR(-ENOMEM); 9916 9917 ret = btrfs_replace_file_extents(&inode->vfs_inode, path, file_offset, 9918 file_offset + len - 1, &extent_info, 9919 &trans); 9920 btrfs_free_path(path); 9921 if (ret) 9922 return ERR_PTR(ret); 9923 9924 return trans; 9925 } 9926 9927 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 9928 u64 start, u64 num_bytes, u64 min_size, 9929 loff_t actual_len, u64 *alloc_hint, 9930 struct btrfs_trans_handle *trans) 9931 { 9932 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9933 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 9934 struct extent_map *em; 9935 struct btrfs_root *root = BTRFS_I(inode)->root; 9936 struct btrfs_key ins; 9937 u64 cur_offset = start; 9938 u64 clear_offset = start; 9939 u64 i_size; 9940 u64 cur_bytes; 9941 u64 last_alloc = (u64)-1; 9942 int ret = 0; 9943 bool own_trans = true; 9944 u64 end = start + num_bytes - 1; 9945 9946 if (trans) 9947 own_trans = false; 9948 while (num_bytes > 0) { 9949 cur_bytes = min_t(u64, num_bytes, SZ_256M); 9950 cur_bytes = max(cur_bytes, min_size); 9951 /* 9952 * If we are severely fragmented we could end up with really 9953 * small allocations, so if the allocator is returning small 9954 * chunks lets make its job easier by only searching for those 9955 * sized chunks. 9956 */ 9957 cur_bytes = min(cur_bytes, last_alloc); 9958 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 9959 min_size, 0, *alloc_hint, &ins, 1, 0); 9960 if (ret) 9961 break; 9962 9963 /* 9964 * We've reserved this space, and thus converted it from 9965 * ->bytes_may_use to ->bytes_reserved. Any error that happens 9966 * from here on out we will only need to clear our reservation 9967 * for the remaining unreserved area, so advance our 9968 * clear_offset by our extent size. 9969 */ 9970 clear_offset += ins.offset; 9971 9972 last_alloc = ins.offset; 9973 trans = insert_prealloc_file_extent(trans, BTRFS_I(inode), 9974 &ins, cur_offset); 9975 /* 9976 * Now that we inserted the prealloc extent we can finally 9977 * decrement the number of reservations in the block group. 9978 * If we did it before, we could race with relocation and have 9979 * relocation miss the reserved extent, making it fail later. 9980 */ 9981 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 9982 if (IS_ERR(trans)) { 9983 ret = PTR_ERR(trans); 9984 btrfs_free_reserved_extent(fs_info, ins.objectid, 9985 ins.offset, 0); 9986 break; 9987 } 9988 9989 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 9990 cur_offset + ins.offset -1, 0); 9991 9992 em = alloc_extent_map(); 9993 if (!em) { 9994 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 9995 &BTRFS_I(inode)->runtime_flags); 9996 goto next; 9997 } 9998 9999 em->start = cur_offset; 10000 em->orig_start = cur_offset; 10001 em->len = ins.offset; 10002 em->block_start = ins.objectid; 10003 em->block_len = ins.offset; 10004 em->orig_block_len = ins.offset; 10005 em->ram_bytes = ins.offset; 10006 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 10007 em->generation = trans->transid; 10008 10009 while (1) { 10010 write_lock(&em_tree->lock); 10011 ret = add_extent_mapping(em_tree, em, 1); 10012 write_unlock(&em_tree->lock); 10013 if (ret != -EEXIST) 10014 break; 10015 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, 10016 cur_offset + ins.offset - 1, 10017 0); 10018 } 10019 free_extent_map(em); 10020 next: 10021 num_bytes -= ins.offset; 10022 cur_offset += ins.offset; 10023 *alloc_hint = ins.objectid + ins.offset; 10024 10025 inode_inc_iversion(inode); 10026 inode->i_ctime = current_time(inode); 10027 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 10028 if (!(mode & FALLOC_FL_KEEP_SIZE) && 10029 (actual_len > inode->i_size) && 10030 (cur_offset > inode->i_size)) { 10031 if (cur_offset > actual_len) 10032 i_size = actual_len; 10033 else 10034 i_size = cur_offset; 10035 i_size_write(inode, i_size); 10036 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); 10037 } 10038 10039 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 10040 10041 if (ret) { 10042 btrfs_abort_transaction(trans, ret); 10043 if (own_trans) 10044 btrfs_end_transaction(trans); 10045 break; 10046 } 10047 10048 if (own_trans) { 10049 btrfs_end_transaction(trans); 10050 trans = NULL; 10051 } 10052 } 10053 if (clear_offset < end) 10054 btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset, 10055 end - clear_offset + 1); 10056 return ret; 10057 } 10058 10059 int btrfs_prealloc_file_range(struct inode *inode, int mode, 10060 u64 start, u64 num_bytes, u64 min_size, 10061 loff_t actual_len, u64 *alloc_hint) 10062 { 10063 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10064 min_size, actual_len, alloc_hint, 10065 NULL); 10066 } 10067 10068 int btrfs_prealloc_file_range_trans(struct inode *inode, 10069 struct btrfs_trans_handle *trans, int mode, 10070 u64 start, u64 num_bytes, u64 min_size, 10071 loff_t actual_len, u64 *alloc_hint) 10072 { 10073 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 10074 min_size, actual_len, alloc_hint, trans); 10075 } 10076 10077 static int btrfs_set_page_dirty(struct page *page) 10078 { 10079 return __set_page_dirty_nobuffers(page); 10080 } 10081 10082 static int btrfs_permission(struct inode *inode, int mask) 10083 { 10084 struct btrfs_root *root = BTRFS_I(inode)->root; 10085 umode_t mode = inode->i_mode; 10086 10087 if (mask & MAY_WRITE && 10088 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 10089 if (btrfs_root_readonly(root)) 10090 return -EROFS; 10091 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 10092 return -EACCES; 10093 } 10094 return generic_permission(inode, mask); 10095 } 10096 10097 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 10098 { 10099 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 10100 struct btrfs_trans_handle *trans; 10101 struct btrfs_root *root = BTRFS_I(dir)->root; 10102 struct inode *inode = NULL; 10103 u64 objectid; 10104 u64 index; 10105 int ret = 0; 10106 10107 /* 10108 * 5 units required for adding orphan entry 10109 */ 10110 trans = btrfs_start_transaction(root, 5); 10111 if (IS_ERR(trans)) 10112 return PTR_ERR(trans); 10113 10114 ret = btrfs_get_free_objectid(root, &objectid); 10115 if (ret) 10116 goto out; 10117 10118 inode = btrfs_new_inode(trans, root, dir, NULL, 0, 10119 btrfs_ino(BTRFS_I(dir)), objectid, mode, &index); 10120 if (IS_ERR(inode)) { 10121 ret = PTR_ERR(inode); 10122 inode = NULL; 10123 goto out; 10124 } 10125 10126 inode->i_fop = &btrfs_file_operations; 10127 inode->i_op = &btrfs_file_inode_operations; 10128 10129 inode->i_mapping->a_ops = &btrfs_aops; 10130 10131 ret = btrfs_init_inode_security(trans, inode, dir, NULL); 10132 if (ret) 10133 goto out; 10134 10135 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 10136 if (ret) 10137 goto out; 10138 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 10139 if (ret) 10140 goto out; 10141 10142 /* 10143 * We set number of links to 0 in btrfs_new_inode(), and here we set 10144 * it to 1 because d_tmpfile() will issue a warning if the count is 0, 10145 * through: 10146 * 10147 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 10148 */ 10149 set_nlink(inode, 1); 10150 d_tmpfile(dentry, inode); 10151 unlock_new_inode(inode); 10152 mark_inode_dirty(inode); 10153 out: 10154 btrfs_end_transaction(trans); 10155 if (ret && inode) 10156 discard_new_inode(inode); 10157 btrfs_btree_balance_dirty(fs_info); 10158 return ret; 10159 } 10160 10161 void btrfs_set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end) 10162 { 10163 struct inode *inode = tree->private_data; 10164 unsigned long index = start >> PAGE_SHIFT; 10165 unsigned long end_index = end >> PAGE_SHIFT; 10166 struct page *page; 10167 10168 while (index <= end_index) { 10169 page = find_get_page(inode->i_mapping, index); 10170 ASSERT(page); /* Pages should be in the extent_io_tree */ 10171 set_page_writeback(page); 10172 put_page(page); 10173 index++; 10174 } 10175 } 10176 10177 #ifdef CONFIG_SWAP 10178 /* 10179 * Add an entry indicating a block group or device which is pinned by a 10180 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a 10181 * negative errno on failure. 10182 */ 10183 static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr, 10184 bool is_block_group) 10185 { 10186 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10187 struct btrfs_swapfile_pin *sp, *entry; 10188 struct rb_node **p; 10189 struct rb_node *parent = NULL; 10190 10191 sp = kmalloc(sizeof(*sp), GFP_NOFS); 10192 if (!sp) 10193 return -ENOMEM; 10194 sp->ptr = ptr; 10195 sp->inode = inode; 10196 sp->is_block_group = is_block_group; 10197 10198 spin_lock(&fs_info->swapfile_pins_lock); 10199 p = &fs_info->swapfile_pins.rb_node; 10200 while (*p) { 10201 parent = *p; 10202 entry = rb_entry(parent, struct btrfs_swapfile_pin, node); 10203 if (sp->ptr < entry->ptr || 10204 (sp->ptr == entry->ptr && sp->inode < entry->inode)) { 10205 p = &(*p)->rb_left; 10206 } else if (sp->ptr > entry->ptr || 10207 (sp->ptr == entry->ptr && sp->inode > entry->inode)) { 10208 p = &(*p)->rb_right; 10209 } else { 10210 spin_unlock(&fs_info->swapfile_pins_lock); 10211 kfree(sp); 10212 return 1; 10213 } 10214 } 10215 rb_link_node(&sp->node, parent, p); 10216 rb_insert_color(&sp->node, &fs_info->swapfile_pins); 10217 spin_unlock(&fs_info->swapfile_pins_lock); 10218 return 0; 10219 } 10220 10221 /* Free all of the entries pinned by this swapfile. */ 10222 static void btrfs_free_swapfile_pins(struct inode *inode) 10223 { 10224 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10225 struct btrfs_swapfile_pin *sp; 10226 struct rb_node *node, *next; 10227 10228 spin_lock(&fs_info->swapfile_pins_lock); 10229 node = rb_first(&fs_info->swapfile_pins); 10230 while (node) { 10231 next = rb_next(node); 10232 sp = rb_entry(node, struct btrfs_swapfile_pin, node); 10233 if (sp->inode == inode) { 10234 rb_erase(&sp->node, &fs_info->swapfile_pins); 10235 if (sp->is_block_group) 10236 btrfs_put_block_group(sp->ptr); 10237 kfree(sp); 10238 } 10239 node = next; 10240 } 10241 spin_unlock(&fs_info->swapfile_pins_lock); 10242 } 10243 10244 struct btrfs_swap_info { 10245 u64 start; 10246 u64 block_start; 10247 u64 block_len; 10248 u64 lowest_ppage; 10249 u64 highest_ppage; 10250 unsigned long nr_pages; 10251 int nr_extents; 10252 }; 10253 10254 static int btrfs_add_swap_extent(struct swap_info_struct *sis, 10255 struct btrfs_swap_info *bsi) 10256 { 10257 unsigned long nr_pages; 10258 u64 first_ppage, first_ppage_reported, next_ppage; 10259 int ret; 10260 10261 first_ppage = ALIGN(bsi->block_start, PAGE_SIZE) >> PAGE_SHIFT; 10262 next_ppage = ALIGN_DOWN(bsi->block_start + bsi->block_len, 10263 PAGE_SIZE) >> PAGE_SHIFT; 10264 10265 if (first_ppage >= next_ppage) 10266 return 0; 10267 nr_pages = next_ppage - first_ppage; 10268 10269 first_ppage_reported = first_ppage; 10270 if (bsi->start == 0) 10271 first_ppage_reported++; 10272 if (bsi->lowest_ppage > first_ppage_reported) 10273 bsi->lowest_ppage = first_ppage_reported; 10274 if (bsi->highest_ppage < (next_ppage - 1)) 10275 bsi->highest_ppage = next_ppage - 1; 10276 10277 ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage); 10278 if (ret < 0) 10279 return ret; 10280 bsi->nr_extents += ret; 10281 bsi->nr_pages += nr_pages; 10282 return 0; 10283 } 10284 10285 static void btrfs_swap_deactivate(struct file *file) 10286 { 10287 struct inode *inode = file_inode(file); 10288 10289 btrfs_free_swapfile_pins(inode); 10290 atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles); 10291 } 10292 10293 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10294 sector_t *span) 10295 { 10296 struct inode *inode = file_inode(file); 10297 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10298 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 10299 struct extent_state *cached_state = NULL; 10300 struct extent_map *em = NULL; 10301 struct btrfs_device *device = NULL; 10302 struct btrfs_swap_info bsi = { 10303 .lowest_ppage = (sector_t)-1ULL, 10304 }; 10305 int ret = 0; 10306 u64 isize; 10307 u64 start; 10308 10309 /* 10310 * If the swap file was just created, make sure delalloc is done. If the 10311 * file changes again after this, the user is doing something stupid and 10312 * we don't really care. 10313 */ 10314 ret = btrfs_wait_ordered_range(inode, 0, (u64)-1); 10315 if (ret) 10316 return ret; 10317 10318 /* 10319 * The inode is locked, so these flags won't change after we check them. 10320 */ 10321 if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) { 10322 btrfs_warn(fs_info, "swapfile must not be compressed"); 10323 return -EINVAL; 10324 } 10325 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) { 10326 btrfs_warn(fs_info, "swapfile must not be copy-on-write"); 10327 return -EINVAL; 10328 } 10329 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { 10330 btrfs_warn(fs_info, "swapfile must not be checksummed"); 10331 return -EINVAL; 10332 } 10333 10334 /* 10335 * Balance or device remove/replace/resize can move stuff around from 10336 * under us. The exclop protection makes sure they aren't running/won't 10337 * run concurrently while we are mapping the swap extents, and 10338 * fs_info->swapfile_pins prevents them from running while the swap 10339 * file is active and moving the extents. Note that this also prevents 10340 * a concurrent device add which isn't actually necessary, but it's not 10341 * really worth the trouble to allow it. 10342 */ 10343 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) { 10344 btrfs_warn(fs_info, 10345 "cannot activate swapfile while exclusive operation is running"); 10346 return -EBUSY; 10347 } 10348 /* 10349 * Snapshots can create extents which require COW even if NODATACOW is 10350 * set. We use this counter to prevent snapshots. We must increment it 10351 * before walking the extents because we don't want a concurrent 10352 * snapshot to run after we've already checked the extents. 10353 */ 10354 atomic_inc(&BTRFS_I(inode)->root->nr_swapfiles); 10355 10356 isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); 10357 10358 lock_extent_bits(io_tree, 0, isize - 1, &cached_state); 10359 start = 0; 10360 while (start < isize) { 10361 u64 logical_block_start, physical_block_start; 10362 struct btrfs_block_group *bg; 10363 u64 len = isize - start; 10364 10365 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len); 10366 if (IS_ERR(em)) { 10367 ret = PTR_ERR(em); 10368 goto out; 10369 } 10370 10371 if (em->block_start == EXTENT_MAP_HOLE) { 10372 btrfs_warn(fs_info, "swapfile must not have holes"); 10373 ret = -EINVAL; 10374 goto out; 10375 } 10376 if (em->block_start == EXTENT_MAP_INLINE) { 10377 /* 10378 * It's unlikely we'll ever actually find ourselves 10379 * here, as a file small enough to fit inline won't be 10380 * big enough to store more than the swap header, but in 10381 * case something changes in the future, let's catch it 10382 * here rather than later. 10383 */ 10384 btrfs_warn(fs_info, "swapfile must not be inline"); 10385 ret = -EINVAL; 10386 goto out; 10387 } 10388 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 10389 btrfs_warn(fs_info, "swapfile must not be compressed"); 10390 ret = -EINVAL; 10391 goto out; 10392 } 10393 10394 logical_block_start = em->block_start + (start - em->start); 10395 len = min(len, em->len - (start - em->start)); 10396 free_extent_map(em); 10397 em = NULL; 10398 10399 ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, true); 10400 if (ret < 0) { 10401 goto out; 10402 } else if (ret) { 10403 ret = 0; 10404 } else { 10405 btrfs_warn(fs_info, 10406 "swapfile must not be copy-on-write"); 10407 ret = -EINVAL; 10408 goto out; 10409 } 10410 10411 em = btrfs_get_chunk_map(fs_info, logical_block_start, len); 10412 if (IS_ERR(em)) { 10413 ret = PTR_ERR(em); 10414 goto out; 10415 } 10416 10417 if (em->map_lookup->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { 10418 btrfs_warn(fs_info, 10419 "swapfile must have single data profile"); 10420 ret = -EINVAL; 10421 goto out; 10422 } 10423 10424 if (device == NULL) { 10425 device = em->map_lookup->stripes[0].dev; 10426 ret = btrfs_add_swapfile_pin(inode, device, false); 10427 if (ret == 1) 10428 ret = 0; 10429 else if (ret) 10430 goto out; 10431 } else if (device != em->map_lookup->stripes[0].dev) { 10432 btrfs_warn(fs_info, "swapfile must be on one device"); 10433 ret = -EINVAL; 10434 goto out; 10435 } 10436 10437 physical_block_start = (em->map_lookup->stripes[0].physical + 10438 (logical_block_start - em->start)); 10439 len = min(len, em->len - (logical_block_start - em->start)); 10440 free_extent_map(em); 10441 em = NULL; 10442 10443 bg = btrfs_lookup_block_group(fs_info, logical_block_start); 10444 if (!bg) { 10445 btrfs_warn(fs_info, 10446 "could not find block group containing swapfile"); 10447 ret = -EINVAL; 10448 goto out; 10449 } 10450 10451 ret = btrfs_add_swapfile_pin(inode, bg, true); 10452 if (ret) { 10453 btrfs_put_block_group(bg); 10454 if (ret == 1) 10455 ret = 0; 10456 else 10457 goto out; 10458 } 10459 10460 if (bsi.block_len && 10461 bsi.block_start + bsi.block_len == physical_block_start) { 10462 bsi.block_len += len; 10463 } else { 10464 if (bsi.block_len) { 10465 ret = btrfs_add_swap_extent(sis, &bsi); 10466 if (ret) 10467 goto out; 10468 } 10469 bsi.start = start; 10470 bsi.block_start = physical_block_start; 10471 bsi.block_len = len; 10472 } 10473 10474 start += len; 10475 } 10476 10477 if (bsi.block_len) 10478 ret = btrfs_add_swap_extent(sis, &bsi); 10479 10480 out: 10481 if (!IS_ERR_OR_NULL(em)) 10482 free_extent_map(em); 10483 10484 unlock_extent_cached(io_tree, 0, isize - 1, &cached_state); 10485 10486 if (ret) 10487 btrfs_swap_deactivate(file); 10488 10489 btrfs_exclop_finish(fs_info); 10490 10491 if (ret) 10492 return ret; 10493 10494 if (device) 10495 sis->bdev = device->bdev; 10496 *span = bsi.highest_ppage - bsi.lowest_ppage + 1; 10497 sis->max = bsi.nr_pages; 10498 sis->pages = bsi.nr_pages - 1; 10499 sis->highest_bit = bsi.nr_pages - 1; 10500 return bsi.nr_extents; 10501 } 10502 #else 10503 static void btrfs_swap_deactivate(struct file *file) 10504 { 10505 } 10506 10507 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10508 sector_t *span) 10509 { 10510 return -EOPNOTSUPP; 10511 } 10512 #endif 10513 10514 /* 10515 * Update the number of bytes used in the VFS' inode. When we replace extents in 10516 * a range (clone, dedupe, fallocate's zero range), we must update the number of 10517 * bytes used by the inode in an atomic manner, so that concurrent stat(2) calls 10518 * always get a correct value. 10519 */ 10520 void btrfs_update_inode_bytes(struct btrfs_inode *inode, 10521 const u64 add_bytes, 10522 const u64 del_bytes) 10523 { 10524 if (add_bytes == del_bytes) 10525 return; 10526 10527 spin_lock(&inode->lock); 10528 if (del_bytes > 0) 10529 inode_sub_bytes(&inode->vfs_inode, del_bytes); 10530 if (add_bytes > 0) 10531 inode_add_bytes(&inode->vfs_inode, add_bytes); 10532 spin_unlock(&inode->lock); 10533 } 10534 10535 static const struct inode_operations btrfs_dir_inode_operations = { 10536 .getattr = btrfs_getattr, 10537 .lookup = btrfs_lookup, 10538 .create = btrfs_create, 10539 .unlink = btrfs_unlink, 10540 .link = btrfs_link, 10541 .mkdir = btrfs_mkdir, 10542 .rmdir = btrfs_rmdir, 10543 .rename = btrfs_rename2, 10544 .symlink = btrfs_symlink, 10545 .setattr = btrfs_setattr, 10546 .mknod = btrfs_mknod, 10547 .listxattr = btrfs_listxattr, 10548 .permission = btrfs_permission, 10549 .get_acl = btrfs_get_acl, 10550 .set_acl = btrfs_set_acl, 10551 .update_time = btrfs_update_time, 10552 .tmpfile = btrfs_tmpfile, 10553 }; 10554 10555 static const struct file_operations btrfs_dir_file_operations = { 10556 .llseek = generic_file_llseek, 10557 .read = generic_read_dir, 10558 .iterate_shared = btrfs_real_readdir, 10559 .open = btrfs_opendir, 10560 .unlocked_ioctl = btrfs_ioctl, 10561 #ifdef CONFIG_COMPAT 10562 .compat_ioctl = btrfs_compat_ioctl, 10563 #endif 10564 .release = btrfs_release_file, 10565 .fsync = btrfs_sync_file, 10566 }; 10567 10568 /* 10569 * btrfs doesn't support the bmap operation because swapfiles 10570 * use bmap to make a mapping of extents in the file. They assume 10571 * these extents won't change over the life of the file and they 10572 * use the bmap result to do IO directly to the drive. 10573 * 10574 * the btrfs bmap call would return logical addresses that aren't 10575 * suitable for IO and they also will change frequently as COW 10576 * operations happen. So, swapfile + btrfs == corruption. 10577 * 10578 * For now we're avoiding this by dropping bmap. 10579 */ 10580 static const struct address_space_operations btrfs_aops = { 10581 .readpage = btrfs_readpage, 10582 .writepage = btrfs_writepage, 10583 .writepages = btrfs_writepages, 10584 .readahead = btrfs_readahead, 10585 .direct_IO = noop_direct_IO, 10586 .invalidatepage = btrfs_invalidatepage, 10587 .releasepage = btrfs_releasepage, 10588 #ifdef CONFIG_MIGRATION 10589 .migratepage = btrfs_migratepage, 10590 #endif 10591 .set_page_dirty = btrfs_set_page_dirty, 10592 .error_remove_page = generic_error_remove_page, 10593 .swap_activate = btrfs_swap_activate, 10594 .swap_deactivate = btrfs_swap_deactivate, 10595 }; 10596 10597 static const struct inode_operations btrfs_file_inode_operations = { 10598 .getattr = btrfs_getattr, 10599 .setattr = btrfs_setattr, 10600 .listxattr = btrfs_listxattr, 10601 .permission = btrfs_permission, 10602 .fiemap = btrfs_fiemap, 10603 .get_acl = btrfs_get_acl, 10604 .set_acl = btrfs_set_acl, 10605 .update_time = btrfs_update_time, 10606 }; 10607 static const struct inode_operations btrfs_special_inode_operations = { 10608 .getattr = btrfs_getattr, 10609 .setattr = btrfs_setattr, 10610 .permission = btrfs_permission, 10611 .listxattr = btrfs_listxattr, 10612 .get_acl = btrfs_get_acl, 10613 .set_acl = btrfs_set_acl, 10614 .update_time = btrfs_update_time, 10615 }; 10616 static const struct inode_operations btrfs_symlink_inode_operations = { 10617 .get_link = page_get_link, 10618 .getattr = btrfs_getattr, 10619 .setattr = btrfs_setattr, 10620 .permission = btrfs_permission, 10621 .listxattr = btrfs_listxattr, 10622 .update_time = btrfs_update_time, 10623 }; 10624 10625 const struct dentry_operations btrfs_dentry_operations = { 10626 .d_delete = btrfs_dentry_delete, 10627 }; 10628