1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 */ 5 6 #include <crypto/hash.h> 7 #include <linux/kernel.h> 8 #include <linux/bio.h> 9 #include <linux/blk-cgroup.h> 10 #include <linux/file.h> 11 #include <linux/fs.h> 12 #include <linux/pagemap.h> 13 #include <linux/highmem.h> 14 #include <linux/time.h> 15 #include <linux/init.h> 16 #include <linux/string.h> 17 #include <linux/backing-dev.h> 18 #include <linux/writeback.h> 19 #include <linux/compat.h> 20 #include <linux/xattr.h> 21 #include <linux/posix_acl.h> 22 #include <linux/falloc.h> 23 #include <linux/slab.h> 24 #include <linux/ratelimit.h> 25 #include <linux/btrfs.h> 26 #include <linux/blkdev.h> 27 #include <linux/posix_acl_xattr.h> 28 #include <linux/uio.h> 29 #include <linux/magic.h> 30 #include <linux/iversion.h> 31 #include <linux/swap.h> 32 #include <linux/migrate.h> 33 #include <linux/sched/mm.h> 34 #include <linux/iomap.h> 35 #include <asm/unaligned.h> 36 #include <linux/fsverity.h> 37 #include "misc.h" 38 #include "ctree.h" 39 #include "disk-io.h" 40 #include "transaction.h" 41 #include "btrfs_inode.h" 42 #include "print-tree.h" 43 #include "ordered-data.h" 44 #include "xattr.h" 45 #include "tree-log.h" 46 #include "bio.h" 47 #include "compression.h" 48 #include "locking.h" 49 #include "free-space-cache.h" 50 #include "props.h" 51 #include "qgroup.h" 52 #include "delalloc-space.h" 53 #include "block-group.h" 54 #include "space-info.h" 55 #include "zoned.h" 56 #include "subpage.h" 57 #include "inode-item.h" 58 #include "fs.h" 59 #include "accessors.h" 60 #include "extent-tree.h" 61 #include "root-tree.h" 62 #include "defrag.h" 63 #include "dir-item.h" 64 #include "file-item.h" 65 #include "uuid-tree.h" 66 #include "ioctl.h" 67 #include "file.h" 68 #include "acl.h" 69 #include "relocation.h" 70 #include "verity.h" 71 #include "super.h" 72 #include "orphan.h" 73 74 struct btrfs_iget_args { 75 u64 ino; 76 struct btrfs_root *root; 77 }; 78 79 struct btrfs_dio_data { 80 ssize_t submitted; 81 struct extent_changeset *data_reserved; 82 struct btrfs_ordered_extent *ordered; 83 bool data_space_reserved; 84 bool nocow_done; 85 }; 86 87 struct btrfs_dio_private { 88 /* Range of I/O */ 89 u64 file_offset; 90 u32 bytes; 91 92 /* This must be last */ 93 struct btrfs_bio bbio; 94 }; 95 96 static struct bio_set btrfs_dio_bioset; 97 98 struct btrfs_rename_ctx { 99 /* Output field. Stores the index number of the old directory entry. */ 100 u64 index; 101 }; 102 103 static const struct inode_operations btrfs_dir_inode_operations; 104 static const struct inode_operations btrfs_symlink_inode_operations; 105 static const struct inode_operations btrfs_special_inode_operations; 106 static const struct inode_operations btrfs_file_inode_operations; 107 static const struct address_space_operations btrfs_aops; 108 static const struct file_operations btrfs_dir_file_operations; 109 110 static struct kmem_cache *btrfs_inode_cachep; 111 112 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 113 static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback); 114 static noinline int cow_file_range(struct btrfs_inode *inode, 115 struct page *locked_page, 116 u64 start, u64 end, int *page_started, 117 unsigned long *nr_written, int unlock, 118 u64 *done_offset); 119 static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, 120 u64 len, u64 orig_start, u64 block_start, 121 u64 block_len, u64 orig_block_len, 122 u64 ram_bytes, int compress_type, 123 int type); 124 125 static void __cold btrfs_print_data_csum_error(struct btrfs_inode *inode, 126 u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num) 127 { 128 struct btrfs_root *root = inode->root; 129 const u32 csum_size = root->fs_info->csum_size; 130 131 /* Output without objectid, which is more meaningful */ 132 if (root->root_key.objectid >= BTRFS_LAST_FREE_OBJECTID) { 133 btrfs_warn_rl(root->fs_info, 134 "csum failed root %lld ino %lld off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d", 135 root->root_key.objectid, btrfs_ino(inode), 136 logical_start, 137 CSUM_FMT_VALUE(csum_size, csum), 138 CSUM_FMT_VALUE(csum_size, csum_expected), 139 mirror_num); 140 } else { 141 btrfs_warn_rl(root->fs_info, 142 "csum failed root %llu ino %llu off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d", 143 root->root_key.objectid, btrfs_ino(inode), 144 logical_start, 145 CSUM_FMT_VALUE(csum_size, csum), 146 CSUM_FMT_VALUE(csum_size, csum_expected), 147 mirror_num); 148 } 149 } 150 151 /* 152 * btrfs_inode_lock - lock inode i_rwsem based on arguments passed 153 * 154 * ilock_flags can have the following bit set: 155 * 156 * BTRFS_ILOCK_SHARED - acquire a shared lock on the inode 157 * BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt 158 * return -EAGAIN 159 * BTRFS_ILOCK_MMAP - acquire a write lock on the i_mmap_lock 160 */ 161 int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags) 162 { 163 if (ilock_flags & BTRFS_ILOCK_SHARED) { 164 if (ilock_flags & BTRFS_ILOCK_TRY) { 165 if (!inode_trylock_shared(&inode->vfs_inode)) 166 return -EAGAIN; 167 else 168 return 0; 169 } 170 inode_lock_shared(&inode->vfs_inode); 171 } else { 172 if (ilock_flags & BTRFS_ILOCK_TRY) { 173 if (!inode_trylock(&inode->vfs_inode)) 174 return -EAGAIN; 175 else 176 return 0; 177 } 178 inode_lock(&inode->vfs_inode); 179 } 180 if (ilock_flags & BTRFS_ILOCK_MMAP) 181 down_write(&inode->i_mmap_lock); 182 return 0; 183 } 184 185 /* 186 * btrfs_inode_unlock - unock inode i_rwsem 187 * 188 * ilock_flags should contain the same bits set as passed to btrfs_inode_lock() 189 * to decide whether the lock acquired is shared or exclusive. 190 */ 191 void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags) 192 { 193 if (ilock_flags & BTRFS_ILOCK_MMAP) 194 up_write(&inode->i_mmap_lock); 195 if (ilock_flags & BTRFS_ILOCK_SHARED) 196 inode_unlock_shared(&inode->vfs_inode); 197 else 198 inode_unlock(&inode->vfs_inode); 199 } 200 201 /* 202 * Cleanup all submitted ordered extents in specified range to handle errors 203 * from the btrfs_run_delalloc_range() callback. 204 * 205 * NOTE: caller must ensure that when an error happens, it can not call 206 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING 207 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata 208 * to be released, which we want to happen only when finishing the ordered 209 * extent (btrfs_finish_ordered_io()). 210 */ 211 static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, 212 struct page *locked_page, 213 u64 offset, u64 bytes) 214 { 215 unsigned long index = offset >> PAGE_SHIFT; 216 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT; 217 u64 page_start = 0, page_end = 0; 218 struct page *page; 219 220 if (locked_page) { 221 page_start = page_offset(locked_page); 222 page_end = page_start + PAGE_SIZE - 1; 223 } 224 225 while (index <= end_index) { 226 /* 227 * For locked page, we will call end_extent_writepage() on it 228 * in run_delalloc_range() for the error handling. That 229 * end_extent_writepage() function will call 230 * btrfs_mark_ordered_io_finished() to clear page Ordered and 231 * run the ordered extent accounting. 232 * 233 * Here we can't just clear the Ordered bit, or 234 * btrfs_mark_ordered_io_finished() would skip the accounting 235 * for the page range, and the ordered extent will never finish. 236 */ 237 if (locked_page && index == (page_start >> PAGE_SHIFT)) { 238 index++; 239 continue; 240 } 241 page = find_get_page(inode->vfs_inode.i_mapping, index); 242 index++; 243 if (!page) 244 continue; 245 246 /* 247 * Here we just clear all Ordered bits for every page in the 248 * range, then btrfs_mark_ordered_io_finished() will handle 249 * the ordered extent accounting for the range. 250 */ 251 btrfs_page_clamp_clear_ordered(inode->root->fs_info, page, 252 offset, bytes); 253 put_page(page); 254 } 255 256 if (locked_page) { 257 /* The locked page covers the full range, nothing needs to be done */ 258 if (bytes + offset <= page_start + PAGE_SIZE) 259 return; 260 /* 261 * In case this page belongs to the delalloc range being 262 * instantiated then skip it, since the first page of a range is 263 * going to be properly cleaned up by the caller of 264 * run_delalloc_range 265 */ 266 if (page_start >= offset && page_end <= (offset + bytes - 1)) { 267 bytes = offset + bytes - page_offset(locked_page) - PAGE_SIZE; 268 offset = page_offset(locked_page) + PAGE_SIZE; 269 } 270 } 271 272 return btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes, false); 273 } 274 275 static int btrfs_dirty_inode(struct btrfs_inode *inode); 276 277 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 278 struct btrfs_new_inode_args *args) 279 { 280 int err; 281 282 if (args->default_acl) { 283 err = __btrfs_set_acl(trans, args->inode, args->default_acl, 284 ACL_TYPE_DEFAULT); 285 if (err) 286 return err; 287 } 288 if (args->acl) { 289 err = __btrfs_set_acl(trans, args->inode, args->acl, ACL_TYPE_ACCESS); 290 if (err) 291 return err; 292 } 293 if (!args->default_acl && !args->acl) 294 cache_no_acl(args->inode); 295 return btrfs_xattr_security_init(trans, args->inode, args->dir, 296 &args->dentry->d_name); 297 } 298 299 /* 300 * this does all the hard work for inserting an inline extent into 301 * the btree. The caller should have done a btrfs_drop_extents so that 302 * no overlapping inline items exist in the btree 303 */ 304 static int insert_inline_extent(struct btrfs_trans_handle *trans, 305 struct btrfs_path *path, 306 struct btrfs_inode *inode, bool extent_inserted, 307 size_t size, size_t compressed_size, 308 int compress_type, 309 struct page **compressed_pages, 310 bool update_i_size) 311 { 312 struct btrfs_root *root = inode->root; 313 struct extent_buffer *leaf; 314 struct page *page = NULL; 315 char *kaddr; 316 unsigned long ptr; 317 struct btrfs_file_extent_item *ei; 318 int ret; 319 size_t cur_size = size; 320 u64 i_size; 321 322 ASSERT((compressed_size > 0 && compressed_pages) || 323 (compressed_size == 0 && !compressed_pages)); 324 325 if (compressed_size && compressed_pages) 326 cur_size = compressed_size; 327 328 if (!extent_inserted) { 329 struct btrfs_key key; 330 size_t datasize; 331 332 key.objectid = btrfs_ino(inode); 333 key.offset = 0; 334 key.type = BTRFS_EXTENT_DATA_KEY; 335 336 datasize = btrfs_file_extent_calc_inline_size(cur_size); 337 ret = btrfs_insert_empty_item(trans, root, path, &key, 338 datasize); 339 if (ret) 340 goto fail; 341 } 342 leaf = path->nodes[0]; 343 ei = btrfs_item_ptr(leaf, path->slots[0], 344 struct btrfs_file_extent_item); 345 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 346 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 347 btrfs_set_file_extent_encryption(leaf, ei, 0); 348 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 349 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 350 ptr = btrfs_file_extent_inline_start(ei); 351 352 if (compress_type != BTRFS_COMPRESS_NONE) { 353 struct page *cpage; 354 int i = 0; 355 while (compressed_size > 0) { 356 cpage = compressed_pages[i]; 357 cur_size = min_t(unsigned long, compressed_size, 358 PAGE_SIZE); 359 360 kaddr = kmap_local_page(cpage); 361 write_extent_buffer(leaf, kaddr, ptr, cur_size); 362 kunmap_local(kaddr); 363 364 i++; 365 ptr += cur_size; 366 compressed_size -= cur_size; 367 } 368 btrfs_set_file_extent_compression(leaf, ei, 369 compress_type); 370 } else { 371 page = find_get_page(inode->vfs_inode.i_mapping, 0); 372 btrfs_set_file_extent_compression(leaf, ei, 0); 373 kaddr = kmap_local_page(page); 374 write_extent_buffer(leaf, kaddr, ptr, size); 375 kunmap_local(kaddr); 376 put_page(page); 377 } 378 btrfs_mark_buffer_dirty(leaf); 379 btrfs_release_path(path); 380 381 /* 382 * We align size to sectorsize for inline extents just for simplicity 383 * sake. 384 */ 385 ret = btrfs_inode_set_file_extent_range(inode, 0, 386 ALIGN(size, root->fs_info->sectorsize)); 387 if (ret) 388 goto fail; 389 390 /* 391 * We're an inline extent, so nobody can extend the file past i_size 392 * without locking a page we already have locked. 393 * 394 * We must do any i_size and inode updates before we unlock the pages. 395 * Otherwise we could end up racing with unlink. 396 */ 397 i_size = i_size_read(&inode->vfs_inode); 398 if (update_i_size && size > i_size) { 399 i_size_write(&inode->vfs_inode, size); 400 i_size = size; 401 } 402 inode->disk_i_size = i_size; 403 404 fail: 405 return ret; 406 } 407 408 409 /* 410 * conditionally insert an inline extent into the file. This 411 * does the checks required to make sure the data is small enough 412 * to fit as an inline extent. 413 */ 414 static noinline int cow_file_range_inline(struct btrfs_inode *inode, u64 size, 415 size_t compressed_size, 416 int compress_type, 417 struct page **compressed_pages, 418 bool update_i_size) 419 { 420 struct btrfs_drop_extents_args drop_args = { 0 }; 421 struct btrfs_root *root = inode->root; 422 struct btrfs_fs_info *fs_info = root->fs_info; 423 struct btrfs_trans_handle *trans; 424 u64 data_len = (compressed_size ?: size); 425 int ret; 426 struct btrfs_path *path; 427 428 /* 429 * We can create an inline extent if it ends at or beyond the current 430 * i_size, is no larger than a sector (decompressed), and the (possibly 431 * compressed) data fits in a leaf and the configured maximum inline 432 * size. 433 */ 434 if (size < i_size_read(&inode->vfs_inode) || 435 size > fs_info->sectorsize || 436 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 437 data_len > fs_info->max_inline) 438 return 1; 439 440 path = btrfs_alloc_path(); 441 if (!path) 442 return -ENOMEM; 443 444 trans = btrfs_join_transaction(root); 445 if (IS_ERR(trans)) { 446 btrfs_free_path(path); 447 return PTR_ERR(trans); 448 } 449 trans->block_rsv = &inode->block_rsv; 450 451 drop_args.path = path; 452 drop_args.start = 0; 453 drop_args.end = fs_info->sectorsize; 454 drop_args.drop_cache = true; 455 drop_args.replace_extent = true; 456 drop_args.extent_item_size = btrfs_file_extent_calc_inline_size(data_len); 457 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 458 if (ret) { 459 btrfs_abort_transaction(trans, ret); 460 goto out; 461 } 462 463 ret = insert_inline_extent(trans, path, inode, drop_args.extent_inserted, 464 size, compressed_size, compress_type, 465 compressed_pages, update_i_size); 466 if (ret && ret != -ENOSPC) { 467 btrfs_abort_transaction(trans, ret); 468 goto out; 469 } else if (ret == -ENOSPC) { 470 ret = 1; 471 goto out; 472 } 473 474 btrfs_update_inode_bytes(inode, size, drop_args.bytes_found); 475 ret = btrfs_update_inode(trans, root, inode); 476 if (ret && ret != -ENOSPC) { 477 btrfs_abort_transaction(trans, ret); 478 goto out; 479 } else if (ret == -ENOSPC) { 480 ret = 1; 481 goto out; 482 } 483 484 btrfs_set_inode_full_sync(inode); 485 out: 486 /* 487 * Don't forget to free the reserved space, as for inlined extent 488 * it won't count as data extent, free them directly here. 489 * And at reserve time, it's always aligned to page size, so 490 * just free one page here. 491 */ 492 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE); 493 btrfs_free_path(path); 494 btrfs_end_transaction(trans); 495 return ret; 496 } 497 498 struct async_extent { 499 u64 start; 500 u64 ram_size; 501 u64 compressed_size; 502 struct page **pages; 503 unsigned long nr_pages; 504 int compress_type; 505 struct list_head list; 506 }; 507 508 struct async_chunk { 509 struct btrfs_inode *inode; 510 struct page *locked_page; 511 u64 start; 512 u64 end; 513 blk_opf_t write_flags; 514 struct list_head extents; 515 struct cgroup_subsys_state *blkcg_css; 516 struct btrfs_work work; 517 struct async_cow *async_cow; 518 }; 519 520 struct async_cow { 521 atomic_t num_chunks; 522 struct async_chunk chunks[]; 523 }; 524 525 static noinline int add_async_extent(struct async_chunk *cow, 526 u64 start, u64 ram_size, 527 u64 compressed_size, 528 struct page **pages, 529 unsigned long nr_pages, 530 int compress_type) 531 { 532 struct async_extent *async_extent; 533 534 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 535 BUG_ON(!async_extent); /* -ENOMEM */ 536 async_extent->start = start; 537 async_extent->ram_size = ram_size; 538 async_extent->compressed_size = compressed_size; 539 async_extent->pages = pages; 540 async_extent->nr_pages = nr_pages; 541 async_extent->compress_type = compress_type; 542 list_add_tail(&async_extent->list, &cow->extents); 543 return 0; 544 } 545 546 /* 547 * Check if the inode needs to be submitted to compression, based on mount 548 * options, defragmentation, properties or heuristics. 549 */ 550 static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, 551 u64 end) 552 { 553 struct btrfs_fs_info *fs_info = inode->root->fs_info; 554 555 if (!btrfs_inode_can_compress(inode)) { 556 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG), 557 KERN_ERR "BTRFS: unexpected compression for ino %llu\n", 558 btrfs_ino(inode)); 559 return 0; 560 } 561 /* 562 * Special check for subpage. 563 * 564 * We lock the full page then run each delalloc range in the page, thus 565 * for the following case, we will hit some subpage specific corner case: 566 * 567 * 0 32K 64K 568 * | |///////| |///////| 569 * \- A \- B 570 * 571 * In above case, both range A and range B will try to unlock the full 572 * page [0, 64K), causing the one finished later will have page 573 * unlocked already, triggering various page lock requirement BUG_ON()s. 574 * 575 * So here we add an artificial limit that subpage compression can only 576 * if the range is fully page aligned. 577 * 578 * In theory we only need to ensure the first page is fully covered, but 579 * the tailing partial page will be locked until the full compression 580 * finishes, delaying the write of other range. 581 * 582 * TODO: Make btrfs_run_delalloc_range() to lock all delalloc range 583 * first to prevent any submitted async extent to unlock the full page. 584 * By this, we can ensure for subpage case that only the last async_cow 585 * will unlock the full page. 586 */ 587 if (fs_info->sectorsize < PAGE_SIZE) { 588 if (!PAGE_ALIGNED(start) || 589 !PAGE_ALIGNED(end + 1)) 590 return 0; 591 } 592 593 /* force compress */ 594 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 595 return 1; 596 /* defrag ioctl */ 597 if (inode->defrag_compress) 598 return 1; 599 /* bad compression ratios */ 600 if (inode->flags & BTRFS_INODE_NOCOMPRESS) 601 return 0; 602 if (btrfs_test_opt(fs_info, COMPRESS) || 603 inode->flags & BTRFS_INODE_COMPRESS || 604 inode->prop_compress) 605 return btrfs_compress_heuristic(&inode->vfs_inode, start, end); 606 return 0; 607 } 608 609 static inline void inode_should_defrag(struct btrfs_inode *inode, 610 u64 start, u64 end, u64 num_bytes, u32 small_write) 611 { 612 /* If this is a small write inside eof, kick off a defrag */ 613 if (num_bytes < small_write && 614 (start > 0 || end + 1 < inode->disk_i_size)) 615 btrfs_add_inode_defrag(NULL, inode, small_write); 616 } 617 618 /* 619 * we create compressed extents in two phases. The first 620 * phase compresses a range of pages that have already been 621 * locked (both pages and state bits are locked). 622 * 623 * This is done inside an ordered work queue, and the compression 624 * is spread across many cpus. The actual IO submission is step 625 * two, and the ordered work queue takes care of making sure that 626 * happens in the same order things were put onto the queue by 627 * writepages and friends. 628 * 629 * If this code finds it can't get good compression, it puts an 630 * entry onto the work queue to write the uncompressed bytes. This 631 * makes sure that both compressed inodes and uncompressed inodes 632 * are written in the same order that the flusher thread sent them 633 * down. 634 */ 635 static noinline int compress_file_range(struct async_chunk *async_chunk) 636 { 637 struct btrfs_inode *inode = async_chunk->inode; 638 struct btrfs_fs_info *fs_info = inode->root->fs_info; 639 u64 blocksize = fs_info->sectorsize; 640 u64 start = async_chunk->start; 641 u64 end = async_chunk->end; 642 u64 actual_end; 643 u64 i_size; 644 int ret = 0; 645 struct page **pages = NULL; 646 unsigned long nr_pages; 647 unsigned long total_compressed = 0; 648 unsigned long total_in = 0; 649 int i; 650 int will_compress; 651 int compress_type = fs_info->compress_type; 652 int compressed_extents = 0; 653 int redirty = 0; 654 655 inode_should_defrag(inode, start, end, end - start + 1, SZ_16K); 656 657 /* 658 * We need to save i_size before now because it could change in between 659 * us evaluating the size and assigning it. This is because we lock and 660 * unlock the page in truncate and fallocate, and then modify the i_size 661 * later on. 662 * 663 * The barriers are to emulate READ_ONCE, remove that once i_size_read 664 * does that for us. 665 */ 666 barrier(); 667 i_size = i_size_read(&inode->vfs_inode); 668 barrier(); 669 actual_end = min_t(u64, i_size, end + 1); 670 again: 671 will_compress = 0; 672 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; 673 nr_pages = min_t(unsigned long, nr_pages, BTRFS_MAX_COMPRESSED_PAGES); 674 675 /* 676 * we don't want to send crud past the end of i_size through 677 * compression, that's just a waste of CPU time. So, if the 678 * end of the file is before the start of our current 679 * requested range of bytes, we bail out to the uncompressed 680 * cleanup code that can deal with all of this. 681 * 682 * It isn't really the fastest way to fix things, but this is a 683 * very uncommon corner. 684 */ 685 if (actual_end <= start) 686 goto cleanup_and_bail_uncompressed; 687 688 total_compressed = actual_end - start; 689 690 /* 691 * Skip compression for a small file range(<=blocksize) that 692 * isn't an inline extent, since it doesn't save disk space at all. 693 */ 694 if (total_compressed <= blocksize && 695 (start > 0 || end + 1 < inode->disk_i_size)) 696 goto cleanup_and_bail_uncompressed; 697 698 /* 699 * For subpage case, we require full page alignment for the sector 700 * aligned range. 701 * Thus we must also check against @actual_end, not just @end. 702 */ 703 if (blocksize < PAGE_SIZE) { 704 if (!PAGE_ALIGNED(start) || 705 !PAGE_ALIGNED(round_up(actual_end, blocksize))) 706 goto cleanup_and_bail_uncompressed; 707 } 708 709 total_compressed = min_t(unsigned long, total_compressed, 710 BTRFS_MAX_UNCOMPRESSED); 711 total_in = 0; 712 ret = 0; 713 714 /* 715 * we do compression for mount -o compress and when the 716 * inode has not been flagged as nocompress. This flag can 717 * change at any time if we discover bad compression ratios. 718 */ 719 if (inode_need_compress(inode, start, end)) { 720 WARN_ON(pages); 721 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 722 if (!pages) { 723 /* just bail out to the uncompressed code */ 724 nr_pages = 0; 725 goto cont; 726 } 727 728 if (inode->defrag_compress) 729 compress_type = inode->defrag_compress; 730 else if (inode->prop_compress) 731 compress_type = inode->prop_compress; 732 733 /* 734 * we need to call clear_page_dirty_for_io on each 735 * page in the range. Otherwise applications with the file 736 * mmap'd can wander in and change the page contents while 737 * we are compressing them. 738 * 739 * If the compression fails for any reason, we set the pages 740 * dirty again later on. 741 * 742 * Note that the remaining part is redirtied, the start pointer 743 * has moved, the end is the original one. 744 */ 745 if (!redirty) { 746 extent_range_clear_dirty_for_io(&inode->vfs_inode, start, end); 747 redirty = 1; 748 } 749 750 /* Compression level is applied here and only here */ 751 ret = btrfs_compress_pages( 752 compress_type | (fs_info->compress_level << 4), 753 inode->vfs_inode.i_mapping, start, 754 pages, 755 &nr_pages, 756 &total_in, 757 &total_compressed); 758 759 if (!ret) { 760 unsigned long offset = offset_in_page(total_compressed); 761 struct page *page = pages[nr_pages - 1]; 762 763 /* zero the tail end of the last page, we might be 764 * sending it down to disk 765 */ 766 if (offset) 767 memzero_page(page, offset, PAGE_SIZE - offset); 768 will_compress = 1; 769 } 770 } 771 cont: 772 /* 773 * Check cow_file_range() for why we don't even try to create inline 774 * extent for subpage case. 775 */ 776 if (start == 0 && fs_info->sectorsize == PAGE_SIZE) { 777 /* lets try to make an inline extent */ 778 if (ret || total_in < actual_end) { 779 /* we didn't compress the entire range, try 780 * to make an uncompressed inline extent. 781 */ 782 ret = cow_file_range_inline(inode, actual_end, 783 0, BTRFS_COMPRESS_NONE, 784 NULL, false); 785 } else { 786 /* try making a compressed inline extent */ 787 ret = cow_file_range_inline(inode, actual_end, 788 total_compressed, 789 compress_type, pages, 790 false); 791 } 792 if (ret <= 0) { 793 unsigned long clear_flags = EXTENT_DELALLOC | 794 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 795 EXTENT_DO_ACCOUNTING; 796 unsigned long page_error_op; 797 798 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 799 800 /* 801 * inline extent creation worked or returned error, 802 * we don't need to create any more async work items. 803 * Unlock and free up our temp pages. 804 * 805 * We use DO_ACCOUNTING here because we need the 806 * delalloc_release_metadata to be done _after_ we drop 807 * our outstanding extent for clearing delalloc for this 808 * range. 809 */ 810 extent_clear_unlock_delalloc(inode, start, end, 811 NULL, 812 clear_flags, 813 PAGE_UNLOCK | 814 PAGE_START_WRITEBACK | 815 page_error_op | 816 PAGE_END_WRITEBACK); 817 818 /* 819 * Ensure we only free the compressed pages if we have 820 * them allocated, as we can still reach here with 821 * inode_need_compress() == false. 822 */ 823 if (pages) { 824 for (i = 0; i < nr_pages; i++) { 825 WARN_ON(pages[i]->mapping); 826 put_page(pages[i]); 827 } 828 kfree(pages); 829 } 830 return 0; 831 } 832 } 833 834 if (will_compress) { 835 /* 836 * we aren't doing an inline extent round the compressed size 837 * up to a block size boundary so the allocator does sane 838 * things 839 */ 840 total_compressed = ALIGN(total_compressed, blocksize); 841 842 /* 843 * one last check to make sure the compression is really a 844 * win, compare the page count read with the blocks on disk, 845 * compression must free at least one sector size 846 */ 847 total_in = round_up(total_in, fs_info->sectorsize); 848 if (total_compressed + blocksize <= total_in) { 849 compressed_extents++; 850 851 /* 852 * The async work queues will take care of doing actual 853 * allocation on disk for these compressed pages, and 854 * will submit them to the elevator. 855 */ 856 add_async_extent(async_chunk, start, total_in, 857 total_compressed, pages, nr_pages, 858 compress_type); 859 860 if (start + total_in < end) { 861 start += total_in; 862 pages = NULL; 863 cond_resched(); 864 goto again; 865 } 866 return compressed_extents; 867 } 868 } 869 if (pages) { 870 /* 871 * the compression code ran but failed to make things smaller, 872 * free any pages it allocated and our page pointer array 873 */ 874 for (i = 0; i < nr_pages; i++) { 875 WARN_ON(pages[i]->mapping); 876 put_page(pages[i]); 877 } 878 kfree(pages); 879 pages = NULL; 880 total_compressed = 0; 881 nr_pages = 0; 882 883 /* flag the file so we don't compress in the future */ 884 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && 885 !(inode->prop_compress)) { 886 inode->flags |= BTRFS_INODE_NOCOMPRESS; 887 } 888 } 889 cleanup_and_bail_uncompressed: 890 /* 891 * No compression, but we still need to write the pages in the file 892 * we've been given so far. redirty the locked page if it corresponds 893 * to our extent and set things up for the async work queue to run 894 * cow_file_range to do the normal delalloc dance. 895 */ 896 if (async_chunk->locked_page && 897 (page_offset(async_chunk->locked_page) >= start && 898 page_offset(async_chunk->locked_page)) <= end) { 899 __set_page_dirty_nobuffers(async_chunk->locked_page); 900 /* unlocked later on in the async handlers */ 901 } 902 903 if (redirty) 904 extent_range_redirty_for_io(&inode->vfs_inode, start, end); 905 add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0, 906 BTRFS_COMPRESS_NONE); 907 compressed_extents++; 908 909 return compressed_extents; 910 } 911 912 static void free_async_extent_pages(struct async_extent *async_extent) 913 { 914 int i; 915 916 if (!async_extent->pages) 917 return; 918 919 for (i = 0; i < async_extent->nr_pages; i++) { 920 WARN_ON(async_extent->pages[i]->mapping); 921 put_page(async_extent->pages[i]); 922 } 923 kfree(async_extent->pages); 924 async_extent->nr_pages = 0; 925 async_extent->pages = NULL; 926 } 927 928 static int submit_uncompressed_range(struct btrfs_inode *inode, 929 struct async_extent *async_extent, 930 struct page *locked_page) 931 { 932 u64 start = async_extent->start; 933 u64 end = async_extent->start + async_extent->ram_size - 1; 934 unsigned long nr_written = 0; 935 int page_started = 0; 936 int ret; 937 938 /* 939 * Call cow_file_range() to run the delalloc range directly, since we 940 * won't go to NOCOW or async path again. 941 * 942 * Also we call cow_file_range() with @unlock_page == 0, so that we 943 * can directly submit them without interruption. 944 */ 945 ret = cow_file_range(inode, locked_page, start, end, &page_started, 946 &nr_written, 0, NULL); 947 /* Inline extent inserted, page gets unlocked and everything is done */ 948 if (page_started) 949 return 0; 950 951 if (ret < 0) { 952 btrfs_cleanup_ordered_extents(inode, locked_page, start, end - start + 1); 953 if (locked_page) { 954 const u64 page_start = page_offset(locked_page); 955 const u64 page_end = page_start + PAGE_SIZE - 1; 956 957 btrfs_page_set_error(inode->root->fs_info, locked_page, 958 page_start, PAGE_SIZE); 959 set_page_writeback(locked_page); 960 end_page_writeback(locked_page); 961 end_extent_writepage(locked_page, ret, page_start, page_end); 962 unlock_page(locked_page); 963 } 964 return ret; 965 } 966 967 /* All pages will be unlocked, including @locked_page */ 968 return extent_write_locked_range(&inode->vfs_inode, start, end); 969 } 970 971 static int submit_one_async_extent(struct btrfs_inode *inode, 972 struct async_chunk *async_chunk, 973 struct async_extent *async_extent, 974 u64 *alloc_hint) 975 { 976 struct extent_io_tree *io_tree = &inode->io_tree; 977 struct btrfs_root *root = inode->root; 978 struct btrfs_fs_info *fs_info = root->fs_info; 979 struct btrfs_key ins; 980 struct page *locked_page = NULL; 981 struct extent_map *em; 982 int ret = 0; 983 u64 start = async_extent->start; 984 u64 end = async_extent->start + async_extent->ram_size - 1; 985 986 if (async_chunk->blkcg_css) 987 kthread_associate_blkcg(async_chunk->blkcg_css); 988 989 /* 990 * If async_chunk->locked_page is in the async_extent range, we need to 991 * handle it. 992 */ 993 if (async_chunk->locked_page) { 994 u64 locked_page_start = page_offset(async_chunk->locked_page); 995 u64 locked_page_end = locked_page_start + PAGE_SIZE - 1; 996 997 if (!(start >= locked_page_end || end <= locked_page_start)) 998 locked_page = async_chunk->locked_page; 999 } 1000 lock_extent(io_tree, start, end, NULL); 1001 1002 /* We have fall back to uncompressed write */ 1003 if (!async_extent->pages) { 1004 ret = submit_uncompressed_range(inode, async_extent, locked_page); 1005 goto done; 1006 } 1007 1008 ret = btrfs_reserve_extent(root, async_extent->ram_size, 1009 async_extent->compressed_size, 1010 async_extent->compressed_size, 1011 0, *alloc_hint, &ins, 1, 1); 1012 if (ret) { 1013 free_async_extent_pages(async_extent); 1014 /* 1015 * Here we used to try again by going back to non-compressed 1016 * path for ENOSPC. But we can't reserve space even for 1017 * compressed size, how could it work for uncompressed size 1018 * which requires larger size? So here we directly go error 1019 * path. 1020 */ 1021 goto out_free; 1022 } 1023 1024 /* Here we're doing allocation and writeback of the compressed pages */ 1025 em = create_io_em(inode, start, 1026 async_extent->ram_size, /* len */ 1027 start, /* orig_start */ 1028 ins.objectid, /* block_start */ 1029 ins.offset, /* block_len */ 1030 ins.offset, /* orig_block_len */ 1031 async_extent->ram_size, /* ram_bytes */ 1032 async_extent->compress_type, 1033 BTRFS_ORDERED_COMPRESSED); 1034 if (IS_ERR(em)) { 1035 ret = PTR_ERR(em); 1036 goto out_free_reserve; 1037 } 1038 free_extent_map(em); 1039 1040 ret = btrfs_add_ordered_extent(inode, start, /* file_offset */ 1041 async_extent->ram_size, /* num_bytes */ 1042 async_extent->ram_size, /* ram_bytes */ 1043 ins.objectid, /* disk_bytenr */ 1044 ins.offset, /* disk_num_bytes */ 1045 0, /* offset */ 1046 1 << BTRFS_ORDERED_COMPRESSED, 1047 async_extent->compress_type); 1048 if (ret) { 1049 btrfs_drop_extent_map_range(inode, start, end, false); 1050 goto out_free_reserve; 1051 } 1052 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1053 1054 /* Clear dirty, set writeback and unlock the pages. */ 1055 extent_clear_unlock_delalloc(inode, start, end, 1056 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 1057 PAGE_UNLOCK | PAGE_START_WRITEBACK); 1058 1059 btrfs_submit_compressed_write(inode, start, /* file_offset */ 1060 async_extent->ram_size, /* num_bytes */ 1061 ins.objectid, /* disk_bytenr */ 1062 ins.offset, /* compressed_len */ 1063 async_extent->pages, /* compressed_pages */ 1064 async_extent->nr_pages, 1065 async_chunk->write_flags, true); 1066 *alloc_hint = ins.objectid + ins.offset; 1067 done: 1068 if (async_chunk->blkcg_css) 1069 kthread_associate_blkcg(NULL); 1070 kfree(async_extent); 1071 return ret; 1072 1073 out_free_reserve: 1074 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1075 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1076 out_free: 1077 extent_clear_unlock_delalloc(inode, start, end, 1078 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 1079 EXTENT_DELALLOC_NEW | 1080 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 1081 PAGE_UNLOCK | PAGE_START_WRITEBACK | 1082 PAGE_END_WRITEBACK | PAGE_SET_ERROR); 1083 free_async_extent_pages(async_extent); 1084 goto done; 1085 } 1086 1087 /* 1088 * Phase two of compressed writeback. This is the ordered portion of the code, 1089 * which only gets called in the order the work was queued. We walk all the 1090 * async extents created by compress_file_range and send them down to the disk. 1091 */ 1092 static noinline void submit_compressed_extents(struct async_chunk *async_chunk) 1093 { 1094 struct btrfs_inode *inode = async_chunk->inode; 1095 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1096 struct async_extent *async_extent; 1097 u64 alloc_hint = 0; 1098 int ret = 0; 1099 1100 while (!list_empty(&async_chunk->extents)) { 1101 u64 extent_start; 1102 u64 ram_size; 1103 1104 async_extent = list_entry(async_chunk->extents.next, 1105 struct async_extent, list); 1106 list_del(&async_extent->list); 1107 extent_start = async_extent->start; 1108 ram_size = async_extent->ram_size; 1109 1110 ret = submit_one_async_extent(inode, async_chunk, async_extent, 1111 &alloc_hint); 1112 btrfs_debug(fs_info, 1113 "async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d", 1114 inode->root->root_key.objectid, 1115 btrfs_ino(inode), extent_start, ram_size, ret); 1116 } 1117 } 1118 1119 static u64 get_extent_allocation_hint(struct btrfs_inode *inode, u64 start, 1120 u64 num_bytes) 1121 { 1122 struct extent_map_tree *em_tree = &inode->extent_tree; 1123 struct extent_map *em; 1124 u64 alloc_hint = 0; 1125 1126 read_lock(&em_tree->lock); 1127 em = search_extent_mapping(em_tree, start, num_bytes); 1128 if (em) { 1129 /* 1130 * if block start isn't an actual block number then find the 1131 * first block in this inode and use that as a hint. If that 1132 * block is also bogus then just don't worry about it. 1133 */ 1134 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 1135 free_extent_map(em); 1136 em = search_extent_mapping(em_tree, 0, 0); 1137 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 1138 alloc_hint = em->block_start; 1139 if (em) 1140 free_extent_map(em); 1141 } else { 1142 alloc_hint = em->block_start; 1143 free_extent_map(em); 1144 } 1145 } 1146 read_unlock(&em_tree->lock); 1147 1148 return alloc_hint; 1149 } 1150 1151 /* 1152 * when extent_io.c finds a delayed allocation range in the file, 1153 * the call backs end up in this code. The basic idea is to 1154 * allocate extents on disk for the range, and create ordered data structs 1155 * in ram to track those extents. 1156 * 1157 * locked_page is the page that writepage had locked already. We use 1158 * it to make sure we don't do extra locks or unlocks. 1159 * 1160 * *page_started is set to one if we unlock locked_page and do everything 1161 * required to start IO on it. It may be clean and already done with 1162 * IO when we return. 1163 * 1164 * When unlock == 1, we unlock the pages in successfully allocated regions. 1165 * When unlock == 0, we leave them locked for writing them out. 1166 * 1167 * However, we unlock all the pages except @locked_page in case of failure. 1168 * 1169 * In summary, page locking state will be as follow: 1170 * 1171 * - page_started == 1 (return value) 1172 * - All the pages are unlocked. IO is started. 1173 * - Note that this can happen only on success 1174 * - unlock == 1 1175 * - All the pages except @locked_page are unlocked in any case 1176 * - unlock == 0 1177 * - On success, all the pages are locked for writing out them 1178 * - On failure, all the pages except @locked_page are unlocked 1179 * 1180 * When a failure happens in the second or later iteration of the 1181 * while-loop, the ordered extents created in previous iterations are kept 1182 * intact. So, the caller must clean them up by calling 1183 * btrfs_cleanup_ordered_extents(). See btrfs_run_delalloc_range() for 1184 * example. 1185 */ 1186 static noinline int cow_file_range(struct btrfs_inode *inode, 1187 struct page *locked_page, 1188 u64 start, u64 end, int *page_started, 1189 unsigned long *nr_written, int unlock, 1190 u64 *done_offset) 1191 { 1192 struct btrfs_root *root = inode->root; 1193 struct btrfs_fs_info *fs_info = root->fs_info; 1194 u64 alloc_hint = 0; 1195 u64 orig_start = start; 1196 u64 num_bytes; 1197 unsigned long ram_size; 1198 u64 cur_alloc_size = 0; 1199 u64 min_alloc_size; 1200 u64 blocksize = fs_info->sectorsize; 1201 struct btrfs_key ins; 1202 struct extent_map *em; 1203 unsigned clear_bits; 1204 unsigned long page_ops; 1205 bool extent_reserved = false; 1206 int ret = 0; 1207 1208 if (btrfs_is_free_space_inode(inode)) { 1209 ret = -EINVAL; 1210 goto out_unlock; 1211 } 1212 1213 num_bytes = ALIGN(end - start + 1, blocksize); 1214 num_bytes = max(blocksize, num_bytes); 1215 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); 1216 1217 inode_should_defrag(inode, start, end, num_bytes, SZ_64K); 1218 1219 /* 1220 * Due to the page size limit, for subpage we can only trigger the 1221 * writeback for the dirty sectors of page, that means data writeback 1222 * is doing more writeback than what we want. 1223 * 1224 * This is especially unexpected for some call sites like fallocate, 1225 * where we only increase i_size after everything is done. 1226 * This means we can trigger inline extent even if we didn't want to. 1227 * So here we skip inline extent creation completely. 1228 */ 1229 if (start == 0 && fs_info->sectorsize == PAGE_SIZE) { 1230 u64 actual_end = min_t(u64, i_size_read(&inode->vfs_inode), 1231 end + 1); 1232 1233 /* lets try to make an inline extent */ 1234 ret = cow_file_range_inline(inode, actual_end, 0, 1235 BTRFS_COMPRESS_NONE, NULL, false); 1236 if (ret == 0) { 1237 /* 1238 * We use DO_ACCOUNTING here because we need the 1239 * delalloc_release_metadata to be run _after_ we drop 1240 * our outstanding extent for clearing delalloc for this 1241 * range. 1242 */ 1243 extent_clear_unlock_delalloc(inode, start, end, 1244 locked_page, 1245 EXTENT_LOCKED | EXTENT_DELALLOC | 1246 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 1247 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1248 PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); 1249 *nr_written = *nr_written + 1250 (end - start + PAGE_SIZE) / PAGE_SIZE; 1251 *page_started = 1; 1252 /* 1253 * locked_page is locked by the caller of 1254 * writepage_delalloc(), not locked by 1255 * __process_pages_contig(). 1256 * 1257 * We can't let __process_pages_contig() to unlock it, 1258 * as it doesn't have any subpage::writers recorded. 1259 * 1260 * Here we manually unlock the page, since the caller 1261 * can't use page_started to determine if it's an 1262 * inline extent or a compressed extent. 1263 */ 1264 unlock_page(locked_page); 1265 goto out; 1266 } else if (ret < 0) { 1267 goto out_unlock; 1268 } 1269 } 1270 1271 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 1272 1273 /* 1274 * Relocation relies on the relocated extents to have exactly the same 1275 * size as the original extents. Normally writeback for relocation data 1276 * extents follows a NOCOW path because relocation preallocates the 1277 * extents. However, due to an operation such as scrub turning a block 1278 * group to RO mode, it may fallback to COW mode, so we must make sure 1279 * an extent allocated during COW has exactly the requested size and can 1280 * not be split into smaller extents, otherwise relocation breaks and 1281 * fails during the stage where it updates the bytenr of file extent 1282 * items. 1283 */ 1284 if (btrfs_is_data_reloc_root(root)) 1285 min_alloc_size = num_bytes; 1286 else 1287 min_alloc_size = fs_info->sectorsize; 1288 1289 while (num_bytes > 0) { 1290 cur_alloc_size = num_bytes; 1291 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size, 1292 min_alloc_size, 0, alloc_hint, 1293 &ins, 1, 1); 1294 if (ret < 0) 1295 goto out_unlock; 1296 cur_alloc_size = ins.offset; 1297 extent_reserved = true; 1298 1299 ram_size = ins.offset; 1300 em = create_io_em(inode, start, ins.offset, /* len */ 1301 start, /* orig_start */ 1302 ins.objectid, /* block_start */ 1303 ins.offset, /* block_len */ 1304 ins.offset, /* orig_block_len */ 1305 ram_size, /* ram_bytes */ 1306 BTRFS_COMPRESS_NONE, /* compress_type */ 1307 BTRFS_ORDERED_REGULAR /* type */); 1308 if (IS_ERR(em)) { 1309 ret = PTR_ERR(em); 1310 goto out_reserve; 1311 } 1312 free_extent_map(em); 1313 1314 ret = btrfs_add_ordered_extent(inode, start, ram_size, ram_size, 1315 ins.objectid, cur_alloc_size, 0, 1316 1 << BTRFS_ORDERED_REGULAR, 1317 BTRFS_COMPRESS_NONE); 1318 if (ret) 1319 goto out_drop_extent_cache; 1320 1321 if (btrfs_is_data_reloc_root(root)) { 1322 ret = btrfs_reloc_clone_csums(inode, start, 1323 cur_alloc_size); 1324 /* 1325 * Only drop cache here, and process as normal. 1326 * 1327 * We must not allow extent_clear_unlock_delalloc() 1328 * at out_unlock label to free meta of this ordered 1329 * extent, as its meta should be freed by 1330 * btrfs_finish_ordered_io(). 1331 * 1332 * So we must continue until @start is increased to 1333 * skip current ordered extent. 1334 */ 1335 if (ret) 1336 btrfs_drop_extent_map_range(inode, start, 1337 start + ram_size - 1, 1338 false); 1339 } 1340 1341 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1342 1343 /* 1344 * We're not doing compressed IO, don't unlock the first page 1345 * (which the caller expects to stay locked), don't clear any 1346 * dirty bits and don't set any writeback bits 1347 * 1348 * Do set the Ordered (Private2) bit so we know this page was 1349 * properly setup for writepage. 1350 */ 1351 page_ops = unlock ? PAGE_UNLOCK : 0; 1352 page_ops |= PAGE_SET_ORDERED; 1353 1354 extent_clear_unlock_delalloc(inode, start, start + ram_size - 1, 1355 locked_page, 1356 EXTENT_LOCKED | EXTENT_DELALLOC, 1357 page_ops); 1358 if (num_bytes < cur_alloc_size) 1359 num_bytes = 0; 1360 else 1361 num_bytes -= cur_alloc_size; 1362 alloc_hint = ins.objectid + ins.offset; 1363 start += cur_alloc_size; 1364 extent_reserved = false; 1365 1366 /* 1367 * btrfs_reloc_clone_csums() error, since start is increased 1368 * extent_clear_unlock_delalloc() at out_unlock label won't 1369 * free metadata of current ordered extent, we're OK to exit. 1370 */ 1371 if (ret) 1372 goto out_unlock; 1373 } 1374 out: 1375 return ret; 1376 1377 out_drop_extent_cache: 1378 btrfs_drop_extent_map_range(inode, start, start + ram_size - 1, false); 1379 out_reserve: 1380 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1381 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 1382 out_unlock: 1383 /* 1384 * If done_offset is non-NULL and ret == -EAGAIN, we expect the 1385 * caller to write out the successfully allocated region and retry. 1386 */ 1387 if (done_offset && ret == -EAGAIN) { 1388 if (orig_start < start) 1389 *done_offset = start - 1; 1390 else 1391 *done_offset = start; 1392 return ret; 1393 } else if (ret == -EAGAIN) { 1394 /* Convert to -ENOSPC since the caller cannot retry. */ 1395 ret = -ENOSPC; 1396 } 1397 1398 /* 1399 * Now, we have three regions to clean up: 1400 * 1401 * |-------(1)----|---(2)---|-------------(3)----------| 1402 * `- orig_start `- start `- start + cur_alloc_size `- end 1403 * 1404 * We process each region below. 1405 */ 1406 1407 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 1408 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; 1409 page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK; 1410 1411 /* 1412 * For the range (1). We have already instantiated the ordered extents 1413 * for this region. They are cleaned up by 1414 * btrfs_cleanup_ordered_extents() in e.g, 1415 * btrfs_run_delalloc_range(). EXTENT_LOCKED | EXTENT_DELALLOC are 1416 * already cleared in the above loop. And, EXTENT_DELALLOC_NEW | 1417 * EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV are handled by the cleanup 1418 * function. 1419 * 1420 * However, in case of unlock == 0, we still need to unlock the pages 1421 * (except @locked_page) to ensure all the pages are unlocked. 1422 */ 1423 if (!unlock && orig_start < start) { 1424 if (!locked_page) 1425 mapping_set_error(inode->vfs_inode.i_mapping, ret); 1426 extent_clear_unlock_delalloc(inode, orig_start, start - 1, 1427 locked_page, 0, page_ops); 1428 } 1429 1430 /* 1431 * For the range (2). If we reserved an extent for our delalloc range 1432 * (or a subrange) and failed to create the respective ordered extent, 1433 * then it means that when we reserved the extent we decremented the 1434 * extent's size from the data space_info's bytes_may_use counter and 1435 * incremented the space_info's bytes_reserved counter by the same 1436 * amount. We must make sure extent_clear_unlock_delalloc() does not try 1437 * to decrement again the data space_info's bytes_may_use counter, 1438 * therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV. 1439 */ 1440 if (extent_reserved) { 1441 extent_clear_unlock_delalloc(inode, start, 1442 start + cur_alloc_size - 1, 1443 locked_page, 1444 clear_bits, 1445 page_ops); 1446 start += cur_alloc_size; 1447 if (start >= end) 1448 return ret; 1449 } 1450 1451 /* 1452 * For the range (3). We never touched the region. In addition to the 1453 * clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data 1454 * space_info's bytes_may_use counter, reserved in 1455 * btrfs_check_data_free_space(). 1456 */ 1457 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1458 clear_bits | EXTENT_CLEAR_DATA_RESV, 1459 page_ops); 1460 return ret; 1461 } 1462 1463 /* 1464 * work queue call back to started compression on a file and pages 1465 */ 1466 static noinline void async_cow_start(struct btrfs_work *work) 1467 { 1468 struct async_chunk *async_chunk; 1469 int compressed_extents; 1470 1471 async_chunk = container_of(work, struct async_chunk, work); 1472 1473 compressed_extents = compress_file_range(async_chunk); 1474 if (compressed_extents == 0) { 1475 btrfs_add_delayed_iput(async_chunk->inode); 1476 async_chunk->inode = NULL; 1477 } 1478 } 1479 1480 /* 1481 * work queue call back to submit previously compressed pages 1482 */ 1483 static noinline void async_cow_submit(struct btrfs_work *work) 1484 { 1485 struct async_chunk *async_chunk = container_of(work, struct async_chunk, 1486 work); 1487 struct btrfs_fs_info *fs_info = btrfs_work_owner(work); 1488 unsigned long nr_pages; 1489 1490 nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >> 1491 PAGE_SHIFT; 1492 1493 /* 1494 * ->inode could be NULL if async_chunk_start has failed to compress, 1495 * in which case we don't have anything to submit, yet we need to 1496 * always adjust ->async_delalloc_pages as its paired with the init 1497 * happening in cow_file_range_async 1498 */ 1499 if (async_chunk->inode) 1500 submit_compressed_extents(async_chunk); 1501 1502 /* atomic_sub_return implies a barrier */ 1503 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1504 5 * SZ_1M) 1505 cond_wake_up_nomb(&fs_info->async_submit_wait); 1506 } 1507 1508 static noinline void async_cow_free(struct btrfs_work *work) 1509 { 1510 struct async_chunk *async_chunk; 1511 struct async_cow *async_cow; 1512 1513 async_chunk = container_of(work, struct async_chunk, work); 1514 if (async_chunk->inode) 1515 btrfs_add_delayed_iput(async_chunk->inode); 1516 if (async_chunk->blkcg_css) 1517 css_put(async_chunk->blkcg_css); 1518 1519 async_cow = async_chunk->async_cow; 1520 if (atomic_dec_and_test(&async_cow->num_chunks)) 1521 kvfree(async_cow); 1522 } 1523 1524 static int cow_file_range_async(struct btrfs_inode *inode, 1525 struct writeback_control *wbc, 1526 struct page *locked_page, 1527 u64 start, u64 end, int *page_started, 1528 unsigned long *nr_written) 1529 { 1530 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1531 struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc); 1532 struct async_cow *ctx; 1533 struct async_chunk *async_chunk; 1534 unsigned long nr_pages; 1535 u64 cur_end; 1536 u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K); 1537 int i; 1538 bool should_compress; 1539 unsigned nofs_flag; 1540 const blk_opf_t write_flags = wbc_to_write_flags(wbc); 1541 1542 unlock_extent(&inode->io_tree, start, end, NULL); 1543 1544 if (inode->flags & BTRFS_INODE_NOCOMPRESS && 1545 !btrfs_test_opt(fs_info, FORCE_COMPRESS)) { 1546 num_chunks = 1; 1547 should_compress = false; 1548 } else { 1549 should_compress = true; 1550 } 1551 1552 nofs_flag = memalloc_nofs_save(); 1553 ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL); 1554 memalloc_nofs_restore(nofs_flag); 1555 1556 if (!ctx) { 1557 unsigned clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | 1558 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | 1559 EXTENT_DO_ACCOUNTING; 1560 unsigned long page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | 1561 PAGE_END_WRITEBACK | PAGE_SET_ERROR; 1562 1563 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1564 clear_bits, page_ops); 1565 return -ENOMEM; 1566 } 1567 1568 async_chunk = ctx->chunks; 1569 atomic_set(&ctx->num_chunks, num_chunks); 1570 1571 for (i = 0; i < num_chunks; i++) { 1572 if (should_compress) 1573 cur_end = min(end, start + SZ_512K - 1); 1574 else 1575 cur_end = end; 1576 1577 /* 1578 * igrab is called higher up in the call chain, take only the 1579 * lightweight reference for the callback lifetime 1580 */ 1581 ihold(&inode->vfs_inode); 1582 async_chunk[i].async_cow = ctx; 1583 async_chunk[i].inode = inode; 1584 async_chunk[i].start = start; 1585 async_chunk[i].end = cur_end; 1586 async_chunk[i].write_flags = write_flags; 1587 INIT_LIST_HEAD(&async_chunk[i].extents); 1588 1589 /* 1590 * The locked_page comes all the way from writepage and its 1591 * the original page we were actually given. As we spread 1592 * this large delalloc region across multiple async_chunk 1593 * structs, only the first struct needs a pointer to locked_page 1594 * 1595 * This way we don't need racey decisions about who is supposed 1596 * to unlock it. 1597 */ 1598 if (locked_page) { 1599 /* 1600 * Depending on the compressibility, the pages might or 1601 * might not go through async. We want all of them to 1602 * be accounted against wbc once. Let's do it here 1603 * before the paths diverge. wbc accounting is used 1604 * only for foreign writeback detection and doesn't 1605 * need full accuracy. Just account the whole thing 1606 * against the first page. 1607 */ 1608 wbc_account_cgroup_owner(wbc, locked_page, 1609 cur_end - start); 1610 async_chunk[i].locked_page = locked_page; 1611 locked_page = NULL; 1612 } else { 1613 async_chunk[i].locked_page = NULL; 1614 } 1615 1616 if (blkcg_css != blkcg_root_css) { 1617 css_get(blkcg_css); 1618 async_chunk[i].blkcg_css = blkcg_css; 1619 async_chunk[i].write_flags |= REQ_BTRFS_CGROUP_PUNT; 1620 } else { 1621 async_chunk[i].blkcg_css = NULL; 1622 } 1623 1624 btrfs_init_work(&async_chunk[i].work, async_cow_start, 1625 async_cow_submit, async_cow_free); 1626 1627 nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE); 1628 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1629 1630 btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work); 1631 1632 *nr_written += nr_pages; 1633 start = cur_end + 1; 1634 } 1635 *page_started = 1; 1636 return 0; 1637 } 1638 1639 static noinline int run_delalloc_zoned(struct btrfs_inode *inode, 1640 struct page *locked_page, u64 start, 1641 u64 end, int *page_started, 1642 unsigned long *nr_written) 1643 { 1644 u64 done_offset = end; 1645 int ret; 1646 bool locked_page_done = false; 1647 1648 while (start <= end) { 1649 ret = cow_file_range(inode, locked_page, start, end, page_started, 1650 nr_written, 0, &done_offset); 1651 if (ret && ret != -EAGAIN) 1652 return ret; 1653 1654 if (*page_started) { 1655 ASSERT(ret == 0); 1656 return 0; 1657 } 1658 1659 if (ret == 0) 1660 done_offset = end; 1661 1662 if (done_offset == start) { 1663 wait_on_bit_io(&inode->root->fs_info->flags, 1664 BTRFS_FS_NEED_ZONE_FINISH, 1665 TASK_UNINTERRUPTIBLE); 1666 continue; 1667 } 1668 1669 if (!locked_page_done) { 1670 __set_page_dirty_nobuffers(locked_page); 1671 account_page_redirty(locked_page); 1672 } 1673 locked_page_done = true; 1674 extent_write_locked_range(&inode->vfs_inode, start, done_offset); 1675 1676 start = done_offset + 1; 1677 } 1678 1679 *page_started = 1; 1680 1681 return 0; 1682 } 1683 1684 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, 1685 u64 bytenr, u64 num_bytes, bool nowait) 1686 { 1687 struct btrfs_root *csum_root = btrfs_csum_root(fs_info, bytenr); 1688 struct btrfs_ordered_sum *sums; 1689 int ret; 1690 LIST_HEAD(list); 1691 1692 ret = btrfs_lookup_csums_list(csum_root, bytenr, bytenr + num_bytes - 1, 1693 &list, 0, nowait); 1694 if (ret == 0 && list_empty(&list)) 1695 return 0; 1696 1697 while (!list_empty(&list)) { 1698 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1699 list_del(&sums->list); 1700 kfree(sums); 1701 } 1702 if (ret < 0) 1703 return ret; 1704 return 1; 1705 } 1706 1707 static int fallback_to_cow(struct btrfs_inode *inode, struct page *locked_page, 1708 const u64 start, const u64 end, 1709 int *page_started, unsigned long *nr_written) 1710 { 1711 const bool is_space_ino = btrfs_is_free_space_inode(inode); 1712 const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root); 1713 const u64 range_bytes = end + 1 - start; 1714 struct extent_io_tree *io_tree = &inode->io_tree; 1715 u64 range_start = start; 1716 u64 count; 1717 1718 /* 1719 * If EXTENT_NORESERVE is set it means that when the buffered write was 1720 * made we had not enough available data space and therefore we did not 1721 * reserve data space for it, since we though we could do NOCOW for the 1722 * respective file range (either there is prealloc extent or the inode 1723 * has the NOCOW bit set). 1724 * 1725 * However when we need to fallback to COW mode (because for example the 1726 * block group for the corresponding extent was turned to RO mode by a 1727 * scrub or relocation) we need to do the following: 1728 * 1729 * 1) We increment the bytes_may_use counter of the data space info. 1730 * If COW succeeds, it allocates a new data extent and after doing 1731 * that it decrements the space info's bytes_may_use counter and 1732 * increments its bytes_reserved counter by the same amount (we do 1733 * this at btrfs_add_reserved_bytes()). So we need to increment the 1734 * bytes_may_use counter to compensate (when space is reserved at 1735 * buffered write time, the bytes_may_use counter is incremented); 1736 * 1737 * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so 1738 * that if the COW path fails for any reason, it decrements (through 1739 * extent_clear_unlock_delalloc()) the bytes_may_use counter of the 1740 * data space info, which we incremented in the step above. 1741 * 1742 * If we need to fallback to cow and the inode corresponds to a free 1743 * space cache inode or an inode of the data relocation tree, we must 1744 * also increment bytes_may_use of the data space_info for the same 1745 * reason. Space caches and relocated data extents always get a prealloc 1746 * extent for them, however scrub or balance may have set the block 1747 * group that contains that extent to RO mode and therefore force COW 1748 * when starting writeback. 1749 */ 1750 count = count_range_bits(io_tree, &range_start, end, range_bytes, 1751 EXTENT_NORESERVE, 0, NULL); 1752 if (count > 0 || is_space_ino || is_reloc_ino) { 1753 u64 bytes = count; 1754 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1755 struct btrfs_space_info *sinfo = fs_info->data_sinfo; 1756 1757 if (is_space_ino || is_reloc_ino) 1758 bytes = range_bytes; 1759 1760 spin_lock(&sinfo->lock); 1761 btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes); 1762 spin_unlock(&sinfo->lock); 1763 1764 if (count > 0) 1765 clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, 1766 NULL); 1767 } 1768 1769 return cow_file_range(inode, locked_page, start, end, page_started, 1770 nr_written, 1, NULL); 1771 } 1772 1773 struct can_nocow_file_extent_args { 1774 /* Input fields. */ 1775 1776 /* Start file offset of the range we want to NOCOW. */ 1777 u64 start; 1778 /* End file offset (inclusive) of the range we want to NOCOW. */ 1779 u64 end; 1780 bool writeback_path; 1781 bool strict; 1782 /* 1783 * Free the path passed to can_nocow_file_extent() once it's not needed 1784 * anymore. 1785 */ 1786 bool free_path; 1787 1788 /* Output fields. Only set when can_nocow_file_extent() returns 1. */ 1789 1790 u64 disk_bytenr; 1791 u64 disk_num_bytes; 1792 u64 extent_offset; 1793 /* Number of bytes that can be written to in NOCOW mode. */ 1794 u64 num_bytes; 1795 }; 1796 1797 /* 1798 * Check if we can NOCOW the file extent that the path points to. 1799 * This function may return with the path released, so the caller should check 1800 * if path->nodes[0] is NULL or not if it needs to use the path afterwards. 1801 * 1802 * Returns: < 0 on error 1803 * 0 if we can not NOCOW 1804 * 1 if we can NOCOW 1805 */ 1806 static int can_nocow_file_extent(struct btrfs_path *path, 1807 struct btrfs_key *key, 1808 struct btrfs_inode *inode, 1809 struct can_nocow_file_extent_args *args) 1810 { 1811 const bool is_freespace_inode = btrfs_is_free_space_inode(inode); 1812 struct extent_buffer *leaf = path->nodes[0]; 1813 struct btrfs_root *root = inode->root; 1814 struct btrfs_file_extent_item *fi; 1815 u64 extent_end; 1816 u8 extent_type; 1817 int can_nocow = 0; 1818 int ret = 0; 1819 bool nowait = path->nowait; 1820 1821 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); 1822 extent_type = btrfs_file_extent_type(leaf, fi); 1823 1824 if (extent_type == BTRFS_FILE_EXTENT_INLINE) 1825 goto out; 1826 1827 /* Can't access these fields unless we know it's not an inline extent. */ 1828 args->disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1829 args->disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); 1830 args->extent_offset = btrfs_file_extent_offset(leaf, fi); 1831 1832 if (!(inode->flags & BTRFS_INODE_NODATACOW) && 1833 extent_type == BTRFS_FILE_EXTENT_REG) 1834 goto out; 1835 1836 /* 1837 * If the extent was created before the generation where the last snapshot 1838 * for its subvolume was created, then this implies the extent is shared, 1839 * hence we must COW. 1840 */ 1841 if (!args->strict && 1842 btrfs_file_extent_generation(leaf, fi) <= 1843 btrfs_root_last_snapshot(&root->root_item)) 1844 goto out; 1845 1846 /* An explicit hole, must COW. */ 1847 if (args->disk_bytenr == 0) 1848 goto out; 1849 1850 /* Compressed/encrypted/encoded extents must be COWed. */ 1851 if (btrfs_file_extent_compression(leaf, fi) || 1852 btrfs_file_extent_encryption(leaf, fi) || 1853 btrfs_file_extent_other_encoding(leaf, fi)) 1854 goto out; 1855 1856 extent_end = btrfs_file_extent_end(path); 1857 1858 /* 1859 * The following checks can be expensive, as they need to take other 1860 * locks and do btree or rbtree searches, so release the path to avoid 1861 * blocking other tasks for too long. 1862 */ 1863 btrfs_release_path(path); 1864 1865 ret = btrfs_cross_ref_exist(root, btrfs_ino(inode), 1866 key->offset - args->extent_offset, 1867 args->disk_bytenr, false, path); 1868 WARN_ON_ONCE(ret > 0 && is_freespace_inode); 1869 if (ret != 0) 1870 goto out; 1871 1872 if (args->free_path) { 1873 /* 1874 * We don't need the path anymore, plus through the 1875 * csum_exist_in_range() call below we will end up allocating 1876 * another path. So free the path to avoid unnecessary extra 1877 * memory usage. 1878 */ 1879 btrfs_free_path(path); 1880 path = NULL; 1881 } 1882 1883 /* If there are pending snapshots for this root, we must COW. */ 1884 if (args->writeback_path && !is_freespace_inode && 1885 atomic_read(&root->snapshot_force_cow)) 1886 goto out; 1887 1888 args->disk_bytenr += args->extent_offset; 1889 args->disk_bytenr += args->start - key->offset; 1890 args->num_bytes = min(args->end + 1, extent_end) - args->start; 1891 1892 /* 1893 * Force COW if csums exist in the range. This ensures that csums for a 1894 * given extent are either valid or do not exist. 1895 */ 1896 ret = csum_exist_in_range(root->fs_info, args->disk_bytenr, args->num_bytes, 1897 nowait); 1898 WARN_ON_ONCE(ret > 0 && is_freespace_inode); 1899 if (ret != 0) 1900 goto out; 1901 1902 can_nocow = 1; 1903 out: 1904 if (args->free_path && path) 1905 btrfs_free_path(path); 1906 1907 return ret < 0 ? ret : can_nocow; 1908 } 1909 1910 /* 1911 * when nowcow writeback call back. This checks for snapshots or COW copies 1912 * of the extents that exist in the file, and COWs the file as required. 1913 * 1914 * If no cow copies or snapshots exist, we write directly to the existing 1915 * blocks on disk 1916 */ 1917 static noinline int run_delalloc_nocow(struct btrfs_inode *inode, 1918 struct page *locked_page, 1919 const u64 start, const u64 end, 1920 int *page_started, 1921 unsigned long *nr_written) 1922 { 1923 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1924 struct btrfs_root *root = inode->root; 1925 struct btrfs_path *path; 1926 u64 cow_start = (u64)-1; 1927 u64 cur_offset = start; 1928 int ret; 1929 bool check_prev = true; 1930 u64 ino = btrfs_ino(inode); 1931 struct btrfs_block_group *bg; 1932 bool nocow = false; 1933 struct can_nocow_file_extent_args nocow_args = { 0 }; 1934 1935 path = btrfs_alloc_path(); 1936 if (!path) { 1937 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1938 EXTENT_LOCKED | EXTENT_DELALLOC | 1939 EXTENT_DO_ACCOUNTING | 1940 EXTENT_DEFRAG, PAGE_UNLOCK | 1941 PAGE_START_WRITEBACK | 1942 PAGE_END_WRITEBACK); 1943 return -ENOMEM; 1944 } 1945 1946 nocow_args.end = end; 1947 nocow_args.writeback_path = true; 1948 1949 while (1) { 1950 struct btrfs_key found_key; 1951 struct btrfs_file_extent_item *fi; 1952 struct extent_buffer *leaf; 1953 u64 extent_end; 1954 u64 ram_bytes; 1955 u64 nocow_end; 1956 int extent_type; 1957 1958 nocow = false; 1959 1960 ret = btrfs_lookup_file_extent(NULL, root, path, ino, 1961 cur_offset, 0); 1962 if (ret < 0) 1963 goto error; 1964 1965 /* 1966 * If there is no extent for our range when doing the initial 1967 * search, then go back to the previous slot as it will be the 1968 * one containing the search offset 1969 */ 1970 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1971 leaf = path->nodes[0]; 1972 btrfs_item_key_to_cpu(leaf, &found_key, 1973 path->slots[0] - 1); 1974 if (found_key.objectid == ino && 1975 found_key.type == BTRFS_EXTENT_DATA_KEY) 1976 path->slots[0]--; 1977 } 1978 check_prev = false; 1979 next_slot: 1980 /* Go to next leaf if we have exhausted the current one */ 1981 leaf = path->nodes[0]; 1982 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1983 ret = btrfs_next_leaf(root, path); 1984 if (ret < 0) { 1985 if (cow_start != (u64)-1) 1986 cur_offset = cow_start; 1987 goto error; 1988 } 1989 if (ret > 0) 1990 break; 1991 leaf = path->nodes[0]; 1992 } 1993 1994 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1995 1996 /* Didn't find anything for our INO */ 1997 if (found_key.objectid > ino) 1998 break; 1999 /* 2000 * Keep searching until we find an EXTENT_ITEM or there are no 2001 * more extents for this inode 2002 */ 2003 if (WARN_ON_ONCE(found_key.objectid < ino) || 2004 found_key.type < BTRFS_EXTENT_DATA_KEY) { 2005 path->slots[0]++; 2006 goto next_slot; 2007 } 2008 2009 /* Found key is not EXTENT_DATA_KEY or starts after req range */ 2010 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 2011 found_key.offset > end) 2012 break; 2013 2014 /* 2015 * If the found extent starts after requested offset, then 2016 * adjust extent_end to be right before this extent begins 2017 */ 2018 if (found_key.offset > cur_offset) { 2019 extent_end = found_key.offset; 2020 extent_type = 0; 2021 goto out_check; 2022 } 2023 2024 /* 2025 * Found extent which begins before our range and potentially 2026 * intersect it 2027 */ 2028 fi = btrfs_item_ptr(leaf, path->slots[0], 2029 struct btrfs_file_extent_item); 2030 extent_type = btrfs_file_extent_type(leaf, fi); 2031 /* If this is triggered then we have a memory corruption. */ 2032 ASSERT(extent_type < BTRFS_NR_FILE_EXTENT_TYPES); 2033 if (WARN_ON(extent_type >= BTRFS_NR_FILE_EXTENT_TYPES)) { 2034 ret = -EUCLEAN; 2035 goto error; 2036 } 2037 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 2038 extent_end = btrfs_file_extent_end(path); 2039 2040 /* 2041 * If the extent we got ends before our current offset, skip to 2042 * the next extent. 2043 */ 2044 if (extent_end <= cur_offset) { 2045 path->slots[0]++; 2046 goto next_slot; 2047 } 2048 2049 nocow_args.start = cur_offset; 2050 ret = can_nocow_file_extent(path, &found_key, inode, &nocow_args); 2051 if (ret < 0) { 2052 if (cow_start != (u64)-1) 2053 cur_offset = cow_start; 2054 goto error; 2055 } else if (ret == 0) { 2056 goto out_check; 2057 } 2058 2059 ret = 0; 2060 bg = btrfs_inc_nocow_writers(fs_info, nocow_args.disk_bytenr); 2061 if (bg) 2062 nocow = true; 2063 out_check: 2064 /* 2065 * If nocow is false then record the beginning of the range 2066 * that needs to be COWed 2067 */ 2068 if (!nocow) { 2069 if (cow_start == (u64)-1) 2070 cow_start = cur_offset; 2071 cur_offset = extent_end; 2072 if (cur_offset > end) 2073 break; 2074 if (!path->nodes[0]) 2075 continue; 2076 path->slots[0]++; 2077 goto next_slot; 2078 } 2079 2080 /* 2081 * COW range from cow_start to found_key.offset - 1. As the key 2082 * will contain the beginning of the first extent that can be 2083 * NOCOW, following one which needs to be COW'ed 2084 */ 2085 if (cow_start != (u64)-1) { 2086 ret = fallback_to_cow(inode, locked_page, 2087 cow_start, found_key.offset - 1, 2088 page_started, nr_written); 2089 if (ret) 2090 goto error; 2091 cow_start = (u64)-1; 2092 } 2093 2094 nocow_end = cur_offset + nocow_args.num_bytes - 1; 2095 2096 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 2097 u64 orig_start = found_key.offset - nocow_args.extent_offset; 2098 struct extent_map *em; 2099 2100 em = create_io_em(inode, cur_offset, nocow_args.num_bytes, 2101 orig_start, 2102 nocow_args.disk_bytenr, /* block_start */ 2103 nocow_args.num_bytes, /* block_len */ 2104 nocow_args.disk_num_bytes, /* orig_block_len */ 2105 ram_bytes, BTRFS_COMPRESS_NONE, 2106 BTRFS_ORDERED_PREALLOC); 2107 if (IS_ERR(em)) { 2108 ret = PTR_ERR(em); 2109 goto error; 2110 } 2111 free_extent_map(em); 2112 ret = btrfs_add_ordered_extent(inode, 2113 cur_offset, nocow_args.num_bytes, 2114 nocow_args.num_bytes, 2115 nocow_args.disk_bytenr, 2116 nocow_args.num_bytes, 0, 2117 1 << BTRFS_ORDERED_PREALLOC, 2118 BTRFS_COMPRESS_NONE); 2119 if (ret) { 2120 btrfs_drop_extent_map_range(inode, cur_offset, 2121 nocow_end, false); 2122 goto error; 2123 } 2124 } else { 2125 ret = btrfs_add_ordered_extent(inode, cur_offset, 2126 nocow_args.num_bytes, 2127 nocow_args.num_bytes, 2128 nocow_args.disk_bytenr, 2129 nocow_args.num_bytes, 2130 0, 2131 1 << BTRFS_ORDERED_NOCOW, 2132 BTRFS_COMPRESS_NONE); 2133 if (ret) 2134 goto error; 2135 } 2136 2137 if (nocow) { 2138 btrfs_dec_nocow_writers(bg); 2139 nocow = false; 2140 } 2141 2142 if (btrfs_is_data_reloc_root(root)) 2143 /* 2144 * Error handled later, as we must prevent 2145 * extent_clear_unlock_delalloc() in error handler 2146 * from freeing metadata of created ordered extent. 2147 */ 2148 ret = btrfs_reloc_clone_csums(inode, cur_offset, 2149 nocow_args.num_bytes); 2150 2151 extent_clear_unlock_delalloc(inode, cur_offset, nocow_end, 2152 locked_page, EXTENT_LOCKED | 2153 EXTENT_DELALLOC | 2154 EXTENT_CLEAR_DATA_RESV, 2155 PAGE_UNLOCK | PAGE_SET_ORDERED); 2156 2157 cur_offset = extent_end; 2158 2159 /* 2160 * btrfs_reloc_clone_csums() error, now we're OK to call error 2161 * handler, as metadata for created ordered extent will only 2162 * be freed by btrfs_finish_ordered_io(). 2163 */ 2164 if (ret) 2165 goto error; 2166 if (cur_offset > end) 2167 break; 2168 } 2169 btrfs_release_path(path); 2170 2171 if (cur_offset <= end && cow_start == (u64)-1) 2172 cow_start = cur_offset; 2173 2174 if (cow_start != (u64)-1) { 2175 cur_offset = end; 2176 ret = fallback_to_cow(inode, locked_page, cow_start, end, 2177 page_started, nr_written); 2178 if (ret) 2179 goto error; 2180 } 2181 2182 error: 2183 if (nocow) 2184 btrfs_dec_nocow_writers(bg); 2185 2186 if (ret && cur_offset < end) 2187 extent_clear_unlock_delalloc(inode, cur_offset, end, 2188 locked_page, EXTENT_LOCKED | 2189 EXTENT_DELALLOC | EXTENT_DEFRAG | 2190 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 2191 PAGE_START_WRITEBACK | 2192 PAGE_END_WRITEBACK); 2193 btrfs_free_path(path); 2194 return ret; 2195 } 2196 2197 static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end) 2198 { 2199 if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) { 2200 if (inode->defrag_bytes && 2201 test_range_bit(&inode->io_tree, start, end, EXTENT_DEFRAG, 2202 0, NULL)) 2203 return false; 2204 return true; 2205 } 2206 return false; 2207 } 2208 2209 /* 2210 * Function to process delayed allocation (create CoW) for ranges which are 2211 * being touched for the first time. 2212 */ 2213 int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct page *locked_page, 2214 u64 start, u64 end, int *page_started, unsigned long *nr_written, 2215 struct writeback_control *wbc) 2216 { 2217 int ret; 2218 const bool zoned = btrfs_is_zoned(inode->root->fs_info); 2219 2220 /* 2221 * The range must cover part of the @locked_page, or the returned 2222 * @page_started can confuse the caller. 2223 */ 2224 ASSERT(!(end <= page_offset(locked_page) || 2225 start >= page_offset(locked_page) + PAGE_SIZE)); 2226 2227 if (should_nocow(inode, start, end)) { 2228 /* 2229 * Normally on a zoned device we're only doing COW writes, but 2230 * in case of relocation on a zoned filesystem we have taken 2231 * precaution, that we're only writing sequentially. It's safe 2232 * to use run_delalloc_nocow() here, like for regular 2233 * preallocated inodes. 2234 */ 2235 ASSERT(!zoned || btrfs_is_data_reloc_root(inode->root)); 2236 ret = run_delalloc_nocow(inode, locked_page, start, end, 2237 page_started, nr_written); 2238 } else if (!btrfs_inode_can_compress(inode) || 2239 !inode_need_compress(inode, start, end)) { 2240 if (zoned) 2241 ret = run_delalloc_zoned(inode, locked_page, start, end, 2242 page_started, nr_written); 2243 else 2244 ret = cow_file_range(inode, locked_page, start, end, 2245 page_started, nr_written, 1, NULL); 2246 } else { 2247 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags); 2248 ret = cow_file_range_async(inode, wbc, locked_page, start, end, 2249 page_started, nr_written); 2250 } 2251 ASSERT(ret <= 0); 2252 if (ret) 2253 btrfs_cleanup_ordered_extents(inode, locked_page, start, 2254 end - start + 1); 2255 return ret; 2256 } 2257 2258 void btrfs_split_delalloc_extent(struct btrfs_inode *inode, 2259 struct extent_state *orig, u64 split) 2260 { 2261 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2262 u64 size; 2263 2264 /* not delalloc, ignore it */ 2265 if (!(orig->state & EXTENT_DELALLOC)) 2266 return; 2267 2268 size = orig->end - orig->start + 1; 2269 if (size > fs_info->max_extent_size) { 2270 u32 num_extents; 2271 u64 new_size; 2272 2273 /* 2274 * See the explanation in btrfs_merge_delalloc_extent, the same 2275 * applies here, just in reverse. 2276 */ 2277 new_size = orig->end - split + 1; 2278 num_extents = count_max_extents(fs_info, new_size); 2279 new_size = split - orig->start; 2280 num_extents += count_max_extents(fs_info, new_size); 2281 if (count_max_extents(fs_info, size) >= num_extents) 2282 return; 2283 } 2284 2285 spin_lock(&inode->lock); 2286 btrfs_mod_outstanding_extents(inode, 1); 2287 spin_unlock(&inode->lock); 2288 } 2289 2290 /* 2291 * Handle merged delayed allocation extents so we can keep track of new extents 2292 * that are just merged onto old extents, such as when we are doing sequential 2293 * writes, so we can properly account for the metadata space we'll need. 2294 */ 2295 void btrfs_merge_delalloc_extent(struct btrfs_inode *inode, struct extent_state *new, 2296 struct extent_state *other) 2297 { 2298 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2299 u64 new_size, old_size; 2300 u32 num_extents; 2301 2302 /* not delalloc, ignore it */ 2303 if (!(other->state & EXTENT_DELALLOC)) 2304 return; 2305 2306 if (new->start > other->start) 2307 new_size = new->end - other->start + 1; 2308 else 2309 new_size = other->end - new->start + 1; 2310 2311 /* we're not bigger than the max, unreserve the space and go */ 2312 if (new_size <= fs_info->max_extent_size) { 2313 spin_lock(&inode->lock); 2314 btrfs_mod_outstanding_extents(inode, -1); 2315 spin_unlock(&inode->lock); 2316 return; 2317 } 2318 2319 /* 2320 * We have to add up either side to figure out how many extents were 2321 * accounted for before we merged into one big extent. If the number of 2322 * extents we accounted for is <= the amount we need for the new range 2323 * then we can return, otherwise drop. Think of it like this 2324 * 2325 * [ 4k][MAX_SIZE] 2326 * 2327 * So we've grown the extent by a MAX_SIZE extent, this would mean we 2328 * need 2 outstanding extents, on one side we have 1 and the other side 2329 * we have 1 so they are == and we can return. But in this case 2330 * 2331 * [MAX_SIZE+4k][MAX_SIZE+4k] 2332 * 2333 * Each range on their own accounts for 2 extents, but merged together 2334 * they are only 3 extents worth of accounting, so we need to drop in 2335 * this case. 2336 */ 2337 old_size = other->end - other->start + 1; 2338 num_extents = count_max_extents(fs_info, old_size); 2339 old_size = new->end - new->start + 1; 2340 num_extents += count_max_extents(fs_info, old_size); 2341 if (count_max_extents(fs_info, new_size) >= num_extents) 2342 return; 2343 2344 spin_lock(&inode->lock); 2345 btrfs_mod_outstanding_extents(inode, -1); 2346 spin_unlock(&inode->lock); 2347 } 2348 2349 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 2350 struct btrfs_inode *inode) 2351 { 2352 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2353 2354 spin_lock(&root->delalloc_lock); 2355 if (list_empty(&inode->delalloc_inodes)) { 2356 list_add_tail(&inode->delalloc_inodes, &root->delalloc_inodes); 2357 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, &inode->runtime_flags); 2358 root->nr_delalloc_inodes++; 2359 if (root->nr_delalloc_inodes == 1) { 2360 spin_lock(&fs_info->delalloc_root_lock); 2361 BUG_ON(!list_empty(&root->delalloc_root)); 2362 list_add_tail(&root->delalloc_root, 2363 &fs_info->delalloc_roots); 2364 spin_unlock(&fs_info->delalloc_root_lock); 2365 } 2366 } 2367 spin_unlock(&root->delalloc_lock); 2368 } 2369 2370 void __btrfs_del_delalloc_inode(struct btrfs_root *root, 2371 struct btrfs_inode *inode) 2372 { 2373 struct btrfs_fs_info *fs_info = root->fs_info; 2374 2375 if (!list_empty(&inode->delalloc_inodes)) { 2376 list_del_init(&inode->delalloc_inodes); 2377 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2378 &inode->runtime_flags); 2379 root->nr_delalloc_inodes--; 2380 if (!root->nr_delalloc_inodes) { 2381 ASSERT(list_empty(&root->delalloc_inodes)); 2382 spin_lock(&fs_info->delalloc_root_lock); 2383 BUG_ON(list_empty(&root->delalloc_root)); 2384 list_del_init(&root->delalloc_root); 2385 spin_unlock(&fs_info->delalloc_root_lock); 2386 } 2387 } 2388 } 2389 2390 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 2391 struct btrfs_inode *inode) 2392 { 2393 spin_lock(&root->delalloc_lock); 2394 __btrfs_del_delalloc_inode(root, inode); 2395 spin_unlock(&root->delalloc_lock); 2396 } 2397 2398 /* 2399 * Properly track delayed allocation bytes in the inode and to maintain the 2400 * list of inodes that have pending delalloc work to be done. 2401 */ 2402 void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state, 2403 u32 bits) 2404 { 2405 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2406 2407 if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC)) 2408 WARN_ON(1); 2409 /* 2410 * set_bit and clear bit hooks normally require _irqsave/restore 2411 * but in this case, we are only testing for the DELALLOC 2412 * bit, which is only set or cleared with irqs on 2413 */ 2414 if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) { 2415 struct btrfs_root *root = inode->root; 2416 u64 len = state->end + 1 - state->start; 2417 u32 num_extents = count_max_extents(fs_info, len); 2418 bool do_list = !btrfs_is_free_space_inode(inode); 2419 2420 spin_lock(&inode->lock); 2421 btrfs_mod_outstanding_extents(inode, num_extents); 2422 spin_unlock(&inode->lock); 2423 2424 /* For sanity tests */ 2425 if (btrfs_is_testing(fs_info)) 2426 return; 2427 2428 percpu_counter_add_batch(&fs_info->delalloc_bytes, len, 2429 fs_info->delalloc_batch); 2430 spin_lock(&inode->lock); 2431 inode->delalloc_bytes += len; 2432 if (bits & EXTENT_DEFRAG) 2433 inode->defrag_bytes += len; 2434 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2435 &inode->runtime_flags)) 2436 btrfs_add_delalloc_inodes(root, inode); 2437 spin_unlock(&inode->lock); 2438 } 2439 2440 if (!(state->state & EXTENT_DELALLOC_NEW) && 2441 (bits & EXTENT_DELALLOC_NEW)) { 2442 spin_lock(&inode->lock); 2443 inode->new_delalloc_bytes += state->end + 1 - state->start; 2444 spin_unlock(&inode->lock); 2445 } 2446 } 2447 2448 /* 2449 * Once a range is no longer delalloc this function ensures that proper 2450 * accounting happens. 2451 */ 2452 void btrfs_clear_delalloc_extent(struct btrfs_inode *inode, 2453 struct extent_state *state, u32 bits) 2454 { 2455 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2456 u64 len = state->end + 1 - state->start; 2457 u32 num_extents = count_max_extents(fs_info, len); 2458 2459 if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) { 2460 spin_lock(&inode->lock); 2461 inode->defrag_bytes -= len; 2462 spin_unlock(&inode->lock); 2463 } 2464 2465 /* 2466 * set_bit and clear bit hooks normally require _irqsave/restore 2467 * but in this case, we are only testing for the DELALLOC 2468 * bit, which is only set or cleared with irqs on 2469 */ 2470 if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) { 2471 struct btrfs_root *root = inode->root; 2472 bool do_list = !btrfs_is_free_space_inode(inode); 2473 2474 spin_lock(&inode->lock); 2475 btrfs_mod_outstanding_extents(inode, -num_extents); 2476 spin_unlock(&inode->lock); 2477 2478 /* 2479 * We don't reserve metadata space for space cache inodes so we 2480 * don't need to call delalloc_release_metadata if there is an 2481 * error. 2482 */ 2483 if (bits & EXTENT_CLEAR_META_RESV && 2484 root != fs_info->tree_root) 2485 btrfs_delalloc_release_metadata(inode, len, false); 2486 2487 /* For sanity tests. */ 2488 if (btrfs_is_testing(fs_info)) 2489 return; 2490 2491 if (!btrfs_is_data_reloc_root(root) && 2492 do_list && !(state->state & EXTENT_NORESERVE) && 2493 (bits & EXTENT_CLEAR_DATA_RESV)) 2494 btrfs_free_reserved_data_space_noquota(fs_info, len); 2495 2496 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, 2497 fs_info->delalloc_batch); 2498 spin_lock(&inode->lock); 2499 inode->delalloc_bytes -= len; 2500 if (do_list && inode->delalloc_bytes == 0 && 2501 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 2502 &inode->runtime_flags)) 2503 btrfs_del_delalloc_inode(root, inode); 2504 spin_unlock(&inode->lock); 2505 } 2506 2507 if ((state->state & EXTENT_DELALLOC_NEW) && 2508 (bits & EXTENT_DELALLOC_NEW)) { 2509 spin_lock(&inode->lock); 2510 ASSERT(inode->new_delalloc_bytes >= len); 2511 inode->new_delalloc_bytes -= len; 2512 if (bits & EXTENT_ADD_INODE_BYTES) 2513 inode_add_bytes(&inode->vfs_inode, len); 2514 spin_unlock(&inode->lock); 2515 } 2516 } 2517 2518 /* 2519 * Split off the first pre bytes from the extent_map at [start, start + len] 2520 * 2521 * This function is intended to be used only for extract_ordered_extent(). 2522 */ 2523 static int split_extent_map(struct btrfs_inode *inode, u64 start, u64 len, u64 pre) 2524 { 2525 struct extent_map_tree *em_tree = &inode->extent_tree; 2526 struct extent_map *em; 2527 struct extent_map *split_pre = NULL; 2528 struct extent_map *split_mid = NULL; 2529 int ret = 0; 2530 unsigned long flags; 2531 2532 ASSERT(pre != 0); 2533 ASSERT(pre < len); 2534 2535 split_pre = alloc_extent_map(); 2536 if (!split_pre) 2537 return -ENOMEM; 2538 split_mid = alloc_extent_map(); 2539 if (!split_mid) { 2540 ret = -ENOMEM; 2541 goto out_free_pre; 2542 } 2543 2544 lock_extent(&inode->io_tree, start, start + len - 1, NULL); 2545 write_lock(&em_tree->lock); 2546 em = lookup_extent_mapping(em_tree, start, len); 2547 if (!em) { 2548 ret = -EIO; 2549 goto out_unlock; 2550 } 2551 2552 ASSERT(em->len == len); 2553 ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)); 2554 ASSERT(em->block_start < EXTENT_MAP_LAST_BYTE); 2555 ASSERT(test_bit(EXTENT_FLAG_PINNED, &em->flags)); 2556 ASSERT(!test_bit(EXTENT_FLAG_LOGGING, &em->flags)); 2557 ASSERT(!list_empty(&em->list)); 2558 2559 flags = em->flags; 2560 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 2561 2562 /* First, replace the em with a new extent_map starting from * em->start */ 2563 split_pre->start = em->start; 2564 split_pre->len = pre; 2565 split_pre->orig_start = split_pre->start; 2566 split_pre->block_start = em->block_start; 2567 split_pre->block_len = split_pre->len; 2568 split_pre->orig_block_len = split_pre->block_len; 2569 split_pre->ram_bytes = split_pre->len; 2570 split_pre->flags = flags; 2571 split_pre->compress_type = em->compress_type; 2572 split_pre->generation = em->generation; 2573 2574 replace_extent_mapping(em_tree, em, split_pre, 1); 2575 2576 /* 2577 * Now we only have an extent_map at: 2578 * [em->start, em->start + pre] 2579 */ 2580 2581 /* Insert the middle extent_map. */ 2582 split_mid->start = em->start + pre; 2583 split_mid->len = em->len - pre; 2584 split_mid->orig_start = split_mid->start; 2585 split_mid->block_start = em->block_start + pre; 2586 split_mid->block_len = split_mid->len; 2587 split_mid->orig_block_len = split_mid->block_len; 2588 split_mid->ram_bytes = split_mid->len; 2589 split_mid->flags = flags; 2590 split_mid->compress_type = em->compress_type; 2591 split_mid->generation = em->generation; 2592 add_extent_mapping(em_tree, split_mid, 1); 2593 2594 /* Once for us */ 2595 free_extent_map(em); 2596 /* Once for the tree */ 2597 free_extent_map(em); 2598 2599 out_unlock: 2600 write_unlock(&em_tree->lock); 2601 unlock_extent(&inode->io_tree, start, start + len - 1, NULL); 2602 free_extent_map(split_mid); 2603 out_free_pre: 2604 free_extent_map(split_pre); 2605 return ret; 2606 } 2607 2608 int btrfs_extract_ordered_extent(struct btrfs_bio *bbio, 2609 struct btrfs_ordered_extent *ordered) 2610 { 2611 u64 start = (u64)bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; 2612 u64 len = bbio->bio.bi_iter.bi_size; 2613 struct btrfs_inode *inode = bbio->inode; 2614 u64 ordered_len = ordered->num_bytes; 2615 int ret = 0; 2616 2617 /* Must always be called for the beginning of an ordered extent. */ 2618 if (WARN_ON_ONCE(start != ordered->disk_bytenr)) 2619 return -EINVAL; 2620 2621 /* No need to split if the ordered extent covers the entire bio. */ 2622 if (ordered->disk_num_bytes == len) 2623 return 0; 2624 2625 ret = btrfs_split_ordered_extent(ordered, len); 2626 if (ret) 2627 return ret; 2628 2629 /* 2630 * Don't split the extent_map for NOCOW extents, as we're writing into 2631 * a pre-existing one. 2632 */ 2633 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) 2634 return 0; 2635 2636 return split_extent_map(inode, bbio->file_offset, ordered_len, len); 2637 } 2638 2639 /* 2640 * given a list of ordered sums record them in the inode. This happens 2641 * at IO completion time based on sums calculated at bio submission time. 2642 */ 2643 static int add_pending_csums(struct btrfs_trans_handle *trans, 2644 struct list_head *list) 2645 { 2646 struct btrfs_ordered_sum *sum; 2647 struct btrfs_root *csum_root = NULL; 2648 int ret; 2649 2650 list_for_each_entry(sum, list, list) { 2651 trans->adding_csums = true; 2652 if (!csum_root) 2653 csum_root = btrfs_csum_root(trans->fs_info, 2654 sum->bytenr); 2655 ret = btrfs_csum_file_blocks(trans, csum_root, sum); 2656 trans->adding_csums = false; 2657 if (ret) 2658 return ret; 2659 } 2660 return 0; 2661 } 2662 2663 static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode, 2664 const u64 start, 2665 const u64 len, 2666 struct extent_state **cached_state) 2667 { 2668 u64 search_start = start; 2669 const u64 end = start + len - 1; 2670 2671 while (search_start < end) { 2672 const u64 search_len = end - search_start + 1; 2673 struct extent_map *em; 2674 u64 em_len; 2675 int ret = 0; 2676 2677 em = btrfs_get_extent(inode, NULL, 0, search_start, search_len); 2678 if (IS_ERR(em)) 2679 return PTR_ERR(em); 2680 2681 if (em->block_start != EXTENT_MAP_HOLE) 2682 goto next; 2683 2684 em_len = em->len; 2685 if (em->start < search_start) 2686 em_len -= search_start - em->start; 2687 if (em_len > search_len) 2688 em_len = search_len; 2689 2690 ret = set_extent_bit(&inode->io_tree, search_start, 2691 search_start + em_len - 1, 2692 EXTENT_DELALLOC_NEW, cached_state, 2693 GFP_NOFS); 2694 next: 2695 search_start = extent_map_end(em); 2696 free_extent_map(em); 2697 if (ret) 2698 return ret; 2699 } 2700 return 0; 2701 } 2702 2703 int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, 2704 unsigned int extra_bits, 2705 struct extent_state **cached_state) 2706 { 2707 WARN_ON(PAGE_ALIGNED(end)); 2708 2709 if (start >= i_size_read(&inode->vfs_inode) && 2710 !(inode->flags & BTRFS_INODE_PREALLOC)) { 2711 /* 2712 * There can't be any extents following eof in this case so just 2713 * set the delalloc new bit for the range directly. 2714 */ 2715 extra_bits |= EXTENT_DELALLOC_NEW; 2716 } else { 2717 int ret; 2718 2719 ret = btrfs_find_new_delalloc_bytes(inode, start, 2720 end + 1 - start, 2721 cached_state); 2722 if (ret) 2723 return ret; 2724 } 2725 2726 return set_extent_delalloc(&inode->io_tree, start, end, extra_bits, 2727 cached_state); 2728 } 2729 2730 /* see btrfs_writepage_start_hook for details on why this is required */ 2731 struct btrfs_writepage_fixup { 2732 struct page *page; 2733 struct btrfs_inode *inode; 2734 struct btrfs_work work; 2735 }; 2736 2737 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2738 { 2739 struct btrfs_writepage_fixup *fixup; 2740 struct btrfs_ordered_extent *ordered; 2741 struct extent_state *cached_state = NULL; 2742 struct extent_changeset *data_reserved = NULL; 2743 struct page *page; 2744 struct btrfs_inode *inode; 2745 u64 page_start; 2746 u64 page_end; 2747 int ret = 0; 2748 bool free_delalloc_space = true; 2749 2750 fixup = container_of(work, struct btrfs_writepage_fixup, work); 2751 page = fixup->page; 2752 inode = fixup->inode; 2753 page_start = page_offset(page); 2754 page_end = page_offset(page) + PAGE_SIZE - 1; 2755 2756 /* 2757 * This is similar to page_mkwrite, we need to reserve the space before 2758 * we take the page lock. 2759 */ 2760 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 2761 PAGE_SIZE); 2762 again: 2763 lock_page(page); 2764 2765 /* 2766 * Before we queued this fixup, we took a reference on the page. 2767 * page->mapping may go NULL, but it shouldn't be moved to a different 2768 * address space. 2769 */ 2770 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 2771 /* 2772 * Unfortunately this is a little tricky, either 2773 * 2774 * 1) We got here and our page had already been dealt with and 2775 * we reserved our space, thus ret == 0, so we need to just 2776 * drop our space reservation and bail. This can happen the 2777 * first time we come into the fixup worker, or could happen 2778 * while waiting for the ordered extent. 2779 * 2) Our page was already dealt with, but we happened to get an 2780 * ENOSPC above from the btrfs_delalloc_reserve_space. In 2781 * this case we obviously don't have anything to release, but 2782 * because the page was already dealt with we don't want to 2783 * mark the page with an error, so make sure we're resetting 2784 * ret to 0. This is why we have this check _before_ the ret 2785 * check, because we do not want to have a surprise ENOSPC 2786 * when the page was already properly dealt with. 2787 */ 2788 if (!ret) { 2789 btrfs_delalloc_release_extents(inode, PAGE_SIZE); 2790 btrfs_delalloc_release_space(inode, data_reserved, 2791 page_start, PAGE_SIZE, 2792 true); 2793 } 2794 ret = 0; 2795 goto out_page; 2796 } 2797 2798 /* 2799 * We can't mess with the page state unless it is locked, so now that 2800 * it is locked bail if we failed to make our space reservation. 2801 */ 2802 if (ret) 2803 goto out_page; 2804 2805 lock_extent(&inode->io_tree, page_start, page_end, &cached_state); 2806 2807 /* already ordered? We're done */ 2808 if (PageOrdered(page)) 2809 goto out_reserved; 2810 2811 ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); 2812 if (ordered) { 2813 unlock_extent(&inode->io_tree, page_start, page_end, 2814 &cached_state); 2815 unlock_page(page); 2816 btrfs_start_ordered_extent(ordered); 2817 btrfs_put_ordered_extent(ordered); 2818 goto again; 2819 } 2820 2821 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, 2822 &cached_state); 2823 if (ret) 2824 goto out_reserved; 2825 2826 /* 2827 * Everything went as planned, we're now the owner of a dirty page with 2828 * delayed allocation bits set and space reserved for our COW 2829 * destination. 2830 * 2831 * The page was dirty when we started, nothing should have cleaned it. 2832 */ 2833 BUG_ON(!PageDirty(page)); 2834 free_delalloc_space = false; 2835 out_reserved: 2836 btrfs_delalloc_release_extents(inode, PAGE_SIZE); 2837 if (free_delalloc_space) 2838 btrfs_delalloc_release_space(inode, data_reserved, page_start, 2839 PAGE_SIZE, true); 2840 unlock_extent(&inode->io_tree, page_start, page_end, &cached_state); 2841 out_page: 2842 if (ret) { 2843 /* 2844 * We hit ENOSPC or other errors. Update the mapping and page 2845 * to reflect the errors and clean the page. 2846 */ 2847 mapping_set_error(page->mapping, ret); 2848 end_extent_writepage(page, ret, page_start, page_end); 2849 clear_page_dirty_for_io(page); 2850 SetPageError(page); 2851 } 2852 btrfs_page_clear_checked(inode->root->fs_info, page, page_start, PAGE_SIZE); 2853 unlock_page(page); 2854 put_page(page); 2855 kfree(fixup); 2856 extent_changeset_free(data_reserved); 2857 /* 2858 * As a precaution, do a delayed iput in case it would be the last iput 2859 * that could need flushing space. Recursing back to fixup worker would 2860 * deadlock. 2861 */ 2862 btrfs_add_delayed_iput(inode); 2863 } 2864 2865 /* 2866 * There are a few paths in the higher layers of the kernel that directly 2867 * set the page dirty bit without asking the filesystem if it is a 2868 * good idea. This causes problems because we want to make sure COW 2869 * properly happens and the data=ordered rules are followed. 2870 * 2871 * In our case any range that doesn't have the ORDERED bit set 2872 * hasn't been properly setup for IO. We kick off an async process 2873 * to fix it up. The async helper will wait for ordered extents, set 2874 * the delalloc bit and make it safe to write the page. 2875 */ 2876 int btrfs_writepage_cow_fixup(struct page *page) 2877 { 2878 struct inode *inode = page->mapping->host; 2879 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 2880 struct btrfs_writepage_fixup *fixup; 2881 2882 /* This page has ordered extent covering it already */ 2883 if (PageOrdered(page)) 2884 return 0; 2885 2886 /* 2887 * PageChecked is set below when we create a fixup worker for this page, 2888 * don't try to create another one if we're already PageChecked() 2889 * 2890 * The extent_io writepage code will redirty the page if we send back 2891 * EAGAIN. 2892 */ 2893 if (PageChecked(page)) 2894 return -EAGAIN; 2895 2896 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2897 if (!fixup) 2898 return -EAGAIN; 2899 2900 /* 2901 * We are already holding a reference to this inode from 2902 * write_cache_pages. We need to hold it because the space reservation 2903 * takes place outside of the page lock, and we can't trust 2904 * page->mapping outside of the page lock. 2905 */ 2906 ihold(inode); 2907 btrfs_page_set_checked(fs_info, page, page_offset(page), PAGE_SIZE); 2908 get_page(page); 2909 btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL, NULL); 2910 fixup->page = page; 2911 fixup->inode = BTRFS_I(inode); 2912 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2913 2914 return -EAGAIN; 2915 } 2916 2917 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2918 struct btrfs_inode *inode, u64 file_pos, 2919 struct btrfs_file_extent_item *stack_fi, 2920 const bool update_inode_bytes, 2921 u64 qgroup_reserved) 2922 { 2923 struct btrfs_root *root = inode->root; 2924 const u64 sectorsize = root->fs_info->sectorsize; 2925 struct btrfs_path *path; 2926 struct extent_buffer *leaf; 2927 struct btrfs_key ins; 2928 u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi); 2929 u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi); 2930 u64 offset = btrfs_stack_file_extent_offset(stack_fi); 2931 u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi); 2932 u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi); 2933 struct btrfs_drop_extents_args drop_args = { 0 }; 2934 int ret; 2935 2936 path = btrfs_alloc_path(); 2937 if (!path) 2938 return -ENOMEM; 2939 2940 /* 2941 * we may be replacing one extent in the tree with another. 2942 * The new extent is pinned in the extent map, and we don't want 2943 * to drop it from the cache until it is completely in the btree. 2944 * 2945 * So, tell btrfs_drop_extents to leave this extent in the cache. 2946 * the caller is expected to unpin it and allow it to be merged 2947 * with the others. 2948 */ 2949 drop_args.path = path; 2950 drop_args.start = file_pos; 2951 drop_args.end = file_pos + num_bytes; 2952 drop_args.replace_extent = true; 2953 drop_args.extent_item_size = sizeof(*stack_fi); 2954 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 2955 if (ret) 2956 goto out; 2957 2958 if (!drop_args.extent_inserted) { 2959 ins.objectid = btrfs_ino(inode); 2960 ins.offset = file_pos; 2961 ins.type = BTRFS_EXTENT_DATA_KEY; 2962 2963 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2964 sizeof(*stack_fi)); 2965 if (ret) 2966 goto out; 2967 } 2968 leaf = path->nodes[0]; 2969 btrfs_set_stack_file_extent_generation(stack_fi, trans->transid); 2970 write_extent_buffer(leaf, stack_fi, 2971 btrfs_item_ptr_offset(leaf, path->slots[0]), 2972 sizeof(struct btrfs_file_extent_item)); 2973 2974 btrfs_mark_buffer_dirty(leaf); 2975 btrfs_release_path(path); 2976 2977 /* 2978 * If we dropped an inline extent here, we know the range where it is 2979 * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the 2980 * number of bytes only for that range containing the inline extent. 2981 * The remaining of the range will be processed when clearning the 2982 * EXTENT_DELALLOC_BIT bit through the ordered extent completion. 2983 */ 2984 if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) { 2985 u64 inline_size = round_down(drop_args.bytes_found, sectorsize); 2986 2987 inline_size = drop_args.bytes_found - inline_size; 2988 btrfs_update_inode_bytes(inode, sectorsize, inline_size); 2989 drop_args.bytes_found -= inline_size; 2990 num_bytes -= sectorsize; 2991 } 2992 2993 if (update_inode_bytes) 2994 btrfs_update_inode_bytes(inode, num_bytes, drop_args.bytes_found); 2995 2996 ins.objectid = disk_bytenr; 2997 ins.offset = disk_num_bytes; 2998 ins.type = BTRFS_EXTENT_ITEM_KEY; 2999 3000 ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes); 3001 if (ret) 3002 goto out; 3003 3004 ret = btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode), 3005 file_pos - offset, 3006 qgroup_reserved, &ins); 3007 out: 3008 btrfs_free_path(path); 3009 3010 return ret; 3011 } 3012 3013 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 3014 u64 start, u64 len) 3015 { 3016 struct btrfs_block_group *cache; 3017 3018 cache = btrfs_lookup_block_group(fs_info, start); 3019 ASSERT(cache); 3020 3021 spin_lock(&cache->lock); 3022 cache->delalloc_bytes -= len; 3023 spin_unlock(&cache->lock); 3024 3025 btrfs_put_block_group(cache); 3026 } 3027 3028 static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans, 3029 struct btrfs_ordered_extent *oe) 3030 { 3031 struct btrfs_file_extent_item stack_fi; 3032 bool update_inode_bytes; 3033 u64 num_bytes = oe->num_bytes; 3034 u64 ram_bytes = oe->ram_bytes; 3035 3036 memset(&stack_fi, 0, sizeof(stack_fi)); 3037 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG); 3038 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr); 3039 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, 3040 oe->disk_num_bytes); 3041 btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset); 3042 if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags)) { 3043 num_bytes = oe->truncated_len; 3044 ram_bytes = num_bytes; 3045 } 3046 btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes); 3047 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes); 3048 btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type); 3049 /* Encryption and other encoding is reserved and all 0 */ 3050 3051 /* 3052 * For delalloc, when completing an ordered extent we update the inode's 3053 * bytes when clearing the range in the inode's io tree, so pass false 3054 * as the argument 'update_inode_bytes' to insert_reserved_file_extent(), 3055 * except if the ordered extent was truncated. 3056 */ 3057 update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) || 3058 test_bit(BTRFS_ORDERED_ENCODED, &oe->flags) || 3059 test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags); 3060 3061 return insert_reserved_file_extent(trans, BTRFS_I(oe->inode), 3062 oe->file_offset, &stack_fi, 3063 update_inode_bytes, oe->qgroup_rsv); 3064 } 3065 3066 /* 3067 * As ordered data IO finishes, this gets called so we can finish 3068 * an ordered extent if the range of bytes in the file it covers are 3069 * fully written. 3070 */ 3071 int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 3072 { 3073 struct btrfs_inode *inode = BTRFS_I(ordered_extent->inode); 3074 struct btrfs_root *root = inode->root; 3075 struct btrfs_fs_info *fs_info = root->fs_info; 3076 struct btrfs_trans_handle *trans = NULL; 3077 struct extent_io_tree *io_tree = &inode->io_tree; 3078 struct extent_state *cached_state = NULL; 3079 u64 start, end; 3080 int compress_type = 0; 3081 int ret = 0; 3082 u64 logical_len = ordered_extent->num_bytes; 3083 bool freespace_inode; 3084 bool truncated = false; 3085 bool clear_reserved_extent = true; 3086 unsigned int clear_bits = EXTENT_DEFRAG; 3087 3088 start = ordered_extent->file_offset; 3089 end = start + ordered_extent->num_bytes - 1; 3090 3091 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3092 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && 3093 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags) && 3094 !test_bit(BTRFS_ORDERED_ENCODED, &ordered_extent->flags)) 3095 clear_bits |= EXTENT_DELALLOC_NEW; 3096 3097 freespace_inode = btrfs_is_free_space_inode(inode); 3098 if (!freespace_inode) 3099 btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent); 3100 3101 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 3102 ret = -EIO; 3103 goto out; 3104 } 3105 3106 /* A valid ->physical implies a write on a sequential zone. */ 3107 if (ordered_extent->physical != (u64)-1) { 3108 btrfs_rewrite_logical_zoned(ordered_extent); 3109 btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr, 3110 ordered_extent->disk_num_bytes); 3111 } 3112 3113 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 3114 truncated = true; 3115 logical_len = ordered_extent->truncated_len; 3116 /* Truncated the entire extent, don't bother adding */ 3117 if (!logical_len) 3118 goto out; 3119 } 3120 3121 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 3122 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 3123 3124 btrfs_inode_safe_disk_i_size_write(inode, 0); 3125 if (freespace_inode) 3126 trans = btrfs_join_transaction_spacecache(root); 3127 else 3128 trans = btrfs_join_transaction(root); 3129 if (IS_ERR(trans)) { 3130 ret = PTR_ERR(trans); 3131 trans = NULL; 3132 goto out; 3133 } 3134 trans->block_rsv = &inode->block_rsv; 3135 ret = btrfs_update_inode_fallback(trans, root, inode); 3136 if (ret) /* -ENOMEM or corruption */ 3137 btrfs_abort_transaction(trans, ret); 3138 goto out; 3139 } 3140 3141 clear_bits |= EXTENT_LOCKED; 3142 lock_extent(io_tree, start, end, &cached_state); 3143 3144 if (freespace_inode) 3145 trans = btrfs_join_transaction_spacecache(root); 3146 else 3147 trans = btrfs_join_transaction(root); 3148 if (IS_ERR(trans)) { 3149 ret = PTR_ERR(trans); 3150 trans = NULL; 3151 goto out; 3152 } 3153 3154 trans->block_rsv = &inode->block_rsv; 3155 3156 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 3157 compress_type = ordered_extent->compress_type; 3158 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3159 BUG_ON(compress_type); 3160 ret = btrfs_mark_extent_written(trans, inode, 3161 ordered_extent->file_offset, 3162 ordered_extent->file_offset + 3163 logical_len); 3164 btrfs_zoned_release_data_reloc_bg(fs_info, ordered_extent->disk_bytenr, 3165 ordered_extent->disk_num_bytes); 3166 } else { 3167 BUG_ON(root == fs_info->tree_root); 3168 ret = insert_ordered_extent_file_extent(trans, ordered_extent); 3169 if (!ret) { 3170 clear_reserved_extent = false; 3171 btrfs_release_delalloc_bytes(fs_info, 3172 ordered_extent->disk_bytenr, 3173 ordered_extent->disk_num_bytes); 3174 } 3175 } 3176 unpin_extent_cache(&inode->extent_tree, ordered_extent->file_offset, 3177 ordered_extent->num_bytes, trans->transid); 3178 if (ret < 0) { 3179 btrfs_abort_transaction(trans, ret); 3180 goto out; 3181 } 3182 3183 ret = add_pending_csums(trans, &ordered_extent->list); 3184 if (ret) { 3185 btrfs_abort_transaction(trans, ret); 3186 goto out; 3187 } 3188 3189 /* 3190 * If this is a new delalloc range, clear its new delalloc flag to 3191 * update the inode's number of bytes. This needs to be done first 3192 * before updating the inode item. 3193 */ 3194 if ((clear_bits & EXTENT_DELALLOC_NEW) && 3195 !test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) 3196 clear_extent_bit(&inode->io_tree, start, end, 3197 EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES, 3198 &cached_state); 3199 3200 btrfs_inode_safe_disk_i_size_write(inode, 0); 3201 ret = btrfs_update_inode_fallback(trans, root, inode); 3202 if (ret) { /* -ENOMEM or corruption */ 3203 btrfs_abort_transaction(trans, ret); 3204 goto out; 3205 } 3206 ret = 0; 3207 out: 3208 clear_extent_bit(&inode->io_tree, start, end, clear_bits, 3209 &cached_state); 3210 3211 if (trans) 3212 btrfs_end_transaction(trans); 3213 3214 if (ret || truncated) { 3215 u64 unwritten_start = start; 3216 3217 /* 3218 * If we failed to finish this ordered extent for any reason we 3219 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered 3220 * extent, and mark the inode with the error if it wasn't 3221 * already set. Any error during writeback would have already 3222 * set the mapping error, so we need to set it if we're the ones 3223 * marking this ordered extent as failed. 3224 */ 3225 if (ret && !test_and_set_bit(BTRFS_ORDERED_IOERR, 3226 &ordered_extent->flags)) 3227 mapping_set_error(ordered_extent->inode->i_mapping, -EIO); 3228 3229 if (truncated) 3230 unwritten_start += logical_len; 3231 clear_extent_uptodate(io_tree, unwritten_start, end, NULL); 3232 3233 /* Drop extent maps for the part of the extent we didn't write. */ 3234 btrfs_drop_extent_map_range(inode, unwritten_start, end, false); 3235 3236 /* 3237 * If the ordered extent had an IOERR or something else went 3238 * wrong we need to return the space for this ordered extent 3239 * back to the allocator. We only free the extent in the 3240 * truncated case if we didn't write out the extent at all. 3241 * 3242 * If we made it past insert_reserved_file_extent before we 3243 * errored out then we don't need to do this as the accounting 3244 * has already been done. 3245 */ 3246 if ((ret || !logical_len) && 3247 clear_reserved_extent && 3248 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3249 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3250 /* 3251 * Discard the range before returning it back to the 3252 * free space pool 3253 */ 3254 if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC)) 3255 btrfs_discard_extent(fs_info, 3256 ordered_extent->disk_bytenr, 3257 ordered_extent->disk_num_bytes, 3258 NULL); 3259 btrfs_free_reserved_extent(fs_info, 3260 ordered_extent->disk_bytenr, 3261 ordered_extent->disk_num_bytes, 1); 3262 } 3263 } 3264 3265 /* 3266 * This needs to be done to make sure anybody waiting knows we are done 3267 * updating everything for this ordered extent. 3268 */ 3269 btrfs_remove_ordered_extent(inode, ordered_extent); 3270 3271 /* once for us */ 3272 btrfs_put_ordered_extent(ordered_extent); 3273 /* once for the tree */ 3274 btrfs_put_ordered_extent(ordered_extent); 3275 3276 return ret; 3277 } 3278 3279 void btrfs_writepage_endio_finish_ordered(struct btrfs_inode *inode, 3280 struct page *page, u64 start, 3281 u64 end, bool uptodate) 3282 { 3283 trace_btrfs_writepage_end_io_hook(inode, start, end, uptodate); 3284 3285 btrfs_mark_ordered_io_finished(inode, page, start, end + 1 - start, uptodate); 3286 } 3287 3288 /* 3289 * Verify the checksum for a single sector without any extra action that depend 3290 * on the type of I/O. 3291 */ 3292 int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page, 3293 u32 pgoff, u8 *csum, const u8 * const csum_expected) 3294 { 3295 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 3296 char *kaddr; 3297 3298 ASSERT(pgoff + fs_info->sectorsize <= PAGE_SIZE); 3299 3300 shash->tfm = fs_info->csum_shash; 3301 3302 kaddr = kmap_local_page(page) + pgoff; 3303 crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum); 3304 kunmap_local(kaddr); 3305 3306 if (memcmp(csum, csum_expected, fs_info->csum_size)) 3307 return -EIO; 3308 return 0; 3309 } 3310 3311 /* 3312 * Verify the checksum of a single data sector. 3313 * 3314 * @bbio: btrfs_io_bio which contains the csum 3315 * @dev: device the sector is on 3316 * @bio_offset: offset to the beginning of the bio (in bytes) 3317 * @bv: bio_vec to check 3318 * 3319 * Check if the checksum on a data block is valid. When a checksum mismatch is 3320 * detected, report the error and fill the corrupted range with zero. 3321 * 3322 * Return %true if the sector is ok or had no checksum to start with, else %false. 3323 */ 3324 bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, 3325 u32 bio_offset, struct bio_vec *bv) 3326 { 3327 struct btrfs_inode *inode = bbio->inode; 3328 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3329 u64 file_offset = bbio->file_offset + bio_offset; 3330 u64 end = file_offset + bv->bv_len - 1; 3331 u8 *csum_expected; 3332 u8 csum[BTRFS_CSUM_SIZE]; 3333 3334 ASSERT(bv->bv_len == fs_info->sectorsize); 3335 3336 if (!bbio->csum) 3337 return true; 3338 3339 if (btrfs_is_data_reloc_root(inode->root) && 3340 test_range_bit(&inode->io_tree, file_offset, end, EXTENT_NODATASUM, 3341 1, NULL)) { 3342 /* Skip the range without csum for data reloc inode */ 3343 clear_extent_bits(&inode->io_tree, file_offset, end, 3344 EXTENT_NODATASUM); 3345 return true; 3346 } 3347 3348 csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) * 3349 fs_info->csum_size; 3350 if (btrfs_check_sector_csum(fs_info, bv->bv_page, bv->bv_offset, csum, 3351 csum_expected)) 3352 goto zeroit; 3353 return true; 3354 3355 zeroit: 3356 btrfs_print_data_csum_error(inode, file_offset, csum, csum_expected, 3357 bbio->mirror_num); 3358 if (dev) 3359 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); 3360 memzero_bvec(bv); 3361 return false; 3362 } 3363 3364 /* 3365 * btrfs_add_delayed_iput - perform a delayed iput on @inode 3366 * 3367 * @inode: The inode we want to perform iput on 3368 * 3369 * This function uses the generic vfs_inode::i_count to track whether we should 3370 * just decrement it (in case it's > 1) or if this is the last iput then link 3371 * the inode to the delayed iput machinery. Delayed iputs are processed at 3372 * transaction commit time/superblock commit/cleaner kthread. 3373 */ 3374 void btrfs_add_delayed_iput(struct btrfs_inode *inode) 3375 { 3376 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3377 3378 if (atomic_add_unless(&inode->vfs_inode.i_count, -1, 1)) 3379 return; 3380 3381 atomic_inc(&fs_info->nr_delayed_iputs); 3382 spin_lock(&fs_info->delayed_iput_lock); 3383 ASSERT(list_empty(&inode->delayed_iput)); 3384 list_add_tail(&inode->delayed_iput, &fs_info->delayed_iputs); 3385 spin_unlock(&fs_info->delayed_iput_lock); 3386 if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags)) 3387 wake_up_process(fs_info->cleaner_kthread); 3388 } 3389 3390 static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info, 3391 struct btrfs_inode *inode) 3392 { 3393 list_del_init(&inode->delayed_iput); 3394 spin_unlock(&fs_info->delayed_iput_lock); 3395 iput(&inode->vfs_inode); 3396 if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) 3397 wake_up(&fs_info->delayed_iputs_wait); 3398 spin_lock(&fs_info->delayed_iput_lock); 3399 } 3400 3401 static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info, 3402 struct btrfs_inode *inode) 3403 { 3404 if (!list_empty(&inode->delayed_iput)) { 3405 spin_lock(&fs_info->delayed_iput_lock); 3406 if (!list_empty(&inode->delayed_iput)) 3407 run_delayed_iput_locked(fs_info, inode); 3408 spin_unlock(&fs_info->delayed_iput_lock); 3409 } 3410 } 3411 3412 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 3413 { 3414 3415 spin_lock(&fs_info->delayed_iput_lock); 3416 while (!list_empty(&fs_info->delayed_iputs)) { 3417 struct btrfs_inode *inode; 3418 3419 inode = list_first_entry(&fs_info->delayed_iputs, 3420 struct btrfs_inode, delayed_iput); 3421 run_delayed_iput_locked(fs_info, inode); 3422 cond_resched_lock(&fs_info->delayed_iput_lock); 3423 } 3424 spin_unlock(&fs_info->delayed_iput_lock); 3425 } 3426 3427 /* 3428 * Wait for flushing all delayed iputs 3429 * 3430 * @fs_info: the filesystem 3431 * 3432 * This will wait on any delayed iputs that are currently running with KILLABLE 3433 * set. Once they are all done running we will return, unless we are killed in 3434 * which case we return EINTR. This helps in user operations like fallocate etc 3435 * that might get blocked on the iputs. 3436 * 3437 * Return EINTR if we were killed, 0 if nothing's pending 3438 */ 3439 int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info) 3440 { 3441 int ret = wait_event_killable(fs_info->delayed_iputs_wait, 3442 atomic_read(&fs_info->nr_delayed_iputs) == 0); 3443 if (ret) 3444 return -EINTR; 3445 return 0; 3446 } 3447 3448 /* 3449 * This creates an orphan entry for the given inode in case something goes wrong 3450 * in the middle of an unlink. 3451 */ 3452 int btrfs_orphan_add(struct btrfs_trans_handle *trans, 3453 struct btrfs_inode *inode) 3454 { 3455 int ret; 3456 3457 ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode)); 3458 if (ret && ret != -EEXIST) { 3459 btrfs_abort_transaction(trans, ret); 3460 return ret; 3461 } 3462 3463 return 0; 3464 } 3465 3466 /* 3467 * We have done the delete so we can go ahead and remove the orphan item for 3468 * this particular inode. 3469 */ 3470 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3471 struct btrfs_inode *inode) 3472 { 3473 return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode)); 3474 } 3475 3476 /* 3477 * this cleans up any orphans that may be left on the list from the last use 3478 * of this root. 3479 */ 3480 int btrfs_orphan_cleanup(struct btrfs_root *root) 3481 { 3482 struct btrfs_fs_info *fs_info = root->fs_info; 3483 struct btrfs_path *path; 3484 struct extent_buffer *leaf; 3485 struct btrfs_key key, found_key; 3486 struct btrfs_trans_handle *trans; 3487 struct inode *inode; 3488 u64 last_objectid = 0; 3489 int ret = 0, nr_unlink = 0; 3490 3491 if (test_and_set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &root->state)) 3492 return 0; 3493 3494 path = btrfs_alloc_path(); 3495 if (!path) { 3496 ret = -ENOMEM; 3497 goto out; 3498 } 3499 path->reada = READA_BACK; 3500 3501 key.objectid = BTRFS_ORPHAN_OBJECTID; 3502 key.type = BTRFS_ORPHAN_ITEM_KEY; 3503 key.offset = (u64)-1; 3504 3505 while (1) { 3506 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3507 if (ret < 0) 3508 goto out; 3509 3510 /* 3511 * if ret == 0 means we found what we were searching for, which 3512 * is weird, but possible, so only screw with path if we didn't 3513 * find the key and see if we have stuff that matches 3514 */ 3515 if (ret > 0) { 3516 ret = 0; 3517 if (path->slots[0] == 0) 3518 break; 3519 path->slots[0]--; 3520 } 3521 3522 /* pull out the item */ 3523 leaf = path->nodes[0]; 3524 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3525 3526 /* make sure the item matches what we want */ 3527 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3528 break; 3529 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3530 break; 3531 3532 /* release the path since we're done with it */ 3533 btrfs_release_path(path); 3534 3535 /* 3536 * this is where we are basically btrfs_lookup, without the 3537 * crossing root thing. we store the inode number in the 3538 * offset of the orphan item. 3539 */ 3540 3541 if (found_key.offset == last_objectid) { 3542 btrfs_err(fs_info, 3543 "Error removing orphan entry, stopping orphan cleanup"); 3544 ret = -EINVAL; 3545 goto out; 3546 } 3547 3548 last_objectid = found_key.offset; 3549 3550 found_key.objectid = found_key.offset; 3551 found_key.type = BTRFS_INODE_ITEM_KEY; 3552 found_key.offset = 0; 3553 inode = btrfs_iget(fs_info->sb, last_objectid, root); 3554 ret = PTR_ERR_OR_ZERO(inode); 3555 if (ret && ret != -ENOENT) 3556 goto out; 3557 3558 if (ret == -ENOENT && root == fs_info->tree_root) { 3559 struct btrfs_root *dead_root; 3560 int is_dead_root = 0; 3561 3562 /* 3563 * This is an orphan in the tree root. Currently these 3564 * could come from 2 sources: 3565 * a) a root (snapshot/subvolume) deletion in progress 3566 * b) a free space cache inode 3567 * We need to distinguish those two, as the orphan item 3568 * for a root must not get deleted before the deletion 3569 * of the snapshot/subvolume's tree completes. 3570 * 3571 * btrfs_find_orphan_roots() ran before us, which has 3572 * found all deleted roots and loaded them into 3573 * fs_info->fs_roots_radix. So here we can find if an 3574 * orphan item corresponds to a deleted root by looking 3575 * up the root from that radix tree. 3576 */ 3577 3578 spin_lock(&fs_info->fs_roots_radix_lock); 3579 dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, 3580 (unsigned long)found_key.objectid); 3581 if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) 3582 is_dead_root = 1; 3583 spin_unlock(&fs_info->fs_roots_radix_lock); 3584 3585 if (is_dead_root) { 3586 /* prevent this orphan from being found again */ 3587 key.offset = found_key.objectid - 1; 3588 continue; 3589 } 3590 3591 } 3592 3593 /* 3594 * If we have an inode with links, there are a couple of 3595 * possibilities: 3596 * 3597 * 1. We were halfway through creating fsverity metadata for the 3598 * file. In that case, the orphan item represents incomplete 3599 * fsverity metadata which must be cleaned up with 3600 * btrfs_drop_verity_items and deleting the orphan item. 3601 3602 * 2. Old kernels (before v3.12) used to create an 3603 * orphan item for truncate indicating that there were possibly 3604 * extent items past i_size that needed to be deleted. In v3.12, 3605 * truncate was changed to update i_size in sync with the extent 3606 * items, but the (useless) orphan item was still created. Since 3607 * v4.18, we don't create the orphan item for truncate at all. 3608 * 3609 * So, this item could mean that we need to do a truncate, but 3610 * only if this filesystem was last used on a pre-v3.12 kernel 3611 * and was not cleanly unmounted. The odds of that are quite 3612 * slim, and it's a pain to do the truncate now, so just delete 3613 * the orphan item. 3614 * 3615 * It's also possible that this orphan item was supposed to be 3616 * deleted but wasn't. The inode number may have been reused, 3617 * but either way, we can delete the orphan item. 3618 */ 3619 if (ret == -ENOENT || inode->i_nlink) { 3620 if (!ret) { 3621 ret = btrfs_drop_verity_items(BTRFS_I(inode)); 3622 iput(inode); 3623 if (ret) 3624 goto out; 3625 } 3626 trans = btrfs_start_transaction(root, 1); 3627 if (IS_ERR(trans)) { 3628 ret = PTR_ERR(trans); 3629 iput(inode); 3630 goto out; 3631 } 3632 btrfs_debug(fs_info, "auto deleting %Lu", 3633 found_key.objectid); 3634 ret = btrfs_del_orphan_item(trans, root, 3635 found_key.objectid); 3636 btrfs_end_transaction(trans); 3637 if (ret) { 3638 iput(inode); 3639 goto out; 3640 } 3641 continue; 3642 } 3643 3644 nr_unlink++; 3645 3646 /* this will do delete_inode and everything for us */ 3647 iput(inode); 3648 } 3649 /* release the path since we're done with it */ 3650 btrfs_release_path(path); 3651 3652 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3653 trans = btrfs_join_transaction(root); 3654 if (!IS_ERR(trans)) 3655 btrfs_end_transaction(trans); 3656 } 3657 3658 if (nr_unlink) 3659 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3660 3661 out: 3662 if (ret) 3663 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3664 btrfs_free_path(path); 3665 return ret; 3666 } 3667 3668 /* 3669 * very simple check to peek ahead in the leaf looking for xattrs. If we 3670 * don't find any xattrs, we know there can't be any acls. 3671 * 3672 * slot is the slot the inode is in, objectid is the objectid of the inode 3673 */ 3674 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3675 int slot, u64 objectid, 3676 int *first_xattr_slot) 3677 { 3678 u32 nritems = btrfs_header_nritems(leaf); 3679 struct btrfs_key found_key; 3680 static u64 xattr_access = 0; 3681 static u64 xattr_default = 0; 3682 int scanned = 0; 3683 3684 if (!xattr_access) { 3685 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3686 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3687 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3688 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3689 } 3690 3691 slot++; 3692 *first_xattr_slot = -1; 3693 while (slot < nritems) { 3694 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3695 3696 /* we found a different objectid, there must not be acls */ 3697 if (found_key.objectid != objectid) 3698 return 0; 3699 3700 /* we found an xattr, assume we've got an acl */ 3701 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3702 if (*first_xattr_slot == -1) 3703 *first_xattr_slot = slot; 3704 if (found_key.offset == xattr_access || 3705 found_key.offset == xattr_default) 3706 return 1; 3707 } 3708 3709 /* 3710 * we found a key greater than an xattr key, there can't 3711 * be any acls later on 3712 */ 3713 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3714 return 0; 3715 3716 slot++; 3717 scanned++; 3718 3719 /* 3720 * it goes inode, inode backrefs, xattrs, extents, 3721 * so if there are a ton of hard links to an inode there can 3722 * be a lot of backrefs. Don't waste time searching too hard, 3723 * this is just an optimization 3724 */ 3725 if (scanned >= 8) 3726 break; 3727 } 3728 /* we hit the end of the leaf before we found an xattr or 3729 * something larger than an xattr. We have to assume the inode 3730 * has acls 3731 */ 3732 if (*first_xattr_slot == -1) 3733 *first_xattr_slot = slot; 3734 return 1; 3735 } 3736 3737 /* 3738 * read an inode from the btree into the in-memory inode 3739 */ 3740 static int btrfs_read_locked_inode(struct inode *inode, 3741 struct btrfs_path *in_path) 3742 { 3743 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 3744 struct btrfs_path *path = in_path; 3745 struct extent_buffer *leaf; 3746 struct btrfs_inode_item *inode_item; 3747 struct btrfs_root *root = BTRFS_I(inode)->root; 3748 struct btrfs_key location; 3749 unsigned long ptr; 3750 int maybe_acls; 3751 u32 rdev; 3752 int ret; 3753 bool filled = false; 3754 int first_xattr_slot; 3755 3756 ret = btrfs_fill_inode(inode, &rdev); 3757 if (!ret) 3758 filled = true; 3759 3760 if (!path) { 3761 path = btrfs_alloc_path(); 3762 if (!path) 3763 return -ENOMEM; 3764 } 3765 3766 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3767 3768 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3769 if (ret) { 3770 if (path != in_path) 3771 btrfs_free_path(path); 3772 return ret; 3773 } 3774 3775 leaf = path->nodes[0]; 3776 3777 if (filled) 3778 goto cache_index; 3779 3780 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3781 struct btrfs_inode_item); 3782 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3783 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3784 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3785 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3786 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item)); 3787 btrfs_inode_set_file_extent_range(BTRFS_I(inode), 0, 3788 round_up(i_size_read(inode), fs_info->sectorsize)); 3789 3790 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime); 3791 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime); 3792 3793 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime); 3794 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime); 3795 3796 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime); 3797 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime); 3798 3799 BTRFS_I(inode)->i_otime.tv_sec = 3800 btrfs_timespec_sec(leaf, &inode_item->otime); 3801 BTRFS_I(inode)->i_otime.tv_nsec = 3802 btrfs_timespec_nsec(leaf, &inode_item->otime); 3803 3804 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3805 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3806 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3807 3808 inode_set_iversion_queried(inode, 3809 btrfs_inode_sequence(leaf, inode_item)); 3810 inode->i_generation = BTRFS_I(inode)->generation; 3811 inode->i_rdev = 0; 3812 rdev = btrfs_inode_rdev(leaf, inode_item); 3813 3814 BTRFS_I(inode)->index_cnt = (u64)-1; 3815 btrfs_inode_split_flags(btrfs_inode_flags(leaf, inode_item), 3816 &BTRFS_I(inode)->flags, &BTRFS_I(inode)->ro_flags); 3817 3818 cache_index: 3819 /* 3820 * If we were modified in the current generation and evicted from memory 3821 * and then re-read we need to do a full sync since we don't have any 3822 * idea about which extents were modified before we were evicted from 3823 * cache. 3824 * 3825 * This is required for both inode re-read from disk and delayed inode 3826 * in delayed_nodes_tree. 3827 */ 3828 if (BTRFS_I(inode)->last_trans == fs_info->generation) 3829 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3830 &BTRFS_I(inode)->runtime_flags); 3831 3832 /* 3833 * We don't persist the id of the transaction where an unlink operation 3834 * against the inode was last made. So here we assume the inode might 3835 * have been evicted, and therefore the exact value of last_unlink_trans 3836 * lost, and set it to last_trans to avoid metadata inconsistencies 3837 * between the inode and its parent if the inode is fsync'ed and the log 3838 * replayed. For example, in the scenario: 3839 * 3840 * touch mydir/foo 3841 * ln mydir/foo mydir/bar 3842 * sync 3843 * unlink mydir/bar 3844 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 3845 * xfs_io -c fsync mydir/foo 3846 * <power failure> 3847 * mount fs, triggers fsync log replay 3848 * 3849 * We must make sure that when we fsync our inode foo we also log its 3850 * parent inode, otherwise after log replay the parent still has the 3851 * dentry with the "bar" name but our inode foo has a link count of 1 3852 * and doesn't have an inode ref with the name "bar" anymore. 3853 * 3854 * Setting last_unlink_trans to last_trans is a pessimistic approach, 3855 * but it guarantees correctness at the expense of occasional full 3856 * transaction commits on fsync if our inode is a directory, or if our 3857 * inode is not a directory, logging its parent unnecessarily. 3858 */ 3859 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans; 3860 3861 /* 3862 * Same logic as for last_unlink_trans. We don't persist the generation 3863 * of the last transaction where this inode was used for a reflink 3864 * operation, so after eviction and reloading the inode we must be 3865 * pessimistic and assume the last transaction that modified the inode. 3866 */ 3867 BTRFS_I(inode)->last_reflink_trans = BTRFS_I(inode)->last_trans; 3868 3869 path->slots[0]++; 3870 if (inode->i_nlink != 1 || 3871 path->slots[0] >= btrfs_header_nritems(leaf)) 3872 goto cache_acl; 3873 3874 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3875 if (location.objectid != btrfs_ino(BTRFS_I(inode))) 3876 goto cache_acl; 3877 3878 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3879 if (location.type == BTRFS_INODE_REF_KEY) { 3880 struct btrfs_inode_ref *ref; 3881 3882 ref = (struct btrfs_inode_ref *)ptr; 3883 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3884 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3885 struct btrfs_inode_extref *extref; 3886 3887 extref = (struct btrfs_inode_extref *)ptr; 3888 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3889 extref); 3890 } 3891 cache_acl: 3892 /* 3893 * try to precache a NULL acl entry for files that don't have 3894 * any xattrs or acls 3895 */ 3896 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3897 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot); 3898 if (first_xattr_slot != -1) { 3899 path->slots[0] = first_xattr_slot; 3900 ret = btrfs_load_inode_props(inode, path); 3901 if (ret) 3902 btrfs_err(fs_info, 3903 "error loading props for ino %llu (root %llu): %d", 3904 btrfs_ino(BTRFS_I(inode)), 3905 root->root_key.objectid, ret); 3906 } 3907 if (path != in_path) 3908 btrfs_free_path(path); 3909 3910 if (!maybe_acls) 3911 cache_no_acl(inode); 3912 3913 switch (inode->i_mode & S_IFMT) { 3914 case S_IFREG: 3915 inode->i_mapping->a_ops = &btrfs_aops; 3916 inode->i_fop = &btrfs_file_operations; 3917 inode->i_op = &btrfs_file_inode_operations; 3918 break; 3919 case S_IFDIR: 3920 inode->i_fop = &btrfs_dir_file_operations; 3921 inode->i_op = &btrfs_dir_inode_operations; 3922 break; 3923 case S_IFLNK: 3924 inode->i_op = &btrfs_symlink_inode_operations; 3925 inode_nohighmem(inode); 3926 inode->i_mapping->a_ops = &btrfs_aops; 3927 break; 3928 default: 3929 inode->i_op = &btrfs_special_inode_operations; 3930 init_special_inode(inode, inode->i_mode, rdev); 3931 break; 3932 } 3933 3934 btrfs_sync_inode_flags_to_i_flags(inode); 3935 return 0; 3936 } 3937 3938 /* 3939 * given a leaf and an inode, copy the inode fields into the leaf 3940 */ 3941 static void fill_inode_item(struct btrfs_trans_handle *trans, 3942 struct extent_buffer *leaf, 3943 struct btrfs_inode_item *item, 3944 struct inode *inode) 3945 { 3946 struct btrfs_map_token token; 3947 u64 flags; 3948 3949 btrfs_init_map_token(&token, leaf); 3950 3951 btrfs_set_token_inode_uid(&token, item, i_uid_read(inode)); 3952 btrfs_set_token_inode_gid(&token, item, i_gid_read(inode)); 3953 btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size); 3954 btrfs_set_token_inode_mode(&token, item, inode->i_mode); 3955 btrfs_set_token_inode_nlink(&token, item, inode->i_nlink); 3956 3957 btrfs_set_token_timespec_sec(&token, &item->atime, 3958 inode->i_atime.tv_sec); 3959 btrfs_set_token_timespec_nsec(&token, &item->atime, 3960 inode->i_atime.tv_nsec); 3961 3962 btrfs_set_token_timespec_sec(&token, &item->mtime, 3963 inode->i_mtime.tv_sec); 3964 btrfs_set_token_timespec_nsec(&token, &item->mtime, 3965 inode->i_mtime.tv_nsec); 3966 3967 btrfs_set_token_timespec_sec(&token, &item->ctime, 3968 inode->i_ctime.tv_sec); 3969 btrfs_set_token_timespec_nsec(&token, &item->ctime, 3970 inode->i_ctime.tv_nsec); 3971 3972 btrfs_set_token_timespec_sec(&token, &item->otime, 3973 BTRFS_I(inode)->i_otime.tv_sec); 3974 btrfs_set_token_timespec_nsec(&token, &item->otime, 3975 BTRFS_I(inode)->i_otime.tv_nsec); 3976 3977 btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode)); 3978 btrfs_set_token_inode_generation(&token, item, 3979 BTRFS_I(inode)->generation); 3980 btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode)); 3981 btrfs_set_token_inode_transid(&token, item, trans->transid); 3982 btrfs_set_token_inode_rdev(&token, item, inode->i_rdev); 3983 flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags, 3984 BTRFS_I(inode)->ro_flags); 3985 btrfs_set_token_inode_flags(&token, item, flags); 3986 btrfs_set_token_inode_block_group(&token, item, 0); 3987 } 3988 3989 /* 3990 * copy everything in the in-memory inode into the btree. 3991 */ 3992 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 3993 struct btrfs_root *root, 3994 struct btrfs_inode *inode) 3995 { 3996 struct btrfs_inode_item *inode_item; 3997 struct btrfs_path *path; 3998 struct extent_buffer *leaf; 3999 int ret; 4000 4001 path = btrfs_alloc_path(); 4002 if (!path) 4003 return -ENOMEM; 4004 4005 ret = btrfs_lookup_inode(trans, root, path, &inode->location, 1); 4006 if (ret) { 4007 if (ret > 0) 4008 ret = -ENOENT; 4009 goto failed; 4010 } 4011 4012 leaf = path->nodes[0]; 4013 inode_item = btrfs_item_ptr(leaf, path->slots[0], 4014 struct btrfs_inode_item); 4015 4016 fill_inode_item(trans, leaf, inode_item, &inode->vfs_inode); 4017 btrfs_mark_buffer_dirty(leaf); 4018 btrfs_set_inode_last_trans(trans, inode); 4019 ret = 0; 4020 failed: 4021 btrfs_free_path(path); 4022 return ret; 4023 } 4024 4025 /* 4026 * copy everything in the in-memory inode into the btree. 4027 */ 4028 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 4029 struct btrfs_root *root, 4030 struct btrfs_inode *inode) 4031 { 4032 struct btrfs_fs_info *fs_info = root->fs_info; 4033 int ret; 4034 4035 /* 4036 * If the inode is a free space inode, we can deadlock during commit 4037 * if we put it into the delayed code. 4038 * 4039 * The data relocation inode should also be directly updated 4040 * without delay 4041 */ 4042 if (!btrfs_is_free_space_inode(inode) 4043 && !btrfs_is_data_reloc_root(root) 4044 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 4045 btrfs_update_root_times(trans, root); 4046 4047 ret = btrfs_delayed_update_inode(trans, root, inode); 4048 if (!ret) 4049 btrfs_set_inode_last_trans(trans, inode); 4050 return ret; 4051 } 4052 4053 return btrfs_update_inode_item(trans, root, inode); 4054 } 4055 4056 int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 4057 struct btrfs_root *root, struct btrfs_inode *inode) 4058 { 4059 int ret; 4060 4061 ret = btrfs_update_inode(trans, root, inode); 4062 if (ret == -ENOSPC) 4063 return btrfs_update_inode_item(trans, root, inode); 4064 return ret; 4065 } 4066 4067 /* 4068 * unlink helper that gets used here in inode.c and in the tree logging 4069 * recovery code. It remove a link in a directory with a given name, and 4070 * also drops the back refs in the inode to the directory 4071 */ 4072 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4073 struct btrfs_inode *dir, 4074 struct btrfs_inode *inode, 4075 const struct fscrypt_str *name, 4076 struct btrfs_rename_ctx *rename_ctx) 4077 { 4078 struct btrfs_root *root = dir->root; 4079 struct btrfs_fs_info *fs_info = root->fs_info; 4080 struct btrfs_path *path; 4081 int ret = 0; 4082 struct btrfs_dir_item *di; 4083 u64 index; 4084 u64 ino = btrfs_ino(inode); 4085 u64 dir_ino = btrfs_ino(dir); 4086 4087 path = btrfs_alloc_path(); 4088 if (!path) { 4089 ret = -ENOMEM; 4090 goto out; 4091 } 4092 4093 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, name, -1); 4094 if (IS_ERR_OR_NULL(di)) { 4095 ret = di ? PTR_ERR(di) : -ENOENT; 4096 goto err; 4097 } 4098 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4099 if (ret) 4100 goto err; 4101 btrfs_release_path(path); 4102 4103 /* 4104 * If we don't have dir index, we have to get it by looking up 4105 * the inode ref, since we get the inode ref, remove it directly, 4106 * it is unnecessary to do delayed deletion. 4107 * 4108 * But if we have dir index, needn't search inode ref to get it. 4109 * Since the inode ref is close to the inode item, it is better 4110 * that we delay to delete it, and just do this deletion when 4111 * we update the inode item. 4112 */ 4113 if (inode->dir_index) { 4114 ret = btrfs_delayed_delete_inode_ref(inode); 4115 if (!ret) { 4116 index = inode->dir_index; 4117 goto skip_backref; 4118 } 4119 } 4120 4121 ret = btrfs_del_inode_ref(trans, root, name, ino, dir_ino, &index); 4122 if (ret) { 4123 btrfs_info(fs_info, 4124 "failed to delete reference to %.*s, inode %llu parent %llu", 4125 name->len, name->name, ino, dir_ino); 4126 btrfs_abort_transaction(trans, ret); 4127 goto err; 4128 } 4129 skip_backref: 4130 if (rename_ctx) 4131 rename_ctx->index = index; 4132 4133 ret = btrfs_delete_delayed_dir_index(trans, dir, index); 4134 if (ret) { 4135 btrfs_abort_transaction(trans, ret); 4136 goto err; 4137 } 4138 4139 /* 4140 * If we are in a rename context, we don't need to update anything in the 4141 * log. That will be done later during the rename by btrfs_log_new_name(). 4142 * Besides that, doing it here would only cause extra unnecessary btree 4143 * operations on the log tree, increasing latency for applications. 4144 */ 4145 if (!rename_ctx) { 4146 btrfs_del_inode_ref_in_log(trans, root, name, inode, dir_ino); 4147 btrfs_del_dir_entries_in_log(trans, root, name, dir, index); 4148 } 4149 4150 /* 4151 * If we have a pending delayed iput we could end up with the final iput 4152 * being run in btrfs-cleaner context. If we have enough of these built 4153 * up we can end up burning a lot of time in btrfs-cleaner without any 4154 * way to throttle the unlinks. Since we're currently holding a ref on 4155 * the inode we can run the delayed iput here without any issues as the 4156 * final iput won't be done until after we drop the ref we're currently 4157 * holding. 4158 */ 4159 btrfs_run_delayed_iput(fs_info, inode); 4160 err: 4161 btrfs_free_path(path); 4162 if (ret) 4163 goto out; 4164 4165 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name->len * 2); 4166 inode_inc_iversion(&inode->vfs_inode); 4167 inode_inc_iversion(&dir->vfs_inode); 4168 inode->vfs_inode.i_ctime = current_time(&inode->vfs_inode); 4169 dir->vfs_inode.i_mtime = inode->vfs_inode.i_ctime; 4170 dir->vfs_inode.i_ctime = inode->vfs_inode.i_ctime; 4171 ret = btrfs_update_inode(trans, root, dir); 4172 out: 4173 return ret; 4174 } 4175 4176 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4177 struct btrfs_inode *dir, struct btrfs_inode *inode, 4178 const struct fscrypt_str *name) 4179 { 4180 int ret; 4181 4182 ret = __btrfs_unlink_inode(trans, dir, inode, name, NULL); 4183 if (!ret) { 4184 drop_nlink(&inode->vfs_inode); 4185 ret = btrfs_update_inode(trans, inode->root, inode); 4186 } 4187 return ret; 4188 } 4189 4190 /* 4191 * helper to start transaction for unlink and rmdir. 4192 * 4193 * unlink and rmdir are special in btrfs, they do not always free space, so 4194 * if we cannot make our reservations the normal way try and see if there is 4195 * plenty of slack room in the global reserve to migrate, otherwise we cannot 4196 * allow the unlink to occur. 4197 */ 4198 static struct btrfs_trans_handle *__unlink_start_trans(struct btrfs_inode *dir) 4199 { 4200 struct btrfs_root *root = dir->root; 4201 4202 return btrfs_start_transaction_fallback_global_rsv(root, 4203 BTRFS_UNLINK_METADATA_UNITS); 4204 } 4205 4206 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 4207 { 4208 struct btrfs_trans_handle *trans; 4209 struct inode *inode = d_inode(dentry); 4210 int ret; 4211 struct fscrypt_name fname; 4212 4213 ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname); 4214 if (ret) 4215 return ret; 4216 4217 /* This needs to handle no-key deletions later on */ 4218 4219 trans = __unlink_start_trans(BTRFS_I(dir)); 4220 if (IS_ERR(trans)) { 4221 ret = PTR_ERR(trans); 4222 goto fscrypt_free; 4223 } 4224 4225 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4226 0); 4227 4228 ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4229 &fname.disk_name); 4230 if (ret) 4231 goto end_trans; 4232 4233 if (inode->i_nlink == 0) { 4234 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 4235 if (ret) 4236 goto end_trans; 4237 } 4238 4239 end_trans: 4240 btrfs_end_transaction(trans); 4241 btrfs_btree_balance_dirty(BTRFS_I(dir)->root->fs_info); 4242 fscrypt_free: 4243 fscrypt_free_filename(&fname); 4244 return ret; 4245 } 4246 4247 static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 4248 struct btrfs_inode *dir, struct dentry *dentry) 4249 { 4250 struct btrfs_root *root = dir->root; 4251 struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); 4252 struct btrfs_path *path; 4253 struct extent_buffer *leaf; 4254 struct btrfs_dir_item *di; 4255 struct btrfs_key key; 4256 u64 index; 4257 int ret; 4258 u64 objectid; 4259 u64 dir_ino = btrfs_ino(dir); 4260 struct fscrypt_name fname; 4261 4262 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname); 4263 if (ret) 4264 return ret; 4265 4266 /* This needs to handle no-key deletions later on */ 4267 4268 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) { 4269 objectid = inode->root->root_key.objectid; 4270 } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 4271 objectid = inode->location.objectid; 4272 } else { 4273 WARN_ON(1); 4274 fscrypt_free_filename(&fname); 4275 return -EINVAL; 4276 } 4277 4278 path = btrfs_alloc_path(); 4279 if (!path) { 4280 ret = -ENOMEM; 4281 goto out; 4282 } 4283 4284 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4285 &fname.disk_name, -1); 4286 if (IS_ERR_OR_NULL(di)) { 4287 ret = di ? PTR_ERR(di) : -ENOENT; 4288 goto out; 4289 } 4290 4291 leaf = path->nodes[0]; 4292 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4293 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 4294 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4295 if (ret) { 4296 btrfs_abort_transaction(trans, ret); 4297 goto out; 4298 } 4299 btrfs_release_path(path); 4300 4301 /* 4302 * This is a placeholder inode for a subvolume we didn't have a 4303 * reference to at the time of the snapshot creation. In the meantime 4304 * we could have renamed the real subvol link into our snapshot, so 4305 * depending on btrfs_del_root_ref to return -ENOENT here is incorrect. 4306 * Instead simply lookup the dir_index_item for this entry so we can 4307 * remove it. Otherwise we know we have a ref to the root and we can 4308 * call btrfs_del_root_ref, and it _shouldn't_ fail. 4309 */ 4310 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 4311 di = btrfs_search_dir_index_item(root, path, dir_ino, &fname.disk_name); 4312 if (IS_ERR_OR_NULL(di)) { 4313 if (!di) 4314 ret = -ENOENT; 4315 else 4316 ret = PTR_ERR(di); 4317 btrfs_abort_transaction(trans, ret); 4318 goto out; 4319 } 4320 4321 leaf = path->nodes[0]; 4322 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4323 index = key.offset; 4324 btrfs_release_path(path); 4325 } else { 4326 ret = btrfs_del_root_ref(trans, objectid, 4327 root->root_key.objectid, dir_ino, 4328 &index, &fname.disk_name); 4329 if (ret) { 4330 btrfs_abort_transaction(trans, ret); 4331 goto out; 4332 } 4333 } 4334 4335 ret = btrfs_delete_delayed_dir_index(trans, dir, index); 4336 if (ret) { 4337 btrfs_abort_transaction(trans, ret); 4338 goto out; 4339 } 4340 4341 btrfs_i_size_write(dir, dir->vfs_inode.i_size - fname.disk_name.len * 2); 4342 inode_inc_iversion(&dir->vfs_inode); 4343 dir->vfs_inode.i_mtime = current_time(&dir->vfs_inode); 4344 dir->vfs_inode.i_ctime = dir->vfs_inode.i_mtime; 4345 ret = btrfs_update_inode_fallback(trans, root, dir); 4346 if (ret) 4347 btrfs_abort_transaction(trans, ret); 4348 out: 4349 btrfs_free_path(path); 4350 fscrypt_free_filename(&fname); 4351 return ret; 4352 } 4353 4354 /* 4355 * Helper to check if the subvolume references other subvolumes or if it's 4356 * default. 4357 */ 4358 static noinline int may_destroy_subvol(struct btrfs_root *root) 4359 { 4360 struct btrfs_fs_info *fs_info = root->fs_info; 4361 struct btrfs_path *path; 4362 struct btrfs_dir_item *di; 4363 struct btrfs_key key; 4364 struct fscrypt_str name = FSTR_INIT("default", 7); 4365 u64 dir_id; 4366 int ret; 4367 4368 path = btrfs_alloc_path(); 4369 if (!path) 4370 return -ENOMEM; 4371 4372 /* Make sure this root isn't set as the default subvol */ 4373 dir_id = btrfs_super_root_dir(fs_info->super_copy); 4374 di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path, 4375 dir_id, &name, 0); 4376 if (di && !IS_ERR(di)) { 4377 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key); 4378 if (key.objectid == root->root_key.objectid) { 4379 ret = -EPERM; 4380 btrfs_err(fs_info, 4381 "deleting default subvolume %llu is not allowed", 4382 key.objectid); 4383 goto out; 4384 } 4385 btrfs_release_path(path); 4386 } 4387 4388 key.objectid = root->root_key.objectid; 4389 key.type = BTRFS_ROOT_REF_KEY; 4390 key.offset = (u64)-1; 4391 4392 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 4393 if (ret < 0) 4394 goto out; 4395 BUG_ON(ret == 0); 4396 4397 ret = 0; 4398 if (path->slots[0] > 0) { 4399 path->slots[0]--; 4400 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); 4401 if (key.objectid == root->root_key.objectid && 4402 key.type == BTRFS_ROOT_REF_KEY) 4403 ret = -ENOTEMPTY; 4404 } 4405 out: 4406 btrfs_free_path(path); 4407 return ret; 4408 } 4409 4410 /* Delete all dentries for inodes belonging to the root */ 4411 static void btrfs_prune_dentries(struct btrfs_root *root) 4412 { 4413 struct btrfs_fs_info *fs_info = root->fs_info; 4414 struct rb_node *node; 4415 struct rb_node *prev; 4416 struct btrfs_inode *entry; 4417 struct inode *inode; 4418 u64 objectid = 0; 4419 4420 if (!BTRFS_FS_ERROR(fs_info)) 4421 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 4422 4423 spin_lock(&root->inode_lock); 4424 again: 4425 node = root->inode_tree.rb_node; 4426 prev = NULL; 4427 while (node) { 4428 prev = node; 4429 entry = rb_entry(node, struct btrfs_inode, rb_node); 4430 4431 if (objectid < btrfs_ino(entry)) 4432 node = node->rb_left; 4433 else if (objectid > btrfs_ino(entry)) 4434 node = node->rb_right; 4435 else 4436 break; 4437 } 4438 if (!node) { 4439 while (prev) { 4440 entry = rb_entry(prev, struct btrfs_inode, rb_node); 4441 if (objectid <= btrfs_ino(entry)) { 4442 node = prev; 4443 break; 4444 } 4445 prev = rb_next(prev); 4446 } 4447 } 4448 while (node) { 4449 entry = rb_entry(node, struct btrfs_inode, rb_node); 4450 objectid = btrfs_ino(entry) + 1; 4451 inode = igrab(&entry->vfs_inode); 4452 if (inode) { 4453 spin_unlock(&root->inode_lock); 4454 if (atomic_read(&inode->i_count) > 1) 4455 d_prune_aliases(inode); 4456 /* 4457 * btrfs_drop_inode will have it removed from the inode 4458 * cache when its usage count hits zero. 4459 */ 4460 iput(inode); 4461 cond_resched(); 4462 spin_lock(&root->inode_lock); 4463 goto again; 4464 } 4465 4466 if (cond_resched_lock(&root->inode_lock)) 4467 goto again; 4468 4469 node = rb_next(node); 4470 } 4471 spin_unlock(&root->inode_lock); 4472 } 4473 4474 int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry) 4475 { 4476 struct btrfs_fs_info *fs_info = btrfs_sb(dentry->d_sb); 4477 struct btrfs_root *root = dir->root; 4478 struct inode *inode = d_inode(dentry); 4479 struct btrfs_root *dest = BTRFS_I(inode)->root; 4480 struct btrfs_trans_handle *trans; 4481 struct btrfs_block_rsv block_rsv; 4482 u64 root_flags; 4483 int ret; 4484 4485 /* 4486 * Don't allow to delete a subvolume with send in progress. This is 4487 * inside the inode lock so the error handling that has to drop the bit 4488 * again is not run concurrently. 4489 */ 4490 spin_lock(&dest->root_item_lock); 4491 if (dest->send_in_progress) { 4492 spin_unlock(&dest->root_item_lock); 4493 btrfs_warn(fs_info, 4494 "attempt to delete subvolume %llu during send", 4495 dest->root_key.objectid); 4496 return -EPERM; 4497 } 4498 if (atomic_read(&dest->nr_swapfiles)) { 4499 spin_unlock(&dest->root_item_lock); 4500 btrfs_warn(fs_info, 4501 "attempt to delete subvolume %llu with active swapfile", 4502 root->root_key.objectid); 4503 return -EPERM; 4504 } 4505 root_flags = btrfs_root_flags(&dest->root_item); 4506 btrfs_set_root_flags(&dest->root_item, 4507 root_flags | BTRFS_ROOT_SUBVOL_DEAD); 4508 spin_unlock(&dest->root_item_lock); 4509 4510 down_write(&fs_info->subvol_sem); 4511 4512 ret = may_destroy_subvol(dest); 4513 if (ret) 4514 goto out_up_write; 4515 4516 btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP); 4517 /* 4518 * One for dir inode, 4519 * two for dir entries, 4520 * two for root ref/backref. 4521 */ 4522 ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true); 4523 if (ret) 4524 goto out_up_write; 4525 4526 trans = btrfs_start_transaction(root, 0); 4527 if (IS_ERR(trans)) { 4528 ret = PTR_ERR(trans); 4529 goto out_release; 4530 } 4531 trans->block_rsv = &block_rsv; 4532 trans->bytes_reserved = block_rsv.size; 4533 4534 btrfs_record_snapshot_destroy(trans, dir); 4535 4536 ret = btrfs_unlink_subvol(trans, dir, dentry); 4537 if (ret) { 4538 btrfs_abort_transaction(trans, ret); 4539 goto out_end_trans; 4540 } 4541 4542 ret = btrfs_record_root_in_trans(trans, dest); 4543 if (ret) { 4544 btrfs_abort_transaction(trans, ret); 4545 goto out_end_trans; 4546 } 4547 4548 memset(&dest->root_item.drop_progress, 0, 4549 sizeof(dest->root_item.drop_progress)); 4550 btrfs_set_root_drop_level(&dest->root_item, 0); 4551 btrfs_set_root_refs(&dest->root_item, 0); 4552 4553 if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) { 4554 ret = btrfs_insert_orphan_item(trans, 4555 fs_info->tree_root, 4556 dest->root_key.objectid); 4557 if (ret) { 4558 btrfs_abort_transaction(trans, ret); 4559 goto out_end_trans; 4560 } 4561 } 4562 4563 ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid, 4564 BTRFS_UUID_KEY_SUBVOL, 4565 dest->root_key.objectid); 4566 if (ret && ret != -ENOENT) { 4567 btrfs_abort_transaction(trans, ret); 4568 goto out_end_trans; 4569 } 4570 if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) { 4571 ret = btrfs_uuid_tree_remove(trans, 4572 dest->root_item.received_uuid, 4573 BTRFS_UUID_KEY_RECEIVED_SUBVOL, 4574 dest->root_key.objectid); 4575 if (ret && ret != -ENOENT) { 4576 btrfs_abort_transaction(trans, ret); 4577 goto out_end_trans; 4578 } 4579 } 4580 4581 free_anon_bdev(dest->anon_dev); 4582 dest->anon_dev = 0; 4583 out_end_trans: 4584 trans->block_rsv = NULL; 4585 trans->bytes_reserved = 0; 4586 ret = btrfs_end_transaction(trans); 4587 inode->i_flags |= S_DEAD; 4588 out_release: 4589 btrfs_subvolume_release_metadata(root, &block_rsv); 4590 out_up_write: 4591 up_write(&fs_info->subvol_sem); 4592 if (ret) { 4593 spin_lock(&dest->root_item_lock); 4594 root_flags = btrfs_root_flags(&dest->root_item); 4595 btrfs_set_root_flags(&dest->root_item, 4596 root_flags & ~BTRFS_ROOT_SUBVOL_DEAD); 4597 spin_unlock(&dest->root_item_lock); 4598 } else { 4599 d_invalidate(dentry); 4600 btrfs_prune_dentries(dest); 4601 ASSERT(dest->send_in_progress == 0); 4602 } 4603 4604 return ret; 4605 } 4606 4607 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4608 { 4609 struct inode *inode = d_inode(dentry); 4610 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 4611 int err = 0; 4612 struct btrfs_trans_handle *trans; 4613 u64 last_unlink_trans; 4614 struct fscrypt_name fname; 4615 4616 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4617 return -ENOTEMPTY; 4618 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) { 4619 if (unlikely(btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))) { 4620 btrfs_err(fs_info, 4621 "extent tree v2 doesn't support snapshot deletion yet"); 4622 return -EOPNOTSUPP; 4623 } 4624 return btrfs_delete_subvolume(BTRFS_I(dir), dentry); 4625 } 4626 4627 err = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname); 4628 if (err) 4629 return err; 4630 4631 /* This needs to handle no-key deletions later on */ 4632 4633 trans = __unlink_start_trans(BTRFS_I(dir)); 4634 if (IS_ERR(trans)) { 4635 err = PTR_ERR(trans); 4636 goto out_notrans; 4637 } 4638 4639 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4640 err = btrfs_unlink_subvol(trans, BTRFS_I(dir), dentry); 4641 goto out; 4642 } 4643 4644 err = btrfs_orphan_add(trans, BTRFS_I(inode)); 4645 if (err) 4646 goto out; 4647 4648 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans; 4649 4650 /* now the directory is empty */ 4651 err = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4652 &fname.disk_name); 4653 if (!err) { 4654 btrfs_i_size_write(BTRFS_I(inode), 0); 4655 /* 4656 * Propagate the last_unlink_trans value of the deleted dir to 4657 * its parent directory. This is to prevent an unrecoverable 4658 * log tree in the case we do something like this: 4659 * 1) create dir foo 4660 * 2) create snapshot under dir foo 4661 * 3) delete the snapshot 4662 * 4) rmdir foo 4663 * 5) mkdir foo 4664 * 6) fsync foo or some file inside foo 4665 */ 4666 if (last_unlink_trans >= trans->transid) 4667 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans; 4668 } 4669 out: 4670 btrfs_end_transaction(trans); 4671 out_notrans: 4672 btrfs_btree_balance_dirty(fs_info); 4673 fscrypt_free_filename(&fname); 4674 4675 return err; 4676 } 4677 4678 /* 4679 * btrfs_truncate_block - read, zero a chunk and write a block 4680 * @inode - inode that we're zeroing 4681 * @from - the offset to start zeroing 4682 * @len - the length to zero, 0 to zero the entire range respective to the 4683 * offset 4684 * @front - zero up to the offset instead of from the offset on 4685 * 4686 * This will find the block for the "from" offset and cow the block and zero the 4687 * part we want to zero. This is used with truncate and hole punching. 4688 */ 4689 int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len, 4690 int front) 4691 { 4692 struct btrfs_fs_info *fs_info = inode->root->fs_info; 4693 struct address_space *mapping = inode->vfs_inode.i_mapping; 4694 struct extent_io_tree *io_tree = &inode->io_tree; 4695 struct btrfs_ordered_extent *ordered; 4696 struct extent_state *cached_state = NULL; 4697 struct extent_changeset *data_reserved = NULL; 4698 bool only_release_metadata = false; 4699 u32 blocksize = fs_info->sectorsize; 4700 pgoff_t index = from >> PAGE_SHIFT; 4701 unsigned offset = from & (blocksize - 1); 4702 struct page *page; 4703 gfp_t mask = btrfs_alloc_write_mask(mapping); 4704 size_t write_bytes = blocksize; 4705 int ret = 0; 4706 u64 block_start; 4707 u64 block_end; 4708 4709 if (IS_ALIGNED(offset, blocksize) && 4710 (!len || IS_ALIGNED(len, blocksize))) 4711 goto out; 4712 4713 block_start = round_down(from, blocksize); 4714 block_end = block_start + blocksize - 1; 4715 4716 ret = btrfs_check_data_free_space(inode, &data_reserved, block_start, 4717 blocksize, false); 4718 if (ret < 0) { 4719 if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) { 4720 /* For nocow case, no need to reserve data space */ 4721 only_release_metadata = true; 4722 } else { 4723 goto out; 4724 } 4725 } 4726 ret = btrfs_delalloc_reserve_metadata(inode, blocksize, blocksize, false); 4727 if (ret < 0) { 4728 if (!only_release_metadata) 4729 btrfs_free_reserved_data_space(inode, data_reserved, 4730 block_start, blocksize); 4731 goto out; 4732 } 4733 again: 4734 page = find_or_create_page(mapping, index, mask); 4735 if (!page) { 4736 btrfs_delalloc_release_space(inode, data_reserved, block_start, 4737 blocksize, true); 4738 btrfs_delalloc_release_extents(inode, blocksize); 4739 ret = -ENOMEM; 4740 goto out; 4741 } 4742 ret = set_page_extent_mapped(page); 4743 if (ret < 0) 4744 goto out_unlock; 4745 4746 if (!PageUptodate(page)) { 4747 ret = btrfs_read_folio(NULL, page_folio(page)); 4748 lock_page(page); 4749 if (page->mapping != mapping) { 4750 unlock_page(page); 4751 put_page(page); 4752 goto again; 4753 } 4754 if (!PageUptodate(page)) { 4755 ret = -EIO; 4756 goto out_unlock; 4757 } 4758 } 4759 wait_on_page_writeback(page); 4760 4761 lock_extent(io_tree, block_start, block_end, &cached_state); 4762 4763 ordered = btrfs_lookup_ordered_extent(inode, block_start); 4764 if (ordered) { 4765 unlock_extent(io_tree, block_start, block_end, &cached_state); 4766 unlock_page(page); 4767 put_page(page); 4768 btrfs_start_ordered_extent(ordered); 4769 btrfs_put_ordered_extent(ordered); 4770 goto again; 4771 } 4772 4773 clear_extent_bit(&inode->io_tree, block_start, block_end, 4774 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4775 &cached_state); 4776 4777 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, 4778 &cached_state); 4779 if (ret) { 4780 unlock_extent(io_tree, block_start, block_end, &cached_state); 4781 goto out_unlock; 4782 } 4783 4784 if (offset != blocksize) { 4785 if (!len) 4786 len = blocksize - offset; 4787 if (front) 4788 memzero_page(page, (block_start - page_offset(page)), 4789 offset); 4790 else 4791 memzero_page(page, (block_start - page_offset(page)) + offset, 4792 len); 4793 } 4794 btrfs_page_clear_checked(fs_info, page, block_start, 4795 block_end + 1 - block_start); 4796 btrfs_page_set_dirty(fs_info, page, block_start, block_end + 1 - block_start); 4797 unlock_extent(io_tree, block_start, block_end, &cached_state); 4798 4799 if (only_release_metadata) 4800 set_extent_bit(&inode->io_tree, block_start, block_end, 4801 EXTENT_NORESERVE, NULL, GFP_NOFS); 4802 4803 out_unlock: 4804 if (ret) { 4805 if (only_release_metadata) 4806 btrfs_delalloc_release_metadata(inode, blocksize, true); 4807 else 4808 btrfs_delalloc_release_space(inode, data_reserved, 4809 block_start, blocksize, true); 4810 } 4811 btrfs_delalloc_release_extents(inode, blocksize); 4812 unlock_page(page); 4813 put_page(page); 4814 out: 4815 if (only_release_metadata) 4816 btrfs_check_nocow_unlock(inode); 4817 extent_changeset_free(data_reserved); 4818 return ret; 4819 } 4820 4821 static int maybe_insert_hole(struct btrfs_root *root, struct btrfs_inode *inode, 4822 u64 offset, u64 len) 4823 { 4824 struct btrfs_fs_info *fs_info = root->fs_info; 4825 struct btrfs_trans_handle *trans; 4826 struct btrfs_drop_extents_args drop_args = { 0 }; 4827 int ret; 4828 4829 /* 4830 * If NO_HOLES is enabled, we don't need to do anything. 4831 * Later, up in the call chain, either btrfs_set_inode_last_sub_trans() 4832 * or btrfs_update_inode() will be called, which guarantee that the next 4833 * fsync will know this inode was changed and needs to be logged. 4834 */ 4835 if (btrfs_fs_incompat(fs_info, NO_HOLES)) 4836 return 0; 4837 4838 /* 4839 * 1 - for the one we're dropping 4840 * 1 - for the one we're adding 4841 * 1 - for updating the inode. 4842 */ 4843 trans = btrfs_start_transaction(root, 3); 4844 if (IS_ERR(trans)) 4845 return PTR_ERR(trans); 4846 4847 drop_args.start = offset; 4848 drop_args.end = offset + len; 4849 drop_args.drop_cache = true; 4850 4851 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 4852 if (ret) { 4853 btrfs_abort_transaction(trans, ret); 4854 btrfs_end_transaction(trans); 4855 return ret; 4856 } 4857 4858 ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, len); 4859 if (ret) { 4860 btrfs_abort_transaction(trans, ret); 4861 } else { 4862 btrfs_update_inode_bytes(inode, 0, drop_args.bytes_found); 4863 btrfs_update_inode(trans, root, inode); 4864 } 4865 btrfs_end_transaction(trans); 4866 return ret; 4867 } 4868 4869 /* 4870 * This function puts in dummy file extents for the area we're creating a hole 4871 * for. So if we are truncating this file to a larger size we need to insert 4872 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 4873 * the range between oldsize and size 4874 */ 4875 int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size) 4876 { 4877 struct btrfs_root *root = inode->root; 4878 struct btrfs_fs_info *fs_info = root->fs_info; 4879 struct extent_io_tree *io_tree = &inode->io_tree; 4880 struct extent_map *em = NULL; 4881 struct extent_state *cached_state = NULL; 4882 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 4883 u64 block_end = ALIGN(size, fs_info->sectorsize); 4884 u64 last_byte; 4885 u64 cur_offset; 4886 u64 hole_size; 4887 int err = 0; 4888 4889 /* 4890 * If our size started in the middle of a block we need to zero out the 4891 * rest of the block before we expand the i_size, otherwise we could 4892 * expose stale data. 4893 */ 4894 err = btrfs_truncate_block(inode, oldsize, 0, 0); 4895 if (err) 4896 return err; 4897 4898 if (size <= hole_start) 4899 return 0; 4900 4901 btrfs_lock_and_flush_ordered_range(inode, hole_start, block_end - 1, 4902 &cached_state); 4903 cur_offset = hole_start; 4904 while (1) { 4905 em = btrfs_get_extent(inode, NULL, 0, cur_offset, 4906 block_end - cur_offset); 4907 if (IS_ERR(em)) { 4908 err = PTR_ERR(em); 4909 em = NULL; 4910 break; 4911 } 4912 last_byte = min(extent_map_end(em), block_end); 4913 last_byte = ALIGN(last_byte, fs_info->sectorsize); 4914 hole_size = last_byte - cur_offset; 4915 4916 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 4917 struct extent_map *hole_em; 4918 4919 err = maybe_insert_hole(root, inode, cur_offset, 4920 hole_size); 4921 if (err) 4922 break; 4923 4924 err = btrfs_inode_set_file_extent_range(inode, 4925 cur_offset, hole_size); 4926 if (err) 4927 break; 4928 4929 hole_em = alloc_extent_map(); 4930 if (!hole_em) { 4931 btrfs_drop_extent_map_range(inode, cur_offset, 4932 cur_offset + hole_size - 1, 4933 false); 4934 btrfs_set_inode_full_sync(inode); 4935 goto next; 4936 } 4937 hole_em->start = cur_offset; 4938 hole_em->len = hole_size; 4939 hole_em->orig_start = cur_offset; 4940 4941 hole_em->block_start = EXTENT_MAP_HOLE; 4942 hole_em->block_len = 0; 4943 hole_em->orig_block_len = 0; 4944 hole_em->ram_bytes = hole_size; 4945 hole_em->compress_type = BTRFS_COMPRESS_NONE; 4946 hole_em->generation = fs_info->generation; 4947 4948 err = btrfs_replace_extent_map_range(inode, hole_em, true); 4949 free_extent_map(hole_em); 4950 } else { 4951 err = btrfs_inode_set_file_extent_range(inode, 4952 cur_offset, hole_size); 4953 if (err) 4954 break; 4955 } 4956 next: 4957 free_extent_map(em); 4958 em = NULL; 4959 cur_offset = last_byte; 4960 if (cur_offset >= block_end) 4961 break; 4962 } 4963 free_extent_map(em); 4964 unlock_extent(io_tree, hole_start, block_end - 1, &cached_state); 4965 return err; 4966 } 4967 4968 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 4969 { 4970 struct btrfs_root *root = BTRFS_I(inode)->root; 4971 struct btrfs_trans_handle *trans; 4972 loff_t oldsize = i_size_read(inode); 4973 loff_t newsize = attr->ia_size; 4974 int mask = attr->ia_valid; 4975 int ret; 4976 4977 /* 4978 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 4979 * special case where we need to update the times despite not having 4980 * these flags set. For all other operations the VFS set these flags 4981 * explicitly if it wants a timestamp update. 4982 */ 4983 if (newsize != oldsize) { 4984 inode_inc_iversion(inode); 4985 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) { 4986 inode->i_mtime = current_time(inode); 4987 inode->i_ctime = inode->i_mtime; 4988 } 4989 } 4990 4991 if (newsize > oldsize) { 4992 /* 4993 * Don't do an expanding truncate while snapshotting is ongoing. 4994 * This is to ensure the snapshot captures a fully consistent 4995 * state of this file - if the snapshot captures this expanding 4996 * truncation, it must capture all writes that happened before 4997 * this truncation. 4998 */ 4999 btrfs_drew_write_lock(&root->snapshot_lock); 5000 ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, newsize); 5001 if (ret) { 5002 btrfs_drew_write_unlock(&root->snapshot_lock); 5003 return ret; 5004 } 5005 5006 trans = btrfs_start_transaction(root, 1); 5007 if (IS_ERR(trans)) { 5008 btrfs_drew_write_unlock(&root->snapshot_lock); 5009 return PTR_ERR(trans); 5010 } 5011 5012 i_size_write(inode, newsize); 5013 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); 5014 pagecache_isize_extended(inode, oldsize, newsize); 5015 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 5016 btrfs_drew_write_unlock(&root->snapshot_lock); 5017 btrfs_end_transaction(trans); 5018 } else { 5019 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5020 5021 if (btrfs_is_zoned(fs_info)) { 5022 ret = btrfs_wait_ordered_range(inode, 5023 ALIGN(newsize, fs_info->sectorsize), 5024 (u64)-1); 5025 if (ret) 5026 return ret; 5027 } 5028 5029 /* 5030 * We're truncating a file that used to have good data down to 5031 * zero. Make sure any new writes to the file get on disk 5032 * on close. 5033 */ 5034 if (newsize == 0) 5035 set_bit(BTRFS_INODE_FLUSH_ON_CLOSE, 5036 &BTRFS_I(inode)->runtime_flags); 5037 5038 truncate_setsize(inode, newsize); 5039 5040 inode_dio_wait(inode); 5041 5042 ret = btrfs_truncate(BTRFS_I(inode), newsize == oldsize); 5043 if (ret && inode->i_nlink) { 5044 int err; 5045 5046 /* 5047 * Truncate failed, so fix up the in-memory size. We 5048 * adjusted disk_i_size down as we removed extents, so 5049 * wait for disk_i_size to be stable and then update the 5050 * in-memory size to match. 5051 */ 5052 err = btrfs_wait_ordered_range(inode, 0, (u64)-1); 5053 if (err) 5054 return err; 5055 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 5056 } 5057 } 5058 5059 return ret; 5060 } 5061 5062 static int btrfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, 5063 struct iattr *attr) 5064 { 5065 struct inode *inode = d_inode(dentry); 5066 struct btrfs_root *root = BTRFS_I(inode)->root; 5067 int err; 5068 5069 if (btrfs_root_readonly(root)) 5070 return -EROFS; 5071 5072 err = setattr_prepare(idmap, dentry, attr); 5073 if (err) 5074 return err; 5075 5076 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 5077 err = btrfs_setsize(inode, attr); 5078 if (err) 5079 return err; 5080 } 5081 5082 if (attr->ia_valid) { 5083 setattr_copy(idmap, inode, attr); 5084 inode_inc_iversion(inode); 5085 err = btrfs_dirty_inode(BTRFS_I(inode)); 5086 5087 if (!err && attr->ia_valid & ATTR_MODE) 5088 err = posix_acl_chmod(idmap, dentry, inode->i_mode); 5089 } 5090 5091 return err; 5092 } 5093 5094 /* 5095 * While truncating the inode pages during eviction, we get the VFS 5096 * calling btrfs_invalidate_folio() against each folio of the inode. This 5097 * is slow because the calls to btrfs_invalidate_folio() result in a 5098 * huge amount of calls to lock_extent() and clear_extent_bit(), 5099 * which keep merging and splitting extent_state structures over and over, 5100 * wasting lots of time. 5101 * 5102 * Therefore if the inode is being evicted, let btrfs_invalidate_folio() 5103 * skip all those expensive operations on a per folio basis and do only 5104 * the ordered io finishing, while we release here the extent_map and 5105 * extent_state structures, without the excessive merging and splitting. 5106 */ 5107 static void evict_inode_truncate_pages(struct inode *inode) 5108 { 5109 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 5110 struct rb_node *node; 5111 5112 ASSERT(inode->i_state & I_FREEING); 5113 truncate_inode_pages_final(&inode->i_data); 5114 5115 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); 5116 5117 /* 5118 * Keep looping until we have no more ranges in the io tree. 5119 * We can have ongoing bios started by readahead that have 5120 * their endio callback (extent_io.c:end_bio_extent_readpage) 5121 * still in progress (unlocked the pages in the bio but did not yet 5122 * unlocked the ranges in the io tree). Therefore this means some 5123 * ranges can still be locked and eviction started because before 5124 * submitting those bios, which are executed by a separate task (work 5125 * queue kthread), inode references (inode->i_count) were not taken 5126 * (which would be dropped in the end io callback of each bio). 5127 * Therefore here we effectively end up waiting for those bios and 5128 * anyone else holding locked ranges without having bumped the inode's 5129 * reference count - if we don't do it, when they access the inode's 5130 * io_tree to unlock a range it may be too late, leading to an 5131 * use-after-free issue. 5132 */ 5133 spin_lock(&io_tree->lock); 5134 while (!RB_EMPTY_ROOT(&io_tree->state)) { 5135 struct extent_state *state; 5136 struct extent_state *cached_state = NULL; 5137 u64 start; 5138 u64 end; 5139 unsigned state_flags; 5140 5141 node = rb_first(&io_tree->state); 5142 state = rb_entry(node, struct extent_state, rb_node); 5143 start = state->start; 5144 end = state->end; 5145 state_flags = state->state; 5146 spin_unlock(&io_tree->lock); 5147 5148 lock_extent(io_tree, start, end, &cached_state); 5149 5150 /* 5151 * If still has DELALLOC flag, the extent didn't reach disk, 5152 * and its reserved space won't be freed by delayed_ref. 5153 * So we need to free its reserved space here. 5154 * (Refer to comment in btrfs_invalidate_folio, case 2) 5155 * 5156 * Note, end is the bytenr of last byte, so we need + 1 here. 5157 */ 5158 if (state_flags & EXTENT_DELALLOC) 5159 btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start, 5160 end - start + 1); 5161 5162 clear_extent_bit(io_tree, start, end, 5163 EXTENT_CLEAR_ALL_BITS | EXTENT_DO_ACCOUNTING, 5164 &cached_state); 5165 5166 cond_resched(); 5167 spin_lock(&io_tree->lock); 5168 } 5169 spin_unlock(&io_tree->lock); 5170 } 5171 5172 static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root, 5173 struct btrfs_block_rsv *rsv) 5174 { 5175 struct btrfs_fs_info *fs_info = root->fs_info; 5176 struct btrfs_trans_handle *trans; 5177 u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, 1); 5178 int ret; 5179 5180 /* 5181 * Eviction should be taking place at some place safe because of our 5182 * delayed iputs. However the normal flushing code will run delayed 5183 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock. 5184 * 5185 * We reserve the delayed_refs_extra here again because we can't use 5186 * btrfs_start_transaction(root, 0) for the same deadlocky reason as 5187 * above. We reserve our extra bit here because we generate a ton of 5188 * delayed refs activity by truncating. 5189 * 5190 * BTRFS_RESERVE_FLUSH_EVICT will steal from the global_rsv if it can, 5191 * if we fail to make this reservation we can re-try without the 5192 * delayed_refs_extra so we can make some forward progress. 5193 */ 5194 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size + delayed_refs_extra, 5195 BTRFS_RESERVE_FLUSH_EVICT); 5196 if (ret) { 5197 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size, 5198 BTRFS_RESERVE_FLUSH_EVICT); 5199 if (ret) { 5200 btrfs_warn(fs_info, 5201 "could not allocate space for delete; will truncate on mount"); 5202 return ERR_PTR(-ENOSPC); 5203 } 5204 delayed_refs_extra = 0; 5205 } 5206 5207 trans = btrfs_join_transaction(root); 5208 if (IS_ERR(trans)) 5209 return trans; 5210 5211 if (delayed_refs_extra) { 5212 trans->block_rsv = &fs_info->trans_block_rsv; 5213 trans->bytes_reserved = delayed_refs_extra; 5214 btrfs_block_rsv_migrate(rsv, trans->block_rsv, 5215 delayed_refs_extra, true); 5216 } 5217 return trans; 5218 } 5219 5220 void btrfs_evict_inode(struct inode *inode) 5221 { 5222 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 5223 struct btrfs_trans_handle *trans; 5224 struct btrfs_root *root = BTRFS_I(inode)->root; 5225 struct btrfs_block_rsv *rsv = NULL; 5226 int ret; 5227 5228 trace_btrfs_inode_evict(inode); 5229 5230 if (!root) { 5231 fsverity_cleanup_inode(inode); 5232 clear_inode(inode); 5233 return; 5234 } 5235 5236 evict_inode_truncate_pages(inode); 5237 5238 if (inode->i_nlink && 5239 ((btrfs_root_refs(&root->root_item) != 0 && 5240 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 5241 btrfs_is_free_space_inode(BTRFS_I(inode)))) 5242 goto out; 5243 5244 if (is_bad_inode(inode)) 5245 goto out; 5246 5247 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) 5248 goto out; 5249 5250 if (inode->i_nlink > 0) { 5251 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5252 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 5253 goto out; 5254 } 5255 5256 /* 5257 * This makes sure the inode item in tree is uptodate and the space for 5258 * the inode update is released. 5259 */ 5260 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); 5261 if (ret) 5262 goto out; 5263 5264 /* 5265 * This drops any pending insert or delete operations we have for this 5266 * inode. We could have a delayed dir index deletion queued up, but 5267 * we're removing the inode completely so that'll be taken care of in 5268 * the truncate. 5269 */ 5270 btrfs_kill_delayed_inode_items(BTRFS_I(inode)); 5271 5272 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 5273 if (!rsv) 5274 goto out; 5275 rsv->size = btrfs_calc_metadata_size(fs_info, 1); 5276 rsv->failfast = true; 5277 5278 btrfs_i_size_write(BTRFS_I(inode), 0); 5279 5280 while (1) { 5281 struct btrfs_truncate_control control = { 5282 .inode = BTRFS_I(inode), 5283 .ino = btrfs_ino(BTRFS_I(inode)), 5284 .new_size = 0, 5285 .min_type = 0, 5286 }; 5287 5288 trans = evict_refill_and_join(root, rsv); 5289 if (IS_ERR(trans)) 5290 goto out; 5291 5292 trans->block_rsv = rsv; 5293 5294 ret = btrfs_truncate_inode_items(trans, root, &control); 5295 trans->block_rsv = &fs_info->trans_block_rsv; 5296 btrfs_end_transaction(trans); 5297 /* 5298 * We have not added new delayed items for our inode after we 5299 * have flushed its delayed items, so no need to throttle on 5300 * delayed items. However we have modified extent buffers. 5301 */ 5302 btrfs_btree_balance_dirty_nodelay(fs_info); 5303 if (ret && ret != -ENOSPC && ret != -EAGAIN) 5304 goto out; 5305 else if (!ret) 5306 break; 5307 } 5308 5309 /* 5310 * Errors here aren't a big deal, it just means we leave orphan items in 5311 * the tree. They will be cleaned up on the next mount. If the inode 5312 * number gets reused, cleanup deletes the orphan item without doing 5313 * anything, and unlink reuses the existing orphan item. 5314 * 5315 * If it turns out that we are dropping too many of these, we might want 5316 * to add a mechanism for retrying these after a commit. 5317 */ 5318 trans = evict_refill_and_join(root, rsv); 5319 if (!IS_ERR(trans)) { 5320 trans->block_rsv = rsv; 5321 btrfs_orphan_del(trans, BTRFS_I(inode)); 5322 trans->block_rsv = &fs_info->trans_block_rsv; 5323 btrfs_end_transaction(trans); 5324 } 5325 5326 out: 5327 btrfs_free_block_rsv(fs_info, rsv); 5328 /* 5329 * If we didn't successfully delete, the orphan item will still be in 5330 * the tree and we'll retry on the next mount. Again, we might also want 5331 * to retry these periodically in the future. 5332 */ 5333 btrfs_remove_delayed_node(BTRFS_I(inode)); 5334 fsverity_cleanup_inode(inode); 5335 clear_inode(inode); 5336 } 5337 5338 /* 5339 * Return the key found in the dir entry in the location pointer, fill @type 5340 * with BTRFS_FT_*, and return 0. 5341 * 5342 * If no dir entries were found, returns -ENOENT. 5343 * If found a corrupted location in dir entry, returns -EUCLEAN. 5344 */ 5345 static int btrfs_inode_by_name(struct btrfs_inode *dir, struct dentry *dentry, 5346 struct btrfs_key *location, u8 *type) 5347 { 5348 struct btrfs_dir_item *di; 5349 struct btrfs_path *path; 5350 struct btrfs_root *root = dir->root; 5351 int ret = 0; 5352 struct fscrypt_name fname; 5353 5354 path = btrfs_alloc_path(); 5355 if (!path) 5356 return -ENOMEM; 5357 5358 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname); 5359 if (ret < 0) 5360 goto out; 5361 /* 5362 * fscrypt_setup_filename() should never return a positive value, but 5363 * gcc on sparc/parisc thinks it can, so assert that doesn't happen. 5364 */ 5365 ASSERT(ret == 0); 5366 5367 /* This needs to handle no-key deletions later on */ 5368 5369 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), 5370 &fname.disk_name, 0); 5371 if (IS_ERR_OR_NULL(di)) { 5372 ret = di ? PTR_ERR(di) : -ENOENT; 5373 goto out; 5374 } 5375 5376 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5377 if (location->type != BTRFS_INODE_ITEM_KEY && 5378 location->type != BTRFS_ROOT_ITEM_KEY) { 5379 ret = -EUCLEAN; 5380 btrfs_warn(root->fs_info, 5381 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))", 5382 __func__, fname.disk_name.name, btrfs_ino(dir), 5383 location->objectid, location->type, location->offset); 5384 } 5385 if (!ret) 5386 *type = btrfs_dir_ftype(path->nodes[0], di); 5387 out: 5388 fscrypt_free_filename(&fname); 5389 btrfs_free_path(path); 5390 return ret; 5391 } 5392 5393 /* 5394 * when we hit a tree root in a directory, the btrfs part of the inode 5395 * needs to be changed to reflect the root directory of the tree root. This 5396 * is kind of like crossing a mount point. 5397 */ 5398 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5399 struct btrfs_inode *dir, 5400 struct dentry *dentry, 5401 struct btrfs_key *location, 5402 struct btrfs_root **sub_root) 5403 { 5404 struct btrfs_path *path; 5405 struct btrfs_root *new_root; 5406 struct btrfs_root_ref *ref; 5407 struct extent_buffer *leaf; 5408 struct btrfs_key key; 5409 int ret; 5410 int err = 0; 5411 struct fscrypt_name fname; 5412 5413 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 0, &fname); 5414 if (ret) 5415 return ret; 5416 5417 path = btrfs_alloc_path(); 5418 if (!path) { 5419 err = -ENOMEM; 5420 goto out; 5421 } 5422 5423 err = -ENOENT; 5424 key.objectid = dir->root->root_key.objectid; 5425 key.type = BTRFS_ROOT_REF_KEY; 5426 key.offset = location->objectid; 5427 5428 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5429 if (ret) { 5430 if (ret < 0) 5431 err = ret; 5432 goto out; 5433 } 5434 5435 leaf = path->nodes[0]; 5436 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5437 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) || 5438 btrfs_root_ref_name_len(leaf, ref) != fname.disk_name.len) 5439 goto out; 5440 5441 ret = memcmp_extent_buffer(leaf, fname.disk_name.name, 5442 (unsigned long)(ref + 1), fname.disk_name.len); 5443 if (ret) 5444 goto out; 5445 5446 btrfs_release_path(path); 5447 5448 new_root = btrfs_get_fs_root(fs_info, location->objectid, true); 5449 if (IS_ERR(new_root)) { 5450 err = PTR_ERR(new_root); 5451 goto out; 5452 } 5453 5454 *sub_root = new_root; 5455 location->objectid = btrfs_root_dirid(&new_root->root_item); 5456 location->type = BTRFS_INODE_ITEM_KEY; 5457 location->offset = 0; 5458 err = 0; 5459 out: 5460 btrfs_free_path(path); 5461 fscrypt_free_filename(&fname); 5462 return err; 5463 } 5464 5465 static void inode_tree_add(struct btrfs_inode *inode) 5466 { 5467 struct btrfs_root *root = inode->root; 5468 struct btrfs_inode *entry; 5469 struct rb_node **p; 5470 struct rb_node *parent; 5471 struct rb_node *new = &inode->rb_node; 5472 u64 ino = btrfs_ino(inode); 5473 5474 if (inode_unhashed(&inode->vfs_inode)) 5475 return; 5476 parent = NULL; 5477 spin_lock(&root->inode_lock); 5478 p = &root->inode_tree.rb_node; 5479 while (*p) { 5480 parent = *p; 5481 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5482 5483 if (ino < btrfs_ino(entry)) 5484 p = &parent->rb_left; 5485 else if (ino > btrfs_ino(entry)) 5486 p = &parent->rb_right; 5487 else { 5488 WARN_ON(!(entry->vfs_inode.i_state & 5489 (I_WILL_FREE | I_FREEING))); 5490 rb_replace_node(parent, new, &root->inode_tree); 5491 RB_CLEAR_NODE(parent); 5492 spin_unlock(&root->inode_lock); 5493 return; 5494 } 5495 } 5496 rb_link_node(new, parent, p); 5497 rb_insert_color(new, &root->inode_tree); 5498 spin_unlock(&root->inode_lock); 5499 } 5500 5501 static void inode_tree_del(struct btrfs_inode *inode) 5502 { 5503 struct btrfs_root *root = inode->root; 5504 int empty = 0; 5505 5506 spin_lock(&root->inode_lock); 5507 if (!RB_EMPTY_NODE(&inode->rb_node)) { 5508 rb_erase(&inode->rb_node, &root->inode_tree); 5509 RB_CLEAR_NODE(&inode->rb_node); 5510 empty = RB_EMPTY_ROOT(&root->inode_tree); 5511 } 5512 spin_unlock(&root->inode_lock); 5513 5514 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5515 spin_lock(&root->inode_lock); 5516 empty = RB_EMPTY_ROOT(&root->inode_tree); 5517 spin_unlock(&root->inode_lock); 5518 if (empty) 5519 btrfs_add_dead_root(root); 5520 } 5521 } 5522 5523 5524 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5525 { 5526 struct btrfs_iget_args *args = p; 5527 5528 inode->i_ino = args->ino; 5529 BTRFS_I(inode)->location.objectid = args->ino; 5530 BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY; 5531 BTRFS_I(inode)->location.offset = 0; 5532 BTRFS_I(inode)->root = btrfs_grab_root(args->root); 5533 BUG_ON(args->root && !BTRFS_I(inode)->root); 5534 5535 if (args->root && args->root == args->root->fs_info->tree_root && 5536 args->ino != BTRFS_BTREE_INODE_OBJECTID) 5537 set_bit(BTRFS_INODE_FREE_SPACE_INODE, 5538 &BTRFS_I(inode)->runtime_flags); 5539 return 0; 5540 } 5541 5542 static int btrfs_find_actor(struct inode *inode, void *opaque) 5543 { 5544 struct btrfs_iget_args *args = opaque; 5545 5546 return args->ino == BTRFS_I(inode)->location.objectid && 5547 args->root == BTRFS_I(inode)->root; 5548 } 5549 5550 static struct inode *btrfs_iget_locked(struct super_block *s, u64 ino, 5551 struct btrfs_root *root) 5552 { 5553 struct inode *inode; 5554 struct btrfs_iget_args args; 5555 unsigned long hashval = btrfs_inode_hash(ino, root); 5556 5557 args.ino = ino; 5558 args.root = root; 5559 5560 inode = iget5_locked(s, hashval, btrfs_find_actor, 5561 btrfs_init_locked_inode, 5562 (void *)&args); 5563 return inode; 5564 } 5565 5566 /* 5567 * Get an inode object given its inode number and corresponding root. 5568 * Path can be preallocated to prevent recursing back to iget through 5569 * allocator. NULL is also valid but may require an additional allocation 5570 * later. 5571 */ 5572 struct inode *btrfs_iget_path(struct super_block *s, u64 ino, 5573 struct btrfs_root *root, struct btrfs_path *path) 5574 { 5575 struct inode *inode; 5576 5577 inode = btrfs_iget_locked(s, ino, root); 5578 if (!inode) 5579 return ERR_PTR(-ENOMEM); 5580 5581 if (inode->i_state & I_NEW) { 5582 int ret; 5583 5584 ret = btrfs_read_locked_inode(inode, path); 5585 if (!ret) { 5586 inode_tree_add(BTRFS_I(inode)); 5587 unlock_new_inode(inode); 5588 } else { 5589 iget_failed(inode); 5590 /* 5591 * ret > 0 can come from btrfs_search_slot called by 5592 * btrfs_read_locked_inode, this means the inode item 5593 * was not found. 5594 */ 5595 if (ret > 0) 5596 ret = -ENOENT; 5597 inode = ERR_PTR(ret); 5598 } 5599 } 5600 5601 return inode; 5602 } 5603 5604 struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root) 5605 { 5606 return btrfs_iget_path(s, ino, root, NULL); 5607 } 5608 5609 static struct inode *new_simple_dir(struct super_block *s, 5610 struct btrfs_key *key, 5611 struct btrfs_root *root) 5612 { 5613 struct inode *inode = new_inode(s); 5614 5615 if (!inode) 5616 return ERR_PTR(-ENOMEM); 5617 5618 BTRFS_I(inode)->root = btrfs_grab_root(root); 5619 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5620 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5621 5622 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5623 /* 5624 * We only need lookup, the rest is read-only and there's no inode 5625 * associated with the dentry 5626 */ 5627 inode->i_op = &simple_dir_inode_operations; 5628 inode->i_opflags &= ~IOP_XATTR; 5629 inode->i_fop = &simple_dir_operations; 5630 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5631 inode->i_mtime = current_time(inode); 5632 inode->i_atime = inode->i_mtime; 5633 inode->i_ctime = inode->i_mtime; 5634 BTRFS_I(inode)->i_otime = inode->i_mtime; 5635 5636 return inode; 5637 } 5638 5639 static_assert(BTRFS_FT_UNKNOWN == FT_UNKNOWN); 5640 static_assert(BTRFS_FT_REG_FILE == FT_REG_FILE); 5641 static_assert(BTRFS_FT_DIR == FT_DIR); 5642 static_assert(BTRFS_FT_CHRDEV == FT_CHRDEV); 5643 static_assert(BTRFS_FT_BLKDEV == FT_BLKDEV); 5644 static_assert(BTRFS_FT_FIFO == FT_FIFO); 5645 static_assert(BTRFS_FT_SOCK == FT_SOCK); 5646 static_assert(BTRFS_FT_SYMLINK == FT_SYMLINK); 5647 5648 static inline u8 btrfs_inode_type(struct inode *inode) 5649 { 5650 return fs_umode_to_ftype(inode->i_mode); 5651 } 5652 5653 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5654 { 5655 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 5656 struct inode *inode; 5657 struct btrfs_root *root = BTRFS_I(dir)->root; 5658 struct btrfs_root *sub_root = root; 5659 struct btrfs_key location; 5660 u8 di_type = 0; 5661 int ret = 0; 5662 5663 if (dentry->d_name.len > BTRFS_NAME_LEN) 5664 return ERR_PTR(-ENAMETOOLONG); 5665 5666 ret = btrfs_inode_by_name(BTRFS_I(dir), dentry, &location, &di_type); 5667 if (ret < 0) 5668 return ERR_PTR(ret); 5669 5670 if (location.type == BTRFS_INODE_ITEM_KEY) { 5671 inode = btrfs_iget(dir->i_sb, location.objectid, root); 5672 if (IS_ERR(inode)) 5673 return inode; 5674 5675 /* Do extra check against inode mode with di_type */ 5676 if (btrfs_inode_type(inode) != di_type) { 5677 btrfs_crit(fs_info, 5678 "inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u", 5679 inode->i_mode, btrfs_inode_type(inode), 5680 di_type); 5681 iput(inode); 5682 return ERR_PTR(-EUCLEAN); 5683 } 5684 return inode; 5685 } 5686 5687 ret = fixup_tree_root_location(fs_info, BTRFS_I(dir), dentry, 5688 &location, &sub_root); 5689 if (ret < 0) { 5690 if (ret != -ENOENT) 5691 inode = ERR_PTR(ret); 5692 else 5693 inode = new_simple_dir(dir->i_sb, &location, root); 5694 } else { 5695 inode = btrfs_iget(dir->i_sb, location.objectid, sub_root); 5696 btrfs_put_root(sub_root); 5697 5698 if (IS_ERR(inode)) 5699 return inode; 5700 5701 down_read(&fs_info->cleanup_work_sem); 5702 if (!sb_rdonly(inode->i_sb)) 5703 ret = btrfs_orphan_cleanup(sub_root); 5704 up_read(&fs_info->cleanup_work_sem); 5705 if (ret) { 5706 iput(inode); 5707 inode = ERR_PTR(ret); 5708 } 5709 } 5710 5711 return inode; 5712 } 5713 5714 static int btrfs_dentry_delete(const struct dentry *dentry) 5715 { 5716 struct btrfs_root *root; 5717 struct inode *inode = d_inode(dentry); 5718 5719 if (!inode && !IS_ROOT(dentry)) 5720 inode = d_inode(dentry->d_parent); 5721 5722 if (inode) { 5723 root = BTRFS_I(inode)->root; 5724 if (btrfs_root_refs(&root->root_item) == 0) 5725 return 1; 5726 5727 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5728 return 1; 5729 } 5730 return 0; 5731 } 5732 5733 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5734 unsigned int flags) 5735 { 5736 struct inode *inode = btrfs_lookup_dentry(dir, dentry); 5737 5738 if (inode == ERR_PTR(-ENOENT)) 5739 inode = NULL; 5740 return d_splice_alias(inode, dentry); 5741 } 5742 5743 /* 5744 * All this infrastructure exists because dir_emit can fault, and we are holding 5745 * the tree lock when doing readdir. For now just allocate a buffer and copy 5746 * our information into that, and then dir_emit from the buffer. This is 5747 * similar to what NFS does, only we don't keep the buffer around in pagecache 5748 * because I'm afraid I'll mess that up. Long term we need to make filldir do 5749 * copy_to_user_inatomic so we don't have to worry about page faulting under the 5750 * tree lock. 5751 */ 5752 static int btrfs_opendir(struct inode *inode, struct file *file) 5753 { 5754 struct btrfs_file_private *private; 5755 5756 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL); 5757 if (!private) 5758 return -ENOMEM; 5759 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 5760 if (!private->filldir_buf) { 5761 kfree(private); 5762 return -ENOMEM; 5763 } 5764 file->private_data = private; 5765 return 0; 5766 } 5767 5768 struct dir_entry { 5769 u64 ino; 5770 u64 offset; 5771 unsigned type; 5772 int name_len; 5773 }; 5774 5775 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) 5776 { 5777 while (entries--) { 5778 struct dir_entry *entry = addr; 5779 char *name = (char *)(entry + 1); 5780 5781 ctx->pos = get_unaligned(&entry->offset); 5782 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len), 5783 get_unaligned(&entry->ino), 5784 get_unaligned(&entry->type))) 5785 return 1; 5786 addr += sizeof(struct dir_entry) + 5787 get_unaligned(&entry->name_len); 5788 ctx->pos++; 5789 } 5790 return 0; 5791 } 5792 5793 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5794 { 5795 struct inode *inode = file_inode(file); 5796 struct btrfs_root *root = BTRFS_I(inode)->root; 5797 struct btrfs_file_private *private = file->private_data; 5798 struct btrfs_dir_item *di; 5799 struct btrfs_key key; 5800 struct btrfs_key found_key; 5801 struct btrfs_path *path; 5802 void *addr; 5803 struct list_head ins_list; 5804 struct list_head del_list; 5805 int ret; 5806 char *name_ptr; 5807 int name_len; 5808 int entries = 0; 5809 int total_len = 0; 5810 bool put = false; 5811 struct btrfs_key location; 5812 5813 if (!dir_emit_dots(file, ctx)) 5814 return 0; 5815 5816 path = btrfs_alloc_path(); 5817 if (!path) 5818 return -ENOMEM; 5819 5820 addr = private->filldir_buf; 5821 path->reada = READA_FORWARD; 5822 5823 INIT_LIST_HEAD(&ins_list); 5824 INIT_LIST_HEAD(&del_list); 5825 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list); 5826 5827 again: 5828 key.type = BTRFS_DIR_INDEX_KEY; 5829 key.offset = ctx->pos; 5830 key.objectid = btrfs_ino(BTRFS_I(inode)); 5831 5832 btrfs_for_each_slot(root, &key, &found_key, path, ret) { 5833 struct dir_entry *entry; 5834 struct extent_buffer *leaf = path->nodes[0]; 5835 u8 ftype; 5836 5837 if (found_key.objectid != key.objectid) 5838 break; 5839 if (found_key.type != BTRFS_DIR_INDEX_KEY) 5840 break; 5841 if (found_key.offset < ctx->pos) 5842 continue; 5843 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 5844 continue; 5845 di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item); 5846 name_len = btrfs_dir_name_len(leaf, di); 5847 if ((total_len + sizeof(struct dir_entry) + name_len) >= 5848 PAGE_SIZE) { 5849 btrfs_release_path(path); 5850 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 5851 if (ret) 5852 goto nopos; 5853 addr = private->filldir_buf; 5854 entries = 0; 5855 total_len = 0; 5856 goto again; 5857 } 5858 5859 ftype = btrfs_dir_flags_to_ftype(btrfs_dir_flags(leaf, di)); 5860 entry = addr; 5861 name_ptr = (char *)(entry + 1); 5862 read_extent_buffer(leaf, name_ptr, 5863 (unsigned long)(di + 1), name_len); 5864 put_unaligned(name_len, &entry->name_len); 5865 put_unaligned(fs_ftype_to_dtype(ftype), &entry->type); 5866 btrfs_dir_item_key_to_cpu(leaf, di, &location); 5867 put_unaligned(location.objectid, &entry->ino); 5868 put_unaligned(found_key.offset, &entry->offset); 5869 entries++; 5870 addr += sizeof(struct dir_entry) + name_len; 5871 total_len += sizeof(struct dir_entry) + name_len; 5872 } 5873 /* Catch error encountered during iteration */ 5874 if (ret < 0) 5875 goto err; 5876 5877 btrfs_release_path(path); 5878 5879 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 5880 if (ret) 5881 goto nopos; 5882 5883 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 5884 if (ret) 5885 goto nopos; 5886 5887 /* 5888 * Stop new entries from being returned after we return the last 5889 * entry. 5890 * 5891 * New directory entries are assigned a strictly increasing 5892 * offset. This means that new entries created during readdir 5893 * are *guaranteed* to be seen in the future by that readdir. 5894 * This has broken buggy programs which operate on names as 5895 * they're returned by readdir. Until we re-use freed offsets 5896 * we have this hack to stop new entries from being returned 5897 * under the assumption that they'll never reach this huge 5898 * offset. 5899 * 5900 * This is being careful not to overflow 32bit loff_t unless the 5901 * last entry requires it because doing so has broken 32bit apps 5902 * in the past. 5903 */ 5904 if (ctx->pos >= INT_MAX) 5905 ctx->pos = LLONG_MAX; 5906 else 5907 ctx->pos = INT_MAX; 5908 nopos: 5909 ret = 0; 5910 err: 5911 if (put) 5912 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list); 5913 btrfs_free_path(path); 5914 return ret; 5915 } 5916 5917 /* 5918 * This is somewhat expensive, updating the tree every time the 5919 * inode changes. But, it is most likely to find the inode in cache. 5920 * FIXME, needs more benchmarking...there are no reasons other than performance 5921 * to keep or drop this code. 5922 */ 5923 static int btrfs_dirty_inode(struct btrfs_inode *inode) 5924 { 5925 struct btrfs_root *root = inode->root; 5926 struct btrfs_fs_info *fs_info = root->fs_info; 5927 struct btrfs_trans_handle *trans; 5928 int ret; 5929 5930 if (test_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags)) 5931 return 0; 5932 5933 trans = btrfs_join_transaction(root); 5934 if (IS_ERR(trans)) 5935 return PTR_ERR(trans); 5936 5937 ret = btrfs_update_inode(trans, root, inode); 5938 if (ret && (ret == -ENOSPC || ret == -EDQUOT)) { 5939 /* whoops, lets try again with the full transaction */ 5940 btrfs_end_transaction(trans); 5941 trans = btrfs_start_transaction(root, 1); 5942 if (IS_ERR(trans)) 5943 return PTR_ERR(trans); 5944 5945 ret = btrfs_update_inode(trans, root, inode); 5946 } 5947 btrfs_end_transaction(trans); 5948 if (inode->delayed_node) 5949 btrfs_balance_delayed_items(fs_info); 5950 5951 return ret; 5952 } 5953 5954 /* 5955 * This is a copy of file_update_time. We need this so we can return error on 5956 * ENOSPC for updating the inode in the case of file write and mmap writes. 5957 */ 5958 static int btrfs_update_time(struct inode *inode, struct timespec64 *now, 5959 int flags) 5960 { 5961 struct btrfs_root *root = BTRFS_I(inode)->root; 5962 bool dirty = flags & ~S_VERSION; 5963 5964 if (btrfs_root_readonly(root)) 5965 return -EROFS; 5966 5967 if (flags & S_VERSION) 5968 dirty |= inode_maybe_inc_iversion(inode, dirty); 5969 if (flags & S_CTIME) 5970 inode->i_ctime = *now; 5971 if (flags & S_MTIME) 5972 inode->i_mtime = *now; 5973 if (flags & S_ATIME) 5974 inode->i_atime = *now; 5975 return dirty ? btrfs_dirty_inode(BTRFS_I(inode)) : 0; 5976 } 5977 5978 /* 5979 * find the highest existing sequence number in a directory 5980 * and then set the in-memory index_cnt variable to reflect 5981 * free sequence numbers 5982 */ 5983 static int btrfs_set_inode_index_count(struct btrfs_inode *inode) 5984 { 5985 struct btrfs_root *root = inode->root; 5986 struct btrfs_key key, found_key; 5987 struct btrfs_path *path; 5988 struct extent_buffer *leaf; 5989 int ret; 5990 5991 key.objectid = btrfs_ino(inode); 5992 key.type = BTRFS_DIR_INDEX_KEY; 5993 key.offset = (u64)-1; 5994 5995 path = btrfs_alloc_path(); 5996 if (!path) 5997 return -ENOMEM; 5998 5999 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 6000 if (ret < 0) 6001 goto out; 6002 /* FIXME: we should be able to handle this */ 6003 if (ret == 0) 6004 goto out; 6005 ret = 0; 6006 6007 if (path->slots[0] == 0) { 6008 inode->index_cnt = BTRFS_DIR_START_INDEX; 6009 goto out; 6010 } 6011 6012 path->slots[0]--; 6013 6014 leaf = path->nodes[0]; 6015 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6016 6017 if (found_key.objectid != btrfs_ino(inode) || 6018 found_key.type != BTRFS_DIR_INDEX_KEY) { 6019 inode->index_cnt = BTRFS_DIR_START_INDEX; 6020 goto out; 6021 } 6022 6023 inode->index_cnt = found_key.offset + 1; 6024 out: 6025 btrfs_free_path(path); 6026 return ret; 6027 } 6028 6029 /* 6030 * helper to find a free sequence number in a given directory. This current 6031 * code is very simple, later versions will do smarter things in the btree 6032 */ 6033 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) 6034 { 6035 int ret = 0; 6036 6037 if (dir->index_cnt == (u64)-1) { 6038 ret = btrfs_inode_delayed_dir_index_count(dir); 6039 if (ret) { 6040 ret = btrfs_set_inode_index_count(dir); 6041 if (ret) 6042 return ret; 6043 } 6044 } 6045 6046 *index = dir->index_cnt; 6047 dir->index_cnt++; 6048 6049 return ret; 6050 } 6051 6052 static int btrfs_insert_inode_locked(struct inode *inode) 6053 { 6054 struct btrfs_iget_args args; 6055 6056 args.ino = BTRFS_I(inode)->location.objectid; 6057 args.root = BTRFS_I(inode)->root; 6058 6059 return insert_inode_locked4(inode, 6060 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 6061 btrfs_find_actor, &args); 6062 } 6063 6064 int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args, 6065 unsigned int *trans_num_items) 6066 { 6067 struct inode *dir = args->dir; 6068 struct inode *inode = args->inode; 6069 int ret; 6070 6071 if (!args->orphan) { 6072 ret = fscrypt_setup_filename(dir, &args->dentry->d_name, 0, 6073 &args->fname); 6074 if (ret) 6075 return ret; 6076 } 6077 6078 ret = posix_acl_create(dir, &inode->i_mode, &args->default_acl, &args->acl); 6079 if (ret) { 6080 fscrypt_free_filename(&args->fname); 6081 return ret; 6082 } 6083 6084 /* 1 to add inode item */ 6085 *trans_num_items = 1; 6086 /* 1 to add compression property */ 6087 if (BTRFS_I(dir)->prop_compress) 6088 (*trans_num_items)++; 6089 /* 1 to add default ACL xattr */ 6090 if (args->default_acl) 6091 (*trans_num_items)++; 6092 /* 1 to add access ACL xattr */ 6093 if (args->acl) 6094 (*trans_num_items)++; 6095 #ifdef CONFIG_SECURITY 6096 /* 1 to add LSM xattr */ 6097 if (dir->i_security) 6098 (*trans_num_items)++; 6099 #endif 6100 if (args->orphan) { 6101 /* 1 to add orphan item */ 6102 (*trans_num_items)++; 6103 } else { 6104 /* 6105 * 1 to add dir item 6106 * 1 to add dir index 6107 * 1 to update parent inode item 6108 * 6109 * No need for 1 unit for the inode ref item because it is 6110 * inserted in a batch together with the inode item at 6111 * btrfs_create_new_inode(). 6112 */ 6113 *trans_num_items += 3; 6114 } 6115 return 0; 6116 } 6117 6118 void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args) 6119 { 6120 posix_acl_release(args->acl); 6121 posix_acl_release(args->default_acl); 6122 fscrypt_free_filename(&args->fname); 6123 } 6124 6125 /* 6126 * Inherit flags from the parent inode. 6127 * 6128 * Currently only the compression flags and the cow flags are inherited. 6129 */ 6130 static void btrfs_inherit_iflags(struct btrfs_inode *inode, struct btrfs_inode *dir) 6131 { 6132 unsigned int flags; 6133 6134 flags = dir->flags; 6135 6136 if (flags & BTRFS_INODE_NOCOMPRESS) { 6137 inode->flags &= ~BTRFS_INODE_COMPRESS; 6138 inode->flags |= BTRFS_INODE_NOCOMPRESS; 6139 } else if (flags & BTRFS_INODE_COMPRESS) { 6140 inode->flags &= ~BTRFS_INODE_NOCOMPRESS; 6141 inode->flags |= BTRFS_INODE_COMPRESS; 6142 } 6143 6144 if (flags & BTRFS_INODE_NODATACOW) { 6145 inode->flags |= BTRFS_INODE_NODATACOW; 6146 if (S_ISREG(inode->vfs_inode.i_mode)) 6147 inode->flags |= BTRFS_INODE_NODATASUM; 6148 } 6149 6150 btrfs_sync_inode_flags_to_i_flags(&inode->vfs_inode); 6151 } 6152 6153 int btrfs_create_new_inode(struct btrfs_trans_handle *trans, 6154 struct btrfs_new_inode_args *args) 6155 { 6156 struct inode *dir = args->dir; 6157 struct inode *inode = args->inode; 6158 const struct fscrypt_str *name = args->orphan ? NULL : &args->fname.disk_name; 6159 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6160 struct btrfs_root *root; 6161 struct btrfs_inode_item *inode_item; 6162 struct btrfs_key *location; 6163 struct btrfs_path *path; 6164 u64 objectid; 6165 struct btrfs_inode_ref *ref; 6166 struct btrfs_key key[2]; 6167 u32 sizes[2]; 6168 struct btrfs_item_batch batch; 6169 unsigned long ptr; 6170 int ret; 6171 6172 path = btrfs_alloc_path(); 6173 if (!path) 6174 return -ENOMEM; 6175 6176 if (!args->subvol) 6177 BTRFS_I(inode)->root = btrfs_grab_root(BTRFS_I(dir)->root); 6178 root = BTRFS_I(inode)->root; 6179 6180 ret = btrfs_get_free_objectid(root, &objectid); 6181 if (ret) 6182 goto out; 6183 inode->i_ino = objectid; 6184 6185 if (args->orphan) { 6186 /* 6187 * O_TMPFILE, set link count to 0, so that after this point, we 6188 * fill in an inode item with the correct link count. 6189 */ 6190 set_nlink(inode, 0); 6191 } else { 6192 trace_btrfs_inode_request(dir); 6193 6194 ret = btrfs_set_inode_index(BTRFS_I(dir), &BTRFS_I(inode)->dir_index); 6195 if (ret) 6196 goto out; 6197 } 6198 /* index_cnt is ignored for everything but a dir. */ 6199 BTRFS_I(inode)->index_cnt = BTRFS_DIR_START_INDEX; 6200 BTRFS_I(inode)->generation = trans->transid; 6201 inode->i_generation = BTRFS_I(inode)->generation; 6202 6203 /* 6204 * Subvolumes don't inherit flags from their parent directory. 6205 * Originally this was probably by accident, but we probably can't 6206 * change it now without compatibility issues. 6207 */ 6208 if (!args->subvol) 6209 btrfs_inherit_iflags(BTRFS_I(inode), BTRFS_I(dir)); 6210 6211 if (S_ISREG(inode->i_mode)) { 6212 if (btrfs_test_opt(fs_info, NODATASUM)) 6213 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6214 if (btrfs_test_opt(fs_info, NODATACOW)) 6215 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6216 BTRFS_INODE_NODATASUM; 6217 } 6218 6219 location = &BTRFS_I(inode)->location; 6220 location->objectid = objectid; 6221 location->offset = 0; 6222 location->type = BTRFS_INODE_ITEM_KEY; 6223 6224 ret = btrfs_insert_inode_locked(inode); 6225 if (ret < 0) { 6226 if (!args->orphan) 6227 BTRFS_I(dir)->index_cnt--; 6228 goto out; 6229 } 6230 6231 /* 6232 * We could have gotten an inode number from somebody who was fsynced 6233 * and then removed in this same transaction, so let's just set full 6234 * sync since it will be a full sync anyway and this will blow away the 6235 * old info in the log. 6236 */ 6237 btrfs_set_inode_full_sync(BTRFS_I(inode)); 6238 6239 key[0].objectid = objectid; 6240 key[0].type = BTRFS_INODE_ITEM_KEY; 6241 key[0].offset = 0; 6242 6243 sizes[0] = sizeof(struct btrfs_inode_item); 6244 6245 if (!args->orphan) { 6246 /* 6247 * Start new inodes with an inode_ref. This is slightly more 6248 * efficient for small numbers of hard links since they will 6249 * be packed into one item. Extended refs will kick in if we 6250 * add more hard links than can fit in the ref item. 6251 */ 6252 key[1].objectid = objectid; 6253 key[1].type = BTRFS_INODE_REF_KEY; 6254 if (args->subvol) { 6255 key[1].offset = objectid; 6256 sizes[1] = 2 + sizeof(*ref); 6257 } else { 6258 key[1].offset = btrfs_ino(BTRFS_I(dir)); 6259 sizes[1] = name->len + sizeof(*ref); 6260 } 6261 } 6262 6263 batch.keys = &key[0]; 6264 batch.data_sizes = &sizes[0]; 6265 batch.total_data_size = sizes[0] + (args->orphan ? 0 : sizes[1]); 6266 batch.nr = args->orphan ? 1 : 2; 6267 ret = btrfs_insert_empty_items(trans, root, path, &batch); 6268 if (ret != 0) { 6269 btrfs_abort_transaction(trans, ret); 6270 goto discard; 6271 } 6272 6273 inode->i_mtime = current_time(inode); 6274 inode->i_atime = inode->i_mtime; 6275 inode->i_ctime = inode->i_mtime; 6276 BTRFS_I(inode)->i_otime = inode->i_mtime; 6277 6278 /* 6279 * We're going to fill the inode item now, so at this point the inode 6280 * must be fully initialized. 6281 */ 6282 6283 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6284 struct btrfs_inode_item); 6285 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6286 sizeof(*inode_item)); 6287 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6288 6289 if (!args->orphan) { 6290 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6291 struct btrfs_inode_ref); 6292 ptr = (unsigned long)(ref + 1); 6293 if (args->subvol) { 6294 btrfs_set_inode_ref_name_len(path->nodes[0], ref, 2); 6295 btrfs_set_inode_ref_index(path->nodes[0], ref, 0); 6296 write_extent_buffer(path->nodes[0], "..", ptr, 2); 6297 } else { 6298 btrfs_set_inode_ref_name_len(path->nodes[0], ref, 6299 name->len); 6300 btrfs_set_inode_ref_index(path->nodes[0], ref, 6301 BTRFS_I(inode)->dir_index); 6302 write_extent_buffer(path->nodes[0], name->name, ptr, 6303 name->len); 6304 } 6305 } 6306 6307 btrfs_mark_buffer_dirty(path->nodes[0]); 6308 /* 6309 * We don't need the path anymore, plus inheriting properties, adding 6310 * ACLs, security xattrs, orphan item or adding the link, will result in 6311 * allocating yet another path. So just free our path. 6312 */ 6313 btrfs_free_path(path); 6314 path = NULL; 6315 6316 if (args->subvol) { 6317 struct inode *parent; 6318 6319 /* 6320 * Subvolumes inherit properties from their parent subvolume, 6321 * not the directory they were created in. 6322 */ 6323 parent = btrfs_iget(fs_info->sb, BTRFS_FIRST_FREE_OBJECTID, 6324 BTRFS_I(dir)->root); 6325 if (IS_ERR(parent)) { 6326 ret = PTR_ERR(parent); 6327 } else { 6328 ret = btrfs_inode_inherit_props(trans, inode, parent); 6329 iput(parent); 6330 } 6331 } else { 6332 ret = btrfs_inode_inherit_props(trans, inode, dir); 6333 } 6334 if (ret) { 6335 btrfs_err(fs_info, 6336 "error inheriting props for ino %llu (root %llu): %d", 6337 btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, 6338 ret); 6339 } 6340 6341 /* 6342 * Subvolumes don't inherit ACLs or get passed to the LSM. This is 6343 * probably a bug. 6344 */ 6345 if (!args->subvol) { 6346 ret = btrfs_init_inode_security(trans, args); 6347 if (ret) { 6348 btrfs_abort_transaction(trans, ret); 6349 goto discard; 6350 } 6351 } 6352 6353 inode_tree_add(BTRFS_I(inode)); 6354 6355 trace_btrfs_inode_new(inode); 6356 btrfs_set_inode_last_trans(trans, BTRFS_I(inode)); 6357 6358 btrfs_update_root_times(trans, root); 6359 6360 if (args->orphan) { 6361 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 6362 } else { 6363 ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name, 6364 0, BTRFS_I(inode)->dir_index); 6365 } 6366 if (ret) { 6367 btrfs_abort_transaction(trans, ret); 6368 goto discard; 6369 } 6370 6371 return 0; 6372 6373 discard: 6374 /* 6375 * discard_new_inode() calls iput(), but the caller owns the reference 6376 * to the inode. 6377 */ 6378 ihold(inode); 6379 discard_new_inode(inode); 6380 out: 6381 btrfs_free_path(path); 6382 return ret; 6383 } 6384 6385 /* 6386 * utility function to add 'inode' into 'parent_inode' with 6387 * a give name and a given sequence number. 6388 * if 'add_backref' is true, also insert a backref from the 6389 * inode to the parent directory. 6390 */ 6391 int btrfs_add_link(struct btrfs_trans_handle *trans, 6392 struct btrfs_inode *parent_inode, struct btrfs_inode *inode, 6393 const struct fscrypt_str *name, int add_backref, u64 index) 6394 { 6395 int ret = 0; 6396 struct btrfs_key key; 6397 struct btrfs_root *root = parent_inode->root; 6398 u64 ino = btrfs_ino(inode); 6399 u64 parent_ino = btrfs_ino(parent_inode); 6400 6401 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6402 memcpy(&key, &inode->root->root_key, sizeof(key)); 6403 } else { 6404 key.objectid = ino; 6405 key.type = BTRFS_INODE_ITEM_KEY; 6406 key.offset = 0; 6407 } 6408 6409 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6410 ret = btrfs_add_root_ref(trans, key.objectid, 6411 root->root_key.objectid, parent_ino, 6412 index, name); 6413 } else if (add_backref) { 6414 ret = btrfs_insert_inode_ref(trans, root, name, 6415 ino, parent_ino, index); 6416 } 6417 6418 /* Nothing to clean up yet */ 6419 if (ret) 6420 return ret; 6421 6422 ret = btrfs_insert_dir_item(trans, name, parent_inode, &key, 6423 btrfs_inode_type(&inode->vfs_inode), index); 6424 if (ret == -EEXIST || ret == -EOVERFLOW) 6425 goto fail_dir_item; 6426 else if (ret) { 6427 btrfs_abort_transaction(trans, ret); 6428 return ret; 6429 } 6430 6431 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + 6432 name->len * 2); 6433 inode_inc_iversion(&parent_inode->vfs_inode); 6434 /* 6435 * If we are replaying a log tree, we do not want to update the mtime 6436 * and ctime of the parent directory with the current time, since the 6437 * log replay procedure is responsible for setting them to their correct 6438 * values (the ones it had when the fsync was done). 6439 */ 6440 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags)) { 6441 struct timespec64 now = current_time(&parent_inode->vfs_inode); 6442 6443 parent_inode->vfs_inode.i_mtime = now; 6444 parent_inode->vfs_inode.i_ctime = now; 6445 } 6446 ret = btrfs_update_inode(trans, root, parent_inode); 6447 if (ret) 6448 btrfs_abort_transaction(trans, ret); 6449 return ret; 6450 6451 fail_dir_item: 6452 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6453 u64 local_index; 6454 int err; 6455 err = btrfs_del_root_ref(trans, key.objectid, 6456 root->root_key.objectid, parent_ino, 6457 &local_index, name); 6458 if (err) 6459 btrfs_abort_transaction(trans, err); 6460 } else if (add_backref) { 6461 u64 local_index; 6462 int err; 6463 6464 err = btrfs_del_inode_ref(trans, root, name, ino, parent_ino, 6465 &local_index); 6466 if (err) 6467 btrfs_abort_transaction(trans, err); 6468 } 6469 6470 /* Return the original error code */ 6471 return ret; 6472 } 6473 6474 static int btrfs_create_common(struct inode *dir, struct dentry *dentry, 6475 struct inode *inode) 6476 { 6477 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 6478 struct btrfs_root *root = BTRFS_I(dir)->root; 6479 struct btrfs_new_inode_args new_inode_args = { 6480 .dir = dir, 6481 .dentry = dentry, 6482 .inode = inode, 6483 }; 6484 unsigned int trans_num_items; 6485 struct btrfs_trans_handle *trans; 6486 int err; 6487 6488 err = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); 6489 if (err) 6490 goto out_inode; 6491 6492 trans = btrfs_start_transaction(root, trans_num_items); 6493 if (IS_ERR(trans)) { 6494 err = PTR_ERR(trans); 6495 goto out_new_inode_args; 6496 } 6497 6498 err = btrfs_create_new_inode(trans, &new_inode_args); 6499 if (!err) 6500 d_instantiate_new(dentry, inode); 6501 6502 btrfs_end_transaction(trans); 6503 btrfs_btree_balance_dirty(fs_info); 6504 out_new_inode_args: 6505 btrfs_new_inode_args_destroy(&new_inode_args); 6506 out_inode: 6507 if (err) 6508 iput(inode); 6509 return err; 6510 } 6511 6512 static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir, 6513 struct dentry *dentry, umode_t mode, dev_t rdev) 6514 { 6515 struct inode *inode; 6516 6517 inode = new_inode(dir->i_sb); 6518 if (!inode) 6519 return -ENOMEM; 6520 inode_init_owner(idmap, inode, dir, mode); 6521 inode->i_op = &btrfs_special_inode_operations; 6522 init_special_inode(inode, inode->i_mode, rdev); 6523 return btrfs_create_common(dir, dentry, inode); 6524 } 6525 6526 static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir, 6527 struct dentry *dentry, umode_t mode, bool excl) 6528 { 6529 struct inode *inode; 6530 6531 inode = new_inode(dir->i_sb); 6532 if (!inode) 6533 return -ENOMEM; 6534 inode_init_owner(idmap, inode, dir, mode); 6535 inode->i_fop = &btrfs_file_operations; 6536 inode->i_op = &btrfs_file_inode_operations; 6537 inode->i_mapping->a_ops = &btrfs_aops; 6538 return btrfs_create_common(dir, dentry, inode); 6539 } 6540 6541 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6542 struct dentry *dentry) 6543 { 6544 struct btrfs_trans_handle *trans = NULL; 6545 struct btrfs_root *root = BTRFS_I(dir)->root; 6546 struct inode *inode = d_inode(old_dentry); 6547 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 6548 struct fscrypt_name fname; 6549 u64 index; 6550 int err; 6551 int drop_inode = 0; 6552 6553 /* do not allow sys_link's with other subvols of the same device */ 6554 if (root->root_key.objectid != BTRFS_I(inode)->root->root_key.objectid) 6555 return -EXDEV; 6556 6557 if (inode->i_nlink >= BTRFS_LINK_MAX) 6558 return -EMLINK; 6559 6560 err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &fname); 6561 if (err) 6562 goto fail; 6563 6564 err = btrfs_set_inode_index(BTRFS_I(dir), &index); 6565 if (err) 6566 goto fail; 6567 6568 /* 6569 * 2 items for inode and inode ref 6570 * 2 items for dir items 6571 * 1 item for parent inode 6572 * 1 item for orphan item deletion if O_TMPFILE 6573 */ 6574 trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6); 6575 if (IS_ERR(trans)) { 6576 err = PTR_ERR(trans); 6577 trans = NULL; 6578 goto fail; 6579 } 6580 6581 /* There are several dir indexes for this inode, clear the cache. */ 6582 BTRFS_I(inode)->dir_index = 0ULL; 6583 inc_nlink(inode); 6584 inode_inc_iversion(inode); 6585 inode->i_ctime = current_time(inode); 6586 ihold(inode); 6587 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6588 6589 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), 6590 &fname.disk_name, 1, index); 6591 6592 if (err) { 6593 drop_inode = 1; 6594 } else { 6595 struct dentry *parent = dentry->d_parent; 6596 6597 err = btrfs_update_inode(trans, root, BTRFS_I(inode)); 6598 if (err) 6599 goto fail; 6600 if (inode->i_nlink == 1) { 6601 /* 6602 * If new hard link count is 1, it's a file created 6603 * with open(2) O_TMPFILE flag. 6604 */ 6605 err = btrfs_orphan_del(trans, BTRFS_I(inode)); 6606 if (err) 6607 goto fail; 6608 } 6609 d_instantiate(dentry, inode); 6610 btrfs_log_new_name(trans, old_dentry, NULL, 0, parent); 6611 } 6612 6613 fail: 6614 fscrypt_free_filename(&fname); 6615 if (trans) 6616 btrfs_end_transaction(trans); 6617 if (drop_inode) { 6618 inode_dec_link_count(inode); 6619 iput(inode); 6620 } 6621 btrfs_btree_balance_dirty(fs_info); 6622 return err; 6623 } 6624 6625 static int btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, 6626 struct dentry *dentry, umode_t mode) 6627 { 6628 struct inode *inode; 6629 6630 inode = new_inode(dir->i_sb); 6631 if (!inode) 6632 return -ENOMEM; 6633 inode_init_owner(idmap, inode, dir, S_IFDIR | mode); 6634 inode->i_op = &btrfs_dir_inode_operations; 6635 inode->i_fop = &btrfs_dir_file_operations; 6636 return btrfs_create_common(dir, dentry, inode); 6637 } 6638 6639 static noinline int uncompress_inline(struct btrfs_path *path, 6640 struct page *page, 6641 struct btrfs_file_extent_item *item) 6642 { 6643 int ret; 6644 struct extent_buffer *leaf = path->nodes[0]; 6645 char *tmp; 6646 size_t max_size; 6647 unsigned long inline_size; 6648 unsigned long ptr; 6649 int compress_type; 6650 6651 compress_type = btrfs_file_extent_compression(leaf, item); 6652 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6653 inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]); 6654 tmp = kmalloc(inline_size, GFP_NOFS); 6655 if (!tmp) 6656 return -ENOMEM; 6657 ptr = btrfs_file_extent_inline_start(item); 6658 6659 read_extent_buffer(leaf, tmp, ptr, inline_size); 6660 6661 max_size = min_t(unsigned long, PAGE_SIZE, max_size); 6662 ret = btrfs_decompress(compress_type, tmp, page, 0, inline_size, max_size); 6663 6664 /* 6665 * decompression code contains a memset to fill in any space between the end 6666 * of the uncompressed data and the end of max_size in case the decompressed 6667 * data ends up shorter than ram_bytes. That doesn't cover the hole between 6668 * the end of an inline extent and the beginning of the next block, so we 6669 * cover that region here. 6670 */ 6671 6672 if (max_size < PAGE_SIZE) 6673 memzero_page(page, max_size, PAGE_SIZE - max_size); 6674 kfree(tmp); 6675 return ret; 6676 } 6677 6678 static int read_inline_extent(struct btrfs_inode *inode, struct btrfs_path *path, 6679 struct page *page) 6680 { 6681 struct btrfs_file_extent_item *fi; 6682 void *kaddr; 6683 size_t copy_size; 6684 6685 if (!page || PageUptodate(page)) 6686 return 0; 6687 6688 ASSERT(page_offset(page) == 0); 6689 6690 fi = btrfs_item_ptr(path->nodes[0], path->slots[0], 6691 struct btrfs_file_extent_item); 6692 if (btrfs_file_extent_compression(path->nodes[0], fi) != BTRFS_COMPRESS_NONE) 6693 return uncompress_inline(path, page, fi); 6694 6695 copy_size = min_t(u64, PAGE_SIZE, 6696 btrfs_file_extent_ram_bytes(path->nodes[0], fi)); 6697 kaddr = kmap_local_page(page); 6698 read_extent_buffer(path->nodes[0], kaddr, 6699 btrfs_file_extent_inline_start(fi), copy_size); 6700 kunmap_local(kaddr); 6701 if (copy_size < PAGE_SIZE) 6702 memzero_page(page, copy_size, PAGE_SIZE - copy_size); 6703 return 0; 6704 } 6705 6706 /* 6707 * Lookup the first extent overlapping a range in a file. 6708 * 6709 * @inode: file to search in 6710 * @page: page to read extent data into if the extent is inline 6711 * @pg_offset: offset into @page to copy to 6712 * @start: file offset 6713 * @len: length of range starting at @start 6714 * 6715 * Return the first &struct extent_map which overlaps the given range, reading 6716 * it from the B-tree and caching it if necessary. Note that there may be more 6717 * extents which overlap the given range after the returned extent_map. 6718 * 6719 * If @page is not NULL and the extent is inline, this also reads the extent 6720 * data directly into the page and marks the extent up to date in the io_tree. 6721 * 6722 * Return: ERR_PTR on error, non-NULL extent_map on success. 6723 */ 6724 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, 6725 struct page *page, size_t pg_offset, 6726 u64 start, u64 len) 6727 { 6728 struct btrfs_fs_info *fs_info = inode->root->fs_info; 6729 int ret = 0; 6730 u64 extent_start = 0; 6731 u64 extent_end = 0; 6732 u64 objectid = btrfs_ino(inode); 6733 int extent_type = -1; 6734 struct btrfs_path *path = NULL; 6735 struct btrfs_root *root = inode->root; 6736 struct btrfs_file_extent_item *item; 6737 struct extent_buffer *leaf; 6738 struct btrfs_key found_key; 6739 struct extent_map *em = NULL; 6740 struct extent_map_tree *em_tree = &inode->extent_tree; 6741 6742 read_lock(&em_tree->lock); 6743 em = lookup_extent_mapping(em_tree, start, len); 6744 read_unlock(&em_tree->lock); 6745 6746 if (em) { 6747 if (em->start > start || em->start + em->len <= start) 6748 free_extent_map(em); 6749 else if (em->block_start == EXTENT_MAP_INLINE && page) 6750 free_extent_map(em); 6751 else 6752 goto out; 6753 } 6754 em = alloc_extent_map(); 6755 if (!em) { 6756 ret = -ENOMEM; 6757 goto out; 6758 } 6759 em->start = EXTENT_MAP_HOLE; 6760 em->orig_start = EXTENT_MAP_HOLE; 6761 em->len = (u64)-1; 6762 em->block_len = (u64)-1; 6763 6764 path = btrfs_alloc_path(); 6765 if (!path) { 6766 ret = -ENOMEM; 6767 goto out; 6768 } 6769 6770 /* Chances are we'll be called again, so go ahead and do readahead */ 6771 path->reada = READA_FORWARD; 6772 6773 /* 6774 * The same explanation in load_free_space_cache applies here as well, 6775 * we only read when we're loading the free space cache, and at that 6776 * point the commit_root has everything we need. 6777 */ 6778 if (btrfs_is_free_space_inode(inode)) { 6779 path->search_commit_root = 1; 6780 path->skip_locking = 1; 6781 } 6782 6783 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); 6784 if (ret < 0) { 6785 goto out; 6786 } else if (ret > 0) { 6787 if (path->slots[0] == 0) 6788 goto not_found; 6789 path->slots[0]--; 6790 ret = 0; 6791 } 6792 6793 leaf = path->nodes[0]; 6794 item = btrfs_item_ptr(leaf, path->slots[0], 6795 struct btrfs_file_extent_item); 6796 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6797 if (found_key.objectid != objectid || 6798 found_key.type != BTRFS_EXTENT_DATA_KEY) { 6799 /* 6800 * If we backup past the first extent we want to move forward 6801 * and see if there is an extent in front of us, otherwise we'll 6802 * say there is a hole for our whole search range which can 6803 * cause problems. 6804 */ 6805 extent_end = start; 6806 goto next; 6807 } 6808 6809 extent_type = btrfs_file_extent_type(leaf, item); 6810 extent_start = found_key.offset; 6811 extent_end = btrfs_file_extent_end(path); 6812 if (extent_type == BTRFS_FILE_EXTENT_REG || 6813 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 6814 /* Only regular file could have regular/prealloc extent */ 6815 if (!S_ISREG(inode->vfs_inode.i_mode)) { 6816 ret = -EUCLEAN; 6817 btrfs_crit(fs_info, 6818 "regular/prealloc extent found for non-regular inode %llu", 6819 btrfs_ino(inode)); 6820 goto out; 6821 } 6822 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, 6823 extent_start); 6824 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 6825 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, 6826 path->slots[0], 6827 extent_start); 6828 } 6829 next: 6830 if (start >= extent_end) { 6831 path->slots[0]++; 6832 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 6833 ret = btrfs_next_leaf(root, path); 6834 if (ret < 0) 6835 goto out; 6836 else if (ret > 0) 6837 goto not_found; 6838 6839 leaf = path->nodes[0]; 6840 } 6841 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6842 if (found_key.objectid != objectid || 6843 found_key.type != BTRFS_EXTENT_DATA_KEY) 6844 goto not_found; 6845 if (start + len <= found_key.offset) 6846 goto not_found; 6847 if (start > found_key.offset) 6848 goto next; 6849 6850 /* New extent overlaps with existing one */ 6851 em->start = start; 6852 em->orig_start = start; 6853 em->len = found_key.offset - start; 6854 em->block_start = EXTENT_MAP_HOLE; 6855 goto insert; 6856 } 6857 6858 btrfs_extent_item_to_extent_map(inode, path, item, em); 6859 6860 if (extent_type == BTRFS_FILE_EXTENT_REG || 6861 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 6862 goto insert; 6863 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 6864 /* 6865 * Inline extent can only exist at file offset 0. This is 6866 * ensured by tree-checker and inline extent creation path. 6867 * Thus all members representing file offsets should be zero. 6868 */ 6869 ASSERT(pg_offset == 0); 6870 ASSERT(extent_start == 0); 6871 ASSERT(em->start == 0); 6872 6873 /* 6874 * btrfs_extent_item_to_extent_map() should have properly 6875 * initialized em members already. 6876 * 6877 * Other members are not utilized for inline extents. 6878 */ 6879 ASSERT(em->block_start == EXTENT_MAP_INLINE); 6880 ASSERT(em->len == fs_info->sectorsize); 6881 6882 ret = read_inline_extent(inode, path, page); 6883 if (ret < 0) 6884 goto out; 6885 goto insert; 6886 } 6887 not_found: 6888 em->start = start; 6889 em->orig_start = start; 6890 em->len = len; 6891 em->block_start = EXTENT_MAP_HOLE; 6892 insert: 6893 ret = 0; 6894 btrfs_release_path(path); 6895 if (em->start > start || extent_map_end(em) <= start) { 6896 btrfs_err(fs_info, 6897 "bad extent! em: [%llu %llu] passed [%llu %llu]", 6898 em->start, em->len, start, len); 6899 ret = -EIO; 6900 goto out; 6901 } 6902 6903 write_lock(&em_tree->lock); 6904 ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len); 6905 write_unlock(&em_tree->lock); 6906 out: 6907 btrfs_free_path(path); 6908 6909 trace_btrfs_get_extent(root, inode, em); 6910 6911 if (ret) { 6912 free_extent_map(em); 6913 return ERR_PTR(ret); 6914 } 6915 return em; 6916 } 6917 6918 static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, 6919 struct btrfs_dio_data *dio_data, 6920 const u64 start, 6921 const u64 len, 6922 const u64 orig_start, 6923 const u64 block_start, 6924 const u64 block_len, 6925 const u64 orig_block_len, 6926 const u64 ram_bytes, 6927 const int type) 6928 { 6929 struct extent_map *em = NULL; 6930 struct btrfs_ordered_extent *ordered; 6931 6932 if (type != BTRFS_ORDERED_NOCOW) { 6933 em = create_io_em(inode, start, len, orig_start, block_start, 6934 block_len, orig_block_len, ram_bytes, 6935 BTRFS_COMPRESS_NONE, /* compress_type */ 6936 type); 6937 if (IS_ERR(em)) 6938 goto out; 6939 } 6940 ordered = btrfs_alloc_ordered_extent(inode, start, len, len, 6941 block_start, block_len, 0, 6942 (1 << type) | 6943 (1 << BTRFS_ORDERED_DIRECT), 6944 BTRFS_COMPRESS_NONE); 6945 if (IS_ERR(ordered)) { 6946 if (em) { 6947 free_extent_map(em); 6948 btrfs_drop_extent_map_range(inode, start, 6949 start + len - 1, false); 6950 } 6951 em = ERR_CAST(ordered); 6952 } else { 6953 ASSERT(!dio_data->ordered); 6954 dio_data->ordered = ordered; 6955 } 6956 out: 6957 6958 return em; 6959 } 6960 6961 static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode, 6962 struct btrfs_dio_data *dio_data, 6963 u64 start, u64 len) 6964 { 6965 struct btrfs_root *root = inode->root; 6966 struct btrfs_fs_info *fs_info = root->fs_info; 6967 struct extent_map *em; 6968 struct btrfs_key ins; 6969 u64 alloc_hint; 6970 int ret; 6971 6972 alloc_hint = get_extent_allocation_hint(inode, start, len); 6973 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, 6974 0, alloc_hint, &ins, 1, 1); 6975 if (ret) 6976 return ERR_PTR(ret); 6977 6978 em = btrfs_create_dio_extent(inode, dio_data, start, ins.offset, start, 6979 ins.objectid, ins.offset, ins.offset, 6980 ins.offset, BTRFS_ORDERED_REGULAR); 6981 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 6982 if (IS_ERR(em)) 6983 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 6984 1); 6985 6986 return em; 6987 } 6988 6989 static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr) 6990 { 6991 struct btrfs_block_group *block_group; 6992 bool readonly = false; 6993 6994 block_group = btrfs_lookup_block_group(fs_info, bytenr); 6995 if (!block_group || block_group->ro) 6996 readonly = true; 6997 if (block_group) 6998 btrfs_put_block_group(block_group); 6999 return readonly; 7000 } 7001 7002 /* 7003 * Check if we can do nocow write into the range [@offset, @offset + @len) 7004 * 7005 * @offset: File offset 7006 * @len: The length to write, will be updated to the nocow writeable 7007 * range 7008 * @orig_start: (optional) Return the original file offset of the file extent 7009 * @orig_len: (optional) Return the original on-disk length of the file extent 7010 * @ram_bytes: (optional) Return the ram_bytes of the file extent 7011 * @strict: if true, omit optimizations that might force us into unnecessary 7012 * cow. e.g., don't trust generation number. 7013 * 7014 * Return: 7015 * >0 and update @len if we can do nocow write 7016 * 0 if we can't do nocow write 7017 * <0 if error happened 7018 * 7019 * NOTE: This only checks the file extents, caller is responsible to wait for 7020 * any ordered extents. 7021 */ 7022 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 7023 u64 *orig_start, u64 *orig_block_len, 7024 u64 *ram_bytes, bool nowait, bool strict) 7025 { 7026 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7027 struct can_nocow_file_extent_args nocow_args = { 0 }; 7028 struct btrfs_path *path; 7029 int ret; 7030 struct extent_buffer *leaf; 7031 struct btrfs_root *root = BTRFS_I(inode)->root; 7032 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7033 struct btrfs_file_extent_item *fi; 7034 struct btrfs_key key; 7035 int found_type; 7036 7037 path = btrfs_alloc_path(); 7038 if (!path) 7039 return -ENOMEM; 7040 path->nowait = nowait; 7041 7042 ret = btrfs_lookup_file_extent(NULL, root, path, 7043 btrfs_ino(BTRFS_I(inode)), offset, 0); 7044 if (ret < 0) 7045 goto out; 7046 7047 if (ret == 1) { 7048 if (path->slots[0] == 0) { 7049 /* can't find the item, must cow */ 7050 ret = 0; 7051 goto out; 7052 } 7053 path->slots[0]--; 7054 } 7055 ret = 0; 7056 leaf = path->nodes[0]; 7057 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 7058 if (key.objectid != btrfs_ino(BTRFS_I(inode)) || 7059 key.type != BTRFS_EXTENT_DATA_KEY) { 7060 /* not our file or wrong item type, must cow */ 7061 goto out; 7062 } 7063 7064 if (key.offset > offset) { 7065 /* Wrong offset, must cow */ 7066 goto out; 7067 } 7068 7069 if (btrfs_file_extent_end(path) <= offset) 7070 goto out; 7071 7072 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); 7073 found_type = btrfs_file_extent_type(leaf, fi); 7074 if (ram_bytes) 7075 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 7076 7077 nocow_args.start = offset; 7078 nocow_args.end = offset + *len - 1; 7079 nocow_args.strict = strict; 7080 nocow_args.free_path = true; 7081 7082 ret = can_nocow_file_extent(path, &key, BTRFS_I(inode), &nocow_args); 7083 /* can_nocow_file_extent() has freed the path. */ 7084 path = NULL; 7085 7086 if (ret != 1) { 7087 /* Treat errors as not being able to NOCOW. */ 7088 ret = 0; 7089 goto out; 7090 } 7091 7092 ret = 0; 7093 if (btrfs_extent_readonly(fs_info, nocow_args.disk_bytenr)) 7094 goto out; 7095 7096 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7097 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7098 u64 range_end; 7099 7100 range_end = round_up(offset + nocow_args.num_bytes, 7101 root->fs_info->sectorsize) - 1; 7102 ret = test_range_bit(io_tree, offset, range_end, 7103 EXTENT_DELALLOC, 0, NULL); 7104 if (ret) { 7105 ret = -EAGAIN; 7106 goto out; 7107 } 7108 } 7109 7110 if (orig_start) 7111 *orig_start = key.offset - nocow_args.extent_offset; 7112 if (orig_block_len) 7113 *orig_block_len = nocow_args.disk_num_bytes; 7114 7115 *len = nocow_args.num_bytes; 7116 ret = 1; 7117 out: 7118 btrfs_free_path(path); 7119 return ret; 7120 } 7121 7122 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7123 struct extent_state **cached_state, 7124 unsigned int iomap_flags) 7125 { 7126 const bool writing = (iomap_flags & IOMAP_WRITE); 7127 const bool nowait = (iomap_flags & IOMAP_NOWAIT); 7128 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 7129 struct btrfs_ordered_extent *ordered; 7130 int ret = 0; 7131 7132 while (1) { 7133 if (nowait) { 7134 if (!try_lock_extent(io_tree, lockstart, lockend, 7135 cached_state)) 7136 return -EAGAIN; 7137 } else { 7138 lock_extent(io_tree, lockstart, lockend, cached_state); 7139 } 7140 /* 7141 * We're concerned with the entire range that we're going to be 7142 * doing DIO to, so we need to make sure there's no ordered 7143 * extents in this range. 7144 */ 7145 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart, 7146 lockend - lockstart + 1); 7147 7148 /* 7149 * We need to make sure there are no buffered pages in this 7150 * range either, we could have raced between the invalidate in 7151 * generic_file_direct_write and locking the extent. The 7152 * invalidate needs to happen so that reads after a write do not 7153 * get stale data. 7154 */ 7155 if (!ordered && 7156 (!writing || !filemap_range_has_page(inode->i_mapping, 7157 lockstart, lockend))) 7158 break; 7159 7160 unlock_extent(io_tree, lockstart, lockend, cached_state); 7161 7162 if (ordered) { 7163 if (nowait) { 7164 btrfs_put_ordered_extent(ordered); 7165 ret = -EAGAIN; 7166 break; 7167 } 7168 /* 7169 * If we are doing a DIO read and the ordered extent we 7170 * found is for a buffered write, we can not wait for it 7171 * to complete and retry, because if we do so we can 7172 * deadlock with concurrent buffered writes on page 7173 * locks. This happens only if our DIO read covers more 7174 * than one extent map, if at this point has already 7175 * created an ordered extent for a previous extent map 7176 * and locked its range in the inode's io tree, and a 7177 * concurrent write against that previous extent map's 7178 * range and this range started (we unlock the ranges 7179 * in the io tree only when the bios complete and 7180 * buffered writes always lock pages before attempting 7181 * to lock range in the io tree). 7182 */ 7183 if (writing || 7184 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) 7185 btrfs_start_ordered_extent(ordered); 7186 else 7187 ret = nowait ? -EAGAIN : -ENOTBLK; 7188 btrfs_put_ordered_extent(ordered); 7189 } else { 7190 /* 7191 * We could trigger writeback for this range (and wait 7192 * for it to complete) and then invalidate the pages for 7193 * this range (through invalidate_inode_pages2_range()), 7194 * but that can lead us to a deadlock with a concurrent 7195 * call to readahead (a buffered read or a defrag call 7196 * triggered a readahead) on a page lock due to an 7197 * ordered dio extent we created before but did not have 7198 * yet a corresponding bio submitted (whence it can not 7199 * complete), which makes readahead wait for that 7200 * ordered extent to complete while holding a lock on 7201 * that page. 7202 */ 7203 ret = nowait ? -EAGAIN : -ENOTBLK; 7204 } 7205 7206 if (ret) 7207 break; 7208 7209 cond_resched(); 7210 } 7211 7212 return ret; 7213 } 7214 7215 /* The callers of this must take lock_extent() */ 7216 static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, 7217 u64 len, u64 orig_start, u64 block_start, 7218 u64 block_len, u64 orig_block_len, 7219 u64 ram_bytes, int compress_type, 7220 int type) 7221 { 7222 struct extent_map *em; 7223 int ret; 7224 7225 ASSERT(type == BTRFS_ORDERED_PREALLOC || 7226 type == BTRFS_ORDERED_COMPRESSED || 7227 type == BTRFS_ORDERED_NOCOW || 7228 type == BTRFS_ORDERED_REGULAR); 7229 7230 em = alloc_extent_map(); 7231 if (!em) 7232 return ERR_PTR(-ENOMEM); 7233 7234 em->start = start; 7235 em->orig_start = orig_start; 7236 em->len = len; 7237 em->block_len = block_len; 7238 em->block_start = block_start; 7239 em->orig_block_len = orig_block_len; 7240 em->ram_bytes = ram_bytes; 7241 em->generation = -1; 7242 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7243 if (type == BTRFS_ORDERED_PREALLOC) { 7244 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7245 } else if (type == BTRFS_ORDERED_COMPRESSED) { 7246 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 7247 em->compress_type = compress_type; 7248 } 7249 7250 ret = btrfs_replace_extent_map_range(inode, em, true); 7251 if (ret) { 7252 free_extent_map(em); 7253 return ERR_PTR(ret); 7254 } 7255 7256 /* em got 2 refs now, callers needs to do free_extent_map once. */ 7257 return em; 7258 } 7259 7260 7261 static int btrfs_get_blocks_direct_write(struct extent_map **map, 7262 struct inode *inode, 7263 struct btrfs_dio_data *dio_data, 7264 u64 start, u64 len, 7265 unsigned int iomap_flags) 7266 { 7267 const bool nowait = (iomap_flags & IOMAP_NOWAIT); 7268 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7269 struct extent_map *em = *map; 7270 int type; 7271 u64 block_start, orig_start, orig_block_len, ram_bytes; 7272 struct btrfs_block_group *bg; 7273 bool can_nocow = false; 7274 bool space_reserved = false; 7275 u64 prev_len; 7276 int ret = 0; 7277 7278 /* 7279 * We don't allocate a new extent in the following cases 7280 * 7281 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7282 * existing extent. 7283 * 2) The extent is marked as PREALLOC. We're good to go here and can 7284 * just use the extent. 7285 * 7286 */ 7287 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7288 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7289 em->block_start != EXTENT_MAP_HOLE)) { 7290 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7291 type = BTRFS_ORDERED_PREALLOC; 7292 else 7293 type = BTRFS_ORDERED_NOCOW; 7294 len = min(len, em->len - (start - em->start)); 7295 block_start = em->block_start + (start - em->start); 7296 7297 if (can_nocow_extent(inode, start, &len, &orig_start, 7298 &orig_block_len, &ram_bytes, false, false) == 1) { 7299 bg = btrfs_inc_nocow_writers(fs_info, block_start); 7300 if (bg) 7301 can_nocow = true; 7302 } 7303 } 7304 7305 prev_len = len; 7306 if (can_nocow) { 7307 struct extent_map *em2; 7308 7309 /* We can NOCOW, so only need to reserve metadata space. */ 7310 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len, len, 7311 nowait); 7312 if (ret < 0) { 7313 /* Our caller expects us to free the input extent map. */ 7314 free_extent_map(em); 7315 *map = NULL; 7316 btrfs_dec_nocow_writers(bg); 7317 if (nowait && (ret == -ENOSPC || ret == -EDQUOT)) 7318 ret = -EAGAIN; 7319 goto out; 7320 } 7321 space_reserved = true; 7322 7323 em2 = btrfs_create_dio_extent(BTRFS_I(inode), dio_data, start, len, 7324 orig_start, block_start, 7325 len, orig_block_len, 7326 ram_bytes, type); 7327 btrfs_dec_nocow_writers(bg); 7328 if (type == BTRFS_ORDERED_PREALLOC) { 7329 free_extent_map(em); 7330 *map = em2; 7331 em = em2; 7332 } 7333 7334 if (IS_ERR(em2)) { 7335 ret = PTR_ERR(em2); 7336 goto out; 7337 } 7338 7339 dio_data->nocow_done = true; 7340 } else { 7341 /* Our caller expects us to free the input extent map. */ 7342 free_extent_map(em); 7343 *map = NULL; 7344 7345 if (nowait) 7346 return -EAGAIN; 7347 7348 /* 7349 * If we could not allocate data space before locking the file 7350 * range and we can't do a NOCOW write, then we have to fail. 7351 */ 7352 if (!dio_data->data_space_reserved) 7353 return -ENOSPC; 7354 7355 /* 7356 * We have to COW and we have already reserved data space before, 7357 * so now we reserve only metadata. 7358 */ 7359 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len, len, 7360 false); 7361 if (ret < 0) 7362 goto out; 7363 space_reserved = true; 7364 7365 em = btrfs_new_extent_direct(BTRFS_I(inode), dio_data, start, len); 7366 if (IS_ERR(em)) { 7367 ret = PTR_ERR(em); 7368 goto out; 7369 } 7370 *map = em; 7371 len = min(len, em->len - (start - em->start)); 7372 if (len < prev_len) 7373 btrfs_delalloc_release_metadata(BTRFS_I(inode), 7374 prev_len - len, true); 7375 } 7376 7377 /* 7378 * We have created our ordered extent, so we can now release our reservation 7379 * for an outstanding extent. 7380 */ 7381 btrfs_delalloc_release_extents(BTRFS_I(inode), prev_len); 7382 7383 /* 7384 * Need to update the i_size under the extent lock so buffered 7385 * readers will get the updated i_size when we unlock. 7386 */ 7387 if (start + len > i_size_read(inode)) 7388 i_size_write(inode, start + len); 7389 out: 7390 if (ret && space_reserved) { 7391 btrfs_delalloc_release_extents(BTRFS_I(inode), len); 7392 btrfs_delalloc_release_metadata(BTRFS_I(inode), len, true); 7393 } 7394 return ret; 7395 } 7396 7397 static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, 7398 loff_t length, unsigned int flags, struct iomap *iomap, 7399 struct iomap *srcmap) 7400 { 7401 struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); 7402 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 7403 struct extent_map *em; 7404 struct extent_state *cached_state = NULL; 7405 struct btrfs_dio_data *dio_data = iter->private; 7406 u64 lockstart, lockend; 7407 const bool write = !!(flags & IOMAP_WRITE); 7408 int ret = 0; 7409 u64 len = length; 7410 const u64 data_alloc_len = length; 7411 bool unlock_extents = false; 7412 7413 /* 7414 * We could potentially fault if we have a buffer > PAGE_SIZE, and if 7415 * we're NOWAIT we may submit a bio for a partial range and return 7416 * EIOCBQUEUED, which would result in an errant short read. 7417 * 7418 * The best way to handle this would be to allow for partial completions 7419 * of iocb's, so we could submit the partial bio, return and fault in 7420 * the rest of the pages, and then submit the io for the rest of the 7421 * range. However we don't have that currently, so simply return 7422 * -EAGAIN at this point so that the normal path is used. 7423 */ 7424 if (!write && (flags & IOMAP_NOWAIT) && length > PAGE_SIZE) 7425 return -EAGAIN; 7426 7427 /* 7428 * Cap the size of reads to that usually seen in buffered I/O as we need 7429 * to allocate a contiguous array for the checksums. 7430 */ 7431 if (!write) 7432 len = min_t(u64, len, fs_info->sectorsize * BTRFS_MAX_BIO_SECTORS); 7433 7434 lockstart = start; 7435 lockend = start + len - 1; 7436 7437 /* 7438 * iomap_dio_rw() only does filemap_write_and_wait_range(), which isn't 7439 * enough if we've written compressed pages to this area, so we need to 7440 * flush the dirty pages again to make absolutely sure that any 7441 * outstanding dirty pages are on disk - the first flush only starts 7442 * compression on the data, while keeping the pages locked, so by the 7443 * time the second flush returns we know bios for the compressed pages 7444 * were submitted and finished, and the pages no longer under writeback. 7445 * 7446 * If we have a NOWAIT request and we have any pages in the range that 7447 * are locked, likely due to compression still in progress, we don't want 7448 * to block on page locks. We also don't want to block on pages marked as 7449 * dirty or under writeback (same as for the non-compression case). 7450 * iomap_dio_rw() did the same check, but after that and before we got 7451 * here, mmap'ed writes may have happened or buffered reads started 7452 * (readpage() and readahead(), which lock pages), as we haven't locked 7453 * the file range yet. 7454 */ 7455 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 7456 &BTRFS_I(inode)->runtime_flags)) { 7457 if (flags & IOMAP_NOWAIT) { 7458 if (filemap_range_needs_writeback(inode->i_mapping, 7459 lockstart, lockend)) 7460 return -EAGAIN; 7461 } else { 7462 ret = filemap_fdatawrite_range(inode->i_mapping, start, 7463 start + length - 1); 7464 if (ret) 7465 return ret; 7466 } 7467 } 7468 7469 memset(dio_data, 0, sizeof(*dio_data)); 7470 7471 /* 7472 * We always try to allocate data space and must do it before locking 7473 * the file range, to avoid deadlocks with concurrent writes to the same 7474 * range if the range has several extents and the writes don't expand the 7475 * current i_size (the inode lock is taken in shared mode). If we fail to 7476 * allocate data space here we continue and later, after locking the 7477 * file range, we fail with ENOSPC only if we figure out we can not do a 7478 * NOCOW write. 7479 */ 7480 if (write && !(flags & IOMAP_NOWAIT)) { 7481 ret = btrfs_check_data_free_space(BTRFS_I(inode), 7482 &dio_data->data_reserved, 7483 start, data_alloc_len, false); 7484 if (!ret) 7485 dio_data->data_space_reserved = true; 7486 else if (ret && !(BTRFS_I(inode)->flags & 7487 (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC))) 7488 goto err; 7489 } 7490 7491 /* 7492 * If this errors out it's because we couldn't invalidate pagecache for 7493 * this range and we need to fallback to buffered IO, or we are doing a 7494 * NOWAIT read/write and we need to block. 7495 */ 7496 ret = lock_extent_direct(inode, lockstart, lockend, &cached_state, flags); 7497 if (ret < 0) 7498 goto err; 7499 7500 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len); 7501 if (IS_ERR(em)) { 7502 ret = PTR_ERR(em); 7503 goto unlock_err; 7504 } 7505 7506 /* 7507 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7508 * io. INLINE is special, and we could probably kludge it in here, but 7509 * it's still buffered so for safety lets just fall back to the generic 7510 * buffered path. 7511 * 7512 * For COMPRESSED we _have_ to read the entire extent in so we can 7513 * decompress it, so there will be buffering required no matter what we 7514 * do, so go ahead and fallback to buffered. 7515 * 7516 * We return -ENOTBLK because that's what makes DIO go ahead and go back 7517 * to buffered IO. Don't blame me, this is the price we pay for using 7518 * the generic code. 7519 */ 7520 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7521 em->block_start == EXTENT_MAP_INLINE) { 7522 free_extent_map(em); 7523 /* 7524 * If we are in a NOWAIT context, return -EAGAIN in order to 7525 * fallback to buffered IO. This is not only because we can 7526 * block with buffered IO (no support for NOWAIT semantics at 7527 * the moment) but also to avoid returning short reads to user 7528 * space - this happens if we were able to read some data from 7529 * previous non-compressed extents and then when we fallback to 7530 * buffered IO, at btrfs_file_read_iter() by calling 7531 * filemap_read(), we fail to fault in pages for the read buffer, 7532 * in which case filemap_read() returns a short read (the number 7533 * of bytes previously read is > 0, so it does not return -EFAULT). 7534 */ 7535 ret = (flags & IOMAP_NOWAIT) ? -EAGAIN : -ENOTBLK; 7536 goto unlock_err; 7537 } 7538 7539 len = min(len, em->len - (start - em->start)); 7540 7541 /* 7542 * If we have a NOWAIT request and the range contains multiple extents 7543 * (or a mix of extents and holes), then we return -EAGAIN to make the 7544 * caller fallback to a context where it can do a blocking (without 7545 * NOWAIT) request. This way we avoid doing partial IO and returning 7546 * success to the caller, which is not optimal for writes and for reads 7547 * it can result in unexpected behaviour for an application. 7548 * 7549 * When doing a read, because we use IOMAP_DIO_PARTIAL when calling 7550 * iomap_dio_rw(), we can end up returning less data then what the caller 7551 * asked for, resulting in an unexpected, and incorrect, short read. 7552 * That is, the caller asked to read N bytes and we return less than that, 7553 * which is wrong unless we are crossing EOF. This happens if we get a 7554 * page fault error when trying to fault in pages for the buffer that is 7555 * associated to the struct iov_iter passed to iomap_dio_rw(), and we 7556 * have previously submitted bios for other extents in the range, in 7557 * which case iomap_dio_rw() may return us EIOCBQUEUED if not all of 7558 * those bios have completed by the time we get the page fault error, 7559 * which we return back to our caller - we should only return EIOCBQUEUED 7560 * after we have submitted bios for all the extents in the range. 7561 */ 7562 if ((flags & IOMAP_NOWAIT) && len < length) { 7563 free_extent_map(em); 7564 ret = -EAGAIN; 7565 goto unlock_err; 7566 } 7567 7568 if (write) { 7569 ret = btrfs_get_blocks_direct_write(&em, inode, dio_data, 7570 start, len, flags); 7571 if (ret < 0) 7572 goto unlock_err; 7573 unlock_extents = true; 7574 /* Recalc len in case the new em is smaller than requested */ 7575 len = min(len, em->len - (start - em->start)); 7576 if (dio_data->data_space_reserved) { 7577 u64 release_offset; 7578 u64 release_len = 0; 7579 7580 if (dio_data->nocow_done) { 7581 release_offset = start; 7582 release_len = data_alloc_len; 7583 } else if (len < data_alloc_len) { 7584 release_offset = start + len; 7585 release_len = data_alloc_len - len; 7586 } 7587 7588 if (release_len > 0) 7589 btrfs_free_reserved_data_space(BTRFS_I(inode), 7590 dio_data->data_reserved, 7591 release_offset, 7592 release_len); 7593 } 7594 } else { 7595 /* 7596 * We need to unlock only the end area that we aren't using. 7597 * The rest is going to be unlocked by the endio routine. 7598 */ 7599 lockstart = start + len; 7600 if (lockstart < lockend) 7601 unlock_extents = true; 7602 } 7603 7604 if (unlock_extents) 7605 unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7606 &cached_state); 7607 else 7608 free_extent_state(cached_state); 7609 7610 /* 7611 * Translate extent map information to iomap. 7612 * We trim the extents (and move the addr) even though iomap code does 7613 * that, since we have locked only the parts we are performing I/O in. 7614 */ 7615 if ((em->block_start == EXTENT_MAP_HOLE) || 7616 (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) && !write)) { 7617 iomap->addr = IOMAP_NULL_ADDR; 7618 iomap->type = IOMAP_HOLE; 7619 } else { 7620 iomap->addr = em->block_start + (start - em->start); 7621 iomap->type = IOMAP_MAPPED; 7622 } 7623 iomap->offset = start; 7624 iomap->bdev = fs_info->fs_devices->latest_dev->bdev; 7625 iomap->length = len; 7626 free_extent_map(em); 7627 7628 return 0; 7629 7630 unlock_err: 7631 unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7632 &cached_state); 7633 err: 7634 if (dio_data->data_space_reserved) { 7635 btrfs_free_reserved_data_space(BTRFS_I(inode), 7636 dio_data->data_reserved, 7637 start, data_alloc_len); 7638 extent_changeset_free(dio_data->data_reserved); 7639 } 7640 7641 return ret; 7642 } 7643 7644 static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, 7645 ssize_t written, unsigned int flags, struct iomap *iomap) 7646 { 7647 struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); 7648 struct btrfs_dio_data *dio_data = iter->private; 7649 size_t submitted = dio_data->submitted; 7650 const bool write = !!(flags & IOMAP_WRITE); 7651 int ret = 0; 7652 7653 if (!write && (iomap->type == IOMAP_HOLE)) { 7654 /* If reading from a hole, unlock and return */ 7655 unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1, 7656 NULL); 7657 return 0; 7658 } 7659 7660 if (submitted < length) { 7661 pos += submitted; 7662 length -= submitted; 7663 if (write) 7664 btrfs_mark_ordered_io_finished(BTRFS_I(inode), NULL, 7665 pos, length, false); 7666 else 7667 unlock_extent(&BTRFS_I(inode)->io_tree, pos, 7668 pos + length - 1, NULL); 7669 ret = -ENOTBLK; 7670 } 7671 if (write) { 7672 btrfs_put_ordered_extent(dio_data->ordered); 7673 dio_data->ordered = NULL; 7674 } 7675 7676 if (write) 7677 extent_changeset_free(dio_data->data_reserved); 7678 return ret; 7679 } 7680 7681 static void btrfs_dio_end_io(struct btrfs_bio *bbio) 7682 { 7683 struct btrfs_dio_private *dip = 7684 container_of(bbio, struct btrfs_dio_private, bbio); 7685 struct btrfs_inode *inode = bbio->inode; 7686 struct bio *bio = &bbio->bio; 7687 7688 if (bio->bi_status) { 7689 btrfs_warn(inode->root->fs_info, 7690 "direct IO failed ino %llu op 0x%0x offset %#llx len %u err no %d", 7691 btrfs_ino(inode), bio->bi_opf, 7692 dip->file_offset, dip->bytes, bio->bi_status); 7693 } 7694 7695 if (btrfs_op(bio) == BTRFS_MAP_WRITE) 7696 btrfs_mark_ordered_io_finished(inode, NULL, dip->file_offset, 7697 dip->bytes, !bio->bi_status); 7698 else 7699 unlock_extent(&inode->io_tree, dip->file_offset, 7700 dip->file_offset + dip->bytes - 1, NULL); 7701 7702 bbio->bio.bi_private = bbio->private; 7703 iomap_dio_bio_end_io(bio); 7704 } 7705 7706 static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, 7707 loff_t file_offset) 7708 { 7709 struct btrfs_bio *bbio = btrfs_bio(bio); 7710 struct btrfs_dio_private *dip = 7711 container_of(bbio, struct btrfs_dio_private, bbio); 7712 struct btrfs_dio_data *dio_data = iter->private; 7713 7714 btrfs_bio_init(bbio, BTRFS_I(iter->inode)->root->fs_info, 7715 btrfs_dio_end_io, bio->bi_private); 7716 bbio->inode = BTRFS_I(iter->inode); 7717 bbio->file_offset = file_offset; 7718 7719 dip->file_offset = file_offset; 7720 dip->bytes = bio->bi_iter.bi_size; 7721 7722 dio_data->submitted += bio->bi_iter.bi_size; 7723 7724 /* 7725 * Check if we are doing a partial write. If we are, we need to split 7726 * the ordered extent to match the submitted bio. Hang on to the 7727 * remaining unfinishable ordered_extent in dio_data so that it can be 7728 * cancelled in iomap_end to avoid a deadlock wherein faulting the 7729 * remaining pages is blocked on the outstanding ordered extent. 7730 */ 7731 if (iter->flags & IOMAP_WRITE) { 7732 int ret; 7733 7734 ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered); 7735 if (ret) { 7736 btrfs_bio_end_io(bbio, errno_to_blk_status(ret)); 7737 return; 7738 } 7739 } 7740 7741 btrfs_submit_bio(bbio, 0); 7742 } 7743 7744 static const struct iomap_ops btrfs_dio_iomap_ops = { 7745 .iomap_begin = btrfs_dio_iomap_begin, 7746 .iomap_end = btrfs_dio_iomap_end, 7747 }; 7748 7749 static const struct iomap_dio_ops btrfs_dio_ops = { 7750 .submit_io = btrfs_dio_submit_io, 7751 .bio_set = &btrfs_dio_bioset, 7752 }; 7753 7754 ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, size_t done_before) 7755 { 7756 struct btrfs_dio_data data = { 0 }; 7757 7758 return iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, 7759 IOMAP_DIO_PARTIAL, &data, done_before); 7760 } 7761 7762 struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter, 7763 size_t done_before) 7764 { 7765 struct btrfs_dio_data data = { 0 }; 7766 7767 return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, 7768 IOMAP_DIO_PARTIAL, &data, done_before); 7769 } 7770 7771 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 7772 u64 start, u64 len) 7773 { 7774 int ret; 7775 7776 ret = fiemap_prep(inode, fieinfo, start, &len, 0); 7777 if (ret) 7778 return ret; 7779 7780 /* 7781 * fiemap_prep() called filemap_write_and_wait() for the whole possible 7782 * file range (0 to LLONG_MAX), but that is not enough if we have 7783 * compression enabled. The first filemap_fdatawrite_range() only kicks 7784 * in the compression of data (in an async thread) and will return 7785 * before the compression is done and writeback is started. A second 7786 * filemap_fdatawrite_range() is needed to wait for the compression to 7787 * complete and writeback to start. We also need to wait for ordered 7788 * extents to complete, because our fiemap implementation uses mainly 7789 * file extent items to list the extents, searching for extent maps 7790 * only for file ranges with holes or prealloc extents to figure out 7791 * if we have delalloc in those ranges. 7792 */ 7793 if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { 7794 ret = btrfs_wait_ordered_range(inode, 0, LLONG_MAX); 7795 if (ret) 7796 return ret; 7797 } 7798 7799 return extent_fiemap(BTRFS_I(inode), fieinfo, start, len); 7800 } 7801 7802 static int btrfs_writepages(struct address_space *mapping, 7803 struct writeback_control *wbc) 7804 { 7805 return extent_writepages(mapping, wbc); 7806 } 7807 7808 static void btrfs_readahead(struct readahead_control *rac) 7809 { 7810 extent_readahead(rac); 7811 } 7812 7813 /* 7814 * For release_folio() and invalidate_folio() we have a race window where 7815 * folio_end_writeback() is called but the subpage spinlock is not yet released. 7816 * If we continue to release/invalidate the page, we could cause use-after-free 7817 * for subpage spinlock. So this function is to spin and wait for subpage 7818 * spinlock. 7819 */ 7820 static void wait_subpage_spinlock(struct page *page) 7821 { 7822 struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); 7823 struct btrfs_subpage *subpage; 7824 7825 if (!btrfs_is_subpage(fs_info, page)) 7826 return; 7827 7828 ASSERT(PagePrivate(page) && page->private); 7829 subpage = (struct btrfs_subpage *)page->private; 7830 7831 /* 7832 * This may look insane as we just acquire the spinlock and release it, 7833 * without doing anything. But we just want to make sure no one is 7834 * still holding the subpage spinlock. 7835 * And since the page is not dirty nor writeback, and we have page 7836 * locked, the only possible way to hold a spinlock is from the endio 7837 * function to clear page writeback. 7838 * 7839 * Here we just acquire the spinlock so that all existing callers 7840 * should exit and we're safe to release/invalidate the page. 7841 */ 7842 spin_lock_irq(&subpage->lock); 7843 spin_unlock_irq(&subpage->lock); 7844 } 7845 7846 static bool __btrfs_release_folio(struct folio *folio, gfp_t gfp_flags) 7847 { 7848 int ret = try_release_extent_mapping(&folio->page, gfp_flags); 7849 7850 if (ret == 1) { 7851 wait_subpage_spinlock(&folio->page); 7852 clear_page_extent_mapped(&folio->page); 7853 } 7854 return ret; 7855 } 7856 7857 static bool btrfs_release_folio(struct folio *folio, gfp_t gfp_flags) 7858 { 7859 if (folio_test_writeback(folio) || folio_test_dirty(folio)) 7860 return false; 7861 return __btrfs_release_folio(folio, gfp_flags); 7862 } 7863 7864 #ifdef CONFIG_MIGRATION 7865 static int btrfs_migrate_folio(struct address_space *mapping, 7866 struct folio *dst, struct folio *src, 7867 enum migrate_mode mode) 7868 { 7869 int ret = filemap_migrate_folio(mapping, dst, src, mode); 7870 7871 if (ret != MIGRATEPAGE_SUCCESS) 7872 return ret; 7873 7874 if (folio_test_ordered(src)) { 7875 folio_clear_ordered(src); 7876 folio_set_ordered(dst); 7877 } 7878 7879 return MIGRATEPAGE_SUCCESS; 7880 } 7881 #else 7882 #define btrfs_migrate_folio NULL 7883 #endif 7884 7885 static void btrfs_invalidate_folio(struct folio *folio, size_t offset, 7886 size_t length) 7887 { 7888 struct btrfs_inode *inode = BTRFS_I(folio->mapping->host); 7889 struct btrfs_fs_info *fs_info = inode->root->fs_info; 7890 struct extent_io_tree *tree = &inode->io_tree; 7891 struct extent_state *cached_state = NULL; 7892 u64 page_start = folio_pos(folio); 7893 u64 page_end = page_start + folio_size(folio) - 1; 7894 u64 cur; 7895 int inode_evicting = inode->vfs_inode.i_state & I_FREEING; 7896 7897 /* 7898 * We have folio locked so no new ordered extent can be created on this 7899 * page, nor bio can be submitted for this folio. 7900 * 7901 * But already submitted bio can still be finished on this folio. 7902 * Furthermore, endio function won't skip folio which has Ordered 7903 * (Private2) already cleared, so it's possible for endio and 7904 * invalidate_folio to do the same ordered extent accounting twice 7905 * on one folio. 7906 * 7907 * So here we wait for any submitted bios to finish, so that we won't 7908 * do double ordered extent accounting on the same folio. 7909 */ 7910 folio_wait_writeback(folio); 7911 wait_subpage_spinlock(&folio->page); 7912 7913 /* 7914 * For subpage case, we have call sites like 7915 * btrfs_punch_hole_lock_range() which passes range not aligned to 7916 * sectorsize. 7917 * If the range doesn't cover the full folio, we don't need to and 7918 * shouldn't clear page extent mapped, as folio->private can still 7919 * record subpage dirty bits for other part of the range. 7920 * 7921 * For cases that invalidate the full folio even the range doesn't 7922 * cover the full folio, like invalidating the last folio, we're 7923 * still safe to wait for ordered extent to finish. 7924 */ 7925 if (!(offset == 0 && length == folio_size(folio))) { 7926 btrfs_release_folio(folio, GFP_NOFS); 7927 return; 7928 } 7929 7930 if (!inode_evicting) 7931 lock_extent(tree, page_start, page_end, &cached_state); 7932 7933 cur = page_start; 7934 while (cur < page_end) { 7935 struct btrfs_ordered_extent *ordered; 7936 u64 range_end; 7937 u32 range_len; 7938 u32 extra_flags = 0; 7939 7940 ordered = btrfs_lookup_first_ordered_range(inode, cur, 7941 page_end + 1 - cur); 7942 if (!ordered) { 7943 range_end = page_end; 7944 /* 7945 * No ordered extent covering this range, we are safe 7946 * to delete all extent states in the range. 7947 */ 7948 extra_flags = EXTENT_CLEAR_ALL_BITS; 7949 goto next; 7950 } 7951 if (ordered->file_offset > cur) { 7952 /* 7953 * There is a range between [cur, oe->file_offset) not 7954 * covered by any ordered extent. 7955 * We are safe to delete all extent states, and handle 7956 * the ordered extent in the next iteration. 7957 */ 7958 range_end = ordered->file_offset - 1; 7959 extra_flags = EXTENT_CLEAR_ALL_BITS; 7960 goto next; 7961 } 7962 7963 range_end = min(ordered->file_offset + ordered->num_bytes - 1, 7964 page_end); 7965 ASSERT(range_end + 1 - cur < U32_MAX); 7966 range_len = range_end + 1 - cur; 7967 if (!btrfs_page_test_ordered(fs_info, &folio->page, cur, range_len)) { 7968 /* 7969 * If Ordered (Private2) is cleared, it means endio has 7970 * already been executed for the range. 7971 * We can't delete the extent states as 7972 * btrfs_finish_ordered_io() may still use some of them. 7973 */ 7974 goto next; 7975 } 7976 btrfs_page_clear_ordered(fs_info, &folio->page, cur, range_len); 7977 7978 /* 7979 * IO on this page will never be started, so we need to account 7980 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW 7981 * here, must leave that up for the ordered extent completion. 7982 * 7983 * This will also unlock the range for incoming 7984 * btrfs_finish_ordered_io(). 7985 */ 7986 if (!inode_evicting) 7987 clear_extent_bit(tree, cur, range_end, 7988 EXTENT_DELALLOC | 7989 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 7990 EXTENT_DEFRAG, &cached_state); 7991 7992 spin_lock_irq(&inode->ordered_tree.lock); 7993 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 7994 ordered->truncated_len = min(ordered->truncated_len, 7995 cur - ordered->file_offset); 7996 spin_unlock_irq(&inode->ordered_tree.lock); 7997 7998 /* 7999 * If the ordered extent has finished, we're safe to delete all 8000 * the extent states of the range, otherwise 8001 * btrfs_finish_ordered_io() will get executed by endio for 8002 * other pages, so we can't delete extent states. 8003 */ 8004 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8005 cur, range_end + 1 - cur)) { 8006 btrfs_finish_ordered_io(ordered); 8007 /* 8008 * The ordered extent has finished, now we're again 8009 * safe to delete all extent states of the range. 8010 */ 8011 extra_flags = EXTENT_CLEAR_ALL_BITS; 8012 } 8013 next: 8014 if (ordered) 8015 btrfs_put_ordered_extent(ordered); 8016 /* 8017 * Qgroup reserved space handler 8018 * Sector(s) here will be either: 8019 * 8020 * 1) Already written to disk or bio already finished 8021 * Then its QGROUP_RESERVED bit in io_tree is already cleared. 8022 * Qgroup will be handled by its qgroup_record then. 8023 * btrfs_qgroup_free_data() call will do nothing here. 8024 * 8025 * 2) Not written to disk yet 8026 * Then btrfs_qgroup_free_data() call will clear the 8027 * QGROUP_RESERVED bit of its io_tree, and free the qgroup 8028 * reserved data space. 8029 * Since the IO will never happen for this page. 8030 */ 8031 btrfs_qgroup_free_data(inode, NULL, cur, range_end + 1 - cur); 8032 if (!inode_evicting) { 8033 clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED | 8034 EXTENT_DELALLOC | EXTENT_UPTODATE | 8035 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG | 8036 extra_flags, &cached_state); 8037 } 8038 cur = range_end + 1; 8039 } 8040 /* 8041 * We have iterated through all ordered extents of the page, the page 8042 * should not have Ordered (Private2) anymore, or the above iteration 8043 * did something wrong. 8044 */ 8045 ASSERT(!folio_test_ordered(folio)); 8046 btrfs_page_clear_checked(fs_info, &folio->page, folio_pos(folio), folio_size(folio)); 8047 if (!inode_evicting) 8048 __btrfs_release_folio(folio, GFP_NOFS); 8049 clear_page_extent_mapped(&folio->page); 8050 } 8051 8052 /* 8053 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8054 * called from a page fault handler when a page is first dirtied. Hence we must 8055 * be careful to check for EOF conditions here. We set the page up correctly 8056 * for a written page which means we get ENOSPC checking when writing into 8057 * holes and correct delalloc and unwritten extent mapping on filesystems that 8058 * support these features. 8059 * 8060 * We are not allowed to take the i_mutex here so we have to play games to 8061 * protect against truncate races as the page could now be beyond EOF. Because 8062 * truncate_setsize() writes the inode size before removing pages, once we have 8063 * the page lock we can determine safely if the page is beyond EOF. If it is not 8064 * beyond EOF, then the page is guaranteed safe against truncation until we 8065 * unlock the page. 8066 */ 8067 vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) 8068 { 8069 struct page *page = vmf->page; 8070 struct inode *inode = file_inode(vmf->vma->vm_file); 8071 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 8072 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8073 struct btrfs_ordered_extent *ordered; 8074 struct extent_state *cached_state = NULL; 8075 struct extent_changeset *data_reserved = NULL; 8076 unsigned long zero_start; 8077 loff_t size; 8078 vm_fault_t ret; 8079 int ret2; 8080 int reserved = 0; 8081 u64 reserved_space; 8082 u64 page_start; 8083 u64 page_end; 8084 u64 end; 8085 8086 reserved_space = PAGE_SIZE; 8087 8088 sb_start_pagefault(inode->i_sb); 8089 page_start = page_offset(page); 8090 page_end = page_start + PAGE_SIZE - 1; 8091 end = page_end; 8092 8093 /* 8094 * Reserving delalloc space after obtaining the page lock can lead to 8095 * deadlock. For example, if a dirty page is locked by this function 8096 * and the call to btrfs_delalloc_reserve_space() ends up triggering 8097 * dirty page write out, then the btrfs_writepages() function could 8098 * end up waiting indefinitely to get a lock on the page currently 8099 * being processed by btrfs_page_mkwrite() function. 8100 */ 8101 ret2 = btrfs_delalloc_reserve_space(BTRFS_I(inode), &data_reserved, 8102 page_start, reserved_space); 8103 if (!ret2) { 8104 ret2 = file_update_time(vmf->vma->vm_file); 8105 reserved = 1; 8106 } 8107 if (ret2) { 8108 ret = vmf_error(ret2); 8109 if (reserved) 8110 goto out; 8111 goto out_noreserve; 8112 } 8113 8114 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 8115 again: 8116 down_read(&BTRFS_I(inode)->i_mmap_lock); 8117 lock_page(page); 8118 size = i_size_read(inode); 8119 8120 if ((page->mapping != inode->i_mapping) || 8121 (page_start >= size)) { 8122 /* page got truncated out from underneath us */ 8123 goto out_unlock; 8124 } 8125 wait_on_page_writeback(page); 8126 8127 lock_extent(io_tree, page_start, page_end, &cached_state); 8128 ret2 = set_page_extent_mapped(page); 8129 if (ret2 < 0) { 8130 ret = vmf_error(ret2); 8131 unlock_extent(io_tree, page_start, page_end, &cached_state); 8132 goto out_unlock; 8133 } 8134 8135 /* 8136 * we can't set the delalloc bits if there are pending ordered 8137 * extents. Drop our locks and wait for them to finish 8138 */ 8139 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, 8140 PAGE_SIZE); 8141 if (ordered) { 8142 unlock_extent(io_tree, page_start, page_end, &cached_state); 8143 unlock_page(page); 8144 up_read(&BTRFS_I(inode)->i_mmap_lock); 8145 btrfs_start_ordered_extent(ordered); 8146 btrfs_put_ordered_extent(ordered); 8147 goto again; 8148 } 8149 8150 if (page->index == ((size - 1) >> PAGE_SHIFT)) { 8151 reserved_space = round_up(size - page_start, 8152 fs_info->sectorsize); 8153 if (reserved_space < PAGE_SIZE) { 8154 end = page_start + reserved_space - 1; 8155 btrfs_delalloc_release_space(BTRFS_I(inode), 8156 data_reserved, page_start, 8157 PAGE_SIZE - reserved_space, true); 8158 } 8159 } 8160 8161 /* 8162 * page_mkwrite gets called when the page is firstly dirtied after it's 8163 * faulted in, but write(2) could also dirty a page and set delalloc 8164 * bits, thus in this case for space account reason, we still need to 8165 * clear any delalloc bits within this page range since we have to 8166 * reserve data&meta space before lock_page() (see above comments). 8167 */ 8168 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, 8169 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 8170 EXTENT_DEFRAG, &cached_state); 8171 8172 ret2 = btrfs_set_extent_delalloc(BTRFS_I(inode), page_start, end, 0, 8173 &cached_state); 8174 if (ret2) { 8175 unlock_extent(io_tree, page_start, page_end, &cached_state); 8176 ret = VM_FAULT_SIGBUS; 8177 goto out_unlock; 8178 } 8179 8180 /* page is wholly or partially inside EOF */ 8181 if (page_start + PAGE_SIZE > size) 8182 zero_start = offset_in_page(size); 8183 else 8184 zero_start = PAGE_SIZE; 8185 8186 if (zero_start != PAGE_SIZE) 8187 memzero_page(page, zero_start, PAGE_SIZE - zero_start); 8188 8189 btrfs_page_clear_checked(fs_info, page, page_start, PAGE_SIZE); 8190 btrfs_page_set_dirty(fs_info, page, page_start, end + 1 - page_start); 8191 btrfs_page_set_uptodate(fs_info, page, page_start, end + 1 - page_start); 8192 8193 btrfs_set_inode_last_sub_trans(BTRFS_I(inode)); 8194 8195 unlock_extent(io_tree, page_start, page_end, &cached_state); 8196 up_read(&BTRFS_I(inode)->i_mmap_lock); 8197 8198 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); 8199 sb_end_pagefault(inode->i_sb); 8200 extent_changeset_free(data_reserved); 8201 return VM_FAULT_LOCKED; 8202 8203 out_unlock: 8204 unlock_page(page); 8205 up_read(&BTRFS_I(inode)->i_mmap_lock); 8206 out: 8207 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); 8208 btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved, page_start, 8209 reserved_space, (ret != 0)); 8210 out_noreserve: 8211 sb_end_pagefault(inode->i_sb); 8212 extent_changeset_free(data_reserved); 8213 return ret; 8214 } 8215 8216 static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback) 8217 { 8218 struct btrfs_truncate_control control = { 8219 .inode = inode, 8220 .ino = btrfs_ino(inode), 8221 .min_type = BTRFS_EXTENT_DATA_KEY, 8222 .clear_extent_range = true, 8223 }; 8224 struct btrfs_root *root = inode->root; 8225 struct btrfs_fs_info *fs_info = root->fs_info; 8226 struct btrfs_block_rsv *rsv; 8227 int ret; 8228 struct btrfs_trans_handle *trans; 8229 u64 mask = fs_info->sectorsize - 1; 8230 u64 min_size = btrfs_calc_metadata_size(fs_info, 1); 8231 8232 if (!skip_writeback) { 8233 ret = btrfs_wait_ordered_range(&inode->vfs_inode, 8234 inode->vfs_inode.i_size & (~mask), 8235 (u64)-1); 8236 if (ret) 8237 return ret; 8238 } 8239 8240 /* 8241 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of 8242 * things going on here: 8243 * 8244 * 1) We need to reserve space to update our inode. 8245 * 8246 * 2) We need to have something to cache all the space that is going to 8247 * be free'd up by the truncate operation, but also have some slack 8248 * space reserved in case it uses space during the truncate (thank you 8249 * very much snapshotting). 8250 * 8251 * And we need these to be separate. The fact is we can use a lot of 8252 * space doing the truncate, and we have no earthly idea how much space 8253 * we will use, so we need the truncate reservation to be separate so it 8254 * doesn't end up using space reserved for updating the inode. We also 8255 * need to be able to stop the transaction and start a new one, which 8256 * means we need to be able to update the inode several times, and we 8257 * have no idea of knowing how many times that will be, so we can't just 8258 * reserve 1 item for the entirety of the operation, so that has to be 8259 * done separately as well. 8260 * 8261 * So that leaves us with 8262 * 8263 * 1) rsv - for the truncate reservation, which we will steal from the 8264 * transaction reservation. 8265 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for 8266 * updating the inode. 8267 */ 8268 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP); 8269 if (!rsv) 8270 return -ENOMEM; 8271 rsv->size = min_size; 8272 rsv->failfast = true; 8273 8274 /* 8275 * 1 for the truncate slack space 8276 * 1 for updating the inode. 8277 */ 8278 trans = btrfs_start_transaction(root, 2); 8279 if (IS_ERR(trans)) { 8280 ret = PTR_ERR(trans); 8281 goto out; 8282 } 8283 8284 /* Migrate the slack space for the truncate to our reserve */ 8285 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv, 8286 min_size, false); 8287 BUG_ON(ret); 8288 8289 trans->block_rsv = rsv; 8290 8291 while (1) { 8292 struct extent_state *cached_state = NULL; 8293 const u64 new_size = inode->vfs_inode.i_size; 8294 const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize); 8295 8296 control.new_size = new_size; 8297 lock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state); 8298 /* 8299 * We want to drop from the next block forward in case this new 8300 * size is not block aligned since we will be keeping the last 8301 * block of the extent just the way it is. 8302 */ 8303 btrfs_drop_extent_map_range(inode, 8304 ALIGN(new_size, fs_info->sectorsize), 8305 (u64)-1, false); 8306 8307 ret = btrfs_truncate_inode_items(trans, root, &control); 8308 8309 inode_sub_bytes(&inode->vfs_inode, control.sub_bytes); 8310 btrfs_inode_safe_disk_i_size_write(inode, control.last_size); 8311 8312 unlock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state); 8313 8314 trans->block_rsv = &fs_info->trans_block_rsv; 8315 if (ret != -ENOSPC && ret != -EAGAIN) 8316 break; 8317 8318 ret = btrfs_update_inode(trans, root, inode); 8319 if (ret) 8320 break; 8321 8322 btrfs_end_transaction(trans); 8323 btrfs_btree_balance_dirty(fs_info); 8324 8325 trans = btrfs_start_transaction(root, 2); 8326 if (IS_ERR(trans)) { 8327 ret = PTR_ERR(trans); 8328 trans = NULL; 8329 break; 8330 } 8331 8332 btrfs_block_rsv_release(fs_info, rsv, -1, NULL); 8333 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 8334 rsv, min_size, false); 8335 BUG_ON(ret); /* shouldn't happen */ 8336 trans->block_rsv = rsv; 8337 } 8338 8339 /* 8340 * We can't call btrfs_truncate_block inside a trans handle as we could 8341 * deadlock with freeze, if we got BTRFS_NEED_TRUNCATE_BLOCK then we 8342 * know we've truncated everything except the last little bit, and can 8343 * do btrfs_truncate_block and then update the disk_i_size. 8344 */ 8345 if (ret == BTRFS_NEED_TRUNCATE_BLOCK) { 8346 btrfs_end_transaction(trans); 8347 btrfs_btree_balance_dirty(fs_info); 8348 8349 ret = btrfs_truncate_block(inode, inode->vfs_inode.i_size, 0, 0); 8350 if (ret) 8351 goto out; 8352 trans = btrfs_start_transaction(root, 1); 8353 if (IS_ERR(trans)) { 8354 ret = PTR_ERR(trans); 8355 goto out; 8356 } 8357 btrfs_inode_safe_disk_i_size_write(inode, 0); 8358 } 8359 8360 if (trans) { 8361 int ret2; 8362 8363 trans->block_rsv = &fs_info->trans_block_rsv; 8364 ret2 = btrfs_update_inode(trans, root, inode); 8365 if (ret2 && !ret) 8366 ret = ret2; 8367 8368 ret2 = btrfs_end_transaction(trans); 8369 if (ret2 && !ret) 8370 ret = ret2; 8371 btrfs_btree_balance_dirty(fs_info); 8372 } 8373 out: 8374 btrfs_free_block_rsv(fs_info, rsv); 8375 /* 8376 * So if we truncate and then write and fsync we normally would just 8377 * write the extents that changed, which is a problem if we need to 8378 * first truncate that entire inode. So set this flag so we write out 8379 * all of the extents in the inode to the sync log so we're completely 8380 * safe. 8381 * 8382 * If no extents were dropped or trimmed we don't need to force the next 8383 * fsync to truncate all the inode's items from the log and re-log them 8384 * all. This means the truncate operation did not change the file size, 8385 * or changed it to a smaller size but there was only an implicit hole 8386 * between the old i_size and the new i_size, and there were no prealloc 8387 * extents beyond i_size to drop. 8388 */ 8389 if (control.extents_found > 0) 8390 btrfs_set_inode_full_sync(inode); 8391 8392 return ret; 8393 } 8394 8395 struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap, 8396 struct inode *dir) 8397 { 8398 struct inode *inode; 8399 8400 inode = new_inode(dir->i_sb); 8401 if (inode) { 8402 /* 8403 * Subvolumes don't inherit the sgid bit or the parent's gid if 8404 * the parent's sgid bit is set. This is probably a bug. 8405 */ 8406 inode_init_owner(idmap, inode, NULL, 8407 S_IFDIR | (~current_umask() & S_IRWXUGO)); 8408 inode->i_op = &btrfs_dir_inode_operations; 8409 inode->i_fop = &btrfs_dir_file_operations; 8410 } 8411 return inode; 8412 } 8413 8414 struct inode *btrfs_alloc_inode(struct super_block *sb) 8415 { 8416 struct btrfs_fs_info *fs_info = btrfs_sb(sb); 8417 struct btrfs_inode *ei; 8418 struct inode *inode; 8419 8420 ei = alloc_inode_sb(sb, btrfs_inode_cachep, GFP_KERNEL); 8421 if (!ei) 8422 return NULL; 8423 8424 ei->root = NULL; 8425 ei->generation = 0; 8426 ei->last_trans = 0; 8427 ei->last_sub_trans = 0; 8428 ei->logged_trans = 0; 8429 ei->delalloc_bytes = 0; 8430 ei->new_delalloc_bytes = 0; 8431 ei->defrag_bytes = 0; 8432 ei->disk_i_size = 0; 8433 ei->flags = 0; 8434 ei->ro_flags = 0; 8435 ei->csum_bytes = 0; 8436 ei->index_cnt = (u64)-1; 8437 ei->dir_index = 0; 8438 ei->last_unlink_trans = 0; 8439 ei->last_reflink_trans = 0; 8440 ei->last_log_commit = 0; 8441 8442 spin_lock_init(&ei->lock); 8443 ei->outstanding_extents = 0; 8444 if (sb->s_magic != BTRFS_TEST_MAGIC) 8445 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, 8446 BTRFS_BLOCK_RSV_DELALLOC); 8447 ei->runtime_flags = 0; 8448 ei->prop_compress = BTRFS_COMPRESS_NONE; 8449 ei->defrag_compress = BTRFS_COMPRESS_NONE; 8450 8451 ei->delayed_node = NULL; 8452 8453 ei->i_otime.tv_sec = 0; 8454 ei->i_otime.tv_nsec = 0; 8455 8456 inode = &ei->vfs_inode; 8457 extent_map_tree_init(&ei->extent_tree); 8458 extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO); 8459 ei->io_tree.inode = ei; 8460 extent_io_tree_init(fs_info, &ei->file_extent_tree, 8461 IO_TREE_INODE_FILE_EXTENT); 8462 atomic_set(&ei->sync_writers, 0); 8463 mutex_init(&ei->log_mutex); 8464 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 8465 INIT_LIST_HEAD(&ei->delalloc_inodes); 8466 INIT_LIST_HEAD(&ei->delayed_iput); 8467 RB_CLEAR_NODE(&ei->rb_node); 8468 init_rwsem(&ei->i_mmap_lock); 8469 8470 return inode; 8471 } 8472 8473 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 8474 void btrfs_test_destroy_inode(struct inode *inode) 8475 { 8476 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); 8477 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8478 } 8479 #endif 8480 8481 void btrfs_free_inode(struct inode *inode) 8482 { 8483 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8484 } 8485 8486 void btrfs_destroy_inode(struct inode *vfs_inode) 8487 { 8488 struct btrfs_ordered_extent *ordered; 8489 struct btrfs_inode *inode = BTRFS_I(vfs_inode); 8490 struct btrfs_root *root = inode->root; 8491 bool freespace_inode; 8492 8493 WARN_ON(!hlist_empty(&vfs_inode->i_dentry)); 8494 WARN_ON(vfs_inode->i_data.nrpages); 8495 WARN_ON(inode->block_rsv.reserved); 8496 WARN_ON(inode->block_rsv.size); 8497 WARN_ON(inode->outstanding_extents); 8498 if (!S_ISDIR(vfs_inode->i_mode)) { 8499 WARN_ON(inode->delalloc_bytes); 8500 WARN_ON(inode->new_delalloc_bytes); 8501 } 8502 WARN_ON(inode->csum_bytes); 8503 WARN_ON(inode->defrag_bytes); 8504 8505 /* 8506 * This can happen where we create an inode, but somebody else also 8507 * created the same inode and we need to destroy the one we already 8508 * created. 8509 */ 8510 if (!root) 8511 return; 8512 8513 /* 8514 * If this is a free space inode do not take the ordered extents lockdep 8515 * map. 8516 */ 8517 freespace_inode = btrfs_is_free_space_inode(inode); 8518 8519 while (1) { 8520 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 8521 if (!ordered) 8522 break; 8523 else { 8524 btrfs_err(root->fs_info, 8525 "found ordered extent %llu %llu on inode cleanup", 8526 ordered->file_offset, ordered->num_bytes); 8527 8528 if (!freespace_inode) 8529 btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent); 8530 8531 btrfs_remove_ordered_extent(inode, ordered); 8532 btrfs_put_ordered_extent(ordered); 8533 btrfs_put_ordered_extent(ordered); 8534 } 8535 } 8536 btrfs_qgroup_check_reserved_leak(inode); 8537 inode_tree_del(inode); 8538 btrfs_drop_extent_map_range(inode, 0, (u64)-1, false); 8539 btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1); 8540 btrfs_put_root(inode->root); 8541 } 8542 8543 int btrfs_drop_inode(struct inode *inode) 8544 { 8545 struct btrfs_root *root = BTRFS_I(inode)->root; 8546 8547 if (root == NULL) 8548 return 1; 8549 8550 /* the snap/subvol tree is on deleting */ 8551 if (btrfs_root_refs(&root->root_item) == 0) 8552 return 1; 8553 else 8554 return generic_drop_inode(inode); 8555 } 8556 8557 static void init_once(void *foo) 8558 { 8559 struct btrfs_inode *ei = foo; 8560 8561 inode_init_once(&ei->vfs_inode); 8562 } 8563 8564 void __cold btrfs_destroy_cachep(void) 8565 { 8566 /* 8567 * Make sure all delayed rcu free inodes are flushed before we 8568 * destroy cache. 8569 */ 8570 rcu_barrier(); 8571 bioset_exit(&btrfs_dio_bioset); 8572 kmem_cache_destroy(btrfs_inode_cachep); 8573 } 8574 8575 int __init btrfs_init_cachep(void) 8576 { 8577 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 8578 sizeof(struct btrfs_inode), 0, 8579 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, 8580 init_once); 8581 if (!btrfs_inode_cachep) 8582 goto fail; 8583 8584 if (bioset_init(&btrfs_dio_bioset, BIO_POOL_SIZE, 8585 offsetof(struct btrfs_dio_private, bbio.bio), 8586 BIOSET_NEED_BVECS)) 8587 goto fail; 8588 8589 return 0; 8590 fail: 8591 btrfs_destroy_cachep(); 8592 return -ENOMEM; 8593 } 8594 8595 static int btrfs_getattr(struct mnt_idmap *idmap, 8596 const struct path *path, struct kstat *stat, 8597 u32 request_mask, unsigned int flags) 8598 { 8599 u64 delalloc_bytes; 8600 u64 inode_bytes; 8601 struct inode *inode = d_inode(path->dentry); 8602 u32 blocksize = inode->i_sb->s_blocksize; 8603 u32 bi_flags = BTRFS_I(inode)->flags; 8604 u32 bi_ro_flags = BTRFS_I(inode)->ro_flags; 8605 8606 stat->result_mask |= STATX_BTIME; 8607 stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec; 8608 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec; 8609 if (bi_flags & BTRFS_INODE_APPEND) 8610 stat->attributes |= STATX_ATTR_APPEND; 8611 if (bi_flags & BTRFS_INODE_COMPRESS) 8612 stat->attributes |= STATX_ATTR_COMPRESSED; 8613 if (bi_flags & BTRFS_INODE_IMMUTABLE) 8614 stat->attributes |= STATX_ATTR_IMMUTABLE; 8615 if (bi_flags & BTRFS_INODE_NODUMP) 8616 stat->attributes |= STATX_ATTR_NODUMP; 8617 if (bi_ro_flags & BTRFS_INODE_RO_VERITY) 8618 stat->attributes |= STATX_ATTR_VERITY; 8619 8620 stat->attributes_mask |= (STATX_ATTR_APPEND | 8621 STATX_ATTR_COMPRESSED | 8622 STATX_ATTR_IMMUTABLE | 8623 STATX_ATTR_NODUMP); 8624 8625 generic_fillattr(idmap, inode, stat); 8626 stat->dev = BTRFS_I(inode)->root->anon_dev; 8627 8628 spin_lock(&BTRFS_I(inode)->lock); 8629 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; 8630 inode_bytes = inode_get_bytes(inode); 8631 spin_unlock(&BTRFS_I(inode)->lock); 8632 stat->blocks = (ALIGN(inode_bytes, blocksize) + 8633 ALIGN(delalloc_bytes, blocksize)) >> 9; 8634 return 0; 8635 } 8636 8637 static int btrfs_rename_exchange(struct inode *old_dir, 8638 struct dentry *old_dentry, 8639 struct inode *new_dir, 8640 struct dentry *new_dentry) 8641 { 8642 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 8643 struct btrfs_trans_handle *trans; 8644 unsigned int trans_num_items; 8645 struct btrfs_root *root = BTRFS_I(old_dir)->root; 8646 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 8647 struct inode *new_inode = new_dentry->d_inode; 8648 struct inode *old_inode = old_dentry->d_inode; 8649 struct timespec64 ctime = current_time(old_inode); 8650 struct btrfs_rename_ctx old_rename_ctx; 8651 struct btrfs_rename_ctx new_rename_ctx; 8652 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 8653 u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); 8654 u64 old_idx = 0; 8655 u64 new_idx = 0; 8656 int ret; 8657 int ret2; 8658 bool need_abort = false; 8659 struct fscrypt_name old_fname, new_fname; 8660 struct fscrypt_str *old_name, *new_name; 8661 8662 /* 8663 * For non-subvolumes allow exchange only within one subvolume, in the 8664 * same inode namespace. Two subvolumes (represented as directory) can 8665 * be exchanged as they're a logical link and have a fixed inode number. 8666 */ 8667 if (root != dest && 8668 (old_ino != BTRFS_FIRST_FREE_OBJECTID || 8669 new_ino != BTRFS_FIRST_FREE_OBJECTID)) 8670 return -EXDEV; 8671 8672 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname); 8673 if (ret) 8674 return ret; 8675 8676 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname); 8677 if (ret) { 8678 fscrypt_free_filename(&old_fname); 8679 return ret; 8680 } 8681 8682 old_name = &old_fname.disk_name; 8683 new_name = &new_fname.disk_name; 8684 8685 /* close the race window with snapshot create/destroy ioctl */ 8686 if (old_ino == BTRFS_FIRST_FREE_OBJECTID || 8687 new_ino == BTRFS_FIRST_FREE_OBJECTID) 8688 down_read(&fs_info->subvol_sem); 8689 8690 /* 8691 * For each inode: 8692 * 1 to remove old dir item 8693 * 1 to remove old dir index 8694 * 1 to add new dir item 8695 * 1 to add new dir index 8696 * 1 to update parent inode 8697 * 8698 * If the parents are the same, we only need to account for one 8699 */ 8700 trans_num_items = (old_dir == new_dir ? 9 : 10); 8701 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8702 /* 8703 * 1 to remove old root ref 8704 * 1 to remove old root backref 8705 * 1 to add new root ref 8706 * 1 to add new root backref 8707 */ 8708 trans_num_items += 4; 8709 } else { 8710 /* 8711 * 1 to update inode item 8712 * 1 to remove old inode ref 8713 * 1 to add new inode ref 8714 */ 8715 trans_num_items += 3; 8716 } 8717 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 8718 trans_num_items += 4; 8719 else 8720 trans_num_items += 3; 8721 trans = btrfs_start_transaction(root, trans_num_items); 8722 if (IS_ERR(trans)) { 8723 ret = PTR_ERR(trans); 8724 goto out_notrans; 8725 } 8726 8727 if (dest != root) { 8728 ret = btrfs_record_root_in_trans(trans, dest); 8729 if (ret) 8730 goto out_fail; 8731 } 8732 8733 /* 8734 * We need to find a free sequence number both in the source and 8735 * in the destination directory for the exchange. 8736 */ 8737 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); 8738 if (ret) 8739 goto out_fail; 8740 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); 8741 if (ret) 8742 goto out_fail; 8743 8744 BTRFS_I(old_inode)->dir_index = 0ULL; 8745 BTRFS_I(new_inode)->dir_index = 0ULL; 8746 8747 /* Reference for the source. */ 8748 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8749 /* force full log commit if subvolume involved. */ 8750 btrfs_set_log_full_commit(trans); 8751 } else { 8752 ret = btrfs_insert_inode_ref(trans, dest, new_name, old_ino, 8753 btrfs_ino(BTRFS_I(new_dir)), 8754 old_idx); 8755 if (ret) 8756 goto out_fail; 8757 need_abort = true; 8758 } 8759 8760 /* And now for the dest. */ 8761 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 8762 /* force full log commit if subvolume involved. */ 8763 btrfs_set_log_full_commit(trans); 8764 } else { 8765 ret = btrfs_insert_inode_ref(trans, root, old_name, new_ino, 8766 btrfs_ino(BTRFS_I(old_dir)), 8767 new_idx); 8768 if (ret) { 8769 if (need_abort) 8770 btrfs_abort_transaction(trans, ret); 8771 goto out_fail; 8772 } 8773 } 8774 8775 /* Update inode version and ctime/mtime. */ 8776 inode_inc_iversion(old_dir); 8777 inode_inc_iversion(new_dir); 8778 inode_inc_iversion(old_inode); 8779 inode_inc_iversion(new_inode); 8780 old_dir->i_mtime = ctime; 8781 old_dir->i_ctime = ctime; 8782 new_dir->i_mtime = ctime; 8783 new_dir->i_ctime = ctime; 8784 old_inode->i_ctime = ctime; 8785 new_inode->i_ctime = ctime; 8786 8787 if (old_dentry->d_parent != new_dentry->d_parent) { 8788 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 8789 BTRFS_I(old_inode), 1); 8790 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), 8791 BTRFS_I(new_inode), 1); 8792 } 8793 8794 /* src is a subvolume */ 8795 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8796 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry); 8797 } else { /* src is an inode */ 8798 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir), 8799 BTRFS_I(old_dentry->d_inode), 8800 old_name, &old_rename_ctx); 8801 if (!ret) 8802 ret = btrfs_update_inode(trans, root, BTRFS_I(old_inode)); 8803 } 8804 if (ret) { 8805 btrfs_abort_transaction(trans, ret); 8806 goto out_fail; 8807 } 8808 8809 /* dest is a subvolume */ 8810 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 8811 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry); 8812 } else { /* dest is an inode */ 8813 ret = __btrfs_unlink_inode(trans, BTRFS_I(new_dir), 8814 BTRFS_I(new_dentry->d_inode), 8815 new_name, &new_rename_ctx); 8816 if (!ret) 8817 ret = btrfs_update_inode(trans, dest, BTRFS_I(new_inode)); 8818 } 8819 if (ret) { 8820 btrfs_abort_transaction(trans, ret); 8821 goto out_fail; 8822 } 8823 8824 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 8825 new_name, 0, old_idx); 8826 if (ret) { 8827 btrfs_abort_transaction(trans, ret); 8828 goto out_fail; 8829 } 8830 8831 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), 8832 old_name, 0, new_idx); 8833 if (ret) { 8834 btrfs_abort_transaction(trans, ret); 8835 goto out_fail; 8836 } 8837 8838 if (old_inode->i_nlink == 1) 8839 BTRFS_I(old_inode)->dir_index = old_idx; 8840 if (new_inode->i_nlink == 1) 8841 BTRFS_I(new_inode)->dir_index = new_idx; 8842 8843 /* 8844 * Now pin the logs of the roots. We do it to ensure that no other task 8845 * can sync the logs while we are in progress with the rename, because 8846 * that could result in an inconsistency in case any of the inodes that 8847 * are part of this rename operation were logged before. 8848 */ 8849 if (old_ino != BTRFS_FIRST_FREE_OBJECTID) 8850 btrfs_pin_log_trans(root); 8851 if (new_ino != BTRFS_FIRST_FREE_OBJECTID) 8852 btrfs_pin_log_trans(dest); 8853 8854 /* Do the log updates for all inodes. */ 8855 if (old_ino != BTRFS_FIRST_FREE_OBJECTID) 8856 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), 8857 old_rename_ctx.index, new_dentry->d_parent); 8858 if (new_ino != BTRFS_FIRST_FREE_OBJECTID) 8859 btrfs_log_new_name(trans, new_dentry, BTRFS_I(new_dir), 8860 new_rename_ctx.index, old_dentry->d_parent); 8861 8862 /* Now unpin the logs. */ 8863 if (old_ino != BTRFS_FIRST_FREE_OBJECTID) 8864 btrfs_end_log_trans(root); 8865 if (new_ino != BTRFS_FIRST_FREE_OBJECTID) 8866 btrfs_end_log_trans(dest); 8867 out_fail: 8868 ret2 = btrfs_end_transaction(trans); 8869 ret = ret ? ret : ret2; 8870 out_notrans: 8871 if (new_ino == BTRFS_FIRST_FREE_OBJECTID || 8872 old_ino == BTRFS_FIRST_FREE_OBJECTID) 8873 up_read(&fs_info->subvol_sem); 8874 8875 fscrypt_free_filename(&new_fname); 8876 fscrypt_free_filename(&old_fname); 8877 return ret; 8878 } 8879 8880 static struct inode *new_whiteout_inode(struct mnt_idmap *idmap, 8881 struct inode *dir) 8882 { 8883 struct inode *inode; 8884 8885 inode = new_inode(dir->i_sb); 8886 if (inode) { 8887 inode_init_owner(idmap, inode, dir, 8888 S_IFCHR | WHITEOUT_MODE); 8889 inode->i_op = &btrfs_special_inode_operations; 8890 init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); 8891 } 8892 return inode; 8893 } 8894 8895 static int btrfs_rename(struct mnt_idmap *idmap, 8896 struct inode *old_dir, struct dentry *old_dentry, 8897 struct inode *new_dir, struct dentry *new_dentry, 8898 unsigned int flags) 8899 { 8900 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb); 8901 struct btrfs_new_inode_args whiteout_args = { 8902 .dir = old_dir, 8903 .dentry = old_dentry, 8904 }; 8905 struct btrfs_trans_handle *trans; 8906 unsigned int trans_num_items; 8907 struct btrfs_root *root = BTRFS_I(old_dir)->root; 8908 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 8909 struct inode *new_inode = d_inode(new_dentry); 8910 struct inode *old_inode = d_inode(old_dentry); 8911 struct btrfs_rename_ctx rename_ctx; 8912 u64 index = 0; 8913 int ret; 8914 int ret2; 8915 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 8916 struct fscrypt_name old_fname, new_fname; 8917 8918 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 8919 return -EPERM; 8920 8921 /* we only allow rename subvolume link between subvolumes */ 8922 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 8923 return -EXDEV; 8924 8925 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 8926 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) 8927 return -ENOTEMPTY; 8928 8929 if (S_ISDIR(old_inode->i_mode) && new_inode && 8930 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 8931 return -ENOTEMPTY; 8932 8933 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname); 8934 if (ret) 8935 return ret; 8936 8937 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname); 8938 if (ret) { 8939 fscrypt_free_filename(&old_fname); 8940 return ret; 8941 } 8942 8943 /* check for collisions, even if the name isn't there */ 8944 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, &new_fname.disk_name); 8945 if (ret) { 8946 if (ret == -EEXIST) { 8947 /* we shouldn't get 8948 * eexist without a new_inode */ 8949 if (WARN_ON(!new_inode)) { 8950 goto out_fscrypt_names; 8951 } 8952 } else { 8953 /* maybe -EOVERFLOW */ 8954 goto out_fscrypt_names; 8955 } 8956 } 8957 ret = 0; 8958 8959 /* 8960 * we're using rename to replace one file with another. Start IO on it 8961 * now so we don't add too much work to the end of the transaction 8962 */ 8963 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 8964 filemap_flush(old_inode->i_mapping); 8965 8966 if (flags & RENAME_WHITEOUT) { 8967 whiteout_args.inode = new_whiteout_inode(idmap, old_dir); 8968 if (!whiteout_args.inode) { 8969 ret = -ENOMEM; 8970 goto out_fscrypt_names; 8971 } 8972 ret = btrfs_new_inode_prepare(&whiteout_args, &trans_num_items); 8973 if (ret) 8974 goto out_whiteout_inode; 8975 } else { 8976 /* 1 to update the old parent inode. */ 8977 trans_num_items = 1; 8978 } 8979 8980 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8981 /* Close the race window with snapshot create/destroy ioctl */ 8982 down_read(&fs_info->subvol_sem); 8983 /* 8984 * 1 to remove old root ref 8985 * 1 to remove old root backref 8986 * 1 to add new root ref 8987 * 1 to add new root backref 8988 */ 8989 trans_num_items += 4; 8990 } else { 8991 /* 8992 * 1 to update inode 8993 * 1 to remove old inode ref 8994 * 1 to add new inode ref 8995 */ 8996 trans_num_items += 3; 8997 } 8998 /* 8999 * 1 to remove old dir item 9000 * 1 to remove old dir index 9001 * 1 to add new dir item 9002 * 1 to add new dir index 9003 */ 9004 trans_num_items += 4; 9005 /* 1 to update new parent inode if it's not the same as the old parent */ 9006 if (new_dir != old_dir) 9007 trans_num_items++; 9008 if (new_inode) { 9009 /* 9010 * 1 to update inode 9011 * 1 to remove inode ref 9012 * 1 to remove dir item 9013 * 1 to remove dir index 9014 * 1 to possibly add orphan item 9015 */ 9016 trans_num_items += 5; 9017 } 9018 trans = btrfs_start_transaction(root, trans_num_items); 9019 if (IS_ERR(trans)) { 9020 ret = PTR_ERR(trans); 9021 goto out_notrans; 9022 } 9023 9024 if (dest != root) { 9025 ret = btrfs_record_root_in_trans(trans, dest); 9026 if (ret) 9027 goto out_fail; 9028 } 9029 9030 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); 9031 if (ret) 9032 goto out_fail; 9033 9034 BTRFS_I(old_inode)->dir_index = 0ULL; 9035 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9036 /* force full log commit if subvolume involved. */ 9037 btrfs_set_log_full_commit(trans); 9038 } else { 9039 ret = btrfs_insert_inode_ref(trans, dest, &new_fname.disk_name, 9040 old_ino, btrfs_ino(BTRFS_I(new_dir)), 9041 index); 9042 if (ret) 9043 goto out_fail; 9044 } 9045 9046 inode_inc_iversion(old_dir); 9047 inode_inc_iversion(new_dir); 9048 inode_inc_iversion(old_inode); 9049 old_dir->i_mtime = current_time(old_dir); 9050 old_dir->i_ctime = old_dir->i_mtime; 9051 new_dir->i_mtime = old_dir->i_mtime; 9052 new_dir->i_ctime = old_dir->i_mtime; 9053 old_inode->i_ctime = old_dir->i_mtime; 9054 9055 if (old_dentry->d_parent != new_dentry->d_parent) 9056 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 9057 BTRFS_I(old_inode), 1); 9058 9059 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 9060 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry); 9061 } else { 9062 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir), 9063 BTRFS_I(d_inode(old_dentry)), 9064 &old_fname.disk_name, &rename_ctx); 9065 if (!ret) 9066 ret = btrfs_update_inode(trans, root, BTRFS_I(old_inode)); 9067 } 9068 if (ret) { 9069 btrfs_abort_transaction(trans, ret); 9070 goto out_fail; 9071 } 9072 9073 if (new_inode) { 9074 inode_inc_iversion(new_inode); 9075 new_inode->i_ctime = current_time(new_inode); 9076 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == 9077 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 9078 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry); 9079 BUG_ON(new_inode->i_nlink == 0); 9080 } else { 9081 ret = btrfs_unlink_inode(trans, BTRFS_I(new_dir), 9082 BTRFS_I(d_inode(new_dentry)), 9083 &new_fname.disk_name); 9084 } 9085 if (!ret && new_inode->i_nlink == 0) 9086 ret = btrfs_orphan_add(trans, 9087 BTRFS_I(d_inode(new_dentry))); 9088 if (ret) { 9089 btrfs_abort_transaction(trans, ret); 9090 goto out_fail; 9091 } 9092 } 9093 9094 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 9095 &new_fname.disk_name, 0, index); 9096 if (ret) { 9097 btrfs_abort_transaction(trans, ret); 9098 goto out_fail; 9099 } 9100 9101 if (old_inode->i_nlink == 1) 9102 BTRFS_I(old_inode)->dir_index = index; 9103 9104 if (old_ino != BTRFS_FIRST_FREE_OBJECTID) 9105 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), 9106 rename_ctx.index, new_dentry->d_parent); 9107 9108 if (flags & RENAME_WHITEOUT) { 9109 ret = btrfs_create_new_inode(trans, &whiteout_args); 9110 if (ret) { 9111 btrfs_abort_transaction(trans, ret); 9112 goto out_fail; 9113 } else { 9114 unlock_new_inode(whiteout_args.inode); 9115 iput(whiteout_args.inode); 9116 whiteout_args.inode = NULL; 9117 } 9118 } 9119 out_fail: 9120 ret2 = btrfs_end_transaction(trans); 9121 ret = ret ? ret : ret2; 9122 out_notrans: 9123 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 9124 up_read(&fs_info->subvol_sem); 9125 if (flags & RENAME_WHITEOUT) 9126 btrfs_new_inode_args_destroy(&whiteout_args); 9127 out_whiteout_inode: 9128 if (flags & RENAME_WHITEOUT) 9129 iput(whiteout_args.inode); 9130 out_fscrypt_names: 9131 fscrypt_free_filename(&old_fname); 9132 fscrypt_free_filename(&new_fname); 9133 return ret; 9134 } 9135 9136 static int btrfs_rename2(struct mnt_idmap *idmap, struct inode *old_dir, 9137 struct dentry *old_dentry, struct inode *new_dir, 9138 struct dentry *new_dentry, unsigned int flags) 9139 { 9140 int ret; 9141 9142 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 9143 return -EINVAL; 9144 9145 if (flags & RENAME_EXCHANGE) 9146 ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir, 9147 new_dentry); 9148 else 9149 ret = btrfs_rename(idmap, old_dir, old_dentry, new_dir, 9150 new_dentry, flags); 9151 9152 btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info); 9153 9154 return ret; 9155 } 9156 9157 struct btrfs_delalloc_work { 9158 struct inode *inode; 9159 struct completion completion; 9160 struct list_head list; 9161 struct btrfs_work work; 9162 }; 9163 9164 static void btrfs_run_delalloc_work(struct btrfs_work *work) 9165 { 9166 struct btrfs_delalloc_work *delalloc_work; 9167 struct inode *inode; 9168 9169 delalloc_work = container_of(work, struct btrfs_delalloc_work, 9170 work); 9171 inode = delalloc_work->inode; 9172 filemap_flush(inode->i_mapping); 9173 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 9174 &BTRFS_I(inode)->runtime_flags)) 9175 filemap_flush(inode->i_mapping); 9176 9177 iput(inode); 9178 complete(&delalloc_work->completion); 9179 } 9180 9181 static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode) 9182 { 9183 struct btrfs_delalloc_work *work; 9184 9185 work = kmalloc(sizeof(*work), GFP_NOFS); 9186 if (!work) 9187 return NULL; 9188 9189 init_completion(&work->completion); 9190 INIT_LIST_HEAD(&work->list); 9191 work->inode = inode; 9192 btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL, NULL); 9193 9194 return work; 9195 } 9196 9197 /* 9198 * some fairly slow code that needs optimization. This walks the list 9199 * of all the inodes with pending delalloc and forces them to disk. 9200 */ 9201 static int start_delalloc_inodes(struct btrfs_root *root, 9202 struct writeback_control *wbc, bool snapshot, 9203 bool in_reclaim_context) 9204 { 9205 struct btrfs_inode *binode; 9206 struct inode *inode; 9207 struct btrfs_delalloc_work *work, *next; 9208 struct list_head works; 9209 struct list_head splice; 9210 int ret = 0; 9211 bool full_flush = wbc->nr_to_write == LONG_MAX; 9212 9213 INIT_LIST_HEAD(&works); 9214 INIT_LIST_HEAD(&splice); 9215 9216 mutex_lock(&root->delalloc_mutex); 9217 spin_lock(&root->delalloc_lock); 9218 list_splice_init(&root->delalloc_inodes, &splice); 9219 while (!list_empty(&splice)) { 9220 binode = list_entry(splice.next, struct btrfs_inode, 9221 delalloc_inodes); 9222 9223 list_move_tail(&binode->delalloc_inodes, 9224 &root->delalloc_inodes); 9225 9226 if (in_reclaim_context && 9227 test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &binode->runtime_flags)) 9228 continue; 9229 9230 inode = igrab(&binode->vfs_inode); 9231 if (!inode) { 9232 cond_resched_lock(&root->delalloc_lock); 9233 continue; 9234 } 9235 spin_unlock(&root->delalloc_lock); 9236 9237 if (snapshot) 9238 set_bit(BTRFS_INODE_SNAPSHOT_FLUSH, 9239 &binode->runtime_flags); 9240 if (full_flush) { 9241 work = btrfs_alloc_delalloc_work(inode); 9242 if (!work) { 9243 iput(inode); 9244 ret = -ENOMEM; 9245 goto out; 9246 } 9247 list_add_tail(&work->list, &works); 9248 btrfs_queue_work(root->fs_info->flush_workers, 9249 &work->work); 9250 } else { 9251 ret = filemap_fdatawrite_wbc(inode->i_mapping, wbc); 9252 btrfs_add_delayed_iput(BTRFS_I(inode)); 9253 if (ret || wbc->nr_to_write <= 0) 9254 goto out; 9255 } 9256 cond_resched(); 9257 spin_lock(&root->delalloc_lock); 9258 } 9259 spin_unlock(&root->delalloc_lock); 9260 9261 out: 9262 list_for_each_entry_safe(work, next, &works, list) { 9263 list_del_init(&work->list); 9264 wait_for_completion(&work->completion); 9265 kfree(work); 9266 } 9267 9268 if (!list_empty(&splice)) { 9269 spin_lock(&root->delalloc_lock); 9270 list_splice_tail(&splice, &root->delalloc_inodes); 9271 spin_unlock(&root->delalloc_lock); 9272 } 9273 mutex_unlock(&root->delalloc_mutex); 9274 return ret; 9275 } 9276 9277 int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context) 9278 { 9279 struct writeback_control wbc = { 9280 .nr_to_write = LONG_MAX, 9281 .sync_mode = WB_SYNC_NONE, 9282 .range_start = 0, 9283 .range_end = LLONG_MAX, 9284 }; 9285 struct btrfs_fs_info *fs_info = root->fs_info; 9286 9287 if (BTRFS_FS_ERROR(fs_info)) 9288 return -EROFS; 9289 9290 return start_delalloc_inodes(root, &wbc, true, in_reclaim_context); 9291 } 9292 9293 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, 9294 bool in_reclaim_context) 9295 { 9296 struct writeback_control wbc = { 9297 .nr_to_write = nr, 9298 .sync_mode = WB_SYNC_NONE, 9299 .range_start = 0, 9300 .range_end = LLONG_MAX, 9301 }; 9302 struct btrfs_root *root; 9303 struct list_head splice; 9304 int ret; 9305 9306 if (BTRFS_FS_ERROR(fs_info)) 9307 return -EROFS; 9308 9309 INIT_LIST_HEAD(&splice); 9310 9311 mutex_lock(&fs_info->delalloc_root_mutex); 9312 spin_lock(&fs_info->delalloc_root_lock); 9313 list_splice_init(&fs_info->delalloc_roots, &splice); 9314 while (!list_empty(&splice)) { 9315 /* 9316 * Reset nr_to_write here so we know that we're doing a full 9317 * flush. 9318 */ 9319 if (nr == LONG_MAX) 9320 wbc.nr_to_write = LONG_MAX; 9321 9322 root = list_first_entry(&splice, struct btrfs_root, 9323 delalloc_root); 9324 root = btrfs_grab_root(root); 9325 BUG_ON(!root); 9326 list_move_tail(&root->delalloc_root, 9327 &fs_info->delalloc_roots); 9328 spin_unlock(&fs_info->delalloc_root_lock); 9329 9330 ret = start_delalloc_inodes(root, &wbc, false, in_reclaim_context); 9331 btrfs_put_root(root); 9332 if (ret < 0 || wbc.nr_to_write <= 0) 9333 goto out; 9334 spin_lock(&fs_info->delalloc_root_lock); 9335 } 9336 spin_unlock(&fs_info->delalloc_root_lock); 9337 9338 ret = 0; 9339 out: 9340 if (!list_empty(&splice)) { 9341 spin_lock(&fs_info->delalloc_root_lock); 9342 list_splice_tail(&splice, &fs_info->delalloc_roots); 9343 spin_unlock(&fs_info->delalloc_root_lock); 9344 } 9345 mutex_unlock(&fs_info->delalloc_root_mutex); 9346 return ret; 9347 } 9348 9349 static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir, 9350 struct dentry *dentry, const char *symname) 9351 { 9352 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 9353 struct btrfs_trans_handle *trans; 9354 struct btrfs_root *root = BTRFS_I(dir)->root; 9355 struct btrfs_path *path; 9356 struct btrfs_key key; 9357 struct inode *inode; 9358 struct btrfs_new_inode_args new_inode_args = { 9359 .dir = dir, 9360 .dentry = dentry, 9361 }; 9362 unsigned int trans_num_items; 9363 int err; 9364 int name_len; 9365 int datasize; 9366 unsigned long ptr; 9367 struct btrfs_file_extent_item *ei; 9368 struct extent_buffer *leaf; 9369 9370 name_len = strlen(symname); 9371 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 9372 return -ENAMETOOLONG; 9373 9374 inode = new_inode(dir->i_sb); 9375 if (!inode) 9376 return -ENOMEM; 9377 inode_init_owner(idmap, inode, dir, S_IFLNK | S_IRWXUGO); 9378 inode->i_op = &btrfs_symlink_inode_operations; 9379 inode_nohighmem(inode); 9380 inode->i_mapping->a_ops = &btrfs_aops; 9381 btrfs_i_size_write(BTRFS_I(inode), name_len); 9382 inode_set_bytes(inode, name_len); 9383 9384 new_inode_args.inode = inode; 9385 err = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); 9386 if (err) 9387 goto out_inode; 9388 /* 1 additional item for the inline extent */ 9389 trans_num_items++; 9390 9391 trans = btrfs_start_transaction(root, trans_num_items); 9392 if (IS_ERR(trans)) { 9393 err = PTR_ERR(trans); 9394 goto out_new_inode_args; 9395 } 9396 9397 err = btrfs_create_new_inode(trans, &new_inode_args); 9398 if (err) 9399 goto out; 9400 9401 path = btrfs_alloc_path(); 9402 if (!path) { 9403 err = -ENOMEM; 9404 btrfs_abort_transaction(trans, err); 9405 discard_new_inode(inode); 9406 inode = NULL; 9407 goto out; 9408 } 9409 key.objectid = btrfs_ino(BTRFS_I(inode)); 9410 key.offset = 0; 9411 key.type = BTRFS_EXTENT_DATA_KEY; 9412 datasize = btrfs_file_extent_calc_inline_size(name_len); 9413 err = btrfs_insert_empty_item(trans, root, path, &key, 9414 datasize); 9415 if (err) { 9416 btrfs_abort_transaction(trans, err); 9417 btrfs_free_path(path); 9418 discard_new_inode(inode); 9419 inode = NULL; 9420 goto out; 9421 } 9422 leaf = path->nodes[0]; 9423 ei = btrfs_item_ptr(leaf, path->slots[0], 9424 struct btrfs_file_extent_item); 9425 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 9426 btrfs_set_file_extent_type(leaf, ei, 9427 BTRFS_FILE_EXTENT_INLINE); 9428 btrfs_set_file_extent_encryption(leaf, ei, 0); 9429 btrfs_set_file_extent_compression(leaf, ei, 0); 9430 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 9431 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 9432 9433 ptr = btrfs_file_extent_inline_start(ei); 9434 write_extent_buffer(leaf, symname, ptr, name_len); 9435 btrfs_mark_buffer_dirty(leaf); 9436 btrfs_free_path(path); 9437 9438 d_instantiate_new(dentry, inode); 9439 err = 0; 9440 out: 9441 btrfs_end_transaction(trans); 9442 btrfs_btree_balance_dirty(fs_info); 9443 out_new_inode_args: 9444 btrfs_new_inode_args_destroy(&new_inode_args); 9445 out_inode: 9446 if (err) 9447 iput(inode); 9448 return err; 9449 } 9450 9451 static struct btrfs_trans_handle *insert_prealloc_file_extent( 9452 struct btrfs_trans_handle *trans_in, 9453 struct btrfs_inode *inode, 9454 struct btrfs_key *ins, 9455 u64 file_offset) 9456 { 9457 struct btrfs_file_extent_item stack_fi; 9458 struct btrfs_replace_extent_info extent_info; 9459 struct btrfs_trans_handle *trans = trans_in; 9460 struct btrfs_path *path; 9461 u64 start = ins->objectid; 9462 u64 len = ins->offset; 9463 int qgroup_released; 9464 int ret; 9465 9466 memset(&stack_fi, 0, sizeof(stack_fi)); 9467 9468 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC); 9469 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start); 9470 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len); 9471 btrfs_set_stack_file_extent_num_bytes(&stack_fi, len); 9472 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len); 9473 btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE); 9474 /* Encryption and other encoding is reserved and all 0 */ 9475 9476 qgroup_released = btrfs_qgroup_release_data(inode, file_offset, len); 9477 if (qgroup_released < 0) 9478 return ERR_PTR(qgroup_released); 9479 9480 if (trans) { 9481 ret = insert_reserved_file_extent(trans, inode, 9482 file_offset, &stack_fi, 9483 true, qgroup_released); 9484 if (ret) 9485 goto free_qgroup; 9486 return trans; 9487 } 9488 9489 extent_info.disk_offset = start; 9490 extent_info.disk_len = len; 9491 extent_info.data_offset = 0; 9492 extent_info.data_len = len; 9493 extent_info.file_offset = file_offset; 9494 extent_info.extent_buf = (char *)&stack_fi; 9495 extent_info.is_new_extent = true; 9496 extent_info.update_times = true; 9497 extent_info.qgroup_reserved = qgroup_released; 9498 extent_info.insertions = 0; 9499 9500 path = btrfs_alloc_path(); 9501 if (!path) { 9502 ret = -ENOMEM; 9503 goto free_qgroup; 9504 } 9505 9506 ret = btrfs_replace_file_extents(inode, path, file_offset, 9507 file_offset + len - 1, &extent_info, 9508 &trans); 9509 btrfs_free_path(path); 9510 if (ret) 9511 goto free_qgroup; 9512 return trans; 9513 9514 free_qgroup: 9515 /* 9516 * We have released qgroup data range at the beginning of the function, 9517 * and normally qgroup_released bytes will be freed when committing 9518 * transaction. 9519 * But if we error out early, we have to free what we have released 9520 * or we leak qgroup data reservation. 9521 */ 9522 btrfs_qgroup_free_refroot(inode->root->fs_info, 9523 inode->root->root_key.objectid, qgroup_released, 9524 BTRFS_QGROUP_RSV_DATA); 9525 return ERR_PTR(ret); 9526 } 9527 9528 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 9529 u64 start, u64 num_bytes, u64 min_size, 9530 loff_t actual_len, u64 *alloc_hint, 9531 struct btrfs_trans_handle *trans) 9532 { 9533 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); 9534 struct extent_map *em; 9535 struct btrfs_root *root = BTRFS_I(inode)->root; 9536 struct btrfs_key ins; 9537 u64 cur_offset = start; 9538 u64 clear_offset = start; 9539 u64 i_size; 9540 u64 cur_bytes; 9541 u64 last_alloc = (u64)-1; 9542 int ret = 0; 9543 bool own_trans = true; 9544 u64 end = start + num_bytes - 1; 9545 9546 if (trans) 9547 own_trans = false; 9548 while (num_bytes > 0) { 9549 cur_bytes = min_t(u64, num_bytes, SZ_256M); 9550 cur_bytes = max(cur_bytes, min_size); 9551 /* 9552 * If we are severely fragmented we could end up with really 9553 * small allocations, so if the allocator is returning small 9554 * chunks lets make its job easier by only searching for those 9555 * sized chunks. 9556 */ 9557 cur_bytes = min(cur_bytes, last_alloc); 9558 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 9559 min_size, 0, *alloc_hint, &ins, 1, 0); 9560 if (ret) 9561 break; 9562 9563 /* 9564 * We've reserved this space, and thus converted it from 9565 * ->bytes_may_use to ->bytes_reserved. Any error that happens 9566 * from here on out we will only need to clear our reservation 9567 * for the remaining unreserved area, so advance our 9568 * clear_offset by our extent size. 9569 */ 9570 clear_offset += ins.offset; 9571 9572 last_alloc = ins.offset; 9573 trans = insert_prealloc_file_extent(trans, BTRFS_I(inode), 9574 &ins, cur_offset); 9575 /* 9576 * Now that we inserted the prealloc extent we can finally 9577 * decrement the number of reservations in the block group. 9578 * If we did it before, we could race with relocation and have 9579 * relocation miss the reserved extent, making it fail later. 9580 */ 9581 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 9582 if (IS_ERR(trans)) { 9583 ret = PTR_ERR(trans); 9584 btrfs_free_reserved_extent(fs_info, ins.objectid, 9585 ins.offset, 0); 9586 break; 9587 } 9588 9589 em = alloc_extent_map(); 9590 if (!em) { 9591 btrfs_drop_extent_map_range(BTRFS_I(inode), cur_offset, 9592 cur_offset + ins.offset - 1, false); 9593 btrfs_set_inode_full_sync(BTRFS_I(inode)); 9594 goto next; 9595 } 9596 9597 em->start = cur_offset; 9598 em->orig_start = cur_offset; 9599 em->len = ins.offset; 9600 em->block_start = ins.objectid; 9601 em->block_len = ins.offset; 9602 em->orig_block_len = ins.offset; 9603 em->ram_bytes = ins.offset; 9604 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 9605 em->generation = trans->transid; 9606 9607 ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, true); 9608 free_extent_map(em); 9609 next: 9610 num_bytes -= ins.offset; 9611 cur_offset += ins.offset; 9612 *alloc_hint = ins.objectid + ins.offset; 9613 9614 inode_inc_iversion(inode); 9615 inode->i_ctime = current_time(inode); 9616 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 9617 if (!(mode & FALLOC_FL_KEEP_SIZE) && 9618 (actual_len > inode->i_size) && 9619 (cur_offset > inode->i_size)) { 9620 if (cur_offset > actual_len) 9621 i_size = actual_len; 9622 else 9623 i_size = cur_offset; 9624 i_size_write(inode, i_size); 9625 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); 9626 } 9627 9628 ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); 9629 9630 if (ret) { 9631 btrfs_abort_transaction(trans, ret); 9632 if (own_trans) 9633 btrfs_end_transaction(trans); 9634 break; 9635 } 9636 9637 if (own_trans) { 9638 btrfs_end_transaction(trans); 9639 trans = NULL; 9640 } 9641 } 9642 if (clear_offset < end) 9643 btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset, 9644 end - clear_offset + 1); 9645 return ret; 9646 } 9647 9648 int btrfs_prealloc_file_range(struct inode *inode, int mode, 9649 u64 start, u64 num_bytes, u64 min_size, 9650 loff_t actual_len, u64 *alloc_hint) 9651 { 9652 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9653 min_size, actual_len, alloc_hint, 9654 NULL); 9655 } 9656 9657 int btrfs_prealloc_file_range_trans(struct inode *inode, 9658 struct btrfs_trans_handle *trans, int mode, 9659 u64 start, u64 num_bytes, u64 min_size, 9660 loff_t actual_len, u64 *alloc_hint) 9661 { 9662 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9663 min_size, actual_len, alloc_hint, trans); 9664 } 9665 9666 static int btrfs_permission(struct mnt_idmap *idmap, 9667 struct inode *inode, int mask) 9668 { 9669 struct btrfs_root *root = BTRFS_I(inode)->root; 9670 umode_t mode = inode->i_mode; 9671 9672 if (mask & MAY_WRITE && 9673 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 9674 if (btrfs_root_readonly(root)) 9675 return -EROFS; 9676 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 9677 return -EACCES; 9678 } 9679 return generic_permission(idmap, inode, mask); 9680 } 9681 9682 static int btrfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, 9683 struct file *file, umode_t mode) 9684 { 9685 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); 9686 struct btrfs_trans_handle *trans; 9687 struct btrfs_root *root = BTRFS_I(dir)->root; 9688 struct inode *inode; 9689 struct btrfs_new_inode_args new_inode_args = { 9690 .dir = dir, 9691 .dentry = file->f_path.dentry, 9692 .orphan = true, 9693 }; 9694 unsigned int trans_num_items; 9695 int ret; 9696 9697 inode = new_inode(dir->i_sb); 9698 if (!inode) 9699 return -ENOMEM; 9700 inode_init_owner(idmap, inode, dir, mode); 9701 inode->i_fop = &btrfs_file_operations; 9702 inode->i_op = &btrfs_file_inode_operations; 9703 inode->i_mapping->a_ops = &btrfs_aops; 9704 9705 new_inode_args.inode = inode; 9706 ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); 9707 if (ret) 9708 goto out_inode; 9709 9710 trans = btrfs_start_transaction(root, trans_num_items); 9711 if (IS_ERR(trans)) { 9712 ret = PTR_ERR(trans); 9713 goto out_new_inode_args; 9714 } 9715 9716 ret = btrfs_create_new_inode(trans, &new_inode_args); 9717 9718 /* 9719 * We set number of links to 0 in btrfs_create_new_inode(), and here we 9720 * set it to 1 because d_tmpfile() will issue a warning if the count is 9721 * 0, through: 9722 * 9723 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 9724 */ 9725 set_nlink(inode, 1); 9726 9727 if (!ret) { 9728 d_tmpfile(file, inode); 9729 unlock_new_inode(inode); 9730 mark_inode_dirty(inode); 9731 } 9732 9733 btrfs_end_transaction(trans); 9734 btrfs_btree_balance_dirty(fs_info); 9735 out_new_inode_args: 9736 btrfs_new_inode_args_destroy(&new_inode_args); 9737 out_inode: 9738 if (ret) 9739 iput(inode); 9740 return finish_open_simple(file, ret); 9741 } 9742 9743 void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end) 9744 { 9745 struct btrfs_fs_info *fs_info = inode->root->fs_info; 9746 unsigned long index = start >> PAGE_SHIFT; 9747 unsigned long end_index = end >> PAGE_SHIFT; 9748 struct page *page; 9749 u32 len; 9750 9751 ASSERT(end + 1 - start <= U32_MAX); 9752 len = end + 1 - start; 9753 while (index <= end_index) { 9754 page = find_get_page(inode->vfs_inode.i_mapping, index); 9755 ASSERT(page); /* Pages should be in the extent_io_tree */ 9756 9757 btrfs_page_set_writeback(fs_info, page, start, len); 9758 put_page(page); 9759 index++; 9760 } 9761 } 9762 9763 int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info, 9764 int compress_type) 9765 { 9766 switch (compress_type) { 9767 case BTRFS_COMPRESS_NONE: 9768 return BTRFS_ENCODED_IO_COMPRESSION_NONE; 9769 case BTRFS_COMPRESS_ZLIB: 9770 return BTRFS_ENCODED_IO_COMPRESSION_ZLIB; 9771 case BTRFS_COMPRESS_LZO: 9772 /* 9773 * The LZO format depends on the sector size. 64K is the maximum 9774 * sector size that we support. 9775 */ 9776 if (fs_info->sectorsize < SZ_4K || fs_info->sectorsize > SZ_64K) 9777 return -EINVAL; 9778 return BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 9779 (fs_info->sectorsize_bits - 12); 9780 case BTRFS_COMPRESS_ZSTD: 9781 return BTRFS_ENCODED_IO_COMPRESSION_ZSTD; 9782 default: 9783 return -EUCLEAN; 9784 } 9785 } 9786 9787 static ssize_t btrfs_encoded_read_inline( 9788 struct kiocb *iocb, 9789 struct iov_iter *iter, u64 start, 9790 u64 lockend, 9791 struct extent_state **cached_state, 9792 u64 extent_start, size_t count, 9793 struct btrfs_ioctl_encoded_io_args *encoded, 9794 bool *unlocked) 9795 { 9796 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 9797 struct btrfs_root *root = inode->root; 9798 struct btrfs_fs_info *fs_info = root->fs_info; 9799 struct extent_io_tree *io_tree = &inode->io_tree; 9800 struct btrfs_path *path; 9801 struct extent_buffer *leaf; 9802 struct btrfs_file_extent_item *item; 9803 u64 ram_bytes; 9804 unsigned long ptr; 9805 void *tmp; 9806 ssize_t ret; 9807 9808 path = btrfs_alloc_path(); 9809 if (!path) { 9810 ret = -ENOMEM; 9811 goto out; 9812 } 9813 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), 9814 extent_start, 0); 9815 if (ret) { 9816 if (ret > 0) { 9817 /* The extent item disappeared? */ 9818 ret = -EIO; 9819 } 9820 goto out; 9821 } 9822 leaf = path->nodes[0]; 9823 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); 9824 9825 ram_bytes = btrfs_file_extent_ram_bytes(leaf, item); 9826 ptr = btrfs_file_extent_inline_start(item); 9827 9828 encoded->len = min_t(u64, extent_start + ram_bytes, 9829 inode->vfs_inode.i_size) - iocb->ki_pos; 9830 ret = btrfs_encoded_io_compression_from_extent(fs_info, 9831 btrfs_file_extent_compression(leaf, item)); 9832 if (ret < 0) 9833 goto out; 9834 encoded->compression = ret; 9835 if (encoded->compression) { 9836 size_t inline_size; 9837 9838 inline_size = btrfs_file_extent_inline_item_len(leaf, 9839 path->slots[0]); 9840 if (inline_size > count) { 9841 ret = -ENOBUFS; 9842 goto out; 9843 } 9844 count = inline_size; 9845 encoded->unencoded_len = ram_bytes; 9846 encoded->unencoded_offset = iocb->ki_pos - extent_start; 9847 } else { 9848 count = min_t(u64, count, encoded->len); 9849 encoded->len = count; 9850 encoded->unencoded_len = count; 9851 ptr += iocb->ki_pos - extent_start; 9852 } 9853 9854 tmp = kmalloc(count, GFP_NOFS); 9855 if (!tmp) { 9856 ret = -ENOMEM; 9857 goto out; 9858 } 9859 read_extent_buffer(leaf, tmp, ptr, count); 9860 btrfs_release_path(path); 9861 unlock_extent(io_tree, start, lockend, cached_state); 9862 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 9863 *unlocked = true; 9864 9865 ret = copy_to_iter(tmp, count, iter); 9866 if (ret != count) 9867 ret = -EFAULT; 9868 kfree(tmp); 9869 out: 9870 btrfs_free_path(path); 9871 return ret; 9872 } 9873 9874 struct btrfs_encoded_read_private { 9875 wait_queue_head_t wait; 9876 atomic_t pending; 9877 blk_status_t status; 9878 }; 9879 9880 static void btrfs_encoded_read_endio(struct btrfs_bio *bbio) 9881 { 9882 struct btrfs_encoded_read_private *priv = bbio->private; 9883 9884 if (bbio->bio.bi_status) { 9885 /* 9886 * The memory barrier implied by the atomic_dec_return() here 9887 * pairs with the memory barrier implied by the 9888 * atomic_dec_return() or io_wait_event() in 9889 * btrfs_encoded_read_regular_fill_pages() to ensure that this 9890 * write is observed before the load of status in 9891 * btrfs_encoded_read_regular_fill_pages(). 9892 */ 9893 WRITE_ONCE(priv->status, bbio->bio.bi_status); 9894 } 9895 if (!atomic_dec_return(&priv->pending)) 9896 wake_up(&priv->wait); 9897 bio_put(&bbio->bio); 9898 } 9899 9900 int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, 9901 u64 file_offset, u64 disk_bytenr, 9902 u64 disk_io_size, struct page **pages) 9903 { 9904 struct btrfs_fs_info *fs_info = inode->root->fs_info; 9905 struct btrfs_encoded_read_private priv = { 9906 .pending = ATOMIC_INIT(1), 9907 }; 9908 unsigned long i = 0; 9909 struct btrfs_bio *bbio; 9910 9911 init_waitqueue_head(&priv.wait); 9912 9913 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info, 9914 btrfs_encoded_read_endio, &priv); 9915 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; 9916 bbio->inode = inode; 9917 9918 do { 9919 size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE); 9920 9921 if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) { 9922 atomic_inc(&priv.pending); 9923 btrfs_submit_bio(bbio, 0); 9924 9925 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info, 9926 btrfs_encoded_read_endio, &priv); 9927 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; 9928 bbio->inode = inode; 9929 continue; 9930 } 9931 9932 i++; 9933 disk_bytenr += bytes; 9934 disk_io_size -= bytes; 9935 } while (disk_io_size); 9936 9937 atomic_inc(&priv.pending); 9938 btrfs_submit_bio(bbio, 0); 9939 9940 if (atomic_dec_return(&priv.pending)) 9941 io_wait_event(priv.wait, !atomic_read(&priv.pending)); 9942 /* See btrfs_encoded_read_endio() for ordering. */ 9943 return blk_status_to_errno(READ_ONCE(priv.status)); 9944 } 9945 9946 static ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, 9947 struct iov_iter *iter, 9948 u64 start, u64 lockend, 9949 struct extent_state **cached_state, 9950 u64 disk_bytenr, u64 disk_io_size, 9951 size_t count, bool compressed, 9952 bool *unlocked) 9953 { 9954 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 9955 struct extent_io_tree *io_tree = &inode->io_tree; 9956 struct page **pages; 9957 unsigned long nr_pages, i; 9958 u64 cur; 9959 size_t page_offset; 9960 ssize_t ret; 9961 9962 nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE); 9963 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 9964 if (!pages) 9965 return -ENOMEM; 9966 ret = btrfs_alloc_page_array(nr_pages, pages); 9967 if (ret) { 9968 ret = -ENOMEM; 9969 goto out; 9970 } 9971 9972 ret = btrfs_encoded_read_regular_fill_pages(inode, start, disk_bytenr, 9973 disk_io_size, pages); 9974 if (ret) 9975 goto out; 9976 9977 unlock_extent(io_tree, start, lockend, cached_state); 9978 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 9979 *unlocked = true; 9980 9981 if (compressed) { 9982 i = 0; 9983 page_offset = 0; 9984 } else { 9985 i = (iocb->ki_pos - start) >> PAGE_SHIFT; 9986 page_offset = (iocb->ki_pos - start) & (PAGE_SIZE - 1); 9987 } 9988 cur = 0; 9989 while (cur < count) { 9990 size_t bytes = min_t(size_t, count - cur, 9991 PAGE_SIZE - page_offset); 9992 9993 if (copy_page_to_iter(pages[i], page_offset, bytes, 9994 iter) != bytes) { 9995 ret = -EFAULT; 9996 goto out; 9997 } 9998 i++; 9999 cur += bytes; 10000 page_offset = 0; 10001 } 10002 ret = count; 10003 out: 10004 for (i = 0; i < nr_pages; i++) { 10005 if (pages[i]) 10006 __free_page(pages[i]); 10007 } 10008 kfree(pages); 10009 return ret; 10010 } 10011 10012 ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter, 10013 struct btrfs_ioctl_encoded_io_args *encoded) 10014 { 10015 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 10016 struct btrfs_fs_info *fs_info = inode->root->fs_info; 10017 struct extent_io_tree *io_tree = &inode->io_tree; 10018 ssize_t ret; 10019 size_t count = iov_iter_count(iter); 10020 u64 start, lockend, disk_bytenr, disk_io_size; 10021 struct extent_state *cached_state = NULL; 10022 struct extent_map *em; 10023 bool unlocked = false; 10024 10025 file_accessed(iocb->ki_filp); 10026 10027 btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED); 10028 10029 if (iocb->ki_pos >= inode->vfs_inode.i_size) { 10030 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 10031 return 0; 10032 } 10033 start = ALIGN_DOWN(iocb->ki_pos, fs_info->sectorsize); 10034 /* 10035 * We don't know how long the extent containing iocb->ki_pos is, but if 10036 * it's compressed we know that it won't be longer than this. 10037 */ 10038 lockend = start + BTRFS_MAX_UNCOMPRESSED - 1; 10039 10040 for (;;) { 10041 struct btrfs_ordered_extent *ordered; 10042 10043 ret = btrfs_wait_ordered_range(&inode->vfs_inode, start, 10044 lockend - start + 1); 10045 if (ret) 10046 goto out_unlock_inode; 10047 lock_extent(io_tree, start, lockend, &cached_state); 10048 ordered = btrfs_lookup_ordered_range(inode, start, 10049 lockend - start + 1); 10050 if (!ordered) 10051 break; 10052 btrfs_put_ordered_extent(ordered); 10053 unlock_extent(io_tree, start, lockend, &cached_state); 10054 cond_resched(); 10055 } 10056 10057 em = btrfs_get_extent(inode, NULL, 0, start, lockend - start + 1); 10058 if (IS_ERR(em)) { 10059 ret = PTR_ERR(em); 10060 goto out_unlock_extent; 10061 } 10062 10063 if (em->block_start == EXTENT_MAP_INLINE) { 10064 u64 extent_start = em->start; 10065 10066 /* 10067 * For inline extents we get everything we need out of the 10068 * extent item. 10069 */ 10070 free_extent_map(em); 10071 em = NULL; 10072 ret = btrfs_encoded_read_inline(iocb, iter, start, lockend, 10073 &cached_state, extent_start, 10074 count, encoded, &unlocked); 10075 goto out; 10076 } 10077 10078 /* 10079 * We only want to return up to EOF even if the extent extends beyond 10080 * that. 10081 */ 10082 encoded->len = min_t(u64, extent_map_end(em), 10083 inode->vfs_inode.i_size) - iocb->ki_pos; 10084 if (em->block_start == EXTENT_MAP_HOLE || 10085 test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 10086 disk_bytenr = EXTENT_MAP_HOLE; 10087 count = min_t(u64, count, encoded->len); 10088 encoded->len = count; 10089 encoded->unencoded_len = count; 10090 } else if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 10091 disk_bytenr = em->block_start; 10092 /* 10093 * Bail if the buffer isn't large enough to return the whole 10094 * compressed extent. 10095 */ 10096 if (em->block_len > count) { 10097 ret = -ENOBUFS; 10098 goto out_em; 10099 } 10100 disk_io_size = em->block_len; 10101 count = em->block_len; 10102 encoded->unencoded_len = em->ram_bytes; 10103 encoded->unencoded_offset = iocb->ki_pos - em->orig_start; 10104 ret = btrfs_encoded_io_compression_from_extent(fs_info, 10105 em->compress_type); 10106 if (ret < 0) 10107 goto out_em; 10108 encoded->compression = ret; 10109 } else { 10110 disk_bytenr = em->block_start + (start - em->start); 10111 if (encoded->len > count) 10112 encoded->len = count; 10113 /* 10114 * Don't read beyond what we locked. This also limits the page 10115 * allocations that we'll do. 10116 */ 10117 disk_io_size = min(lockend + 1, iocb->ki_pos + encoded->len) - start; 10118 count = start + disk_io_size - iocb->ki_pos; 10119 encoded->len = count; 10120 encoded->unencoded_len = count; 10121 disk_io_size = ALIGN(disk_io_size, fs_info->sectorsize); 10122 } 10123 free_extent_map(em); 10124 em = NULL; 10125 10126 if (disk_bytenr == EXTENT_MAP_HOLE) { 10127 unlock_extent(io_tree, start, lockend, &cached_state); 10128 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 10129 unlocked = true; 10130 ret = iov_iter_zero(count, iter); 10131 if (ret != count) 10132 ret = -EFAULT; 10133 } else { 10134 ret = btrfs_encoded_read_regular(iocb, iter, start, lockend, 10135 &cached_state, disk_bytenr, 10136 disk_io_size, count, 10137 encoded->compression, 10138 &unlocked); 10139 } 10140 10141 out: 10142 if (ret >= 0) 10143 iocb->ki_pos += encoded->len; 10144 out_em: 10145 free_extent_map(em); 10146 out_unlock_extent: 10147 if (!unlocked) 10148 unlock_extent(io_tree, start, lockend, &cached_state); 10149 out_unlock_inode: 10150 if (!unlocked) 10151 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 10152 return ret; 10153 } 10154 10155 ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, 10156 const struct btrfs_ioctl_encoded_io_args *encoded) 10157 { 10158 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 10159 struct btrfs_root *root = inode->root; 10160 struct btrfs_fs_info *fs_info = root->fs_info; 10161 struct extent_io_tree *io_tree = &inode->io_tree; 10162 struct extent_changeset *data_reserved = NULL; 10163 struct extent_state *cached_state = NULL; 10164 int compression; 10165 size_t orig_count; 10166 u64 start, end; 10167 u64 num_bytes, ram_bytes, disk_num_bytes; 10168 unsigned long nr_pages, i; 10169 struct page **pages; 10170 struct btrfs_key ins; 10171 bool extent_reserved = false; 10172 struct extent_map *em; 10173 ssize_t ret; 10174 10175 switch (encoded->compression) { 10176 case BTRFS_ENCODED_IO_COMPRESSION_ZLIB: 10177 compression = BTRFS_COMPRESS_ZLIB; 10178 break; 10179 case BTRFS_ENCODED_IO_COMPRESSION_ZSTD: 10180 compression = BTRFS_COMPRESS_ZSTD; 10181 break; 10182 case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K: 10183 case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K: 10184 case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K: 10185 case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K: 10186 case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K: 10187 /* The sector size must match for LZO. */ 10188 if (encoded->compression - 10189 BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12 != 10190 fs_info->sectorsize_bits) 10191 return -EINVAL; 10192 compression = BTRFS_COMPRESS_LZO; 10193 break; 10194 default: 10195 return -EINVAL; 10196 } 10197 if (encoded->encryption != BTRFS_ENCODED_IO_ENCRYPTION_NONE) 10198 return -EINVAL; 10199 10200 orig_count = iov_iter_count(from); 10201 10202 /* The extent size must be sane. */ 10203 if (encoded->unencoded_len > BTRFS_MAX_UNCOMPRESSED || 10204 orig_count > BTRFS_MAX_COMPRESSED || orig_count == 0) 10205 return -EINVAL; 10206 10207 /* 10208 * The compressed data must be smaller than the decompressed data. 10209 * 10210 * It's of course possible for data to compress to larger or the same 10211 * size, but the buffered I/O path falls back to no compression for such 10212 * data, and we don't want to break any assumptions by creating these 10213 * extents. 10214 * 10215 * Note that this is less strict than the current check we have that the 10216 * compressed data must be at least one sector smaller than the 10217 * decompressed data. We only want to enforce the weaker requirement 10218 * from old kernels that it is at least one byte smaller. 10219 */ 10220 if (orig_count >= encoded->unencoded_len) 10221 return -EINVAL; 10222 10223 /* The extent must start on a sector boundary. */ 10224 start = iocb->ki_pos; 10225 if (!IS_ALIGNED(start, fs_info->sectorsize)) 10226 return -EINVAL; 10227 10228 /* 10229 * The extent must end on a sector boundary. However, we allow a write 10230 * which ends at or extends i_size to have an unaligned length; we round 10231 * up the extent size and set i_size to the unaligned end. 10232 */ 10233 if (start + encoded->len < inode->vfs_inode.i_size && 10234 !IS_ALIGNED(start + encoded->len, fs_info->sectorsize)) 10235 return -EINVAL; 10236 10237 /* Finally, the offset in the unencoded data must be sector-aligned. */ 10238 if (!IS_ALIGNED(encoded->unencoded_offset, fs_info->sectorsize)) 10239 return -EINVAL; 10240 10241 num_bytes = ALIGN(encoded->len, fs_info->sectorsize); 10242 ram_bytes = ALIGN(encoded->unencoded_len, fs_info->sectorsize); 10243 end = start + num_bytes - 1; 10244 10245 /* 10246 * If the extent cannot be inline, the compressed data on disk must be 10247 * sector-aligned. For convenience, we extend it with zeroes if it 10248 * isn't. 10249 */ 10250 disk_num_bytes = ALIGN(orig_count, fs_info->sectorsize); 10251 nr_pages = DIV_ROUND_UP(disk_num_bytes, PAGE_SIZE); 10252 pages = kvcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL_ACCOUNT); 10253 if (!pages) 10254 return -ENOMEM; 10255 for (i = 0; i < nr_pages; i++) { 10256 size_t bytes = min_t(size_t, PAGE_SIZE, iov_iter_count(from)); 10257 char *kaddr; 10258 10259 pages[i] = alloc_page(GFP_KERNEL_ACCOUNT); 10260 if (!pages[i]) { 10261 ret = -ENOMEM; 10262 goto out_pages; 10263 } 10264 kaddr = kmap_local_page(pages[i]); 10265 if (copy_from_iter(kaddr, bytes, from) != bytes) { 10266 kunmap_local(kaddr); 10267 ret = -EFAULT; 10268 goto out_pages; 10269 } 10270 if (bytes < PAGE_SIZE) 10271 memset(kaddr + bytes, 0, PAGE_SIZE - bytes); 10272 kunmap_local(kaddr); 10273 } 10274 10275 for (;;) { 10276 struct btrfs_ordered_extent *ordered; 10277 10278 ret = btrfs_wait_ordered_range(&inode->vfs_inode, start, num_bytes); 10279 if (ret) 10280 goto out_pages; 10281 ret = invalidate_inode_pages2_range(inode->vfs_inode.i_mapping, 10282 start >> PAGE_SHIFT, 10283 end >> PAGE_SHIFT); 10284 if (ret) 10285 goto out_pages; 10286 lock_extent(io_tree, start, end, &cached_state); 10287 ordered = btrfs_lookup_ordered_range(inode, start, num_bytes); 10288 if (!ordered && 10289 !filemap_range_has_page(inode->vfs_inode.i_mapping, start, end)) 10290 break; 10291 if (ordered) 10292 btrfs_put_ordered_extent(ordered); 10293 unlock_extent(io_tree, start, end, &cached_state); 10294 cond_resched(); 10295 } 10296 10297 /* 10298 * We don't use the higher-level delalloc space functions because our 10299 * num_bytes and disk_num_bytes are different. 10300 */ 10301 ret = btrfs_alloc_data_chunk_ondemand(inode, disk_num_bytes); 10302 if (ret) 10303 goto out_unlock; 10304 ret = btrfs_qgroup_reserve_data(inode, &data_reserved, start, num_bytes); 10305 if (ret) 10306 goto out_free_data_space; 10307 ret = btrfs_delalloc_reserve_metadata(inode, num_bytes, disk_num_bytes, 10308 false); 10309 if (ret) 10310 goto out_qgroup_free_data; 10311 10312 /* Try an inline extent first. */ 10313 if (start == 0 && encoded->unencoded_len == encoded->len && 10314 encoded->unencoded_offset == 0) { 10315 ret = cow_file_range_inline(inode, encoded->len, orig_count, 10316 compression, pages, true); 10317 if (ret <= 0) { 10318 if (ret == 0) 10319 ret = orig_count; 10320 goto out_delalloc_release; 10321 } 10322 } 10323 10324 ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes, 10325 disk_num_bytes, 0, 0, &ins, 1, 1); 10326 if (ret) 10327 goto out_delalloc_release; 10328 extent_reserved = true; 10329 10330 em = create_io_em(inode, start, num_bytes, 10331 start - encoded->unencoded_offset, ins.objectid, 10332 ins.offset, ins.offset, ram_bytes, compression, 10333 BTRFS_ORDERED_COMPRESSED); 10334 if (IS_ERR(em)) { 10335 ret = PTR_ERR(em); 10336 goto out_free_reserved; 10337 } 10338 free_extent_map(em); 10339 10340 ret = btrfs_add_ordered_extent(inode, start, num_bytes, ram_bytes, 10341 ins.objectid, ins.offset, 10342 encoded->unencoded_offset, 10343 (1 << BTRFS_ORDERED_ENCODED) | 10344 (1 << BTRFS_ORDERED_COMPRESSED), 10345 compression); 10346 if (ret) { 10347 btrfs_drop_extent_map_range(inode, start, end, false); 10348 goto out_free_reserved; 10349 } 10350 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10351 10352 if (start + encoded->len > inode->vfs_inode.i_size) 10353 i_size_write(&inode->vfs_inode, start + encoded->len); 10354 10355 unlock_extent(io_tree, start, end, &cached_state); 10356 10357 btrfs_delalloc_release_extents(inode, num_bytes); 10358 10359 btrfs_submit_compressed_write(inode, start, num_bytes, ins.objectid, 10360 ins.offset, pages, nr_pages, 0, false); 10361 ret = orig_count; 10362 goto out; 10363 10364 out_free_reserved: 10365 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10366 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); 10367 out_delalloc_release: 10368 btrfs_delalloc_release_extents(inode, num_bytes); 10369 btrfs_delalloc_release_metadata(inode, disk_num_bytes, ret < 0); 10370 out_qgroup_free_data: 10371 if (ret < 0) 10372 btrfs_qgroup_free_data(inode, data_reserved, start, num_bytes); 10373 out_free_data_space: 10374 /* 10375 * If btrfs_reserve_extent() succeeded, then we already decremented 10376 * bytes_may_use. 10377 */ 10378 if (!extent_reserved) 10379 btrfs_free_reserved_data_space_noquota(fs_info, disk_num_bytes); 10380 out_unlock: 10381 unlock_extent(io_tree, start, end, &cached_state); 10382 out_pages: 10383 for (i = 0; i < nr_pages; i++) { 10384 if (pages[i]) 10385 __free_page(pages[i]); 10386 } 10387 kvfree(pages); 10388 out: 10389 if (ret >= 0) 10390 iocb->ki_pos += encoded->len; 10391 return ret; 10392 } 10393 10394 #ifdef CONFIG_SWAP 10395 /* 10396 * Add an entry indicating a block group or device which is pinned by a 10397 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a 10398 * negative errno on failure. 10399 */ 10400 static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr, 10401 bool is_block_group) 10402 { 10403 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10404 struct btrfs_swapfile_pin *sp, *entry; 10405 struct rb_node **p; 10406 struct rb_node *parent = NULL; 10407 10408 sp = kmalloc(sizeof(*sp), GFP_NOFS); 10409 if (!sp) 10410 return -ENOMEM; 10411 sp->ptr = ptr; 10412 sp->inode = inode; 10413 sp->is_block_group = is_block_group; 10414 sp->bg_extent_count = 1; 10415 10416 spin_lock(&fs_info->swapfile_pins_lock); 10417 p = &fs_info->swapfile_pins.rb_node; 10418 while (*p) { 10419 parent = *p; 10420 entry = rb_entry(parent, struct btrfs_swapfile_pin, node); 10421 if (sp->ptr < entry->ptr || 10422 (sp->ptr == entry->ptr && sp->inode < entry->inode)) { 10423 p = &(*p)->rb_left; 10424 } else if (sp->ptr > entry->ptr || 10425 (sp->ptr == entry->ptr && sp->inode > entry->inode)) { 10426 p = &(*p)->rb_right; 10427 } else { 10428 if (is_block_group) 10429 entry->bg_extent_count++; 10430 spin_unlock(&fs_info->swapfile_pins_lock); 10431 kfree(sp); 10432 return 1; 10433 } 10434 } 10435 rb_link_node(&sp->node, parent, p); 10436 rb_insert_color(&sp->node, &fs_info->swapfile_pins); 10437 spin_unlock(&fs_info->swapfile_pins_lock); 10438 return 0; 10439 } 10440 10441 /* Free all of the entries pinned by this swapfile. */ 10442 static void btrfs_free_swapfile_pins(struct inode *inode) 10443 { 10444 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10445 struct btrfs_swapfile_pin *sp; 10446 struct rb_node *node, *next; 10447 10448 spin_lock(&fs_info->swapfile_pins_lock); 10449 node = rb_first(&fs_info->swapfile_pins); 10450 while (node) { 10451 next = rb_next(node); 10452 sp = rb_entry(node, struct btrfs_swapfile_pin, node); 10453 if (sp->inode == inode) { 10454 rb_erase(&sp->node, &fs_info->swapfile_pins); 10455 if (sp->is_block_group) { 10456 btrfs_dec_block_group_swap_extents(sp->ptr, 10457 sp->bg_extent_count); 10458 btrfs_put_block_group(sp->ptr); 10459 } 10460 kfree(sp); 10461 } 10462 node = next; 10463 } 10464 spin_unlock(&fs_info->swapfile_pins_lock); 10465 } 10466 10467 struct btrfs_swap_info { 10468 u64 start; 10469 u64 block_start; 10470 u64 block_len; 10471 u64 lowest_ppage; 10472 u64 highest_ppage; 10473 unsigned long nr_pages; 10474 int nr_extents; 10475 }; 10476 10477 static int btrfs_add_swap_extent(struct swap_info_struct *sis, 10478 struct btrfs_swap_info *bsi) 10479 { 10480 unsigned long nr_pages; 10481 unsigned long max_pages; 10482 u64 first_ppage, first_ppage_reported, next_ppage; 10483 int ret; 10484 10485 /* 10486 * Our swapfile may have had its size extended after the swap header was 10487 * written. In that case activating the swapfile should not go beyond 10488 * the max size set in the swap header. 10489 */ 10490 if (bsi->nr_pages >= sis->max) 10491 return 0; 10492 10493 max_pages = sis->max - bsi->nr_pages; 10494 first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT; 10495 next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT; 10496 10497 if (first_ppage >= next_ppage) 10498 return 0; 10499 nr_pages = next_ppage - first_ppage; 10500 nr_pages = min(nr_pages, max_pages); 10501 10502 first_ppage_reported = first_ppage; 10503 if (bsi->start == 0) 10504 first_ppage_reported++; 10505 if (bsi->lowest_ppage > first_ppage_reported) 10506 bsi->lowest_ppage = first_ppage_reported; 10507 if (bsi->highest_ppage < (next_ppage - 1)) 10508 bsi->highest_ppage = next_ppage - 1; 10509 10510 ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage); 10511 if (ret < 0) 10512 return ret; 10513 bsi->nr_extents += ret; 10514 bsi->nr_pages += nr_pages; 10515 return 0; 10516 } 10517 10518 static void btrfs_swap_deactivate(struct file *file) 10519 { 10520 struct inode *inode = file_inode(file); 10521 10522 btrfs_free_swapfile_pins(inode); 10523 atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles); 10524 } 10525 10526 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10527 sector_t *span) 10528 { 10529 struct inode *inode = file_inode(file); 10530 struct btrfs_root *root = BTRFS_I(inode)->root; 10531 struct btrfs_fs_info *fs_info = root->fs_info; 10532 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 10533 struct extent_state *cached_state = NULL; 10534 struct extent_map *em = NULL; 10535 struct btrfs_device *device = NULL; 10536 struct btrfs_swap_info bsi = { 10537 .lowest_ppage = (sector_t)-1ULL, 10538 }; 10539 int ret = 0; 10540 u64 isize; 10541 u64 start; 10542 10543 /* 10544 * If the swap file was just created, make sure delalloc is done. If the 10545 * file changes again after this, the user is doing something stupid and 10546 * we don't really care. 10547 */ 10548 ret = btrfs_wait_ordered_range(inode, 0, (u64)-1); 10549 if (ret) 10550 return ret; 10551 10552 /* 10553 * The inode is locked, so these flags won't change after we check them. 10554 */ 10555 if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) { 10556 btrfs_warn(fs_info, "swapfile must not be compressed"); 10557 return -EINVAL; 10558 } 10559 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) { 10560 btrfs_warn(fs_info, "swapfile must not be copy-on-write"); 10561 return -EINVAL; 10562 } 10563 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { 10564 btrfs_warn(fs_info, "swapfile must not be checksummed"); 10565 return -EINVAL; 10566 } 10567 10568 /* 10569 * Balance or device remove/replace/resize can move stuff around from 10570 * under us. The exclop protection makes sure they aren't running/won't 10571 * run concurrently while we are mapping the swap extents, and 10572 * fs_info->swapfile_pins prevents them from running while the swap 10573 * file is active and moving the extents. Note that this also prevents 10574 * a concurrent device add which isn't actually necessary, but it's not 10575 * really worth the trouble to allow it. 10576 */ 10577 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) { 10578 btrfs_warn(fs_info, 10579 "cannot activate swapfile while exclusive operation is running"); 10580 return -EBUSY; 10581 } 10582 10583 /* 10584 * Prevent snapshot creation while we are activating the swap file. 10585 * We do not want to race with snapshot creation. If snapshot creation 10586 * already started before we bumped nr_swapfiles from 0 to 1 and 10587 * completes before the first write into the swap file after it is 10588 * activated, than that write would fallback to COW. 10589 */ 10590 if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) { 10591 btrfs_exclop_finish(fs_info); 10592 btrfs_warn(fs_info, 10593 "cannot activate swapfile because snapshot creation is in progress"); 10594 return -EINVAL; 10595 } 10596 /* 10597 * Snapshots can create extents which require COW even if NODATACOW is 10598 * set. We use this counter to prevent snapshots. We must increment it 10599 * before walking the extents because we don't want a concurrent 10600 * snapshot to run after we've already checked the extents. 10601 * 10602 * It is possible that subvolume is marked for deletion but still not 10603 * removed yet. To prevent this race, we check the root status before 10604 * activating the swapfile. 10605 */ 10606 spin_lock(&root->root_item_lock); 10607 if (btrfs_root_dead(root)) { 10608 spin_unlock(&root->root_item_lock); 10609 10610 btrfs_exclop_finish(fs_info); 10611 btrfs_warn(fs_info, 10612 "cannot activate swapfile because subvolume %llu is being deleted", 10613 root->root_key.objectid); 10614 return -EPERM; 10615 } 10616 atomic_inc(&root->nr_swapfiles); 10617 spin_unlock(&root->root_item_lock); 10618 10619 isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); 10620 10621 lock_extent(io_tree, 0, isize - 1, &cached_state); 10622 start = 0; 10623 while (start < isize) { 10624 u64 logical_block_start, physical_block_start; 10625 struct btrfs_block_group *bg; 10626 u64 len = isize - start; 10627 10628 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len); 10629 if (IS_ERR(em)) { 10630 ret = PTR_ERR(em); 10631 goto out; 10632 } 10633 10634 if (em->block_start == EXTENT_MAP_HOLE) { 10635 btrfs_warn(fs_info, "swapfile must not have holes"); 10636 ret = -EINVAL; 10637 goto out; 10638 } 10639 if (em->block_start == EXTENT_MAP_INLINE) { 10640 /* 10641 * It's unlikely we'll ever actually find ourselves 10642 * here, as a file small enough to fit inline won't be 10643 * big enough to store more than the swap header, but in 10644 * case something changes in the future, let's catch it 10645 * here rather than later. 10646 */ 10647 btrfs_warn(fs_info, "swapfile must not be inline"); 10648 ret = -EINVAL; 10649 goto out; 10650 } 10651 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 10652 btrfs_warn(fs_info, "swapfile must not be compressed"); 10653 ret = -EINVAL; 10654 goto out; 10655 } 10656 10657 logical_block_start = em->block_start + (start - em->start); 10658 len = min(len, em->len - (start - em->start)); 10659 free_extent_map(em); 10660 em = NULL; 10661 10662 ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, false, true); 10663 if (ret < 0) { 10664 goto out; 10665 } else if (ret) { 10666 ret = 0; 10667 } else { 10668 btrfs_warn(fs_info, 10669 "swapfile must not be copy-on-write"); 10670 ret = -EINVAL; 10671 goto out; 10672 } 10673 10674 em = btrfs_get_chunk_map(fs_info, logical_block_start, len); 10675 if (IS_ERR(em)) { 10676 ret = PTR_ERR(em); 10677 goto out; 10678 } 10679 10680 if (em->map_lookup->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { 10681 btrfs_warn(fs_info, 10682 "swapfile must have single data profile"); 10683 ret = -EINVAL; 10684 goto out; 10685 } 10686 10687 if (device == NULL) { 10688 device = em->map_lookup->stripes[0].dev; 10689 ret = btrfs_add_swapfile_pin(inode, device, false); 10690 if (ret == 1) 10691 ret = 0; 10692 else if (ret) 10693 goto out; 10694 } else if (device != em->map_lookup->stripes[0].dev) { 10695 btrfs_warn(fs_info, "swapfile must be on one device"); 10696 ret = -EINVAL; 10697 goto out; 10698 } 10699 10700 physical_block_start = (em->map_lookup->stripes[0].physical + 10701 (logical_block_start - em->start)); 10702 len = min(len, em->len - (logical_block_start - em->start)); 10703 free_extent_map(em); 10704 em = NULL; 10705 10706 bg = btrfs_lookup_block_group(fs_info, logical_block_start); 10707 if (!bg) { 10708 btrfs_warn(fs_info, 10709 "could not find block group containing swapfile"); 10710 ret = -EINVAL; 10711 goto out; 10712 } 10713 10714 if (!btrfs_inc_block_group_swap_extents(bg)) { 10715 btrfs_warn(fs_info, 10716 "block group for swapfile at %llu is read-only%s", 10717 bg->start, 10718 atomic_read(&fs_info->scrubs_running) ? 10719 " (scrub running)" : ""); 10720 btrfs_put_block_group(bg); 10721 ret = -EINVAL; 10722 goto out; 10723 } 10724 10725 ret = btrfs_add_swapfile_pin(inode, bg, true); 10726 if (ret) { 10727 btrfs_put_block_group(bg); 10728 if (ret == 1) 10729 ret = 0; 10730 else 10731 goto out; 10732 } 10733 10734 if (bsi.block_len && 10735 bsi.block_start + bsi.block_len == physical_block_start) { 10736 bsi.block_len += len; 10737 } else { 10738 if (bsi.block_len) { 10739 ret = btrfs_add_swap_extent(sis, &bsi); 10740 if (ret) 10741 goto out; 10742 } 10743 bsi.start = start; 10744 bsi.block_start = physical_block_start; 10745 bsi.block_len = len; 10746 } 10747 10748 start += len; 10749 } 10750 10751 if (bsi.block_len) 10752 ret = btrfs_add_swap_extent(sis, &bsi); 10753 10754 out: 10755 if (!IS_ERR_OR_NULL(em)) 10756 free_extent_map(em); 10757 10758 unlock_extent(io_tree, 0, isize - 1, &cached_state); 10759 10760 if (ret) 10761 btrfs_swap_deactivate(file); 10762 10763 btrfs_drew_write_unlock(&root->snapshot_lock); 10764 10765 btrfs_exclop_finish(fs_info); 10766 10767 if (ret) 10768 return ret; 10769 10770 if (device) 10771 sis->bdev = device->bdev; 10772 *span = bsi.highest_ppage - bsi.lowest_ppage + 1; 10773 sis->max = bsi.nr_pages; 10774 sis->pages = bsi.nr_pages - 1; 10775 sis->highest_bit = bsi.nr_pages - 1; 10776 return bsi.nr_extents; 10777 } 10778 #else 10779 static void btrfs_swap_deactivate(struct file *file) 10780 { 10781 } 10782 10783 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10784 sector_t *span) 10785 { 10786 return -EOPNOTSUPP; 10787 } 10788 #endif 10789 10790 /* 10791 * Update the number of bytes used in the VFS' inode. When we replace extents in 10792 * a range (clone, dedupe, fallocate's zero range), we must update the number of 10793 * bytes used by the inode in an atomic manner, so that concurrent stat(2) calls 10794 * always get a correct value. 10795 */ 10796 void btrfs_update_inode_bytes(struct btrfs_inode *inode, 10797 const u64 add_bytes, 10798 const u64 del_bytes) 10799 { 10800 if (add_bytes == del_bytes) 10801 return; 10802 10803 spin_lock(&inode->lock); 10804 if (del_bytes > 0) 10805 inode_sub_bytes(&inode->vfs_inode, del_bytes); 10806 if (add_bytes > 0) 10807 inode_add_bytes(&inode->vfs_inode, add_bytes); 10808 spin_unlock(&inode->lock); 10809 } 10810 10811 /* 10812 * Verify that there are no ordered extents for a given file range. 10813 * 10814 * @inode: The target inode. 10815 * @start: Start offset of the file range, should be sector size aligned. 10816 * @end: End offset (inclusive) of the file range, its value +1 should be 10817 * sector size aligned. 10818 * 10819 * This should typically be used for cases where we locked an inode's VFS lock in 10820 * exclusive mode, we have also locked the inode's i_mmap_lock in exclusive mode, 10821 * we have flushed all delalloc in the range, we have waited for all ordered 10822 * extents in the range to complete and finally we have locked the file range in 10823 * the inode's io_tree. 10824 */ 10825 void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end) 10826 { 10827 struct btrfs_root *root = inode->root; 10828 struct btrfs_ordered_extent *ordered; 10829 10830 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 10831 return; 10832 10833 ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start); 10834 if (ordered) { 10835 btrfs_err(root->fs_info, 10836 "found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range [%llu, %llu])", 10837 start, end, btrfs_ino(inode), root->root_key.objectid, 10838 ordered->file_offset, 10839 ordered->file_offset + ordered->num_bytes - 1); 10840 btrfs_put_ordered_extent(ordered); 10841 } 10842 10843 ASSERT(ordered == NULL); 10844 } 10845 10846 static const struct inode_operations btrfs_dir_inode_operations = { 10847 .getattr = btrfs_getattr, 10848 .lookup = btrfs_lookup, 10849 .create = btrfs_create, 10850 .unlink = btrfs_unlink, 10851 .link = btrfs_link, 10852 .mkdir = btrfs_mkdir, 10853 .rmdir = btrfs_rmdir, 10854 .rename = btrfs_rename2, 10855 .symlink = btrfs_symlink, 10856 .setattr = btrfs_setattr, 10857 .mknod = btrfs_mknod, 10858 .listxattr = btrfs_listxattr, 10859 .permission = btrfs_permission, 10860 .get_inode_acl = btrfs_get_acl, 10861 .set_acl = btrfs_set_acl, 10862 .update_time = btrfs_update_time, 10863 .tmpfile = btrfs_tmpfile, 10864 .fileattr_get = btrfs_fileattr_get, 10865 .fileattr_set = btrfs_fileattr_set, 10866 }; 10867 10868 static const struct file_operations btrfs_dir_file_operations = { 10869 .llseek = generic_file_llseek, 10870 .read = generic_read_dir, 10871 .iterate_shared = btrfs_real_readdir, 10872 .open = btrfs_opendir, 10873 .unlocked_ioctl = btrfs_ioctl, 10874 #ifdef CONFIG_COMPAT 10875 .compat_ioctl = btrfs_compat_ioctl, 10876 #endif 10877 .release = btrfs_release_file, 10878 .fsync = btrfs_sync_file, 10879 }; 10880 10881 /* 10882 * btrfs doesn't support the bmap operation because swapfiles 10883 * use bmap to make a mapping of extents in the file. They assume 10884 * these extents won't change over the life of the file and they 10885 * use the bmap result to do IO directly to the drive. 10886 * 10887 * the btrfs bmap call would return logical addresses that aren't 10888 * suitable for IO and they also will change frequently as COW 10889 * operations happen. So, swapfile + btrfs == corruption. 10890 * 10891 * For now we're avoiding this by dropping bmap. 10892 */ 10893 static const struct address_space_operations btrfs_aops = { 10894 .read_folio = btrfs_read_folio, 10895 .writepages = btrfs_writepages, 10896 .readahead = btrfs_readahead, 10897 .direct_IO = noop_direct_IO, 10898 .invalidate_folio = btrfs_invalidate_folio, 10899 .release_folio = btrfs_release_folio, 10900 .migrate_folio = btrfs_migrate_folio, 10901 .dirty_folio = filemap_dirty_folio, 10902 .error_remove_page = generic_error_remove_page, 10903 .swap_activate = btrfs_swap_activate, 10904 .swap_deactivate = btrfs_swap_deactivate, 10905 }; 10906 10907 static const struct inode_operations btrfs_file_inode_operations = { 10908 .getattr = btrfs_getattr, 10909 .setattr = btrfs_setattr, 10910 .listxattr = btrfs_listxattr, 10911 .permission = btrfs_permission, 10912 .fiemap = btrfs_fiemap, 10913 .get_inode_acl = btrfs_get_acl, 10914 .set_acl = btrfs_set_acl, 10915 .update_time = btrfs_update_time, 10916 .fileattr_get = btrfs_fileattr_get, 10917 .fileattr_set = btrfs_fileattr_set, 10918 }; 10919 static const struct inode_operations btrfs_special_inode_operations = { 10920 .getattr = btrfs_getattr, 10921 .setattr = btrfs_setattr, 10922 .permission = btrfs_permission, 10923 .listxattr = btrfs_listxattr, 10924 .get_inode_acl = btrfs_get_acl, 10925 .set_acl = btrfs_set_acl, 10926 .update_time = btrfs_update_time, 10927 }; 10928 static const struct inode_operations btrfs_symlink_inode_operations = { 10929 .get_link = page_get_link, 10930 .getattr = btrfs_getattr, 10931 .setattr = btrfs_setattr, 10932 .permission = btrfs_permission, 10933 .listxattr = btrfs_listxattr, 10934 .update_time = btrfs_update_time, 10935 }; 10936 10937 const struct dentry_operations btrfs_dentry_operations = { 10938 .d_delete = btrfs_dentry_delete, 10939 }; 10940