1 /* 2 * Copyright (C) 2007 Oracle. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public 6 * License v2 as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public 14 * License along with this program; if not, write to the 15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 16 * Boston, MA 021110-1307, USA. 17 */ 18 19 #include <linux/kernel.h> 20 #include <linux/bio.h> 21 #include <linux/buffer_head.h> 22 #include <linux/file.h> 23 #include <linux/fs.h> 24 #include <linux/pagemap.h> 25 #include <linux/highmem.h> 26 #include <linux/time.h> 27 #include <linux/init.h> 28 #include <linux/string.h> 29 #include <linux/backing-dev.h> 30 #include <linux/mpage.h> 31 #include <linux/swap.h> 32 #include <linux/writeback.h> 33 #include <linux/statfs.h> 34 #include <linux/compat.h> 35 #include <linux/aio.h> 36 #include <linux/bit_spinlock.h> 37 #include <linux/xattr.h> 38 #include <linux/posix_acl.h> 39 #include <linux/falloc.h> 40 #include <linux/slab.h> 41 #include <linux/ratelimit.h> 42 #include <linux/mount.h> 43 #include <linux/btrfs.h> 44 #include <linux/blkdev.h> 45 #include <linux/posix_acl_xattr.h> 46 #include "ctree.h" 47 #include "disk-io.h" 48 #include "transaction.h" 49 #include "btrfs_inode.h" 50 #include "print-tree.h" 51 #include "ordered-data.h" 52 #include "xattr.h" 53 #include "tree-log.h" 54 #include "volumes.h" 55 #include "compression.h" 56 #include "locking.h" 57 #include "free-space-cache.h" 58 #include "inode-map.h" 59 #include "backref.h" 60 #include "hash.h" 61 #include "props.h" 62 63 struct btrfs_iget_args { 64 struct btrfs_key *location; 65 struct btrfs_root *root; 66 }; 67 68 static const struct inode_operations btrfs_dir_inode_operations; 69 static const struct inode_operations btrfs_symlink_inode_operations; 70 static const struct inode_operations btrfs_dir_ro_inode_operations; 71 static const struct inode_operations btrfs_special_inode_operations; 72 static const struct inode_operations btrfs_file_inode_operations; 73 static const struct address_space_operations btrfs_aops; 74 static const struct address_space_operations btrfs_symlink_aops; 75 static const struct file_operations btrfs_dir_file_operations; 76 static struct extent_io_ops btrfs_extent_io_ops; 77 78 static struct kmem_cache *btrfs_inode_cachep; 79 static struct kmem_cache *btrfs_delalloc_work_cachep; 80 struct kmem_cache *btrfs_trans_handle_cachep; 81 struct kmem_cache *btrfs_transaction_cachep; 82 struct kmem_cache *btrfs_path_cachep; 83 struct kmem_cache *btrfs_free_space_cachep; 84 85 #define S_SHIFT 12 86 static unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = { 87 [S_IFREG >> S_SHIFT] = BTRFS_FT_REG_FILE, 88 [S_IFDIR >> S_SHIFT] = BTRFS_FT_DIR, 89 [S_IFCHR >> S_SHIFT] = BTRFS_FT_CHRDEV, 90 [S_IFBLK >> S_SHIFT] = BTRFS_FT_BLKDEV, 91 [S_IFIFO >> S_SHIFT] = BTRFS_FT_FIFO, 92 [S_IFSOCK >> S_SHIFT] = BTRFS_FT_SOCK, 93 [S_IFLNK >> S_SHIFT] = BTRFS_FT_SYMLINK, 94 }; 95 96 static int btrfs_setsize(struct inode *inode, struct iattr *attr); 97 static int btrfs_truncate(struct inode *inode); 98 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); 99 static noinline int cow_file_range(struct inode *inode, 100 struct page *locked_page, 101 u64 start, u64 end, int *page_started, 102 unsigned long *nr_written, int unlock); 103 static struct extent_map *create_pinned_em(struct inode *inode, u64 start, 104 u64 len, u64 orig_start, 105 u64 block_start, u64 block_len, 106 u64 orig_block_len, u64 ram_bytes, 107 int type); 108 109 static int btrfs_dirty_inode(struct inode *inode); 110 111 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 112 struct inode *inode, struct inode *dir, 113 const struct qstr *qstr) 114 { 115 int err; 116 117 err = btrfs_init_acl(trans, inode, dir); 118 if (!err) 119 err = btrfs_xattr_security_init(trans, inode, dir, qstr); 120 return err; 121 } 122 123 /* 124 * this does all the hard work for inserting an inline extent into 125 * the btree. The caller should have done a btrfs_drop_extents so that 126 * no overlapping inline items exist in the btree 127 */ 128 static int insert_inline_extent(struct btrfs_trans_handle *trans, 129 struct btrfs_path *path, int extent_inserted, 130 struct btrfs_root *root, struct inode *inode, 131 u64 start, size_t size, size_t compressed_size, 132 int compress_type, 133 struct page **compressed_pages) 134 { 135 struct extent_buffer *leaf; 136 struct page *page = NULL; 137 char *kaddr; 138 unsigned long ptr; 139 struct btrfs_file_extent_item *ei; 140 int err = 0; 141 int ret; 142 size_t cur_size = size; 143 unsigned long offset; 144 145 if (compressed_size && compressed_pages) 146 cur_size = compressed_size; 147 148 inode_add_bytes(inode, size); 149 150 if (!extent_inserted) { 151 struct btrfs_key key; 152 size_t datasize; 153 154 key.objectid = btrfs_ino(inode); 155 key.offset = start; 156 key.type = BTRFS_EXTENT_DATA_KEY; 157 158 datasize = btrfs_file_extent_calc_inline_size(cur_size); 159 path->leave_spinning = 1; 160 ret = btrfs_insert_empty_item(trans, root, path, &key, 161 datasize); 162 if (ret) { 163 err = ret; 164 goto fail; 165 } 166 } 167 leaf = path->nodes[0]; 168 ei = btrfs_item_ptr(leaf, path->slots[0], 169 struct btrfs_file_extent_item); 170 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 171 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 172 btrfs_set_file_extent_encryption(leaf, ei, 0); 173 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 174 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 175 ptr = btrfs_file_extent_inline_start(ei); 176 177 if (compress_type != BTRFS_COMPRESS_NONE) { 178 struct page *cpage; 179 int i = 0; 180 while (compressed_size > 0) { 181 cpage = compressed_pages[i]; 182 cur_size = min_t(unsigned long, compressed_size, 183 PAGE_CACHE_SIZE); 184 185 kaddr = kmap_atomic(cpage); 186 write_extent_buffer(leaf, kaddr, ptr, cur_size); 187 kunmap_atomic(kaddr); 188 189 i++; 190 ptr += cur_size; 191 compressed_size -= cur_size; 192 } 193 btrfs_set_file_extent_compression(leaf, ei, 194 compress_type); 195 } else { 196 page = find_get_page(inode->i_mapping, 197 start >> PAGE_CACHE_SHIFT); 198 btrfs_set_file_extent_compression(leaf, ei, 0); 199 kaddr = kmap_atomic(page); 200 offset = start & (PAGE_CACHE_SIZE - 1); 201 write_extent_buffer(leaf, kaddr + offset, ptr, size); 202 kunmap_atomic(kaddr); 203 page_cache_release(page); 204 } 205 btrfs_mark_buffer_dirty(leaf); 206 btrfs_release_path(path); 207 208 /* 209 * we're an inline extent, so nobody can 210 * extend the file past i_size without locking 211 * a page we already have locked. 212 * 213 * We must do any isize and inode updates 214 * before we unlock the pages. Otherwise we 215 * could end up racing with unlink. 216 */ 217 BTRFS_I(inode)->disk_i_size = inode->i_size; 218 ret = btrfs_update_inode(trans, root, inode); 219 220 return ret; 221 fail: 222 return err; 223 } 224 225 226 /* 227 * conditionally insert an inline extent into the file. This 228 * does the checks required to make sure the data is small enough 229 * to fit as an inline extent. 230 */ 231 static noinline int cow_file_range_inline(struct btrfs_root *root, 232 struct inode *inode, u64 start, 233 u64 end, size_t compressed_size, 234 int compress_type, 235 struct page **compressed_pages) 236 { 237 struct btrfs_trans_handle *trans; 238 u64 isize = i_size_read(inode); 239 u64 actual_end = min(end + 1, isize); 240 u64 inline_len = actual_end - start; 241 u64 aligned_end = ALIGN(end, root->sectorsize); 242 u64 data_len = inline_len; 243 int ret; 244 struct btrfs_path *path; 245 int extent_inserted = 0; 246 u32 extent_item_size; 247 248 if (compressed_size) 249 data_len = compressed_size; 250 251 if (start > 0 || 252 actual_end > PAGE_CACHE_SIZE || 253 data_len > BTRFS_MAX_INLINE_DATA_SIZE(root) || 254 (!compressed_size && 255 (actual_end & (root->sectorsize - 1)) == 0) || 256 end + 1 < isize || 257 data_len > root->fs_info->max_inline) { 258 return 1; 259 } 260 261 path = btrfs_alloc_path(); 262 if (!path) 263 return -ENOMEM; 264 265 trans = btrfs_join_transaction(root); 266 if (IS_ERR(trans)) { 267 btrfs_free_path(path); 268 return PTR_ERR(trans); 269 } 270 trans->block_rsv = &root->fs_info->delalloc_block_rsv; 271 272 if (compressed_size && compressed_pages) 273 extent_item_size = btrfs_file_extent_calc_inline_size( 274 compressed_size); 275 else 276 extent_item_size = btrfs_file_extent_calc_inline_size( 277 inline_len); 278 279 ret = __btrfs_drop_extents(trans, root, inode, path, 280 start, aligned_end, NULL, 281 1, 1, extent_item_size, &extent_inserted); 282 if (ret) { 283 btrfs_abort_transaction(trans, root, ret); 284 goto out; 285 } 286 287 if (isize > actual_end) 288 inline_len = min_t(u64, isize, actual_end); 289 ret = insert_inline_extent(trans, path, extent_inserted, 290 root, inode, start, 291 inline_len, compressed_size, 292 compress_type, compressed_pages); 293 if (ret && ret != -ENOSPC) { 294 btrfs_abort_transaction(trans, root, ret); 295 goto out; 296 } else if (ret == -ENOSPC) { 297 ret = 1; 298 goto out; 299 } 300 301 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 302 btrfs_delalloc_release_metadata(inode, end + 1 - start); 303 btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0); 304 out: 305 btrfs_free_path(path); 306 btrfs_end_transaction(trans, root); 307 return ret; 308 } 309 310 struct async_extent { 311 u64 start; 312 u64 ram_size; 313 u64 compressed_size; 314 struct page **pages; 315 unsigned long nr_pages; 316 int compress_type; 317 struct list_head list; 318 }; 319 320 struct async_cow { 321 struct inode *inode; 322 struct btrfs_root *root; 323 struct page *locked_page; 324 u64 start; 325 u64 end; 326 struct list_head extents; 327 struct btrfs_work work; 328 }; 329 330 static noinline int add_async_extent(struct async_cow *cow, 331 u64 start, u64 ram_size, 332 u64 compressed_size, 333 struct page **pages, 334 unsigned long nr_pages, 335 int compress_type) 336 { 337 struct async_extent *async_extent; 338 339 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 340 BUG_ON(!async_extent); /* -ENOMEM */ 341 async_extent->start = start; 342 async_extent->ram_size = ram_size; 343 async_extent->compressed_size = compressed_size; 344 async_extent->pages = pages; 345 async_extent->nr_pages = nr_pages; 346 async_extent->compress_type = compress_type; 347 list_add_tail(&async_extent->list, &cow->extents); 348 return 0; 349 } 350 351 static inline int inode_need_compress(struct inode *inode) 352 { 353 struct btrfs_root *root = BTRFS_I(inode)->root; 354 355 /* force compress */ 356 if (btrfs_test_opt(root, FORCE_COMPRESS)) 357 return 1; 358 /* bad compression ratios */ 359 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS) 360 return 0; 361 if (btrfs_test_opt(root, COMPRESS) || 362 BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS || 363 BTRFS_I(inode)->force_compress) 364 return 1; 365 return 0; 366 } 367 368 /* 369 * we create compressed extents in two phases. The first 370 * phase compresses a range of pages that have already been 371 * locked (both pages and state bits are locked). 372 * 373 * This is done inside an ordered work queue, and the compression 374 * is spread across many cpus. The actual IO submission is step 375 * two, and the ordered work queue takes care of making sure that 376 * happens in the same order things were put onto the queue by 377 * writepages and friends. 378 * 379 * If this code finds it can't get good compression, it puts an 380 * entry onto the work queue to write the uncompressed bytes. This 381 * makes sure that both compressed inodes and uncompressed inodes 382 * are written in the same order that the flusher thread sent them 383 * down. 384 */ 385 static noinline void compress_file_range(struct inode *inode, 386 struct page *locked_page, 387 u64 start, u64 end, 388 struct async_cow *async_cow, 389 int *num_added) 390 { 391 struct btrfs_root *root = BTRFS_I(inode)->root; 392 u64 num_bytes; 393 u64 blocksize = root->sectorsize; 394 u64 actual_end; 395 u64 isize = i_size_read(inode); 396 int ret = 0; 397 struct page **pages = NULL; 398 unsigned long nr_pages; 399 unsigned long nr_pages_ret = 0; 400 unsigned long total_compressed = 0; 401 unsigned long total_in = 0; 402 unsigned long max_compressed = 128 * 1024; 403 unsigned long max_uncompressed = 128 * 1024; 404 int i; 405 int will_compress; 406 int compress_type = root->fs_info->compress_type; 407 int redirty = 0; 408 409 /* if this is a small write inside eof, kick off a defrag */ 410 if ((end - start + 1) < 16 * 1024 && 411 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 412 btrfs_add_inode_defrag(NULL, inode); 413 414 actual_end = min_t(u64, isize, end + 1); 415 again: 416 will_compress = 0; 417 nr_pages = (end >> PAGE_CACHE_SHIFT) - (start >> PAGE_CACHE_SHIFT) + 1; 418 nr_pages = min(nr_pages, (128 * 1024UL) / PAGE_CACHE_SIZE); 419 420 /* 421 * we don't want to send crud past the end of i_size through 422 * compression, that's just a waste of CPU time. So, if the 423 * end of the file is before the start of our current 424 * requested range of bytes, we bail out to the uncompressed 425 * cleanup code that can deal with all of this. 426 * 427 * It isn't really the fastest way to fix things, but this is a 428 * very uncommon corner. 429 */ 430 if (actual_end <= start) 431 goto cleanup_and_bail_uncompressed; 432 433 total_compressed = actual_end - start; 434 435 /* 436 * skip compression for a small file range(<=blocksize) that 437 * isn't an inline extent, since it dosen't save disk space at all. 438 */ 439 if (total_compressed <= blocksize && 440 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 441 goto cleanup_and_bail_uncompressed; 442 443 /* we want to make sure that amount of ram required to uncompress 444 * an extent is reasonable, so we limit the total size in ram 445 * of a compressed extent to 128k. This is a crucial number 446 * because it also controls how easily we can spread reads across 447 * cpus for decompression. 448 * 449 * We also want to make sure the amount of IO required to do 450 * a random read is reasonably small, so we limit the size of 451 * a compressed extent to 128k. 452 */ 453 total_compressed = min(total_compressed, max_uncompressed); 454 num_bytes = ALIGN(end - start + 1, blocksize); 455 num_bytes = max(blocksize, num_bytes); 456 total_in = 0; 457 ret = 0; 458 459 /* 460 * we do compression for mount -o compress and when the 461 * inode has not been flagged as nocompress. This flag can 462 * change at any time if we discover bad compression ratios. 463 */ 464 if (inode_need_compress(inode)) { 465 WARN_ON(pages); 466 pages = kzalloc(sizeof(struct page *) * nr_pages, GFP_NOFS); 467 if (!pages) { 468 /* just bail out to the uncompressed code */ 469 goto cont; 470 } 471 472 if (BTRFS_I(inode)->force_compress) 473 compress_type = BTRFS_I(inode)->force_compress; 474 475 /* 476 * we need to call clear_page_dirty_for_io on each 477 * page in the range. Otherwise applications with the file 478 * mmap'd can wander in and change the page contents while 479 * we are compressing them. 480 * 481 * If the compression fails for any reason, we set the pages 482 * dirty again later on. 483 */ 484 extent_range_clear_dirty_for_io(inode, start, end); 485 redirty = 1; 486 ret = btrfs_compress_pages(compress_type, 487 inode->i_mapping, start, 488 total_compressed, pages, 489 nr_pages, &nr_pages_ret, 490 &total_in, 491 &total_compressed, 492 max_compressed); 493 494 if (!ret) { 495 unsigned long offset = total_compressed & 496 (PAGE_CACHE_SIZE - 1); 497 struct page *page = pages[nr_pages_ret - 1]; 498 char *kaddr; 499 500 /* zero the tail end of the last page, we might be 501 * sending it down to disk 502 */ 503 if (offset) { 504 kaddr = kmap_atomic(page); 505 memset(kaddr + offset, 0, 506 PAGE_CACHE_SIZE - offset); 507 kunmap_atomic(kaddr); 508 } 509 will_compress = 1; 510 } 511 } 512 cont: 513 if (start == 0) { 514 /* lets try to make an inline extent */ 515 if (ret || total_in < (actual_end - start)) { 516 /* we didn't compress the entire range, try 517 * to make an uncompressed inline extent. 518 */ 519 ret = cow_file_range_inline(root, inode, start, end, 520 0, 0, NULL); 521 } else { 522 /* try making a compressed inline extent */ 523 ret = cow_file_range_inline(root, inode, start, end, 524 total_compressed, 525 compress_type, pages); 526 } 527 if (ret <= 0) { 528 unsigned long clear_flags = EXTENT_DELALLOC | 529 EXTENT_DEFRAG; 530 unsigned long page_error_op; 531 532 clear_flags |= (ret < 0) ? EXTENT_DO_ACCOUNTING : 0; 533 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0; 534 535 /* 536 * inline extent creation worked or returned error, 537 * we don't need to create any more async work items. 538 * Unlock and free up our temp pages. 539 */ 540 extent_clear_unlock_delalloc(inode, start, end, NULL, 541 clear_flags, PAGE_UNLOCK | 542 PAGE_CLEAR_DIRTY | 543 PAGE_SET_WRITEBACK | 544 page_error_op | 545 PAGE_END_WRITEBACK); 546 goto free_pages_out; 547 } 548 } 549 550 if (will_compress) { 551 /* 552 * we aren't doing an inline extent round the compressed size 553 * up to a block size boundary so the allocator does sane 554 * things 555 */ 556 total_compressed = ALIGN(total_compressed, blocksize); 557 558 /* 559 * one last check to make sure the compression is really a 560 * win, compare the page count read with the blocks on disk 561 */ 562 total_in = ALIGN(total_in, PAGE_CACHE_SIZE); 563 if (total_compressed >= total_in) { 564 will_compress = 0; 565 } else { 566 num_bytes = total_in; 567 } 568 } 569 if (!will_compress && pages) { 570 /* 571 * the compression code ran but failed to make things smaller, 572 * free any pages it allocated and our page pointer array 573 */ 574 for (i = 0; i < nr_pages_ret; i++) { 575 WARN_ON(pages[i]->mapping); 576 page_cache_release(pages[i]); 577 } 578 kfree(pages); 579 pages = NULL; 580 total_compressed = 0; 581 nr_pages_ret = 0; 582 583 /* flag the file so we don't compress in the future */ 584 if (!btrfs_test_opt(root, FORCE_COMPRESS) && 585 !(BTRFS_I(inode)->force_compress)) { 586 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; 587 } 588 } 589 if (will_compress) { 590 *num_added += 1; 591 592 /* the async work queues will take care of doing actual 593 * allocation on disk for these compressed pages, 594 * and will submit them to the elevator. 595 */ 596 add_async_extent(async_cow, start, num_bytes, 597 total_compressed, pages, nr_pages_ret, 598 compress_type); 599 600 if (start + num_bytes < end) { 601 start += num_bytes; 602 pages = NULL; 603 cond_resched(); 604 goto again; 605 } 606 } else { 607 cleanup_and_bail_uncompressed: 608 /* 609 * No compression, but we still need to write the pages in 610 * the file we've been given so far. redirty the locked 611 * page if it corresponds to our extent and set things up 612 * for the async work queue to run cow_file_range to do 613 * the normal delalloc dance 614 */ 615 if (page_offset(locked_page) >= start && 616 page_offset(locked_page) <= end) { 617 __set_page_dirty_nobuffers(locked_page); 618 /* unlocked later on in the async handlers */ 619 } 620 if (redirty) 621 extent_range_redirty_for_io(inode, start, end); 622 add_async_extent(async_cow, start, end - start + 1, 623 0, NULL, 0, BTRFS_COMPRESS_NONE); 624 *num_added += 1; 625 } 626 627 return; 628 629 free_pages_out: 630 for (i = 0; i < nr_pages_ret; i++) { 631 WARN_ON(pages[i]->mapping); 632 page_cache_release(pages[i]); 633 } 634 kfree(pages); 635 } 636 637 static void free_async_extent_pages(struct async_extent *async_extent) 638 { 639 int i; 640 641 if (!async_extent->pages) 642 return; 643 644 for (i = 0; i < async_extent->nr_pages; i++) { 645 WARN_ON(async_extent->pages[i]->mapping); 646 page_cache_release(async_extent->pages[i]); 647 } 648 kfree(async_extent->pages); 649 async_extent->nr_pages = 0; 650 async_extent->pages = NULL; 651 } 652 653 /* 654 * phase two of compressed writeback. This is the ordered portion 655 * of the code, which only gets called in the order the work was 656 * queued. We walk all the async extents created by compress_file_range 657 * and send them down to the disk. 658 */ 659 static noinline void submit_compressed_extents(struct inode *inode, 660 struct async_cow *async_cow) 661 { 662 struct async_extent *async_extent; 663 u64 alloc_hint = 0; 664 struct btrfs_key ins; 665 struct extent_map *em; 666 struct btrfs_root *root = BTRFS_I(inode)->root; 667 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 668 struct extent_io_tree *io_tree; 669 int ret = 0; 670 671 again: 672 while (!list_empty(&async_cow->extents)) { 673 async_extent = list_entry(async_cow->extents.next, 674 struct async_extent, list); 675 list_del(&async_extent->list); 676 677 io_tree = &BTRFS_I(inode)->io_tree; 678 679 retry: 680 /* did the compression code fall back to uncompressed IO? */ 681 if (!async_extent->pages) { 682 int page_started = 0; 683 unsigned long nr_written = 0; 684 685 lock_extent(io_tree, async_extent->start, 686 async_extent->start + 687 async_extent->ram_size - 1); 688 689 /* allocate blocks */ 690 ret = cow_file_range(inode, async_cow->locked_page, 691 async_extent->start, 692 async_extent->start + 693 async_extent->ram_size - 1, 694 &page_started, &nr_written, 0); 695 696 /* JDM XXX */ 697 698 /* 699 * if page_started, cow_file_range inserted an 700 * inline extent and took care of all the unlocking 701 * and IO for us. Otherwise, we need to submit 702 * all those pages down to the drive. 703 */ 704 if (!page_started && !ret) 705 extent_write_locked_range(io_tree, 706 inode, async_extent->start, 707 async_extent->start + 708 async_extent->ram_size - 1, 709 btrfs_get_extent, 710 WB_SYNC_ALL); 711 else if (ret) 712 unlock_page(async_cow->locked_page); 713 kfree(async_extent); 714 cond_resched(); 715 continue; 716 } 717 718 lock_extent(io_tree, async_extent->start, 719 async_extent->start + async_extent->ram_size - 1); 720 721 ret = btrfs_reserve_extent(root, 722 async_extent->compressed_size, 723 async_extent->compressed_size, 724 0, alloc_hint, &ins, 1, 1); 725 if (ret) { 726 free_async_extent_pages(async_extent); 727 728 if (ret == -ENOSPC) { 729 unlock_extent(io_tree, async_extent->start, 730 async_extent->start + 731 async_extent->ram_size - 1); 732 733 /* 734 * we need to redirty the pages if we decide to 735 * fallback to uncompressed IO, otherwise we 736 * will not submit these pages down to lower 737 * layers. 738 */ 739 extent_range_redirty_for_io(inode, 740 async_extent->start, 741 async_extent->start + 742 async_extent->ram_size - 1); 743 744 goto retry; 745 } 746 goto out_free; 747 } 748 749 /* 750 * here we're doing allocation and writeback of the 751 * compressed pages 752 */ 753 btrfs_drop_extent_cache(inode, async_extent->start, 754 async_extent->start + 755 async_extent->ram_size - 1, 0); 756 757 em = alloc_extent_map(); 758 if (!em) { 759 ret = -ENOMEM; 760 goto out_free_reserve; 761 } 762 em->start = async_extent->start; 763 em->len = async_extent->ram_size; 764 em->orig_start = em->start; 765 em->mod_start = em->start; 766 em->mod_len = em->len; 767 768 em->block_start = ins.objectid; 769 em->block_len = ins.offset; 770 em->orig_block_len = ins.offset; 771 em->ram_bytes = async_extent->ram_size; 772 em->bdev = root->fs_info->fs_devices->latest_bdev; 773 em->compress_type = async_extent->compress_type; 774 set_bit(EXTENT_FLAG_PINNED, &em->flags); 775 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 776 em->generation = -1; 777 778 while (1) { 779 write_lock(&em_tree->lock); 780 ret = add_extent_mapping(em_tree, em, 1); 781 write_unlock(&em_tree->lock); 782 if (ret != -EEXIST) { 783 free_extent_map(em); 784 break; 785 } 786 btrfs_drop_extent_cache(inode, async_extent->start, 787 async_extent->start + 788 async_extent->ram_size - 1, 0); 789 } 790 791 if (ret) 792 goto out_free_reserve; 793 794 ret = btrfs_add_ordered_extent_compress(inode, 795 async_extent->start, 796 ins.objectid, 797 async_extent->ram_size, 798 ins.offset, 799 BTRFS_ORDERED_COMPRESSED, 800 async_extent->compress_type); 801 if (ret) { 802 btrfs_drop_extent_cache(inode, async_extent->start, 803 async_extent->start + 804 async_extent->ram_size - 1, 0); 805 goto out_free_reserve; 806 } 807 808 /* 809 * clear dirty, set writeback and unlock the pages. 810 */ 811 extent_clear_unlock_delalloc(inode, async_extent->start, 812 async_extent->start + 813 async_extent->ram_size - 1, 814 NULL, EXTENT_LOCKED | EXTENT_DELALLOC, 815 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 816 PAGE_SET_WRITEBACK); 817 ret = btrfs_submit_compressed_write(inode, 818 async_extent->start, 819 async_extent->ram_size, 820 ins.objectid, 821 ins.offset, async_extent->pages, 822 async_extent->nr_pages); 823 if (ret) { 824 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 825 struct page *p = async_extent->pages[0]; 826 const u64 start = async_extent->start; 827 const u64 end = start + async_extent->ram_size - 1; 828 829 p->mapping = inode->i_mapping; 830 tree->ops->writepage_end_io_hook(p, start, end, 831 NULL, 0); 832 p->mapping = NULL; 833 extent_clear_unlock_delalloc(inode, start, end, NULL, 0, 834 PAGE_END_WRITEBACK | 835 PAGE_SET_ERROR); 836 free_async_extent_pages(async_extent); 837 } 838 alloc_hint = ins.objectid + ins.offset; 839 kfree(async_extent); 840 cond_resched(); 841 } 842 return; 843 out_free_reserve: 844 btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1); 845 out_free: 846 extent_clear_unlock_delalloc(inode, async_extent->start, 847 async_extent->start + 848 async_extent->ram_size - 1, 849 NULL, EXTENT_LOCKED | EXTENT_DELALLOC | 850 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 851 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 852 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | 853 PAGE_SET_ERROR); 854 free_async_extent_pages(async_extent); 855 kfree(async_extent); 856 goto again; 857 } 858 859 static u64 get_extent_allocation_hint(struct inode *inode, u64 start, 860 u64 num_bytes) 861 { 862 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 863 struct extent_map *em; 864 u64 alloc_hint = 0; 865 866 read_lock(&em_tree->lock); 867 em = search_extent_mapping(em_tree, start, num_bytes); 868 if (em) { 869 /* 870 * if block start isn't an actual block number then find the 871 * first block in this inode and use that as a hint. If that 872 * block is also bogus then just don't worry about it. 873 */ 874 if (em->block_start >= EXTENT_MAP_LAST_BYTE) { 875 free_extent_map(em); 876 em = search_extent_mapping(em_tree, 0, 0); 877 if (em && em->block_start < EXTENT_MAP_LAST_BYTE) 878 alloc_hint = em->block_start; 879 if (em) 880 free_extent_map(em); 881 } else { 882 alloc_hint = em->block_start; 883 free_extent_map(em); 884 } 885 } 886 read_unlock(&em_tree->lock); 887 888 return alloc_hint; 889 } 890 891 /* 892 * when extent_io.c finds a delayed allocation range in the file, 893 * the call backs end up in this code. The basic idea is to 894 * allocate extents on disk for the range, and create ordered data structs 895 * in ram to track those extents. 896 * 897 * locked_page is the page that writepage had locked already. We use 898 * it to make sure we don't do extra locks or unlocks. 899 * 900 * *page_started is set to one if we unlock locked_page and do everything 901 * required to start IO on it. It may be clean and already done with 902 * IO when we return. 903 */ 904 static noinline int cow_file_range(struct inode *inode, 905 struct page *locked_page, 906 u64 start, u64 end, int *page_started, 907 unsigned long *nr_written, 908 int unlock) 909 { 910 struct btrfs_root *root = BTRFS_I(inode)->root; 911 u64 alloc_hint = 0; 912 u64 num_bytes; 913 unsigned long ram_size; 914 u64 disk_num_bytes; 915 u64 cur_alloc_size; 916 u64 blocksize = root->sectorsize; 917 struct btrfs_key ins; 918 struct extent_map *em; 919 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 920 int ret = 0; 921 922 if (btrfs_is_free_space_inode(inode)) { 923 WARN_ON_ONCE(1); 924 ret = -EINVAL; 925 goto out_unlock; 926 } 927 928 num_bytes = ALIGN(end - start + 1, blocksize); 929 num_bytes = max(blocksize, num_bytes); 930 disk_num_bytes = num_bytes; 931 932 /* if this is a small write inside eof, kick off defrag */ 933 if (num_bytes < 64 * 1024 && 934 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size)) 935 btrfs_add_inode_defrag(NULL, inode); 936 937 if (start == 0) { 938 /* lets try to make an inline extent */ 939 ret = cow_file_range_inline(root, inode, start, end, 0, 0, 940 NULL); 941 if (ret == 0) { 942 extent_clear_unlock_delalloc(inode, start, end, NULL, 943 EXTENT_LOCKED | EXTENT_DELALLOC | 944 EXTENT_DEFRAG, PAGE_UNLOCK | 945 PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | 946 PAGE_END_WRITEBACK); 947 948 *nr_written = *nr_written + 949 (end - start + PAGE_CACHE_SIZE) / PAGE_CACHE_SIZE; 950 *page_started = 1; 951 goto out; 952 } else if (ret < 0) { 953 goto out_unlock; 954 } 955 } 956 957 BUG_ON(disk_num_bytes > 958 btrfs_super_total_bytes(root->fs_info->super_copy)); 959 960 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); 961 btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0); 962 963 while (disk_num_bytes > 0) { 964 unsigned long op; 965 966 cur_alloc_size = disk_num_bytes; 967 ret = btrfs_reserve_extent(root, cur_alloc_size, 968 root->sectorsize, 0, alloc_hint, 969 &ins, 1, 1); 970 if (ret < 0) 971 goto out_unlock; 972 973 em = alloc_extent_map(); 974 if (!em) { 975 ret = -ENOMEM; 976 goto out_reserve; 977 } 978 em->start = start; 979 em->orig_start = em->start; 980 ram_size = ins.offset; 981 em->len = ins.offset; 982 em->mod_start = em->start; 983 em->mod_len = em->len; 984 985 em->block_start = ins.objectid; 986 em->block_len = ins.offset; 987 em->orig_block_len = ins.offset; 988 em->ram_bytes = ram_size; 989 em->bdev = root->fs_info->fs_devices->latest_bdev; 990 set_bit(EXTENT_FLAG_PINNED, &em->flags); 991 em->generation = -1; 992 993 while (1) { 994 write_lock(&em_tree->lock); 995 ret = add_extent_mapping(em_tree, em, 1); 996 write_unlock(&em_tree->lock); 997 if (ret != -EEXIST) { 998 free_extent_map(em); 999 break; 1000 } 1001 btrfs_drop_extent_cache(inode, start, 1002 start + ram_size - 1, 0); 1003 } 1004 if (ret) 1005 goto out_reserve; 1006 1007 cur_alloc_size = ins.offset; 1008 ret = btrfs_add_ordered_extent(inode, start, ins.objectid, 1009 ram_size, cur_alloc_size, 0); 1010 if (ret) 1011 goto out_drop_extent_cache; 1012 1013 if (root->root_key.objectid == 1014 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1015 ret = btrfs_reloc_clone_csums(inode, start, 1016 cur_alloc_size); 1017 if (ret) 1018 goto out_drop_extent_cache; 1019 } 1020 1021 if (disk_num_bytes < cur_alloc_size) 1022 break; 1023 1024 /* we're not doing compressed IO, don't unlock the first 1025 * page (which the caller expects to stay locked), don't 1026 * clear any dirty bits and don't set any writeback bits 1027 * 1028 * Do set the Private2 bit so we know this page was properly 1029 * setup for writepage 1030 */ 1031 op = unlock ? PAGE_UNLOCK : 0; 1032 op |= PAGE_SET_PRIVATE2; 1033 1034 extent_clear_unlock_delalloc(inode, start, 1035 start + ram_size - 1, locked_page, 1036 EXTENT_LOCKED | EXTENT_DELALLOC, 1037 op); 1038 disk_num_bytes -= cur_alloc_size; 1039 num_bytes -= cur_alloc_size; 1040 alloc_hint = ins.objectid + ins.offset; 1041 start += cur_alloc_size; 1042 } 1043 out: 1044 return ret; 1045 1046 out_drop_extent_cache: 1047 btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0); 1048 out_reserve: 1049 btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1); 1050 out_unlock: 1051 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1052 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 1053 EXTENT_DELALLOC | EXTENT_DEFRAG, 1054 PAGE_UNLOCK | PAGE_CLEAR_DIRTY | 1055 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK); 1056 goto out; 1057 } 1058 1059 /* 1060 * work queue call back to started compression on a file and pages 1061 */ 1062 static noinline void async_cow_start(struct btrfs_work *work) 1063 { 1064 struct async_cow *async_cow; 1065 int num_added = 0; 1066 async_cow = container_of(work, struct async_cow, work); 1067 1068 compress_file_range(async_cow->inode, async_cow->locked_page, 1069 async_cow->start, async_cow->end, async_cow, 1070 &num_added); 1071 if (num_added == 0) { 1072 btrfs_add_delayed_iput(async_cow->inode); 1073 async_cow->inode = NULL; 1074 } 1075 } 1076 1077 /* 1078 * work queue call back to submit previously compressed pages 1079 */ 1080 static noinline void async_cow_submit(struct btrfs_work *work) 1081 { 1082 struct async_cow *async_cow; 1083 struct btrfs_root *root; 1084 unsigned long nr_pages; 1085 1086 async_cow = container_of(work, struct async_cow, work); 1087 1088 root = async_cow->root; 1089 nr_pages = (async_cow->end - async_cow->start + PAGE_CACHE_SIZE) >> 1090 PAGE_CACHE_SHIFT; 1091 1092 if (atomic_sub_return(nr_pages, &root->fs_info->async_delalloc_pages) < 1093 5 * 1024 * 1024 && 1094 waitqueue_active(&root->fs_info->async_submit_wait)) 1095 wake_up(&root->fs_info->async_submit_wait); 1096 1097 if (async_cow->inode) 1098 submit_compressed_extents(async_cow->inode, async_cow); 1099 } 1100 1101 static noinline void async_cow_free(struct btrfs_work *work) 1102 { 1103 struct async_cow *async_cow; 1104 async_cow = container_of(work, struct async_cow, work); 1105 if (async_cow->inode) 1106 btrfs_add_delayed_iput(async_cow->inode); 1107 kfree(async_cow); 1108 } 1109 1110 static int cow_file_range_async(struct inode *inode, struct page *locked_page, 1111 u64 start, u64 end, int *page_started, 1112 unsigned long *nr_written) 1113 { 1114 struct async_cow *async_cow; 1115 struct btrfs_root *root = BTRFS_I(inode)->root; 1116 unsigned long nr_pages; 1117 u64 cur_end; 1118 int limit = 10 * 1024 * 1024; 1119 1120 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED, 1121 1, 0, NULL, GFP_NOFS); 1122 while (start < end) { 1123 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS); 1124 BUG_ON(!async_cow); /* -ENOMEM */ 1125 async_cow->inode = igrab(inode); 1126 async_cow->root = root; 1127 async_cow->locked_page = locked_page; 1128 async_cow->start = start; 1129 1130 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS && 1131 !btrfs_test_opt(root, FORCE_COMPRESS)) 1132 cur_end = end; 1133 else 1134 cur_end = min(end, start + 512 * 1024 - 1); 1135 1136 async_cow->end = cur_end; 1137 INIT_LIST_HEAD(&async_cow->extents); 1138 1139 btrfs_init_work(&async_cow->work, 1140 btrfs_delalloc_helper, 1141 async_cow_start, async_cow_submit, 1142 async_cow_free); 1143 1144 nr_pages = (cur_end - start + PAGE_CACHE_SIZE) >> 1145 PAGE_CACHE_SHIFT; 1146 atomic_add(nr_pages, &root->fs_info->async_delalloc_pages); 1147 1148 btrfs_queue_work(root->fs_info->delalloc_workers, 1149 &async_cow->work); 1150 1151 if (atomic_read(&root->fs_info->async_delalloc_pages) > limit) { 1152 wait_event(root->fs_info->async_submit_wait, 1153 (atomic_read(&root->fs_info->async_delalloc_pages) < 1154 limit)); 1155 } 1156 1157 while (atomic_read(&root->fs_info->async_submit_draining) && 1158 atomic_read(&root->fs_info->async_delalloc_pages)) { 1159 wait_event(root->fs_info->async_submit_wait, 1160 (atomic_read(&root->fs_info->async_delalloc_pages) == 1161 0)); 1162 } 1163 1164 *nr_written += nr_pages; 1165 start = cur_end + 1; 1166 } 1167 *page_started = 1; 1168 return 0; 1169 } 1170 1171 static noinline int csum_exist_in_range(struct btrfs_root *root, 1172 u64 bytenr, u64 num_bytes) 1173 { 1174 int ret; 1175 struct btrfs_ordered_sum *sums; 1176 LIST_HEAD(list); 1177 1178 ret = btrfs_lookup_csums_range(root->fs_info->csum_root, bytenr, 1179 bytenr + num_bytes - 1, &list, 0); 1180 if (ret == 0 && list_empty(&list)) 1181 return 0; 1182 1183 while (!list_empty(&list)) { 1184 sums = list_entry(list.next, struct btrfs_ordered_sum, list); 1185 list_del(&sums->list); 1186 kfree(sums); 1187 } 1188 return 1; 1189 } 1190 1191 /* 1192 * when nowcow writeback call back. This checks for snapshots or COW copies 1193 * of the extents that exist in the file, and COWs the file as required. 1194 * 1195 * If no cow copies or snapshots exist, we write directly to the existing 1196 * blocks on disk 1197 */ 1198 static noinline int run_delalloc_nocow(struct inode *inode, 1199 struct page *locked_page, 1200 u64 start, u64 end, int *page_started, int force, 1201 unsigned long *nr_written) 1202 { 1203 struct btrfs_root *root = BTRFS_I(inode)->root; 1204 struct btrfs_trans_handle *trans; 1205 struct extent_buffer *leaf; 1206 struct btrfs_path *path; 1207 struct btrfs_file_extent_item *fi; 1208 struct btrfs_key found_key; 1209 u64 cow_start; 1210 u64 cur_offset; 1211 u64 extent_end; 1212 u64 extent_offset; 1213 u64 disk_bytenr; 1214 u64 num_bytes; 1215 u64 disk_num_bytes; 1216 u64 ram_bytes; 1217 int extent_type; 1218 int ret, err; 1219 int type; 1220 int nocow; 1221 int check_prev = 1; 1222 bool nolock; 1223 u64 ino = btrfs_ino(inode); 1224 1225 path = btrfs_alloc_path(); 1226 if (!path) { 1227 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1228 EXTENT_LOCKED | EXTENT_DELALLOC | 1229 EXTENT_DO_ACCOUNTING | 1230 EXTENT_DEFRAG, PAGE_UNLOCK | 1231 PAGE_CLEAR_DIRTY | 1232 PAGE_SET_WRITEBACK | 1233 PAGE_END_WRITEBACK); 1234 return -ENOMEM; 1235 } 1236 1237 nolock = btrfs_is_free_space_inode(inode); 1238 1239 if (nolock) 1240 trans = btrfs_join_transaction_nolock(root); 1241 else 1242 trans = btrfs_join_transaction(root); 1243 1244 if (IS_ERR(trans)) { 1245 extent_clear_unlock_delalloc(inode, start, end, locked_page, 1246 EXTENT_LOCKED | EXTENT_DELALLOC | 1247 EXTENT_DO_ACCOUNTING | 1248 EXTENT_DEFRAG, PAGE_UNLOCK | 1249 PAGE_CLEAR_DIRTY | 1250 PAGE_SET_WRITEBACK | 1251 PAGE_END_WRITEBACK); 1252 btrfs_free_path(path); 1253 return PTR_ERR(trans); 1254 } 1255 1256 trans->block_rsv = &root->fs_info->delalloc_block_rsv; 1257 1258 cow_start = (u64)-1; 1259 cur_offset = start; 1260 while (1) { 1261 ret = btrfs_lookup_file_extent(trans, root, path, ino, 1262 cur_offset, 0); 1263 if (ret < 0) 1264 goto error; 1265 if (ret > 0 && path->slots[0] > 0 && check_prev) { 1266 leaf = path->nodes[0]; 1267 btrfs_item_key_to_cpu(leaf, &found_key, 1268 path->slots[0] - 1); 1269 if (found_key.objectid == ino && 1270 found_key.type == BTRFS_EXTENT_DATA_KEY) 1271 path->slots[0]--; 1272 } 1273 check_prev = 0; 1274 next_slot: 1275 leaf = path->nodes[0]; 1276 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 1277 ret = btrfs_next_leaf(root, path); 1278 if (ret < 0) 1279 goto error; 1280 if (ret > 0) 1281 break; 1282 leaf = path->nodes[0]; 1283 } 1284 1285 nocow = 0; 1286 disk_bytenr = 0; 1287 num_bytes = 0; 1288 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 1289 1290 if (found_key.objectid > ino || 1291 found_key.type > BTRFS_EXTENT_DATA_KEY || 1292 found_key.offset > end) 1293 break; 1294 1295 if (found_key.offset > cur_offset) { 1296 extent_end = found_key.offset; 1297 extent_type = 0; 1298 goto out_check; 1299 } 1300 1301 fi = btrfs_item_ptr(leaf, path->slots[0], 1302 struct btrfs_file_extent_item); 1303 extent_type = btrfs_file_extent_type(leaf, fi); 1304 1305 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1306 if (extent_type == BTRFS_FILE_EXTENT_REG || 1307 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1308 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1309 extent_offset = btrfs_file_extent_offset(leaf, fi); 1310 extent_end = found_key.offset + 1311 btrfs_file_extent_num_bytes(leaf, fi); 1312 disk_num_bytes = 1313 btrfs_file_extent_disk_num_bytes(leaf, fi); 1314 if (extent_end <= start) { 1315 path->slots[0]++; 1316 goto next_slot; 1317 } 1318 if (disk_bytenr == 0) 1319 goto out_check; 1320 if (btrfs_file_extent_compression(leaf, fi) || 1321 btrfs_file_extent_encryption(leaf, fi) || 1322 btrfs_file_extent_other_encoding(leaf, fi)) 1323 goto out_check; 1324 if (extent_type == BTRFS_FILE_EXTENT_REG && !force) 1325 goto out_check; 1326 if (btrfs_extent_readonly(root, disk_bytenr)) 1327 goto out_check; 1328 if (btrfs_cross_ref_exist(trans, root, ino, 1329 found_key.offset - 1330 extent_offset, disk_bytenr)) 1331 goto out_check; 1332 disk_bytenr += extent_offset; 1333 disk_bytenr += cur_offset - found_key.offset; 1334 num_bytes = min(end + 1, extent_end) - cur_offset; 1335 /* 1336 * if there are pending snapshots for this root, 1337 * we fall into common COW way. 1338 */ 1339 if (!nolock) { 1340 err = btrfs_start_write_no_snapshoting(root); 1341 if (!err) 1342 goto out_check; 1343 } 1344 /* 1345 * force cow if csum exists in the range. 1346 * this ensure that csum for a given extent are 1347 * either valid or do not exist. 1348 */ 1349 if (csum_exist_in_range(root, disk_bytenr, num_bytes)) 1350 goto out_check; 1351 nocow = 1; 1352 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 1353 extent_end = found_key.offset + 1354 btrfs_file_extent_inline_len(leaf, 1355 path->slots[0], fi); 1356 extent_end = ALIGN(extent_end, root->sectorsize); 1357 } else { 1358 BUG_ON(1); 1359 } 1360 out_check: 1361 if (extent_end <= start) { 1362 path->slots[0]++; 1363 if (!nolock && nocow) 1364 btrfs_end_write_no_snapshoting(root); 1365 goto next_slot; 1366 } 1367 if (!nocow) { 1368 if (cow_start == (u64)-1) 1369 cow_start = cur_offset; 1370 cur_offset = extent_end; 1371 if (cur_offset > end) 1372 break; 1373 path->slots[0]++; 1374 goto next_slot; 1375 } 1376 1377 btrfs_release_path(path); 1378 if (cow_start != (u64)-1) { 1379 ret = cow_file_range(inode, locked_page, 1380 cow_start, found_key.offset - 1, 1381 page_started, nr_written, 1); 1382 if (ret) { 1383 if (!nolock && nocow) 1384 btrfs_end_write_no_snapshoting(root); 1385 goto error; 1386 } 1387 cow_start = (u64)-1; 1388 } 1389 1390 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 1391 struct extent_map *em; 1392 struct extent_map_tree *em_tree; 1393 em_tree = &BTRFS_I(inode)->extent_tree; 1394 em = alloc_extent_map(); 1395 BUG_ON(!em); /* -ENOMEM */ 1396 em->start = cur_offset; 1397 em->orig_start = found_key.offset - extent_offset; 1398 em->len = num_bytes; 1399 em->block_len = num_bytes; 1400 em->block_start = disk_bytenr; 1401 em->orig_block_len = disk_num_bytes; 1402 em->ram_bytes = ram_bytes; 1403 em->bdev = root->fs_info->fs_devices->latest_bdev; 1404 em->mod_start = em->start; 1405 em->mod_len = em->len; 1406 set_bit(EXTENT_FLAG_PINNED, &em->flags); 1407 set_bit(EXTENT_FLAG_FILLING, &em->flags); 1408 em->generation = -1; 1409 while (1) { 1410 write_lock(&em_tree->lock); 1411 ret = add_extent_mapping(em_tree, em, 1); 1412 write_unlock(&em_tree->lock); 1413 if (ret != -EEXIST) { 1414 free_extent_map(em); 1415 break; 1416 } 1417 btrfs_drop_extent_cache(inode, em->start, 1418 em->start + em->len - 1, 0); 1419 } 1420 type = BTRFS_ORDERED_PREALLOC; 1421 } else { 1422 type = BTRFS_ORDERED_NOCOW; 1423 } 1424 1425 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr, 1426 num_bytes, num_bytes, type); 1427 BUG_ON(ret); /* -ENOMEM */ 1428 1429 if (root->root_key.objectid == 1430 BTRFS_DATA_RELOC_TREE_OBJECTID) { 1431 ret = btrfs_reloc_clone_csums(inode, cur_offset, 1432 num_bytes); 1433 if (ret) { 1434 if (!nolock && nocow) 1435 btrfs_end_write_no_snapshoting(root); 1436 goto error; 1437 } 1438 } 1439 1440 extent_clear_unlock_delalloc(inode, cur_offset, 1441 cur_offset + num_bytes - 1, 1442 locked_page, EXTENT_LOCKED | 1443 EXTENT_DELALLOC, PAGE_UNLOCK | 1444 PAGE_SET_PRIVATE2); 1445 if (!nolock && nocow) 1446 btrfs_end_write_no_snapshoting(root); 1447 cur_offset = extent_end; 1448 if (cur_offset > end) 1449 break; 1450 } 1451 btrfs_release_path(path); 1452 1453 if (cur_offset <= end && cow_start == (u64)-1) { 1454 cow_start = cur_offset; 1455 cur_offset = end; 1456 } 1457 1458 if (cow_start != (u64)-1) { 1459 ret = cow_file_range(inode, locked_page, cow_start, end, 1460 page_started, nr_written, 1); 1461 if (ret) 1462 goto error; 1463 } 1464 1465 error: 1466 err = btrfs_end_transaction(trans, root); 1467 if (!ret) 1468 ret = err; 1469 1470 if (ret && cur_offset < end) 1471 extent_clear_unlock_delalloc(inode, cur_offset, end, 1472 locked_page, EXTENT_LOCKED | 1473 EXTENT_DELALLOC | EXTENT_DEFRAG | 1474 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 1475 PAGE_CLEAR_DIRTY | 1476 PAGE_SET_WRITEBACK | 1477 PAGE_END_WRITEBACK); 1478 btrfs_free_path(path); 1479 return ret; 1480 } 1481 1482 static inline int need_force_cow(struct inode *inode, u64 start, u64 end) 1483 { 1484 1485 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 1486 !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC)) 1487 return 0; 1488 1489 /* 1490 * @defrag_bytes is a hint value, no spinlock held here, 1491 * if is not zero, it means the file is defragging. 1492 * Force cow if given extent needs to be defragged. 1493 */ 1494 if (BTRFS_I(inode)->defrag_bytes && 1495 test_range_bit(&BTRFS_I(inode)->io_tree, start, end, 1496 EXTENT_DEFRAG, 0, NULL)) 1497 return 1; 1498 1499 return 0; 1500 } 1501 1502 /* 1503 * extent_io.c call back to do delayed allocation processing 1504 */ 1505 static int run_delalloc_range(struct inode *inode, struct page *locked_page, 1506 u64 start, u64 end, int *page_started, 1507 unsigned long *nr_written) 1508 { 1509 int ret; 1510 int force_cow = need_force_cow(inode, start, end); 1511 1512 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) { 1513 ret = run_delalloc_nocow(inode, locked_page, start, end, 1514 page_started, 1, nr_written); 1515 } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) { 1516 ret = run_delalloc_nocow(inode, locked_page, start, end, 1517 page_started, 0, nr_written); 1518 } else if (!inode_need_compress(inode)) { 1519 ret = cow_file_range(inode, locked_page, start, end, 1520 page_started, nr_written, 1); 1521 } else { 1522 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 1523 &BTRFS_I(inode)->runtime_flags); 1524 ret = cow_file_range_async(inode, locked_page, start, end, 1525 page_started, nr_written); 1526 } 1527 return ret; 1528 } 1529 1530 static void btrfs_split_extent_hook(struct inode *inode, 1531 struct extent_state *orig, u64 split) 1532 { 1533 /* not delalloc, ignore it */ 1534 if (!(orig->state & EXTENT_DELALLOC)) 1535 return; 1536 1537 spin_lock(&BTRFS_I(inode)->lock); 1538 BTRFS_I(inode)->outstanding_extents++; 1539 spin_unlock(&BTRFS_I(inode)->lock); 1540 } 1541 1542 /* 1543 * extent_io.c merge_extent_hook, used to track merged delayed allocation 1544 * extents so we can keep track of new extents that are just merged onto old 1545 * extents, such as when we are doing sequential writes, so we can properly 1546 * account for the metadata space we'll need. 1547 */ 1548 static void btrfs_merge_extent_hook(struct inode *inode, 1549 struct extent_state *new, 1550 struct extent_state *other) 1551 { 1552 /* not delalloc, ignore it */ 1553 if (!(other->state & EXTENT_DELALLOC)) 1554 return; 1555 1556 spin_lock(&BTRFS_I(inode)->lock); 1557 BTRFS_I(inode)->outstanding_extents--; 1558 spin_unlock(&BTRFS_I(inode)->lock); 1559 } 1560 1561 static void btrfs_add_delalloc_inodes(struct btrfs_root *root, 1562 struct inode *inode) 1563 { 1564 spin_lock(&root->delalloc_lock); 1565 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1566 list_add_tail(&BTRFS_I(inode)->delalloc_inodes, 1567 &root->delalloc_inodes); 1568 set_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1569 &BTRFS_I(inode)->runtime_flags); 1570 root->nr_delalloc_inodes++; 1571 if (root->nr_delalloc_inodes == 1) { 1572 spin_lock(&root->fs_info->delalloc_root_lock); 1573 BUG_ON(!list_empty(&root->delalloc_root)); 1574 list_add_tail(&root->delalloc_root, 1575 &root->fs_info->delalloc_roots); 1576 spin_unlock(&root->fs_info->delalloc_root_lock); 1577 } 1578 } 1579 spin_unlock(&root->delalloc_lock); 1580 } 1581 1582 static void btrfs_del_delalloc_inode(struct btrfs_root *root, 1583 struct inode *inode) 1584 { 1585 spin_lock(&root->delalloc_lock); 1586 if (!list_empty(&BTRFS_I(inode)->delalloc_inodes)) { 1587 list_del_init(&BTRFS_I(inode)->delalloc_inodes); 1588 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1589 &BTRFS_I(inode)->runtime_flags); 1590 root->nr_delalloc_inodes--; 1591 if (!root->nr_delalloc_inodes) { 1592 spin_lock(&root->fs_info->delalloc_root_lock); 1593 BUG_ON(list_empty(&root->delalloc_root)); 1594 list_del_init(&root->delalloc_root); 1595 spin_unlock(&root->fs_info->delalloc_root_lock); 1596 } 1597 } 1598 spin_unlock(&root->delalloc_lock); 1599 } 1600 1601 /* 1602 * extent_io.c set_bit_hook, used to track delayed allocation 1603 * bytes in this file, and to maintain the list of inodes that 1604 * have pending delalloc work to be done. 1605 */ 1606 static void btrfs_set_bit_hook(struct inode *inode, 1607 struct extent_state *state, unsigned long *bits) 1608 { 1609 1610 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC)) 1611 WARN_ON(1); 1612 /* 1613 * set_bit and clear bit hooks normally require _irqsave/restore 1614 * but in this case, we are only testing for the DELALLOC 1615 * bit, which is only set or cleared with irqs on 1616 */ 1617 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1618 struct btrfs_root *root = BTRFS_I(inode)->root; 1619 u64 len = state->end + 1 - state->start; 1620 bool do_list = !btrfs_is_free_space_inode(inode); 1621 1622 if (*bits & EXTENT_FIRST_DELALLOC) { 1623 *bits &= ~EXTENT_FIRST_DELALLOC; 1624 } else { 1625 spin_lock(&BTRFS_I(inode)->lock); 1626 BTRFS_I(inode)->outstanding_extents++; 1627 spin_unlock(&BTRFS_I(inode)->lock); 1628 } 1629 1630 __percpu_counter_add(&root->fs_info->delalloc_bytes, len, 1631 root->fs_info->delalloc_batch); 1632 spin_lock(&BTRFS_I(inode)->lock); 1633 BTRFS_I(inode)->delalloc_bytes += len; 1634 if (*bits & EXTENT_DEFRAG) 1635 BTRFS_I(inode)->defrag_bytes += len; 1636 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1637 &BTRFS_I(inode)->runtime_flags)) 1638 btrfs_add_delalloc_inodes(root, inode); 1639 spin_unlock(&BTRFS_I(inode)->lock); 1640 } 1641 } 1642 1643 /* 1644 * extent_io.c clear_bit_hook, see set_bit_hook for why 1645 */ 1646 static void btrfs_clear_bit_hook(struct inode *inode, 1647 struct extent_state *state, 1648 unsigned long *bits) 1649 { 1650 u64 len = state->end + 1 - state->start; 1651 1652 spin_lock(&BTRFS_I(inode)->lock); 1653 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) 1654 BTRFS_I(inode)->defrag_bytes -= len; 1655 spin_unlock(&BTRFS_I(inode)->lock); 1656 1657 /* 1658 * set_bit and clear bit hooks normally require _irqsave/restore 1659 * but in this case, we are only testing for the DELALLOC 1660 * bit, which is only set or cleared with irqs on 1661 */ 1662 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) { 1663 struct btrfs_root *root = BTRFS_I(inode)->root; 1664 bool do_list = !btrfs_is_free_space_inode(inode); 1665 1666 if (*bits & EXTENT_FIRST_DELALLOC) { 1667 *bits &= ~EXTENT_FIRST_DELALLOC; 1668 } else if (!(*bits & EXTENT_DO_ACCOUNTING)) { 1669 spin_lock(&BTRFS_I(inode)->lock); 1670 BTRFS_I(inode)->outstanding_extents--; 1671 spin_unlock(&BTRFS_I(inode)->lock); 1672 } 1673 1674 /* 1675 * We don't reserve metadata space for space cache inodes so we 1676 * don't need to call dellalloc_release_metadata if there is an 1677 * error. 1678 */ 1679 if (*bits & EXTENT_DO_ACCOUNTING && 1680 root != root->fs_info->tree_root) 1681 btrfs_delalloc_release_metadata(inode, len); 1682 1683 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 1684 && do_list && !(state->state & EXTENT_NORESERVE)) 1685 btrfs_free_reserved_data_space(inode, len); 1686 1687 __percpu_counter_add(&root->fs_info->delalloc_bytes, -len, 1688 root->fs_info->delalloc_batch); 1689 spin_lock(&BTRFS_I(inode)->lock); 1690 BTRFS_I(inode)->delalloc_bytes -= len; 1691 if (do_list && BTRFS_I(inode)->delalloc_bytes == 0 && 1692 test_bit(BTRFS_INODE_IN_DELALLOC_LIST, 1693 &BTRFS_I(inode)->runtime_flags)) 1694 btrfs_del_delalloc_inode(root, inode); 1695 spin_unlock(&BTRFS_I(inode)->lock); 1696 } 1697 } 1698 1699 /* 1700 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure 1701 * we don't create bios that span stripes or chunks 1702 */ 1703 int btrfs_merge_bio_hook(int rw, struct page *page, unsigned long offset, 1704 size_t size, struct bio *bio, 1705 unsigned long bio_flags) 1706 { 1707 struct btrfs_root *root = BTRFS_I(page->mapping->host)->root; 1708 u64 logical = (u64)bio->bi_iter.bi_sector << 9; 1709 u64 length = 0; 1710 u64 map_length; 1711 int ret; 1712 1713 if (bio_flags & EXTENT_BIO_COMPRESSED) 1714 return 0; 1715 1716 length = bio->bi_iter.bi_size; 1717 map_length = length; 1718 ret = btrfs_map_block(root->fs_info, rw, logical, 1719 &map_length, NULL, 0); 1720 /* Will always return 0 with map_multi == NULL */ 1721 BUG_ON(ret < 0); 1722 if (map_length < length + size) 1723 return 1; 1724 return 0; 1725 } 1726 1727 /* 1728 * in order to insert checksums into the metadata in large chunks, 1729 * we wait until bio submission time. All the pages in the bio are 1730 * checksummed and sums are attached onto the ordered extent record. 1731 * 1732 * At IO completion time the cums attached on the ordered extent record 1733 * are inserted into the btree 1734 */ 1735 static int __btrfs_submit_bio_start(struct inode *inode, int rw, 1736 struct bio *bio, int mirror_num, 1737 unsigned long bio_flags, 1738 u64 bio_offset) 1739 { 1740 struct btrfs_root *root = BTRFS_I(inode)->root; 1741 int ret = 0; 1742 1743 ret = btrfs_csum_one_bio(root, inode, bio, 0, 0); 1744 BUG_ON(ret); /* -ENOMEM */ 1745 return 0; 1746 } 1747 1748 /* 1749 * in order to insert checksums into the metadata in large chunks, 1750 * we wait until bio submission time. All the pages in the bio are 1751 * checksummed and sums are attached onto the ordered extent record. 1752 * 1753 * At IO completion time the cums attached on the ordered extent record 1754 * are inserted into the btree 1755 */ 1756 static int __btrfs_submit_bio_done(struct inode *inode, int rw, struct bio *bio, 1757 int mirror_num, unsigned long bio_flags, 1758 u64 bio_offset) 1759 { 1760 struct btrfs_root *root = BTRFS_I(inode)->root; 1761 int ret; 1762 1763 ret = btrfs_map_bio(root, rw, bio, mirror_num, 1); 1764 if (ret) 1765 bio_endio(bio, ret); 1766 return ret; 1767 } 1768 1769 /* 1770 * extent_io.c submission hook. This does the right thing for csum calculation 1771 * on write, or reading the csums from the tree before a read 1772 */ 1773 static int btrfs_submit_bio_hook(struct inode *inode, int rw, struct bio *bio, 1774 int mirror_num, unsigned long bio_flags, 1775 u64 bio_offset) 1776 { 1777 struct btrfs_root *root = BTRFS_I(inode)->root; 1778 int ret = 0; 1779 int skip_sum; 1780 int metadata = 0; 1781 int async = !atomic_read(&BTRFS_I(inode)->sync_writers); 1782 1783 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 1784 1785 if (btrfs_is_free_space_inode(inode)) 1786 metadata = 2; 1787 1788 if (!(rw & REQ_WRITE)) { 1789 ret = btrfs_bio_wq_end_io(root->fs_info, bio, metadata); 1790 if (ret) 1791 goto out; 1792 1793 if (bio_flags & EXTENT_BIO_COMPRESSED) { 1794 ret = btrfs_submit_compressed_read(inode, bio, 1795 mirror_num, 1796 bio_flags); 1797 goto out; 1798 } else if (!skip_sum) { 1799 ret = btrfs_lookup_bio_sums(root, inode, bio, NULL); 1800 if (ret) 1801 goto out; 1802 } 1803 goto mapit; 1804 } else if (async && !skip_sum) { 1805 /* csum items have already been cloned */ 1806 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) 1807 goto mapit; 1808 /* we're doing a write, do the async checksumming */ 1809 ret = btrfs_wq_submit_bio(BTRFS_I(inode)->root->fs_info, 1810 inode, rw, bio, mirror_num, 1811 bio_flags, bio_offset, 1812 __btrfs_submit_bio_start, 1813 __btrfs_submit_bio_done); 1814 goto out; 1815 } else if (!skip_sum) { 1816 ret = btrfs_csum_one_bio(root, inode, bio, 0, 0); 1817 if (ret) 1818 goto out; 1819 } 1820 1821 mapit: 1822 ret = btrfs_map_bio(root, rw, bio, mirror_num, 0); 1823 1824 out: 1825 if (ret < 0) 1826 bio_endio(bio, ret); 1827 return ret; 1828 } 1829 1830 /* 1831 * given a list of ordered sums record them in the inode. This happens 1832 * at IO completion time based on sums calculated at bio submission time. 1833 */ 1834 static noinline int add_pending_csums(struct btrfs_trans_handle *trans, 1835 struct inode *inode, u64 file_offset, 1836 struct list_head *list) 1837 { 1838 struct btrfs_ordered_sum *sum; 1839 1840 list_for_each_entry(sum, list, list) { 1841 trans->adding_csums = 1; 1842 btrfs_csum_file_blocks(trans, 1843 BTRFS_I(inode)->root->fs_info->csum_root, sum); 1844 trans->adding_csums = 0; 1845 } 1846 return 0; 1847 } 1848 1849 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end, 1850 struct extent_state **cached_state) 1851 { 1852 WARN_ON((end & (PAGE_CACHE_SIZE - 1)) == 0); 1853 return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end, 1854 cached_state, GFP_NOFS); 1855 } 1856 1857 /* see btrfs_writepage_start_hook for details on why this is required */ 1858 struct btrfs_writepage_fixup { 1859 struct page *page; 1860 struct btrfs_work work; 1861 }; 1862 1863 static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 1864 { 1865 struct btrfs_writepage_fixup *fixup; 1866 struct btrfs_ordered_extent *ordered; 1867 struct extent_state *cached_state = NULL; 1868 struct page *page; 1869 struct inode *inode; 1870 u64 page_start; 1871 u64 page_end; 1872 int ret; 1873 1874 fixup = container_of(work, struct btrfs_writepage_fixup, work); 1875 page = fixup->page; 1876 again: 1877 lock_page(page); 1878 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) { 1879 ClearPageChecked(page); 1880 goto out_page; 1881 } 1882 1883 inode = page->mapping->host; 1884 page_start = page_offset(page); 1885 page_end = page_offset(page) + PAGE_CACHE_SIZE - 1; 1886 1887 lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end, 0, 1888 &cached_state); 1889 1890 /* already ordered? We're done */ 1891 if (PagePrivate2(page)) 1892 goto out; 1893 1894 ordered = btrfs_lookup_ordered_extent(inode, page_start); 1895 if (ordered) { 1896 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, 1897 page_end, &cached_state, GFP_NOFS); 1898 unlock_page(page); 1899 btrfs_start_ordered_extent(inode, ordered, 1); 1900 btrfs_put_ordered_extent(ordered); 1901 goto again; 1902 } 1903 1904 ret = btrfs_delalloc_reserve_space(inode, PAGE_CACHE_SIZE); 1905 if (ret) { 1906 mapping_set_error(page->mapping, ret); 1907 end_extent_writepage(page, ret, page_start, page_end); 1908 ClearPageChecked(page); 1909 goto out; 1910 } 1911 1912 btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state); 1913 ClearPageChecked(page); 1914 set_page_dirty(page); 1915 out: 1916 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end, 1917 &cached_state, GFP_NOFS); 1918 out_page: 1919 unlock_page(page); 1920 page_cache_release(page); 1921 kfree(fixup); 1922 } 1923 1924 /* 1925 * There are a few paths in the higher layers of the kernel that directly 1926 * set the page dirty bit without asking the filesystem if it is a 1927 * good idea. This causes problems because we want to make sure COW 1928 * properly happens and the data=ordered rules are followed. 1929 * 1930 * In our case any range that doesn't have the ORDERED bit set 1931 * hasn't been properly setup for IO. We kick off an async process 1932 * to fix it up. The async helper will wait for ordered extents, set 1933 * the delalloc bit and make it safe to write the page. 1934 */ 1935 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end) 1936 { 1937 struct inode *inode = page->mapping->host; 1938 struct btrfs_writepage_fixup *fixup; 1939 struct btrfs_root *root = BTRFS_I(inode)->root; 1940 1941 /* this page is properly in the ordered list */ 1942 if (TestClearPagePrivate2(page)) 1943 return 0; 1944 1945 if (PageChecked(page)) 1946 return -EAGAIN; 1947 1948 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 1949 if (!fixup) 1950 return -EAGAIN; 1951 1952 SetPageChecked(page); 1953 page_cache_get(page); 1954 btrfs_init_work(&fixup->work, btrfs_fixup_helper, 1955 btrfs_writepage_fixup_worker, NULL, NULL); 1956 fixup->page = page; 1957 btrfs_queue_work(root->fs_info->fixup_workers, &fixup->work); 1958 return -EBUSY; 1959 } 1960 1961 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 1962 struct inode *inode, u64 file_pos, 1963 u64 disk_bytenr, u64 disk_num_bytes, 1964 u64 num_bytes, u64 ram_bytes, 1965 u8 compression, u8 encryption, 1966 u16 other_encoding, int extent_type) 1967 { 1968 struct btrfs_root *root = BTRFS_I(inode)->root; 1969 struct btrfs_file_extent_item *fi; 1970 struct btrfs_path *path; 1971 struct extent_buffer *leaf; 1972 struct btrfs_key ins; 1973 int extent_inserted = 0; 1974 int ret; 1975 1976 path = btrfs_alloc_path(); 1977 if (!path) 1978 return -ENOMEM; 1979 1980 /* 1981 * we may be replacing one extent in the tree with another. 1982 * The new extent is pinned in the extent map, and we don't want 1983 * to drop it from the cache until it is completely in the btree. 1984 * 1985 * So, tell btrfs_drop_extents to leave this extent in the cache. 1986 * the caller is expected to unpin it and allow it to be merged 1987 * with the others. 1988 */ 1989 ret = __btrfs_drop_extents(trans, root, inode, path, file_pos, 1990 file_pos + num_bytes, NULL, 0, 1991 1, sizeof(*fi), &extent_inserted); 1992 if (ret) 1993 goto out; 1994 1995 if (!extent_inserted) { 1996 ins.objectid = btrfs_ino(inode); 1997 ins.offset = file_pos; 1998 ins.type = BTRFS_EXTENT_DATA_KEY; 1999 2000 path->leave_spinning = 1; 2001 ret = btrfs_insert_empty_item(trans, root, path, &ins, 2002 sizeof(*fi)); 2003 if (ret) 2004 goto out; 2005 } 2006 leaf = path->nodes[0]; 2007 fi = btrfs_item_ptr(leaf, path->slots[0], 2008 struct btrfs_file_extent_item); 2009 btrfs_set_file_extent_generation(leaf, fi, trans->transid); 2010 btrfs_set_file_extent_type(leaf, fi, extent_type); 2011 btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr); 2012 btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes); 2013 btrfs_set_file_extent_offset(leaf, fi, 0); 2014 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); 2015 btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes); 2016 btrfs_set_file_extent_compression(leaf, fi, compression); 2017 btrfs_set_file_extent_encryption(leaf, fi, encryption); 2018 btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding); 2019 2020 btrfs_mark_buffer_dirty(leaf); 2021 btrfs_release_path(path); 2022 2023 inode_add_bytes(inode, num_bytes); 2024 2025 ins.objectid = disk_bytenr; 2026 ins.offset = disk_num_bytes; 2027 ins.type = BTRFS_EXTENT_ITEM_KEY; 2028 ret = btrfs_alloc_reserved_file_extent(trans, root, 2029 root->root_key.objectid, 2030 btrfs_ino(inode), file_pos, &ins); 2031 out: 2032 btrfs_free_path(path); 2033 2034 return ret; 2035 } 2036 2037 /* snapshot-aware defrag */ 2038 struct sa_defrag_extent_backref { 2039 struct rb_node node; 2040 struct old_sa_defrag_extent *old; 2041 u64 root_id; 2042 u64 inum; 2043 u64 file_pos; 2044 u64 extent_offset; 2045 u64 num_bytes; 2046 u64 generation; 2047 }; 2048 2049 struct old_sa_defrag_extent { 2050 struct list_head list; 2051 struct new_sa_defrag_extent *new; 2052 2053 u64 extent_offset; 2054 u64 bytenr; 2055 u64 offset; 2056 u64 len; 2057 int count; 2058 }; 2059 2060 struct new_sa_defrag_extent { 2061 struct rb_root root; 2062 struct list_head head; 2063 struct btrfs_path *path; 2064 struct inode *inode; 2065 u64 file_pos; 2066 u64 len; 2067 u64 bytenr; 2068 u64 disk_len; 2069 u8 compress_type; 2070 }; 2071 2072 static int backref_comp(struct sa_defrag_extent_backref *b1, 2073 struct sa_defrag_extent_backref *b2) 2074 { 2075 if (b1->root_id < b2->root_id) 2076 return -1; 2077 else if (b1->root_id > b2->root_id) 2078 return 1; 2079 2080 if (b1->inum < b2->inum) 2081 return -1; 2082 else if (b1->inum > b2->inum) 2083 return 1; 2084 2085 if (b1->file_pos < b2->file_pos) 2086 return -1; 2087 else if (b1->file_pos > b2->file_pos) 2088 return 1; 2089 2090 /* 2091 * [------------------------------] ===> (a range of space) 2092 * |<--->| |<---->| =============> (fs/file tree A) 2093 * |<---------------------------->| ===> (fs/file tree B) 2094 * 2095 * A range of space can refer to two file extents in one tree while 2096 * refer to only one file extent in another tree. 2097 * 2098 * So we may process a disk offset more than one time(two extents in A) 2099 * and locate at the same extent(one extent in B), then insert two same 2100 * backrefs(both refer to the extent in B). 2101 */ 2102 return 0; 2103 } 2104 2105 static void backref_insert(struct rb_root *root, 2106 struct sa_defrag_extent_backref *backref) 2107 { 2108 struct rb_node **p = &root->rb_node; 2109 struct rb_node *parent = NULL; 2110 struct sa_defrag_extent_backref *entry; 2111 int ret; 2112 2113 while (*p) { 2114 parent = *p; 2115 entry = rb_entry(parent, struct sa_defrag_extent_backref, node); 2116 2117 ret = backref_comp(backref, entry); 2118 if (ret < 0) 2119 p = &(*p)->rb_left; 2120 else 2121 p = &(*p)->rb_right; 2122 } 2123 2124 rb_link_node(&backref->node, parent, p); 2125 rb_insert_color(&backref->node, root); 2126 } 2127 2128 /* 2129 * Note the backref might has changed, and in this case we just return 0. 2130 */ 2131 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id, 2132 void *ctx) 2133 { 2134 struct btrfs_file_extent_item *extent; 2135 struct btrfs_fs_info *fs_info; 2136 struct old_sa_defrag_extent *old = ctx; 2137 struct new_sa_defrag_extent *new = old->new; 2138 struct btrfs_path *path = new->path; 2139 struct btrfs_key key; 2140 struct btrfs_root *root; 2141 struct sa_defrag_extent_backref *backref; 2142 struct extent_buffer *leaf; 2143 struct inode *inode = new->inode; 2144 int slot; 2145 int ret; 2146 u64 extent_offset; 2147 u64 num_bytes; 2148 2149 if (BTRFS_I(inode)->root->root_key.objectid == root_id && 2150 inum == btrfs_ino(inode)) 2151 return 0; 2152 2153 key.objectid = root_id; 2154 key.type = BTRFS_ROOT_ITEM_KEY; 2155 key.offset = (u64)-1; 2156 2157 fs_info = BTRFS_I(inode)->root->fs_info; 2158 root = btrfs_read_fs_root_no_name(fs_info, &key); 2159 if (IS_ERR(root)) { 2160 if (PTR_ERR(root) == -ENOENT) 2161 return 0; 2162 WARN_ON(1); 2163 pr_debug("inum=%llu, offset=%llu, root_id=%llu\n", 2164 inum, offset, root_id); 2165 return PTR_ERR(root); 2166 } 2167 2168 key.objectid = inum; 2169 key.type = BTRFS_EXTENT_DATA_KEY; 2170 if (offset > (u64)-1 << 32) 2171 key.offset = 0; 2172 else 2173 key.offset = offset; 2174 2175 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2176 if (WARN_ON(ret < 0)) 2177 return ret; 2178 ret = 0; 2179 2180 while (1) { 2181 cond_resched(); 2182 2183 leaf = path->nodes[0]; 2184 slot = path->slots[0]; 2185 2186 if (slot >= btrfs_header_nritems(leaf)) { 2187 ret = btrfs_next_leaf(root, path); 2188 if (ret < 0) { 2189 goto out; 2190 } else if (ret > 0) { 2191 ret = 0; 2192 goto out; 2193 } 2194 continue; 2195 } 2196 2197 path->slots[0]++; 2198 2199 btrfs_item_key_to_cpu(leaf, &key, slot); 2200 2201 if (key.objectid > inum) 2202 goto out; 2203 2204 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY) 2205 continue; 2206 2207 extent = btrfs_item_ptr(leaf, slot, 2208 struct btrfs_file_extent_item); 2209 2210 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr) 2211 continue; 2212 2213 /* 2214 * 'offset' refers to the exact key.offset, 2215 * NOT the 'offset' field in btrfs_extent_data_ref, ie. 2216 * (key.offset - extent_offset). 2217 */ 2218 if (key.offset != offset) 2219 continue; 2220 2221 extent_offset = btrfs_file_extent_offset(leaf, extent); 2222 num_bytes = btrfs_file_extent_num_bytes(leaf, extent); 2223 2224 if (extent_offset >= old->extent_offset + old->offset + 2225 old->len || extent_offset + num_bytes <= 2226 old->extent_offset + old->offset) 2227 continue; 2228 break; 2229 } 2230 2231 backref = kmalloc(sizeof(*backref), GFP_NOFS); 2232 if (!backref) { 2233 ret = -ENOENT; 2234 goto out; 2235 } 2236 2237 backref->root_id = root_id; 2238 backref->inum = inum; 2239 backref->file_pos = offset; 2240 backref->num_bytes = num_bytes; 2241 backref->extent_offset = extent_offset; 2242 backref->generation = btrfs_file_extent_generation(leaf, extent); 2243 backref->old = old; 2244 backref_insert(&new->root, backref); 2245 old->count++; 2246 out: 2247 btrfs_release_path(path); 2248 WARN_ON(ret); 2249 return ret; 2250 } 2251 2252 static noinline bool record_extent_backrefs(struct btrfs_path *path, 2253 struct new_sa_defrag_extent *new) 2254 { 2255 struct btrfs_fs_info *fs_info = BTRFS_I(new->inode)->root->fs_info; 2256 struct old_sa_defrag_extent *old, *tmp; 2257 int ret; 2258 2259 new->path = path; 2260 2261 list_for_each_entry_safe(old, tmp, &new->head, list) { 2262 ret = iterate_inodes_from_logical(old->bytenr + 2263 old->extent_offset, fs_info, 2264 path, record_one_backref, 2265 old); 2266 if (ret < 0 && ret != -ENOENT) 2267 return false; 2268 2269 /* no backref to be processed for this extent */ 2270 if (!old->count) { 2271 list_del(&old->list); 2272 kfree(old); 2273 } 2274 } 2275 2276 if (list_empty(&new->head)) 2277 return false; 2278 2279 return true; 2280 } 2281 2282 static int relink_is_mergable(struct extent_buffer *leaf, 2283 struct btrfs_file_extent_item *fi, 2284 struct new_sa_defrag_extent *new) 2285 { 2286 if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr) 2287 return 0; 2288 2289 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG) 2290 return 0; 2291 2292 if (btrfs_file_extent_compression(leaf, fi) != new->compress_type) 2293 return 0; 2294 2295 if (btrfs_file_extent_encryption(leaf, fi) || 2296 btrfs_file_extent_other_encoding(leaf, fi)) 2297 return 0; 2298 2299 return 1; 2300 } 2301 2302 /* 2303 * Note the backref might has changed, and in this case we just return 0. 2304 */ 2305 static noinline int relink_extent_backref(struct btrfs_path *path, 2306 struct sa_defrag_extent_backref *prev, 2307 struct sa_defrag_extent_backref *backref) 2308 { 2309 struct btrfs_file_extent_item *extent; 2310 struct btrfs_file_extent_item *item; 2311 struct btrfs_ordered_extent *ordered; 2312 struct btrfs_trans_handle *trans; 2313 struct btrfs_fs_info *fs_info; 2314 struct btrfs_root *root; 2315 struct btrfs_key key; 2316 struct extent_buffer *leaf; 2317 struct old_sa_defrag_extent *old = backref->old; 2318 struct new_sa_defrag_extent *new = old->new; 2319 struct inode *src_inode = new->inode; 2320 struct inode *inode; 2321 struct extent_state *cached = NULL; 2322 int ret = 0; 2323 u64 start; 2324 u64 len; 2325 u64 lock_start; 2326 u64 lock_end; 2327 bool merge = false; 2328 int index; 2329 2330 if (prev && prev->root_id == backref->root_id && 2331 prev->inum == backref->inum && 2332 prev->file_pos + prev->num_bytes == backref->file_pos) 2333 merge = true; 2334 2335 /* step 1: get root */ 2336 key.objectid = backref->root_id; 2337 key.type = BTRFS_ROOT_ITEM_KEY; 2338 key.offset = (u64)-1; 2339 2340 fs_info = BTRFS_I(src_inode)->root->fs_info; 2341 index = srcu_read_lock(&fs_info->subvol_srcu); 2342 2343 root = btrfs_read_fs_root_no_name(fs_info, &key); 2344 if (IS_ERR(root)) { 2345 srcu_read_unlock(&fs_info->subvol_srcu, index); 2346 if (PTR_ERR(root) == -ENOENT) 2347 return 0; 2348 return PTR_ERR(root); 2349 } 2350 2351 if (btrfs_root_readonly(root)) { 2352 srcu_read_unlock(&fs_info->subvol_srcu, index); 2353 return 0; 2354 } 2355 2356 /* step 2: get inode */ 2357 key.objectid = backref->inum; 2358 key.type = BTRFS_INODE_ITEM_KEY; 2359 key.offset = 0; 2360 2361 inode = btrfs_iget(fs_info->sb, &key, root, NULL); 2362 if (IS_ERR(inode)) { 2363 srcu_read_unlock(&fs_info->subvol_srcu, index); 2364 return 0; 2365 } 2366 2367 srcu_read_unlock(&fs_info->subvol_srcu, index); 2368 2369 /* step 3: relink backref */ 2370 lock_start = backref->file_pos; 2371 lock_end = backref->file_pos + backref->num_bytes - 1; 2372 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2373 0, &cached); 2374 2375 ordered = btrfs_lookup_first_ordered_extent(inode, lock_end); 2376 if (ordered) { 2377 btrfs_put_ordered_extent(ordered); 2378 goto out_unlock; 2379 } 2380 2381 trans = btrfs_join_transaction(root); 2382 if (IS_ERR(trans)) { 2383 ret = PTR_ERR(trans); 2384 goto out_unlock; 2385 } 2386 2387 key.objectid = backref->inum; 2388 key.type = BTRFS_EXTENT_DATA_KEY; 2389 key.offset = backref->file_pos; 2390 2391 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2392 if (ret < 0) { 2393 goto out_free_path; 2394 } else if (ret > 0) { 2395 ret = 0; 2396 goto out_free_path; 2397 } 2398 2399 extent = btrfs_item_ptr(path->nodes[0], path->slots[0], 2400 struct btrfs_file_extent_item); 2401 2402 if (btrfs_file_extent_generation(path->nodes[0], extent) != 2403 backref->generation) 2404 goto out_free_path; 2405 2406 btrfs_release_path(path); 2407 2408 start = backref->file_pos; 2409 if (backref->extent_offset < old->extent_offset + old->offset) 2410 start += old->extent_offset + old->offset - 2411 backref->extent_offset; 2412 2413 len = min(backref->extent_offset + backref->num_bytes, 2414 old->extent_offset + old->offset + old->len); 2415 len -= max(backref->extent_offset, old->extent_offset + old->offset); 2416 2417 ret = btrfs_drop_extents(trans, root, inode, start, 2418 start + len, 1); 2419 if (ret) 2420 goto out_free_path; 2421 again: 2422 key.objectid = btrfs_ino(inode); 2423 key.type = BTRFS_EXTENT_DATA_KEY; 2424 key.offset = start; 2425 2426 path->leave_spinning = 1; 2427 if (merge) { 2428 struct btrfs_file_extent_item *fi; 2429 u64 extent_len; 2430 struct btrfs_key found_key; 2431 2432 ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 2433 if (ret < 0) 2434 goto out_free_path; 2435 2436 path->slots[0]--; 2437 leaf = path->nodes[0]; 2438 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 2439 2440 fi = btrfs_item_ptr(leaf, path->slots[0], 2441 struct btrfs_file_extent_item); 2442 extent_len = btrfs_file_extent_num_bytes(leaf, fi); 2443 2444 if (extent_len + found_key.offset == start && 2445 relink_is_mergable(leaf, fi, new)) { 2446 btrfs_set_file_extent_num_bytes(leaf, fi, 2447 extent_len + len); 2448 btrfs_mark_buffer_dirty(leaf); 2449 inode_add_bytes(inode, len); 2450 2451 ret = 1; 2452 goto out_free_path; 2453 } else { 2454 merge = false; 2455 btrfs_release_path(path); 2456 goto again; 2457 } 2458 } 2459 2460 ret = btrfs_insert_empty_item(trans, root, path, &key, 2461 sizeof(*extent)); 2462 if (ret) { 2463 btrfs_abort_transaction(trans, root, ret); 2464 goto out_free_path; 2465 } 2466 2467 leaf = path->nodes[0]; 2468 item = btrfs_item_ptr(leaf, path->slots[0], 2469 struct btrfs_file_extent_item); 2470 btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr); 2471 btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len); 2472 btrfs_set_file_extent_offset(leaf, item, start - new->file_pos); 2473 btrfs_set_file_extent_num_bytes(leaf, item, len); 2474 btrfs_set_file_extent_ram_bytes(leaf, item, new->len); 2475 btrfs_set_file_extent_generation(leaf, item, trans->transid); 2476 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); 2477 btrfs_set_file_extent_compression(leaf, item, new->compress_type); 2478 btrfs_set_file_extent_encryption(leaf, item, 0); 2479 btrfs_set_file_extent_other_encoding(leaf, item, 0); 2480 2481 btrfs_mark_buffer_dirty(leaf); 2482 inode_add_bytes(inode, len); 2483 btrfs_release_path(path); 2484 2485 ret = btrfs_inc_extent_ref(trans, root, new->bytenr, 2486 new->disk_len, 0, 2487 backref->root_id, backref->inum, 2488 new->file_pos, 0); /* start - extent_offset */ 2489 if (ret) { 2490 btrfs_abort_transaction(trans, root, ret); 2491 goto out_free_path; 2492 } 2493 2494 ret = 1; 2495 out_free_path: 2496 btrfs_release_path(path); 2497 path->leave_spinning = 0; 2498 btrfs_end_transaction(trans, root); 2499 out_unlock: 2500 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end, 2501 &cached, GFP_NOFS); 2502 iput(inode); 2503 return ret; 2504 } 2505 2506 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new) 2507 { 2508 struct old_sa_defrag_extent *old, *tmp; 2509 2510 if (!new) 2511 return; 2512 2513 list_for_each_entry_safe(old, tmp, &new->head, list) { 2514 list_del(&old->list); 2515 kfree(old); 2516 } 2517 kfree(new); 2518 } 2519 2520 static void relink_file_extents(struct new_sa_defrag_extent *new) 2521 { 2522 struct btrfs_path *path; 2523 struct sa_defrag_extent_backref *backref; 2524 struct sa_defrag_extent_backref *prev = NULL; 2525 struct inode *inode; 2526 struct btrfs_root *root; 2527 struct rb_node *node; 2528 int ret; 2529 2530 inode = new->inode; 2531 root = BTRFS_I(inode)->root; 2532 2533 path = btrfs_alloc_path(); 2534 if (!path) 2535 return; 2536 2537 if (!record_extent_backrefs(path, new)) { 2538 btrfs_free_path(path); 2539 goto out; 2540 } 2541 btrfs_release_path(path); 2542 2543 while (1) { 2544 node = rb_first(&new->root); 2545 if (!node) 2546 break; 2547 rb_erase(node, &new->root); 2548 2549 backref = rb_entry(node, struct sa_defrag_extent_backref, node); 2550 2551 ret = relink_extent_backref(path, prev, backref); 2552 WARN_ON(ret < 0); 2553 2554 kfree(prev); 2555 2556 if (ret == 1) 2557 prev = backref; 2558 else 2559 prev = NULL; 2560 cond_resched(); 2561 } 2562 kfree(prev); 2563 2564 btrfs_free_path(path); 2565 out: 2566 free_sa_defrag_extent(new); 2567 2568 atomic_dec(&root->fs_info->defrag_running); 2569 wake_up(&root->fs_info->transaction_wait); 2570 } 2571 2572 static struct new_sa_defrag_extent * 2573 record_old_file_extents(struct inode *inode, 2574 struct btrfs_ordered_extent *ordered) 2575 { 2576 struct btrfs_root *root = BTRFS_I(inode)->root; 2577 struct btrfs_path *path; 2578 struct btrfs_key key; 2579 struct old_sa_defrag_extent *old; 2580 struct new_sa_defrag_extent *new; 2581 int ret; 2582 2583 new = kmalloc(sizeof(*new), GFP_NOFS); 2584 if (!new) 2585 return NULL; 2586 2587 new->inode = inode; 2588 new->file_pos = ordered->file_offset; 2589 new->len = ordered->len; 2590 new->bytenr = ordered->start; 2591 new->disk_len = ordered->disk_len; 2592 new->compress_type = ordered->compress_type; 2593 new->root = RB_ROOT; 2594 INIT_LIST_HEAD(&new->head); 2595 2596 path = btrfs_alloc_path(); 2597 if (!path) 2598 goto out_kfree; 2599 2600 key.objectid = btrfs_ino(inode); 2601 key.type = BTRFS_EXTENT_DATA_KEY; 2602 key.offset = new->file_pos; 2603 2604 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2605 if (ret < 0) 2606 goto out_free_path; 2607 if (ret > 0 && path->slots[0] > 0) 2608 path->slots[0]--; 2609 2610 /* find out all the old extents for the file range */ 2611 while (1) { 2612 struct btrfs_file_extent_item *extent; 2613 struct extent_buffer *l; 2614 int slot; 2615 u64 num_bytes; 2616 u64 offset; 2617 u64 end; 2618 u64 disk_bytenr; 2619 u64 extent_offset; 2620 2621 l = path->nodes[0]; 2622 slot = path->slots[0]; 2623 2624 if (slot >= btrfs_header_nritems(l)) { 2625 ret = btrfs_next_leaf(root, path); 2626 if (ret < 0) 2627 goto out_free_path; 2628 else if (ret > 0) 2629 break; 2630 continue; 2631 } 2632 2633 btrfs_item_key_to_cpu(l, &key, slot); 2634 2635 if (key.objectid != btrfs_ino(inode)) 2636 break; 2637 if (key.type != BTRFS_EXTENT_DATA_KEY) 2638 break; 2639 if (key.offset >= new->file_pos + new->len) 2640 break; 2641 2642 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item); 2643 2644 num_bytes = btrfs_file_extent_num_bytes(l, extent); 2645 if (key.offset + num_bytes < new->file_pos) 2646 goto next; 2647 2648 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent); 2649 if (!disk_bytenr) 2650 goto next; 2651 2652 extent_offset = btrfs_file_extent_offset(l, extent); 2653 2654 old = kmalloc(sizeof(*old), GFP_NOFS); 2655 if (!old) 2656 goto out_free_path; 2657 2658 offset = max(new->file_pos, key.offset); 2659 end = min(new->file_pos + new->len, key.offset + num_bytes); 2660 2661 old->bytenr = disk_bytenr; 2662 old->extent_offset = extent_offset; 2663 old->offset = offset - key.offset; 2664 old->len = end - offset; 2665 old->new = new; 2666 old->count = 0; 2667 list_add_tail(&old->list, &new->head); 2668 next: 2669 path->slots[0]++; 2670 cond_resched(); 2671 } 2672 2673 btrfs_free_path(path); 2674 atomic_inc(&root->fs_info->defrag_running); 2675 2676 return new; 2677 2678 out_free_path: 2679 btrfs_free_path(path); 2680 out_kfree: 2681 free_sa_defrag_extent(new); 2682 return NULL; 2683 } 2684 2685 static void btrfs_release_delalloc_bytes(struct btrfs_root *root, 2686 u64 start, u64 len) 2687 { 2688 struct btrfs_block_group_cache *cache; 2689 2690 cache = btrfs_lookup_block_group(root->fs_info, start); 2691 ASSERT(cache); 2692 2693 spin_lock(&cache->lock); 2694 cache->delalloc_bytes -= len; 2695 spin_unlock(&cache->lock); 2696 2697 btrfs_put_block_group(cache); 2698 } 2699 2700 /* as ordered data IO finishes, this gets called so we can finish 2701 * an ordered extent if the range of bytes in the file it covers are 2702 * fully written. 2703 */ 2704 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) 2705 { 2706 struct inode *inode = ordered_extent->inode; 2707 struct btrfs_root *root = BTRFS_I(inode)->root; 2708 struct btrfs_trans_handle *trans = NULL; 2709 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2710 struct extent_state *cached_state = NULL; 2711 struct new_sa_defrag_extent *new = NULL; 2712 int compress_type = 0; 2713 int ret = 0; 2714 u64 logical_len = ordered_extent->len; 2715 bool nolock; 2716 bool truncated = false; 2717 2718 nolock = btrfs_is_free_space_inode(inode); 2719 2720 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { 2721 ret = -EIO; 2722 goto out; 2723 } 2724 2725 btrfs_free_io_failure_record(inode, ordered_extent->file_offset, 2726 ordered_extent->file_offset + 2727 ordered_extent->len - 1); 2728 2729 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 2730 truncated = true; 2731 logical_len = ordered_extent->truncated_len; 2732 /* Truncated the entire extent, don't bother adding */ 2733 if (!logical_len) 2734 goto out; 2735 } 2736 2737 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 2738 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */ 2739 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 2740 if (nolock) 2741 trans = btrfs_join_transaction_nolock(root); 2742 else 2743 trans = btrfs_join_transaction(root); 2744 if (IS_ERR(trans)) { 2745 ret = PTR_ERR(trans); 2746 trans = NULL; 2747 goto out; 2748 } 2749 trans->block_rsv = &root->fs_info->delalloc_block_rsv; 2750 ret = btrfs_update_inode_fallback(trans, root, inode); 2751 if (ret) /* -ENOMEM or corruption */ 2752 btrfs_abort_transaction(trans, root, ret); 2753 goto out; 2754 } 2755 2756 lock_extent_bits(io_tree, ordered_extent->file_offset, 2757 ordered_extent->file_offset + ordered_extent->len - 1, 2758 0, &cached_state); 2759 2760 ret = test_range_bit(io_tree, ordered_extent->file_offset, 2761 ordered_extent->file_offset + ordered_extent->len - 1, 2762 EXTENT_DEFRAG, 1, cached_state); 2763 if (ret) { 2764 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item); 2765 if (0 && last_snapshot >= BTRFS_I(inode)->generation) 2766 /* the inode is shared */ 2767 new = record_old_file_extents(inode, ordered_extent); 2768 2769 clear_extent_bit(io_tree, ordered_extent->file_offset, 2770 ordered_extent->file_offset + ordered_extent->len - 1, 2771 EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS); 2772 } 2773 2774 if (nolock) 2775 trans = btrfs_join_transaction_nolock(root); 2776 else 2777 trans = btrfs_join_transaction(root); 2778 if (IS_ERR(trans)) { 2779 ret = PTR_ERR(trans); 2780 trans = NULL; 2781 goto out_unlock; 2782 } 2783 2784 trans->block_rsv = &root->fs_info->delalloc_block_rsv; 2785 2786 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 2787 compress_type = ordered_extent->compress_type; 2788 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 2789 BUG_ON(compress_type); 2790 ret = btrfs_mark_extent_written(trans, inode, 2791 ordered_extent->file_offset, 2792 ordered_extent->file_offset + 2793 logical_len); 2794 } else { 2795 BUG_ON(root == root->fs_info->tree_root); 2796 ret = insert_reserved_file_extent(trans, inode, 2797 ordered_extent->file_offset, 2798 ordered_extent->start, 2799 ordered_extent->disk_len, 2800 logical_len, logical_len, 2801 compress_type, 0, 0, 2802 BTRFS_FILE_EXTENT_REG); 2803 if (!ret) 2804 btrfs_release_delalloc_bytes(root, 2805 ordered_extent->start, 2806 ordered_extent->disk_len); 2807 } 2808 unpin_extent_cache(&BTRFS_I(inode)->extent_tree, 2809 ordered_extent->file_offset, ordered_extent->len, 2810 trans->transid); 2811 if (ret < 0) { 2812 btrfs_abort_transaction(trans, root, ret); 2813 goto out_unlock; 2814 } 2815 2816 add_pending_csums(trans, inode, ordered_extent->file_offset, 2817 &ordered_extent->list); 2818 2819 btrfs_ordered_update_i_size(inode, 0, ordered_extent); 2820 ret = btrfs_update_inode_fallback(trans, root, inode); 2821 if (ret) { /* -ENOMEM or corruption */ 2822 btrfs_abort_transaction(trans, root, ret); 2823 goto out_unlock; 2824 } 2825 ret = 0; 2826 out_unlock: 2827 unlock_extent_cached(io_tree, ordered_extent->file_offset, 2828 ordered_extent->file_offset + 2829 ordered_extent->len - 1, &cached_state, GFP_NOFS); 2830 out: 2831 if (root != root->fs_info->tree_root) 2832 btrfs_delalloc_release_metadata(inode, ordered_extent->len); 2833 if (trans) 2834 btrfs_end_transaction(trans, root); 2835 2836 if (ret || truncated) { 2837 u64 start, end; 2838 2839 if (truncated) 2840 start = ordered_extent->file_offset + logical_len; 2841 else 2842 start = ordered_extent->file_offset; 2843 end = ordered_extent->file_offset + ordered_extent->len - 1; 2844 clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS); 2845 2846 /* Drop the cache for the part of the extent we didn't write. */ 2847 btrfs_drop_extent_cache(inode, start, end, 0); 2848 2849 /* 2850 * If the ordered extent had an IOERR or something else went 2851 * wrong we need to return the space for this ordered extent 2852 * back to the allocator. We only free the extent in the 2853 * truncated case if we didn't write out the extent at all. 2854 */ 2855 if ((ret || !logical_len) && 2856 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 2857 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) 2858 btrfs_free_reserved_extent(root, ordered_extent->start, 2859 ordered_extent->disk_len, 1); 2860 } 2861 2862 2863 /* 2864 * This needs to be done to make sure anybody waiting knows we are done 2865 * updating everything for this ordered extent. 2866 */ 2867 btrfs_remove_ordered_extent(inode, ordered_extent); 2868 2869 /* for snapshot-aware defrag */ 2870 if (new) { 2871 if (ret) { 2872 free_sa_defrag_extent(new); 2873 atomic_dec(&root->fs_info->defrag_running); 2874 } else { 2875 relink_file_extents(new); 2876 } 2877 } 2878 2879 /* once for us */ 2880 btrfs_put_ordered_extent(ordered_extent); 2881 /* once for the tree */ 2882 btrfs_put_ordered_extent(ordered_extent); 2883 2884 return ret; 2885 } 2886 2887 static void finish_ordered_fn(struct btrfs_work *work) 2888 { 2889 struct btrfs_ordered_extent *ordered_extent; 2890 ordered_extent = container_of(work, struct btrfs_ordered_extent, work); 2891 btrfs_finish_ordered_io(ordered_extent); 2892 } 2893 2894 static int btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end, 2895 struct extent_state *state, int uptodate) 2896 { 2897 struct inode *inode = page->mapping->host; 2898 struct btrfs_root *root = BTRFS_I(inode)->root; 2899 struct btrfs_ordered_extent *ordered_extent = NULL; 2900 struct btrfs_workqueue *wq; 2901 btrfs_work_func_t func; 2902 2903 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); 2904 2905 ClearPagePrivate2(page); 2906 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, 2907 end - start + 1, uptodate)) 2908 return 0; 2909 2910 if (btrfs_is_free_space_inode(inode)) { 2911 wq = root->fs_info->endio_freespace_worker; 2912 func = btrfs_freespace_write_helper; 2913 } else { 2914 wq = root->fs_info->endio_write_workers; 2915 func = btrfs_endio_write_helper; 2916 } 2917 2918 btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL, 2919 NULL); 2920 btrfs_queue_work(wq, &ordered_extent->work); 2921 2922 return 0; 2923 } 2924 2925 static int __readpage_endio_check(struct inode *inode, 2926 struct btrfs_io_bio *io_bio, 2927 int icsum, struct page *page, 2928 int pgoff, u64 start, size_t len) 2929 { 2930 char *kaddr; 2931 u32 csum_expected; 2932 u32 csum = ~(u32)0; 2933 static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL, 2934 DEFAULT_RATELIMIT_BURST); 2935 2936 csum_expected = *(((u32 *)io_bio->csum) + icsum); 2937 2938 kaddr = kmap_atomic(page); 2939 csum = btrfs_csum_data(kaddr + pgoff, csum, len); 2940 btrfs_csum_final(csum, (char *)&csum); 2941 if (csum != csum_expected) 2942 goto zeroit; 2943 2944 kunmap_atomic(kaddr); 2945 return 0; 2946 zeroit: 2947 if (__ratelimit(&_rs)) 2948 btrfs_info(BTRFS_I(inode)->root->fs_info, 2949 "csum failed ino %llu off %llu csum %u expected csum %u", 2950 btrfs_ino(inode), start, csum, csum_expected); 2951 memset(kaddr + pgoff, 1, len); 2952 flush_dcache_page(page); 2953 kunmap_atomic(kaddr); 2954 if (csum_expected == 0) 2955 return 0; 2956 return -EIO; 2957 } 2958 2959 /* 2960 * when reads are done, we need to check csums to verify the data is correct 2961 * if there's a match, we allow the bio to finish. If not, the code in 2962 * extent_io.c will try to find good copies for us. 2963 */ 2964 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, 2965 u64 phy_offset, struct page *page, 2966 u64 start, u64 end, int mirror) 2967 { 2968 size_t offset = start - page_offset(page); 2969 struct inode *inode = page->mapping->host; 2970 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 2971 struct btrfs_root *root = BTRFS_I(inode)->root; 2972 2973 if (PageChecked(page)) { 2974 ClearPageChecked(page); 2975 return 0; 2976 } 2977 2978 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) 2979 return 0; 2980 2981 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID && 2982 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) { 2983 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM, 2984 GFP_NOFS); 2985 return 0; 2986 } 2987 2988 phy_offset >>= inode->i_sb->s_blocksize_bits; 2989 return __readpage_endio_check(inode, io_bio, phy_offset, page, offset, 2990 start, (size_t)(end - start + 1)); 2991 } 2992 2993 struct delayed_iput { 2994 struct list_head list; 2995 struct inode *inode; 2996 }; 2997 2998 /* JDM: If this is fs-wide, why can't we add a pointer to 2999 * btrfs_inode instead and avoid the allocation? */ 3000 void btrfs_add_delayed_iput(struct inode *inode) 3001 { 3002 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 3003 struct delayed_iput *delayed; 3004 3005 if (atomic_add_unless(&inode->i_count, -1, 1)) 3006 return; 3007 3008 delayed = kmalloc(sizeof(*delayed), GFP_NOFS | __GFP_NOFAIL); 3009 delayed->inode = inode; 3010 3011 spin_lock(&fs_info->delayed_iput_lock); 3012 list_add_tail(&delayed->list, &fs_info->delayed_iputs); 3013 spin_unlock(&fs_info->delayed_iput_lock); 3014 } 3015 3016 void btrfs_run_delayed_iputs(struct btrfs_root *root) 3017 { 3018 LIST_HEAD(list); 3019 struct btrfs_fs_info *fs_info = root->fs_info; 3020 struct delayed_iput *delayed; 3021 int empty; 3022 3023 spin_lock(&fs_info->delayed_iput_lock); 3024 empty = list_empty(&fs_info->delayed_iputs); 3025 spin_unlock(&fs_info->delayed_iput_lock); 3026 if (empty) 3027 return; 3028 3029 spin_lock(&fs_info->delayed_iput_lock); 3030 list_splice_init(&fs_info->delayed_iputs, &list); 3031 spin_unlock(&fs_info->delayed_iput_lock); 3032 3033 while (!list_empty(&list)) { 3034 delayed = list_entry(list.next, struct delayed_iput, list); 3035 list_del(&delayed->list); 3036 iput(delayed->inode); 3037 kfree(delayed); 3038 } 3039 } 3040 3041 /* 3042 * This is called in transaction commit time. If there are no orphan 3043 * files in the subvolume, it removes orphan item and frees block_rsv 3044 * structure. 3045 */ 3046 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans, 3047 struct btrfs_root *root) 3048 { 3049 struct btrfs_block_rsv *block_rsv; 3050 int ret; 3051 3052 if (atomic_read(&root->orphan_inodes) || 3053 root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) 3054 return; 3055 3056 spin_lock(&root->orphan_lock); 3057 if (atomic_read(&root->orphan_inodes)) { 3058 spin_unlock(&root->orphan_lock); 3059 return; 3060 } 3061 3062 if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) { 3063 spin_unlock(&root->orphan_lock); 3064 return; 3065 } 3066 3067 block_rsv = root->orphan_block_rsv; 3068 root->orphan_block_rsv = NULL; 3069 spin_unlock(&root->orphan_lock); 3070 3071 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) && 3072 btrfs_root_refs(&root->root_item) > 0) { 3073 ret = btrfs_del_orphan_item(trans, root->fs_info->tree_root, 3074 root->root_key.objectid); 3075 if (ret) 3076 btrfs_abort_transaction(trans, root, ret); 3077 else 3078 clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, 3079 &root->state); 3080 } 3081 3082 if (block_rsv) { 3083 WARN_ON(block_rsv->size > 0); 3084 btrfs_free_block_rsv(root, block_rsv); 3085 } 3086 } 3087 3088 /* 3089 * This creates an orphan entry for the given inode in case something goes 3090 * wrong in the middle of an unlink/truncate. 3091 * 3092 * NOTE: caller of this function should reserve 5 units of metadata for 3093 * this function. 3094 */ 3095 int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode) 3096 { 3097 struct btrfs_root *root = BTRFS_I(inode)->root; 3098 struct btrfs_block_rsv *block_rsv = NULL; 3099 int reserve = 0; 3100 int insert = 0; 3101 int ret; 3102 3103 if (!root->orphan_block_rsv) { 3104 block_rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP); 3105 if (!block_rsv) 3106 return -ENOMEM; 3107 } 3108 3109 spin_lock(&root->orphan_lock); 3110 if (!root->orphan_block_rsv) { 3111 root->orphan_block_rsv = block_rsv; 3112 } else if (block_rsv) { 3113 btrfs_free_block_rsv(root, block_rsv); 3114 block_rsv = NULL; 3115 } 3116 3117 if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3118 &BTRFS_I(inode)->runtime_flags)) { 3119 #if 0 3120 /* 3121 * For proper ENOSPC handling, we should do orphan 3122 * cleanup when mounting. But this introduces backward 3123 * compatibility issue. 3124 */ 3125 if (!xchg(&root->orphan_item_inserted, 1)) 3126 insert = 2; 3127 else 3128 insert = 1; 3129 #endif 3130 insert = 1; 3131 atomic_inc(&root->orphan_inodes); 3132 } 3133 3134 if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3135 &BTRFS_I(inode)->runtime_flags)) 3136 reserve = 1; 3137 spin_unlock(&root->orphan_lock); 3138 3139 /* grab metadata reservation from transaction handle */ 3140 if (reserve) { 3141 ret = btrfs_orphan_reserve_metadata(trans, inode); 3142 BUG_ON(ret); /* -ENOSPC in reservation; Logic error? JDM */ 3143 } 3144 3145 /* insert an orphan item to track this unlinked/truncated file */ 3146 if (insert >= 1) { 3147 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode)); 3148 if (ret) { 3149 atomic_dec(&root->orphan_inodes); 3150 if (reserve) { 3151 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3152 &BTRFS_I(inode)->runtime_flags); 3153 btrfs_orphan_release_metadata(inode); 3154 } 3155 if (ret != -EEXIST) { 3156 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3157 &BTRFS_I(inode)->runtime_flags); 3158 btrfs_abort_transaction(trans, root, ret); 3159 return ret; 3160 } 3161 } 3162 ret = 0; 3163 } 3164 3165 /* insert an orphan item to track subvolume contains orphan files */ 3166 if (insert >= 2) { 3167 ret = btrfs_insert_orphan_item(trans, root->fs_info->tree_root, 3168 root->root_key.objectid); 3169 if (ret && ret != -EEXIST) { 3170 btrfs_abort_transaction(trans, root, ret); 3171 return ret; 3172 } 3173 } 3174 return 0; 3175 } 3176 3177 /* 3178 * We have done the truncate/delete so we can go ahead and remove the orphan 3179 * item for this particular inode. 3180 */ 3181 static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3182 struct inode *inode) 3183 { 3184 struct btrfs_root *root = BTRFS_I(inode)->root; 3185 int delete_item = 0; 3186 int release_rsv = 0; 3187 int ret = 0; 3188 3189 spin_lock(&root->orphan_lock); 3190 if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3191 &BTRFS_I(inode)->runtime_flags)) 3192 delete_item = 1; 3193 3194 if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED, 3195 &BTRFS_I(inode)->runtime_flags)) 3196 release_rsv = 1; 3197 spin_unlock(&root->orphan_lock); 3198 3199 if (delete_item) { 3200 atomic_dec(&root->orphan_inodes); 3201 if (trans) 3202 ret = btrfs_del_orphan_item(trans, root, 3203 btrfs_ino(inode)); 3204 } 3205 3206 if (release_rsv) 3207 btrfs_orphan_release_metadata(inode); 3208 3209 return ret; 3210 } 3211 3212 /* 3213 * this cleans up any orphans that may be left on the list from the last use 3214 * of this root. 3215 */ 3216 int btrfs_orphan_cleanup(struct btrfs_root *root) 3217 { 3218 struct btrfs_path *path; 3219 struct extent_buffer *leaf; 3220 struct btrfs_key key, found_key; 3221 struct btrfs_trans_handle *trans; 3222 struct inode *inode; 3223 u64 last_objectid = 0; 3224 int ret = 0, nr_unlink = 0, nr_truncate = 0; 3225 3226 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED)) 3227 return 0; 3228 3229 path = btrfs_alloc_path(); 3230 if (!path) { 3231 ret = -ENOMEM; 3232 goto out; 3233 } 3234 path->reada = -1; 3235 3236 key.objectid = BTRFS_ORPHAN_OBJECTID; 3237 key.type = BTRFS_ORPHAN_ITEM_KEY; 3238 key.offset = (u64)-1; 3239 3240 while (1) { 3241 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3242 if (ret < 0) 3243 goto out; 3244 3245 /* 3246 * if ret == 0 means we found what we were searching for, which 3247 * is weird, but possible, so only screw with path if we didn't 3248 * find the key and see if we have stuff that matches 3249 */ 3250 if (ret > 0) { 3251 ret = 0; 3252 if (path->slots[0] == 0) 3253 break; 3254 path->slots[0]--; 3255 } 3256 3257 /* pull out the item */ 3258 leaf = path->nodes[0]; 3259 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3260 3261 /* make sure the item matches what we want */ 3262 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3263 break; 3264 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3265 break; 3266 3267 /* release the path since we're done with it */ 3268 btrfs_release_path(path); 3269 3270 /* 3271 * this is where we are basically btrfs_lookup, without the 3272 * crossing root thing. we store the inode number in the 3273 * offset of the orphan item. 3274 */ 3275 3276 if (found_key.offset == last_objectid) { 3277 btrfs_err(root->fs_info, 3278 "Error removing orphan entry, stopping orphan cleanup"); 3279 ret = -EINVAL; 3280 goto out; 3281 } 3282 3283 last_objectid = found_key.offset; 3284 3285 found_key.objectid = found_key.offset; 3286 found_key.type = BTRFS_INODE_ITEM_KEY; 3287 found_key.offset = 0; 3288 inode = btrfs_iget(root->fs_info->sb, &found_key, root, NULL); 3289 ret = PTR_ERR_OR_ZERO(inode); 3290 if (ret && ret != -ESTALE) 3291 goto out; 3292 3293 if (ret == -ESTALE && root == root->fs_info->tree_root) { 3294 struct btrfs_root *dead_root; 3295 struct btrfs_fs_info *fs_info = root->fs_info; 3296 int is_dead_root = 0; 3297 3298 /* 3299 * this is an orphan in the tree root. Currently these 3300 * could come from 2 sources: 3301 * a) a snapshot deletion in progress 3302 * b) a free space cache inode 3303 * We need to distinguish those two, as the snapshot 3304 * orphan must not get deleted. 3305 * find_dead_roots already ran before us, so if this 3306 * is a snapshot deletion, we should find the root 3307 * in the dead_roots list 3308 */ 3309 spin_lock(&fs_info->trans_lock); 3310 list_for_each_entry(dead_root, &fs_info->dead_roots, 3311 root_list) { 3312 if (dead_root->root_key.objectid == 3313 found_key.objectid) { 3314 is_dead_root = 1; 3315 break; 3316 } 3317 } 3318 spin_unlock(&fs_info->trans_lock); 3319 if (is_dead_root) { 3320 /* prevent this orphan from being found again */ 3321 key.offset = found_key.objectid - 1; 3322 continue; 3323 } 3324 } 3325 /* 3326 * Inode is already gone but the orphan item is still there, 3327 * kill the orphan item. 3328 */ 3329 if (ret == -ESTALE) { 3330 trans = btrfs_start_transaction(root, 1); 3331 if (IS_ERR(trans)) { 3332 ret = PTR_ERR(trans); 3333 goto out; 3334 } 3335 btrfs_debug(root->fs_info, "auto deleting %Lu", 3336 found_key.objectid); 3337 ret = btrfs_del_orphan_item(trans, root, 3338 found_key.objectid); 3339 btrfs_end_transaction(trans, root); 3340 if (ret) 3341 goto out; 3342 continue; 3343 } 3344 3345 /* 3346 * add this inode to the orphan list so btrfs_orphan_del does 3347 * the proper thing when we hit it 3348 */ 3349 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 3350 &BTRFS_I(inode)->runtime_flags); 3351 atomic_inc(&root->orphan_inodes); 3352 3353 /* if we have links, this was a truncate, lets do that */ 3354 if (inode->i_nlink) { 3355 if (WARN_ON(!S_ISREG(inode->i_mode))) { 3356 iput(inode); 3357 continue; 3358 } 3359 nr_truncate++; 3360 3361 /* 1 for the orphan item deletion. */ 3362 trans = btrfs_start_transaction(root, 1); 3363 if (IS_ERR(trans)) { 3364 iput(inode); 3365 ret = PTR_ERR(trans); 3366 goto out; 3367 } 3368 ret = btrfs_orphan_add(trans, inode); 3369 btrfs_end_transaction(trans, root); 3370 if (ret) { 3371 iput(inode); 3372 goto out; 3373 } 3374 3375 ret = btrfs_truncate(inode); 3376 if (ret) 3377 btrfs_orphan_del(NULL, inode); 3378 } else { 3379 nr_unlink++; 3380 } 3381 3382 /* this will do delete_inode and everything for us */ 3383 iput(inode); 3384 if (ret) 3385 goto out; 3386 } 3387 /* release the path since we're done with it */ 3388 btrfs_release_path(path); 3389 3390 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE; 3391 3392 if (root->orphan_block_rsv) 3393 btrfs_block_rsv_release(root, root->orphan_block_rsv, 3394 (u64)-1); 3395 3396 if (root->orphan_block_rsv || 3397 test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3398 trans = btrfs_join_transaction(root); 3399 if (!IS_ERR(trans)) 3400 btrfs_end_transaction(trans, root); 3401 } 3402 3403 if (nr_unlink) 3404 btrfs_debug(root->fs_info, "unlinked %d orphans", nr_unlink); 3405 if (nr_truncate) 3406 btrfs_debug(root->fs_info, "truncated %d orphans", nr_truncate); 3407 3408 out: 3409 if (ret) 3410 btrfs_crit(root->fs_info, 3411 "could not do orphan cleanup %d", ret); 3412 btrfs_free_path(path); 3413 return ret; 3414 } 3415 3416 /* 3417 * very simple check to peek ahead in the leaf looking for xattrs. If we 3418 * don't find any xattrs, we know there can't be any acls. 3419 * 3420 * slot is the slot the inode is in, objectid is the objectid of the inode 3421 */ 3422 static noinline int acls_after_inode_item(struct extent_buffer *leaf, 3423 int slot, u64 objectid, 3424 int *first_xattr_slot) 3425 { 3426 u32 nritems = btrfs_header_nritems(leaf); 3427 struct btrfs_key found_key; 3428 static u64 xattr_access = 0; 3429 static u64 xattr_default = 0; 3430 int scanned = 0; 3431 3432 if (!xattr_access) { 3433 xattr_access = btrfs_name_hash(POSIX_ACL_XATTR_ACCESS, 3434 strlen(POSIX_ACL_XATTR_ACCESS)); 3435 xattr_default = btrfs_name_hash(POSIX_ACL_XATTR_DEFAULT, 3436 strlen(POSIX_ACL_XATTR_DEFAULT)); 3437 } 3438 3439 slot++; 3440 *first_xattr_slot = -1; 3441 while (slot < nritems) { 3442 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3443 3444 /* we found a different objectid, there must not be acls */ 3445 if (found_key.objectid != objectid) 3446 return 0; 3447 3448 /* we found an xattr, assume we've got an acl */ 3449 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3450 if (*first_xattr_slot == -1) 3451 *first_xattr_slot = slot; 3452 if (found_key.offset == xattr_access || 3453 found_key.offset == xattr_default) 3454 return 1; 3455 } 3456 3457 /* 3458 * we found a key greater than an xattr key, there can't 3459 * be any acls later on 3460 */ 3461 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3462 return 0; 3463 3464 slot++; 3465 scanned++; 3466 3467 /* 3468 * it goes inode, inode backrefs, xattrs, extents, 3469 * so if there are a ton of hard links to an inode there can 3470 * be a lot of backrefs. Don't waste time searching too hard, 3471 * this is just an optimization 3472 */ 3473 if (scanned >= 8) 3474 break; 3475 } 3476 /* we hit the end of the leaf before we found an xattr or 3477 * something larger than an xattr. We have to assume the inode 3478 * has acls 3479 */ 3480 if (*first_xattr_slot == -1) 3481 *first_xattr_slot = slot; 3482 return 1; 3483 } 3484 3485 /* 3486 * read an inode from the btree into the in-memory inode 3487 */ 3488 static void btrfs_read_locked_inode(struct inode *inode) 3489 { 3490 struct btrfs_path *path; 3491 struct extent_buffer *leaf; 3492 struct btrfs_inode_item *inode_item; 3493 struct btrfs_timespec *tspec; 3494 struct btrfs_root *root = BTRFS_I(inode)->root; 3495 struct btrfs_key location; 3496 unsigned long ptr; 3497 int maybe_acls; 3498 u32 rdev; 3499 int ret; 3500 bool filled = false; 3501 int first_xattr_slot; 3502 3503 ret = btrfs_fill_inode(inode, &rdev); 3504 if (!ret) 3505 filled = true; 3506 3507 path = btrfs_alloc_path(); 3508 if (!path) 3509 goto make_bad; 3510 3511 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location)); 3512 3513 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3514 if (ret) 3515 goto make_bad; 3516 3517 leaf = path->nodes[0]; 3518 3519 if (filled) 3520 goto cache_index; 3521 3522 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3523 struct btrfs_inode_item); 3524 inode->i_mode = btrfs_inode_mode(leaf, inode_item); 3525 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item)); 3526 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item)); 3527 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item)); 3528 btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item)); 3529 3530 tspec = btrfs_inode_atime(inode_item); 3531 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, tspec); 3532 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, tspec); 3533 3534 tspec = btrfs_inode_mtime(inode_item); 3535 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, tspec); 3536 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, tspec); 3537 3538 tspec = btrfs_inode_ctime(inode_item); 3539 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, tspec); 3540 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, tspec); 3541 3542 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item)); 3543 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item); 3544 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item); 3545 3546 /* 3547 * If we were modified in the current generation and evicted from memory 3548 * and then re-read we need to do a full sync since we don't have any 3549 * idea about which extents were modified before we were evicted from 3550 * cache. 3551 */ 3552 if (BTRFS_I(inode)->last_trans == root->fs_info->generation) 3553 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 3554 &BTRFS_I(inode)->runtime_flags); 3555 3556 inode->i_version = btrfs_inode_sequence(leaf, inode_item); 3557 inode->i_generation = BTRFS_I(inode)->generation; 3558 inode->i_rdev = 0; 3559 rdev = btrfs_inode_rdev(leaf, inode_item); 3560 3561 BTRFS_I(inode)->index_cnt = (u64)-1; 3562 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item); 3563 3564 cache_index: 3565 path->slots[0]++; 3566 if (inode->i_nlink != 1 || 3567 path->slots[0] >= btrfs_header_nritems(leaf)) 3568 goto cache_acl; 3569 3570 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 3571 if (location.objectid != btrfs_ino(inode)) 3572 goto cache_acl; 3573 3574 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 3575 if (location.type == BTRFS_INODE_REF_KEY) { 3576 struct btrfs_inode_ref *ref; 3577 3578 ref = (struct btrfs_inode_ref *)ptr; 3579 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref); 3580 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 3581 struct btrfs_inode_extref *extref; 3582 3583 extref = (struct btrfs_inode_extref *)ptr; 3584 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf, 3585 extref); 3586 } 3587 cache_acl: 3588 /* 3589 * try to precache a NULL acl entry for files that don't have 3590 * any xattrs or acls 3591 */ 3592 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 3593 btrfs_ino(inode), &first_xattr_slot); 3594 if (first_xattr_slot != -1) { 3595 path->slots[0] = first_xattr_slot; 3596 ret = btrfs_load_inode_props(inode, path); 3597 if (ret) 3598 btrfs_err(root->fs_info, 3599 "error loading props for ino %llu (root %llu): %d", 3600 btrfs_ino(inode), 3601 root->root_key.objectid, ret); 3602 } 3603 btrfs_free_path(path); 3604 3605 if (!maybe_acls) 3606 cache_no_acl(inode); 3607 3608 switch (inode->i_mode & S_IFMT) { 3609 case S_IFREG: 3610 inode->i_mapping->a_ops = &btrfs_aops; 3611 inode->i_mapping->backing_dev_info = &root->fs_info->bdi; 3612 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 3613 inode->i_fop = &btrfs_file_operations; 3614 inode->i_op = &btrfs_file_inode_operations; 3615 break; 3616 case S_IFDIR: 3617 inode->i_fop = &btrfs_dir_file_operations; 3618 if (root == root->fs_info->tree_root) 3619 inode->i_op = &btrfs_dir_ro_inode_operations; 3620 else 3621 inode->i_op = &btrfs_dir_inode_operations; 3622 break; 3623 case S_IFLNK: 3624 inode->i_op = &btrfs_symlink_inode_operations; 3625 inode->i_mapping->a_ops = &btrfs_symlink_aops; 3626 inode->i_mapping->backing_dev_info = &root->fs_info->bdi; 3627 break; 3628 default: 3629 inode->i_op = &btrfs_special_inode_operations; 3630 init_special_inode(inode, inode->i_mode, rdev); 3631 break; 3632 } 3633 3634 btrfs_update_iflags(inode); 3635 return; 3636 3637 make_bad: 3638 btrfs_free_path(path); 3639 make_bad_inode(inode); 3640 } 3641 3642 /* 3643 * given a leaf and an inode, copy the inode fields into the leaf 3644 */ 3645 static void fill_inode_item(struct btrfs_trans_handle *trans, 3646 struct extent_buffer *leaf, 3647 struct btrfs_inode_item *item, 3648 struct inode *inode) 3649 { 3650 struct btrfs_map_token token; 3651 3652 btrfs_init_map_token(&token); 3653 3654 btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token); 3655 btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token); 3656 btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size, 3657 &token); 3658 btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token); 3659 btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token); 3660 3661 btrfs_set_token_timespec_sec(leaf, btrfs_inode_atime(item), 3662 inode->i_atime.tv_sec, &token); 3663 btrfs_set_token_timespec_nsec(leaf, btrfs_inode_atime(item), 3664 inode->i_atime.tv_nsec, &token); 3665 3666 btrfs_set_token_timespec_sec(leaf, btrfs_inode_mtime(item), 3667 inode->i_mtime.tv_sec, &token); 3668 btrfs_set_token_timespec_nsec(leaf, btrfs_inode_mtime(item), 3669 inode->i_mtime.tv_nsec, &token); 3670 3671 btrfs_set_token_timespec_sec(leaf, btrfs_inode_ctime(item), 3672 inode->i_ctime.tv_sec, &token); 3673 btrfs_set_token_timespec_nsec(leaf, btrfs_inode_ctime(item), 3674 inode->i_ctime.tv_nsec, &token); 3675 3676 btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode), 3677 &token); 3678 btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation, 3679 &token); 3680 btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token); 3681 btrfs_set_token_inode_transid(leaf, item, trans->transid, &token); 3682 btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token); 3683 btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token); 3684 btrfs_set_token_inode_block_group(leaf, item, 0, &token); 3685 } 3686 3687 /* 3688 * copy everything in the in-memory inode into the btree. 3689 */ 3690 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 3691 struct btrfs_root *root, struct inode *inode) 3692 { 3693 struct btrfs_inode_item *inode_item; 3694 struct btrfs_path *path; 3695 struct extent_buffer *leaf; 3696 int ret; 3697 3698 path = btrfs_alloc_path(); 3699 if (!path) 3700 return -ENOMEM; 3701 3702 path->leave_spinning = 1; 3703 ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location, 3704 1); 3705 if (ret) { 3706 if (ret > 0) 3707 ret = -ENOENT; 3708 goto failed; 3709 } 3710 3711 leaf = path->nodes[0]; 3712 inode_item = btrfs_item_ptr(leaf, path->slots[0], 3713 struct btrfs_inode_item); 3714 3715 fill_inode_item(trans, leaf, inode_item, inode); 3716 btrfs_mark_buffer_dirty(leaf); 3717 btrfs_set_inode_last_trans(trans, inode); 3718 ret = 0; 3719 failed: 3720 btrfs_free_path(path); 3721 return ret; 3722 } 3723 3724 /* 3725 * copy everything in the in-memory inode into the btree. 3726 */ 3727 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans, 3728 struct btrfs_root *root, struct inode *inode) 3729 { 3730 int ret; 3731 3732 /* 3733 * If the inode is a free space inode, we can deadlock during commit 3734 * if we put it into the delayed code. 3735 * 3736 * The data relocation inode should also be directly updated 3737 * without delay 3738 */ 3739 if (!btrfs_is_free_space_inode(inode) 3740 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID 3741 && !root->fs_info->log_root_recovering) { 3742 btrfs_update_root_times(trans, root); 3743 3744 ret = btrfs_delayed_update_inode(trans, root, inode); 3745 if (!ret) 3746 btrfs_set_inode_last_trans(trans, inode); 3747 return ret; 3748 } 3749 3750 return btrfs_update_inode_item(trans, root, inode); 3751 } 3752 3753 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 3754 struct btrfs_root *root, 3755 struct inode *inode) 3756 { 3757 int ret; 3758 3759 ret = btrfs_update_inode(trans, root, inode); 3760 if (ret == -ENOSPC) 3761 return btrfs_update_inode_item(trans, root, inode); 3762 return ret; 3763 } 3764 3765 /* 3766 * unlink helper that gets used here in inode.c and in the tree logging 3767 * recovery code. It remove a link in a directory with a given name, and 3768 * also drops the back refs in the inode to the directory 3769 */ 3770 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 3771 struct btrfs_root *root, 3772 struct inode *dir, struct inode *inode, 3773 const char *name, int name_len) 3774 { 3775 struct btrfs_path *path; 3776 int ret = 0; 3777 struct extent_buffer *leaf; 3778 struct btrfs_dir_item *di; 3779 struct btrfs_key key; 3780 u64 index; 3781 u64 ino = btrfs_ino(inode); 3782 u64 dir_ino = btrfs_ino(dir); 3783 3784 path = btrfs_alloc_path(); 3785 if (!path) { 3786 ret = -ENOMEM; 3787 goto out; 3788 } 3789 3790 path->leave_spinning = 1; 3791 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 3792 name, name_len, -1); 3793 if (IS_ERR(di)) { 3794 ret = PTR_ERR(di); 3795 goto err; 3796 } 3797 if (!di) { 3798 ret = -ENOENT; 3799 goto err; 3800 } 3801 leaf = path->nodes[0]; 3802 btrfs_dir_item_key_to_cpu(leaf, di, &key); 3803 ret = btrfs_delete_one_dir_name(trans, root, path, di); 3804 if (ret) 3805 goto err; 3806 btrfs_release_path(path); 3807 3808 /* 3809 * If we don't have dir index, we have to get it by looking up 3810 * the inode ref, since we get the inode ref, remove it directly, 3811 * it is unnecessary to do delayed deletion. 3812 * 3813 * But if we have dir index, needn't search inode ref to get it. 3814 * Since the inode ref is close to the inode item, it is better 3815 * that we delay to delete it, and just do this deletion when 3816 * we update the inode item. 3817 */ 3818 if (BTRFS_I(inode)->dir_index) { 3819 ret = btrfs_delayed_delete_inode_ref(inode); 3820 if (!ret) { 3821 index = BTRFS_I(inode)->dir_index; 3822 goto skip_backref; 3823 } 3824 } 3825 3826 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino, 3827 dir_ino, &index); 3828 if (ret) { 3829 btrfs_info(root->fs_info, 3830 "failed to delete reference to %.*s, inode %llu parent %llu", 3831 name_len, name, ino, dir_ino); 3832 btrfs_abort_transaction(trans, root, ret); 3833 goto err; 3834 } 3835 skip_backref: 3836 ret = btrfs_delete_delayed_dir_index(trans, root, dir, index); 3837 if (ret) { 3838 btrfs_abort_transaction(trans, root, ret); 3839 goto err; 3840 } 3841 3842 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, 3843 inode, dir_ino); 3844 if (ret != 0 && ret != -ENOENT) { 3845 btrfs_abort_transaction(trans, root, ret); 3846 goto err; 3847 } 3848 3849 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, 3850 dir, index); 3851 if (ret == -ENOENT) 3852 ret = 0; 3853 else if (ret) 3854 btrfs_abort_transaction(trans, root, ret); 3855 err: 3856 btrfs_free_path(path); 3857 if (ret) 3858 goto out; 3859 3860 btrfs_i_size_write(dir, dir->i_size - name_len * 2); 3861 inode_inc_iversion(inode); 3862 inode_inc_iversion(dir); 3863 inode->i_ctime = dir->i_mtime = dir->i_ctime = CURRENT_TIME; 3864 ret = btrfs_update_inode(trans, root, dir); 3865 out: 3866 return ret; 3867 } 3868 3869 int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 3870 struct btrfs_root *root, 3871 struct inode *dir, struct inode *inode, 3872 const char *name, int name_len) 3873 { 3874 int ret; 3875 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len); 3876 if (!ret) { 3877 drop_nlink(inode); 3878 ret = btrfs_update_inode(trans, root, inode); 3879 } 3880 return ret; 3881 } 3882 3883 /* 3884 * helper to start transaction for unlink and rmdir. 3885 * 3886 * unlink and rmdir are special in btrfs, they do not always free space, so 3887 * if we cannot make our reservations the normal way try and see if there is 3888 * plenty of slack room in the global reserve to migrate, otherwise we cannot 3889 * allow the unlink to occur. 3890 */ 3891 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) 3892 { 3893 struct btrfs_trans_handle *trans; 3894 struct btrfs_root *root = BTRFS_I(dir)->root; 3895 int ret; 3896 3897 /* 3898 * 1 for the possible orphan item 3899 * 1 for the dir item 3900 * 1 for the dir index 3901 * 1 for the inode ref 3902 * 1 for the inode 3903 */ 3904 trans = btrfs_start_transaction(root, 5); 3905 if (!IS_ERR(trans) || PTR_ERR(trans) != -ENOSPC) 3906 return trans; 3907 3908 if (PTR_ERR(trans) == -ENOSPC) { 3909 u64 num_bytes = btrfs_calc_trans_metadata_size(root, 5); 3910 3911 trans = btrfs_start_transaction(root, 0); 3912 if (IS_ERR(trans)) 3913 return trans; 3914 ret = btrfs_cond_migrate_bytes(root->fs_info, 3915 &root->fs_info->trans_block_rsv, 3916 num_bytes, 5); 3917 if (ret) { 3918 btrfs_end_transaction(trans, root); 3919 return ERR_PTR(ret); 3920 } 3921 trans->block_rsv = &root->fs_info->trans_block_rsv; 3922 trans->bytes_reserved = num_bytes; 3923 } 3924 return trans; 3925 } 3926 3927 static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 3928 { 3929 struct btrfs_root *root = BTRFS_I(dir)->root; 3930 struct btrfs_trans_handle *trans; 3931 struct inode *inode = dentry->d_inode; 3932 int ret; 3933 3934 trans = __unlink_start_trans(dir); 3935 if (IS_ERR(trans)) 3936 return PTR_ERR(trans); 3937 3938 btrfs_record_unlink_dir(trans, dir, dentry->d_inode, 0); 3939 3940 ret = btrfs_unlink_inode(trans, root, dir, dentry->d_inode, 3941 dentry->d_name.name, dentry->d_name.len); 3942 if (ret) 3943 goto out; 3944 3945 if (inode->i_nlink == 0) { 3946 ret = btrfs_orphan_add(trans, inode); 3947 if (ret) 3948 goto out; 3949 } 3950 3951 out: 3952 btrfs_end_transaction(trans, root); 3953 btrfs_btree_balance_dirty(root); 3954 return ret; 3955 } 3956 3957 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 3958 struct btrfs_root *root, 3959 struct inode *dir, u64 objectid, 3960 const char *name, int name_len) 3961 { 3962 struct btrfs_path *path; 3963 struct extent_buffer *leaf; 3964 struct btrfs_dir_item *di; 3965 struct btrfs_key key; 3966 u64 index; 3967 int ret; 3968 u64 dir_ino = btrfs_ino(dir); 3969 3970 path = btrfs_alloc_path(); 3971 if (!path) 3972 return -ENOMEM; 3973 3974 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 3975 name, name_len, -1); 3976 if (IS_ERR_OR_NULL(di)) { 3977 if (!di) 3978 ret = -ENOENT; 3979 else 3980 ret = PTR_ERR(di); 3981 goto out; 3982 } 3983 3984 leaf = path->nodes[0]; 3985 btrfs_dir_item_key_to_cpu(leaf, di, &key); 3986 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 3987 ret = btrfs_delete_one_dir_name(trans, root, path, di); 3988 if (ret) { 3989 btrfs_abort_transaction(trans, root, ret); 3990 goto out; 3991 } 3992 btrfs_release_path(path); 3993 3994 ret = btrfs_del_root_ref(trans, root->fs_info->tree_root, 3995 objectid, root->root_key.objectid, 3996 dir_ino, &index, name, name_len); 3997 if (ret < 0) { 3998 if (ret != -ENOENT) { 3999 btrfs_abort_transaction(trans, root, ret); 4000 goto out; 4001 } 4002 di = btrfs_search_dir_index_item(root, path, dir_ino, 4003 name, name_len); 4004 if (IS_ERR_OR_NULL(di)) { 4005 if (!di) 4006 ret = -ENOENT; 4007 else 4008 ret = PTR_ERR(di); 4009 btrfs_abort_transaction(trans, root, ret); 4010 goto out; 4011 } 4012 4013 leaf = path->nodes[0]; 4014 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4015 btrfs_release_path(path); 4016 index = key.offset; 4017 } 4018 btrfs_release_path(path); 4019 4020 ret = btrfs_delete_delayed_dir_index(trans, root, dir, index); 4021 if (ret) { 4022 btrfs_abort_transaction(trans, root, ret); 4023 goto out; 4024 } 4025 4026 btrfs_i_size_write(dir, dir->i_size - name_len * 2); 4027 inode_inc_iversion(dir); 4028 dir->i_mtime = dir->i_ctime = CURRENT_TIME; 4029 ret = btrfs_update_inode_fallback(trans, root, dir); 4030 if (ret) 4031 btrfs_abort_transaction(trans, root, ret); 4032 out: 4033 btrfs_free_path(path); 4034 return ret; 4035 } 4036 4037 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) 4038 { 4039 struct inode *inode = dentry->d_inode; 4040 int err = 0; 4041 struct btrfs_root *root = BTRFS_I(dir)->root; 4042 struct btrfs_trans_handle *trans; 4043 4044 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE) 4045 return -ENOTEMPTY; 4046 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) 4047 return -EPERM; 4048 4049 trans = __unlink_start_trans(dir); 4050 if (IS_ERR(trans)) 4051 return PTR_ERR(trans); 4052 4053 if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4054 err = btrfs_unlink_subvol(trans, root, dir, 4055 BTRFS_I(inode)->location.objectid, 4056 dentry->d_name.name, 4057 dentry->d_name.len); 4058 goto out; 4059 } 4060 4061 err = btrfs_orphan_add(trans, inode); 4062 if (err) 4063 goto out; 4064 4065 /* now the directory is empty */ 4066 err = btrfs_unlink_inode(trans, root, dir, dentry->d_inode, 4067 dentry->d_name.name, dentry->d_name.len); 4068 if (!err) 4069 btrfs_i_size_write(inode, 0); 4070 out: 4071 btrfs_end_transaction(trans, root); 4072 btrfs_btree_balance_dirty(root); 4073 4074 return err; 4075 } 4076 4077 /* 4078 * this can truncate away extent items, csum items and directory items. 4079 * It starts at a high offset and removes keys until it can't find 4080 * any higher than new_size 4081 * 4082 * csum items that cross the new i_size are truncated to the new size 4083 * as well. 4084 * 4085 * min_type is the minimum key type to truncate down to. If set to 0, this 4086 * will kill all the items on this inode, including the INODE_ITEM_KEY. 4087 */ 4088 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, 4089 struct btrfs_root *root, 4090 struct inode *inode, 4091 u64 new_size, u32 min_type) 4092 { 4093 struct btrfs_path *path; 4094 struct extent_buffer *leaf; 4095 struct btrfs_file_extent_item *fi; 4096 struct btrfs_key key; 4097 struct btrfs_key found_key; 4098 u64 extent_start = 0; 4099 u64 extent_num_bytes = 0; 4100 u64 extent_offset = 0; 4101 u64 item_end = 0; 4102 u64 last_size = (u64)-1; 4103 u32 found_type = (u8)-1; 4104 int found_extent; 4105 int del_item; 4106 int pending_del_nr = 0; 4107 int pending_del_slot = 0; 4108 int extent_type = -1; 4109 int ret; 4110 int err = 0; 4111 u64 ino = btrfs_ino(inode); 4112 4113 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); 4114 4115 path = btrfs_alloc_path(); 4116 if (!path) 4117 return -ENOMEM; 4118 path->reada = -1; 4119 4120 /* 4121 * We want to drop from the next block forward in case this new size is 4122 * not block aligned since we will be keeping the last block of the 4123 * extent just the way it is. 4124 */ 4125 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4126 root == root->fs_info->tree_root) 4127 btrfs_drop_extent_cache(inode, ALIGN(new_size, 4128 root->sectorsize), (u64)-1, 0); 4129 4130 /* 4131 * This function is also used to drop the items in the log tree before 4132 * we relog the inode, so if root != BTRFS_I(inode)->root, it means 4133 * it is used to drop the loged items. So we shouldn't kill the delayed 4134 * items. 4135 */ 4136 if (min_type == 0 && root == BTRFS_I(inode)->root) 4137 btrfs_kill_delayed_inode_items(inode); 4138 4139 key.objectid = ino; 4140 key.offset = (u64)-1; 4141 key.type = (u8)-1; 4142 4143 search_again: 4144 path->leave_spinning = 1; 4145 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); 4146 if (ret < 0) { 4147 err = ret; 4148 goto out; 4149 } 4150 4151 if (ret > 0) { 4152 /* there are no items in the tree for us to truncate, we're 4153 * done 4154 */ 4155 if (path->slots[0] == 0) 4156 goto out; 4157 path->slots[0]--; 4158 } 4159 4160 while (1) { 4161 fi = NULL; 4162 leaf = path->nodes[0]; 4163 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4164 found_type = found_key.type; 4165 4166 if (found_key.objectid != ino) 4167 break; 4168 4169 if (found_type < min_type) 4170 break; 4171 4172 item_end = found_key.offset; 4173 if (found_type == BTRFS_EXTENT_DATA_KEY) { 4174 fi = btrfs_item_ptr(leaf, path->slots[0], 4175 struct btrfs_file_extent_item); 4176 extent_type = btrfs_file_extent_type(leaf, fi); 4177 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4178 item_end += 4179 btrfs_file_extent_num_bytes(leaf, fi); 4180 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4181 item_end += btrfs_file_extent_inline_len(leaf, 4182 path->slots[0], fi); 4183 } 4184 item_end--; 4185 } 4186 if (found_type > min_type) { 4187 del_item = 1; 4188 } else { 4189 if (item_end < new_size) 4190 break; 4191 if (found_key.offset >= new_size) 4192 del_item = 1; 4193 else 4194 del_item = 0; 4195 } 4196 found_extent = 0; 4197 /* FIXME, shrink the extent if the ref count is only 1 */ 4198 if (found_type != BTRFS_EXTENT_DATA_KEY) 4199 goto delete; 4200 4201 if (del_item) 4202 last_size = found_key.offset; 4203 else 4204 last_size = new_size; 4205 4206 if (extent_type != BTRFS_FILE_EXTENT_INLINE) { 4207 u64 num_dec; 4208 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); 4209 if (!del_item) { 4210 u64 orig_num_bytes = 4211 btrfs_file_extent_num_bytes(leaf, fi); 4212 extent_num_bytes = ALIGN(new_size - 4213 found_key.offset, 4214 root->sectorsize); 4215 btrfs_set_file_extent_num_bytes(leaf, fi, 4216 extent_num_bytes); 4217 num_dec = (orig_num_bytes - 4218 extent_num_bytes); 4219 if (test_bit(BTRFS_ROOT_REF_COWS, 4220 &root->state) && 4221 extent_start != 0) 4222 inode_sub_bytes(inode, num_dec); 4223 btrfs_mark_buffer_dirty(leaf); 4224 } else { 4225 extent_num_bytes = 4226 btrfs_file_extent_disk_num_bytes(leaf, 4227 fi); 4228 extent_offset = found_key.offset - 4229 btrfs_file_extent_offset(leaf, fi); 4230 4231 /* FIXME blocksize != 4096 */ 4232 num_dec = btrfs_file_extent_num_bytes(leaf, fi); 4233 if (extent_start != 0) { 4234 found_extent = 1; 4235 if (test_bit(BTRFS_ROOT_REF_COWS, 4236 &root->state)) 4237 inode_sub_bytes(inode, num_dec); 4238 } 4239 } 4240 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 4241 /* 4242 * we can't truncate inline items that have had 4243 * special encodings 4244 */ 4245 if (!del_item && 4246 btrfs_file_extent_compression(leaf, fi) == 0 && 4247 btrfs_file_extent_encryption(leaf, fi) == 0 && 4248 btrfs_file_extent_other_encoding(leaf, fi) == 0) { 4249 u32 size = new_size - found_key.offset; 4250 4251 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) 4252 inode_sub_bytes(inode, item_end + 1 - 4253 new_size); 4254 4255 /* 4256 * update the ram bytes to properly reflect 4257 * the new size of our item 4258 */ 4259 btrfs_set_file_extent_ram_bytes(leaf, fi, size); 4260 size = 4261 btrfs_file_extent_calc_inline_size(size); 4262 btrfs_truncate_item(root, path, size, 1); 4263 } else if (test_bit(BTRFS_ROOT_REF_COWS, 4264 &root->state)) { 4265 inode_sub_bytes(inode, item_end + 1 - 4266 found_key.offset); 4267 } 4268 } 4269 delete: 4270 if (del_item) { 4271 if (!pending_del_nr) { 4272 /* no pending yet, add ourselves */ 4273 pending_del_slot = path->slots[0]; 4274 pending_del_nr = 1; 4275 } else if (pending_del_nr && 4276 path->slots[0] + 1 == pending_del_slot) { 4277 /* hop on the pending chunk */ 4278 pending_del_nr++; 4279 pending_del_slot = path->slots[0]; 4280 } else { 4281 BUG(); 4282 } 4283 } else { 4284 break; 4285 } 4286 if (found_extent && 4287 (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || 4288 root == root->fs_info->tree_root)) { 4289 btrfs_set_path_blocking(path); 4290 ret = btrfs_free_extent(trans, root, extent_start, 4291 extent_num_bytes, 0, 4292 btrfs_header_owner(leaf), 4293 ino, extent_offset, 0); 4294 BUG_ON(ret); 4295 } 4296 4297 if (found_type == BTRFS_INODE_ITEM_KEY) 4298 break; 4299 4300 if (path->slots[0] == 0 || 4301 path->slots[0] != pending_del_slot) { 4302 if (pending_del_nr) { 4303 ret = btrfs_del_items(trans, root, path, 4304 pending_del_slot, 4305 pending_del_nr); 4306 if (ret) { 4307 btrfs_abort_transaction(trans, 4308 root, ret); 4309 goto error; 4310 } 4311 pending_del_nr = 0; 4312 } 4313 btrfs_release_path(path); 4314 goto search_again; 4315 } else { 4316 path->slots[0]--; 4317 } 4318 } 4319 out: 4320 if (pending_del_nr) { 4321 ret = btrfs_del_items(trans, root, path, pending_del_slot, 4322 pending_del_nr); 4323 if (ret) 4324 btrfs_abort_transaction(trans, root, ret); 4325 } 4326 error: 4327 if (last_size != (u64)-1 && 4328 root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) 4329 btrfs_ordered_update_i_size(inode, last_size, NULL); 4330 btrfs_free_path(path); 4331 return err; 4332 } 4333 4334 /* 4335 * btrfs_truncate_page - read, zero a chunk and write a page 4336 * @inode - inode that we're zeroing 4337 * @from - the offset to start zeroing 4338 * @len - the length to zero, 0 to zero the entire range respective to the 4339 * offset 4340 * @front - zero up to the offset instead of from the offset on 4341 * 4342 * This will find the page for the "from" offset and cow the page and zero the 4343 * part we want to zero. This is used with truncate and hole punching. 4344 */ 4345 int btrfs_truncate_page(struct inode *inode, loff_t from, loff_t len, 4346 int front) 4347 { 4348 struct address_space *mapping = inode->i_mapping; 4349 struct btrfs_root *root = BTRFS_I(inode)->root; 4350 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4351 struct btrfs_ordered_extent *ordered; 4352 struct extent_state *cached_state = NULL; 4353 char *kaddr; 4354 u32 blocksize = root->sectorsize; 4355 pgoff_t index = from >> PAGE_CACHE_SHIFT; 4356 unsigned offset = from & (PAGE_CACHE_SIZE-1); 4357 struct page *page; 4358 gfp_t mask = btrfs_alloc_write_mask(mapping); 4359 int ret = 0; 4360 u64 page_start; 4361 u64 page_end; 4362 4363 if ((offset & (blocksize - 1)) == 0 && 4364 (!len || ((len & (blocksize - 1)) == 0))) 4365 goto out; 4366 ret = btrfs_delalloc_reserve_space(inode, PAGE_CACHE_SIZE); 4367 if (ret) 4368 goto out; 4369 4370 again: 4371 page = find_or_create_page(mapping, index, mask); 4372 if (!page) { 4373 btrfs_delalloc_release_space(inode, PAGE_CACHE_SIZE); 4374 ret = -ENOMEM; 4375 goto out; 4376 } 4377 4378 page_start = page_offset(page); 4379 page_end = page_start + PAGE_CACHE_SIZE - 1; 4380 4381 if (!PageUptodate(page)) { 4382 ret = btrfs_readpage(NULL, page); 4383 lock_page(page); 4384 if (page->mapping != mapping) { 4385 unlock_page(page); 4386 page_cache_release(page); 4387 goto again; 4388 } 4389 if (!PageUptodate(page)) { 4390 ret = -EIO; 4391 goto out_unlock; 4392 } 4393 } 4394 wait_on_page_writeback(page); 4395 4396 lock_extent_bits(io_tree, page_start, page_end, 0, &cached_state); 4397 set_page_extent_mapped(page); 4398 4399 ordered = btrfs_lookup_ordered_extent(inode, page_start); 4400 if (ordered) { 4401 unlock_extent_cached(io_tree, page_start, page_end, 4402 &cached_state, GFP_NOFS); 4403 unlock_page(page); 4404 page_cache_release(page); 4405 btrfs_start_ordered_extent(inode, ordered, 1); 4406 btrfs_put_ordered_extent(ordered); 4407 goto again; 4408 } 4409 4410 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, page_end, 4411 EXTENT_DIRTY | EXTENT_DELALLOC | 4412 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 4413 0, 0, &cached_state, GFP_NOFS); 4414 4415 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 4416 &cached_state); 4417 if (ret) { 4418 unlock_extent_cached(io_tree, page_start, page_end, 4419 &cached_state, GFP_NOFS); 4420 goto out_unlock; 4421 } 4422 4423 if (offset != PAGE_CACHE_SIZE) { 4424 if (!len) 4425 len = PAGE_CACHE_SIZE - offset; 4426 kaddr = kmap(page); 4427 if (front) 4428 memset(kaddr, 0, offset); 4429 else 4430 memset(kaddr + offset, 0, len); 4431 flush_dcache_page(page); 4432 kunmap(page); 4433 } 4434 ClearPageChecked(page); 4435 set_page_dirty(page); 4436 unlock_extent_cached(io_tree, page_start, page_end, &cached_state, 4437 GFP_NOFS); 4438 4439 out_unlock: 4440 if (ret) 4441 btrfs_delalloc_release_space(inode, PAGE_CACHE_SIZE); 4442 unlock_page(page); 4443 page_cache_release(page); 4444 out: 4445 return ret; 4446 } 4447 4448 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode, 4449 u64 offset, u64 len) 4450 { 4451 struct btrfs_trans_handle *trans; 4452 int ret; 4453 4454 /* 4455 * Still need to make sure the inode looks like it's been updated so 4456 * that any holes get logged if we fsync. 4457 */ 4458 if (btrfs_fs_incompat(root->fs_info, NO_HOLES)) { 4459 BTRFS_I(inode)->last_trans = root->fs_info->generation; 4460 BTRFS_I(inode)->last_sub_trans = root->log_transid; 4461 BTRFS_I(inode)->last_log_commit = root->last_log_commit; 4462 return 0; 4463 } 4464 4465 /* 4466 * 1 - for the one we're dropping 4467 * 1 - for the one we're adding 4468 * 1 - for updating the inode. 4469 */ 4470 trans = btrfs_start_transaction(root, 3); 4471 if (IS_ERR(trans)) 4472 return PTR_ERR(trans); 4473 4474 ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1); 4475 if (ret) { 4476 btrfs_abort_transaction(trans, root, ret); 4477 btrfs_end_transaction(trans, root); 4478 return ret; 4479 } 4480 4481 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), offset, 4482 0, 0, len, 0, len, 0, 0, 0); 4483 if (ret) 4484 btrfs_abort_transaction(trans, root, ret); 4485 else 4486 btrfs_update_inode(trans, root, inode); 4487 btrfs_end_transaction(trans, root); 4488 return ret; 4489 } 4490 4491 /* 4492 * This function puts in dummy file extents for the area we're creating a hole 4493 * for. So if we are truncating this file to a larger size we need to insert 4494 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 4495 * the range between oldsize and size 4496 */ 4497 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size) 4498 { 4499 struct btrfs_root *root = BTRFS_I(inode)->root; 4500 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4501 struct extent_map *em = NULL; 4502 struct extent_state *cached_state = NULL; 4503 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 4504 u64 hole_start = ALIGN(oldsize, root->sectorsize); 4505 u64 block_end = ALIGN(size, root->sectorsize); 4506 u64 last_byte; 4507 u64 cur_offset; 4508 u64 hole_size; 4509 int err = 0; 4510 4511 /* 4512 * If our size started in the middle of a page we need to zero out the 4513 * rest of the page before we expand the i_size, otherwise we could 4514 * expose stale data. 4515 */ 4516 err = btrfs_truncate_page(inode, oldsize, 0, 0); 4517 if (err) 4518 return err; 4519 4520 if (size <= hole_start) 4521 return 0; 4522 4523 while (1) { 4524 struct btrfs_ordered_extent *ordered; 4525 4526 lock_extent_bits(io_tree, hole_start, block_end - 1, 0, 4527 &cached_state); 4528 ordered = btrfs_lookup_ordered_range(inode, hole_start, 4529 block_end - hole_start); 4530 if (!ordered) 4531 break; 4532 unlock_extent_cached(io_tree, hole_start, block_end - 1, 4533 &cached_state, GFP_NOFS); 4534 btrfs_start_ordered_extent(inode, ordered, 1); 4535 btrfs_put_ordered_extent(ordered); 4536 } 4537 4538 cur_offset = hole_start; 4539 while (1) { 4540 em = btrfs_get_extent(inode, NULL, 0, cur_offset, 4541 block_end - cur_offset, 0); 4542 if (IS_ERR(em)) { 4543 err = PTR_ERR(em); 4544 em = NULL; 4545 break; 4546 } 4547 last_byte = min(extent_map_end(em), block_end); 4548 last_byte = ALIGN(last_byte , root->sectorsize); 4549 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) { 4550 struct extent_map *hole_em; 4551 hole_size = last_byte - cur_offset; 4552 4553 err = maybe_insert_hole(root, inode, cur_offset, 4554 hole_size); 4555 if (err) 4556 break; 4557 btrfs_drop_extent_cache(inode, cur_offset, 4558 cur_offset + hole_size - 1, 0); 4559 hole_em = alloc_extent_map(); 4560 if (!hole_em) { 4561 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 4562 &BTRFS_I(inode)->runtime_flags); 4563 goto next; 4564 } 4565 hole_em->start = cur_offset; 4566 hole_em->len = hole_size; 4567 hole_em->orig_start = cur_offset; 4568 4569 hole_em->block_start = EXTENT_MAP_HOLE; 4570 hole_em->block_len = 0; 4571 hole_em->orig_block_len = 0; 4572 hole_em->ram_bytes = hole_size; 4573 hole_em->bdev = root->fs_info->fs_devices->latest_bdev; 4574 hole_em->compress_type = BTRFS_COMPRESS_NONE; 4575 hole_em->generation = root->fs_info->generation; 4576 4577 while (1) { 4578 write_lock(&em_tree->lock); 4579 err = add_extent_mapping(em_tree, hole_em, 1); 4580 write_unlock(&em_tree->lock); 4581 if (err != -EEXIST) 4582 break; 4583 btrfs_drop_extent_cache(inode, cur_offset, 4584 cur_offset + 4585 hole_size - 1, 0); 4586 } 4587 free_extent_map(hole_em); 4588 } 4589 next: 4590 free_extent_map(em); 4591 em = NULL; 4592 cur_offset = last_byte; 4593 if (cur_offset >= block_end) 4594 break; 4595 } 4596 free_extent_map(em); 4597 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state, 4598 GFP_NOFS); 4599 return err; 4600 } 4601 4602 static int wait_snapshoting_atomic_t(atomic_t *a) 4603 { 4604 schedule(); 4605 return 0; 4606 } 4607 4608 static void wait_for_snapshot_creation(struct btrfs_root *root) 4609 { 4610 while (true) { 4611 int ret; 4612 4613 ret = btrfs_start_write_no_snapshoting(root); 4614 if (ret) 4615 break; 4616 wait_on_atomic_t(&root->will_be_snapshoted, 4617 wait_snapshoting_atomic_t, 4618 TASK_UNINTERRUPTIBLE); 4619 } 4620 } 4621 4622 static int btrfs_setsize(struct inode *inode, struct iattr *attr) 4623 { 4624 struct btrfs_root *root = BTRFS_I(inode)->root; 4625 struct btrfs_trans_handle *trans; 4626 loff_t oldsize = i_size_read(inode); 4627 loff_t newsize = attr->ia_size; 4628 int mask = attr->ia_valid; 4629 int ret; 4630 4631 /* 4632 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 4633 * special case where we need to update the times despite not having 4634 * these flags set. For all other operations the VFS set these flags 4635 * explicitly if it wants a timestamp update. 4636 */ 4637 if (newsize != oldsize) { 4638 inode_inc_iversion(inode); 4639 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) 4640 inode->i_ctime = inode->i_mtime = 4641 current_fs_time(inode->i_sb); 4642 } 4643 4644 if (newsize > oldsize) { 4645 truncate_pagecache(inode, newsize); 4646 /* 4647 * Don't do an expanding truncate while snapshoting is ongoing. 4648 * This is to ensure the snapshot captures a fully consistent 4649 * state of this file - if the snapshot captures this expanding 4650 * truncation, it must capture all writes that happened before 4651 * this truncation. 4652 */ 4653 wait_for_snapshot_creation(root); 4654 ret = btrfs_cont_expand(inode, oldsize, newsize); 4655 if (ret) { 4656 btrfs_end_write_no_snapshoting(root); 4657 return ret; 4658 } 4659 4660 trans = btrfs_start_transaction(root, 1); 4661 if (IS_ERR(trans)) { 4662 btrfs_end_write_no_snapshoting(root); 4663 return PTR_ERR(trans); 4664 } 4665 4666 i_size_write(inode, newsize); 4667 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL); 4668 ret = btrfs_update_inode(trans, root, inode); 4669 btrfs_end_write_no_snapshoting(root); 4670 btrfs_end_transaction(trans, root); 4671 } else { 4672 4673 /* 4674 * We're truncating a file that used to have good data down to 4675 * zero. Make sure it gets into the ordered flush list so that 4676 * any new writes get down to disk quickly. 4677 */ 4678 if (newsize == 0) 4679 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE, 4680 &BTRFS_I(inode)->runtime_flags); 4681 4682 /* 4683 * 1 for the orphan item we're going to add 4684 * 1 for the orphan item deletion. 4685 */ 4686 trans = btrfs_start_transaction(root, 2); 4687 if (IS_ERR(trans)) 4688 return PTR_ERR(trans); 4689 4690 /* 4691 * We need to do this in case we fail at _any_ point during the 4692 * actual truncate. Once we do the truncate_setsize we could 4693 * invalidate pages which forces any outstanding ordered io to 4694 * be instantly completed which will give us extents that need 4695 * to be truncated. If we fail to get an orphan inode down we 4696 * could have left over extents that were never meant to live, 4697 * so we need to garuntee from this point on that everything 4698 * will be consistent. 4699 */ 4700 ret = btrfs_orphan_add(trans, inode); 4701 btrfs_end_transaction(trans, root); 4702 if (ret) 4703 return ret; 4704 4705 /* we don't support swapfiles, so vmtruncate shouldn't fail */ 4706 truncate_setsize(inode, newsize); 4707 4708 /* Disable nonlocked read DIO to avoid the end less truncate */ 4709 btrfs_inode_block_unlocked_dio(inode); 4710 inode_dio_wait(inode); 4711 btrfs_inode_resume_unlocked_dio(inode); 4712 4713 ret = btrfs_truncate(inode); 4714 if (ret && inode->i_nlink) { 4715 int err; 4716 4717 /* 4718 * failed to truncate, disk_i_size is only adjusted down 4719 * as we remove extents, so it should represent the true 4720 * size of the inode, so reset the in memory size and 4721 * delete our orphan entry. 4722 */ 4723 trans = btrfs_join_transaction(root); 4724 if (IS_ERR(trans)) { 4725 btrfs_orphan_del(NULL, inode); 4726 return ret; 4727 } 4728 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 4729 err = btrfs_orphan_del(trans, inode); 4730 if (err) 4731 btrfs_abort_transaction(trans, root, err); 4732 btrfs_end_transaction(trans, root); 4733 } 4734 } 4735 4736 return ret; 4737 } 4738 4739 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) 4740 { 4741 struct inode *inode = dentry->d_inode; 4742 struct btrfs_root *root = BTRFS_I(inode)->root; 4743 int err; 4744 4745 if (btrfs_root_readonly(root)) 4746 return -EROFS; 4747 4748 err = inode_change_ok(inode, attr); 4749 if (err) 4750 return err; 4751 4752 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 4753 err = btrfs_setsize(inode, attr); 4754 if (err) 4755 return err; 4756 } 4757 4758 if (attr->ia_valid) { 4759 setattr_copy(inode, attr); 4760 inode_inc_iversion(inode); 4761 err = btrfs_dirty_inode(inode); 4762 4763 if (!err && attr->ia_valid & ATTR_MODE) 4764 err = posix_acl_chmod(inode, inode->i_mode); 4765 } 4766 4767 return err; 4768 } 4769 4770 /* 4771 * While truncating the inode pages during eviction, we get the VFS calling 4772 * btrfs_invalidatepage() against each page of the inode. This is slow because 4773 * the calls to btrfs_invalidatepage() result in a huge amount of calls to 4774 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting 4775 * extent_state structures over and over, wasting lots of time. 4776 * 4777 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all 4778 * those expensive operations on a per page basis and do only the ordered io 4779 * finishing, while we release here the extent_map and extent_state structures, 4780 * without the excessive merging and splitting. 4781 */ 4782 static void evict_inode_truncate_pages(struct inode *inode) 4783 { 4784 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 4785 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; 4786 struct rb_node *node; 4787 4788 ASSERT(inode->i_state & I_FREEING); 4789 truncate_inode_pages_final(&inode->i_data); 4790 4791 write_lock(&map_tree->lock); 4792 while (!RB_EMPTY_ROOT(&map_tree->map)) { 4793 struct extent_map *em; 4794 4795 node = rb_first(&map_tree->map); 4796 em = rb_entry(node, struct extent_map, rb_node); 4797 clear_bit(EXTENT_FLAG_PINNED, &em->flags); 4798 clear_bit(EXTENT_FLAG_LOGGING, &em->flags); 4799 remove_extent_mapping(map_tree, em); 4800 free_extent_map(em); 4801 if (need_resched()) { 4802 write_unlock(&map_tree->lock); 4803 cond_resched(); 4804 write_lock(&map_tree->lock); 4805 } 4806 } 4807 write_unlock(&map_tree->lock); 4808 4809 spin_lock(&io_tree->lock); 4810 while (!RB_EMPTY_ROOT(&io_tree->state)) { 4811 struct extent_state *state; 4812 struct extent_state *cached_state = NULL; 4813 4814 node = rb_first(&io_tree->state); 4815 state = rb_entry(node, struct extent_state, rb_node); 4816 atomic_inc(&state->refs); 4817 spin_unlock(&io_tree->lock); 4818 4819 lock_extent_bits(io_tree, state->start, state->end, 4820 0, &cached_state); 4821 clear_extent_bit(io_tree, state->start, state->end, 4822 EXTENT_LOCKED | EXTENT_DIRTY | 4823 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 4824 EXTENT_DEFRAG, 1, 1, 4825 &cached_state, GFP_NOFS); 4826 free_extent_state(state); 4827 4828 cond_resched(); 4829 spin_lock(&io_tree->lock); 4830 } 4831 spin_unlock(&io_tree->lock); 4832 } 4833 4834 void btrfs_evict_inode(struct inode *inode) 4835 { 4836 struct btrfs_trans_handle *trans; 4837 struct btrfs_root *root = BTRFS_I(inode)->root; 4838 struct btrfs_block_rsv *rsv, *global_rsv; 4839 u64 min_size = btrfs_calc_trunc_metadata_size(root, 1); 4840 int ret; 4841 4842 trace_btrfs_inode_evict(inode); 4843 4844 evict_inode_truncate_pages(inode); 4845 4846 if (inode->i_nlink && 4847 ((btrfs_root_refs(&root->root_item) != 0 && 4848 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) || 4849 btrfs_is_free_space_inode(inode))) 4850 goto no_delete; 4851 4852 if (is_bad_inode(inode)) { 4853 btrfs_orphan_del(NULL, inode); 4854 goto no_delete; 4855 } 4856 /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */ 4857 btrfs_wait_ordered_range(inode, 0, (u64)-1); 4858 4859 btrfs_free_io_failure_record(inode, 0, (u64)-1); 4860 4861 if (root->fs_info->log_root_recovering) { 4862 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 4863 &BTRFS_I(inode)->runtime_flags)); 4864 goto no_delete; 4865 } 4866 4867 if (inode->i_nlink > 0) { 4868 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 4869 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID); 4870 goto no_delete; 4871 } 4872 4873 ret = btrfs_commit_inode_delayed_inode(inode); 4874 if (ret) { 4875 btrfs_orphan_del(NULL, inode); 4876 goto no_delete; 4877 } 4878 4879 rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP); 4880 if (!rsv) { 4881 btrfs_orphan_del(NULL, inode); 4882 goto no_delete; 4883 } 4884 rsv->size = min_size; 4885 rsv->failfast = 1; 4886 global_rsv = &root->fs_info->global_block_rsv; 4887 4888 btrfs_i_size_write(inode, 0); 4889 4890 /* 4891 * This is a bit simpler than btrfs_truncate since we've already 4892 * reserved our space for our orphan item in the unlink, so we just 4893 * need to reserve some slack space in case we add bytes and update 4894 * inode item when doing the truncate. 4895 */ 4896 while (1) { 4897 ret = btrfs_block_rsv_refill(root, rsv, min_size, 4898 BTRFS_RESERVE_FLUSH_LIMIT); 4899 4900 /* 4901 * Try and steal from the global reserve since we will 4902 * likely not use this space anyway, we want to try as 4903 * hard as possible to get this to work. 4904 */ 4905 if (ret) 4906 ret = btrfs_block_rsv_migrate(global_rsv, rsv, min_size); 4907 4908 if (ret) { 4909 btrfs_warn(root->fs_info, 4910 "Could not get space for a delete, will truncate on mount %d", 4911 ret); 4912 btrfs_orphan_del(NULL, inode); 4913 btrfs_free_block_rsv(root, rsv); 4914 goto no_delete; 4915 } 4916 4917 trans = btrfs_join_transaction(root); 4918 if (IS_ERR(trans)) { 4919 btrfs_orphan_del(NULL, inode); 4920 btrfs_free_block_rsv(root, rsv); 4921 goto no_delete; 4922 } 4923 4924 trans->block_rsv = rsv; 4925 4926 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0); 4927 if (ret != -ENOSPC) 4928 break; 4929 4930 trans->block_rsv = &root->fs_info->trans_block_rsv; 4931 btrfs_end_transaction(trans, root); 4932 trans = NULL; 4933 btrfs_btree_balance_dirty(root); 4934 } 4935 4936 btrfs_free_block_rsv(root, rsv); 4937 4938 /* 4939 * Errors here aren't a big deal, it just means we leave orphan items 4940 * in the tree. They will be cleaned up on the next mount. 4941 */ 4942 if (ret == 0) { 4943 trans->block_rsv = root->orphan_block_rsv; 4944 btrfs_orphan_del(trans, inode); 4945 } else { 4946 btrfs_orphan_del(NULL, inode); 4947 } 4948 4949 trans->block_rsv = &root->fs_info->trans_block_rsv; 4950 if (!(root == root->fs_info->tree_root || 4951 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)) 4952 btrfs_return_ino(root, btrfs_ino(inode)); 4953 4954 btrfs_end_transaction(trans, root); 4955 btrfs_btree_balance_dirty(root); 4956 no_delete: 4957 btrfs_remove_delayed_node(inode); 4958 clear_inode(inode); 4959 return; 4960 } 4961 4962 /* 4963 * this returns the key found in the dir entry in the location pointer. 4964 * If no dir entries were found, location->objectid is 0. 4965 */ 4966 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry, 4967 struct btrfs_key *location) 4968 { 4969 const char *name = dentry->d_name.name; 4970 int namelen = dentry->d_name.len; 4971 struct btrfs_dir_item *di; 4972 struct btrfs_path *path; 4973 struct btrfs_root *root = BTRFS_I(dir)->root; 4974 int ret = 0; 4975 4976 path = btrfs_alloc_path(); 4977 if (!path) 4978 return -ENOMEM; 4979 4980 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), name, 4981 namelen, 0); 4982 if (IS_ERR(di)) 4983 ret = PTR_ERR(di); 4984 4985 if (IS_ERR_OR_NULL(di)) 4986 goto out_err; 4987 4988 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 4989 out: 4990 btrfs_free_path(path); 4991 return ret; 4992 out_err: 4993 location->objectid = 0; 4994 goto out; 4995 } 4996 4997 /* 4998 * when we hit a tree root in a directory, the btrfs part of the inode 4999 * needs to be changed to reflect the root directory of the tree root. This 5000 * is kind of like crossing a mount point. 5001 */ 5002 static int fixup_tree_root_location(struct btrfs_root *root, 5003 struct inode *dir, 5004 struct dentry *dentry, 5005 struct btrfs_key *location, 5006 struct btrfs_root **sub_root) 5007 { 5008 struct btrfs_path *path; 5009 struct btrfs_root *new_root; 5010 struct btrfs_root_ref *ref; 5011 struct extent_buffer *leaf; 5012 int ret; 5013 int err = 0; 5014 5015 path = btrfs_alloc_path(); 5016 if (!path) { 5017 err = -ENOMEM; 5018 goto out; 5019 } 5020 5021 err = -ENOENT; 5022 ret = btrfs_find_item(root->fs_info->tree_root, path, 5023 BTRFS_I(dir)->root->root_key.objectid, 5024 location->objectid, BTRFS_ROOT_REF_KEY, NULL); 5025 if (ret) { 5026 if (ret < 0) 5027 err = ret; 5028 goto out; 5029 } 5030 5031 leaf = path->nodes[0]; 5032 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5033 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) || 5034 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len) 5035 goto out; 5036 5037 ret = memcmp_extent_buffer(leaf, dentry->d_name.name, 5038 (unsigned long)(ref + 1), 5039 dentry->d_name.len); 5040 if (ret) 5041 goto out; 5042 5043 btrfs_release_path(path); 5044 5045 new_root = btrfs_read_fs_root_no_name(root->fs_info, location); 5046 if (IS_ERR(new_root)) { 5047 err = PTR_ERR(new_root); 5048 goto out; 5049 } 5050 5051 *sub_root = new_root; 5052 location->objectid = btrfs_root_dirid(&new_root->root_item); 5053 location->type = BTRFS_INODE_ITEM_KEY; 5054 location->offset = 0; 5055 err = 0; 5056 out: 5057 btrfs_free_path(path); 5058 return err; 5059 } 5060 5061 static void inode_tree_add(struct inode *inode) 5062 { 5063 struct btrfs_root *root = BTRFS_I(inode)->root; 5064 struct btrfs_inode *entry; 5065 struct rb_node **p; 5066 struct rb_node *parent; 5067 struct rb_node *new = &BTRFS_I(inode)->rb_node; 5068 u64 ino = btrfs_ino(inode); 5069 5070 if (inode_unhashed(inode)) 5071 return; 5072 parent = NULL; 5073 spin_lock(&root->inode_lock); 5074 p = &root->inode_tree.rb_node; 5075 while (*p) { 5076 parent = *p; 5077 entry = rb_entry(parent, struct btrfs_inode, rb_node); 5078 5079 if (ino < btrfs_ino(&entry->vfs_inode)) 5080 p = &parent->rb_left; 5081 else if (ino > btrfs_ino(&entry->vfs_inode)) 5082 p = &parent->rb_right; 5083 else { 5084 WARN_ON(!(entry->vfs_inode.i_state & 5085 (I_WILL_FREE | I_FREEING))); 5086 rb_replace_node(parent, new, &root->inode_tree); 5087 RB_CLEAR_NODE(parent); 5088 spin_unlock(&root->inode_lock); 5089 return; 5090 } 5091 } 5092 rb_link_node(new, parent, p); 5093 rb_insert_color(new, &root->inode_tree); 5094 spin_unlock(&root->inode_lock); 5095 } 5096 5097 static void inode_tree_del(struct inode *inode) 5098 { 5099 struct btrfs_root *root = BTRFS_I(inode)->root; 5100 int empty = 0; 5101 5102 spin_lock(&root->inode_lock); 5103 if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) { 5104 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree); 5105 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node); 5106 empty = RB_EMPTY_ROOT(&root->inode_tree); 5107 } 5108 spin_unlock(&root->inode_lock); 5109 5110 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5111 synchronize_srcu(&root->fs_info->subvol_srcu); 5112 spin_lock(&root->inode_lock); 5113 empty = RB_EMPTY_ROOT(&root->inode_tree); 5114 spin_unlock(&root->inode_lock); 5115 if (empty) 5116 btrfs_add_dead_root(root); 5117 } 5118 } 5119 5120 void btrfs_invalidate_inodes(struct btrfs_root *root) 5121 { 5122 struct rb_node *node; 5123 struct rb_node *prev; 5124 struct btrfs_inode *entry; 5125 struct inode *inode; 5126 u64 objectid = 0; 5127 5128 if (!test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state)) 5129 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 5130 5131 spin_lock(&root->inode_lock); 5132 again: 5133 node = root->inode_tree.rb_node; 5134 prev = NULL; 5135 while (node) { 5136 prev = node; 5137 entry = rb_entry(node, struct btrfs_inode, rb_node); 5138 5139 if (objectid < btrfs_ino(&entry->vfs_inode)) 5140 node = node->rb_left; 5141 else if (objectid > btrfs_ino(&entry->vfs_inode)) 5142 node = node->rb_right; 5143 else 5144 break; 5145 } 5146 if (!node) { 5147 while (prev) { 5148 entry = rb_entry(prev, struct btrfs_inode, rb_node); 5149 if (objectid <= btrfs_ino(&entry->vfs_inode)) { 5150 node = prev; 5151 break; 5152 } 5153 prev = rb_next(prev); 5154 } 5155 } 5156 while (node) { 5157 entry = rb_entry(node, struct btrfs_inode, rb_node); 5158 objectid = btrfs_ino(&entry->vfs_inode) + 1; 5159 inode = igrab(&entry->vfs_inode); 5160 if (inode) { 5161 spin_unlock(&root->inode_lock); 5162 if (atomic_read(&inode->i_count) > 1) 5163 d_prune_aliases(inode); 5164 /* 5165 * btrfs_drop_inode will have it removed from 5166 * the inode cache when its usage count 5167 * hits zero. 5168 */ 5169 iput(inode); 5170 cond_resched(); 5171 spin_lock(&root->inode_lock); 5172 goto again; 5173 } 5174 5175 if (cond_resched_lock(&root->inode_lock)) 5176 goto again; 5177 5178 node = rb_next(node); 5179 } 5180 spin_unlock(&root->inode_lock); 5181 } 5182 5183 static int btrfs_init_locked_inode(struct inode *inode, void *p) 5184 { 5185 struct btrfs_iget_args *args = p; 5186 inode->i_ino = args->location->objectid; 5187 memcpy(&BTRFS_I(inode)->location, args->location, 5188 sizeof(*args->location)); 5189 BTRFS_I(inode)->root = args->root; 5190 return 0; 5191 } 5192 5193 static int btrfs_find_actor(struct inode *inode, void *opaque) 5194 { 5195 struct btrfs_iget_args *args = opaque; 5196 return args->location->objectid == BTRFS_I(inode)->location.objectid && 5197 args->root == BTRFS_I(inode)->root; 5198 } 5199 5200 static struct inode *btrfs_iget_locked(struct super_block *s, 5201 struct btrfs_key *location, 5202 struct btrfs_root *root) 5203 { 5204 struct inode *inode; 5205 struct btrfs_iget_args args; 5206 unsigned long hashval = btrfs_inode_hash(location->objectid, root); 5207 5208 args.location = location; 5209 args.root = root; 5210 5211 inode = iget5_locked(s, hashval, btrfs_find_actor, 5212 btrfs_init_locked_inode, 5213 (void *)&args); 5214 return inode; 5215 } 5216 5217 /* Get an inode object given its location and corresponding root. 5218 * Returns in *is_new if the inode was read from disk 5219 */ 5220 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location, 5221 struct btrfs_root *root, int *new) 5222 { 5223 struct inode *inode; 5224 5225 inode = btrfs_iget_locked(s, location, root); 5226 if (!inode) 5227 return ERR_PTR(-ENOMEM); 5228 5229 if (inode->i_state & I_NEW) { 5230 btrfs_read_locked_inode(inode); 5231 if (!is_bad_inode(inode)) { 5232 inode_tree_add(inode); 5233 unlock_new_inode(inode); 5234 if (new) 5235 *new = 1; 5236 } else { 5237 unlock_new_inode(inode); 5238 iput(inode); 5239 inode = ERR_PTR(-ESTALE); 5240 } 5241 } 5242 5243 return inode; 5244 } 5245 5246 static struct inode *new_simple_dir(struct super_block *s, 5247 struct btrfs_key *key, 5248 struct btrfs_root *root) 5249 { 5250 struct inode *inode = new_inode(s); 5251 5252 if (!inode) 5253 return ERR_PTR(-ENOMEM); 5254 5255 BTRFS_I(inode)->root = root; 5256 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key)); 5257 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); 5258 5259 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; 5260 inode->i_op = &btrfs_dir_ro_inode_operations; 5261 inode->i_fop = &simple_dir_operations; 5262 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5263 inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; 5264 5265 return inode; 5266 } 5267 5268 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5269 { 5270 struct inode *inode; 5271 struct btrfs_root *root = BTRFS_I(dir)->root; 5272 struct btrfs_root *sub_root = root; 5273 struct btrfs_key location; 5274 int index; 5275 int ret = 0; 5276 5277 if (dentry->d_name.len > BTRFS_NAME_LEN) 5278 return ERR_PTR(-ENAMETOOLONG); 5279 5280 ret = btrfs_inode_by_name(dir, dentry, &location); 5281 if (ret < 0) 5282 return ERR_PTR(ret); 5283 5284 if (location.objectid == 0) 5285 return ERR_PTR(-ENOENT); 5286 5287 if (location.type == BTRFS_INODE_ITEM_KEY) { 5288 inode = btrfs_iget(dir->i_sb, &location, root, NULL); 5289 return inode; 5290 } 5291 5292 BUG_ON(location.type != BTRFS_ROOT_ITEM_KEY); 5293 5294 index = srcu_read_lock(&root->fs_info->subvol_srcu); 5295 ret = fixup_tree_root_location(root, dir, dentry, 5296 &location, &sub_root); 5297 if (ret < 0) { 5298 if (ret != -ENOENT) 5299 inode = ERR_PTR(ret); 5300 else 5301 inode = new_simple_dir(dir->i_sb, &location, sub_root); 5302 } else { 5303 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL); 5304 } 5305 srcu_read_unlock(&root->fs_info->subvol_srcu, index); 5306 5307 if (!IS_ERR(inode) && root != sub_root) { 5308 down_read(&root->fs_info->cleanup_work_sem); 5309 if (!(inode->i_sb->s_flags & MS_RDONLY)) 5310 ret = btrfs_orphan_cleanup(sub_root); 5311 up_read(&root->fs_info->cleanup_work_sem); 5312 if (ret) { 5313 iput(inode); 5314 inode = ERR_PTR(ret); 5315 } 5316 } 5317 5318 return inode; 5319 } 5320 5321 static int btrfs_dentry_delete(const struct dentry *dentry) 5322 { 5323 struct btrfs_root *root; 5324 struct inode *inode = dentry->d_inode; 5325 5326 if (!inode && !IS_ROOT(dentry)) 5327 inode = dentry->d_parent->d_inode; 5328 5329 if (inode) { 5330 root = BTRFS_I(inode)->root; 5331 if (btrfs_root_refs(&root->root_item) == 0) 5332 return 1; 5333 5334 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 5335 return 1; 5336 } 5337 return 0; 5338 } 5339 5340 static void btrfs_dentry_release(struct dentry *dentry) 5341 { 5342 kfree(dentry->d_fsdata); 5343 } 5344 5345 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 5346 unsigned int flags) 5347 { 5348 struct inode *inode; 5349 5350 inode = btrfs_lookup_dentry(dir, dentry); 5351 if (IS_ERR(inode)) { 5352 if (PTR_ERR(inode) == -ENOENT) 5353 inode = NULL; 5354 else 5355 return ERR_CAST(inode); 5356 } 5357 5358 return d_splice_alias(inode, dentry); 5359 } 5360 5361 unsigned char btrfs_filetype_table[] = { 5362 DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK 5363 }; 5364 5365 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 5366 { 5367 struct inode *inode = file_inode(file); 5368 struct btrfs_root *root = BTRFS_I(inode)->root; 5369 struct btrfs_item *item; 5370 struct btrfs_dir_item *di; 5371 struct btrfs_key key; 5372 struct btrfs_key found_key; 5373 struct btrfs_path *path; 5374 struct list_head ins_list; 5375 struct list_head del_list; 5376 int ret; 5377 struct extent_buffer *leaf; 5378 int slot; 5379 unsigned char d_type; 5380 int over = 0; 5381 u32 di_cur; 5382 u32 di_total; 5383 u32 di_len; 5384 int key_type = BTRFS_DIR_INDEX_KEY; 5385 char tmp_name[32]; 5386 char *name_ptr; 5387 int name_len; 5388 int is_curr = 0; /* ctx->pos points to the current index? */ 5389 5390 /* FIXME, use a real flag for deciding about the key type */ 5391 if (root->fs_info->tree_root == root) 5392 key_type = BTRFS_DIR_ITEM_KEY; 5393 5394 if (!dir_emit_dots(file, ctx)) 5395 return 0; 5396 5397 path = btrfs_alloc_path(); 5398 if (!path) 5399 return -ENOMEM; 5400 5401 path->reada = 1; 5402 5403 if (key_type == BTRFS_DIR_INDEX_KEY) { 5404 INIT_LIST_HEAD(&ins_list); 5405 INIT_LIST_HEAD(&del_list); 5406 btrfs_get_delayed_items(inode, &ins_list, &del_list); 5407 } 5408 5409 key.type = key_type; 5410 key.offset = ctx->pos; 5411 key.objectid = btrfs_ino(inode); 5412 5413 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5414 if (ret < 0) 5415 goto err; 5416 5417 while (1) { 5418 leaf = path->nodes[0]; 5419 slot = path->slots[0]; 5420 if (slot >= btrfs_header_nritems(leaf)) { 5421 ret = btrfs_next_leaf(root, path); 5422 if (ret < 0) 5423 goto err; 5424 else if (ret > 0) 5425 break; 5426 continue; 5427 } 5428 5429 item = btrfs_item_nr(slot); 5430 btrfs_item_key_to_cpu(leaf, &found_key, slot); 5431 5432 if (found_key.objectid != key.objectid) 5433 break; 5434 if (found_key.type != key_type) 5435 break; 5436 if (found_key.offset < ctx->pos) 5437 goto next; 5438 if (key_type == BTRFS_DIR_INDEX_KEY && 5439 btrfs_should_delete_dir_index(&del_list, 5440 found_key.offset)) 5441 goto next; 5442 5443 ctx->pos = found_key.offset; 5444 is_curr = 1; 5445 5446 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); 5447 di_cur = 0; 5448 di_total = btrfs_item_size(leaf, item); 5449 5450 while (di_cur < di_total) { 5451 struct btrfs_key location; 5452 5453 if (verify_dir_item(root, leaf, di)) 5454 break; 5455 5456 name_len = btrfs_dir_name_len(leaf, di); 5457 if (name_len <= sizeof(tmp_name)) { 5458 name_ptr = tmp_name; 5459 } else { 5460 name_ptr = kmalloc(name_len, GFP_NOFS); 5461 if (!name_ptr) { 5462 ret = -ENOMEM; 5463 goto err; 5464 } 5465 } 5466 read_extent_buffer(leaf, name_ptr, 5467 (unsigned long)(di + 1), name_len); 5468 5469 d_type = btrfs_filetype_table[btrfs_dir_type(leaf, di)]; 5470 btrfs_dir_item_key_to_cpu(leaf, di, &location); 5471 5472 5473 /* is this a reference to our own snapshot? If so 5474 * skip it. 5475 * 5476 * In contrast to old kernels, we insert the snapshot's 5477 * dir item and dir index after it has been created, so 5478 * we won't find a reference to our own snapshot. We 5479 * still keep the following code for backward 5480 * compatibility. 5481 */ 5482 if (location.type == BTRFS_ROOT_ITEM_KEY && 5483 location.objectid == root->root_key.objectid) { 5484 over = 0; 5485 goto skip; 5486 } 5487 over = !dir_emit(ctx, name_ptr, name_len, 5488 location.objectid, d_type); 5489 5490 skip: 5491 if (name_ptr != tmp_name) 5492 kfree(name_ptr); 5493 5494 if (over) 5495 goto nopos; 5496 di_len = btrfs_dir_name_len(leaf, di) + 5497 btrfs_dir_data_len(leaf, di) + sizeof(*di); 5498 di_cur += di_len; 5499 di = (struct btrfs_dir_item *)((char *)di + di_len); 5500 } 5501 next: 5502 path->slots[0]++; 5503 } 5504 5505 if (key_type == BTRFS_DIR_INDEX_KEY) { 5506 if (is_curr) 5507 ctx->pos++; 5508 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list); 5509 if (ret) 5510 goto nopos; 5511 } 5512 5513 /* Reached end of directory/root. Bump pos past the last item. */ 5514 ctx->pos++; 5515 5516 /* 5517 * Stop new entries from being returned after we return the last 5518 * entry. 5519 * 5520 * New directory entries are assigned a strictly increasing 5521 * offset. This means that new entries created during readdir 5522 * are *guaranteed* to be seen in the future by that readdir. 5523 * This has broken buggy programs which operate on names as 5524 * they're returned by readdir. Until we re-use freed offsets 5525 * we have this hack to stop new entries from being returned 5526 * under the assumption that they'll never reach this huge 5527 * offset. 5528 * 5529 * This is being careful not to overflow 32bit loff_t unless the 5530 * last entry requires it because doing so has broken 32bit apps 5531 * in the past. 5532 */ 5533 if (key_type == BTRFS_DIR_INDEX_KEY) { 5534 if (ctx->pos >= INT_MAX) 5535 ctx->pos = LLONG_MAX; 5536 else 5537 ctx->pos = INT_MAX; 5538 } 5539 nopos: 5540 ret = 0; 5541 err: 5542 if (key_type == BTRFS_DIR_INDEX_KEY) 5543 btrfs_put_delayed_items(&ins_list, &del_list); 5544 btrfs_free_path(path); 5545 return ret; 5546 } 5547 5548 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc) 5549 { 5550 struct btrfs_root *root = BTRFS_I(inode)->root; 5551 struct btrfs_trans_handle *trans; 5552 int ret = 0; 5553 bool nolock = false; 5554 5555 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 5556 return 0; 5557 5558 if (btrfs_fs_closing(root->fs_info) && btrfs_is_free_space_inode(inode)) 5559 nolock = true; 5560 5561 if (wbc->sync_mode == WB_SYNC_ALL) { 5562 if (nolock) 5563 trans = btrfs_join_transaction_nolock(root); 5564 else 5565 trans = btrfs_join_transaction(root); 5566 if (IS_ERR(trans)) 5567 return PTR_ERR(trans); 5568 ret = btrfs_commit_transaction(trans, root); 5569 } 5570 return ret; 5571 } 5572 5573 /* 5574 * This is somewhat expensive, updating the tree every time the 5575 * inode changes. But, it is most likely to find the inode in cache. 5576 * FIXME, needs more benchmarking...there are no reasons other than performance 5577 * to keep or drop this code. 5578 */ 5579 static int btrfs_dirty_inode(struct inode *inode) 5580 { 5581 struct btrfs_root *root = BTRFS_I(inode)->root; 5582 struct btrfs_trans_handle *trans; 5583 int ret; 5584 5585 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags)) 5586 return 0; 5587 5588 trans = btrfs_join_transaction(root); 5589 if (IS_ERR(trans)) 5590 return PTR_ERR(trans); 5591 5592 ret = btrfs_update_inode(trans, root, inode); 5593 if (ret && ret == -ENOSPC) { 5594 /* whoops, lets try again with the full transaction */ 5595 btrfs_end_transaction(trans, root); 5596 trans = btrfs_start_transaction(root, 1); 5597 if (IS_ERR(trans)) 5598 return PTR_ERR(trans); 5599 5600 ret = btrfs_update_inode(trans, root, inode); 5601 } 5602 btrfs_end_transaction(trans, root); 5603 if (BTRFS_I(inode)->delayed_node) 5604 btrfs_balance_delayed_items(root); 5605 5606 return ret; 5607 } 5608 5609 /* 5610 * This is a copy of file_update_time. We need this so we can return error on 5611 * ENOSPC for updating the inode in the case of file write and mmap writes. 5612 */ 5613 static int btrfs_update_time(struct inode *inode, struct timespec *now, 5614 int flags) 5615 { 5616 struct btrfs_root *root = BTRFS_I(inode)->root; 5617 5618 if (btrfs_root_readonly(root)) 5619 return -EROFS; 5620 5621 if (flags & S_VERSION) 5622 inode_inc_iversion(inode); 5623 if (flags & S_CTIME) 5624 inode->i_ctime = *now; 5625 if (flags & S_MTIME) 5626 inode->i_mtime = *now; 5627 if (flags & S_ATIME) 5628 inode->i_atime = *now; 5629 return btrfs_dirty_inode(inode); 5630 } 5631 5632 /* 5633 * find the highest existing sequence number in a directory 5634 * and then set the in-memory index_cnt variable to reflect 5635 * free sequence numbers 5636 */ 5637 static int btrfs_set_inode_index_count(struct inode *inode) 5638 { 5639 struct btrfs_root *root = BTRFS_I(inode)->root; 5640 struct btrfs_key key, found_key; 5641 struct btrfs_path *path; 5642 struct extent_buffer *leaf; 5643 int ret; 5644 5645 key.objectid = btrfs_ino(inode); 5646 key.type = BTRFS_DIR_INDEX_KEY; 5647 key.offset = (u64)-1; 5648 5649 path = btrfs_alloc_path(); 5650 if (!path) 5651 return -ENOMEM; 5652 5653 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5654 if (ret < 0) 5655 goto out; 5656 /* FIXME: we should be able to handle this */ 5657 if (ret == 0) 5658 goto out; 5659 ret = 0; 5660 5661 /* 5662 * MAGIC NUMBER EXPLANATION: 5663 * since we search a directory based on f_pos we have to start at 2 5664 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody 5665 * else has to start at 2 5666 */ 5667 if (path->slots[0] == 0) { 5668 BTRFS_I(inode)->index_cnt = 2; 5669 goto out; 5670 } 5671 5672 path->slots[0]--; 5673 5674 leaf = path->nodes[0]; 5675 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5676 5677 if (found_key.objectid != btrfs_ino(inode) || 5678 found_key.type != BTRFS_DIR_INDEX_KEY) { 5679 BTRFS_I(inode)->index_cnt = 2; 5680 goto out; 5681 } 5682 5683 BTRFS_I(inode)->index_cnt = found_key.offset + 1; 5684 out: 5685 btrfs_free_path(path); 5686 return ret; 5687 } 5688 5689 /* 5690 * helper to find a free sequence number in a given directory. This current 5691 * code is very simple, later versions will do smarter things in the btree 5692 */ 5693 int btrfs_set_inode_index(struct inode *dir, u64 *index) 5694 { 5695 int ret = 0; 5696 5697 if (BTRFS_I(dir)->index_cnt == (u64)-1) { 5698 ret = btrfs_inode_delayed_dir_index_count(dir); 5699 if (ret) { 5700 ret = btrfs_set_inode_index_count(dir); 5701 if (ret) 5702 return ret; 5703 } 5704 } 5705 5706 *index = BTRFS_I(dir)->index_cnt; 5707 BTRFS_I(dir)->index_cnt++; 5708 5709 return ret; 5710 } 5711 5712 static int btrfs_insert_inode_locked(struct inode *inode) 5713 { 5714 struct btrfs_iget_args args; 5715 args.location = &BTRFS_I(inode)->location; 5716 args.root = BTRFS_I(inode)->root; 5717 5718 return insert_inode_locked4(inode, 5719 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 5720 btrfs_find_actor, &args); 5721 } 5722 5723 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, 5724 struct btrfs_root *root, 5725 struct inode *dir, 5726 const char *name, int name_len, 5727 u64 ref_objectid, u64 objectid, 5728 umode_t mode, u64 *index) 5729 { 5730 struct inode *inode; 5731 struct btrfs_inode_item *inode_item; 5732 struct btrfs_key *location; 5733 struct btrfs_path *path; 5734 struct btrfs_inode_ref *ref; 5735 struct btrfs_key key[2]; 5736 u32 sizes[2]; 5737 int nitems = name ? 2 : 1; 5738 unsigned long ptr; 5739 int ret; 5740 5741 path = btrfs_alloc_path(); 5742 if (!path) 5743 return ERR_PTR(-ENOMEM); 5744 5745 inode = new_inode(root->fs_info->sb); 5746 if (!inode) { 5747 btrfs_free_path(path); 5748 return ERR_PTR(-ENOMEM); 5749 } 5750 5751 /* 5752 * O_TMPFILE, set link count to 0, so that after this point, 5753 * we fill in an inode item with the correct link count. 5754 */ 5755 if (!name) 5756 set_nlink(inode, 0); 5757 5758 /* 5759 * we have to initialize this early, so we can reclaim the inode 5760 * number if we fail afterwards in this function. 5761 */ 5762 inode->i_ino = objectid; 5763 5764 if (dir && name) { 5765 trace_btrfs_inode_request(dir); 5766 5767 ret = btrfs_set_inode_index(dir, index); 5768 if (ret) { 5769 btrfs_free_path(path); 5770 iput(inode); 5771 return ERR_PTR(ret); 5772 } 5773 } else if (dir) { 5774 *index = 0; 5775 } 5776 /* 5777 * index_cnt is ignored for everything but a dir, 5778 * btrfs_get_inode_index_count has an explanation for the magic 5779 * number 5780 */ 5781 BTRFS_I(inode)->index_cnt = 2; 5782 BTRFS_I(inode)->dir_index = *index; 5783 BTRFS_I(inode)->root = root; 5784 BTRFS_I(inode)->generation = trans->transid; 5785 inode->i_generation = BTRFS_I(inode)->generation; 5786 5787 /* 5788 * We could have gotten an inode number from somebody who was fsynced 5789 * and then removed in this same transaction, so let's just set full 5790 * sync since it will be a full sync anyway and this will blow away the 5791 * old info in the log. 5792 */ 5793 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 5794 5795 key[0].objectid = objectid; 5796 key[0].type = BTRFS_INODE_ITEM_KEY; 5797 key[0].offset = 0; 5798 5799 sizes[0] = sizeof(struct btrfs_inode_item); 5800 5801 if (name) { 5802 /* 5803 * Start new inodes with an inode_ref. This is slightly more 5804 * efficient for small numbers of hard links since they will 5805 * be packed into one item. Extended refs will kick in if we 5806 * add more hard links than can fit in the ref item. 5807 */ 5808 key[1].objectid = objectid; 5809 key[1].type = BTRFS_INODE_REF_KEY; 5810 key[1].offset = ref_objectid; 5811 5812 sizes[1] = name_len + sizeof(*ref); 5813 } 5814 5815 location = &BTRFS_I(inode)->location; 5816 location->objectid = objectid; 5817 location->offset = 0; 5818 location->type = BTRFS_INODE_ITEM_KEY; 5819 5820 ret = btrfs_insert_inode_locked(inode); 5821 if (ret < 0) 5822 goto fail; 5823 5824 path->leave_spinning = 1; 5825 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems); 5826 if (ret != 0) 5827 goto fail_unlock; 5828 5829 inode_init_owner(inode, dir, mode); 5830 inode_set_bytes(inode, 0); 5831 inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; 5832 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 5833 struct btrfs_inode_item); 5834 memset_extent_buffer(path->nodes[0], 0, (unsigned long)inode_item, 5835 sizeof(*inode_item)); 5836 fill_inode_item(trans, path->nodes[0], inode_item, inode); 5837 5838 if (name) { 5839 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 5840 struct btrfs_inode_ref); 5841 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len); 5842 btrfs_set_inode_ref_index(path->nodes[0], ref, *index); 5843 ptr = (unsigned long)(ref + 1); 5844 write_extent_buffer(path->nodes[0], name, ptr, name_len); 5845 } 5846 5847 btrfs_mark_buffer_dirty(path->nodes[0]); 5848 btrfs_free_path(path); 5849 5850 btrfs_inherit_iflags(inode, dir); 5851 5852 if (S_ISREG(mode)) { 5853 if (btrfs_test_opt(root, NODATASUM)) 5854 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 5855 if (btrfs_test_opt(root, NODATACOW)) 5856 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 5857 BTRFS_INODE_NODATASUM; 5858 } 5859 5860 inode_tree_add(inode); 5861 5862 trace_btrfs_inode_new(inode); 5863 btrfs_set_inode_last_trans(trans, inode); 5864 5865 btrfs_update_root_times(trans, root); 5866 5867 ret = btrfs_inode_inherit_props(trans, inode, dir); 5868 if (ret) 5869 btrfs_err(root->fs_info, 5870 "error inheriting props for ino %llu (root %llu): %d", 5871 btrfs_ino(inode), root->root_key.objectid, ret); 5872 5873 return inode; 5874 5875 fail_unlock: 5876 unlock_new_inode(inode); 5877 fail: 5878 if (dir && name) 5879 BTRFS_I(dir)->index_cnt--; 5880 btrfs_free_path(path); 5881 iput(inode); 5882 return ERR_PTR(ret); 5883 } 5884 5885 static inline u8 btrfs_inode_type(struct inode *inode) 5886 { 5887 return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT]; 5888 } 5889 5890 /* 5891 * utility function to add 'inode' into 'parent_inode' with 5892 * a give name and a given sequence number. 5893 * if 'add_backref' is true, also insert a backref from the 5894 * inode to the parent directory. 5895 */ 5896 int btrfs_add_link(struct btrfs_trans_handle *trans, 5897 struct inode *parent_inode, struct inode *inode, 5898 const char *name, int name_len, int add_backref, u64 index) 5899 { 5900 int ret = 0; 5901 struct btrfs_key key; 5902 struct btrfs_root *root = BTRFS_I(parent_inode)->root; 5903 u64 ino = btrfs_ino(inode); 5904 u64 parent_ino = btrfs_ino(parent_inode); 5905 5906 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 5907 memcpy(&key, &BTRFS_I(inode)->root->root_key, sizeof(key)); 5908 } else { 5909 key.objectid = ino; 5910 key.type = BTRFS_INODE_ITEM_KEY; 5911 key.offset = 0; 5912 } 5913 5914 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 5915 ret = btrfs_add_root_ref(trans, root->fs_info->tree_root, 5916 key.objectid, root->root_key.objectid, 5917 parent_ino, index, name, name_len); 5918 } else if (add_backref) { 5919 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino, 5920 parent_ino, index); 5921 } 5922 5923 /* Nothing to clean up yet */ 5924 if (ret) 5925 return ret; 5926 5927 ret = btrfs_insert_dir_item(trans, root, name, name_len, 5928 parent_inode, &key, 5929 btrfs_inode_type(inode), index); 5930 if (ret == -EEXIST || ret == -EOVERFLOW) 5931 goto fail_dir_item; 5932 else if (ret) { 5933 btrfs_abort_transaction(trans, root, ret); 5934 return ret; 5935 } 5936 5937 btrfs_i_size_write(parent_inode, parent_inode->i_size + 5938 name_len * 2); 5939 inode_inc_iversion(parent_inode); 5940 parent_inode->i_mtime = parent_inode->i_ctime = CURRENT_TIME; 5941 ret = btrfs_update_inode(trans, root, parent_inode); 5942 if (ret) 5943 btrfs_abort_transaction(trans, root, ret); 5944 return ret; 5945 5946 fail_dir_item: 5947 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 5948 u64 local_index; 5949 int err; 5950 err = btrfs_del_root_ref(trans, root->fs_info->tree_root, 5951 key.objectid, root->root_key.objectid, 5952 parent_ino, &local_index, name, name_len); 5953 5954 } else if (add_backref) { 5955 u64 local_index; 5956 int err; 5957 5958 err = btrfs_del_inode_ref(trans, root, name, name_len, 5959 ino, parent_ino, &local_index); 5960 } 5961 return ret; 5962 } 5963 5964 static int btrfs_add_nondir(struct btrfs_trans_handle *trans, 5965 struct inode *dir, struct dentry *dentry, 5966 struct inode *inode, int backref, u64 index) 5967 { 5968 int err = btrfs_add_link(trans, dir, inode, 5969 dentry->d_name.name, dentry->d_name.len, 5970 backref, index); 5971 if (err > 0) 5972 err = -EEXIST; 5973 return err; 5974 } 5975 5976 static int btrfs_mknod(struct inode *dir, struct dentry *dentry, 5977 umode_t mode, dev_t rdev) 5978 { 5979 struct btrfs_trans_handle *trans; 5980 struct btrfs_root *root = BTRFS_I(dir)->root; 5981 struct inode *inode = NULL; 5982 int err; 5983 int drop_inode = 0; 5984 u64 objectid; 5985 u64 index = 0; 5986 5987 if (!new_valid_dev(rdev)) 5988 return -EINVAL; 5989 5990 /* 5991 * 2 for inode item and ref 5992 * 2 for dir items 5993 * 1 for xattr if selinux is on 5994 */ 5995 trans = btrfs_start_transaction(root, 5); 5996 if (IS_ERR(trans)) 5997 return PTR_ERR(trans); 5998 5999 err = btrfs_find_free_ino(root, &objectid); 6000 if (err) 6001 goto out_unlock; 6002 6003 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6004 dentry->d_name.len, btrfs_ino(dir), objectid, 6005 mode, &index); 6006 if (IS_ERR(inode)) { 6007 err = PTR_ERR(inode); 6008 goto out_unlock; 6009 } 6010 6011 /* 6012 * If the active LSM wants to access the inode during 6013 * d_instantiate it needs these. Smack checks to see 6014 * if the filesystem supports xattrs by looking at the 6015 * ops vector. 6016 */ 6017 inode->i_op = &btrfs_special_inode_operations; 6018 init_special_inode(inode, inode->i_mode, rdev); 6019 6020 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6021 if (err) 6022 goto out_unlock_inode; 6023 6024 err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index); 6025 if (err) { 6026 goto out_unlock_inode; 6027 } else { 6028 btrfs_update_inode(trans, root, inode); 6029 unlock_new_inode(inode); 6030 d_instantiate(dentry, inode); 6031 } 6032 6033 out_unlock: 6034 btrfs_end_transaction(trans, root); 6035 btrfs_balance_delayed_items(root); 6036 btrfs_btree_balance_dirty(root); 6037 if (drop_inode) { 6038 inode_dec_link_count(inode); 6039 iput(inode); 6040 } 6041 return err; 6042 6043 out_unlock_inode: 6044 drop_inode = 1; 6045 unlock_new_inode(inode); 6046 goto out_unlock; 6047 6048 } 6049 6050 static int btrfs_create(struct inode *dir, struct dentry *dentry, 6051 umode_t mode, bool excl) 6052 { 6053 struct btrfs_trans_handle *trans; 6054 struct btrfs_root *root = BTRFS_I(dir)->root; 6055 struct inode *inode = NULL; 6056 int drop_inode_on_err = 0; 6057 int err; 6058 u64 objectid; 6059 u64 index = 0; 6060 6061 /* 6062 * 2 for inode item and ref 6063 * 2 for dir items 6064 * 1 for xattr if selinux is on 6065 */ 6066 trans = btrfs_start_transaction(root, 5); 6067 if (IS_ERR(trans)) 6068 return PTR_ERR(trans); 6069 6070 err = btrfs_find_free_ino(root, &objectid); 6071 if (err) 6072 goto out_unlock; 6073 6074 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6075 dentry->d_name.len, btrfs_ino(dir), objectid, 6076 mode, &index); 6077 if (IS_ERR(inode)) { 6078 err = PTR_ERR(inode); 6079 goto out_unlock; 6080 } 6081 drop_inode_on_err = 1; 6082 /* 6083 * If the active LSM wants to access the inode during 6084 * d_instantiate it needs these. Smack checks to see 6085 * if the filesystem supports xattrs by looking at the 6086 * ops vector. 6087 */ 6088 inode->i_fop = &btrfs_file_operations; 6089 inode->i_op = &btrfs_file_inode_operations; 6090 inode->i_mapping->a_ops = &btrfs_aops; 6091 inode->i_mapping->backing_dev_info = &root->fs_info->bdi; 6092 6093 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6094 if (err) 6095 goto out_unlock_inode; 6096 6097 err = btrfs_update_inode(trans, root, inode); 6098 if (err) 6099 goto out_unlock_inode; 6100 6101 err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index); 6102 if (err) 6103 goto out_unlock_inode; 6104 6105 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 6106 unlock_new_inode(inode); 6107 d_instantiate(dentry, inode); 6108 6109 out_unlock: 6110 btrfs_end_transaction(trans, root); 6111 if (err && drop_inode_on_err) { 6112 inode_dec_link_count(inode); 6113 iput(inode); 6114 } 6115 btrfs_balance_delayed_items(root); 6116 btrfs_btree_balance_dirty(root); 6117 return err; 6118 6119 out_unlock_inode: 6120 unlock_new_inode(inode); 6121 goto out_unlock; 6122 6123 } 6124 6125 static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6126 struct dentry *dentry) 6127 { 6128 struct btrfs_trans_handle *trans; 6129 struct btrfs_root *root = BTRFS_I(dir)->root; 6130 struct inode *inode = old_dentry->d_inode; 6131 u64 index; 6132 int err; 6133 int drop_inode = 0; 6134 6135 /* do not allow sys_link's with other subvols of the same device */ 6136 if (root->objectid != BTRFS_I(inode)->root->objectid) 6137 return -EXDEV; 6138 6139 if (inode->i_nlink >= BTRFS_LINK_MAX) 6140 return -EMLINK; 6141 6142 err = btrfs_set_inode_index(dir, &index); 6143 if (err) 6144 goto fail; 6145 6146 /* 6147 * 2 items for inode and inode ref 6148 * 2 items for dir items 6149 * 1 item for parent inode 6150 */ 6151 trans = btrfs_start_transaction(root, 5); 6152 if (IS_ERR(trans)) { 6153 err = PTR_ERR(trans); 6154 goto fail; 6155 } 6156 6157 /* There are several dir indexes for this inode, clear the cache. */ 6158 BTRFS_I(inode)->dir_index = 0ULL; 6159 inc_nlink(inode); 6160 inode_inc_iversion(inode); 6161 inode->i_ctime = CURRENT_TIME; 6162 ihold(inode); 6163 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags); 6164 6165 err = btrfs_add_nondir(trans, dir, dentry, inode, 1, index); 6166 6167 if (err) { 6168 drop_inode = 1; 6169 } else { 6170 struct dentry *parent = dentry->d_parent; 6171 err = btrfs_update_inode(trans, root, inode); 6172 if (err) 6173 goto fail; 6174 if (inode->i_nlink == 1) { 6175 /* 6176 * If new hard link count is 1, it's a file created 6177 * with open(2) O_TMPFILE flag. 6178 */ 6179 err = btrfs_orphan_del(trans, inode); 6180 if (err) 6181 goto fail; 6182 } 6183 d_instantiate(dentry, inode); 6184 btrfs_log_new_name(trans, inode, NULL, parent); 6185 } 6186 6187 btrfs_end_transaction(trans, root); 6188 btrfs_balance_delayed_items(root); 6189 fail: 6190 if (drop_inode) { 6191 inode_dec_link_count(inode); 6192 iput(inode); 6193 } 6194 btrfs_btree_balance_dirty(root); 6195 return err; 6196 } 6197 6198 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 6199 { 6200 struct inode *inode = NULL; 6201 struct btrfs_trans_handle *trans; 6202 struct btrfs_root *root = BTRFS_I(dir)->root; 6203 int err = 0; 6204 int drop_on_err = 0; 6205 u64 objectid = 0; 6206 u64 index = 0; 6207 6208 /* 6209 * 2 items for inode and ref 6210 * 2 items for dir items 6211 * 1 for xattr if selinux is on 6212 */ 6213 trans = btrfs_start_transaction(root, 5); 6214 if (IS_ERR(trans)) 6215 return PTR_ERR(trans); 6216 6217 err = btrfs_find_free_ino(root, &objectid); 6218 if (err) 6219 goto out_fail; 6220 6221 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 6222 dentry->d_name.len, btrfs_ino(dir), objectid, 6223 S_IFDIR | mode, &index); 6224 if (IS_ERR(inode)) { 6225 err = PTR_ERR(inode); 6226 goto out_fail; 6227 } 6228 6229 drop_on_err = 1; 6230 /* these must be set before we unlock the inode */ 6231 inode->i_op = &btrfs_dir_inode_operations; 6232 inode->i_fop = &btrfs_dir_file_operations; 6233 6234 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 6235 if (err) 6236 goto out_fail_inode; 6237 6238 btrfs_i_size_write(inode, 0); 6239 err = btrfs_update_inode(trans, root, inode); 6240 if (err) 6241 goto out_fail_inode; 6242 6243 err = btrfs_add_link(trans, dir, inode, dentry->d_name.name, 6244 dentry->d_name.len, 0, index); 6245 if (err) 6246 goto out_fail_inode; 6247 6248 d_instantiate(dentry, inode); 6249 /* 6250 * mkdir is special. We're unlocking after we call d_instantiate 6251 * to avoid a race with nfsd calling d_instantiate. 6252 */ 6253 unlock_new_inode(inode); 6254 drop_on_err = 0; 6255 6256 out_fail: 6257 btrfs_end_transaction(trans, root); 6258 if (drop_on_err) 6259 iput(inode); 6260 btrfs_balance_delayed_items(root); 6261 btrfs_btree_balance_dirty(root); 6262 return err; 6263 6264 out_fail_inode: 6265 unlock_new_inode(inode); 6266 goto out_fail; 6267 } 6268 6269 /* Find next extent map of a given extent map, caller needs to ensure locks */ 6270 static struct extent_map *next_extent_map(struct extent_map *em) 6271 { 6272 struct rb_node *next; 6273 6274 next = rb_next(&em->rb_node); 6275 if (!next) 6276 return NULL; 6277 return container_of(next, struct extent_map, rb_node); 6278 } 6279 6280 static struct extent_map *prev_extent_map(struct extent_map *em) 6281 { 6282 struct rb_node *prev; 6283 6284 prev = rb_prev(&em->rb_node); 6285 if (!prev) 6286 return NULL; 6287 return container_of(prev, struct extent_map, rb_node); 6288 } 6289 6290 /* helper for btfs_get_extent. Given an existing extent in the tree, 6291 * the existing extent is the nearest extent to map_start, 6292 * and an extent that you want to insert, deal with overlap and insert 6293 * the best fitted new extent into the tree. 6294 */ 6295 static int merge_extent_mapping(struct extent_map_tree *em_tree, 6296 struct extent_map *existing, 6297 struct extent_map *em, 6298 u64 map_start) 6299 { 6300 struct extent_map *prev; 6301 struct extent_map *next; 6302 u64 start; 6303 u64 end; 6304 u64 start_diff; 6305 6306 BUG_ON(map_start < em->start || map_start >= extent_map_end(em)); 6307 6308 if (existing->start > map_start) { 6309 next = existing; 6310 prev = prev_extent_map(next); 6311 } else { 6312 prev = existing; 6313 next = next_extent_map(prev); 6314 } 6315 6316 start = prev ? extent_map_end(prev) : em->start; 6317 start = max_t(u64, start, em->start); 6318 end = next ? next->start : extent_map_end(em); 6319 end = min_t(u64, end, extent_map_end(em)); 6320 start_diff = start - em->start; 6321 em->start = start; 6322 em->len = end - start; 6323 if (em->block_start < EXTENT_MAP_LAST_BYTE && 6324 !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 6325 em->block_start += start_diff; 6326 em->block_len -= start_diff; 6327 } 6328 return add_extent_mapping(em_tree, em, 0); 6329 } 6330 6331 static noinline int uncompress_inline(struct btrfs_path *path, 6332 struct inode *inode, struct page *page, 6333 size_t pg_offset, u64 extent_offset, 6334 struct btrfs_file_extent_item *item) 6335 { 6336 int ret; 6337 struct extent_buffer *leaf = path->nodes[0]; 6338 char *tmp; 6339 size_t max_size; 6340 unsigned long inline_size; 6341 unsigned long ptr; 6342 int compress_type; 6343 6344 WARN_ON(pg_offset != 0); 6345 compress_type = btrfs_file_extent_compression(leaf, item); 6346 max_size = btrfs_file_extent_ram_bytes(leaf, item); 6347 inline_size = btrfs_file_extent_inline_item_len(leaf, 6348 btrfs_item_nr(path->slots[0])); 6349 tmp = kmalloc(inline_size, GFP_NOFS); 6350 if (!tmp) 6351 return -ENOMEM; 6352 ptr = btrfs_file_extent_inline_start(item); 6353 6354 read_extent_buffer(leaf, tmp, ptr, inline_size); 6355 6356 max_size = min_t(unsigned long, PAGE_CACHE_SIZE, max_size); 6357 ret = btrfs_decompress(compress_type, tmp, page, 6358 extent_offset, inline_size, max_size); 6359 kfree(tmp); 6360 return ret; 6361 } 6362 6363 /* 6364 * a bit scary, this does extent mapping from logical file offset to the disk. 6365 * the ugly parts come from merging extents from the disk with the in-ram 6366 * representation. This gets more complex because of the data=ordered code, 6367 * where the in-ram extents might be locked pending data=ordered completion. 6368 * 6369 * This also copies inline extents directly into the page. 6370 */ 6371 6372 struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page, 6373 size_t pg_offset, u64 start, u64 len, 6374 int create) 6375 { 6376 int ret; 6377 int err = 0; 6378 u64 extent_start = 0; 6379 u64 extent_end = 0; 6380 u64 objectid = btrfs_ino(inode); 6381 u32 found_type; 6382 struct btrfs_path *path = NULL; 6383 struct btrfs_root *root = BTRFS_I(inode)->root; 6384 struct btrfs_file_extent_item *item; 6385 struct extent_buffer *leaf; 6386 struct btrfs_key found_key; 6387 struct extent_map *em = NULL; 6388 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 6389 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 6390 struct btrfs_trans_handle *trans = NULL; 6391 const bool new_inline = !page || create; 6392 6393 again: 6394 read_lock(&em_tree->lock); 6395 em = lookup_extent_mapping(em_tree, start, len); 6396 if (em) 6397 em->bdev = root->fs_info->fs_devices->latest_bdev; 6398 read_unlock(&em_tree->lock); 6399 6400 if (em) { 6401 if (em->start > start || em->start + em->len <= start) 6402 free_extent_map(em); 6403 else if (em->block_start == EXTENT_MAP_INLINE && page) 6404 free_extent_map(em); 6405 else 6406 goto out; 6407 } 6408 em = alloc_extent_map(); 6409 if (!em) { 6410 err = -ENOMEM; 6411 goto out; 6412 } 6413 em->bdev = root->fs_info->fs_devices->latest_bdev; 6414 em->start = EXTENT_MAP_HOLE; 6415 em->orig_start = EXTENT_MAP_HOLE; 6416 em->len = (u64)-1; 6417 em->block_len = (u64)-1; 6418 6419 if (!path) { 6420 path = btrfs_alloc_path(); 6421 if (!path) { 6422 err = -ENOMEM; 6423 goto out; 6424 } 6425 /* 6426 * Chances are we'll be called again, so go ahead and do 6427 * readahead 6428 */ 6429 path->reada = 1; 6430 } 6431 6432 ret = btrfs_lookup_file_extent(trans, root, path, 6433 objectid, start, trans != NULL); 6434 if (ret < 0) { 6435 err = ret; 6436 goto out; 6437 } 6438 6439 if (ret != 0) { 6440 if (path->slots[0] == 0) 6441 goto not_found; 6442 path->slots[0]--; 6443 } 6444 6445 leaf = path->nodes[0]; 6446 item = btrfs_item_ptr(leaf, path->slots[0], 6447 struct btrfs_file_extent_item); 6448 /* are we inside the extent that was found? */ 6449 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6450 found_type = found_key.type; 6451 if (found_key.objectid != objectid || 6452 found_type != BTRFS_EXTENT_DATA_KEY) { 6453 /* 6454 * If we backup past the first extent we want to move forward 6455 * and see if there is an extent in front of us, otherwise we'll 6456 * say there is a hole for our whole search range which can 6457 * cause problems. 6458 */ 6459 extent_end = start; 6460 goto next; 6461 } 6462 6463 found_type = btrfs_file_extent_type(leaf, item); 6464 extent_start = found_key.offset; 6465 if (found_type == BTRFS_FILE_EXTENT_REG || 6466 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6467 extent_end = extent_start + 6468 btrfs_file_extent_num_bytes(leaf, item); 6469 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 6470 size_t size; 6471 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 6472 extent_end = ALIGN(extent_start + size, root->sectorsize); 6473 } 6474 next: 6475 if (start >= extent_end) { 6476 path->slots[0]++; 6477 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 6478 ret = btrfs_next_leaf(root, path); 6479 if (ret < 0) { 6480 err = ret; 6481 goto out; 6482 } 6483 if (ret > 0) 6484 goto not_found; 6485 leaf = path->nodes[0]; 6486 } 6487 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6488 if (found_key.objectid != objectid || 6489 found_key.type != BTRFS_EXTENT_DATA_KEY) 6490 goto not_found; 6491 if (start + len <= found_key.offset) 6492 goto not_found; 6493 if (start > found_key.offset) 6494 goto next; 6495 em->start = start; 6496 em->orig_start = start; 6497 em->len = found_key.offset - start; 6498 goto not_found_em; 6499 } 6500 6501 btrfs_extent_item_to_extent_map(inode, path, item, new_inline, em); 6502 6503 if (found_type == BTRFS_FILE_EXTENT_REG || 6504 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6505 goto insert; 6506 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { 6507 unsigned long ptr; 6508 char *map; 6509 size_t size; 6510 size_t extent_offset; 6511 size_t copy_size; 6512 6513 if (new_inline) 6514 goto out; 6515 6516 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item); 6517 extent_offset = page_offset(page) + pg_offset - extent_start; 6518 copy_size = min_t(u64, PAGE_CACHE_SIZE - pg_offset, 6519 size - extent_offset); 6520 em->start = extent_start + extent_offset; 6521 em->len = ALIGN(copy_size, root->sectorsize); 6522 em->orig_block_len = em->len; 6523 em->orig_start = em->start; 6524 ptr = btrfs_file_extent_inline_start(item) + extent_offset; 6525 if (create == 0 && !PageUptodate(page)) { 6526 if (btrfs_file_extent_compression(leaf, item) != 6527 BTRFS_COMPRESS_NONE) { 6528 ret = uncompress_inline(path, inode, page, 6529 pg_offset, 6530 extent_offset, item); 6531 if (ret) { 6532 err = ret; 6533 goto out; 6534 } 6535 } else { 6536 map = kmap(page); 6537 read_extent_buffer(leaf, map + pg_offset, ptr, 6538 copy_size); 6539 if (pg_offset + copy_size < PAGE_CACHE_SIZE) { 6540 memset(map + pg_offset + copy_size, 0, 6541 PAGE_CACHE_SIZE - pg_offset - 6542 copy_size); 6543 } 6544 kunmap(page); 6545 } 6546 flush_dcache_page(page); 6547 } else if (create && PageUptodate(page)) { 6548 BUG(); 6549 if (!trans) { 6550 kunmap(page); 6551 free_extent_map(em); 6552 em = NULL; 6553 6554 btrfs_release_path(path); 6555 trans = btrfs_join_transaction(root); 6556 6557 if (IS_ERR(trans)) 6558 return ERR_CAST(trans); 6559 goto again; 6560 } 6561 map = kmap(page); 6562 write_extent_buffer(leaf, map + pg_offset, ptr, 6563 copy_size); 6564 kunmap(page); 6565 btrfs_mark_buffer_dirty(leaf); 6566 } 6567 set_extent_uptodate(io_tree, em->start, 6568 extent_map_end(em) - 1, NULL, GFP_NOFS); 6569 goto insert; 6570 } 6571 not_found: 6572 em->start = start; 6573 em->orig_start = start; 6574 em->len = len; 6575 not_found_em: 6576 em->block_start = EXTENT_MAP_HOLE; 6577 set_bit(EXTENT_FLAG_VACANCY, &em->flags); 6578 insert: 6579 btrfs_release_path(path); 6580 if (em->start > start || extent_map_end(em) <= start) { 6581 btrfs_err(root->fs_info, "bad extent! em: [%llu %llu] passed [%llu %llu]", 6582 em->start, em->len, start, len); 6583 err = -EIO; 6584 goto out; 6585 } 6586 6587 err = 0; 6588 write_lock(&em_tree->lock); 6589 ret = add_extent_mapping(em_tree, em, 0); 6590 /* it is possible that someone inserted the extent into the tree 6591 * while we had the lock dropped. It is also possible that 6592 * an overlapping map exists in the tree 6593 */ 6594 if (ret == -EEXIST) { 6595 struct extent_map *existing; 6596 6597 ret = 0; 6598 6599 existing = search_extent_mapping(em_tree, start, len); 6600 /* 6601 * existing will always be non-NULL, since there must be 6602 * extent causing the -EEXIST. 6603 */ 6604 if (start >= extent_map_end(existing) || 6605 start <= existing->start) { 6606 /* 6607 * The existing extent map is the one nearest to 6608 * the [start, start + len) range which overlaps 6609 */ 6610 err = merge_extent_mapping(em_tree, existing, 6611 em, start); 6612 free_extent_map(existing); 6613 if (err) { 6614 free_extent_map(em); 6615 em = NULL; 6616 } 6617 } else { 6618 free_extent_map(em); 6619 em = existing; 6620 err = 0; 6621 } 6622 } 6623 write_unlock(&em_tree->lock); 6624 out: 6625 6626 trace_btrfs_get_extent(root, em); 6627 6628 if (path) 6629 btrfs_free_path(path); 6630 if (trans) { 6631 ret = btrfs_end_transaction(trans, root); 6632 if (!err) 6633 err = ret; 6634 } 6635 if (err) { 6636 free_extent_map(em); 6637 return ERR_PTR(err); 6638 } 6639 BUG_ON(!em); /* Error is always set */ 6640 return em; 6641 } 6642 6643 struct extent_map *btrfs_get_extent_fiemap(struct inode *inode, struct page *page, 6644 size_t pg_offset, u64 start, u64 len, 6645 int create) 6646 { 6647 struct extent_map *em; 6648 struct extent_map *hole_em = NULL; 6649 u64 range_start = start; 6650 u64 end; 6651 u64 found; 6652 u64 found_end; 6653 int err = 0; 6654 6655 em = btrfs_get_extent(inode, page, pg_offset, start, len, create); 6656 if (IS_ERR(em)) 6657 return em; 6658 if (em) { 6659 /* 6660 * if our em maps to 6661 * - a hole or 6662 * - a pre-alloc extent, 6663 * there might actually be delalloc bytes behind it. 6664 */ 6665 if (em->block_start != EXTENT_MAP_HOLE && 6666 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 6667 return em; 6668 else 6669 hole_em = em; 6670 } 6671 6672 /* check to see if we've wrapped (len == -1 or similar) */ 6673 end = start + len; 6674 if (end < start) 6675 end = (u64)-1; 6676 else 6677 end -= 1; 6678 6679 em = NULL; 6680 6681 /* ok, we didn't find anything, lets look for delalloc */ 6682 found = count_range_bits(&BTRFS_I(inode)->io_tree, &range_start, 6683 end, len, EXTENT_DELALLOC, 1); 6684 found_end = range_start + found; 6685 if (found_end < range_start) 6686 found_end = (u64)-1; 6687 6688 /* 6689 * we didn't find anything useful, return 6690 * the original results from get_extent() 6691 */ 6692 if (range_start > end || found_end <= start) { 6693 em = hole_em; 6694 hole_em = NULL; 6695 goto out; 6696 } 6697 6698 /* adjust the range_start to make sure it doesn't 6699 * go backwards from the start they passed in 6700 */ 6701 range_start = max(start, range_start); 6702 found = found_end - range_start; 6703 6704 if (found > 0) { 6705 u64 hole_start = start; 6706 u64 hole_len = len; 6707 6708 em = alloc_extent_map(); 6709 if (!em) { 6710 err = -ENOMEM; 6711 goto out; 6712 } 6713 /* 6714 * when btrfs_get_extent can't find anything it 6715 * returns one huge hole 6716 * 6717 * make sure what it found really fits our range, and 6718 * adjust to make sure it is based on the start from 6719 * the caller 6720 */ 6721 if (hole_em) { 6722 u64 calc_end = extent_map_end(hole_em); 6723 6724 if (calc_end <= start || (hole_em->start > end)) { 6725 free_extent_map(hole_em); 6726 hole_em = NULL; 6727 } else { 6728 hole_start = max(hole_em->start, start); 6729 hole_len = calc_end - hole_start; 6730 } 6731 } 6732 em->bdev = NULL; 6733 if (hole_em && range_start > hole_start) { 6734 /* our hole starts before our delalloc, so we 6735 * have to return just the parts of the hole 6736 * that go until the delalloc starts 6737 */ 6738 em->len = min(hole_len, 6739 range_start - hole_start); 6740 em->start = hole_start; 6741 em->orig_start = hole_start; 6742 /* 6743 * don't adjust block start at all, 6744 * it is fixed at EXTENT_MAP_HOLE 6745 */ 6746 em->block_start = hole_em->block_start; 6747 em->block_len = hole_len; 6748 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) 6749 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 6750 } else { 6751 em->start = range_start; 6752 em->len = found; 6753 em->orig_start = range_start; 6754 em->block_start = EXTENT_MAP_DELALLOC; 6755 em->block_len = found; 6756 } 6757 } else if (hole_em) { 6758 return hole_em; 6759 } 6760 out: 6761 6762 free_extent_map(hole_em); 6763 if (err) { 6764 free_extent_map(em); 6765 return ERR_PTR(err); 6766 } 6767 return em; 6768 } 6769 6770 static struct extent_map *btrfs_new_extent_direct(struct inode *inode, 6771 u64 start, u64 len) 6772 { 6773 struct btrfs_root *root = BTRFS_I(inode)->root; 6774 struct extent_map *em; 6775 struct btrfs_key ins; 6776 u64 alloc_hint; 6777 int ret; 6778 6779 alloc_hint = get_extent_allocation_hint(inode, start, len); 6780 ret = btrfs_reserve_extent(root, len, root->sectorsize, 0, 6781 alloc_hint, &ins, 1, 1); 6782 if (ret) 6783 return ERR_PTR(ret); 6784 6785 em = create_pinned_em(inode, start, ins.offset, start, ins.objectid, 6786 ins.offset, ins.offset, ins.offset, 0); 6787 if (IS_ERR(em)) { 6788 btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1); 6789 return em; 6790 } 6791 6792 ret = btrfs_add_ordered_extent_dio(inode, start, ins.objectid, 6793 ins.offset, ins.offset, 0); 6794 if (ret) { 6795 btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1); 6796 free_extent_map(em); 6797 return ERR_PTR(ret); 6798 } 6799 6800 return em; 6801 } 6802 6803 /* 6804 * returns 1 when the nocow is safe, < 1 on error, 0 if the 6805 * block must be cow'd 6806 */ 6807 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, 6808 u64 *orig_start, u64 *orig_block_len, 6809 u64 *ram_bytes) 6810 { 6811 struct btrfs_trans_handle *trans; 6812 struct btrfs_path *path; 6813 int ret; 6814 struct extent_buffer *leaf; 6815 struct btrfs_root *root = BTRFS_I(inode)->root; 6816 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 6817 struct btrfs_file_extent_item *fi; 6818 struct btrfs_key key; 6819 u64 disk_bytenr; 6820 u64 backref_offset; 6821 u64 extent_end; 6822 u64 num_bytes; 6823 int slot; 6824 int found_type; 6825 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW); 6826 6827 path = btrfs_alloc_path(); 6828 if (!path) 6829 return -ENOMEM; 6830 6831 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), 6832 offset, 0); 6833 if (ret < 0) 6834 goto out; 6835 6836 slot = path->slots[0]; 6837 if (ret == 1) { 6838 if (slot == 0) { 6839 /* can't find the item, must cow */ 6840 ret = 0; 6841 goto out; 6842 } 6843 slot--; 6844 } 6845 ret = 0; 6846 leaf = path->nodes[0]; 6847 btrfs_item_key_to_cpu(leaf, &key, slot); 6848 if (key.objectid != btrfs_ino(inode) || 6849 key.type != BTRFS_EXTENT_DATA_KEY) { 6850 /* not our file or wrong item type, must cow */ 6851 goto out; 6852 } 6853 6854 if (key.offset > offset) { 6855 /* Wrong offset, must cow */ 6856 goto out; 6857 } 6858 6859 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); 6860 found_type = btrfs_file_extent_type(leaf, fi); 6861 if (found_type != BTRFS_FILE_EXTENT_REG && 6862 found_type != BTRFS_FILE_EXTENT_PREALLOC) { 6863 /* not a regular extent, must cow */ 6864 goto out; 6865 } 6866 6867 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG) 6868 goto out; 6869 6870 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); 6871 if (extent_end <= offset) 6872 goto out; 6873 6874 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 6875 if (disk_bytenr == 0) 6876 goto out; 6877 6878 if (btrfs_file_extent_compression(leaf, fi) || 6879 btrfs_file_extent_encryption(leaf, fi) || 6880 btrfs_file_extent_other_encoding(leaf, fi)) 6881 goto out; 6882 6883 backref_offset = btrfs_file_extent_offset(leaf, fi); 6884 6885 if (orig_start) { 6886 *orig_start = key.offset - backref_offset; 6887 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi); 6888 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 6889 } 6890 6891 if (btrfs_extent_readonly(root, disk_bytenr)) 6892 goto out; 6893 6894 num_bytes = min(offset + *len, extent_end) - offset; 6895 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) { 6896 u64 range_end; 6897 6898 range_end = round_up(offset + num_bytes, root->sectorsize) - 1; 6899 ret = test_range_bit(io_tree, offset, range_end, 6900 EXTENT_DELALLOC, 0, NULL); 6901 if (ret) { 6902 ret = -EAGAIN; 6903 goto out; 6904 } 6905 } 6906 6907 btrfs_release_path(path); 6908 6909 /* 6910 * look for other files referencing this extent, if we 6911 * find any we must cow 6912 */ 6913 trans = btrfs_join_transaction(root); 6914 if (IS_ERR(trans)) { 6915 ret = 0; 6916 goto out; 6917 } 6918 6919 ret = btrfs_cross_ref_exist(trans, root, btrfs_ino(inode), 6920 key.offset - backref_offset, disk_bytenr); 6921 btrfs_end_transaction(trans, root); 6922 if (ret) { 6923 ret = 0; 6924 goto out; 6925 } 6926 6927 /* 6928 * adjust disk_bytenr and num_bytes to cover just the bytes 6929 * in this extent we are about to write. If there 6930 * are any csums in that range we have to cow in order 6931 * to keep the csums correct 6932 */ 6933 disk_bytenr += backref_offset; 6934 disk_bytenr += offset - key.offset; 6935 if (csum_exist_in_range(root, disk_bytenr, num_bytes)) 6936 goto out; 6937 /* 6938 * all of the above have passed, it is safe to overwrite this extent 6939 * without cow 6940 */ 6941 *len = num_bytes; 6942 ret = 1; 6943 out: 6944 btrfs_free_path(path); 6945 return ret; 6946 } 6947 6948 bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end) 6949 { 6950 struct radix_tree_root *root = &inode->i_mapping->page_tree; 6951 int found = false; 6952 void **pagep = NULL; 6953 struct page *page = NULL; 6954 int start_idx; 6955 int end_idx; 6956 6957 start_idx = start >> PAGE_CACHE_SHIFT; 6958 6959 /* 6960 * end is the last byte in the last page. end == start is legal 6961 */ 6962 end_idx = end >> PAGE_CACHE_SHIFT; 6963 6964 rcu_read_lock(); 6965 6966 /* Most of the code in this while loop is lifted from 6967 * find_get_page. It's been modified to begin searching from a 6968 * page and return just the first page found in that range. If the 6969 * found idx is less than or equal to the end idx then we know that 6970 * a page exists. If no pages are found or if those pages are 6971 * outside of the range then we're fine (yay!) */ 6972 while (page == NULL && 6973 radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) { 6974 page = radix_tree_deref_slot(pagep); 6975 if (unlikely(!page)) 6976 break; 6977 6978 if (radix_tree_exception(page)) { 6979 if (radix_tree_deref_retry(page)) { 6980 page = NULL; 6981 continue; 6982 } 6983 /* 6984 * Otherwise, shmem/tmpfs must be storing a swap entry 6985 * here as an exceptional entry: so return it without 6986 * attempting to raise page count. 6987 */ 6988 page = NULL; 6989 break; /* TODO: Is this relevant for this use case? */ 6990 } 6991 6992 if (!page_cache_get_speculative(page)) { 6993 page = NULL; 6994 continue; 6995 } 6996 6997 /* 6998 * Has the page moved? 6999 * This is part of the lockless pagecache protocol. See 7000 * include/linux/pagemap.h for details. 7001 */ 7002 if (unlikely(page != *pagep)) { 7003 page_cache_release(page); 7004 page = NULL; 7005 } 7006 } 7007 7008 if (page) { 7009 if (page->index <= end_idx) 7010 found = true; 7011 page_cache_release(page); 7012 } 7013 7014 rcu_read_unlock(); 7015 return found; 7016 } 7017 7018 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, 7019 struct extent_state **cached_state, int writing) 7020 { 7021 struct btrfs_ordered_extent *ordered; 7022 int ret = 0; 7023 7024 while (1) { 7025 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7026 0, cached_state); 7027 /* 7028 * We're concerned with the entire range that we're going to be 7029 * doing DIO to, so we need to make sure theres no ordered 7030 * extents in this range. 7031 */ 7032 ordered = btrfs_lookup_ordered_range(inode, lockstart, 7033 lockend - lockstart + 1); 7034 7035 /* 7036 * We need to make sure there are no buffered pages in this 7037 * range either, we could have raced between the invalidate in 7038 * generic_file_direct_write and locking the extent. The 7039 * invalidate needs to happen so that reads after a write do not 7040 * get stale data. 7041 */ 7042 if (!ordered && 7043 (!writing || 7044 !btrfs_page_exists_in_range(inode, lockstart, lockend))) 7045 break; 7046 7047 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7048 cached_state, GFP_NOFS); 7049 7050 if (ordered) { 7051 btrfs_start_ordered_extent(inode, ordered, 1); 7052 btrfs_put_ordered_extent(ordered); 7053 } else { 7054 /* Screw you mmap */ 7055 ret = btrfs_fdatawrite_range(inode, lockstart, lockend); 7056 if (ret) 7057 break; 7058 ret = filemap_fdatawait_range(inode->i_mapping, 7059 lockstart, 7060 lockend); 7061 if (ret) 7062 break; 7063 7064 /* 7065 * If we found a page that couldn't be invalidated just 7066 * fall back to buffered. 7067 */ 7068 ret = invalidate_inode_pages2_range(inode->i_mapping, 7069 lockstart >> PAGE_CACHE_SHIFT, 7070 lockend >> PAGE_CACHE_SHIFT); 7071 if (ret) 7072 break; 7073 } 7074 7075 cond_resched(); 7076 } 7077 7078 return ret; 7079 } 7080 7081 static struct extent_map *create_pinned_em(struct inode *inode, u64 start, 7082 u64 len, u64 orig_start, 7083 u64 block_start, u64 block_len, 7084 u64 orig_block_len, u64 ram_bytes, 7085 int type) 7086 { 7087 struct extent_map_tree *em_tree; 7088 struct extent_map *em; 7089 struct btrfs_root *root = BTRFS_I(inode)->root; 7090 int ret; 7091 7092 em_tree = &BTRFS_I(inode)->extent_tree; 7093 em = alloc_extent_map(); 7094 if (!em) 7095 return ERR_PTR(-ENOMEM); 7096 7097 em->start = start; 7098 em->orig_start = orig_start; 7099 em->mod_start = start; 7100 em->mod_len = len; 7101 em->len = len; 7102 em->block_len = block_len; 7103 em->block_start = block_start; 7104 em->bdev = root->fs_info->fs_devices->latest_bdev; 7105 em->orig_block_len = orig_block_len; 7106 em->ram_bytes = ram_bytes; 7107 em->generation = -1; 7108 set_bit(EXTENT_FLAG_PINNED, &em->flags); 7109 if (type == BTRFS_ORDERED_PREALLOC) 7110 set_bit(EXTENT_FLAG_FILLING, &em->flags); 7111 7112 do { 7113 btrfs_drop_extent_cache(inode, em->start, 7114 em->start + em->len - 1, 0); 7115 write_lock(&em_tree->lock); 7116 ret = add_extent_mapping(em_tree, em, 1); 7117 write_unlock(&em_tree->lock); 7118 } while (ret == -EEXIST); 7119 7120 if (ret) { 7121 free_extent_map(em); 7122 return ERR_PTR(ret); 7123 } 7124 7125 return em; 7126 } 7127 7128 7129 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, 7130 struct buffer_head *bh_result, int create) 7131 { 7132 struct extent_map *em; 7133 struct btrfs_root *root = BTRFS_I(inode)->root; 7134 struct extent_state *cached_state = NULL; 7135 u64 start = iblock << inode->i_blkbits; 7136 u64 lockstart, lockend; 7137 u64 len = bh_result->b_size; 7138 int unlock_bits = EXTENT_LOCKED; 7139 int ret = 0; 7140 7141 if (create) 7142 unlock_bits |= EXTENT_DELALLOC | EXTENT_DIRTY; 7143 else 7144 len = min_t(u64, len, root->sectorsize); 7145 7146 lockstart = start; 7147 lockend = start + len - 1; 7148 7149 /* 7150 * If this errors out it's because we couldn't invalidate pagecache for 7151 * this range and we need to fallback to buffered. 7152 */ 7153 if (lock_extent_direct(inode, lockstart, lockend, &cached_state, create)) 7154 return -ENOTBLK; 7155 7156 em = btrfs_get_extent(inode, NULL, 0, start, len, 0); 7157 if (IS_ERR(em)) { 7158 ret = PTR_ERR(em); 7159 goto unlock_err; 7160 } 7161 7162 /* 7163 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered 7164 * io. INLINE is special, and we could probably kludge it in here, but 7165 * it's still buffered so for safety lets just fall back to the generic 7166 * buffered path. 7167 * 7168 * For COMPRESSED we _have_ to read the entire extent in so we can 7169 * decompress it, so there will be buffering required no matter what we 7170 * do, so go ahead and fallback to buffered. 7171 * 7172 * We return -ENOTBLK because thats what makes DIO go ahead and go back 7173 * to buffered IO. Don't blame me, this is the price we pay for using 7174 * the generic code. 7175 */ 7176 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) || 7177 em->block_start == EXTENT_MAP_INLINE) { 7178 free_extent_map(em); 7179 ret = -ENOTBLK; 7180 goto unlock_err; 7181 } 7182 7183 /* Just a good old fashioned hole, return */ 7184 if (!create && (em->block_start == EXTENT_MAP_HOLE || 7185 test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) { 7186 free_extent_map(em); 7187 goto unlock_err; 7188 } 7189 7190 /* 7191 * We don't allocate a new extent in the following cases 7192 * 7193 * 1) The inode is marked as NODATACOW. In this case we'll just use the 7194 * existing extent. 7195 * 2) The extent is marked as PREALLOC. We're good to go here and can 7196 * just use the extent. 7197 * 7198 */ 7199 if (!create) { 7200 len = min(len, em->len - (start - em->start)); 7201 lockstart = start + len; 7202 goto unlock; 7203 } 7204 7205 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) || 7206 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && 7207 em->block_start != EXTENT_MAP_HOLE)) { 7208 int type; 7209 int ret; 7210 u64 block_start, orig_start, orig_block_len, ram_bytes; 7211 7212 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7213 type = BTRFS_ORDERED_PREALLOC; 7214 else 7215 type = BTRFS_ORDERED_NOCOW; 7216 len = min(len, em->len - (start - em->start)); 7217 block_start = em->block_start + (start - em->start); 7218 7219 if (can_nocow_extent(inode, start, &len, &orig_start, 7220 &orig_block_len, &ram_bytes) == 1) { 7221 if (type == BTRFS_ORDERED_PREALLOC) { 7222 free_extent_map(em); 7223 em = create_pinned_em(inode, start, len, 7224 orig_start, 7225 block_start, len, 7226 orig_block_len, 7227 ram_bytes, type); 7228 if (IS_ERR(em)) { 7229 ret = PTR_ERR(em); 7230 goto unlock_err; 7231 } 7232 } 7233 7234 ret = btrfs_add_ordered_extent_dio(inode, start, 7235 block_start, len, len, type); 7236 if (ret) { 7237 free_extent_map(em); 7238 goto unlock_err; 7239 } 7240 goto unlock; 7241 } 7242 } 7243 7244 /* 7245 * this will cow the extent, reset the len in case we changed 7246 * it above 7247 */ 7248 len = bh_result->b_size; 7249 free_extent_map(em); 7250 em = btrfs_new_extent_direct(inode, start, len); 7251 if (IS_ERR(em)) { 7252 ret = PTR_ERR(em); 7253 goto unlock_err; 7254 } 7255 len = min(len, em->len - (start - em->start)); 7256 unlock: 7257 bh_result->b_blocknr = (em->block_start + (start - em->start)) >> 7258 inode->i_blkbits; 7259 bh_result->b_size = len; 7260 bh_result->b_bdev = em->bdev; 7261 set_buffer_mapped(bh_result); 7262 if (create) { 7263 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 7264 set_buffer_new(bh_result); 7265 7266 /* 7267 * Need to update the i_size under the extent lock so buffered 7268 * readers will get the updated i_size when we unlock. 7269 */ 7270 if (start + len > i_size_read(inode)) 7271 i_size_write(inode, start + len); 7272 7273 spin_lock(&BTRFS_I(inode)->lock); 7274 BTRFS_I(inode)->outstanding_extents++; 7275 spin_unlock(&BTRFS_I(inode)->lock); 7276 7277 ret = set_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, 7278 lockstart + len - 1, EXTENT_DELALLOC, NULL, 7279 &cached_state, GFP_NOFS); 7280 BUG_ON(ret); 7281 } 7282 7283 /* 7284 * In the case of write we need to clear and unlock the entire range, 7285 * in the case of read we need to unlock only the end area that we 7286 * aren't using if there is any left over space. 7287 */ 7288 if (lockstart < lockend) { 7289 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, 7290 lockend, unlock_bits, 1, 0, 7291 &cached_state, GFP_NOFS); 7292 } else { 7293 free_extent_state(cached_state); 7294 } 7295 7296 free_extent_map(em); 7297 7298 return 0; 7299 7300 unlock_err: 7301 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, 7302 unlock_bits, 1, 0, &cached_state, GFP_NOFS); 7303 return ret; 7304 } 7305 7306 static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio, 7307 int rw, int mirror_num) 7308 { 7309 struct btrfs_root *root = BTRFS_I(inode)->root; 7310 int ret; 7311 7312 BUG_ON(rw & REQ_WRITE); 7313 7314 bio_get(bio); 7315 7316 ret = btrfs_bio_wq_end_io(root->fs_info, bio, 7317 BTRFS_WQ_ENDIO_DIO_REPAIR); 7318 if (ret) 7319 goto err; 7320 7321 ret = btrfs_map_bio(root, rw, bio, mirror_num, 0); 7322 err: 7323 bio_put(bio); 7324 return ret; 7325 } 7326 7327 static int btrfs_check_dio_repairable(struct inode *inode, 7328 struct bio *failed_bio, 7329 struct io_failure_record *failrec, 7330 int failed_mirror) 7331 { 7332 int num_copies; 7333 7334 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info, 7335 failrec->logical, failrec->len); 7336 if (num_copies == 1) { 7337 /* 7338 * we only have a single copy of the data, so don't bother with 7339 * all the retry and error correction code that follows. no 7340 * matter what the error is, it is very likely to persist. 7341 */ 7342 pr_debug("Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n", 7343 num_copies, failrec->this_mirror, failed_mirror); 7344 return 0; 7345 } 7346 7347 failrec->failed_mirror = failed_mirror; 7348 failrec->this_mirror++; 7349 if (failrec->this_mirror == failed_mirror) 7350 failrec->this_mirror++; 7351 7352 if (failrec->this_mirror > num_copies) { 7353 pr_debug("Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n", 7354 num_copies, failrec->this_mirror, failed_mirror); 7355 return 0; 7356 } 7357 7358 return 1; 7359 } 7360 7361 static int dio_read_error(struct inode *inode, struct bio *failed_bio, 7362 struct page *page, u64 start, u64 end, 7363 int failed_mirror, bio_end_io_t *repair_endio, 7364 void *repair_arg) 7365 { 7366 struct io_failure_record *failrec; 7367 struct bio *bio; 7368 int isector; 7369 int read_mode; 7370 int ret; 7371 7372 BUG_ON(failed_bio->bi_rw & REQ_WRITE); 7373 7374 ret = btrfs_get_io_failure_record(inode, start, end, &failrec); 7375 if (ret) 7376 return ret; 7377 7378 ret = btrfs_check_dio_repairable(inode, failed_bio, failrec, 7379 failed_mirror); 7380 if (!ret) { 7381 free_io_failure(inode, failrec); 7382 return -EIO; 7383 } 7384 7385 if (failed_bio->bi_vcnt > 1) 7386 read_mode = READ_SYNC | REQ_FAILFAST_DEV; 7387 else 7388 read_mode = READ_SYNC; 7389 7390 isector = start - btrfs_io_bio(failed_bio)->logical; 7391 isector >>= inode->i_sb->s_blocksize_bits; 7392 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, 7393 0, isector, repair_endio, repair_arg); 7394 if (!bio) { 7395 free_io_failure(inode, failrec); 7396 return -EIO; 7397 } 7398 7399 btrfs_debug(BTRFS_I(inode)->root->fs_info, 7400 "Repair DIO Read Error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d\n", 7401 read_mode, failrec->this_mirror, failrec->in_validation); 7402 7403 ret = submit_dio_repair_bio(inode, bio, read_mode, 7404 failrec->this_mirror); 7405 if (ret) { 7406 free_io_failure(inode, failrec); 7407 bio_put(bio); 7408 } 7409 7410 return ret; 7411 } 7412 7413 struct btrfs_retry_complete { 7414 struct completion done; 7415 struct inode *inode; 7416 u64 start; 7417 int uptodate; 7418 }; 7419 7420 static void btrfs_retry_endio_nocsum(struct bio *bio, int err) 7421 { 7422 struct btrfs_retry_complete *done = bio->bi_private; 7423 struct bio_vec *bvec; 7424 int i; 7425 7426 if (err) 7427 goto end; 7428 7429 done->uptodate = 1; 7430 bio_for_each_segment_all(bvec, bio, i) 7431 clean_io_failure(done->inode, done->start, bvec->bv_page, 0); 7432 end: 7433 complete(&done->done); 7434 bio_put(bio); 7435 } 7436 7437 static int __btrfs_correct_data_nocsum(struct inode *inode, 7438 struct btrfs_io_bio *io_bio) 7439 { 7440 struct bio_vec *bvec; 7441 struct btrfs_retry_complete done; 7442 u64 start; 7443 int i; 7444 int ret; 7445 7446 start = io_bio->logical; 7447 done.inode = inode; 7448 7449 bio_for_each_segment_all(bvec, &io_bio->bio, i) { 7450 try_again: 7451 done.uptodate = 0; 7452 done.start = start; 7453 init_completion(&done.done); 7454 7455 ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, start, 7456 start + bvec->bv_len - 1, 7457 io_bio->mirror_num, 7458 btrfs_retry_endio_nocsum, &done); 7459 if (ret) 7460 return ret; 7461 7462 wait_for_completion(&done.done); 7463 7464 if (!done.uptodate) { 7465 /* We might have another mirror, so try again */ 7466 goto try_again; 7467 } 7468 7469 start += bvec->bv_len; 7470 } 7471 7472 return 0; 7473 } 7474 7475 static void btrfs_retry_endio(struct bio *bio, int err) 7476 { 7477 struct btrfs_retry_complete *done = bio->bi_private; 7478 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7479 struct bio_vec *bvec; 7480 int uptodate; 7481 int ret; 7482 int i; 7483 7484 if (err) 7485 goto end; 7486 7487 uptodate = 1; 7488 bio_for_each_segment_all(bvec, bio, i) { 7489 ret = __readpage_endio_check(done->inode, io_bio, i, 7490 bvec->bv_page, 0, 7491 done->start, bvec->bv_len); 7492 if (!ret) 7493 clean_io_failure(done->inode, done->start, 7494 bvec->bv_page, 0); 7495 else 7496 uptodate = 0; 7497 } 7498 7499 done->uptodate = uptodate; 7500 end: 7501 complete(&done->done); 7502 bio_put(bio); 7503 } 7504 7505 static int __btrfs_subio_endio_read(struct inode *inode, 7506 struct btrfs_io_bio *io_bio, int err) 7507 { 7508 struct bio_vec *bvec; 7509 struct btrfs_retry_complete done; 7510 u64 start; 7511 u64 offset = 0; 7512 int i; 7513 int ret; 7514 7515 err = 0; 7516 start = io_bio->logical; 7517 done.inode = inode; 7518 7519 bio_for_each_segment_all(bvec, &io_bio->bio, i) { 7520 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page, 7521 0, start, bvec->bv_len); 7522 if (likely(!ret)) 7523 goto next; 7524 try_again: 7525 done.uptodate = 0; 7526 done.start = start; 7527 init_completion(&done.done); 7528 7529 ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, start, 7530 start + bvec->bv_len - 1, 7531 io_bio->mirror_num, 7532 btrfs_retry_endio, &done); 7533 if (ret) { 7534 err = ret; 7535 goto next; 7536 } 7537 7538 wait_for_completion(&done.done); 7539 7540 if (!done.uptodate) { 7541 /* We might have another mirror, so try again */ 7542 goto try_again; 7543 } 7544 next: 7545 offset += bvec->bv_len; 7546 start += bvec->bv_len; 7547 } 7548 7549 return err; 7550 } 7551 7552 static int btrfs_subio_endio_read(struct inode *inode, 7553 struct btrfs_io_bio *io_bio, int err) 7554 { 7555 bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 7556 7557 if (skip_csum) { 7558 if (unlikely(err)) 7559 return __btrfs_correct_data_nocsum(inode, io_bio); 7560 else 7561 return 0; 7562 } else { 7563 return __btrfs_subio_endio_read(inode, io_bio, err); 7564 } 7565 } 7566 7567 static void btrfs_endio_direct_read(struct bio *bio, int err) 7568 { 7569 struct btrfs_dio_private *dip = bio->bi_private; 7570 struct inode *inode = dip->inode; 7571 struct bio *dio_bio; 7572 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7573 7574 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) 7575 err = btrfs_subio_endio_read(inode, io_bio, err); 7576 7577 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset, 7578 dip->logical_offset + dip->bytes - 1); 7579 dio_bio = dip->dio_bio; 7580 7581 kfree(dip); 7582 7583 /* If we had a csum failure make sure to clear the uptodate flag */ 7584 if (err) 7585 clear_bit(BIO_UPTODATE, &dio_bio->bi_flags); 7586 dio_end_io(dio_bio, err); 7587 7588 if (io_bio->end_io) 7589 io_bio->end_io(io_bio, err); 7590 bio_put(bio); 7591 } 7592 7593 static void btrfs_endio_direct_write(struct bio *bio, int err) 7594 { 7595 struct btrfs_dio_private *dip = bio->bi_private; 7596 struct inode *inode = dip->inode; 7597 struct btrfs_root *root = BTRFS_I(inode)->root; 7598 struct btrfs_ordered_extent *ordered = NULL; 7599 u64 ordered_offset = dip->logical_offset; 7600 u64 ordered_bytes = dip->bytes; 7601 struct bio *dio_bio; 7602 int ret; 7603 7604 if (err) 7605 goto out_done; 7606 again: 7607 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, 7608 &ordered_offset, 7609 ordered_bytes, !err); 7610 if (!ret) 7611 goto out_test; 7612 7613 btrfs_init_work(&ordered->work, btrfs_endio_write_helper, 7614 finish_ordered_fn, NULL, NULL); 7615 btrfs_queue_work(root->fs_info->endio_write_workers, 7616 &ordered->work); 7617 out_test: 7618 /* 7619 * our bio might span multiple ordered extents. If we haven't 7620 * completed the accounting for the whole dio, go back and try again 7621 */ 7622 if (ordered_offset < dip->logical_offset + dip->bytes) { 7623 ordered_bytes = dip->logical_offset + dip->bytes - 7624 ordered_offset; 7625 ordered = NULL; 7626 goto again; 7627 } 7628 out_done: 7629 dio_bio = dip->dio_bio; 7630 7631 kfree(dip); 7632 7633 /* If we had an error make sure to clear the uptodate flag */ 7634 if (err) 7635 clear_bit(BIO_UPTODATE, &dio_bio->bi_flags); 7636 dio_end_io(dio_bio, err); 7637 bio_put(bio); 7638 } 7639 7640 static int __btrfs_submit_bio_start_direct_io(struct inode *inode, int rw, 7641 struct bio *bio, int mirror_num, 7642 unsigned long bio_flags, u64 offset) 7643 { 7644 int ret; 7645 struct btrfs_root *root = BTRFS_I(inode)->root; 7646 ret = btrfs_csum_one_bio(root, inode, bio, offset, 1); 7647 BUG_ON(ret); /* -ENOMEM */ 7648 return 0; 7649 } 7650 7651 static void btrfs_end_dio_bio(struct bio *bio, int err) 7652 { 7653 struct btrfs_dio_private *dip = bio->bi_private; 7654 7655 if (err) 7656 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 7657 "direct IO failed ino %llu rw %lu sector %#Lx len %u err no %d", 7658 btrfs_ino(dip->inode), bio->bi_rw, 7659 (unsigned long long)bio->bi_iter.bi_sector, 7660 bio->bi_iter.bi_size, err); 7661 7662 if (dip->subio_endio) 7663 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err); 7664 7665 if (err) { 7666 dip->errors = 1; 7667 7668 /* 7669 * before atomic variable goto zero, we must make sure 7670 * dip->errors is perceived to be set. 7671 */ 7672 smp_mb__before_atomic(); 7673 } 7674 7675 /* if there are more bios still pending for this dio, just exit */ 7676 if (!atomic_dec_and_test(&dip->pending_bios)) 7677 goto out; 7678 7679 if (dip->errors) { 7680 bio_io_error(dip->orig_bio); 7681 } else { 7682 set_bit(BIO_UPTODATE, &dip->dio_bio->bi_flags); 7683 bio_endio(dip->orig_bio, 0); 7684 } 7685 out: 7686 bio_put(bio); 7687 } 7688 7689 static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev, 7690 u64 first_sector, gfp_t gfp_flags) 7691 { 7692 int nr_vecs = bio_get_nr_vecs(bdev); 7693 return btrfs_bio_alloc(bdev, first_sector, nr_vecs, gfp_flags); 7694 } 7695 7696 static inline int btrfs_lookup_and_bind_dio_csum(struct btrfs_root *root, 7697 struct inode *inode, 7698 struct btrfs_dio_private *dip, 7699 struct bio *bio, 7700 u64 file_offset) 7701 { 7702 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7703 struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio); 7704 int ret; 7705 7706 /* 7707 * We load all the csum data we need when we submit 7708 * the first bio to reduce the csum tree search and 7709 * contention. 7710 */ 7711 if (dip->logical_offset == file_offset) { 7712 ret = btrfs_lookup_bio_sums_dio(root, inode, dip->orig_bio, 7713 file_offset); 7714 if (ret) 7715 return ret; 7716 } 7717 7718 if (bio == dip->orig_bio) 7719 return 0; 7720 7721 file_offset -= dip->logical_offset; 7722 file_offset >>= inode->i_sb->s_blocksize_bits; 7723 io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset); 7724 7725 return 0; 7726 } 7727 7728 static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode, 7729 int rw, u64 file_offset, int skip_sum, 7730 int async_submit) 7731 { 7732 struct btrfs_dio_private *dip = bio->bi_private; 7733 int write = rw & REQ_WRITE; 7734 struct btrfs_root *root = BTRFS_I(inode)->root; 7735 int ret; 7736 7737 if (async_submit) 7738 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers); 7739 7740 bio_get(bio); 7741 7742 if (!write) { 7743 ret = btrfs_bio_wq_end_io(root->fs_info, bio, 7744 BTRFS_WQ_ENDIO_DATA); 7745 if (ret) 7746 goto err; 7747 } 7748 7749 if (skip_sum) 7750 goto map; 7751 7752 if (write && async_submit) { 7753 ret = btrfs_wq_submit_bio(root->fs_info, 7754 inode, rw, bio, 0, 0, 7755 file_offset, 7756 __btrfs_submit_bio_start_direct_io, 7757 __btrfs_submit_bio_done); 7758 goto err; 7759 } else if (write) { 7760 /* 7761 * If we aren't doing async submit, calculate the csum of the 7762 * bio now. 7763 */ 7764 ret = btrfs_csum_one_bio(root, inode, bio, file_offset, 1); 7765 if (ret) 7766 goto err; 7767 } else { 7768 ret = btrfs_lookup_and_bind_dio_csum(root, inode, dip, bio, 7769 file_offset); 7770 if (ret) 7771 goto err; 7772 } 7773 map: 7774 ret = btrfs_map_bio(root, rw, bio, 0, async_submit); 7775 err: 7776 bio_put(bio); 7777 return ret; 7778 } 7779 7780 static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip, 7781 int skip_sum) 7782 { 7783 struct inode *inode = dip->inode; 7784 struct btrfs_root *root = BTRFS_I(inode)->root; 7785 struct bio *bio; 7786 struct bio *orig_bio = dip->orig_bio; 7787 struct bio_vec *bvec = orig_bio->bi_io_vec; 7788 u64 start_sector = orig_bio->bi_iter.bi_sector; 7789 u64 file_offset = dip->logical_offset; 7790 u64 submit_len = 0; 7791 u64 map_length; 7792 int nr_pages = 0; 7793 int ret; 7794 int async_submit = 0; 7795 7796 map_length = orig_bio->bi_iter.bi_size; 7797 ret = btrfs_map_block(root->fs_info, rw, start_sector << 9, 7798 &map_length, NULL, 0); 7799 if (ret) 7800 return -EIO; 7801 7802 if (map_length >= orig_bio->bi_iter.bi_size) { 7803 bio = orig_bio; 7804 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED; 7805 goto submit; 7806 } 7807 7808 /* async crcs make it difficult to collect full stripe writes. */ 7809 if (btrfs_get_alloc_profile(root, 1) & 7810 (BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6)) 7811 async_submit = 0; 7812 else 7813 async_submit = 1; 7814 7815 bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, start_sector, GFP_NOFS); 7816 if (!bio) 7817 return -ENOMEM; 7818 7819 bio->bi_private = dip; 7820 bio->bi_end_io = btrfs_end_dio_bio; 7821 btrfs_io_bio(bio)->logical = file_offset; 7822 atomic_inc(&dip->pending_bios); 7823 7824 while (bvec <= (orig_bio->bi_io_vec + orig_bio->bi_vcnt - 1)) { 7825 if (map_length < submit_len + bvec->bv_len || 7826 bio_add_page(bio, bvec->bv_page, bvec->bv_len, 7827 bvec->bv_offset) < bvec->bv_len) { 7828 /* 7829 * inc the count before we submit the bio so 7830 * we know the end IO handler won't happen before 7831 * we inc the count. Otherwise, the dip might get freed 7832 * before we're done setting it up 7833 */ 7834 atomic_inc(&dip->pending_bios); 7835 ret = __btrfs_submit_dio_bio(bio, inode, rw, 7836 file_offset, skip_sum, 7837 async_submit); 7838 if (ret) { 7839 bio_put(bio); 7840 atomic_dec(&dip->pending_bios); 7841 goto out_err; 7842 } 7843 7844 start_sector += submit_len >> 9; 7845 file_offset += submit_len; 7846 7847 submit_len = 0; 7848 nr_pages = 0; 7849 7850 bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, 7851 start_sector, GFP_NOFS); 7852 if (!bio) 7853 goto out_err; 7854 bio->bi_private = dip; 7855 bio->bi_end_io = btrfs_end_dio_bio; 7856 btrfs_io_bio(bio)->logical = file_offset; 7857 7858 map_length = orig_bio->bi_iter.bi_size; 7859 ret = btrfs_map_block(root->fs_info, rw, 7860 start_sector << 9, 7861 &map_length, NULL, 0); 7862 if (ret) { 7863 bio_put(bio); 7864 goto out_err; 7865 } 7866 } else { 7867 submit_len += bvec->bv_len; 7868 nr_pages++; 7869 bvec++; 7870 } 7871 } 7872 7873 submit: 7874 ret = __btrfs_submit_dio_bio(bio, inode, rw, file_offset, skip_sum, 7875 async_submit); 7876 if (!ret) 7877 return 0; 7878 7879 bio_put(bio); 7880 out_err: 7881 dip->errors = 1; 7882 /* 7883 * before atomic variable goto zero, we must 7884 * make sure dip->errors is perceived to be set. 7885 */ 7886 smp_mb__before_atomic(); 7887 if (atomic_dec_and_test(&dip->pending_bios)) 7888 bio_io_error(dip->orig_bio); 7889 7890 /* bio_end_io() will handle error, so we needn't return it */ 7891 return 0; 7892 } 7893 7894 static void btrfs_submit_direct(int rw, struct bio *dio_bio, 7895 struct inode *inode, loff_t file_offset) 7896 { 7897 struct btrfs_root *root = BTRFS_I(inode)->root; 7898 struct btrfs_dio_private *dip; 7899 struct bio *io_bio; 7900 struct btrfs_io_bio *btrfs_bio; 7901 int skip_sum; 7902 int write = rw & REQ_WRITE; 7903 int ret = 0; 7904 7905 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; 7906 7907 io_bio = btrfs_bio_clone(dio_bio, GFP_NOFS); 7908 if (!io_bio) { 7909 ret = -ENOMEM; 7910 goto free_ordered; 7911 } 7912 7913 dip = kzalloc(sizeof(*dip), GFP_NOFS); 7914 if (!dip) { 7915 ret = -ENOMEM; 7916 goto free_io_bio; 7917 } 7918 7919 dip->private = dio_bio->bi_private; 7920 dip->inode = inode; 7921 dip->logical_offset = file_offset; 7922 dip->bytes = dio_bio->bi_iter.bi_size; 7923 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; 7924 io_bio->bi_private = dip; 7925 dip->orig_bio = io_bio; 7926 dip->dio_bio = dio_bio; 7927 atomic_set(&dip->pending_bios, 0); 7928 btrfs_bio = btrfs_io_bio(io_bio); 7929 btrfs_bio->logical = file_offset; 7930 7931 if (write) { 7932 io_bio->bi_end_io = btrfs_endio_direct_write; 7933 } else { 7934 io_bio->bi_end_io = btrfs_endio_direct_read; 7935 dip->subio_endio = btrfs_subio_endio_read; 7936 } 7937 7938 ret = btrfs_submit_direct_hook(rw, dip, skip_sum); 7939 if (!ret) 7940 return; 7941 7942 if (btrfs_bio->end_io) 7943 btrfs_bio->end_io(btrfs_bio, ret); 7944 free_io_bio: 7945 bio_put(io_bio); 7946 7947 free_ordered: 7948 /* 7949 * If this is a write, we need to clean up the reserved space and kill 7950 * the ordered extent. 7951 */ 7952 if (write) { 7953 struct btrfs_ordered_extent *ordered; 7954 ordered = btrfs_lookup_ordered_extent(inode, file_offset); 7955 if (!test_bit(BTRFS_ORDERED_PREALLOC, &ordered->flags) && 7956 !test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) 7957 btrfs_free_reserved_extent(root, ordered->start, 7958 ordered->disk_len, 1); 7959 btrfs_put_ordered_extent(ordered); 7960 btrfs_put_ordered_extent(ordered); 7961 } 7962 bio_endio(dio_bio, ret); 7963 } 7964 7965 static ssize_t check_direct_IO(struct btrfs_root *root, int rw, struct kiocb *iocb, 7966 const struct iov_iter *iter, loff_t offset) 7967 { 7968 int seg; 7969 int i; 7970 unsigned blocksize_mask = root->sectorsize - 1; 7971 ssize_t retval = -EINVAL; 7972 7973 if (offset & blocksize_mask) 7974 goto out; 7975 7976 if (iov_iter_alignment(iter) & blocksize_mask) 7977 goto out; 7978 7979 /* If this is a write we don't need to check anymore */ 7980 if (rw & WRITE) 7981 return 0; 7982 /* 7983 * Check to make sure we don't have duplicate iov_base's in this 7984 * iovec, if so return EINVAL, otherwise we'll get csum errors 7985 * when reading back. 7986 */ 7987 for (seg = 0; seg < iter->nr_segs; seg++) { 7988 for (i = seg + 1; i < iter->nr_segs; i++) { 7989 if (iter->iov[seg].iov_base == iter->iov[i].iov_base) 7990 goto out; 7991 } 7992 } 7993 retval = 0; 7994 out: 7995 return retval; 7996 } 7997 7998 static ssize_t btrfs_direct_IO(int rw, struct kiocb *iocb, 7999 struct iov_iter *iter, loff_t offset) 8000 { 8001 struct file *file = iocb->ki_filp; 8002 struct inode *inode = file->f_mapping->host; 8003 size_t count = 0; 8004 int flags = 0; 8005 bool wakeup = true; 8006 bool relock = false; 8007 ssize_t ret; 8008 8009 if (check_direct_IO(BTRFS_I(inode)->root, rw, iocb, iter, offset)) 8010 return 0; 8011 8012 atomic_inc(&inode->i_dio_count); 8013 smp_mb__after_atomic(); 8014 8015 /* 8016 * The generic stuff only does filemap_write_and_wait_range, which 8017 * isn't enough if we've written compressed pages to this area, so 8018 * we need to flush the dirty pages again to make absolutely sure 8019 * that any outstanding dirty pages are on disk. 8020 */ 8021 count = iov_iter_count(iter); 8022 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 8023 &BTRFS_I(inode)->runtime_flags)) 8024 filemap_fdatawrite_range(inode->i_mapping, offset, 8025 offset + count - 1); 8026 8027 if (rw & WRITE) { 8028 /* 8029 * If the write DIO is beyond the EOF, we need update 8030 * the isize, but it is protected by i_mutex. So we can 8031 * not unlock the i_mutex at this case. 8032 */ 8033 if (offset + count <= inode->i_size) { 8034 mutex_unlock(&inode->i_mutex); 8035 relock = true; 8036 } 8037 ret = btrfs_delalloc_reserve_space(inode, count); 8038 if (ret) 8039 goto out; 8040 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK, 8041 &BTRFS_I(inode)->runtime_flags)) { 8042 inode_dio_done(inode); 8043 flags = DIO_LOCKING | DIO_SKIP_HOLES; 8044 wakeup = false; 8045 } 8046 8047 ret = __blockdev_direct_IO(rw, iocb, inode, 8048 BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev, 8049 iter, offset, btrfs_get_blocks_direct, NULL, 8050 btrfs_submit_direct, flags); 8051 if (rw & WRITE) { 8052 if (ret < 0 && ret != -EIOCBQUEUED) 8053 btrfs_delalloc_release_space(inode, count); 8054 else if (ret >= 0 && (size_t)ret < count) 8055 btrfs_delalloc_release_space(inode, 8056 count - (size_t)ret); 8057 else 8058 btrfs_delalloc_release_metadata(inode, 0); 8059 } 8060 out: 8061 if (wakeup) 8062 inode_dio_done(inode); 8063 if (relock) 8064 mutex_lock(&inode->i_mutex); 8065 8066 return ret; 8067 } 8068 8069 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC) 8070 8071 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 8072 __u64 start, __u64 len) 8073 { 8074 int ret; 8075 8076 ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS); 8077 if (ret) 8078 return ret; 8079 8080 return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap); 8081 } 8082 8083 int btrfs_readpage(struct file *file, struct page *page) 8084 { 8085 struct extent_io_tree *tree; 8086 tree = &BTRFS_I(page->mapping->host)->io_tree; 8087 return extent_read_full_page(tree, page, btrfs_get_extent, 0); 8088 } 8089 8090 static int btrfs_writepage(struct page *page, struct writeback_control *wbc) 8091 { 8092 struct extent_io_tree *tree; 8093 8094 8095 if (current->flags & PF_MEMALLOC) { 8096 redirty_page_for_writepage(wbc, page); 8097 unlock_page(page); 8098 return 0; 8099 } 8100 tree = &BTRFS_I(page->mapping->host)->io_tree; 8101 return extent_write_full_page(tree, page, btrfs_get_extent, wbc); 8102 } 8103 8104 static int btrfs_writepages(struct address_space *mapping, 8105 struct writeback_control *wbc) 8106 { 8107 struct extent_io_tree *tree; 8108 8109 tree = &BTRFS_I(mapping->host)->io_tree; 8110 return extent_writepages(tree, mapping, btrfs_get_extent, wbc); 8111 } 8112 8113 static int 8114 btrfs_readpages(struct file *file, struct address_space *mapping, 8115 struct list_head *pages, unsigned nr_pages) 8116 { 8117 struct extent_io_tree *tree; 8118 tree = &BTRFS_I(mapping->host)->io_tree; 8119 return extent_readpages(tree, mapping, pages, nr_pages, 8120 btrfs_get_extent); 8121 } 8122 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8123 { 8124 struct extent_io_tree *tree; 8125 struct extent_map_tree *map; 8126 int ret; 8127 8128 tree = &BTRFS_I(page->mapping->host)->io_tree; 8129 map = &BTRFS_I(page->mapping->host)->extent_tree; 8130 ret = try_release_extent_mapping(map, tree, page, gfp_flags); 8131 if (ret == 1) { 8132 ClearPagePrivate(page); 8133 set_page_private(page, 0); 8134 page_cache_release(page); 8135 } 8136 return ret; 8137 } 8138 8139 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags) 8140 { 8141 if (PageWriteback(page) || PageDirty(page)) 8142 return 0; 8143 return __btrfs_releasepage(page, gfp_flags & GFP_NOFS); 8144 } 8145 8146 static void btrfs_invalidatepage(struct page *page, unsigned int offset, 8147 unsigned int length) 8148 { 8149 struct inode *inode = page->mapping->host; 8150 struct extent_io_tree *tree; 8151 struct btrfs_ordered_extent *ordered; 8152 struct extent_state *cached_state = NULL; 8153 u64 page_start = page_offset(page); 8154 u64 page_end = page_start + PAGE_CACHE_SIZE - 1; 8155 int inode_evicting = inode->i_state & I_FREEING; 8156 8157 /* 8158 * we have the page locked, so new writeback can't start, 8159 * and the dirty bit won't be cleared while we are here. 8160 * 8161 * Wait for IO on this page so that we can safely clear 8162 * the PagePrivate2 bit and do ordered accounting 8163 */ 8164 wait_on_page_writeback(page); 8165 8166 tree = &BTRFS_I(inode)->io_tree; 8167 if (offset) { 8168 btrfs_releasepage(page, GFP_NOFS); 8169 return; 8170 } 8171 8172 if (!inode_evicting) 8173 lock_extent_bits(tree, page_start, page_end, 0, &cached_state); 8174 ordered = btrfs_lookup_ordered_extent(inode, page_start); 8175 if (ordered) { 8176 /* 8177 * IO on this page will never be started, so we need 8178 * to account for any ordered extents now 8179 */ 8180 if (!inode_evicting) 8181 clear_extent_bit(tree, page_start, page_end, 8182 EXTENT_DIRTY | EXTENT_DELALLOC | 8183 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 8184 EXTENT_DEFRAG, 1, 0, &cached_state, 8185 GFP_NOFS); 8186 /* 8187 * whoever cleared the private bit is responsible 8188 * for the finish_ordered_io 8189 */ 8190 if (TestClearPagePrivate2(page)) { 8191 struct btrfs_ordered_inode_tree *tree; 8192 u64 new_len; 8193 8194 tree = &BTRFS_I(inode)->ordered_tree; 8195 8196 spin_lock_irq(&tree->lock); 8197 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 8198 new_len = page_start - ordered->file_offset; 8199 if (new_len < ordered->truncated_len) 8200 ordered->truncated_len = new_len; 8201 spin_unlock_irq(&tree->lock); 8202 8203 if (btrfs_dec_test_ordered_pending(inode, &ordered, 8204 page_start, 8205 PAGE_CACHE_SIZE, 1)) 8206 btrfs_finish_ordered_io(ordered); 8207 } 8208 btrfs_put_ordered_extent(ordered); 8209 if (!inode_evicting) { 8210 cached_state = NULL; 8211 lock_extent_bits(tree, page_start, page_end, 0, 8212 &cached_state); 8213 } 8214 } 8215 8216 if (!inode_evicting) { 8217 clear_extent_bit(tree, page_start, page_end, 8218 EXTENT_LOCKED | EXTENT_DIRTY | 8219 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 8220 EXTENT_DEFRAG, 1, 1, 8221 &cached_state, GFP_NOFS); 8222 8223 __btrfs_releasepage(page, GFP_NOFS); 8224 } 8225 8226 ClearPageChecked(page); 8227 if (PagePrivate(page)) { 8228 ClearPagePrivate(page); 8229 set_page_private(page, 0); 8230 page_cache_release(page); 8231 } 8232 } 8233 8234 /* 8235 * btrfs_page_mkwrite() is not allowed to change the file size as it gets 8236 * called from a page fault handler when a page is first dirtied. Hence we must 8237 * be careful to check for EOF conditions here. We set the page up correctly 8238 * for a written page which means we get ENOSPC checking when writing into 8239 * holes and correct delalloc and unwritten extent mapping on filesystems that 8240 * support these features. 8241 * 8242 * We are not allowed to take the i_mutex here so we have to play games to 8243 * protect against truncate races as the page could now be beyond EOF. Because 8244 * vmtruncate() writes the inode size before removing pages, once we have the 8245 * page lock we can determine safely if the page is beyond EOF. If it is not 8246 * beyond EOF, then the page is guaranteed safe against truncation until we 8247 * unlock the page. 8248 */ 8249 int btrfs_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 8250 { 8251 struct page *page = vmf->page; 8252 struct inode *inode = file_inode(vma->vm_file); 8253 struct btrfs_root *root = BTRFS_I(inode)->root; 8254 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 8255 struct btrfs_ordered_extent *ordered; 8256 struct extent_state *cached_state = NULL; 8257 char *kaddr; 8258 unsigned long zero_start; 8259 loff_t size; 8260 int ret; 8261 int reserved = 0; 8262 u64 page_start; 8263 u64 page_end; 8264 8265 sb_start_pagefault(inode->i_sb); 8266 ret = btrfs_delalloc_reserve_space(inode, PAGE_CACHE_SIZE); 8267 if (!ret) { 8268 ret = file_update_time(vma->vm_file); 8269 reserved = 1; 8270 } 8271 if (ret) { 8272 if (ret == -ENOMEM) 8273 ret = VM_FAULT_OOM; 8274 else /* -ENOSPC, -EIO, etc */ 8275 ret = VM_FAULT_SIGBUS; 8276 if (reserved) 8277 goto out; 8278 goto out_noreserve; 8279 } 8280 8281 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */ 8282 again: 8283 lock_page(page); 8284 size = i_size_read(inode); 8285 page_start = page_offset(page); 8286 page_end = page_start + PAGE_CACHE_SIZE - 1; 8287 8288 if ((page->mapping != inode->i_mapping) || 8289 (page_start >= size)) { 8290 /* page got truncated out from underneath us */ 8291 goto out_unlock; 8292 } 8293 wait_on_page_writeback(page); 8294 8295 lock_extent_bits(io_tree, page_start, page_end, 0, &cached_state); 8296 set_page_extent_mapped(page); 8297 8298 /* 8299 * we can't set the delalloc bits if there are pending ordered 8300 * extents. Drop our locks and wait for them to finish 8301 */ 8302 ordered = btrfs_lookup_ordered_extent(inode, page_start); 8303 if (ordered) { 8304 unlock_extent_cached(io_tree, page_start, page_end, 8305 &cached_state, GFP_NOFS); 8306 unlock_page(page); 8307 btrfs_start_ordered_extent(inode, ordered, 1); 8308 btrfs_put_ordered_extent(ordered); 8309 goto again; 8310 } 8311 8312 /* 8313 * XXX - page_mkwrite gets called every time the page is dirtied, even 8314 * if it was already dirty, so for space accounting reasons we need to 8315 * clear any delalloc bits for the range we are fixing to save. There 8316 * is probably a better way to do this, but for now keep consistent with 8317 * prepare_pages in the normal write path. 8318 */ 8319 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, page_end, 8320 EXTENT_DIRTY | EXTENT_DELALLOC | 8321 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 8322 0, 0, &cached_state, GFP_NOFS); 8323 8324 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 8325 &cached_state); 8326 if (ret) { 8327 unlock_extent_cached(io_tree, page_start, page_end, 8328 &cached_state, GFP_NOFS); 8329 ret = VM_FAULT_SIGBUS; 8330 goto out_unlock; 8331 } 8332 ret = 0; 8333 8334 /* page is wholly or partially inside EOF */ 8335 if (page_start + PAGE_CACHE_SIZE > size) 8336 zero_start = size & ~PAGE_CACHE_MASK; 8337 else 8338 zero_start = PAGE_CACHE_SIZE; 8339 8340 if (zero_start != PAGE_CACHE_SIZE) { 8341 kaddr = kmap(page); 8342 memset(kaddr + zero_start, 0, PAGE_CACHE_SIZE - zero_start); 8343 flush_dcache_page(page); 8344 kunmap(page); 8345 } 8346 ClearPageChecked(page); 8347 set_page_dirty(page); 8348 SetPageUptodate(page); 8349 8350 BTRFS_I(inode)->last_trans = root->fs_info->generation; 8351 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid; 8352 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit; 8353 8354 unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS); 8355 8356 out_unlock: 8357 if (!ret) { 8358 sb_end_pagefault(inode->i_sb); 8359 return VM_FAULT_LOCKED; 8360 } 8361 unlock_page(page); 8362 out: 8363 btrfs_delalloc_release_space(inode, PAGE_CACHE_SIZE); 8364 out_noreserve: 8365 sb_end_pagefault(inode->i_sb); 8366 return ret; 8367 } 8368 8369 static int btrfs_truncate(struct inode *inode) 8370 { 8371 struct btrfs_root *root = BTRFS_I(inode)->root; 8372 struct btrfs_block_rsv *rsv; 8373 int ret = 0; 8374 int err = 0; 8375 struct btrfs_trans_handle *trans; 8376 u64 mask = root->sectorsize - 1; 8377 u64 min_size = btrfs_calc_trunc_metadata_size(root, 1); 8378 8379 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), 8380 (u64)-1); 8381 if (ret) 8382 return ret; 8383 8384 /* 8385 * Yes ladies and gentelment, this is indeed ugly. The fact is we have 8386 * 3 things going on here 8387 * 8388 * 1) We need to reserve space for our orphan item and the space to 8389 * delete our orphan item. Lord knows we don't want to have a dangling 8390 * orphan item because we didn't reserve space to remove it. 8391 * 8392 * 2) We need to reserve space to update our inode. 8393 * 8394 * 3) We need to have something to cache all the space that is going to 8395 * be free'd up by the truncate operation, but also have some slack 8396 * space reserved in case it uses space during the truncate (thank you 8397 * very much snapshotting). 8398 * 8399 * And we need these to all be seperate. The fact is we can use alot of 8400 * space doing the truncate, and we have no earthly idea how much space 8401 * we will use, so we need the truncate reservation to be seperate so it 8402 * doesn't end up using space reserved for updating the inode or 8403 * removing the orphan item. We also need to be able to stop the 8404 * transaction and start a new one, which means we need to be able to 8405 * update the inode several times, and we have no idea of knowing how 8406 * many times that will be, so we can't just reserve 1 item for the 8407 * entirety of the opration, so that has to be done seperately as well. 8408 * Then there is the orphan item, which does indeed need to be held on 8409 * to for the whole operation, and we need nobody to touch this reserved 8410 * space except the orphan code. 8411 * 8412 * So that leaves us with 8413 * 8414 * 1) root->orphan_block_rsv - for the orphan deletion. 8415 * 2) rsv - for the truncate reservation, which we will steal from the 8416 * transaction reservation. 8417 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for 8418 * updating the inode. 8419 */ 8420 rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP); 8421 if (!rsv) 8422 return -ENOMEM; 8423 rsv->size = min_size; 8424 rsv->failfast = 1; 8425 8426 /* 8427 * 1 for the truncate slack space 8428 * 1 for updating the inode. 8429 */ 8430 trans = btrfs_start_transaction(root, 2); 8431 if (IS_ERR(trans)) { 8432 err = PTR_ERR(trans); 8433 goto out; 8434 } 8435 8436 /* Migrate the slack space for the truncate to our reserve */ 8437 ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv, rsv, 8438 min_size); 8439 BUG_ON(ret); 8440 8441 /* 8442 * So if we truncate and then write and fsync we normally would just 8443 * write the extents that changed, which is a problem if we need to 8444 * first truncate that entire inode. So set this flag so we write out 8445 * all of the extents in the inode to the sync log so we're completely 8446 * safe. 8447 */ 8448 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); 8449 trans->block_rsv = rsv; 8450 8451 while (1) { 8452 ret = btrfs_truncate_inode_items(trans, root, inode, 8453 inode->i_size, 8454 BTRFS_EXTENT_DATA_KEY); 8455 if (ret != -ENOSPC) { 8456 err = ret; 8457 break; 8458 } 8459 8460 trans->block_rsv = &root->fs_info->trans_block_rsv; 8461 ret = btrfs_update_inode(trans, root, inode); 8462 if (ret) { 8463 err = ret; 8464 break; 8465 } 8466 8467 btrfs_end_transaction(trans, root); 8468 btrfs_btree_balance_dirty(root); 8469 8470 trans = btrfs_start_transaction(root, 2); 8471 if (IS_ERR(trans)) { 8472 ret = err = PTR_ERR(trans); 8473 trans = NULL; 8474 break; 8475 } 8476 8477 ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv, 8478 rsv, min_size); 8479 BUG_ON(ret); /* shouldn't happen */ 8480 trans->block_rsv = rsv; 8481 } 8482 8483 if (ret == 0 && inode->i_nlink > 0) { 8484 trans->block_rsv = root->orphan_block_rsv; 8485 ret = btrfs_orphan_del(trans, inode); 8486 if (ret) 8487 err = ret; 8488 } 8489 8490 if (trans) { 8491 trans->block_rsv = &root->fs_info->trans_block_rsv; 8492 ret = btrfs_update_inode(trans, root, inode); 8493 if (ret && !err) 8494 err = ret; 8495 8496 ret = btrfs_end_transaction(trans, root); 8497 btrfs_btree_balance_dirty(root); 8498 } 8499 8500 out: 8501 btrfs_free_block_rsv(root, rsv); 8502 8503 if (ret && !err) 8504 err = ret; 8505 8506 return err; 8507 } 8508 8509 /* 8510 * create a new subvolume directory/inode (helper for the ioctl). 8511 */ 8512 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, 8513 struct btrfs_root *new_root, 8514 struct btrfs_root *parent_root, 8515 u64 new_dirid) 8516 { 8517 struct inode *inode; 8518 int err; 8519 u64 index = 0; 8520 8521 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, 8522 new_dirid, new_dirid, 8523 S_IFDIR | (~current_umask() & S_IRWXUGO), 8524 &index); 8525 if (IS_ERR(inode)) 8526 return PTR_ERR(inode); 8527 inode->i_op = &btrfs_dir_inode_operations; 8528 inode->i_fop = &btrfs_dir_file_operations; 8529 8530 set_nlink(inode, 1); 8531 btrfs_i_size_write(inode, 0); 8532 unlock_new_inode(inode); 8533 8534 err = btrfs_subvol_inherit_props(trans, new_root, parent_root); 8535 if (err) 8536 btrfs_err(new_root->fs_info, 8537 "error inheriting subvolume %llu properties: %d", 8538 new_root->root_key.objectid, err); 8539 8540 err = btrfs_update_inode(trans, new_root, inode); 8541 8542 iput(inode); 8543 return err; 8544 } 8545 8546 struct inode *btrfs_alloc_inode(struct super_block *sb) 8547 { 8548 struct btrfs_inode *ei; 8549 struct inode *inode; 8550 8551 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS); 8552 if (!ei) 8553 return NULL; 8554 8555 ei->root = NULL; 8556 ei->generation = 0; 8557 ei->last_trans = 0; 8558 ei->last_sub_trans = 0; 8559 ei->logged_trans = 0; 8560 ei->delalloc_bytes = 0; 8561 ei->defrag_bytes = 0; 8562 ei->disk_i_size = 0; 8563 ei->flags = 0; 8564 ei->csum_bytes = 0; 8565 ei->index_cnt = (u64)-1; 8566 ei->dir_index = 0; 8567 ei->last_unlink_trans = 0; 8568 ei->last_log_commit = 0; 8569 8570 spin_lock_init(&ei->lock); 8571 ei->outstanding_extents = 0; 8572 ei->reserved_extents = 0; 8573 8574 ei->runtime_flags = 0; 8575 ei->force_compress = BTRFS_COMPRESS_NONE; 8576 8577 ei->delayed_node = NULL; 8578 8579 inode = &ei->vfs_inode; 8580 extent_map_tree_init(&ei->extent_tree); 8581 extent_io_tree_init(&ei->io_tree, &inode->i_data); 8582 extent_io_tree_init(&ei->io_failure_tree, &inode->i_data); 8583 ei->io_tree.track_uptodate = 1; 8584 ei->io_failure_tree.track_uptodate = 1; 8585 atomic_set(&ei->sync_writers, 0); 8586 mutex_init(&ei->log_mutex); 8587 mutex_init(&ei->delalloc_mutex); 8588 btrfs_ordered_inode_tree_init(&ei->ordered_tree); 8589 INIT_LIST_HEAD(&ei->delalloc_inodes); 8590 RB_CLEAR_NODE(&ei->rb_node); 8591 8592 return inode; 8593 } 8594 8595 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 8596 void btrfs_test_destroy_inode(struct inode *inode) 8597 { 8598 btrfs_drop_extent_cache(inode, 0, (u64)-1, 0); 8599 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8600 } 8601 #endif 8602 8603 static void btrfs_i_callback(struct rcu_head *head) 8604 { 8605 struct inode *inode = container_of(head, struct inode, i_rcu); 8606 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8607 } 8608 8609 void btrfs_destroy_inode(struct inode *inode) 8610 { 8611 struct btrfs_ordered_extent *ordered; 8612 struct btrfs_root *root = BTRFS_I(inode)->root; 8613 8614 WARN_ON(!hlist_empty(&inode->i_dentry)); 8615 WARN_ON(inode->i_data.nrpages); 8616 WARN_ON(BTRFS_I(inode)->outstanding_extents); 8617 WARN_ON(BTRFS_I(inode)->reserved_extents); 8618 WARN_ON(BTRFS_I(inode)->delalloc_bytes); 8619 WARN_ON(BTRFS_I(inode)->csum_bytes); 8620 WARN_ON(BTRFS_I(inode)->defrag_bytes); 8621 8622 /* 8623 * This can happen where we create an inode, but somebody else also 8624 * created the same inode and we need to destroy the one we already 8625 * created. 8626 */ 8627 if (!root) 8628 goto free; 8629 8630 if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM, 8631 &BTRFS_I(inode)->runtime_flags)) { 8632 btrfs_info(root->fs_info, "inode %llu still on the orphan list", 8633 btrfs_ino(inode)); 8634 atomic_dec(&root->orphan_inodes); 8635 } 8636 8637 while (1) { 8638 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 8639 if (!ordered) 8640 break; 8641 else { 8642 btrfs_err(root->fs_info, "found ordered extent %llu %llu on inode cleanup", 8643 ordered->file_offset, ordered->len); 8644 btrfs_remove_ordered_extent(inode, ordered); 8645 btrfs_put_ordered_extent(ordered); 8646 btrfs_put_ordered_extent(ordered); 8647 } 8648 } 8649 inode_tree_del(inode); 8650 btrfs_drop_extent_cache(inode, 0, (u64)-1, 0); 8651 free: 8652 call_rcu(&inode->i_rcu, btrfs_i_callback); 8653 } 8654 8655 int btrfs_drop_inode(struct inode *inode) 8656 { 8657 struct btrfs_root *root = BTRFS_I(inode)->root; 8658 8659 if (root == NULL) 8660 return 1; 8661 8662 /* the snap/subvol tree is on deleting */ 8663 if (btrfs_root_refs(&root->root_item) == 0) 8664 return 1; 8665 else 8666 return generic_drop_inode(inode); 8667 } 8668 8669 static void init_once(void *foo) 8670 { 8671 struct btrfs_inode *ei = (struct btrfs_inode *) foo; 8672 8673 inode_init_once(&ei->vfs_inode); 8674 } 8675 8676 void btrfs_destroy_cachep(void) 8677 { 8678 /* 8679 * Make sure all delayed rcu free inodes are flushed before we 8680 * destroy cache. 8681 */ 8682 rcu_barrier(); 8683 if (btrfs_inode_cachep) 8684 kmem_cache_destroy(btrfs_inode_cachep); 8685 if (btrfs_trans_handle_cachep) 8686 kmem_cache_destroy(btrfs_trans_handle_cachep); 8687 if (btrfs_transaction_cachep) 8688 kmem_cache_destroy(btrfs_transaction_cachep); 8689 if (btrfs_path_cachep) 8690 kmem_cache_destroy(btrfs_path_cachep); 8691 if (btrfs_free_space_cachep) 8692 kmem_cache_destroy(btrfs_free_space_cachep); 8693 if (btrfs_delalloc_work_cachep) 8694 kmem_cache_destroy(btrfs_delalloc_work_cachep); 8695 } 8696 8697 int btrfs_init_cachep(void) 8698 { 8699 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 8700 sizeof(struct btrfs_inode), 0, 8701 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, init_once); 8702 if (!btrfs_inode_cachep) 8703 goto fail; 8704 8705 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", 8706 sizeof(struct btrfs_trans_handle), 0, 8707 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 8708 if (!btrfs_trans_handle_cachep) 8709 goto fail; 8710 8711 btrfs_transaction_cachep = kmem_cache_create("btrfs_transaction", 8712 sizeof(struct btrfs_transaction), 0, 8713 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 8714 if (!btrfs_transaction_cachep) 8715 goto fail; 8716 8717 btrfs_path_cachep = kmem_cache_create("btrfs_path", 8718 sizeof(struct btrfs_path), 0, 8719 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 8720 if (!btrfs_path_cachep) 8721 goto fail; 8722 8723 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space", 8724 sizeof(struct btrfs_free_space), 0, 8725 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 8726 if (!btrfs_free_space_cachep) 8727 goto fail; 8728 8729 btrfs_delalloc_work_cachep = kmem_cache_create("btrfs_delalloc_work", 8730 sizeof(struct btrfs_delalloc_work), 0, 8731 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, 8732 NULL); 8733 if (!btrfs_delalloc_work_cachep) 8734 goto fail; 8735 8736 return 0; 8737 fail: 8738 btrfs_destroy_cachep(); 8739 return -ENOMEM; 8740 } 8741 8742 static int btrfs_getattr(struct vfsmount *mnt, 8743 struct dentry *dentry, struct kstat *stat) 8744 { 8745 u64 delalloc_bytes; 8746 struct inode *inode = dentry->d_inode; 8747 u32 blocksize = inode->i_sb->s_blocksize; 8748 8749 generic_fillattr(inode, stat); 8750 stat->dev = BTRFS_I(inode)->root->anon_dev; 8751 stat->blksize = PAGE_CACHE_SIZE; 8752 8753 spin_lock(&BTRFS_I(inode)->lock); 8754 delalloc_bytes = BTRFS_I(inode)->delalloc_bytes; 8755 spin_unlock(&BTRFS_I(inode)->lock); 8756 stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) + 8757 ALIGN(delalloc_bytes, blocksize)) >> 9; 8758 return 0; 8759 } 8760 8761 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry, 8762 struct inode *new_dir, struct dentry *new_dentry) 8763 { 8764 struct btrfs_trans_handle *trans; 8765 struct btrfs_root *root = BTRFS_I(old_dir)->root; 8766 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 8767 struct inode *new_inode = new_dentry->d_inode; 8768 struct inode *old_inode = old_dentry->d_inode; 8769 struct timespec ctime = CURRENT_TIME; 8770 u64 index = 0; 8771 u64 root_objectid; 8772 int ret; 8773 u64 old_ino = btrfs_ino(old_inode); 8774 8775 if (btrfs_ino(new_dir) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 8776 return -EPERM; 8777 8778 /* we only allow rename subvolume link between subvolumes */ 8779 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 8780 return -EXDEV; 8781 8782 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 8783 (new_inode && btrfs_ino(new_inode) == BTRFS_FIRST_FREE_OBJECTID)) 8784 return -ENOTEMPTY; 8785 8786 if (S_ISDIR(old_inode->i_mode) && new_inode && 8787 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 8788 return -ENOTEMPTY; 8789 8790 8791 /* check for collisions, even if the name isn't there */ 8792 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, 8793 new_dentry->d_name.name, 8794 new_dentry->d_name.len); 8795 8796 if (ret) { 8797 if (ret == -EEXIST) { 8798 /* we shouldn't get 8799 * eexist without a new_inode */ 8800 if (WARN_ON(!new_inode)) { 8801 return ret; 8802 } 8803 } else { 8804 /* maybe -EOVERFLOW */ 8805 return ret; 8806 } 8807 } 8808 ret = 0; 8809 8810 /* 8811 * we're using rename to replace one file with another. Start IO on it 8812 * now so we don't add too much work to the end of the transaction 8813 */ 8814 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 8815 filemap_flush(old_inode->i_mapping); 8816 8817 /* close the racy window with snapshot create/destroy ioctl */ 8818 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 8819 down_read(&root->fs_info->subvol_sem); 8820 /* 8821 * We want to reserve the absolute worst case amount of items. So if 8822 * both inodes are subvols and we need to unlink them then that would 8823 * require 4 item modifications, but if they are both normal inodes it 8824 * would require 5 item modifications, so we'll assume their normal 8825 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items 8826 * should cover the worst case number of items we'll modify. 8827 */ 8828 trans = btrfs_start_transaction(root, 11); 8829 if (IS_ERR(trans)) { 8830 ret = PTR_ERR(trans); 8831 goto out_notrans; 8832 } 8833 8834 if (dest != root) 8835 btrfs_record_root_in_trans(trans, dest); 8836 8837 ret = btrfs_set_inode_index(new_dir, &index); 8838 if (ret) 8839 goto out_fail; 8840 8841 BTRFS_I(old_inode)->dir_index = 0ULL; 8842 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 8843 /* force full log commit if subvolume involved. */ 8844 btrfs_set_log_full_commit(root->fs_info, trans); 8845 } else { 8846 ret = btrfs_insert_inode_ref(trans, dest, 8847 new_dentry->d_name.name, 8848 new_dentry->d_name.len, 8849 old_ino, 8850 btrfs_ino(new_dir), index); 8851 if (ret) 8852 goto out_fail; 8853 /* 8854 * this is an ugly little race, but the rename is required 8855 * to make sure that if we crash, the inode is either at the 8856 * old name or the new one. pinning the log transaction lets 8857 * us make sure we don't allow a log commit to come in after 8858 * we unlink the name but before we add the new name back in. 8859 */ 8860 btrfs_pin_log_trans(root); 8861 } 8862 8863 inode_inc_iversion(old_dir); 8864 inode_inc_iversion(new_dir); 8865 inode_inc_iversion(old_inode); 8866 old_dir->i_ctime = old_dir->i_mtime = ctime; 8867 new_dir->i_ctime = new_dir->i_mtime = ctime; 8868 old_inode->i_ctime = ctime; 8869 8870 if (old_dentry->d_parent != new_dentry->d_parent) 8871 btrfs_record_unlink_dir(trans, old_dir, old_inode, 1); 8872 8873 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 8874 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid; 8875 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid, 8876 old_dentry->d_name.name, 8877 old_dentry->d_name.len); 8878 } else { 8879 ret = __btrfs_unlink_inode(trans, root, old_dir, 8880 old_dentry->d_inode, 8881 old_dentry->d_name.name, 8882 old_dentry->d_name.len); 8883 if (!ret) 8884 ret = btrfs_update_inode(trans, root, old_inode); 8885 } 8886 if (ret) { 8887 btrfs_abort_transaction(trans, root, ret); 8888 goto out_fail; 8889 } 8890 8891 if (new_inode) { 8892 inode_inc_iversion(new_inode); 8893 new_inode->i_ctime = CURRENT_TIME; 8894 if (unlikely(btrfs_ino(new_inode) == 8895 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 8896 root_objectid = BTRFS_I(new_inode)->location.objectid; 8897 ret = btrfs_unlink_subvol(trans, dest, new_dir, 8898 root_objectid, 8899 new_dentry->d_name.name, 8900 new_dentry->d_name.len); 8901 BUG_ON(new_inode->i_nlink == 0); 8902 } else { 8903 ret = btrfs_unlink_inode(trans, dest, new_dir, 8904 new_dentry->d_inode, 8905 new_dentry->d_name.name, 8906 new_dentry->d_name.len); 8907 } 8908 if (!ret && new_inode->i_nlink == 0) 8909 ret = btrfs_orphan_add(trans, new_dentry->d_inode); 8910 if (ret) { 8911 btrfs_abort_transaction(trans, root, ret); 8912 goto out_fail; 8913 } 8914 } 8915 8916 ret = btrfs_add_link(trans, new_dir, old_inode, 8917 new_dentry->d_name.name, 8918 new_dentry->d_name.len, 0, index); 8919 if (ret) { 8920 btrfs_abort_transaction(trans, root, ret); 8921 goto out_fail; 8922 } 8923 8924 if (old_inode->i_nlink == 1) 8925 BTRFS_I(old_inode)->dir_index = index; 8926 8927 if (old_ino != BTRFS_FIRST_FREE_OBJECTID) { 8928 struct dentry *parent = new_dentry->d_parent; 8929 btrfs_log_new_name(trans, old_inode, old_dir, parent); 8930 btrfs_end_log_trans(root); 8931 } 8932 out_fail: 8933 btrfs_end_transaction(trans, root); 8934 out_notrans: 8935 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 8936 up_read(&root->fs_info->subvol_sem); 8937 8938 return ret; 8939 } 8940 8941 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, 8942 struct inode *new_dir, struct dentry *new_dentry, 8943 unsigned int flags) 8944 { 8945 if (flags & ~RENAME_NOREPLACE) 8946 return -EINVAL; 8947 8948 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry); 8949 } 8950 8951 static void btrfs_run_delalloc_work(struct btrfs_work *work) 8952 { 8953 struct btrfs_delalloc_work *delalloc_work; 8954 struct inode *inode; 8955 8956 delalloc_work = container_of(work, struct btrfs_delalloc_work, 8957 work); 8958 inode = delalloc_work->inode; 8959 if (delalloc_work->wait) { 8960 btrfs_wait_ordered_range(inode, 0, (u64)-1); 8961 } else { 8962 filemap_flush(inode->i_mapping); 8963 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 8964 &BTRFS_I(inode)->runtime_flags)) 8965 filemap_flush(inode->i_mapping); 8966 } 8967 8968 if (delalloc_work->delay_iput) 8969 btrfs_add_delayed_iput(inode); 8970 else 8971 iput(inode); 8972 complete(&delalloc_work->completion); 8973 } 8974 8975 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode, 8976 int wait, int delay_iput) 8977 { 8978 struct btrfs_delalloc_work *work; 8979 8980 work = kmem_cache_zalloc(btrfs_delalloc_work_cachep, GFP_NOFS); 8981 if (!work) 8982 return NULL; 8983 8984 init_completion(&work->completion); 8985 INIT_LIST_HEAD(&work->list); 8986 work->inode = inode; 8987 work->wait = wait; 8988 work->delay_iput = delay_iput; 8989 WARN_ON_ONCE(!inode); 8990 btrfs_init_work(&work->work, btrfs_flush_delalloc_helper, 8991 btrfs_run_delalloc_work, NULL, NULL); 8992 8993 return work; 8994 } 8995 8996 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work) 8997 { 8998 wait_for_completion(&work->completion); 8999 kmem_cache_free(btrfs_delalloc_work_cachep, work); 9000 } 9001 9002 /* 9003 * some fairly slow code that needs optimization. This walks the list 9004 * of all the inodes with pending delalloc and forces them to disk. 9005 */ 9006 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput, 9007 int nr) 9008 { 9009 struct btrfs_inode *binode; 9010 struct inode *inode; 9011 struct btrfs_delalloc_work *work, *next; 9012 struct list_head works; 9013 struct list_head splice; 9014 int ret = 0; 9015 9016 INIT_LIST_HEAD(&works); 9017 INIT_LIST_HEAD(&splice); 9018 9019 mutex_lock(&root->delalloc_mutex); 9020 spin_lock(&root->delalloc_lock); 9021 list_splice_init(&root->delalloc_inodes, &splice); 9022 while (!list_empty(&splice)) { 9023 binode = list_entry(splice.next, struct btrfs_inode, 9024 delalloc_inodes); 9025 9026 list_move_tail(&binode->delalloc_inodes, 9027 &root->delalloc_inodes); 9028 inode = igrab(&binode->vfs_inode); 9029 if (!inode) { 9030 cond_resched_lock(&root->delalloc_lock); 9031 continue; 9032 } 9033 spin_unlock(&root->delalloc_lock); 9034 9035 work = btrfs_alloc_delalloc_work(inode, 0, delay_iput); 9036 if (!work) { 9037 if (delay_iput) 9038 btrfs_add_delayed_iput(inode); 9039 else 9040 iput(inode); 9041 ret = -ENOMEM; 9042 goto out; 9043 } 9044 list_add_tail(&work->list, &works); 9045 btrfs_queue_work(root->fs_info->flush_workers, 9046 &work->work); 9047 ret++; 9048 if (nr != -1 && ret >= nr) 9049 goto out; 9050 cond_resched(); 9051 spin_lock(&root->delalloc_lock); 9052 } 9053 spin_unlock(&root->delalloc_lock); 9054 9055 out: 9056 list_for_each_entry_safe(work, next, &works, list) { 9057 list_del_init(&work->list); 9058 btrfs_wait_and_free_delalloc_work(work); 9059 } 9060 9061 if (!list_empty_careful(&splice)) { 9062 spin_lock(&root->delalloc_lock); 9063 list_splice_tail(&splice, &root->delalloc_inodes); 9064 spin_unlock(&root->delalloc_lock); 9065 } 9066 mutex_unlock(&root->delalloc_mutex); 9067 return ret; 9068 } 9069 9070 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput) 9071 { 9072 int ret; 9073 9074 if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state)) 9075 return -EROFS; 9076 9077 ret = __start_delalloc_inodes(root, delay_iput, -1); 9078 if (ret > 0) 9079 ret = 0; 9080 /* 9081 * the filemap_flush will queue IO into the worker threads, but 9082 * we have to make sure the IO is actually started and that 9083 * ordered extents get created before we return 9084 */ 9085 atomic_inc(&root->fs_info->async_submit_draining); 9086 while (atomic_read(&root->fs_info->nr_async_submits) || 9087 atomic_read(&root->fs_info->async_delalloc_pages)) { 9088 wait_event(root->fs_info->async_submit_wait, 9089 (atomic_read(&root->fs_info->nr_async_submits) == 0 && 9090 atomic_read(&root->fs_info->async_delalloc_pages) == 0)); 9091 } 9092 atomic_dec(&root->fs_info->async_submit_draining); 9093 return ret; 9094 } 9095 9096 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput, 9097 int nr) 9098 { 9099 struct btrfs_root *root; 9100 struct list_head splice; 9101 int ret; 9102 9103 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 9104 return -EROFS; 9105 9106 INIT_LIST_HEAD(&splice); 9107 9108 mutex_lock(&fs_info->delalloc_root_mutex); 9109 spin_lock(&fs_info->delalloc_root_lock); 9110 list_splice_init(&fs_info->delalloc_roots, &splice); 9111 while (!list_empty(&splice) && nr) { 9112 root = list_first_entry(&splice, struct btrfs_root, 9113 delalloc_root); 9114 root = btrfs_grab_fs_root(root); 9115 BUG_ON(!root); 9116 list_move_tail(&root->delalloc_root, 9117 &fs_info->delalloc_roots); 9118 spin_unlock(&fs_info->delalloc_root_lock); 9119 9120 ret = __start_delalloc_inodes(root, delay_iput, nr); 9121 btrfs_put_fs_root(root); 9122 if (ret < 0) 9123 goto out; 9124 9125 if (nr != -1) { 9126 nr -= ret; 9127 WARN_ON(nr < 0); 9128 } 9129 spin_lock(&fs_info->delalloc_root_lock); 9130 } 9131 spin_unlock(&fs_info->delalloc_root_lock); 9132 9133 ret = 0; 9134 atomic_inc(&fs_info->async_submit_draining); 9135 while (atomic_read(&fs_info->nr_async_submits) || 9136 atomic_read(&fs_info->async_delalloc_pages)) { 9137 wait_event(fs_info->async_submit_wait, 9138 (atomic_read(&fs_info->nr_async_submits) == 0 && 9139 atomic_read(&fs_info->async_delalloc_pages) == 0)); 9140 } 9141 atomic_dec(&fs_info->async_submit_draining); 9142 out: 9143 if (!list_empty_careful(&splice)) { 9144 spin_lock(&fs_info->delalloc_root_lock); 9145 list_splice_tail(&splice, &fs_info->delalloc_roots); 9146 spin_unlock(&fs_info->delalloc_root_lock); 9147 } 9148 mutex_unlock(&fs_info->delalloc_root_mutex); 9149 return ret; 9150 } 9151 9152 static int btrfs_symlink(struct inode *dir, struct dentry *dentry, 9153 const char *symname) 9154 { 9155 struct btrfs_trans_handle *trans; 9156 struct btrfs_root *root = BTRFS_I(dir)->root; 9157 struct btrfs_path *path; 9158 struct btrfs_key key; 9159 struct inode *inode = NULL; 9160 int err; 9161 int drop_inode = 0; 9162 u64 objectid; 9163 u64 index = 0; 9164 int name_len; 9165 int datasize; 9166 unsigned long ptr; 9167 struct btrfs_file_extent_item *ei; 9168 struct extent_buffer *leaf; 9169 9170 name_len = strlen(symname); 9171 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(root)) 9172 return -ENAMETOOLONG; 9173 9174 /* 9175 * 2 items for inode item and ref 9176 * 2 items for dir items 9177 * 1 item for xattr if selinux is on 9178 */ 9179 trans = btrfs_start_transaction(root, 5); 9180 if (IS_ERR(trans)) 9181 return PTR_ERR(trans); 9182 9183 err = btrfs_find_free_ino(root, &objectid); 9184 if (err) 9185 goto out_unlock; 9186 9187 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name, 9188 dentry->d_name.len, btrfs_ino(dir), objectid, 9189 S_IFLNK|S_IRWXUGO, &index); 9190 if (IS_ERR(inode)) { 9191 err = PTR_ERR(inode); 9192 goto out_unlock; 9193 } 9194 9195 /* 9196 * If the active LSM wants to access the inode during 9197 * d_instantiate it needs these. Smack checks to see 9198 * if the filesystem supports xattrs by looking at the 9199 * ops vector. 9200 */ 9201 inode->i_fop = &btrfs_file_operations; 9202 inode->i_op = &btrfs_file_inode_operations; 9203 inode->i_mapping->a_ops = &btrfs_aops; 9204 inode->i_mapping->backing_dev_info = &root->fs_info->bdi; 9205 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 9206 9207 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name); 9208 if (err) 9209 goto out_unlock_inode; 9210 9211 err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index); 9212 if (err) 9213 goto out_unlock_inode; 9214 9215 path = btrfs_alloc_path(); 9216 if (!path) { 9217 err = -ENOMEM; 9218 goto out_unlock_inode; 9219 } 9220 key.objectid = btrfs_ino(inode); 9221 key.offset = 0; 9222 key.type = BTRFS_EXTENT_DATA_KEY; 9223 datasize = btrfs_file_extent_calc_inline_size(name_len); 9224 err = btrfs_insert_empty_item(trans, root, path, &key, 9225 datasize); 9226 if (err) { 9227 btrfs_free_path(path); 9228 goto out_unlock_inode; 9229 } 9230 leaf = path->nodes[0]; 9231 ei = btrfs_item_ptr(leaf, path->slots[0], 9232 struct btrfs_file_extent_item); 9233 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 9234 btrfs_set_file_extent_type(leaf, ei, 9235 BTRFS_FILE_EXTENT_INLINE); 9236 btrfs_set_file_extent_encryption(leaf, ei, 0); 9237 btrfs_set_file_extent_compression(leaf, ei, 0); 9238 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 9239 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 9240 9241 ptr = btrfs_file_extent_inline_start(ei); 9242 write_extent_buffer(leaf, symname, ptr, name_len); 9243 btrfs_mark_buffer_dirty(leaf); 9244 btrfs_free_path(path); 9245 9246 inode->i_op = &btrfs_symlink_inode_operations; 9247 inode->i_mapping->a_ops = &btrfs_symlink_aops; 9248 inode->i_mapping->backing_dev_info = &root->fs_info->bdi; 9249 inode_set_bytes(inode, name_len); 9250 btrfs_i_size_write(inode, name_len); 9251 err = btrfs_update_inode(trans, root, inode); 9252 if (err) { 9253 drop_inode = 1; 9254 goto out_unlock_inode; 9255 } 9256 9257 unlock_new_inode(inode); 9258 d_instantiate(dentry, inode); 9259 9260 out_unlock: 9261 btrfs_end_transaction(trans, root); 9262 if (drop_inode) { 9263 inode_dec_link_count(inode); 9264 iput(inode); 9265 } 9266 btrfs_btree_balance_dirty(root); 9267 return err; 9268 9269 out_unlock_inode: 9270 drop_inode = 1; 9271 unlock_new_inode(inode); 9272 goto out_unlock; 9273 } 9274 9275 static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 9276 u64 start, u64 num_bytes, u64 min_size, 9277 loff_t actual_len, u64 *alloc_hint, 9278 struct btrfs_trans_handle *trans) 9279 { 9280 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 9281 struct extent_map *em; 9282 struct btrfs_root *root = BTRFS_I(inode)->root; 9283 struct btrfs_key ins; 9284 u64 cur_offset = start; 9285 u64 i_size; 9286 u64 cur_bytes; 9287 int ret = 0; 9288 bool own_trans = true; 9289 9290 if (trans) 9291 own_trans = false; 9292 while (num_bytes > 0) { 9293 if (own_trans) { 9294 trans = btrfs_start_transaction(root, 3); 9295 if (IS_ERR(trans)) { 9296 ret = PTR_ERR(trans); 9297 break; 9298 } 9299 } 9300 9301 cur_bytes = min(num_bytes, 256ULL * 1024 * 1024); 9302 cur_bytes = max(cur_bytes, min_size); 9303 ret = btrfs_reserve_extent(root, cur_bytes, min_size, 0, 9304 *alloc_hint, &ins, 1, 0); 9305 if (ret) { 9306 if (own_trans) 9307 btrfs_end_transaction(trans, root); 9308 break; 9309 } 9310 9311 ret = insert_reserved_file_extent(trans, inode, 9312 cur_offset, ins.objectid, 9313 ins.offset, ins.offset, 9314 ins.offset, 0, 0, 0, 9315 BTRFS_FILE_EXTENT_PREALLOC); 9316 if (ret) { 9317 btrfs_free_reserved_extent(root, ins.objectid, 9318 ins.offset, 0); 9319 btrfs_abort_transaction(trans, root, ret); 9320 if (own_trans) 9321 btrfs_end_transaction(trans, root); 9322 break; 9323 } 9324 btrfs_drop_extent_cache(inode, cur_offset, 9325 cur_offset + ins.offset -1, 0); 9326 9327 em = alloc_extent_map(); 9328 if (!em) { 9329 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, 9330 &BTRFS_I(inode)->runtime_flags); 9331 goto next; 9332 } 9333 9334 em->start = cur_offset; 9335 em->orig_start = cur_offset; 9336 em->len = ins.offset; 9337 em->block_start = ins.objectid; 9338 em->block_len = ins.offset; 9339 em->orig_block_len = ins.offset; 9340 em->ram_bytes = ins.offset; 9341 em->bdev = root->fs_info->fs_devices->latest_bdev; 9342 set_bit(EXTENT_FLAG_PREALLOC, &em->flags); 9343 em->generation = trans->transid; 9344 9345 while (1) { 9346 write_lock(&em_tree->lock); 9347 ret = add_extent_mapping(em_tree, em, 1); 9348 write_unlock(&em_tree->lock); 9349 if (ret != -EEXIST) 9350 break; 9351 btrfs_drop_extent_cache(inode, cur_offset, 9352 cur_offset + ins.offset - 1, 9353 0); 9354 } 9355 free_extent_map(em); 9356 next: 9357 num_bytes -= ins.offset; 9358 cur_offset += ins.offset; 9359 *alloc_hint = ins.objectid + ins.offset; 9360 9361 inode_inc_iversion(inode); 9362 inode->i_ctime = CURRENT_TIME; 9363 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 9364 if (!(mode & FALLOC_FL_KEEP_SIZE) && 9365 (actual_len > inode->i_size) && 9366 (cur_offset > inode->i_size)) { 9367 if (cur_offset > actual_len) 9368 i_size = actual_len; 9369 else 9370 i_size = cur_offset; 9371 i_size_write(inode, i_size); 9372 btrfs_ordered_update_i_size(inode, i_size, NULL); 9373 } 9374 9375 ret = btrfs_update_inode(trans, root, inode); 9376 9377 if (ret) { 9378 btrfs_abort_transaction(trans, root, ret); 9379 if (own_trans) 9380 btrfs_end_transaction(trans, root); 9381 break; 9382 } 9383 9384 if (own_trans) 9385 btrfs_end_transaction(trans, root); 9386 } 9387 return ret; 9388 } 9389 9390 int btrfs_prealloc_file_range(struct inode *inode, int mode, 9391 u64 start, u64 num_bytes, u64 min_size, 9392 loff_t actual_len, u64 *alloc_hint) 9393 { 9394 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9395 min_size, actual_len, alloc_hint, 9396 NULL); 9397 } 9398 9399 int btrfs_prealloc_file_range_trans(struct inode *inode, 9400 struct btrfs_trans_handle *trans, int mode, 9401 u64 start, u64 num_bytes, u64 min_size, 9402 loff_t actual_len, u64 *alloc_hint) 9403 { 9404 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9405 min_size, actual_len, alloc_hint, trans); 9406 } 9407 9408 static int btrfs_set_page_dirty(struct page *page) 9409 { 9410 return __set_page_dirty_nobuffers(page); 9411 } 9412 9413 static int btrfs_permission(struct inode *inode, int mask) 9414 { 9415 struct btrfs_root *root = BTRFS_I(inode)->root; 9416 umode_t mode = inode->i_mode; 9417 9418 if (mask & MAY_WRITE && 9419 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 9420 if (btrfs_root_readonly(root)) 9421 return -EROFS; 9422 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 9423 return -EACCES; 9424 } 9425 return generic_permission(inode, mask); 9426 } 9427 9428 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 9429 { 9430 struct btrfs_trans_handle *trans; 9431 struct btrfs_root *root = BTRFS_I(dir)->root; 9432 struct inode *inode = NULL; 9433 u64 objectid; 9434 u64 index; 9435 int ret = 0; 9436 9437 /* 9438 * 5 units required for adding orphan entry 9439 */ 9440 trans = btrfs_start_transaction(root, 5); 9441 if (IS_ERR(trans)) 9442 return PTR_ERR(trans); 9443 9444 ret = btrfs_find_free_ino(root, &objectid); 9445 if (ret) 9446 goto out; 9447 9448 inode = btrfs_new_inode(trans, root, dir, NULL, 0, 9449 btrfs_ino(dir), objectid, mode, &index); 9450 if (IS_ERR(inode)) { 9451 ret = PTR_ERR(inode); 9452 inode = NULL; 9453 goto out; 9454 } 9455 9456 inode->i_fop = &btrfs_file_operations; 9457 inode->i_op = &btrfs_file_inode_operations; 9458 9459 inode->i_mapping->a_ops = &btrfs_aops; 9460 inode->i_mapping->backing_dev_info = &root->fs_info->bdi; 9461 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops; 9462 9463 ret = btrfs_init_inode_security(trans, inode, dir, NULL); 9464 if (ret) 9465 goto out_inode; 9466 9467 ret = btrfs_update_inode(trans, root, inode); 9468 if (ret) 9469 goto out_inode; 9470 ret = btrfs_orphan_add(trans, inode); 9471 if (ret) 9472 goto out_inode; 9473 9474 /* 9475 * We set number of links to 0 in btrfs_new_inode(), and here we set 9476 * it to 1 because d_tmpfile() will issue a warning if the count is 0, 9477 * through: 9478 * 9479 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 9480 */ 9481 set_nlink(inode, 1); 9482 unlock_new_inode(inode); 9483 d_tmpfile(dentry, inode); 9484 mark_inode_dirty(inode); 9485 9486 out: 9487 btrfs_end_transaction(trans, root); 9488 if (ret) 9489 iput(inode); 9490 btrfs_balance_delayed_items(root); 9491 btrfs_btree_balance_dirty(root); 9492 return ret; 9493 9494 out_inode: 9495 unlock_new_inode(inode); 9496 goto out; 9497 9498 } 9499 9500 /* Inspired by filemap_check_errors() */ 9501 int btrfs_inode_check_errors(struct inode *inode) 9502 { 9503 int ret = 0; 9504 9505 if (test_bit(AS_ENOSPC, &inode->i_mapping->flags) && 9506 test_and_clear_bit(AS_ENOSPC, &inode->i_mapping->flags)) 9507 ret = -ENOSPC; 9508 if (test_bit(AS_EIO, &inode->i_mapping->flags) && 9509 test_and_clear_bit(AS_EIO, &inode->i_mapping->flags)) 9510 ret = -EIO; 9511 9512 return ret; 9513 } 9514 9515 static const struct inode_operations btrfs_dir_inode_operations = { 9516 .getattr = btrfs_getattr, 9517 .lookup = btrfs_lookup, 9518 .create = btrfs_create, 9519 .unlink = btrfs_unlink, 9520 .link = btrfs_link, 9521 .mkdir = btrfs_mkdir, 9522 .rmdir = btrfs_rmdir, 9523 .rename2 = btrfs_rename2, 9524 .symlink = btrfs_symlink, 9525 .setattr = btrfs_setattr, 9526 .mknod = btrfs_mknod, 9527 .setxattr = btrfs_setxattr, 9528 .getxattr = btrfs_getxattr, 9529 .listxattr = btrfs_listxattr, 9530 .removexattr = btrfs_removexattr, 9531 .permission = btrfs_permission, 9532 .get_acl = btrfs_get_acl, 9533 .set_acl = btrfs_set_acl, 9534 .update_time = btrfs_update_time, 9535 .tmpfile = btrfs_tmpfile, 9536 }; 9537 static const struct inode_operations btrfs_dir_ro_inode_operations = { 9538 .lookup = btrfs_lookup, 9539 .permission = btrfs_permission, 9540 .get_acl = btrfs_get_acl, 9541 .set_acl = btrfs_set_acl, 9542 .update_time = btrfs_update_time, 9543 }; 9544 9545 static const struct file_operations btrfs_dir_file_operations = { 9546 .llseek = generic_file_llseek, 9547 .read = generic_read_dir, 9548 .iterate = btrfs_real_readdir, 9549 .unlocked_ioctl = btrfs_ioctl, 9550 #ifdef CONFIG_COMPAT 9551 .compat_ioctl = btrfs_ioctl, 9552 #endif 9553 .release = btrfs_release_file, 9554 .fsync = btrfs_sync_file, 9555 }; 9556 9557 static struct extent_io_ops btrfs_extent_io_ops = { 9558 .fill_delalloc = run_delalloc_range, 9559 .submit_bio_hook = btrfs_submit_bio_hook, 9560 .merge_bio_hook = btrfs_merge_bio_hook, 9561 .readpage_end_io_hook = btrfs_readpage_end_io_hook, 9562 .writepage_end_io_hook = btrfs_writepage_end_io_hook, 9563 .writepage_start_hook = btrfs_writepage_start_hook, 9564 .set_bit_hook = btrfs_set_bit_hook, 9565 .clear_bit_hook = btrfs_clear_bit_hook, 9566 .merge_extent_hook = btrfs_merge_extent_hook, 9567 .split_extent_hook = btrfs_split_extent_hook, 9568 }; 9569 9570 /* 9571 * btrfs doesn't support the bmap operation because swapfiles 9572 * use bmap to make a mapping of extents in the file. They assume 9573 * these extents won't change over the life of the file and they 9574 * use the bmap result to do IO directly to the drive. 9575 * 9576 * the btrfs bmap call would return logical addresses that aren't 9577 * suitable for IO and they also will change frequently as COW 9578 * operations happen. So, swapfile + btrfs == corruption. 9579 * 9580 * For now we're avoiding this by dropping bmap. 9581 */ 9582 static const struct address_space_operations btrfs_aops = { 9583 .readpage = btrfs_readpage, 9584 .writepage = btrfs_writepage, 9585 .writepages = btrfs_writepages, 9586 .readpages = btrfs_readpages, 9587 .direct_IO = btrfs_direct_IO, 9588 .invalidatepage = btrfs_invalidatepage, 9589 .releasepage = btrfs_releasepage, 9590 .set_page_dirty = btrfs_set_page_dirty, 9591 .error_remove_page = generic_error_remove_page, 9592 }; 9593 9594 static const struct address_space_operations btrfs_symlink_aops = { 9595 .readpage = btrfs_readpage, 9596 .writepage = btrfs_writepage, 9597 .invalidatepage = btrfs_invalidatepage, 9598 .releasepage = btrfs_releasepage, 9599 }; 9600 9601 static const struct inode_operations btrfs_file_inode_operations = { 9602 .getattr = btrfs_getattr, 9603 .setattr = btrfs_setattr, 9604 .setxattr = btrfs_setxattr, 9605 .getxattr = btrfs_getxattr, 9606 .listxattr = btrfs_listxattr, 9607 .removexattr = btrfs_removexattr, 9608 .permission = btrfs_permission, 9609 .fiemap = btrfs_fiemap, 9610 .get_acl = btrfs_get_acl, 9611 .set_acl = btrfs_set_acl, 9612 .update_time = btrfs_update_time, 9613 }; 9614 static const struct inode_operations btrfs_special_inode_operations = { 9615 .getattr = btrfs_getattr, 9616 .setattr = btrfs_setattr, 9617 .permission = btrfs_permission, 9618 .setxattr = btrfs_setxattr, 9619 .getxattr = btrfs_getxattr, 9620 .listxattr = btrfs_listxattr, 9621 .removexattr = btrfs_removexattr, 9622 .get_acl = btrfs_get_acl, 9623 .set_acl = btrfs_set_acl, 9624 .update_time = btrfs_update_time, 9625 }; 9626 static const struct inode_operations btrfs_symlink_inode_operations = { 9627 .readlink = generic_readlink, 9628 .follow_link = page_follow_link_light, 9629 .put_link = page_put_link, 9630 .getattr = btrfs_getattr, 9631 .setattr = btrfs_setattr, 9632 .permission = btrfs_permission, 9633 .setxattr = btrfs_setxattr, 9634 .getxattr = btrfs_getxattr, 9635 .listxattr = btrfs_listxattr, 9636 .removexattr = btrfs_removexattr, 9637 .update_time = btrfs_update_time, 9638 }; 9639 9640 const struct dentry_operations btrfs_dentry_operations = { 9641 .d_delete = btrfs_dentry_delete, 9642 .d_release = btrfs_dentry_release, 9643 }; 9644