1 #include <linux/bitops.h> 2 #include <linux/slab.h> 3 #include <linux/bio.h> 4 #include <linux/mm.h> 5 #include <linux/gfp.h> 6 #include <linux/pagemap.h> 7 #include <linux/page-flags.h> 8 #include <linux/module.h> 9 #include <linux/spinlock.h> 10 #include <linux/blkdev.h> 11 #include <linux/swap.h> 12 #include <linux/writeback.h> 13 #include <linux/pagevec.h> 14 #include "extent_io.h" 15 #include "extent_map.h" 16 #include "compat.h" 17 #include "ctree.h" 18 #include "btrfs_inode.h" 19 20 static struct kmem_cache *extent_state_cache; 21 static struct kmem_cache *extent_buffer_cache; 22 23 static LIST_HEAD(buffers); 24 static LIST_HEAD(states); 25 26 #define LEAK_DEBUG 0 27 #if LEAK_DEBUG 28 static DEFINE_SPINLOCK(leak_lock); 29 #endif 30 31 #define BUFFER_LRU_MAX 64 32 33 struct tree_entry { 34 u64 start; 35 u64 end; 36 struct rb_node rb_node; 37 }; 38 39 struct extent_page_data { 40 struct bio *bio; 41 struct extent_io_tree *tree; 42 get_extent_t *get_extent; 43 44 /* tells writepage not to lock the state bits for this range 45 * it still does the unlocking 46 */ 47 unsigned int extent_locked:1; 48 49 /* tells the submit_bio code to use a WRITE_SYNC */ 50 unsigned int sync_io:1; 51 }; 52 53 int __init extent_io_init(void) 54 { 55 extent_state_cache = kmem_cache_create("extent_state", 56 sizeof(struct extent_state), 0, 57 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 58 if (!extent_state_cache) 59 return -ENOMEM; 60 61 extent_buffer_cache = kmem_cache_create("extent_buffers", 62 sizeof(struct extent_buffer), 0, 63 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 64 if (!extent_buffer_cache) 65 goto free_state_cache; 66 return 0; 67 68 free_state_cache: 69 kmem_cache_destroy(extent_state_cache); 70 return -ENOMEM; 71 } 72 73 void extent_io_exit(void) 74 { 75 struct extent_state *state; 76 struct extent_buffer *eb; 77 78 while (!list_empty(&states)) { 79 state = list_entry(states.next, struct extent_state, leak_list); 80 printk(KERN_ERR "btrfs state leak: start %llu end %llu " 81 "state %lu in tree %p refs %d\n", 82 (unsigned long long)state->start, 83 (unsigned long long)state->end, 84 state->state, state->tree, atomic_read(&state->refs)); 85 list_del(&state->leak_list); 86 kmem_cache_free(extent_state_cache, state); 87 88 } 89 90 while (!list_empty(&buffers)) { 91 eb = list_entry(buffers.next, struct extent_buffer, leak_list); 92 printk(KERN_ERR "btrfs buffer leak start %llu len %lu " 93 "refs %d\n", (unsigned long long)eb->start, 94 eb->len, atomic_read(&eb->refs)); 95 list_del(&eb->leak_list); 96 kmem_cache_free(extent_buffer_cache, eb); 97 } 98 if (extent_state_cache) 99 kmem_cache_destroy(extent_state_cache); 100 if (extent_buffer_cache) 101 kmem_cache_destroy(extent_buffer_cache); 102 } 103 104 void extent_io_tree_init(struct extent_io_tree *tree, 105 struct address_space *mapping, gfp_t mask) 106 { 107 tree->state.rb_node = NULL; 108 tree->buffer.rb_node = NULL; 109 tree->ops = NULL; 110 tree->dirty_bytes = 0; 111 spin_lock_init(&tree->lock); 112 spin_lock_init(&tree->buffer_lock); 113 tree->mapping = mapping; 114 } 115 116 static struct extent_state *alloc_extent_state(gfp_t mask) 117 { 118 struct extent_state *state; 119 #if LEAK_DEBUG 120 unsigned long flags; 121 #endif 122 123 state = kmem_cache_alloc(extent_state_cache, mask); 124 if (!state) 125 return state; 126 state->state = 0; 127 state->private = 0; 128 state->tree = NULL; 129 #if LEAK_DEBUG 130 spin_lock_irqsave(&leak_lock, flags); 131 list_add(&state->leak_list, &states); 132 spin_unlock_irqrestore(&leak_lock, flags); 133 #endif 134 atomic_set(&state->refs, 1); 135 init_waitqueue_head(&state->wq); 136 return state; 137 } 138 139 static void free_extent_state(struct extent_state *state) 140 { 141 if (!state) 142 return; 143 if (atomic_dec_and_test(&state->refs)) { 144 #if LEAK_DEBUG 145 unsigned long flags; 146 #endif 147 WARN_ON(state->tree); 148 #if LEAK_DEBUG 149 spin_lock_irqsave(&leak_lock, flags); 150 list_del(&state->leak_list); 151 spin_unlock_irqrestore(&leak_lock, flags); 152 #endif 153 kmem_cache_free(extent_state_cache, state); 154 } 155 } 156 157 static struct rb_node *tree_insert(struct rb_root *root, u64 offset, 158 struct rb_node *node) 159 { 160 struct rb_node **p = &root->rb_node; 161 struct rb_node *parent = NULL; 162 struct tree_entry *entry; 163 164 while (*p) { 165 parent = *p; 166 entry = rb_entry(parent, struct tree_entry, rb_node); 167 168 if (offset < entry->start) 169 p = &(*p)->rb_left; 170 else if (offset > entry->end) 171 p = &(*p)->rb_right; 172 else 173 return parent; 174 } 175 176 entry = rb_entry(node, struct tree_entry, rb_node); 177 rb_link_node(node, parent, p); 178 rb_insert_color(node, root); 179 return NULL; 180 } 181 182 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset, 183 struct rb_node **prev_ret, 184 struct rb_node **next_ret) 185 { 186 struct rb_root *root = &tree->state; 187 struct rb_node *n = root->rb_node; 188 struct rb_node *prev = NULL; 189 struct rb_node *orig_prev = NULL; 190 struct tree_entry *entry; 191 struct tree_entry *prev_entry = NULL; 192 193 while (n) { 194 entry = rb_entry(n, struct tree_entry, rb_node); 195 prev = n; 196 prev_entry = entry; 197 198 if (offset < entry->start) 199 n = n->rb_left; 200 else if (offset > entry->end) 201 n = n->rb_right; 202 else 203 return n; 204 } 205 206 if (prev_ret) { 207 orig_prev = prev; 208 while (prev && offset > prev_entry->end) { 209 prev = rb_next(prev); 210 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 211 } 212 *prev_ret = prev; 213 prev = orig_prev; 214 } 215 216 if (next_ret) { 217 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 218 while (prev && offset < prev_entry->start) { 219 prev = rb_prev(prev); 220 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 221 } 222 *next_ret = prev; 223 } 224 return NULL; 225 } 226 227 static inline struct rb_node *tree_search(struct extent_io_tree *tree, 228 u64 offset) 229 { 230 struct rb_node *prev = NULL; 231 struct rb_node *ret; 232 233 ret = __etree_search(tree, offset, &prev, NULL); 234 if (!ret) 235 return prev; 236 return ret; 237 } 238 239 static struct extent_buffer *buffer_tree_insert(struct extent_io_tree *tree, 240 u64 offset, struct rb_node *node) 241 { 242 struct rb_root *root = &tree->buffer; 243 struct rb_node **p = &root->rb_node; 244 struct rb_node *parent = NULL; 245 struct extent_buffer *eb; 246 247 while (*p) { 248 parent = *p; 249 eb = rb_entry(parent, struct extent_buffer, rb_node); 250 251 if (offset < eb->start) 252 p = &(*p)->rb_left; 253 else if (offset > eb->start) 254 p = &(*p)->rb_right; 255 else 256 return eb; 257 } 258 259 rb_link_node(node, parent, p); 260 rb_insert_color(node, root); 261 return NULL; 262 } 263 264 static struct extent_buffer *buffer_search(struct extent_io_tree *tree, 265 u64 offset) 266 { 267 struct rb_root *root = &tree->buffer; 268 struct rb_node *n = root->rb_node; 269 struct extent_buffer *eb; 270 271 while (n) { 272 eb = rb_entry(n, struct extent_buffer, rb_node); 273 if (offset < eb->start) 274 n = n->rb_left; 275 else if (offset > eb->start) 276 n = n->rb_right; 277 else 278 return eb; 279 } 280 return NULL; 281 } 282 283 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new, 284 struct extent_state *other) 285 { 286 if (tree->ops && tree->ops->merge_extent_hook) 287 tree->ops->merge_extent_hook(tree->mapping->host, new, 288 other); 289 } 290 291 /* 292 * utility function to look for merge candidates inside a given range. 293 * Any extents with matching state are merged together into a single 294 * extent in the tree. Extents with EXTENT_IO in their state field 295 * are not merged because the end_io handlers need to be able to do 296 * operations on them without sleeping (or doing allocations/splits). 297 * 298 * This should be called with the tree lock held. 299 */ 300 static int merge_state(struct extent_io_tree *tree, 301 struct extent_state *state) 302 { 303 struct extent_state *other; 304 struct rb_node *other_node; 305 306 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) 307 return 0; 308 309 other_node = rb_prev(&state->rb_node); 310 if (other_node) { 311 other = rb_entry(other_node, struct extent_state, rb_node); 312 if (other->end == state->start - 1 && 313 other->state == state->state) { 314 merge_cb(tree, state, other); 315 state->start = other->start; 316 other->tree = NULL; 317 rb_erase(&other->rb_node, &tree->state); 318 free_extent_state(other); 319 } 320 } 321 other_node = rb_next(&state->rb_node); 322 if (other_node) { 323 other = rb_entry(other_node, struct extent_state, rb_node); 324 if (other->start == state->end + 1 && 325 other->state == state->state) { 326 merge_cb(tree, state, other); 327 other->start = state->start; 328 state->tree = NULL; 329 rb_erase(&state->rb_node, &tree->state); 330 free_extent_state(state); 331 state = NULL; 332 } 333 } 334 335 return 0; 336 } 337 338 static int set_state_cb(struct extent_io_tree *tree, 339 struct extent_state *state, 340 unsigned long bits) 341 { 342 if (tree->ops && tree->ops->set_bit_hook) { 343 return tree->ops->set_bit_hook(tree->mapping->host, 344 state->start, state->end, 345 state->state, bits); 346 } 347 348 return 0; 349 } 350 351 static void clear_state_cb(struct extent_io_tree *tree, 352 struct extent_state *state, 353 unsigned long bits) 354 { 355 if (tree->ops && tree->ops->clear_bit_hook) 356 tree->ops->clear_bit_hook(tree->mapping->host, state, bits); 357 } 358 359 /* 360 * insert an extent_state struct into the tree. 'bits' are set on the 361 * struct before it is inserted. 362 * 363 * This may return -EEXIST if the extent is already there, in which case the 364 * state struct is freed. 365 * 366 * The tree lock is not taken internally. This is a utility function and 367 * probably isn't what you want to call (see set/clear_extent_bit). 368 */ 369 static int insert_state(struct extent_io_tree *tree, 370 struct extent_state *state, u64 start, u64 end, 371 int bits) 372 { 373 struct rb_node *node; 374 int ret; 375 376 if (end < start) { 377 printk(KERN_ERR "btrfs end < start %llu %llu\n", 378 (unsigned long long)end, 379 (unsigned long long)start); 380 WARN_ON(1); 381 } 382 state->start = start; 383 state->end = end; 384 ret = set_state_cb(tree, state, bits); 385 if (ret) 386 return ret; 387 388 if (bits & EXTENT_DIRTY) 389 tree->dirty_bytes += end - start + 1; 390 state->state |= bits; 391 node = tree_insert(&tree->state, end, &state->rb_node); 392 if (node) { 393 struct extent_state *found; 394 found = rb_entry(node, struct extent_state, rb_node); 395 printk(KERN_ERR "btrfs found node %llu %llu on insert of " 396 "%llu %llu\n", (unsigned long long)found->start, 397 (unsigned long long)found->end, 398 (unsigned long long)start, (unsigned long long)end); 399 free_extent_state(state); 400 return -EEXIST; 401 } 402 state->tree = tree; 403 merge_state(tree, state); 404 return 0; 405 } 406 407 static int split_cb(struct extent_io_tree *tree, struct extent_state *orig, 408 u64 split) 409 { 410 if (tree->ops && tree->ops->split_extent_hook) 411 return tree->ops->split_extent_hook(tree->mapping->host, 412 orig, split); 413 return 0; 414 } 415 416 /* 417 * split a given extent state struct in two, inserting the preallocated 418 * struct 'prealloc' as the newly created second half. 'split' indicates an 419 * offset inside 'orig' where it should be split. 420 * 421 * Before calling, 422 * the tree has 'orig' at [orig->start, orig->end]. After calling, there 423 * are two extent state structs in the tree: 424 * prealloc: [orig->start, split - 1] 425 * orig: [ split, orig->end ] 426 * 427 * The tree locks are not taken by this function. They need to be held 428 * by the caller. 429 */ 430 static int split_state(struct extent_io_tree *tree, struct extent_state *orig, 431 struct extent_state *prealloc, u64 split) 432 { 433 struct rb_node *node; 434 435 split_cb(tree, orig, split); 436 437 prealloc->start = orig->start; 438 prealloc->end = split - 1; 439 prealloc->state = orig->state; 440 orig->start = split; 441 442 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node); 443 if (node) { 444 free_extent_state(prealloc); 445 return -EEXIST; 446 } 447 prealloc->tree = tree; 448 return 0; 449 } 450 451 /* 452 * utility function to clear some bits in an extent state struct. 453 * it will optionally wake up any one waiting on this state (wake == 1), or 454 * forcibly remove the state from the tree (delete == 1). 455 * 456 * If no bits are set on the state struct after clearing things, the 457 * struct is freed and removed from the tree 458 */ 459 static int clear_state_bit(struct extent_io_tree *tree, 460 struct extent_state *state, int bits, int wake, 461 int delete) 462 { 463 int bits_to_clear = bits & ~EXTENT_DO_ACCOUNTING; 464 int ret = state->state & bits_to_clear; 465 466 if ((bits & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) { 467 u64 range = state->end - state->start + 1; 468 WARN_ON(range > tree->dirty_bytes); 469 tree->dirty_bytes -= range; 470 } 471 clear_state_cb(tree, state, bits); 472 state->state &= ~bits_to_clear; 473 if (wake) 474 wake_up(&state->wq); 475 if (delete || state->state == 0) { 476 if (state->tree) { 477 clear_state_cb(tree, state, state->state); 478 rb_erase(&state->rb_node, &tree->state); 479 state->tree = NULL; 480 free_extent_state(state); 481 } else { 482 WARN_ON(1); 483 } 484 } else { 485 merge_state(tree, state); 486 } 487 return ret; 488 } 489 490 /* 491 * clear some bits on a range in the tree. This may require splitting 492 * or inserting elements in the tree, so the gfp mask is used to 493 * indicate which allocations or sleeping are allowed. 494 * 495 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove 496 * the given range from the tree regardless of state (ie for truncate). 497 * 498 * the range [start, end] is inclusive. 499 * 500 * This takes the tree lock, and returns < 0 on error, > 0 if any of the 501 * bits were already set, or zero if none of the bits were already set. 502 */ 503 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 504 int bits, int wake, int delete, 505 struct extent_state **cached_state, 506 gfp_t mask) 507 { 508 struct extent_state *state; 509 struct extent_state *cached; 510 struct extent_state *prealloc = NULL; 511 struct rb_node *next_node; 512 struct rb_node *node; 513 u64 last_end; 514 int err; 515 int set = 0; 516 517 again: 518 if (!prealloc && (mask & __GFP_WAIT)) { 519 prealloc = alloc_extent_state(mask); 520 if (!prealloc) 521 return -ENOMEM; 522 } 523 524 spin_lock(&tree->lock); 525 if (cached_state) { 526 cached = *cached_state; 527 *cached_state = NULL; 528 cached_state = NULL; 529 if (cached && cached->tree && cached->start == start) { 530 atomic_dec(&cached->refs); 531 state = cached; 532 goto hit_next; 533 } 534 free_extent_state(cached); 535 } 536 /* 537 * this search will find the extents that end after 538 * our range starts 539 */ 540 node = tree_search(tree, start); 541 if (!node) 542 goto out; 543 state = rb_entry(node, struct extent_state, rb_node); 544 hit_next: 545 if (state->start > end) 546 goto out; 547 WARN_ON(state->end < start); 548 last_end = state->end; 549 550 /* 551 * | ---- desired range ---- | 552 * | state | or 553 * | ------------- state -------------- | 554 * 555 * We need to split the extent we found, and may flip 556 * bits on second half. 557 * 558 * If the extent we found extends past our range, we 559 * just split and search again. It'll get split again 560 * the next time though. 561 * 562 * If the extent we found is inside our range, we clear 563 * the desired bit on it. 564 */ 565 566 if (state->start < start) { 567 if (!prealloc) 568 prealloc = alloc_extent_state(GFP_ATOMIC); 569 err = split_state(tree, state, prealloc, start); 570 BUG_ON(err == -EEXIST); 571 prealloc = NULL; 572 if (err) 573 goto out; 574 if (state->end <= end) { 575 set |= clear_state_bit(tree, state, bits, wake, 576 delete); 577 if (last_end == (u64)-1) 578 goto out; 579 start = last_end + 1; 580 } 581 goto search_again; 582 } 583 /* 584 * | ---- desired range ---- | 585 * | state | 586 * We need to split the extent, and clear the bit 587 * on the first half 588 */ 589 if (state->start <= end && state->end > end) { 590 if (!prealloc) 591 prealloc = alloc_extent_state(GFP_ATOMIC); 592 err = split_state(tree, state, prealloc, end + 1); 593 BUG_ON(err == -EEXIST); 594 if (wake) 595 wake_up(&state->wq); 596 597 set |= clear_state_bit(tree, prealloc, bits, wake, delete); 598 599 prealloc = NULL; 600 goto out; 601 } 602 603 if (state->end < end && prealloc && !need_resched()) 604 next_node = rb_next(&state->rb_node); 605 else 606 next_node = NULL; 607 608 set |= clear_state_bit(tree, state, bits, wake, delete); 609 if (last_end == (u64)-1) 610 goto out; 611 start = last_end + 1; 612 if (start <= end && next_node) { 613 state = rb_entry(next_node, struct extent_state, 614 rb_node); 615 if (state->start == start) 616 goto hit_next; 617 } 618 goto search_again; 619 620 out: 621 spin_unlock(&tree->lock); 622 if (prealloc) 623 free_extent_state(prealloc); 624 625 return set; 626 627 search_again: 628 if (start > end) 629 goto out; 630 spin_unlock(&tree->lock); 631 if (mask & __GFP_WAIT) 632 cond_resched(); 633 goto again; 634 } 635 636 static int wait_on_state(struct extent_io_tree *tree, 637 struct extent_state *state) 638 __releases(tree->lock) 639 __acquires(tree->lock) 640 { 641 DEFINE_WAIT(wait); 642 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE); 643 spin_unlock(&tree->lock); 644 schedule(); 645 spin_lock(&tree->lock); 646 finish_wait(&state->wq, &wait); 647 return 0; 648 } 649 650 /* 651 * waits for one or more bits to clear on a range in the state tree. 652 * The range [start, end] is inclusive. 653 * The tree lock is taken by this function 654 */ 655 int wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits) 656 { 657 struct extent_state *state; 658 struct rb_node *node; 659 660 spin_lock(&tree->lock); 661 again: 662 while (1) { 663 /* 664 * this search will find all the extents that end after 665 * our range starts 666 */ 667 node = tree_search(tree, start); 668 if (!node) 669 break; 670 671 state = rb_entry(node, struct extent_state, rb_node); 672 673 if (state->start > end) 674 goto out; 675 676 if (state->state & bits) { 677 start = state->start; 678 atomic_inc(&state->refs); 679 wait_on_state(tree, state); 680 free_extent_state(state); 681 goto again; 682 } 683 start = state->end + 1; 684 685 if (start > end) 686 break; 687 688 if (need_resched()) { 689 spin_unlock(&tree->lock); 690 cond_resched(); 691 spin_lock(&tree->lock); 692 } 693 } 694 out: 695 spin_unlock(&tree->lock); 696 return 0; 697 } 698 699 static int set_state_bits(struct extent_io_tree *tree, 700 struct extent_state *state, 701 int bits) 702 { 703 int ret; 704 705 ret = set_state_cb(tree, state, bits); 706 if (ret) 707 return ret; 708 709 if ((bits & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) { 710 u64 range = state->end - state->start + 1; 711 tree->dirty_bytes += range; 712 } 713 state->state |= bits; 714 715 return 0; 716 } 717 718 static void cache_state(struct extent_state *state, 719 struct extent_state **cached_ptr) 720 { 721 if (cached_ptr && !(*cached_ptr)) { 722 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) { 723 *cached_ptr = state; 724 atomic_inc(&state->refs); 725 } 726 } 727 } 728 729 /* 730 * set some bits on a range in the tree. This may require allocations or 731 * sleeping, so the gfp mask is used to indicate what is allowed. 732 * 733 * If any of the exclusive bits are set, this will fail with -EEXIST if some 734 * part of the range already has the desired bits set. The start of the 735 * existing range is returned in failed_start in this case. 736 * 737 * [start, end] is inclusive This takes the tree lock. 738 */ 739 740 static int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 741 int bits, int exclusive_bits, u64 *failed_start, 742 struct extent_state **cached_state, 743 gfp_t mask) 744 { 745 struct extent_state *state; 746 struct extent_state *prealloc = NULL; 747 struct rb_node *node; 748 int err = 0; 749 u64 last_start; 750 u64 last_end; 751 752 again: 753 if (!prealloc && (mask & __GFP_WAIT)) { 754 prealloc = alloc_extent_state(mask); 755 if (!prealloc) 756 return -ENOMEM; 757 } 758 759 spin_lock(&tree->lock); 760 if (cached_state && *cached_state) { 761 state = *cached_state; 762 if (state->start == start && state->tree) { 763 node = &state->rb_node; 764 goto hit_next; 765 } 766 } 767 /* 768 * this search will find all the extents that end after 769 * our range starts. 770 */ 771 node = tree_search(tree, start); 772 if (!node) { 773 err = insert_state(tree, prealloc, start, end, bits); 774 prealloc = NULL; 775 BUG_ON(err == -EEXIST); 776 goto out; 777 } 778 state = rb_entry(node, struct extent_state, rb_node); 779 hit_next: 780 last_start = state->start; 781 last_end = state->end; 782 783 /* 784 * | ---- desired range ---- | 785 * | state | 786 * 787 * Just lock what we found and keep going 788 */ 789 if (state->start == start && state->end <= end) { 790 struct rb_node *next_node; 791 if (state->state & exclusive_bits) { 792 *failed_start = state->start; 793 err = -EEXIST; 794 goto out; 795 } 796 797 err = set_state_bits(tree, state, bits); 798 if (err) 799 goto out; 800 801 cache_state(state, cached_state); 802 merge_state(tree, state); 803 if (last_end == (u64)-1) 804 goto out; 805 806 start = last_end + 1; 807 if (start < end && prealloc && !need_resched()) { 808 next_node = rb_next(node); 809 if (next_node) { 810 state = rb_entry(next_node, struct extent_state, 811 rb_node); 812 if (state->start == start) 813 goto hit_next; 814 } 815 } 816 goto search_again; 817 } 818 819 /* 820 * | ---- desired range ---- | 821 * | state | 822 * or 823 * | ------------- state -------------- | 824 * 825 * We need to split the extent we found, and may flip bits on 826 * second half. 827 * 828 * If the extent we found extends past our 829 * range, we just split and search again. It'll get split 830 * again the next time though. 831 * 832 * If the extent we found is inside our range, we set the 833 * desired bit on it. 834 */ 835 if (state->start < start) { 836 if (state->state & exclusive_bits) { 837 *failed_start = start; 838 err = -EEXIST; 839 goto out; 840 } 841 err = split_state(tree, state, prealloc, start); 842 BUG_ON(err == -EEXIST); 843 prealloc = NULL; 844 if (err) 845 goto out; 846 if (state->end <= end) { 847 err = set_state_bits(tree, state, bits); 848 if (err) 849 goto out; 850 cache_state(state, cached_state); 851 merge_state(tree, state); 852 if (last_end == (u64)-1) 853 goto out; 854 start = last_end + 1; 855 } 856 goto search_again; 857 } 858 /* 859 * | ---- desired range ---- | 860 * | state | or | state | 861 * 862 * There's a hole, we need to insert something in it and 863 * ignore the extent we found. 864 */ 865 if (state->start > start) { 866 u64 this_end; 867 if (end < last_start) 868 this_end = end; 869 else 870 this_end = last_start - 1; 871 err = insert_state(tree, prealloc, start, this_end, 872 bits); 873 BUG_ON(err == -EEXIST); 874 if (err) { 875 prealloc = NULL; 876 goto out; 877 } 878 cache_state(prealloc, cached_state); 879 prealloc = NULL; 880 start = this_end + 1; 881 goto search_again; 882 } 883 /* 884 * | ---- desired range ---- | 885 * | state | 886 * We need to split the extent, and set the bit 887 * on the first half 888 */ 889 if (state->start <= end && state->end > end) { 890 if (state->state & exclusive_bits) { 891 *failed_start = start; 892 err = -EEXIST; 893 goto out; 894 } 895 err = split_state(tree, state, prealloc, end + 1); 896 BUG_ON(err == -EEXIST); 897 898 err = set_state_bits(tree, prealloc, bits); 899 if (err) { 900 prealloc = NULL; 901 goto out; 902 } 903 cache_state(prealloc, cached_state); 904 merge_state(tree, prealloc); 905 prealloc = NULL; 906 goto out; 907 } 908 909 goto search_again; 910 911 out: 912 spin_unlock(&tree->lock); 913 if (prealloc) 914 free_extent_state(prealloc); 915 916 return err; 917 918 search_again: 919 if (start > end) 920 goto out; 921 spin_unlock(&tree->lock); 922 if (mask & __GFP_WAIT) 923 cond_resched(); 924 goto again; 925 } 926 927 /* wrappers around set/clear extent bit */ 928 int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end, 929 gfp_t mask) 930 { 931 return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL, 932 NULL, mask); 933 } 934 935 int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 936 int bits, gfp_t mask) 937 { 938 return set_extent_bit(tree, start, end, bits, 0, NULL, 939 NULL, mask); 940 } 941 942 int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 943 int bits, gfp_t mask) 944 { 945 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask); 946 } 947 948 int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end, 949 gfp_t mask) 950 { 951 return set_extent_bit(tree, start, end, 952 EXTENT_DELALLOC | EXTENT_DIRTY | EXTENT_UPTODATE, 953 0, NULL, NULL, mask); 954 } 955 956 int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end, 957 gfp_t mask) 958 { 959 return clear_extent_bit(tree, start, end, 960 EXTENT_DIRTY | EXTENT_DELALLOC | 961 EXTENT_DO_ACCOUNTING, 0, 0, 962 NULL, mask); 963 } 964 965 int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end, 966 gfp_t mask) 967 { 968 return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL, 969 NULL, mask); 970 } 971 972 static int clear_extent_new(struct extent_io_tree *tree, u64 start, u64 end, 973 gfp_t mask) 974 { 975 return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, 976 NULL, mask); 977 } 978 979 int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end, 980 gfp_t mask) 981 { 982 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL, 983 NULL, mask); 984 } 985 986 static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, 987 u64 end, gfp_t mask) 988 { 989 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, 990 NULL, mask); 991 } 992 993 int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end) 994 { 995 return wait_extent_bit(tree, start, end, EXTENT_WRITEBACK); 996 } 997 998 /* 999 * either insert or lock state struct between start and end use mask to tell 1000 * us if waiting is desired. 1001 */ 1002 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 1003 int bits, struct extent_state **cached_state, gfp_t mask) 1004 { 1005 int err; 1006 u64 failed_start; 1007 while (1) { 1008 err = set_extent_bit(tree, start, end, EXTENT_LOCKED | bits, 1009 EXTENT_LOCKED, &failed_start, 1010 cached_state, mask); 1011 if (err == -EEXIST && (mask & __GFP_WAIT)) { 1012 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED); 1013 start = failed_start; 1014 } else { 1015 break; 1016 } 1017 WARN_ON(start > end); 1018 } 1019 return err; 1020 } 1021 1022 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask) 1023 { 1024 return lock_extent_bits(tree, start, end, 0, NULL, mask); 1025 } 1026 1027 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end, 1028 gfp_t mask) 1029 { 1030 int err; 1031 u64 failed_start; 1032 1033 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED, 1034 &failed_start, NULL, mask); 1035 if (err == -EEXIST) { 1036 if (failed_start > start) 1037 clear_extent_bit(tree, start, failed_start - 1, 1038 EXTENT_LOCKED, 1, 0, NULL, mask); 1039 return 0; 1040 } 1041 return 1; 1042 } 1043 1044 int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end, 1045 struct extent_state **cached, gfp_t mask) 1046 { 1047 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached, 1048 mask); 1049 } 1050 1051 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end, 1052 gfp_t mask) 1053 { 1054 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL, 1055 mask); 1056 } 1057 1058 /* 1059 * helper function to set pages and extents in the tree dirty 1060 */ 1061 int set_range_dirty(struct extent_io_tree *tree, u64 start, u64 end) 1062 { 1063 unsigned long index = start >> PAGE_CACHE_SHIFT; 1064 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1065 struct page *page; 1066 1067 while (index <= end_index) { 1068 page = find_get_page(tree->mapping, index); 1069 BUG_ON(!page); 1070 __set_page_dirty_nobuffers(page); 1071 page_cache_release(page); 1072 index++; 1073 } 1074 return 0; 1075 } 1076 1077 /* 1078 * helper function to set both pages and extents in the tree writeback 1079 */ 1080 static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end) 1081 { 1082 unsigned long index = start >> PAGE_CACHE_SHIFT; 1083 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1084 struct page *page; 1085 1086 while (index <= end_index) { 1087 page = find_get_page(tree->mapping, index); 1088 BUG_ON(!page); 1089 set_page_writeback(page); 1090 page_cache_release(page); 1091 index++; 1092 } 1093 return 0; 1094 } 1095 1096 /* 1097 * find the first offset in the io tree with 'bits' set. zero is 1098 * returned if we find something, and *start_ret and *end_ret are 1099 * set to reflect the state struct that was found. 1100 * 1101 * If nothing was found, 1 is returned, < 0 on error 1102 */ 1103 int find_first_extent_bit(struct extent_io_tree *tree, u64 start, 1104 u64 *start_ret, u64 *end_ret, int bits) 1105 { 1106 struct rb_node *node; 1107 struct extent_state *state; 1108 int ret = 1; 1109 1110 spin_lock(&tree->lock); 1111 /* 1112 * this search will find all the extents that end after 1113 * our range starts. 1114 */ 1115 node = tree_search(tree, start); 1116 if (!node) 1117 goto out; 1118 1119 while (1) { 1120 state = rb_entry(node, struct extent_state, rb_node); 1121 if (state->end >= start && (state->state & bits)) { 1122 *start_ret = state->start; 1123 *end_ret = state->end; 1124 ret = 0; 1125 break; 1126 } 1127 node = rb_next(node); 1128 if (!node) 1129 break; 1130 } 1131 out: 1132 spin_unlock(&tree->lock); 1133 return ret; 1134 } 1135 1136 /* find the first state struct with 'bits' set after 'start', and 1137 * return it. tree->lock must be held. NULL will returned if 1138 * nothing was found after 'start' 1139 */ 1140 struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree, 1141 u64 start, int bits) 1142 { 1143 struct rb_node *node; 1144 struct extent_state *state; 1145 1146 /* 1147 * this search will find all the extents that end after 1148 * our range starts. 1149 */ 1150 node = tree_search(tree, start); 1151 if (!node) 1152 goto out; 1153 1154 while (1) { 1155 state = rb_entry(node, struct extent_state, rb_node); 1156 if (state->end >= start && (state->state & bits)) 1157 return state; 1158 1159 node = rb_next(node); 1160 if (!node) 1161 break; 1162 } 1163 out: 1164 return NULL; 1165 } 1166 1167 /* 1168 * find a contiguous range of bytes in the file marked as delalloc, not 1169 * more than 'max_bytes'. start and end are used to return the range, 1170 * 1171 * 1 is returned if we find something, 0 if nothing was in the tree 1172 */ 1173 static noinline u64 find_delalloc_range(struct extent_io_tree *tree, 1174 u64 *start, u64 *end, u64 max_bytes) 1175 { 1176 struct rb_node *node; 1177 struct extent_state *state; 1178 u64 cur_start = *start; 1179 u64 found = 0; 1180 u64 total_bytes = 0; 1181 1182 spin_lock(&tree->lock); 1183 1184 /* 1185 * this search will find all the extents that end after 1186 * our range starts. 1187 */ 1188 node = tree_search(tree, cur_start); 1189 if (!node) { 1190 if (!found) 1191 *end = (u64)-1; 1192 goto out; 1193 } 1194 1195 while (1) { 1196 state = rb_entry(node, struct extent_state, rb_node); 1197 if (found && (state->start != cur_start || 1198 (state->state & EXTENT_BOUNDARY))) { 1199 goto out; 1200 } 1201 if (!(state->state & EXTENT_DELALLOC)) { 1202 if (!found) 1203 *end = state->end; 1204 goto out; 1205 } 1206 if (!found) 1207 *start = state->start; 1208 found++; 1209 *end = state->end; 1210 cur_start = state->end + 1; 1211 node = rb_next(node); 1212 if (!node) 1213 break; 1214 total_bytes += state->end - state->start + 1; 1215 if (total_bytes >= max_bytes) 1216 break; 1217 } 1218 out: 1219 spin_unlock(&tree->lock); 1220 return found; 1221 } 1222 1223 static noinline int __unlock_for_delalloc(struct inode *inode, 1224 struct page *locked_page, 1225 u64 start, u64 end) 1226 { 1227 int ret; 1228 struct page *pages[16]; 1229 unsigned long index = start >> PAGE_CACHE_SHIFT; 1230 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1231 unsigned long nr_pages = end_index - index + 1; 1232 int i; 1233 1234 if (index == locked_page->index && end_index == index) 1235 return 0; 1236 1237 while (nr_pages > 0) { 1238 ret = find_get_pages_contig(inode->i_mapping, index, 1239 min_t(unsigned long, nr_pages, 1240 ARRAY_SIZE(pages)), pages); 1241 for (i = 0; i < ret; i++) { 1242 if (pages[i] != locked_page) 1243 unlock_page(pages[i]); 1244 page_cache_release(pages[i]); 1245 } 1246 nr_pages -= ret; 1247 index += ret; 1248 cond_resched(); 1249 } 1250 return 0; 1251 } 1252 1253 static noinline int lock_delalloc_pages(struct inode *inode, 1254 struct page *locked_page, 1255 u64 delalloc_start, 1256 u64 delalloc_end) 1257 { 1258 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT; 1259 unsigned long start_index = index; 1260 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT; 1261 unsigned long pages_locked = 0; 1262 struct page *pages[16]; 1263 unsigned long nrpages; 1264 int ret; 1265 int i; 1266 1267 /* the caller is responsible for locking the start index */ 1268 if (index == locked_page->index && index == end_index) 1269 return 0; 1270 1271 /* skip the page at the start index */ 1272 nrpages = end_index - index + 1; 1273 while (nrpages > 0) { 1274 ret = find_get_pages_contig(inode->i_mapping, index, 1275 min_t(unsigned long, 1276 nrpages, ARRAY_SIZE(pages)), pages); 1277 if (ret == 0) { 1278 ret = -EAGAIN; 1279 goto done; 1280 } 1281 /* now we have an array of pages, lock them all */ 1282 for (i = 0; i < ret; i++) { 1283 /* 1284 * the caller is taking responsibility for 1285 * locked_page 1286 */ 1287 if (pages[i] != locked_page) { 1288 lock_page(pages[i]); 1289 if (!PageDirty(pages[i]) || 1290 pages[i]->mapping != inode->i_mapping) { 1291 ret = -EAGAIN; 1292 unlock_page(pages[i]); 1293 page_cache_release(pages[i]); 1294 goto done; 1295 } 1296 } 1297 page_cache_release(pages[i]); 1298 pages_locked++; 1299 } 1300 nrpages -= ret; 1301 index += ret; 1302 cond_resched(); 1303 } 1304 ret = 0; 1305 done: 1306 if (ret && pages_locked) { 1307 __unlock_for_delalloc(inode, locked_page, 1308 delalloc_start, 1309 ((u64)(start_index + pages_locked - 1)) << 1310 PAGE_CACHE_SHIFT); 1311 } 1312 return ret; 1313 } 1314 1315 /* 1316 * find a contiguous range of bytes in the file marked as delalloc, not 1317 * more than 'max_bytes'. start and end are used to return the range, 1318 * 1319 * 1 is returned if we find something, 0 if nothing was in the tree 1320 */ 1321 static noinline u64 find_lock_delalloc_range(struct inode *inode, 1322 struct extent_io_tree *tree, 1323 struct page *locked_page, 1324 u64 *start, u64 *end, 1325 u64 max_bytes) 1326 { 1327 u64 delalloc_start; 1328 u64 delalloc_end; 1329 u64 found; 1330 struct extent_state *cached_state = NULL; 1331 int ret; 1332 int loops = 0; 1333 1334 again: 1335 /* step one, find a bunch of delalloc bytes starting at start */ 1336 delalloc_start = *start; 1337 delalloc_end = 0; 1338 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end, 1339 max_bytes); 1340 if (!found || delalloc_end <= *start) { 1341 *start = delalloc_start; 1342 *end = delalloc_end; 1343 return found; 1344 } 1345 1346 /* 1347 * start comes from the offset of locked_page. We have to lock 1348 * pages in order, so we can't process delalloc bytes before 1349 * locked_page 1350 */ 1351 if (delalloc_start < *start) 1352 delalloc_start = *start; 1353 1354 /* 1355 * make sure to limit the number of pages we try to lock down 1356 * if we're looping. 1357 */ 1358 if (delalloc_end + 1 - delalloc_start > max_bytes && loops) 1359 delalloc_end = delalloc_start + PAGE_CACHE_SIZE - 1; 1360 1361 /* step two, lock all the pages after the page that has start */ 1362 ret = lock_delalloc_pages(inode, locked_page, 1363 delalloc_start, delalloc_end); 1364 if (ret == -EAGAIN) { 1365 /* some of the pages are gone, lets avoid looping by 1366 * shortening the size of the delalloc range we're searching 1367 */ 1368 free_extent_state(cached_state); 1369 if (!loops) { 1370 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1); 1371 max_bytes = PAGE_CACHE_SIZE - offset; 1372 loops = 1; 1373 goto again; 1374 } else { 1375 found = 0; 1376 goto out_failed; 1377 } 1378 } 1379 BUG_ON(ret); 1380 1381 /* step three, lock the state bits for the whole range */ 1382 lock_extent_bits(tree, delalloc_start, delalloc_end, 1383 0, &cached_state, GFP_NOFS); 1384 1385 /* then test to make sure it is all still delalloc */ 1386 ret = test_range_bit(tree, delalloc_start, delalloc_end, 1387 EXTENT_DELALLOC, 1, cached_state); 1388 if (!ret) { 1389 unlock_extent_cached(tree, delalloc_start, delalloc_end, 1390 &cached_state, GFP_NOFS); 1391 __unlock_for_delalloc(inode, locked_page, 1392 delalloc_start, delalloc_end); 1393 cond_resched(); 1394 goto again; 1395 } 1396 free_extent_state(cached_state); 1397 *start = delalloc_start; 1398 *end = delalloc_end; 1399 out_failed: 1400 return found; 1401 } 1402 1403 int extent_clear_unlock_delalloc(struct inode *inode, 1404 struct extent_io_tree *tree, 1405 u64 start, u64 end, struct page *locked_page, 1406 unsigned long op) 1407 { 1408 int ret; 1409 struct page *pages[16]; 1410 unsigned long index = start >> PAGE_CACHE_SHIFT; 1411 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1412 unsigned long nr_pages = end_index - index + 1; 1413 int i; 1414 int clear_bits = 0; 1415 1416 if (op & EXTENT_CLEAR_UNLOCK) 1417 clear_bits |= EXTENT_LOCKED; 1418 if (op & EXTENT_CLEAR_DIRTY) 1419 clear_bits |= EXTENT_DIRTY; 1420 1421 if (op & EXTENT_CLEAR_DELALLOC) 1422 clear_bits |= EXTENT_DELALLOC; 1423 1424 if (op & EXTENT_CLEAR_ACCOUNTING) 1425 clear_bits |= EXTENT_DO_ACCOUNTING; 1426 1427 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS); 1428 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY | 1429 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK | 1430 EXTENT_SET_PRIVATE2))) 1431 return 0; 1432 1433 while (nr_pages > 0) { 1434 ret = find_get_pages_contig(inode->i_mapping, index, 1435 min_t(unsigned long, 1436 nr_pages, ARRAY_SIZE(pages)), pages); 1437 for (i = 0; i < ret; i++) { 1438 1439 if (op & EXTENT_SET_PRIVATE2) 1440 SetPagePrivate2(pages[i]); 1441 1442 if (pages[i] == locked_page) { 1443 page_cache_release(pages[i]); 1444 continue; 1445 } 1446 if (op & EXTENT_CLEAR_DIRTY) 1447 clear_page_dirty_for_io(pages[i]); 1448 if (op & EXTENT_SET_WRITEBACK) 1449 set_page_writeback(pages[i]); 1450 if (op & EXTENT_END_WRITEBACK) 1451 end_page_writeback(pages[i]); 1452 if (op & EXTENT_CLEAR_UNLOCK_PAGE) 1453 unlock_page(pages[i]); 1454 page_cache_release(pages[i]); 1455 } 1456 nr_pages -= ret; 1457 index += ret; 1458 cond_resched(); 1459 } 1460 return 0; 1461 } 1462 1463 /* 1464 * count the number of bytes in the tree that have a given bit(s) 1465 * set. This can be fairly slow, except for EXTENT_DIRTY which is 1466 * cached. The total number found is returned. 1467 */ 1468 u64 count_range_bits(struct extent_io_tree *tree, 1469 u64 *start, u64 search_end, u64 max_bytes, 1470 unsigned long bits) 1471 { 1472 struct rb_node *node; 1473 struct extent_state *state; 1474 u64 cur_start = *start; 1475 u64 total_bytes = 0; 1476 int found = 0; 1477 1478 if (search_end <= cur_start) { 1479 WARN_ON(1); 1480 return 0; 1481 } 1482 1483 spin_lock(&tree->lock); 1484 if (cur_start == 0 && bits == EXTENT_DIRTY) { 1485 total_bytes = tree->dirty_bytes; 1486 goto out; 1487 } 1488 /* 1489 * this search will find all the extents that end after 1490 * our range starts. 1491 */ 1492 node = tree_search(tree, cur_start); 1493 if (!node) 1494 goto out; 1495 1496 while (1) { 1497 state = rb_entry(node, struct extent_state, rb_node); 1498 if (state->start > search_end) 1499 break; 1500 if (state->end >= cur_start && (state->state & bits)) { 1501 total_bytes += min(search_end, state->end) + 1 - 1502 max(cur_start, state->start); 1503 if (total_bytes >= max_bytes) 1504 break; 1505 if (!found) { 1506 *start = state->start; 1507 found = 1; 1508 } 1509 } 1510 node = rb_next(node); 1511 if (!node) 1512 break; 1513 } 1514 out: 1515 spin_unlock(&tree->lock); 1516 return total_bytes; 1517 } 1518 1519 /* 1520 * set the private field for a given byte offset in the tree. If there isn't 1521 * an extent_state there already, this does nothing. 1522 */ 1523 int set_state_private(struct extent_io_tree *tree, u64 start, u64 private) 1524 { 1525 struct rb_node *node; 1526 struct extent_state *state; 1527 int ret = 0; 1528 1529 spin_lock(&tree->lock); 1530 /* 1531 * this search will find all the extents that end after 1532 * our range starts. 1533 */ 1534 node = tree_search(tree, start); 1535 if (!node) { 1536 ret = -ENOENT; 1537 goto out; 1538 } 1539 state = rb_entry(node, struct extent_state, rb_node); 1540 if (state->start != start) { 1541 ret = -ENOENT; 1542 goto out; 1543 } 1544 state->private = private; 1545 out: 1546 spin_unlock(&tree->lock); 1547 return ret; 1548 } 1549 1550 int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private) 1551 { 1552 struct rb_node *node; 1553 struct extent_state *state; 1554 int ret = 0; 1555 1556 spin_lock(&tree->lock); 1557 /* 1558 * this search will find all the extents that end after 1559 * our range starts. 1560 */ 1561 node = tree_search(tree, start); 1562 if (!node) { 1563 ret = -ENOENT; 1564 goto out; 1565 } 1566 state = rb_entry(node, struct extent_state, rb_node); 1567 if (state->start != start) { 1568 ret = -ENOENT; 1569 goto out; 1570 } 1571 *private = state->private; 1572 out: 1573 spin_unlock(&tree->lock); 1574 return ret; 1575 } 1576 1577 /* 1578 * searches a range in the state tree for a given mask. 1579 * If 'filled' == 1, this returns 1 only if every extent in the tree 1580 * has the bits set. Otherwise, 1 is returned if any bit in the 1581 * range is found set. 1582 */ 1583 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, 1584 int bits, int filled, struct extent_state *cached) 1585 { 1586 struct extent_state *state = NULL; 1587 struct rb_node *node; 1588 int bitset = 0; 1589 1590 spin_lock(&tree->lock); 1591 if (cached && cached->tree && cached->start == start) 1592 node = &cached->rb_node; 1593 else 1594 node = tree_search(tree, start); 1595 while (node && start <= end) { 1596 state = rb_entry(node, struct extent_state, rb_node); 1597 1598 if (filled && state->start > start) { 1599 bitset = 0; 1600 break; 1601 } 1602 1603 if (state->start > end) 1604 break; 1605 1606 if (state->state & bits) { 1607 bitset = 1; 1608 if (!filled) 1609 break; 1610 } else if (filled) { 1611 bitset = 0; 1612 break; 1613 } 1614 1615 if (state->end == (u64)-1) 1616 break; 1617 1618 start = state->end + 1; 1619 if (start > end) 1620 break; 1621 node = rb_next(node); 1622 if (!node) { 1623 if (filled) 1624 bitset = 0; 1625 break; 1626 } 1627 } 1628 spin_unlock(&tree->lock); 1629 return bitset; 1630 } 1631 1632 /* 1633 * helper function to set a given page up to date if all the 1634 * extents in the tree for that page are up to date 1635 */ 1636 static int check_page_uptodate(struct extent_io_tree *tree, 1637 struct page *page) 1638 { 1639 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 1640 u64 end = start + PAGE_CACHE_SIZE - 1; 1641 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL)) 1642 SetPageUptodate(page); 1643 return 0; 1644 } 1645 1646 /* 1647 * helper function to unlock a page if all the extents in the tree 1648 * for that page are unlocked 1649 */ 1650 static int check_page_locked(struct extent_io_tree *tree, 1651 struct page *page) 1652 { 1653 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 1654 u64 end = start + PAGE_CACHE_SIZE - 1; 1655 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) 1656 unlock_page(page); 1657 return 0; 1658 } 1659 1660 /* 1661 * helper function to end page writeback if all the extents 1662 * in the tree for that page are done with writeback 1663 */ 1664 static int check_page_writeback(struct extent_io_tree *tree, 1665 struct page *page) 1666 { 1667 end_page_writeback(page); 1668 return 0; 1669 } 1670 1671 /* lots and lots of room for performance fixes in the end_bio funcs */ 1672 1673 /* 1674 * after a writepage IO is done, we need to: 1675 * clear the uptodate bits on error 1676 * clear the writeback bits in the extent tree for this IO 1677 * end_page_writeback if the page has no more pending IO 1678 * 1679 * Scheduling is not allowed, so the extent state tree is expected 1680 * to have one and only one object corresponding to this IO. 1681 */ 1682 static void end_bio_extent_writepage(struct bio *bio, int err) 1683 { 1684 int uptodate = err == 0; 1685 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 1686 struct extent_io_tree *tree; 1687 u64 start; 1688 u64 end; 1689 int whole_page; 1690 int ret; 1691 1692 do { 1693 struct page *page = bvec->bv_page; 1694 tree = &BTRFS_I(page->mapping->host)->io_tree; 1695 1696 start = ((u64)page->index << PAGE_CACHE_SHIFT) + 1697 bvec->bv_offset; 1698 end = start + bvec->bv_len - 1; 1699 1700 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE) 1701 whole_page = 1; 1702 else 1703 whole_page = 0; 1704 1705 if (--bvec >= bio->bi_io_vec) 1706 prefetchw(&bvec->bv_page->flags); 1707 if (tree->ops && tree->ops->writepage_end_io_hook) { 1708 ret = tree->ops->writepage_end_io_hook(page, start, 1709 end, NULL, uptodate); 1710 if (ret) 1711 uptodate = 0; 1712 } 1713 1714 if (!uptodate && tree->ops && 1715 tree->ops->writepage_io_failed_hook) { 1716 ret = tree->ops->writepage_io_failed_hook(bio, page, 1717 start, end, NULL); 1718 if (ret == 0) { 1719 uptodate = (err == 0); 1720 continue; 1721 } 1722 } 1723 1724 if (!uptodate) { 1725 clear_extent_uptodate(tree, start, end, GFP_NOFS); 1726 ClearPageUptodate(page); 1727 SetPageError(page); 1728 } 1729 1730 if (whole_page) 1731 end_page_writeback(page); 1732 else 1733 check_page_writeback(tree, page); 1734 } while (bvec >= bio->bi_io_vec); 1735 1736 bio_put(bio); 1737 } 1738 1739 /* 1740 * after a readpage IO is done, we need to: 1741 * clear the uptodate bits on error 1742 * set the uptodate bits if things worked 1743 * set the page up to date if all extents in the tree are uptodate 1744 * clear the lock bit in the extent tree 1745 * unlock the page if there are no other extents locked for it 1746 * 1747 * Scheduling is not allowed, so the extent state tree is expected 1748 * to have one and only one object corresponding to this IO. 1749 */ 1750 static void end_bio_extent_readpage(struct bio *bio, int err) 1751 { 1752 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 1753 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 1754 struct extent_io_tree *tree; 1755 u64 start; 1756 u64 end; 1757 int whole_page; 1758 int ret; 1759 1760 if (err) 1761 uptodate = 0; 1762 1763 do { 1764 struct page *page = bvec->bv_page; 1765 tree = &BTRFS_I(page->mapping->host)->io_tree; 1766 1767 start = ((u64)page->index << PAGE_CACHE_SHIFT) + 1768 bvec->bv_offset; 1769 end = start + bvec->bv_len - 1; 1770 1771 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE) 1772 whole_page = 1; 1773 else 1774 whole_page = 0; 1775 1776 if (--bvec >= bio->bi_io_vec) 1777 prefetchw(&bvec->bv_page->flags); 1778 1779 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) { 1780 ret = tree->ops->readpage_end_io_hook(page, start, end, 1781 NULL); 1782 if (ret) 1783 uptodate = 0; 1784 } 1785 if (!uptodate && tree->ops && 1786 tree->ops->readpage_io_failed_hook) { 1787 ret = tree->ops->readpage_io_failed_hook(bio, page, 1788 start, end, NULL); 1789 if (ret == 0) { 1790 uptodate = 1791 test_bit(BIO_UPTODATE, &bio->bi_flags); 1792 if (err) 1793 uptodate = 0; 1794 continue; 1795 } 1796 } 1797 1798 if (uptodate) { 1799 set_extent_uptodate(tree, start, end, 1800 GFP_ATOMIC); 1801 } 1802 unlock_extent(tree, start, end, GFP_ATOMIC); 1803 1804 if (whole_page) { 1805 if (uptodate) { 1806 SetPageUptodate(page); 1807 } else { 1808 ClearPageUptodate(page); 1809 SetPageError(page); 1810 } 1811 unlock_page(page); 1812 } else { 1813 if (uptodate) { 1814 check_page_uptodate(tree, page); 1815 } else { 1816 ClearPageUptodate(page); 1817 SetPageError(page); 1818 } 1819 check_page_locked(tree, page); 1820 } 1821 } while (bvec >= bio->bi_io_vec); 1822 1823 bio_put(bio); 1824 } 1825 1826 /* 1827 * IO done from prepare_write is pretty simple, we just unlock 1828 * the structs in the extent tree when done, and set the uptodate bits 1829 * as appropriate. 1830 */ 1831 static void end_bio_extent_preparewrite(struct bio *bio, int err) 1832 { 1833 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 1834 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 1835 struct extent_io_tree *tree; 1836 u64 start; 1837 u64 end; 1838 1839 do { 1840 struct page *page = bvec->bv_page; 1841 tree = &BTRFS_I(page->mapping->host)->io_tree; 1842 1843 start = ((u64)page->index << PAGE_CACHE_SHIFT) + 1844 bvec->bv_offset; 1845 end = start + bvec->bv_len - 1; 1846 1847 if (--bvec >= bio->bi_io_vec) 1848 prefetchw(&bvec->bv_page->flags); 1849 1850 if (uptodate) { 1851 set_extent_uptodate(tree, start, end, GFP_ATOMIC); 1852 } else { 1853 ClearPageUptodate(page); 1854 SetPageError(page); 1855 } 1856 1857 unlock_extent(tree, start, end, GFP_ATOMIC); 1858 1859 } while (bvec >= bio->bi_io_vec); 1860 1861 bio_put(bio); 1862 } 1863 1864 static struct bio * 1865 extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs, 1866 gfp_t gfp_flags) 1867 { 1868 struct bio *bio; 1869 1870 bio = bio_alloc(gfp_flags, nr_vecs); 1871 1872 if (bio == NULL && (current->flags & PF_MEMALLOC)) { 1873 while (!bio && (nr_vecs /= 2)) 1874 bio = bio_alloc(gfp_flags, nr_vecs); 1875 } 1876 1877 if (bio) { 1878 bio->bi_size = 0; 1879 bio->bi_bdev = bdev; 1880 bio->bi_sector = first_sector; 1881 } 1882 return bio; 1883 } 1884 1885 static int submit_one_bio(int rw, struct bio *bio, int mirror_num, 1886 unsigned long bio_flags) 1887 { 1888 int ret = 0; 1889 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 1890 struct page *page = bvec->bv_page; 1891 struct extent_io_tree *tree = bio->bi_private; 1892 u64 start; 1893 u64 end; 1894 1895 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset; 1896 end = start + bvec->bv_len - 1; 1897 1898 bio->bi_private = NULL; 1899 1900 bio_get(bio); 1901 1902 if (tree->ops && tree->ops->submit_bio_hook) 1903 tree->ops->submit_bio_hook(page->mapping->host, rw, bio, 1904 mirror_num, bio_flags); 1905 else 1906 submit_bio(rw, bio); 1907 if (bio_flagged(bio, BIO_EOPNOTSUPP)) 1908 ret = -EOPNOTSUPP; 1909 bio_put(bio); 1910 return ret; 1911 } 1912 1913 static int submit_extent_page(int rw, struct extent_io_tree *tree, 1914 struct page *page, sector_t sector, 1915 size_t size, unsigned long offset, 1916 struct block_device *bdev, 1917 struct bio **bio_ret, 1918 unsigned long max_pages, 1919 bio_end_io_t end_io_func, 1920 int mirror_num, 1921 unsigned long prev_bio_flags, 1922 unsigned long bio_flags) 1923 { 1924 int ret = 0; 1925 struct bio *bio; 1926 int nr; 1927 int contig = 0; 1928 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED; 1929 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED; 1930 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE); 1931 1932 if (bio_ret && *bio_ret) { 1933 bio = *bio_ret; 1934 if (old_compressed) 1935 contig = bio->bi_sector == sector; 1936 else 1937 contig = bio->bi_sector + (bio->bi_size >> 9) == 1938 sector; 1939 1940 if (prev_bio_flags != bio_flags || !contig || 1941 (tree->ops && tree->ops->merge_bio_hook && 1942 tree->ops->merge_bio_hook(page, offset, page_size, bio, 1943 bio_flags)) || 1944 bio_add_page(bio, page, page_size, offset) < page_size) { 1945 ret = submit_one_bio(rw, bio, mirror_num, 1946 prev_bio_flags); 1947 bio = NULL; 1948 } else { 1949 return 0; 1950 } 1951 } 1952 if (this_compressed) 1953 nr = BIO_MAX_PAGES; 1954 else 1955 nr = bio_get_nr_vecs(bdev); 1956 1957 bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH); 1958 1959 bio_add_page(bio, page, page_size, offset); 1960 bio->bi_end_io = end_io_func; 1961 bio->bi_private = tree; 1962 1963 if (bio_ret) 1964 *bio_ret = bio; 1965 else 1966 ret = submit_one_bio(rw, bio, mirror_num, bio_flags); 1967 1968 return ret; 1969 } 1970 1971 void set_page_extent_mapped(struct page *page) 1972 { 1973 if (!PagePrivate(page)) { 1974 SetPagePrivate(page); 1975 page_cache_get(page); 1976 set_page_private(page, EXTENT_PAGE_PRIVATE); 1977 } 1978 } 1979 1980 static void set_page_extent_head(struct page *page, unsigned long len) 1981 { 1982 set_page_private(page, EXTENT_PAGE_PRIVATE_FIRST_PAGE | len << 2); 1983 } 1984 1985 /* 1986 * basic readpage implementation. Locked extent state structs are inserted 1987 * into the tree that are removed when the IO is done (by the end_io 1988 * handlers) 1989 */ 1990 static int __extent_read_full_page(struct extent_io_tree *tree, 1991 struct page *page, 1992 get_extent_t *get_extent, 1993 struct bio **bio, int mirror_num, 1994 unsigned long *bio_flags) 1995 { 1996 struct inode *inode = page->mapping->host; 1997 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 1998 u64 page_end = start + PAGE_CACHE_SIZE - 1; 1999 u64 end; 2000 u64 cur = start; 2001 u64 extent_offset; 2002 u64 last_byte = i_size_read(inode); 2003 u64 block_start; 2004 u64 cur_end; 2005 sector_t sector; 2006 struct extent_map *em; 2007 struct block_device *bdev; 2008 int ret; 2009 int nr = 0; 2010 size_t page_offset = 0; 2011 size_t iosize; 2012 size_t disk_io_size; 2013 size_t blocksize = inode->i_sb->s_blocksize; 2014 unsigned long this_bio_flag = 0; 2015 2016 set_page_extent_mapped(page); 2017 2018 end = page_end; 2019 lock_extent(tree, start, end, GFP_NOFS); 2020 2021 if (page->index == last_byte >> PAGE_CACHE_SHIFT) { 2022 char *userpage; 2023 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1); 2024 2025 if (zero_offset) { 2026 iosize = PAGE_CACHE_SIZE - zero_offset; 2027 userpage = kmap_atomic(page, KM_USER0); 2028 memset(userpage + zero_offset, 0, iosize); 2029 flush_dcache_page(page); 2030 kunmap_atomic(userpage, KM_USER0); 2031 } 2032 } 2033 while (cur <= end) { 2034 if (cur >= last_byte) { 2035 char *userpage; 2036 iosize = PAGE_CACHE_SIZE - page_offset; 2037 userpage = kmap_atomic(page, KM_USER0); 2038 memset(userpage + page_offset, 0, iosize); 2039 flush_dcache_page(page); 2040 kunmap_atomic(userpage, KM_USER0); 2041 set_extent_uptodate(tree, cur, cur + iosize - 1, 2042 GFP_NOFS); 2043 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS); 2044 break; 2045 } 2046 em = get_extent(inode, page, page_offset, cur, 2047 end - cur + 1, 0); 2048 if (IS_ERR(em) || !em) { 2049 SetPageError(page); 2050 unlock_extent(tree, cur, end, GFP_NOFS); 2051 break; 2052 } 2053 extent_offset = cur - em->start; 2054 BUG_ON(extent_map_end(em) <= cur); 2055 BUG_ON(end < cur); 2056 2057 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 2058 this_bio_flag = EXTENT_BIO_COMPRESSED; 2059 2060 iosize = min(extent_map_end(em) - cur, end - cur + 1); 2061 cur_end = min(extent_map_end(em) - 1, end); 2062 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1); 2063 if (this_bio_flag & EXTENT_BIO_COMPRESSED) { 2064 disk_io_size = em->block_len; 2065 sector = em->block_start >> 9; 2066 } else { 2067 sector = (em->block_start + extent_offset) >> 9; 2068 disk_io_size = iosize; 2069 } 2070 bdev = em->bdev; 2071 block_start = em->block_start; 2072 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 2073 block_start = EXTENT_MAP_HOLE; 2074 free_extent_map(em); 2075 em = NULL; 2076 2077 /* we've found a hole, just zero and go on */ 2078 if (block_start == EXTENT_MAP_HOLE) { 2079 char *userpage; 2080 userpage = kmap_atomic(page, KM_USER0); 2081 memset(userpage + page_offset, 0, iosize); 2082 flush_dcache_page(page); 2083 kunmap_atomic(userpage, KM_USER0); 2084 2085 set_extent_uptodate(tree, cur, cur + iosize - 1, 2086 GFP_NOFS); 2087 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS); 2088 cur = cur + iosize; 2089 page_offset += iosize; 2090 continue; 2091 } 2092 /* the get_extent function already copied into the page */ 2093 if (test_range_bit(tree, cur, cur_end, 2094 EXTENT_UPTODATE, 1, NULL)) { 2095 check_page_uptodate(tree, page); 2096 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS); 2097 cur = cur + iosize; 2098 page_offset += iosize; 2099 continue; 2100 } 2101 /* we have an inline extent but it didn't get marked up 2102 * to date. Error out 2103 */ 2104 if (block_start == EXTENT_MAP_INLINE) { 2105 SetPageError(page); 2106 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS); 2107 cur = cur + iosize; 2108 page_offset += iosize; 2109 continue; 2110 } 2111 2112 ret = 0; 2113 if (tree->ops && tree->ops->readpage_io_hook) { 2114 ret = tree->ops->readpage_io_hook(page, cur, 2115 cur + iosize - 1); 2116 } 2117 if (!ret) { 2118 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1; 2119 pnr -= page->index; 2120 ret = submit_extent_page(READ, tree, page, 2121 sector, disk_io_size, page_offset, 2122 bdev, bio, pnr, 2123 end_bio_extent_readpage, mirror_num, 2124 *bio_flags, 2125 this_bio_flag); 2126 nr++; 2127 *bio_flags = this_bio_flag; 2128 } 2129 if (ret) 2130 SetPageError(page); 2131 cur = cur + iosize; 2132 page_offset += iosize; 2133 } 2134 if (!nr) { 2135 if (!PageError(page)) 2136 SetPageUptodate(page); 2137 unlock_page(page); 2138 } 2139 return 0; 2140 } 2141 2142 int extent_read_full_page(struct extent_io_tree *tree, struct page *page, 2143 get_extent_t *get_extent) 2144 { 2145 struct bio *bio = NULL; 2146 unsigned long bio_flags = 0; 2147 int ret; 2148 2149 ret = __extent_read_full_page(tree, page, get_extent, &bio, 0, 2150 &bio_flags); 2151 if (bio) 2152 submit_one_bio(READ, bio, 0, bio_flags); 2153 return ret; 2154 } 2155 2156 static noinline void update_nr_written(struct page *page, 2157 struct writeback_control *wbc, 2158 unsigned long nr_written) 2159 { 2160 wbc->nr_to_write -= nr_written; 2161 if (wbc->range_cyclic || (wbc->nr_to_write > 0 && 2162 wbc->range_start == 0 && wbc->range_end == LLONG_MAX)) 2163 page->mapping->writeback_index = page->index + nr_written; 2164 } 2165 2166 /* 2167 * the writepage semantics are similar to regular writepage. extent 2168 * records are inserted to lock ranges in the tree, and as dirty areas 2169 * are found, they are marked writeback. Then the lock bits are removed 2170 * and the end_io handler clears the writeback ranges 2171 */ 2172 static int __extent_writepage(struct page *page, struct writeback_control *wbc, 2173 void *data) 2174 { 2175 struct inode *inode = page->mapping->host; 2176 struct extent_page_data *epd = data; 2177 struct extent_io_tree *tree = epd->tree; 2178 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 2179 u64 delalloc_start; 2180 u64 page_end = start + PAGE_CACHE_SIZE - 1; 2181 u64 end; 2182 u64 cur = start; 2183 u64 extent_offset; 2184 u64 last_byte = i_size_read(inode); 2185 u64 block_start; 2186 u64 iosize; 2187 u64 unlock_start; 2188 sector_t sector; 2189 struct extent_state *cached_state = NULL; 2190 struct extent_map *em; 2191 struct block_device *bdev; 2192 int ret; 2193 int nr = 0; 2194 size_t pg_offset = 0; 2195 size_t blocksize; 2196 loff_t i_size = i_size_read(inode); 2197 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT; 2198 u64 nr_delalloc; 2199 u64 delalloc_end; 2200 int page_started; 2201 int compressed; 2202 int write_flags; 2203 unsigned long nr_written = 0; 2204 2205 if (wbc->sync_mode == WB_SYNC_ALL) 2206 write_flags = WRITE_SYNC_PLUG; 2207 else 2208 write_flags = WRITE; 2209 2210 WARN_ON(!PageLocked(page)); 2211 pg_offset = i_size & (PAGE_CACHE_SIZE - 1); 2212 if (page->index > end_index || 2213 (page->index == end_index && !pg_offset)) { 2214 page->mapping->a_ops->invalidatepage(page, 0); 2215 unlock_page(page); 2216 return 0; 2217 } 2218 2219 if (page->index == end_index) { 2220 char *userpage; 2221 2222 userpage = kmap_atomic(page, KM_USER0); 2223 memset(userpage + pg_offset, 0, 2224 PAGE_CACHE_SIZE - pg_offset); 2225 kunmap_atomic(userpage, KM_USER0); 2226 flush_dcache_page(page); 2227 } 2228 pg_offset = 0; 2229 2230 set_page_extent_mapped(page); 2231 2232 delalloc_start = start; 2233 delalloc_end = 0; 2234 page_started = 0; 2235 if (!epd->extent_locked) { 2236 u64 delalloc_to_write = 0; 2237 /* 2238 * make sure the wbc mapping index is at least updated 2239 * to this page. 2240 */ 2241 update_nr_written(page, wbc, 0); 2242 2243 while (delalloc_end < page_end) { 2244 nr_delalloc = find_lock_delalloc_range(inode, tree, 2245 page, 2246 &delalloc_start, 2247 &delalloc_end, 2248 128 * 1024 * 1024); 2249 if (nr_delalloc == 0) { 2250 delalloc_start = delalloc_end + 1; 2251 continue; 2252 } 2253 tree->ops->fill_delalloc(inode, page, delalloc_start, 2254 delalloc_end, &page_started, 2255 &nr_written); 2256 /* 2257 * delalloc_end is already one less than the total 2258 * length, so we don't subtract one from 2259 * PAGE_CACHE_SIZE 2260 */ 2261 delalloc_to_write += (delalloc_end - delalloc_start + 2262 PAGE_CACHE_SIZE) >> 2263 PAGE_CACHE_SHIFT; 2264 delalloc_start = delalloc_end + 1; 2265 } 2266 if (wbc->nr_to_write < delalloc_to_write) { 2267 int thresh = 8192; 2268 2269 if (delalloc_to_write < thresh * 2) 2270 thresh = delalloc_to_write; 2271 wbc->nr_to_write = min_t(u64, delalloc_to_write, 2272 thresh); 2273 } 2274 2275 /* did the fill delalloc function already unlock and start 2276 * the IO? 2277 */ 2278 if (page_started) { 2279 ret = 0; 2280 /* 2281 * we've unlocked the page, so we can't update 2282 * the mapping's writeback index, just update 2283 * nr_to_write. 2284 */ 2285 wbc->nr_to_write -= nr_written; 2286 goto done_unlocked; 2287 } 2288 } 2289 if (tree->ops && tree->ops->writepage_start_hook) { 2290 ret = tree->ops->writepage_start_hook(page, start, 2291 page_end); 2292 if (ret == -EAGAIN) { 2293 redirty_page_for_writepage(wbc, page); 2294 update_nr_written(page, wbc, nr_written); 2295 unlock_page(page); 2296 ret = 0; 2297 goto done_unlocked; 2298 } 2299 } 2300 2301 /* 2302 * we don't want to touch the inode after unlocking the page, 2303 * so we update the mapping writeback index now 2304 */ 2305 update_nr_written(page, wbc, nr_written + 1); 2306 2307 end = page_end; 2308 if (last_byte <= start) { 2309 if (tree->ops && tree->ops->writepage_end_io_hook) 2310 tree->ops->writepage_end_io_hook(page, start, 2311 page_end, NULL, 1); 2312 unlock_start = page_end + 1; 2313 goto done; 2314 } 2315 2316 blocksize = inode->i_sb->s_blocksize; 2317 2318 while (cur <= end) { 2319 if (cur >= last_byte) { 2320 if (tree->ops && tree->ops->writepage_end_io_hook) 2321 tree->ops->writepage_end_io_hook(page, cur, 2322 page_end, NULL, 1); 2323 unlock_start = page_end + 1; 2324 break; 2325 } 2326 em = epd->get_extent(inode, page, pg_offset, cur, 2327 end - cur + 1, 1); 2328 if (IS_ERR(em) || !em) { 2329 SetPageError(page); 2330 break; 2331 } 2332 2333 extent_offset = cur - em->start; 2334 BUG_ON(extent_map_end(em) <= cur); 2335 BUG_ON(end < cur); 2336 iosize = min(extent_map_end(em) - cur, end - cur + 1); 2337 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1); 2338 sector = (em->block_start + extent_offset) >> 9; 2339 bdev = em->bdev; 2340 block_start = em->block_start; 2341 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 2342 free_extent_map(em); 2343 em = NULL; 2344 2345 /* 2346 * compressed and inline extents are written through other 2347 * paths in the FS 2348 */ 2349 if (compressed || block_start == EXTENT_MAP_HOLE || 2350 block_start == EXTENT_MAP_INLINE) { 2351 /* 2352 * end_io notification does not happen here for 2353 * compressed extents 2354 */ 2355 if (!compressed && tree->ops && 2356 tree->ops->writepage_end_io_hook) 2357 tree->ops->writepage_end_io_hook(page, cur, 2358 cur + iosize - 1, 2359 NULL, 1); 2360 else if (compressed) { 2361 /* we don't want to end_page_writeback on 2362 * a compressed extent. this happens 2363 * elsewhere 2364 */ 2365 nr++; 2366 } 2367 2368 cur += iosize; 2369 pg_offset += iosize; 2370 unlock_start = cur; 2371 continue; 2372 } 2373 /* leave this out until we have a page_mkwrite call */ 2374 if (0 && !test_range_bit(tree, cur, cur + iosize - 1, 2375 EXTENT_DIRTY, 0, NULL)) { 2376 cur = cur + iosize; 2377 pg_offset += iosize; 2378 continue; 2379 } 2380 2381 if (tree->ops && tree->ops->writepage_io_hook) { 2382 ret = tree->ops->writepage_io_hook(page, cur, 2383 cur + iosize - 1); 2384 } else { 2385 ret = 0; 2386 } 2387 if (ret) { 2388 SetPageError(page); 2389 } else { 2390 unsigned long max_nr = end_index + 1; 2391 2392 set_range_writeback(tree, cur, cur + iosize - 1); 2393 if (!PageWriteback(page)) { 2394 printk(KERN_ERR "btrfs warning page %lu not " 2395 "writeback, cur %llu end %llu\n", 2396 page->index, (unsigned long long)cur, 2397 (unsigned long long)end); 2398 } 2399 2400 ret = submit_extent_page(write_flags, tree, page, 2401 sector, iosize, pg_offset, 2402 bdev, &epd->bio, max_nr, 2403 end_bio_extent_writepage, 2404 0, 0, 0); 2405 if (ret) 2406 SetPageError(page); 2407 } 2408 cur = cur + iosize; 2409 pg_offset += iosize; 2410 nr++; 2411 } 2412 done: 2413 if (nr == 0) { 2414 /* make sure the mapping tag for page dirty gets cleared */ 2415 set_page_writeback(page); 2416 end_page_writeback(page); 2417 } 2418 unlock_page(page); 2419 2420 done_unlocked: 2421 2422 /* drop our reference on any cached states */ 2423 free_extent_state(cached_state); 2424 return 0; 2425 } 2426 2427 /** 2428 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. 2429 * @mapping: address space structure to write 2430 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 2431 * @writepage: function called for each page 2432 * @data: data passed to writepage function 2433 * 2434 * If a page is already under I/O, write_cache_pages() skips it, even 2435 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 2436 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 2437 * and msync() need to guarantee that all the data which was dirty at the time 2438 * the call was made get new I/O started against them. If wbc->sync_mode is 2439 * WB_SYNC_ALL then we were called for data integrity and we must wait for 2440 * existing IO to complete. 2441 */ 2442 static int extent_write_cache_pages(struct extent_io_tree *tree, 2443 struct address_space *mapping, 2444 struct writeback_control *wbc, 2445 writepage_t writepage, void *data, 2446 void (*flush_fn)(void *)) 2447 { 2448 int ret = 0; 2449 int done = 0; 2450 int nr_to_write_done = 0; 2451 struct pagevec pvec; 2452 int nr_pages; 2453 pgoff_t index; 2454 pgoff_t end; /* Inclusive */ 2455 int scanned = 0; 2456 int range_whole = 0; 2457 2458 pagevec_init(&pvec, 0); 2459 if (wbc->range_cyclic) { 2460 index = mapping->writeback_index; /* Start from prev offset */ 2461 end = -1; 2462 } else { 2463 index = wbc->range_start >> PAGE_CACHE_SHIFT; 2464 end = wbc->range_end >> PAGE_CACHE_SHIFT; 2465 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2466 range_whole = 1; 2467 scanned = 1; 2468 } 2469 retry: 2470 while (!done && !nr_to_write_done && (index <= end) && 2471 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 2472 PAGECACHE_TAG_DIRTY, min(end - index, 2473 (pgoff_t)PAGEVEC_SIZE-1) + 1))) { 2474 unsigned i; 2475 2476 scanned = 1; 2477 for (i = 0; i < nr_pages; i++) { 2478 struct page *page = pvec.pages[i]; 2479 2480 /* 2481 * At this point we hold neither mapping->tree_lock nor 2482 * lock on the page itself: the page may be truncated or 2483 * invalidated (changing page->mapping to NULL), or even 2484 * swizzled back from swapper_space to tmpfs file 2485 * mapping 2486 */ 2487 if (tree->ops && tree->ops->write_cache_pages_lock_hook) 2488 tree->ops->write_cache_pages_lock_hook(page); 2489 else 2490 lock_page(page); 2491 2492 if (unlikely(page->mapping != mapping)) { 2493 unlock_page(page); 2494 continue; 2495 } 2496 2497 if (!wbc->range_cyclic && page->index > end) { 2498 done = 1; 2499 unlock_page(page); 2500 continue; 2501 } 2502 2503 if (wbc->sync_mode != WB_SYNC_NONE) { 2504 if (PageWriteback(page)) 2505 flush_fn(data); 2506 wait_on_page_writeback(page); 2507 } 2508 2509 if (PageWriteback(page) || 2510 !clear_page_dirty_for_io(page)) { 2511 unlock_page(page); 2512 continue; 2513 } 2514 2515 ret = (*writepage)(page, wbc, data); 2516 2517 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) { 2518 unlock_page(page); 2519 ret = 0; 2520 } 2521 if (ret) 2522 done = 1; 2523 2524 /* 2525 * the filesystem may choose to bump up nr_to_write. 2526 * We have to make sure to honor the new nr_to_write 2527 * at any time 2528 */ 2529 nr_to_write_done = wbc->nr_to_write <= 0; 2530 } 2531 pagevec_release(&pvec); 2532 cond_resched(); 2533 } 2534 if (!scanned && !done) { 2535 /* 2536 * We hit the last page and there is more work to be done: wrap 2537 * back to the start of the file 2538 */ 2539 scanned = 1; 2540 index = 0; 2541 goto retry; 2542 } 2543 return ret; 2544 } 2545 2546 static void flush_epd_write_bio(struct extent_page_data *epd) 2547 { 2548 if (epd->bio) { 2549 if (epd->sync_io) 2550 submit_one_bio(WRITE_SYNC, epd->bio, 0, 0); 2551 else 2552 submit_one_bio(WRITE, epd->bio, 0, 0); 2553 epd->bio = NULL; 2554 } 2555 } 2556 2557 static noinline void flush_write_bio(void *data) 2558 { 2559 struct extent_page_data *epd = data; 2560 flush_epd_write_bio(epd); 2561 } 2562 2563 int extent_write_full_page(struct extent_io_tree *tree, struct page *page, 2564 get_extent_t *get_extent, 2565 struct writeback_control *wbc) 2566 { 2567 int ret; 2568 struct address_space *mapping = page->mapping; 2569 struct extent_page_data epd = { 2570 .bio = NULL, 2571 .tree = tree, 2572 .get_extent = get_extent, 2573 .extent_locked = 0, 2574 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 2575 }; 2576 struct writeback_control wbc_writepages = { 2577 .bdi = wbc->bdi, 2578 .sync_mode = wbc->sync_mode, 2579 .older_than_this = NULL, 2580 .nr_to_write = 64, 2581 .range_start = page_offset(page) + PAGE_CACHE_SIZE, 2582 .range_end = (loff_t)-1, 2583 }; 2584 2585 ret = __extent_writepage(page, wbc, &epd); 2586 2587 extent_write_cache_pages(tree, mapping, &wbc_writepages, 2588 __extent_writepage, &epd, flush_write_bio); 2589 flush_epd_write_bio(&epd); 2590 return ret; 2591 } 2592 2593 int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode, 2594 u64 start, u64 end, get_extent_t *get_extent, 2595 int mode) 2596 { 2597 int ret = 0; 2598 struct address_space *mapping = inode->i_mapping; 2599 struct page *page; 2600 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >> 2601 PAGE_CACHE_SHIFT; 2602 2603 struct extent_page_data epd = { 2604 .bio = NULL, 2605 .tree = tree, 2606 .get_extent = get_extent, 2607 .extent_locked = 1, 2608 .sync_io = mode == WB_SYNC_ALL, 2609 }; 2610 struct writeback_control wbc_writepages = { 2611 .bdi = inode->i_mapping->backing_dev_info, 2612 .sync_mode = mode, 2613 .older_than_this = NULL, 2614 .nr_to_write = nr_pages * 2, 2615 .range_start = start, 2616 .range_end = end + 1, 2617 }; 2618 2619 while (start <= end) { 2620 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT); 2621 if (clear_page_dirty_for_io(page)) 2622 ret = __extent_writepage(page, &wbc_writepages, &epd); 2623 else { 2624 if (tree->ops && tree->ops->writepage_end_io_hook) 2625 tree->ops->writepage_end_io_hook(page, start, 2626 start + PAGE_CACHE_SIZE - 1, 2627 NULL, 1); 2628 unlock_page(page); 2629 } 2630 page_cache_release(page); 2631 start += PAGE_CACHE_SIZE; 2632 } 2633 2634 flush_epd_write_bio(&epd); 2635 return ret; 2636 } 2637 2638 int extent_writepages(struct extent_io_tree *tree, 2639 struct address_space *mapping, 2640 get_extent_t *get_extent, 2641 struct writeback_control *wbc) 2642 { 2643 int ret = 0; 2644 struct extent_page_data epd = { 2645 .bio = NULL, 2646 .tree = tree, 2647 .get_extent = get_extent, 2648 .extent_locked = 0, 2649 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 2650 }; 2651 2652 ret = extent_write_cache_pages(tree, mapping, wbc, 2653 __extent_writepage, &epd, 2654 flush_write_bio); 2655 flush_epd_write_bio(&epd); 2656 return ret; 2657 } 2658 2659 int extent_readpages(struct extent_io_tree *tree, 2660 struct address_space *mapping, 2661 struct list_head *pages, unsigned nr_pages, 2662 get_extent_t get_extent) 2663 { 2664 struct bio *bio = NULL; 2665 unsigned page_idx; 2666 struct pagevec pvec; 2667 unsigned long bio_flags = 0; 2668 2669 pagevec_init(&pvec, 0); 2670 for (page_idx = 0; page_idx < nr_pages; page_idx++) { 2671 struct page *page = list_entry(pages->prev, struct page, lru); 2672 2673 prefetchw(&page->flags); 2674 list_del(&page->lru); 2675 /* 2676 * what we want to do here is call add_to_page_cache_lru, 2677 * but that isn't exported, so we reproduce it here 2678 */ 2679 if (!add_to_page_cache(page, mapping, 2680 page->index, GFP_KERNEL)) { 2681 2682 /* open coding of lru_cache_add, also not exported */ 2683 page_cache_get(page); 2684 if (!pagevec_add(&pvec, page)) 2685 __pagevec_lru_add_file(&pvec); 2686 __extent_read_full_page(tree, page, get_extent, 2687 &bio, 0, &bio_flags); 2688 } 2689 page_cache_release(page); 2690 } 2691 if (pagevec_count(&pvec)) 2692 __pagevec_lru_add_file(&pvec); 2693 BUG_ON(!list_empty(pages)); 2694 if (bio) 2695 submit_one_bio(READ, bio, 0, bio_flags); 2696 return 0; 2697 } 2698 2699 /* 2700 * basic invalidatepage code, this waits on any locked or writeback 2701 * ranges corresponding to the page, and then deletes any extent state 2702 * records from the tree 2703 */ 2704 int extent_invalidatepage(struct extent_io_tree *tree, 2705 struct page *page, unsigned long offset) 2706 { 2707 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT); 2708 u64 end = start + PAGE_CACHE_SIZE - 1; 2709 size_t blocksize = page->mapping->host->i_sb->s_blocksize; 2710 2711 start += (offset + blocksize - 1) & ~(blocksize - 1); 2712 if (start > end) 2713 return 0; 2714 2715 lock_extent(tree, start, end, GFP_NOFS); 2716 wait_on_page_writeback(page); 2717 clear_extent_bit(tree, start, end, 2718 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC | 2719 EXTENT_DO_ACCOUNTING, 2720 1, 1, NULL, GFP_NOFS); 2721 return 0; 2722 } 2723 2724 /* 2725 * simple commit_write call, set_range_dirty is used to mark both 2726 * the pages and the extent records as dirty 2727 */ 2728 int extent_commit_write(struct extent_io_tree *tree, 2729 struct inode *inode, struct page *page, 2730 unsigned from, unsigned to) 2731 { 2732 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to; 2733 2734 set_page_extent_mapped(page); 2735 set_page_dirty(page); 2736 2737 if (pos > inode->i_size) { 2738 i_size_write(inode, pos); 2739 mark_inode_dirty(inode); 2740 } 2741 return 0; 2742 } 2743 2744 int extent_prepare_write(struct extent_io_tree *tree, 2745 struct inode *inode, struct page *page, 2746 unsigned from, unsigned to, get_extent_t *get_extent) 2747 { 2748 u64 page_start = (u64)page->index << PAGE_CACHE_SHIFT; 2749 u64 page_end = page_start + PAGE_CACHE_SIZE - 1; 2750 u64 block_start; 2751 u64 orig_block_start; 2752 u64 block_end; 2753 u64 cur_end; 2754 struct extent_map *em; 2755 unsigned blocksize = 1 << inode->i_blkbits; 2756 size_t page_offset = 0; 2757 size_t block_off_start; 2758 size_t block_off_end; 2759 int err = 0; 2760 int iocount = 0; 2761 int ret = 0; 2762 int isnew; 2763 2764 set_page_extent_mapped(page); 2765 2766 block_start = (page_start + from) & ~((u64)blocksize - 1); 2767 block_end = (page_start + to - 1) | (blocksize - 1); 2768 orig_block_start = block_start; 2769 2770 lock_extent(tree, page_start, page_end, GFP_NOFS); 2771 while (block_start <= block_end) { 2772 em = get_extent(inode, page, page_offset, block_start, 2773 block_end - block_start + 1, 1); 2774 if (IS_ERR(em) || !em) 2775 goto err; 2776 2777 cur_end = min(block_end, extent_map_end(em) - 1); 2778 block_off_start = block_start & (PAGE_CACHE_SIZE - 1); 2779 block_off_end = block_off_start + blocksize; 2780 isnew = clear_extent_new(tree, block_start, cur_end, GFP_NOFS); 2781 2782 if (!PageUptodate(page) && isnew && 2783 (block_off_end > to || block_off_start < from)) { 2784 void *kaddr; 2785 2786 kaddr = kmap_atomic(page, KM_USER0); 2787 if (block_off_end > to) 2788 memset(kaddr + to, 0, block_off_end - to); 2789 if (block_off_start < from) 2790 memset(kaddr + block_off_start, 0, 2791 from - block_off_start); 2792 flush_dcache_page(page); 2793 kunmap_atomic(kaddr, KM_USER0); 2794 } 2795 if ((em->block_start != EXTENT_MAP_HOLE && 2796 em->block_start != EXTENT_MAP_INLINE) && 2797 !isnew && !PageUptodate(page) && 2798 (block_off_end > to || block_off_start < from) && 2799 !test_range_bit(tree, block_start, cur_end, 2800 EXTENT_UPTODATE, 1, NULL)) { 2801 u64 sector; 2802 u64 extent_offset = block_start - em->start; 2803 size_t iosize; 2804 sector = (em->block_start + extent_offset) >> 9; 2805 iosize = (cur_end - block_start + blocksize) & 2806 ~((u64)blocksize - 1); 2807 /* 2808 * we've already got the extent locked, but we 2809 * need to split the state such that our end_bio 2810 * handler can clear the lock. 2811 */ 2812 set_extent_bit(tree, block_start, 2813 block_start + iosize - 1, 2814 EXTENT_LOCKED, 0, NULL, NULL, GFP_NOFS); 2815 ret = submit_extent_page(READ, tree, page, 2816 sector, iosize, page_offset, em->bdev, 2817 NULL, 1, 2818 end_bio_extent_preparewrite, 0, 2819 0, 0); 2820 iocount++; 2821 block_start = block_start + iosize; 2822 } else { 2823 set_extent_uptodate(tree, block_start, cur_end, 2824 GFP_NOFS); 2825 unlock_extent(tree, block_start, cur_end, GFP_NOFS); 2826 block_start = cur_end + 1; 2827 } 2828 page_offset = block_start & (PAGE_CACHE_SIZE - 1); 2829 free_extent_map(em); 2830 } 2831 if (iocount) { 2832 wait_extent_bit(tree, orig_block_start, 2833 block_end, EXTENT_LOCKED); 2834 } 2835 check_page_uptodate(tree, page); 2836 err: 2837 /* FIXME, zero out newly allocated blocks on error */ 2838 return err; 2839 } 2840 2841 /* 2842 * a helper for releasepage, this tests for areas of the page that 2843 * are locked or under IO and drops the related state bits if it is safe 2844 * to drop the page. 2845 */ 2846 int try_release_extent_state(struct extent_map_tree *map, 2847 struct extent_io_tree *tree, struct page *page, 2848 gfp_t mask) 2849 { 2850 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 2851 u64 end = start + PAGE_CACHE_SIZE - 1; 2852 int ret = 1; 2853 2854 if (test_range_bit(tree, start, end, 2855 EXTENT_IOBITS, 0, NULL)) 2856 ret = 0; 2857 else { 2858 if ((mask & GFP_NOFS) == GFP_NOFS) 2859 mask = GFP_NOFS; 2860 /* 2861 * at this point we can safely clear everything except the 2862 * locked bit and the nodatasum bit 2863 */ 2864 clear_extent_bit(tree, start, end, 2865 ~(EXTENT_LOCKED | EXTENT_NODATASUM), 2866 0, 0, NULL, mask); 2867 } 2868 return ret; 2869 } 2870 2871 /* 2872 * a helper for releasepage. As long as there are no locked extents 2873 * in the range corresponding to the page, both state records and extent 2874 * map records are removed 2875 */ 2876 int try_release_extent_mapping(struct extent_map_tree *map, 2877 struct extent_io_tree *tree, struct page *page, 2878 gfp_t mask) 2879 { 2880 struct extent_map *em; 2881 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 2882 u64 end = start + PAGE_CACHE_SIZE - 1; 2883 2884 if ((mask & __GFP_WAIT) && 2885 page->mapping->host->i_size > 16 * 1024 * 1024) { 2886 u64 len; 2887 while (start <= end) { 2888 len = end - start + 1; 2889 write_lock(&map->lock); 2890 em = lookup_extent_mapping(map, start, len); 2891 if (!em || IS_ERR(em)) { 2892 write_unlock(&map->lock); 2893 break; 2894 } 2895 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) || 2896 em->start != start) { 2897 write_unlock(&map->lock); 2898 free_extent_map(em); 2899 break; 2900 } 2901 if (!test_range_bit(tree, em->start, 2902 extent_map_end(em) - 1, 2903 EXTENT_LOCKED | EXTENT_WRITEBACK, 2904 0, NULL)) { 2905 remove_extent_mapping(map, em); 2906 /* once for the rb tree */ 2907 free_extent_map(em); 2908 } 2909 start = extent_map_end(em); 2910 write_unlock(&map->lock); 2911 2912 /* once for us */ 2913 free_extent_map(em); 2914 } 2915 } 2916 return try_release_extent_state(map, tree, page, mask); 2917 } 2918 2919 sector_t extent_bmap(struct address_space *mapping, sector_t iblock, 2920 get_extent_t *get_extent) 2921 { 2922 struct inode *inode = mapping->host; 2923 u64 start = iblock << inode->i_blkbits; 2924 sector_t sector = 0; 2925 size_t blksize = (1 << inode->i_blkbits); 2926 struct extent_map *em; 2927 2928 lock_extent(&BTRFS_I(inode)->io_tree, start, start + blksize - 1, 2929 GFP_NOFS); 2930 em = get_extent(inode, NULL, 0, start, blksize, 0); 2931 unlock_extent(&BTRFS_I(inode)->io_tree, start, start + blksize - 1, 2932 GFP_NOFS); 2933 if (!em || IS_ERR(em)) 2934 return 0; 2935 2936 if (em->block_start > EXTENT_MAP_LAST_BYTE) 2937 goto out; 2938 2939 sector = (em->block_start + start - em->start) >> inode->i_blkbits; 2940 out: 2941 free_extent_map(em); 2942 return sector; 2943 } 2944 2945 int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 2946 __u64 start, __u64 len, get_extent_t *get_extent) 2947 { 2948 int ret; 2949 u64 off = start; 2950 u64 max = start + len; 2951 u32 flags = 0; 2952 u64 disko = 0; 2953 struct extent_map *em = NULL; 2954 int end = 0; 2955 u64 em_start = 0, em_len = 0; 2956 unsigned long emflags; 2957 ret = 0; 2958 2959 if (len == 0) 2960 return -EINVAL; 2961 2962 lock_extent(&BTRFS_I(inode)->io_tree, start, start + len, 2963 GFP_NOFS); 2964 em = get_extent(inode, NULL, 0, off, max - off, 0); 2965 if (!em) 2966 goto out; 2967 if (IS_ERR(em)) { 2968 ret = PTR_ERR(em); 2969 goto out; 2970 } 2971 while (!end) { 2972 off = em->start + em->len; 2973 if (off >= max) 2974 end = 1; 2975 2976 em_start = em->start; 2977 em_len = em->len; 2978 2979 disko = 0; 2980 flags = 0; 2981 2982 if (em->block_start == EXTENT_MAP_LAST_BYTE) { 2983 end = 1; 2984 flags |= FIEMAP_EXTENT_LAST; 2985 } else if (em->block_start == EXTENT_MAP_HOLE) { 2986 flags |= FIEMAP_EXTENT_UNWRITTEN; 2987 } else if (em->block_start == EXTENT_MAP_INLINE) { 2988 flags |= (FIEMAP_EXTENT_DATA_INLINE | 2989 FIEMAP_EXTENT_NOT_ALIGNED); 2990 } else if (em->block_start == EXTENT_MAP_DELALLOC) { 2991 flags |= (FIEMAP_EXTENT_DELALLOC | 2992 FIEMAP_EXTENT_UNKNOWN); 2993 } else { 2994 disko = em->block_start; 2995 } 2996 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 2997 flags |= FIEMAP_EXTENT_ENCODED; 2998 2999 emflags = em->flags; 3000 free_extent_map(em); 3001 em = NULL; 3002 3003 if (!end) { 3004 em = get_extent(inode, NULL, 0, off, max - off, 0); 3005 if (!em) 3006 goto out; 3007 if (IS_ERR(em)) { 3008 ret = PTR_ERR(em); 3009 goto out; 3010 } 3011 emflags = em->flags; 3012 } 3013 if (test_bit(EXTENT_FLAG_VACANCY, &emflags)) { 3014 flags |= FIEMAP_EXTENT_LAST; 3015 end = 1; 3016 } 3017 3018 ret = fiemap_fill_next_extent(fieinfo, em_start, disko, 3019 em_len, flags); 3020 if (ret) 3021 goto out_free; 3022 } 3023 out_free: 3024 free_extent_map(em); 3025 out: 3026 unlock_extent(&BTRFS_I(inode)->io_tree, start, start + len, 3027 GFP_NOFS); 3028 return ret; 3029 } 3030 3031 static inline struct page *extent_buffer_page(struct extent_buffer *eb, 3032 unsigned long i) 3033 { 3034 struct page *p; 3035 struct address_space *mapping; 3036 3037 if (i == 0) 3038 return eb->first_page; 3039 i += eb->start >> PAGE_CACHE_SHIFT; 3040 mapping = eb->first_page->mapping; 3041 if (!mapping) 3042 return NULL; 3043 3044 /* 3045 * extent_buffer_page is only called after pinning the page 3046 * by increasing the reference count. So we know the page must 3047 * be in the radix tree. 3048 */ 3049 rcu_read_lock(); 3050 p = radix_tree_lookup(&mapping->page_tree, i); 3051 rcu_read_unlock(); 3052 3053 return p; 3054 } 3055 3056 static inline unsigned long num_extent_pages(u64 start, u64 len) 3057 { 3058 return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) - 3059 (start >> PAGE_CACHE_SHIFT); 3060 } 3061 3062 static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree, 3063 u64 start, 3064 unsigned long len, 3065 gfp_t mask) 3066 { 3067 struct extent_buffer *eb = NULL; 3068 #if LEAK_DEBUG 3069 unsigned long flags; 3070 #endif 3071 3072 eb = kmem_cache_zalloc(extent_buffer_cache, mask); 3073 eb->start = start; 3074 eb->len = len; 3075 spin_lock_init(&eb->lock); 3076 init_waitqueue_head(&eb->lock_wq); 3077 3078 #if LEAK_DEBUG 3079 spin_lock_irqsave(&leak_lock, flags); 3080 list_add(&eb->leak_list, &buffers); 3081 spin_unlock_irqrestore(&leak_lock, flags); 3082 #endif 3083 atomic_set(&eb->refs, 1); 3084 3085 return eb; 3086 } 3087 3088 static void __free_extent_buffer(struct extent_buffer *eb) 3089 { 3090 #if LEAK_DEBUG 3091 unsigned long flags; 3092 spin_lock_irqsave(&leak_lock, flags); 3093 list_del(&eb->leak_list); 3094 spin_unlock_irqrestore(&leak_lock, flags); 3095 #endif 3096 kmem_cache_free(extent_buffer_cache, eb); 3097 } 3098 3099 struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree, 3100 u64 start, unsigned long len, 3101 struct page *page0, 3102 gfp_t mask) 3103 { 3104 unsigned long num_pages = num_extent_pages(start, len); 3105 unsigned long i; 3106 unsigned long index = start >> PAGE_CACHE_SHIFT; 3107 struct extent_buffer *eb; 3108 struct extent_buffer *exists = NULL; 3109 struct page *p; 3110 struct address_space *mapping = tree->mapping; 3111 int uptodate = 1; 3112 3113 spin_lock(&tree->buffer_lock); 3114 eb = buffer_search(tree, start); 3115 if (eb) { 3116 atomic_inc(&eb->refs); 3117 spin_unlock(&tree->buffer_lock); 3118 mark_page_accessed(eb->first_page); 3119 return eb; 3120 } 3121 spin_unlock(&tree->buffer_lock); 3122 3123 eb = __alloc_extent_buffer(tree, start, len, mask); 3124 if (!eb) 3125 return NULL; 3126 3127 if (page0) { 3128 eb->first_page = page0; 3129 i = 1; 3130 index++; 3131 page_cache_get(page0); 3132 mark_page_accessed(page0); 3133 set_page_extent_mapped(page0); 3134 set_page_extent_head(page0, len); 3135 uptodate = PageUptodate(page0); 3136 } else { 3137 i = 0; 3138 } 3139 for (; i < num_pages; i++, index++) { 3140 p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM); 3141 if (!p) { 3142 WARN_ON(1); 3143 goto free_eb; 3144 } 3145 set_page_extent_mapped(p); 3146 mark_page_accessed(p); 3147 if (i == 0) { 3148 eb->first_page = p; 3149 set_page_extent_head(p, len); 3150 } else { 3151 set_page_private(p, EXTENT_PAGE_PRIVATE); 3152 } 3153 if (!PageUptodate(p)) 3154 uptodate = 0; 3155 unlock_page(p); 3156 } 3157 if (uptodate) 3158 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 3159 3160 spin_lock(&tree->buffer_lock); 3161 exists = buffer_tree_insert(tree, start, &eb->rb_node); 3162 if (exists) { 3163 /* add one reference for the caller */ 3164 atomic_inc(&exists->refs); 3165 spin_unlock(&tree->buffer_lock); 3166 goto free_eb; 3167 } 3168 /* add one reference for the tree */ 3169 atomic_inc(&eb->refs); 3170 spin_unlock(&tree->buffer_lock); 3171 return eb; 3172 3173 free_eb: 3174 if (!atomic_dec_and_test(&eb->refs)) 3175 return exists; 3176 for (index = 1; index < i; index++) 3177 page_cache_release(extent_buffer_page(eb, index)); 3178 page_cache_release(extent_buffer_page(eb, 0)); 3179 __free_extent_buffer(eb); 3180 return exists; 3181 } 3182 3183 struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree, 3184 u64 start, unsigned long len, 3185 gfp_t mask) 3186 { 3187 struct extent_buffer *eb; 3188 3189 spin_lock(&tree->buffer_lock); 3190 eb = buffer_search(tree, start); 3191 if (eb) 3192 atomic_inc(&eb->refs); 3193 spin_unlock(&tree->buffer_lock); 3194 3195 if (eb) 3196 mark_page_accessed(eb->first_page); 3197 3198 return eb; 3199 } 3200 3201 void free_extent_buffer(struct extent_buffer *eb) 3202 { 3203 if (!eb) 3204 return; 3205 3206 if (!atomic_dec_and_test(&eb->refs)) 3207 return; 3208 3209 WARN_ON(1); 3210 } 3211 3212 int clear_extent_buffer_dirty(struct extent_io_tree *tree, 3213 struct extent_buffer *eb) 3214 { 3215 unsigned long i; 3216 unsigned long num_pages; 3217 struct page *page; 3218 3219 num_pages = num_extent_pages(eb->start, eb->len); 3220 3221 for (i = 0; i < num_pages; i++) { 3222 page = extent_buffer_page(eb, i); 3223 if (!PageDirty(page)) 3224 continue; 3225 3226 lock_page(page); 3227 if (i == 0) 3228 set_page_extent_head(page, eb->len); 3229 else 3230 set_page_private(page, EXTENT_PAGE_PRIVATE); 3231 3232 clear_page_dirty_for_io(page); 3233 spin_lock_irq(&page->mapping->tree_lock); 3234 if (!PageDirty(page)) { 3235 radix_tree_tag_clear(&page->mapping->page_tree, 3236 page_index(page), 3237 PAGECACHE_TAG_DIRTY); 3238 } 3239 spin_unlock_irq(&page->mapping->tree_lock); 3240 unlock_page(page); 3241 } 3242 return 0; 3243 } 3244 3245 int wait_on_extent_buffer_writeback(struct extent_io_tree *tree, 3246 struct extent_buffer *eb) 3247 { 3248 return wait_on_extent_writeback(tree, eb->start, 3249 eb->start + eb->len - 1); 3250 } 3251 3252 int set_extent_buffer_dirty(struct extent_io_tree *tree, 3253 struct extent_buffer *eb) 3254 { 3255 unsigned long i; 3256 unsigned long num_pages; 3257 int was_dirty = 0; 3258 3259 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags); 3260 num_pages = num_extent_pages(eb->start, eb->len); 3261 for (i = 0; i < num_pages; i++) 3262 __set_page_dirty_nobuffers(extent_buffer_page(eb, i)); 3263 return was_dirty; 3264 } 3265 3266 int clear_extent_buffer_uptodate(struct extent_io_tree *tree, 3267 struct extent_buffer *eb) 3268 { 3269 unsigned long i; 3270 struct page *page; 3271 unsigned long num_pages; 3272 3273 num_pages = num_extent_pages(eb->start, eb->len); 3274 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 3275 3276 clear_extent_uptodate(tree, eb->start, eb->start + eb->len - 1, 3277 GFP_NOFS); 3278 for (i = 0; i < num_pages; i++) { 3279 page = extent_buffer_page(eb, i); 3280 if (page) 3281 ClearPageUptodate(page); 3282 } 3283 return 0; 3284 } 3285 3286 int set_extent_buffer_uptodate(struct extent_io_tree *tree, 3287 struct extent_buffer *eb) 3288 { 3289 unsigned long i; 3290 struct page *page; 3291 unsigned long num_pages; 3292 3293 num_pages = num_extent_pages(eb->start, eb->len); 3294 3295 set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1, 3296 GFP_NOFS); 3297 for (i = 0; i < num_pages; i++) { 3298 page = extent_buffer_page(eb, i); 3299 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) || 3300 ((i == num_pages - 1) && 3301 ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) { 3302 check_page_uptodate(tree, page); 3303 continue; 3304 } 3305 SetPageUptodate(page); 3306 } 3307 return 0; 3308 } 3309 3310 int extent_range_uptodate(struct extent_io_tree *tree, 3311 u64 start, u64 end) 3312 { 3313 struct page *page; 3314 int ret; 3315 int pg_uptodate = 1; 3316 int uptodate; 3317 unsigned long index; 3318 3319 ret = test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL); 3320 if (ret) 3321 return 1; 3322 while (start <= end) { 3323 index = start >> PAGE_CACHE_SHIFT; 3324 page = find_get_page(tree->mapping, index); 3325 uptodate = PageUptodate(page); 3326 page_cache_release(page); 3327 if (!uptodate) { 3328 pg_uptodate = 0; 3329 break; 3330 } 3331 start += PAGE_CACHE_SIZE; 3332 } 3333 return pg_uptodate; 3334 } 3335 3336 int extent_buffer_uptodate(struct extent_io_tree *tree, 3337 struct extent_buffer *eb) 3338 { 3339 int ret = 0; 3340 unsigned long num_pages; 3341 unsigned long i; 3342 struct page *page; 3343 int pg_uptodate = 1; 3344 3345 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) 3346 return 1; 3347 3348 ret = test_range_bit(tree, eb->start, eb->start + eb->len - 1, 3349 EXTENT_UPTODATE, 1, NULL); 3350 if (ret) 3351 return ret; 3352 3353 num_pages = num_extent_pages(eb->start, eb->len); 3354 for (i = 0; i < num_pages; i++) { 3355 page = extent_buffer_page(eb, i); 3356 if (!PageUptodate(page)) { 3357 pg_uptodate = 0; 3358 break; 3359 } 3360 } 3361 return pg_uptodate; 3362 } 3363 3364 int read_extent_buffer_pages(struct extent_io_tree *tree, 3365 struct extent_buffer *eb, 3366 u64 start, int wait, 3367 get_extent_t *get_extent, int mirror_num) 3368 { 3369 unsigned long i; 3370 unsigned long start_i; 3371 struct page *page; 3372 int err; 3373 int ret = 0; 3374 int locked_pages = 0; 3375 int all_uptodate = 1; 3376 int inc_all_pages = 0; 3377 unsigned long num_pages; 3378 struct bio *bio = NULL; 3379 unsigned long bio_flags = 0; 3380 3381 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) 3382 return 0; 3383 3384 if (test_range_bit(tree, eb->start, eb->start + eb->len - 1, 3385 EXTENT_UPTODATE, 1, NULL)) { 3386 return 0; 3387 } 3388 3389 if (start) { 3390 WARN_ON(start < eb->start); 3391 start_i = (start >> PAGE_CACHE_SHIFT) - 3392 (eb->start >> PAGE_CACHE_SHIFT); 3393 } else { 3394 start_i = 0; 3395 } 3396 3397 num_pages = num_extent_pages(eb->start, eb->len); 3398 for (i = start_i; i < num_pages; i++) { 3399 page = extent_buffer_page(eb, i); 3400 if (!wait) { 3401 if (!trylock_page(page)) 3402 goto unlock_exit; 3403 } else { 3404 lock_page(page); 3405 } 3406 locked_pages++; 3407 if (!PageUptodate(page)) 3408 all_uptodate = 0; 3409 } 3410 if (all_uptodate) { 3411 if (start_i == 0) 3412 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 3413 goto unlock_exit; 3414 } 3415 3416 for (i = start_i; i < num_pages; i++) { 3417 page = extent_buffer_page(eb, i); 3418 if (inc_all_pages) 3419 page_cache_get(page); 3420 if (!PageUptodate(page)) { 3421 if (start_i == 0) 3422 inc_all_pages = 1; 3423 ClearPageError(page); 3424 err = __extent_read_full_page(tree, page, 3425 get_extent, &bio, 3426 mirror_num, &bio_flags); 3427 if (err) 3428 ret = err; 3429 } else { 3430 unlock_page(page); 3431 } 3432 } 3433 3434 if (bio) 3435 submit_one_bio(READ, bio, mirror_num, bio_flags); 3436 3437 if (ret || !wait) 3438 return ret; 3439 3440 for (i = start_i; i < num_pages; i++) { 3441 page = extent_buffer_page(eb, i); 3442 wait_on_page_locked(page); 3443 if (!PageUptodate(page)) 3444 ret = -EIO; 3445 } 3446 3447 if (!ret) 3448 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 3449 return ret; 3450 3451 unlock_exit: 3452 i = start_i; 3453 while (locked_pages > 0) { 3454 page = extent_buffer_page(eb, i); 3455 i++; 3456 unlock_page(page); 3457 locked_pages--; 3458 } 3459 return ret; 3460 } 3461 3462 void read_extent_buffer(struct extent_buffer *eb, void *dstv, 3463 unsigned long start, 3464 unsigned long len) 3465 { 3466 size_t cur; 3467 size_t offset; 3468 struct page *page; 3469 char *kaddr; 3470 char *dst = (char *)dstv; 3471 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 3472 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 3473 3474 WARN_ON(start > eb->len); 3475 WARN_ON(start + len > eb->start + eb->len); 3476 3477 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1); 3478 3479 while (len > 0) { 3480 page = extent_buffer_page(eb, i); 3481 3482 cur = min(len, (PAGE_CACHE_SIZE - offset)); 3483 kaddr = kmap_atomic(page, KM_USER1); 3484 memcpy(dst, kaddr + offset, cur); 3485 kunmap_atomic(kaddr, KM_USER1); 3486 3487 dst += cur; 3488 len -= cur; 3489 offset = 0; 3490 i++; 3491 } 3492 } 3493 3494 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start, 3495 unsigned long min_len, char **token, char **map, 3496 unsigned long *map_start, 3497 unsigned long *map_len, int km) 3498 { 3499 size_t offset = start & (PAGE_CACHE_SIZE - 1); 3500 char *kaddr; 3501 struct page *p; 3502 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 3503 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 3504 unsigned long end_i = (start_offset + start + min_len - 1) >> 3505 PAGE_CACHE_SHIFT; 3506 3507 if (i != end_i) 3508 return -EINVAL; 3509 3510 if (i == 0) { 3511 offset = start_offset; 3512 *map_start = 0; 3513 } else { 3514 offset = 0; 3515 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset; 3516 } 3517 3518 if (start + min_len > eb->len) { 3519 printk(KERN_ERR "btrfs bad mapping eb start %llu len %lu, " 3520 "wanted %lu %lu\n", (unsigned long long)eb->start, 3521 eb->len, start, min_len); 3522 WARN_ON(1); 3523 } 3524 3525 p = extent_buffer_page(eb, i); 3526 kaddr = kmap_atomic(p, km); 3527 *token = kaddr; 3528 *map = kaddr + offset; 3529 *map_len = PAGE_CACHE_SIZE - offset; 3530 return 0; 3531 } 3532 3533 int map_extent_buffer(struct extent_buffer *eb, unsigned long start, 3534 unsigned long min_len, 3535 char **token, char **map, 3536 unsigned long *map_start, 3537 unsigned long *map_len, int km) 3538 { 3539 int err; 3540 int save = 0; 3541 if (eb->map_token) { 3542 unmap_extent_buffer(eb, eb->map_token, km); 3543 eb->map_token = NULL; 3544 save = 1; 3545 } 3546 err = map_private_extent_buffer(eb, start, min_len, token, map, 3547 map_start, map_len, km); 3548 if (!err && save) { 3549 eb->map_token = *token; 3550 eb->kaddr = *map; 3551 eb->map_start = *map_start; 3552 eb->map_len = *map_len; 3553 } 3554 return err; 3555 } 3556 3557 void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km) 3558 { 3559 kunmap_atomic(token, km); 3560 } 3561 3562 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv, 3563 unsigned long start, 3564 unsigned long len) 3565 { 3566 size_t cur; 3567 size_t offset; 3568 struct page *page; 3569 char *kaddr; 3570 char *ptr = (char *)ptrv; 3571 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 3572 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 3573 int ret = 0; 3574 3575 WARN_ON(start > eb->len); 3576 WARN_ON(start + len > eb->start + eb->len); 3577 3578 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1); 3579 3580 while (len > 0) { 3581 page = extent_buffer_page(eb, i); 3582 3583 cur = min(len, (PAGE_CACHE_SIZE - offset)); 3584 3585 kaddr = kmap_atomic(page, KM_USER0); 3586 ret = memcmp(ptr, kaddr + offset, cur); 3587 kunmap_atomic(kaddr, KM_USER0); 3588 if (ret) 3589 break; 3590 3591 ptr += cur; 3592 len -= cur; 3593 offset = 0; 3594 i++; 3595 } 3596 return ret; 3597 } 3598 3599 void write_extent_buffer(struct extent_buffer *eb, const void *srcv, 3600 unsigned long start, unsigned long len) 3601 { 3602 size_t cur; 3603 size_t offset; 3604 struct page *page; 3605 char *kaddr; 3606 char *src = (char *)srcv; 3607 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 3608 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 3609 3610 WARN_ON(start > eb->len); 3611 WARN_ON(start + len > eb->start + eb->len); 3612 3613 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1); 3614 3615 while (len > 0) { 3616 page = extent_buffer_page(eb, i); 3617 WARN_ON(!PageUptodate(page)); 3618 3619 cur = min(len, PAGE_CACHE_SIZE - offset); 3620 kaddr = kmap_atomic(page, KM_USER1); 3621 memcpy(kaddr + offset, src, cur); 3622 kunmap_atomic(kaddr, KM_USER1); 3623 3624 src += cur; 3625 len -= cur; 3626 offset = 0; 3627 i++; 3628 } 3629 } 3630 3631 void memset_extent_buffer(struct extent_buffer *eb, char c, 3632 unsigned long start, unsigned long len) 3633 { 3634 size_t cur; 3635 size_t offset; 3636 struct page *page; 3637 char *kaddr; 3638 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 3639 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 3640 3641 WARN_ON(start > eb->len); 3642 WARN_ON(start + len > eb->start + eb->len); 3643 3644 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1); 3645 3646 while (len > 0) { 3647 page = extent_buffer_page(eb, i); 3648 WARN_ON(!PageUptodate(page)); 3649 3650 cur = min(len, PAGE_CACHE_SIZE - offset); 3651 kaddr = kmap_atomic(page, KM_USER0); 3652 memset(kaddr + offset, c, cur); 3653 kunmap_atomic(kaddr, KM_USER0); 3654 3655 len -= cur; 3656 offset = 0; 3657 i++; 3658 } 3659 } 3660 3661 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, 3662 unsigned long dst_offset, unsigned long src_offset, 3663 unsigned long len) 3664 { 3665 u64 dst_len = dst->len; 3666 size_t cur; 3667 size_t offset; 3668 struct page *page; 3669 char *kaddr; 3670 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1); 3671 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT; 3672 3673 WARN_ON(src->len != dst_len); 3674 3675 offset = (start_offset + dst_offset) & 3676 ((unsigned long)PAGE_CACHE_SIZE - 1); 3677 3678 while (len > 0) { 3679 page = extent_buffer_page(dst, i); 3680 WARN_ON(!PageUptodate(page)); 3681 3682 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset)); 3683 3684 kaddr = kmap_atomic(page, KM_USER0); 3685 read_extent_buffer(src, kaddr + offset, src_offset, cur); 3686 kunmap_atomic(kaddr, KM_USER0); 3687 3688 src_offset += cur; 3689 len -= cur; 3690 offset = 0; 3691 i++; 3692 } 3693 } 3694 3695 static void move_pages(struct page *dst_page, struct page *src_page, 3696 unsigned long dst_off, unsigned long src_off, 3697 unsigned long len) 3698 { 3699 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0); 3700 if (dst_page == src_page) { 3701 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len); 3702 } else { 3703 char *src_kaddr = kmap_atomic(src_page, KM_USER1); 3704 char *p = dst_kaddr + dst_off + len; 3705 char *s = src_kaddr + src_off + len; 3706 3707 while (len--) 3708 *--p = *--s; 3709 3710 kunmap_atomic(src_kaddr, KM_USER1); 3711 } 3712 kunmap_atomic(dst_kaddr, KM_USER0); 3713 } 3714 3715 static void copy_pages(struct page *dst_page, struct page *src_page, 3716 unsigned long dst_off, unsigned long src_off, 3717 unsigned long len) 3718 { 3719 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0); 3720 char *src_kaddr; 3721 3722 if (dst_page != src_page) 3723 src_kaddr = kmap_atomic(src_page, KM_USER1); 3724 else 3725 src_kaddr = dst_kaddr; 3726 3727 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len); 3728 kunmap_atomic(dst_kaddr, KM_USER0); 3729 if (dst_page != src_page) 3730 kunmap_atomic(src_kaddr, KM_USER1); 3731 } 3732 3733 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 3734 unsigned long src_offset, unsigned long len) 3735 { 3736 size_t cur; 3737 size_t dst_off_in_page; 3738 size_t src_off_in_page; 3739 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1); 3740 unsigned long dst_i; 3741 unsigned long src_i; 3742 3743 if (src_offset + len > dst->len) { 3744 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move " 3745 "len %lu dst len %lu\n", src_offset, len, dst->len); 3746 BUG_ON(1); 3747 } 3748 if (dst_offset + len > dst->len) { 3749 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move " 3750 "len %lu dst len %lu\n", dst_offset, len, dst->len); 3751 BUG_ON(1); 3752 } 3753 3754 while (len > 0) { 3755 dst_off_in_page = (start_offset + dst_offset) & 3756 ((unsigned long)PAGE_CACHE_SIZE - 1); 3757 src_off_in_page = (start_offset + src_offset) & 3758 ((unsigned long)PAGE_CACHE_SIZE - 1); 3759 3760 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT; 3761 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT; 3762 3763 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - 3764 src_off_in_page)); 3765 cur = min_t(unsigned long, cur, 3766 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page)); 3767 3768 copy_pages(extent_buffer_page(dst, dst_i), 3769 extent_buffer_page(dst, src_i), 3770 dst_off_in_page, src_off_in_page, cur); 3771 3772 src_offset += cur; 3773 dst_offset += cur; 3774 len -= cur; 3775 } 3776 } 3777 3778 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 3779 unsigned long src_offset, unsigned long len) 3780 { 3781 size_t cur; 3782 size_t dst_off_in_page; 3783 size_t src_off_in_page; 3784 unsigned long dst_end = dst_offset + len - 1; 3785 unsigned long src_end = src_offset + len - 1; 3786 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1); 3787 unsigned long dst_i; 3788 unsigned long src_i; 3789 3790 if (src_offset + len > dst->len) { 3791 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move " 3792 "len %lu len %lu\n", src_offset, len, dst->len); 3793 BUG_ON(1); 3794 } 3795 if (dst_offset + len > dst->len) { 3796 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move " 3797 "len %lu len %lu\n", dst_offset, len, dst->len); 3798 BUG_ON(1); 3799 } 3800 if (dst_offset < src_offset) { 3801 memcpy_extent_buffer(dst, dst_offset, src_offset, len); 3802 return; 3803 } 3804 while (len > 0) { 3805 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT; 3806 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT; 3807 3808 dst_off_in_page = (start_offset + dst_end) & 3809 ((unsigned long)PAGE_CACHE_SIZE - 1); 3810 src_off_in_page = (start_offset + src_end) & 3811 ((unsigned long)PAGE_CACHE_SIZE - 1); 3812 3813 cur = min_t(unsigned long, len, src_off_in_page + 1); 3814 cur = min(cur, dst_off_in_page + 1); 3815 move_pages(extent_buffer_page(dst, dst_i), 3816 extent_buffer_page(dst, src_i), 3817 dst_off_in_page - cur + 1, 3818 src_off_in_page - cur + 1, cur); 3819 3820 dst_end -= cur; 3821 src_end -= cur; 3822 len -= cur; 3823 } 3824 } 3825 3826 int try_release_extent_buffer(struct extent_io_tree *tree, struct page *page) 3827 { 3828 u64 start = page_offset(page); 3829 struct extent_buffer *eb; 3830 int ret = 1; 3831 unsigned long i; 3832 unsigned long num_pages; 3833 3834 spin_lock(&tree->buffer_lock); 3835 eb = buffer_search(tree, start); 3836 if (!eb) 3837 goto out; 3838 3839 if (atomic_read(&eb->refs) > 1) { 3840 ret = 0; 3841 goto out; 3842 } 3843 if (test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) { 3844 ret = 0; 3845 goto out; 3846 } 3847 /* at this point we can safely release the extent buffer */ 3848 num_pages = num_extent_pages(eb->start, eb->len); 3849 for (i = 0; i < num_pages; i++) 3850 page_cache_release(extent_buffer_page(eb, i)); 3851 rb_erase(&eb->rb_node, &tree->buffer); 3852 __free_extent_buffer(eb); 3853 out: 3854 spin_unlock(&tree->buffer_lock); 3855 return ret; 3856 } 3857