1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/slab.h> 4 #include "ctree.h" 5 #include "subpage.h" 6 #include "btrfs_inode.h" 7 8 /* 9 * Subpage (sectorsize < PAGE_SIZE) support overview: 10 * 11 * Limitations: 12 * 13 * - Only support 64K page size for now 14 * This is to make metadata handling easier, as 64K page would ensure 15 * all nodesize would fit inside one page, thus we don't need to handle 16 * cases where a tree block crosses several pages. 17 * 18 * - Only metadata read-write for now 19 * The data read-write part is in development. 20 * 21 * - Metadata can't cross 64K page boundary 22 * btrfs-progs and kernel have done that for a while, thus only ancient 23 * filesystems could have such problem. For such case, do a graceful 24 * rejection. 25 * 26 * Special behavior: 27 * 28 * - Metadata 29 * Metadata read is fully supported. 30 * Meaning when reading one tree block will only trigger the read for the 31 * needed range, other unrelated range in the same page will not be touched. 32 * 33 * Metadata write support is partial. 34 * The writeback is still for the full page, but we will only submit 35 * the dirty extent buffers in the page. 36 * 37 * This means, if we have a metadata page like this: 38 * 39 * Page offset 40 * 0 16K 32K 48K 64K 41 * |/////////| |///////////| 42 * \- Tree block A \- Tree block B 43 * 44 * Even if we just want to writeback tree block A, we will also writeback 45 * tree block B if it's also dirty. 46 * 47 * This may cause extra metadata writeback which results more COW. 48 * 49 * Implementation: 50 * 51 * - Common 52 * Both metadata and data will use a new structure, btrfs_subpage, to 53 * record the status of each sector inside a page. This provides the extra 54 * granularity needed. 55 * 56 * - Metadata 57 * Since we have multiple tree blocks inside one page, we can't rely on page 58 * locking anymore, or we will have greatly reduced concurrency or even 59 * deadlocks (hold one tree lock while trying to lock another tree lock in 60 * the same page). 61 * 62 * Thus for metadata locking, subpage support relies on io_tree locking only. 63 * This means a slightly higher tree locking latency. 64 */ 65 66 void btrfs_init_subpage_info(struct btrfs_subpage_info *subpage_info, u32 sectorsize) 67 { 68 unsigned int cur = 0; 69 unsigned int nr_bits; 70 71 ASSERT(IS_ALIGNED(PAGE_SIZE, sectorsize)); 72 73 nr_bits = PAGE_SIZE / sectorsize; 74 subpage_info->bitmap_nr_bits = nr_bits; 75 76 subpage_info->uptodate_offset = cur; 77 cur += nr_bits; 78 79 subpage_info->error_offset = cur; 80 cur += nr_bits; 81 82 subpage_info->dirty_offset = cur; 83 cur += nr_bits; 84 85 subpage_info->writeback_offset = cur; 86 cur += nr_bits; 87 88 subpage_info->ordered_offset = cur; 89 cur += nr_bits; 90 91 subpage_info->checked_offset = cur; 92 cur += nr_bits; 93 94 subpage_info->total_nr_bits = cur; 95 } 96 97 int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, 98 struct page *page, enum btrfs_subpage_type type) 99 { 100 struct btrfs_subpage *subpage; 101 102 /* 103 * We have cases like a dummy extent buffer page, which is not mappped 104 * and doesn't need to be locked. 105 */ 106 if (page->mapping) 107 ASSERT(PageLocked(page)); 108 109 /* Either not subpage, or the page already has private attached */ 110 if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) 111 return 0; 112 113 subpage = btrfs_alloc_subpage(fs_info, type); 114 if (IS_ERR(subpage)) 115 return PTR_ERR(subpage); 116 117 attach_page_private(page, subpage); 118 return 0; 119 } 120 121 void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, 122 struct page *page) 123 { 124 struct btrfs_subpage *subpage; 125 126 /* Either not subpage, or already detached */ 127 if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page)) 128 return; 129 130 subpage = (struct btrfs_subpage *)detach_page_private(page); 131 ASSERT(subpage); 132 btrfs_free_subpage(subpage); 133 } 134 135 struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, 136 enum btrfs_subpage_type type) 137 { 138 struct btrfs_subpage *ret; 139 unsigned int real_size; 140 141 ASSERT(fs_info->sectorsize < PAGE_SIZE); 142 143 real_size = struct_size(ret, bitmaps, 144 BITS_TO_LONGS(fs_info->subpage_info->total_nr_bits)); 145 ret = kzalloc(real_size, GFP_NOFS); 146 if (!ret) 147 return ERR_PTR(-ENOMEM); 148 149 spin_lock_init(&ret->lock); 150 if (type == BTRFS_SUBPAGE_METADATA) { 151 atomic_set(&ret->eb_refs, 0); 152 } else { 153 atomic_set(&ret->readers, 0); 154 atomic_set(&ret->writers, 0); 155 } 156 return ret; 157 } 158 159 void btrfs_free_subpage(struct btrfs_subpage *subpage) 160 { 161 kfree(subpage); 162 } 163 164 /* 165 * Increase the eb_refs of current subpage. 166 * 167 * This is important for eb allocation, to prevent race with last eb freeing 168 * of the same page. 169 * With the eb_refs increased before the eb inserted into radix tree, 170 * detach_extent_buffer_page() won't detach the page private while we're still 171 * allocating the extent buffer. 172 */ 173 void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, 174 struct page *page) 175 { 176 struct btrfs_subpage *subpage; 177 178 if (fs_info->sectorsize == PAGE_SIZE) 179 return; 180 181 ASSERT(PagePrivate(page) && page->mapping); 182 lockdep_assert_held(&page->mapping->private_lock); 183 184 subpage = (struct btrfs_subpage *)page->private; 185 atomic_inc(&subpage->eb_refs); 186 } 187 188 void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, 189 struct page *page) 190 { 191 struct btrfs_subpage *subpage; 192 193 if (fs_info->sectorsize == PAGE_SIZE) 194 return; 195 196 ASSERT(PagePrivate(page) && page->mapping); 197 lockdep_assert_held(&page->mapping->private_lock); 198 199 subpage = (struct btrfs_subpage *)page->private; 200 ASSERT(atomic_read(&subpage->eb_refs)); 201 atomic_dec(&subpage->eb_refs); 202 } 203 204 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 205 struct page *page, u64 start, u32 len) 206 { 207 /* Basic checks */ 208 ASSERT(PagePrivate(page) && page->private); 209 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 210 IS_ALIGNED(len, fs_info->sectorsize)); 211 /* 212 * The range check only works for mapped page, we can still have 213 * unmapped page like dummy extent buffer pages. 214 */ 215 if (page->mapping) 216 ASSERT(page_offset(page) <= start && 217 start + len <= page_offset(page) + PAGE_SIZE); 218 } 219 220 void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, 221 struct page *page, u64 start, u32 len) 222 { 223 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 224 const int nbits = len >> fs_info->sectorsize_bits; 225 226 btrfs_subpage_assert(fs_info, page, start, len); 227 228 atomic_add(nbits, &subpage->readers); 229 } 230 231 void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, 232 struct page *page, u64 start, u32 len) 233 { 234 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 235 const int nbits = len >> fs_info->sectorsize_bits; 236 bool is_data; 237 bool last; 238 239 btrfs_subpage_assert(fs_info, page, start, len); 240 is_data = is_data_inode(page->mapping->host); 241 ASSERT(atomic_read(&subpage->readers) >= nbits); 242 last = atomic_sub_and_test(nbits, &subpage->readers); 243 244 /* 245 * For data we need to unlock the page if the last read has finished. 246 * 247 * And please don't replace @last with atomic_sub_and_test() call 248 * inside if () condition. 249 * As we want the atomic_sub_and_test() to be always executed. 250 */ 251 if (is_data && last) 252 unlock_page(page); 253 } 254 255 static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len) 256 { 257 u64 orig_start = *start; 258 u32 orig_len = *len; 259 260 *start = max_t(u64, page_offset(page), orig_start); 261 /* 262 * For certain call sites like btrfs_drop_pages(), we may have pages 263 * beyond the target range. In that case, just set @len to 0, subpage 264 * helpers can handle @len == 0 without any problem. 265 */ 266 if (page_offset(page) >= orig_start + orig_len) 267 *len = 0; 268 else 269 *len = min_t(u64, page_offset(page) + PAGE_SIZE, 270 orig_start + orig_len) - *start; 271 } 272 273 void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info, 274 struct page *page, u64 start, u32 len) 275 { 276 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 277 const int nbits = (len >> fs_info->sectorsize_bits); 278 int ret; 279 280 btrfs_subpage_assert(fs_info, page, start, len); 281 282 ASSERT(atomic_read(&subpage->readers) == 0); 283 ret = atomic_add_return(nbits, &subpage->writers); 284 ASSERT(ret == nbits); 285 } 286 287 bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, 288 struct page *page, u64 start, u32 len) 289 { 290 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 291 const int nbits = (len >> fs_info->sectorsize_bits); 292 293 btrfs_subpage_assert(fs_info, page, start, len); 294 295 ASSERT(atomic_read(&subpage->writers) >= nbits); 296 return atomic_sub_and_test(nbits, &subpage->writers); 297 } 298 299 /* 300 * Lock a page for delalloc page writeback. 301 * 302 * Return -EAGAIN if the page is not properly initialized. 303 * Return 0 with the page locked, and writer counter updated. 304 * 305 * Even with 0 returned, the page still need extra check to make sure 306 * it's really the correct page, as the caller is using 307 * find_get_pages_contig(), which can race with page invalidating. 308 */ 309 int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, 310 struct page *page, u64 start, u32 len) 311 { 312 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { 313 lock_page(page); 314 return 0; 315 } 316 lock_page(page); 317 if (!PagePrivate(page) || !page->private) { 318 unlock_page(page); 319 return -EAGAIN; 320 } 321 btrfs_subpage_clamp_range(page, &start, &len); 322 btrfs_subpage_start_writer(fs_info, page, start, len); 323 return 0; 324 } 325 326 void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info, 327 struct page *page, u64 start, u32 len) 328 { 329 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) 330 return unlock_page(page); 331 btrfs_subpage_clamp_range(page, &start, &len); 332 if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len)) 333 unlock_page(page); 334 } 335 336 static bool bitmap_test_range_all_set(unsigned long *addr, unsigned int start, 337 unsigned int nbits) 338 { 339 unsigned int found_zero; 340 341 found_zero = find_next_zero_bit(addr, start + nbits, start); 342 if (found_zero == start + nbits) 343 return true; 344 return false; 345 } 346 347 static bool bitmap_test_range_all_zero(unsigned long *addr, unsigned int start, 348 unsigned int nbits) 349 { 350 unsigned int found_set; 351 352 found_set = find_next_bit(addr, start + nbits, start); 353 if (found_set == start + nbits) 354 return true; 355 return false; 356 } 357 358 #define subpage_calc_start_bit(fs_info, page, name, start, len) \ 359 ({ \ 360 unsigned int start_bit; \ 361 \ 362 btrfs_subpage_assert(fs_info, page, start, len); \ 363 start_bit = offset_in_page(start) >> fs_info->sectorsize_bits; \ 364 start_bit += fs_info->subpage_info->name##_offset; \ 365 start_bit; \ 366 }) 367 368 #define subpage_test_bitmap_all_set(fs_info, subpage, name) \ 369 bitmap_test_range_all_set(subpage->bitmaps, \ 370 fs_info->subpage_info->name##_offset, \ 371 fs_info->subpage_info->bitmap_nr_bits) 372 373 #define subpage_test_bitmap_all_zero(fs_info, subpage, name) \ 374 bitmap_test_range_all_zero(subpage->bitmaps, \ 375 fs_info->subpage_info->name##_offset, \ 376 fs_info->subpage_info->bitmap_nr_bits) 377 378 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 379 struct page *page, u64 start, u32 len) 380 { 381 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 382 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 383 uptodate, start, len); 384 unsigned long flags; 385 386 spin_lock_irqsave(&subpage->lock, flags); 387 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 388 if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate)) 389 SetPageUptodate(page); 390 spin_unlock_irqrestore(&subpage->lock, flags); 391 } 392 393 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 394 struct page *page, u64 start, u32 len) 395 { 396 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 397 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 398 uptodate, start, len); 399 unsigned long flags; 400 401 spin_lock_irqsave(&subpage->lock, flags); 402 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 403 ClearPageUptodate(page); 404 spin_unlock_irqrestore(&subpage->lock, flags); 405 } 406 407 void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, 408 struct page *page, u64 start, u32 len) 409 { 410 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 411 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 412 error, start, len); 413 unsigned long flags; 414 415 spin_lock_irqsave(&subpage->lock, flags); 416 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 417 SetPageError(page); 418 spin_unlock_irqrestore(&subpage->lock, flags); 419 } 420 421 void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, 422 struct page *page, u64 start, u32 len) 423 { 424 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 425 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 426 error, start, len); 427 unsigned long flags; 428 429 spin_lock_irqsave(&subpage->lock, flags); 430 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 431 if (subpage_test_bitmap_all_zero(fs_info, subpage, error)) 432 ClearPageError(page); 433 spin_unlock_irqrestore(&subpage->lock, flags); 434 } 435 436 void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, 437 struct page *page, u64 start, u32 len) 438 { 439 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 440 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 441 dirty, start, len); 442 unsigned long flags; 443 444 spin_lock_irqsave(&subpage->lock, flags); 445 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 446 spin_unlock_irqrestore(&subpage->lock, flags); 447 set_page_dirty(page); 448 } 449 450 /* 451 * Extra clear_and_test function for subpage dirty bitmap. 452 * 453 * Return true if we're the last bits in the dirty_bitmap and clear the 454 * dirty_bitmap. 455 * Return false otherwise. 456 * 457 * NOTE: Callers should manually clear page dirty for true case, as we have 458 * extra handling for tree blocks. 459 */ 460 bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 461 struct page *page, u64 start, u32 len) 462 { 463 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 464 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 465 dirty, start, len); 466 unsigned long flags; 467 bool last = false; 468 469 spin_lock_irqsave(&subpage->lock, flags); 470 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 471 if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty)) 472 last = true; 473 spin_unlock_irqrestore(&subpage->lock, flags); 474 return last; 475 } 476 477 void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, 478 struct page *page, u64 start, u32 len) 479 { 480 bool last; 481 482 last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len); 483 if (last) 484 clear_page_dirty_for_io(page); 485 } 486 487 void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, 488 struct page *page, u64 start, u32 len) 489 { 490 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 491 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 492 writeback, start, len); 493 unsigned long flags; 494 495 spin_lock_irqsave(&subpage->lock, flags); 496 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 497 set_page_writeback(page); 498 spin_unlock_irqrestore(&subpage->lock, flags); 499 } 500 501 void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, 502 struct page *page, u64 start, u32 len) 503 { 504 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 505 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 506 writeback, start, len); 507 unsigned long flags; 508 509 spin_lock_irqsave(&subpage->lock, flags); 510 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 511 if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) { 512 ASSERT(PageWriteback(page)); 513 end_page_writeback(page); 514 } 515 spin_unlock_irqrestore(&subpage->lock, flags); 516 } 517 518 void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, 519 struct page *page, u64 start, u32 len) 520 { 521 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 522 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 523 ordered, start, len); 524 unsigned long flags; 525 526 spin_lock_irqsave(&subpage->lock, flags); 527 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 528 SetPageOrdered(page); 529 spin_unlock_irqrestore(&subpage->lock, flags); 530 } 531 532 void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, 533 struct page *page, u64 start, u32 len) 534 { 535 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 536 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 537 ordered, start, len); 538 unsigned long flags; 539 540 spin_lock_irqsave(&subpage->lock, flags); 541 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 542 if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered)) 543 ClearPageOrdered(page); 544 spin_unlock_irqrestore(&subpage->lock, flags); 545 } 546 547 void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info, 548 struct page *page, u64 start, u32 len) 549 { 550 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 551 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 552 checked, start, len); 553 unsigned long flags; 554 555 spin_lock_irqsave(&subpage->lock, flags); 556 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 557 if (subpage_test_bitmap_all_set(fs_info, subpage, checked)) 558 SetPageChecked(page); 559 spin_unlock_irqrestore(&subpage->lock, flags); 560 } 561 562 void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info, 563 struct page *page, u64 start, u32 len) 564 { 565 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 566 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 567 checked, start, len); 568 unsigned long flags; 569 570 spin_lock_irqsave(&subpage->lock, flags); 571 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 572 ClearPageChecked(page); 573 spin_unlock_irqrestore(&subpage->lock, flags); 574 } 575 576 /* 577 * Unlike set/clear which is dependent on each page status, for test all bits 578 * are tested in the same way. 579 */ 580 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 581 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 582 struct page *page, u64 start, u32 len) \ 583 { \ 584 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ 585 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, \ 586 name, start, len); \ 587 unsigned long flags; \ 588 bool ret; \ 589 \ 590 spin_lock_irqsave(&subpage->lock, flags); \ 591 ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit, \ 592 len >> fs_info->sectorsize_bits); \ 593 spin_unlock_irqrestore(&subpage->lock, flags); \ 594 return ret; \ 595 } 596 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 597 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); 598 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); 599 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); 600 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); 601 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked); 602 603 /* 604 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 605 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 606 * back to regular sectorsize branch. 607 */ 608 #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ 609 test_page_func) \ 610 void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ 611 struct page *page, u64 start, u32 len) \ 612 { \ 613 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 614 set_page_func(page); \ 615 return; \ 616 } \ 617 btrfs_subpage_set_##name(fs_info, page, start, len); \ 618 } \ 619 void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ 620 struct page *page, u64 start, u32 len) \ 621 { \ 622 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 623 clear_page_func(page); \ 624 return; \ 625 } \ 626 btrfs_subpage_clear_##name(fs_info, page, start, len); \ 627 } \ 628 bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ 629 struct page *page, u64 start, u32 len) \ 630 { \ 631 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 632 return test_page_func(page); \ 633 return btrfs_subpage_test_##name(fs_info, page, start, len); \ 634 } \ 635 void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 636 struct page *page, u64 start, u32 len) \ 637 { \ 638 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 639 set_page_func(page); \ 640 return; \ 641 } \ 642 btrfs_subpage_clamp_range(page, &start, &len); \ 643 btrfs_subpage_set_##name(fs_info, page, start, len); \ 644 } \ 645 void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 646 struct page *page, u64 start, u32 len) \ 647 { \ 648 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 649 clear_page_func(page); \ 650 return; \ 651 } \ 652 btrfs_subpage_clamp_range(page, &start, &len); \ 653 btrfs_subpage_clear_##name(fs_info, page, start, len); \ 654 } \ 655 bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 656 struct page *page, u64 start, u32 len) \ 657 { \ 658 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 659 return test_page_func(page); \ 660 btrfs_subpage_clamp_range(page, &start, &len); \ 661 return btrfs_subpage_test_##name(fs_info, page, start, len); \ 662 } 663 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, 664 PageUptodate); 665 IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); 666 IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io, 667 PageDirty); 668 IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback, 669 PageWriteback); 670 IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered, 671 PageOrdered); 672 IMPLEMENT_BTRFS_PAGE_OPS(checked, SetPageChecked, ClearPageChecked, PageChecked); 673 674 /* 675 * Make sure not only the page dirty bit is cleared, but also subpage dirty bit 676 * is cleared. 677 */ 678 void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info, 679 struct page *page) 680 { 681 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 682 683 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 684 return; 685 686 ASSERT(!PageDirty(page)); 687 if (fs_info->sectorsize == PAGE_SIZE) 688 return; 689 690 ASSERT(PagePrivate(page) && page->private); 691 ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty)); 692 } 693