1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/slab.h> 4 #include "messages.h" 5 #include "ctree.h" 6 #include "subpage.h" 7 #include "btrfs_inode.h" 8 9 /* 10 * Subpage (sectorsize < PAGE_SIZE) support overview: 11 * 12 * Limitations: 13 * 14 * - Only support 64K page size for now 15 * This is to make metadata handling easier, as 64K page would ensure 16 * all nodesize would fit inside one page, thus we don't need to handle 17 * cases where a tree block crosses several pages. 18 * 19 * - Only metadata read-write for now 20 * The data read-write part is in development. 21 * 22 * - Metadata can't cross 64K page boundary 23 * btrfs-progs and kernel have done that for a while, thus only ancient 24 * filesystems could have such problem. For such case, do a graceful 25 * rejection. 26 * 27 * Special behavior: 28 * 29 * - Metadata 30 * Metadata read is fully supported. 31 * Meaning when reading one tree block will only trigger the read for the 32 * needed range, other unrelated range in the same page will not be touched. 33 * 34 * Metadata write support is partial. 35 * The writeback is still for the full page, but we will only submit 36 * the dirty extent buffers in the page. 37 * 38 * This means, if we have a metadata page like this: 39 * 40 * Page offset 41 * 0 16K 32K 48K 64K 42 * |/////////| |///////////| 43 * \- Tree block A \- Tree block B 44 * 45 * Even if we just want to writeback tree block A, we will also writeback 46 * tree block B if it's also dirty. 47 * 48 * This may cause extra metadata writeback which results more COW. 49 * 50 * Implementation: 51 * 52 * - Common 53 * Both metadata and data will use a new structure, btrfs_subpage, to 54 * record the status of each sector inside a page. This provides the extra 55 * granularity needed. 56 * 57 * - Metadata 58 * Since we have multiple tree blocks inside one page, we can't rely on page 59 * locking anymore, or we will have greatly reduced concurrency or even 60 * deadlocks (hold one tree lock while trying to lock another tree lock in 61 * the same page). 62 * 63 * Thus for metadata locking, subpage support relies on io_tree locking only. 64 * This means a slightly higher tree locking latency. 65 */ 66 67 bool btrfs_is_subpage(const struct btrfs_fs_info *fs_info, struct page *page) 68 { 69 if (fs_info->sectorsize >= PAGE_SIZE) 70 return false; 71 72 /* 73 * Only data pages (either through DIO or compression) can have no 74 * mapping. And if page->mapping->host is data inode, it's subpage. 75 * As we have ruled our sectorsize >= PAGE_SIZE case already. 76 */ 77 if (!page->mapping || !page->mapping->host || 78 is_data_inode(page->mapping->host)) 79 return true; 80 81 /* 82 * Now the only remaining case is metadata, which we only go subpage 83 * routine if nodesize < PAGE_SIZE. 84 */ 85 if (fs_info->nodesize < PAGE_SIZE) 86 return true; 87 return false; 88 } 89 90 void btrfs_init_subpage_info(struct btrfs_subpage_info *subpage_info, u32 sectorsize) 91 { 92 unsigned int cur = 0; 93 unsigned int nr_bits; 94 95 ASSERT(IS_ALIGNED(PAGE_SIZE, sectorsize)); 96 97 nr_bits = PAGE_SIZE / sectorsize; 98 subpage_info->bitmap_nr_bits = nr_bits; 99 100 subpage_info->uptodate_offset = cur; 101 cur += nr_bits; 102 103 subpage_info->error_offset = cur; 104 cur += nr_bits; 105 106 subpage_info->dirty_offset = cur; 107 cur += nr_bits; 108 109 subpage_info->writeback_offset = cur; 110 cur += nr_bits; 111 112 subpage_info->ordered_offset = cur; 113 cur += nr_bits; 114 115 subpage_info->checked_offset = cur; 116 cur += nr_bits; 117 118 subpage_info->total_nr_bits = cur; 119 } 120 121 int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, 122 struct page *page, enum btrfs_subpage_type type) 123 { 124 struct btrfs_subpage *subpage; 125 126 /* 127 * We have cases like a dummy extent buffer page, which is not mapped 128 * and doesn't need to be locked. 129 */ 130 if (page->mapping) 131 ASSERT(PageLocked(page)); 132 133 /* Either not subpage, or the page already has private attached */ 134 if (!btrfs_is_subpage(fs_info, page) || PagePrivate(page)) 135 return 0; 136 137 subpage = btrfs_alloc_subpage(fs_info, type); 138 if (IS_ERR(subpage)) 139 return PTR_ERR(subpage); 140 141 attach_page_private(page, subpage); 142 return 0; 143 } 144 145 void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, 146 struct page *page) 147 { 148 struct btrfs_subpage *subpage; 149 150 /* Either not subpage, or already detached */ 151 if (!btrfs_is_subpage(fs_info, page) || !PagePrivate(page)) 152 return; 153 154 subpage = detach_page_private(page); 155 ASSERT(subpage); 156 btrfs_free_subpage(subpage); 157 } 158 159 struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, 160 enum btrfs_subpage_type type) 161 { 162 struct btrfs_subpage *ret; 163 unsigned int real_size; 164 165 ASSERT(fs_info->sectorsize < PAGE_SIZE); 166 167 real_size = struct_size(ret, bitmaps, 168 BITS_TO_LONGS(fs_info->subpage_info->total_nr_bits)); 169 ret = kzalloc(real_size, GFP_NOFS); 170 if (!ret) 171 return ERR_PTR(-ENOMEM); 172 173 spin_lock_init(&ret->lock); 174 if (type == BTRFS_SUBPAGE_METADATA) { 175 atomic_set(&ret->eb_refs, 0); 176 } else { 177 atomic_set(&ret->readers, 0); 178 atomic_set(&ret->writers, 0); 179 } 180 return ret; 181 } 182 183 void btrfs_free_subpage(struct btrfs_subpage *subpage) 184 { 185 kfree(subpage); 186 } 187 188 /* 189 * Increase the eb_refs of current subpage. 190 * 191 * This is important for eb allocation, to prevent race with last eb freeing 192 * of the same page. 193 * With the eb_refs increased before the eb inserted into radix tree, 194 * detach_extent_buffer_page() won't detach the page private while we're still 195 * allocating the extent buffer. 196 */ 197 void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, 198 struct page *page) 199 { 200 struct btrfs_subpage *subpage; 201 202 if (!btrfs_is_subpage(fs_info, page)) 203 return; 204 205 ASSERT(PagePrivate(page) && page->mapping); 206 lockdep_assert_held(&page->mapping->private_lock); 207 208 subpage = (struct btrfs_subpage *)page->private; 209 atomic_inc(&subpage->eb_refs); 210 } 211 212 void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, 213 struct page *page) 214 { 215 struct btrfs_subpage *subpage; 216 217 if (!btrfs_is_subpage(fs_info, page)) 218 return; 219 220 ASSERT(PagePrivate(page) && page->mapping); 221 lockdep_assert_held(&page->mapping->private_lock); 222 223 subpage = (struct btrfs_subpage *)page->private; 224 ASSERT(atomic_read(&subpage->eb_refs)); 225 atomic_dec(&subpage->eb_refs); 226 } 227 228 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 229 struct page *page, u64 start, u32 len) 230 { 231 /* Basic checks */ 232 ASSERT(PagePrivate(page) && page->private); 233 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 234 IS_ALIGNED(len, fs_info->sectorsize)); 235 /* 236 * The range check only works for mapped page, we can still have 237 * unmapped page like dummy extent buffer pages. 238 */ 239 if (page->mapping) 240 ASSERT(page_offset(page) <= start && 241 start + len <= page_offset(page) + PAGE_SIZE); 242 } 243 244 void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, 245 struct page *page, u64 start, u32 len) 246 { 247 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 248 const int nbits = len >> fs_info->sectorsize_bits; 249 250 btrfs_subpage_assert(fs_info, page, start, len); 251 252 atomic_add(nbits, &subpage->readers); 253 } 254 255 void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, 256 struct page *page, u64 start, u32 len) 257 { 258 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 259 const int nbits = len >> fs_info->sectorsize_bits; 260 bool is_data; 261 bool last; 262 263 btrfs_subpage_assert(fs_info, page, start, len); 264 is_data = is_data_inode(page->mapping->host); 265 ASSERT(atomic_read(&subpage->readers) >= nbits); 266 last = atomic_sub_and_test(nbits, &subpage->readers); 267 268 /* 269 * For data we need to unlock the page if the last read has finished. 270 * 271 * And please don't replace @last with atomic_sub_and_test() call 272 * inside if () condition. 273 * As we want the atomic_sub_and_test() to be always executed. 274 */ 275 if (is_data && last) 276 unlock_page(page); 277 } 278 279 static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len) 280 { 281 u64 orig_start = *start; 282 u32 orig_len = *len; 283 284 *start = max_t(u64, page_offset(page), orig_start); 285 /* 286 * For certain call sites like btrfs_drop_pages(), we may have pages 287 * beyond the target range. In that case, just set @len to 0, subpage 288 * helpers can handle @len == 0 without any problem. 289 */ 290 if (page_offset(page) >= orig_start + orig_len) 291 *len = 0; 292 else 293 *len = min_t(u64, page_offset(page) + PAGE_SIZE, 294 orig_start + orig_len) - *start; 295 } 296 297 void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info, 298 struct page *page, u64 start, u32 len) 299 { 300 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 301 const int nbits = (len >> fs_info->sectorsize_bits); 302 int ret; 303 304 btrfs_subpage_assert(fs_info, page, start, len); 305 306 ASSERT(atomic_read(&subpage->readers) == 0); 307 ret = atomic_add_return(nbits, &subpage->writers); 308 ASSERT(ret == nbits); 309 } 310 311 bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, 312 struct page *page, u64 start, u32 len) 313 { 314 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 315 const int nbits = (len >> fs_info->sectorsize_bits); 316 317 btrfs_subpage_assert(fs_info, page, start, len); 318 319 /* 320 * We have call sites passing @lock_page into 321 * extent_clear_unlock_delalloc() for compression path. 322 * 323 * This @locked_page is locked by plain lock_page(), thus its 324 * subpage::writers is 0. Handle them in a special way. 325 */ 326 if (atomic_read(&subpage->writers) == 0) 327 return true; 328 329 ASSERT(atomic_read(&subpage->writers) >= nbits); 330 return atomic_sub_and_test(nbits, &subpage->writers); 331 } 332 333 /* 334 * Lock a page for delalloc page writeback. 335 * 336 * Return -EAGAIN if the page is not properly initialized. 337 * Return 0 with the page locked, and writer counter updated. 338 * 339 * Even with 0 returned, the page still need extra check to make sure 340 * it's really the correct page, as the caller is using 341 * filemap_get_folios_contig(), which can race with page invalidating. 342 */ 343 int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, 344 struct page *page, u64 start, u32 len) 345 { 346 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) { 347 lock_page(page); 348 return 0; 349 } 350 lock_page(page); 351 if (!PagePrivate(page) || !page->private) { 352 unlock_page(page); 353 return -EAGAIN; 354 } 355 btrfs_subpage_clamp_range(page, &start, &len); 356 btrfs_subpage_start_writer(fs_info, page, start, len); 357 return 0; 358 } 359 360 void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info, 361 struct page *page, u64 start, u32 len) 362 { 363 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) 364 return unlock_page(page); 365 btrfs_subpage_clamp_range(page, &start, &len); 366 if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len)) 367 unlock_page(page); 368 } 369 370 #define subpage_calc_start_bit(fs_info, page, name, start, len) \ 371 ({ \ 372 unsigned int start_bit; \ 373 \ 374 btrfs_subpage_assert(fs_info, page, start, len); \ 375 start_bit = offset_in_page(start) >> fs_info->sectorsize_bits; \ 376 start_bit += fs_info->subpage_info->name##_offset; \ 377 start_bit; \ 378 }) 379 380 #define subpage_test_bitmap_all_set(fs_info, subpage, name) \ 381 bitmap_test_range_all_set(subpage->bitmaps, \ 382 fs_info->subpage_info->name##_offset, \ 383 fs_info->subpage_info->bitmap_nr_bits) 384 385 #define subpage_test_bitmap_all_zero(fs_info, subpage, name) \ 386 bitmap_test_range_all_zero(subpage->bitmaps, \ 387 fs_info->subpage_info->name##_offset, \ 388 fs_info->subpage_info->bitmap_nr_bits) 389 390 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 391 struct page *page, u64 start, u32 len) 392 { 393 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 394 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 395 uptodate, start, len); 396 unsigned long flags; 397 398 spin_lock_irqsave(&subpage->lock, flags); 399 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 400 if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate)) 401 SetPageUptodate(page); 402 spin_unlock_irqrestore(&subpage->lock, flags); 403 } 404 405 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 406 struct page *page, u64 start, u32 len) 407 { 408 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 409 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 410 uptodate, start, len); 411 unsigned long flags; 412 413 spin_lock_irqsave(&subpage->lock, flags); 414 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 415 ClearPageUptodate(page); 416 spin_unlock_irqrestore(&subpage->lock, flags); 417 } 418 419 void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, 420 struct page *page, u64 start, u32 len) 421 { 422 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 423 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 424 error, start, len); 425 unsigned long flags; 426 427 spin_lock_irqsave(&subpage->lock, flags); 428 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 429 SetPageError(page); 430 spin_unlock_irqrestore(&subpage->lock, flags); 431 } 432 433 void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, 434 struct page *page, u64 start, u32 len) 435 { 436 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 437 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 438 error, start, len); 439 unsigned long flags; 440 441 spin_lock_irqsave(&subpage->lock, flags); 442 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 443 if (subpage_test_bitmap_all_zero(fs_info, subpage, error)) 444 ClearPageError(page); 445 spin_unlock_irqrestore(&subpage->lock, flags); 446 } 447 448 void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, 449 struct page *page, u64 start, u32 len) 450 { 451 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 452 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 453 dirty, start, len); 454 unsigned long flags; 455 456 spin_lock_irqsave(&subpage->lock, flags); 457 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 458 spin_unlock_irqrestore(&subpage->lock, flags); 459 set_page_dirty(page); 460 } 461 462 /* 463 * Extra clear_and_test function for subpage dirty bitmap. 464 * 465 * Return true if we're the last bits in the dirty_bitmap and clear the 466 * dirty_bitmap. 467 * Return false otherwise. 468 * 469 * NOTE: Callers should manually clear page dirty for true case, as we have 470 * extra handling for tree blocks. 471 */ 472 bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 473 struct page *page, u64 start, u32 len) 474 { 475 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 476 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 477 dirty, start, len); 478 unsigned long flags; 479 bool last = false; 480 481 spin_lock_irqsave(&subpage->lock, flags); 482 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 483 if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty)) 484 last = true; 485 spin_unlock_irqrestore(&subpage->lock, flags); 486 return last; 487 } 488 489 void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, 490 struct page *page, u64 start, u32 len) 491 { 492 bool last; 493 494 last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len); 495 if (last) 496 clear_page_dirty_for_io(page); 497 } 498 499 void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, 500 struct page *page, u64 start, u32 len) 501 { 502 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 503 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 504 writeback, start, len); 505 unsigned long flags; 506 507 spin_lock_irqsave(&subpage->lock, flags); 508 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 509 set_page_writeback(page); 510 spin_unlock_irqrestore(&subpage->lock, flags); 511 } 512 513 void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, 514 struct page *page, u64 start, u32 len) 515 { 516 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 517 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 518 writeback, start, len); 519 unsigned long flags; 520 521 spin_lock_irqsave(&subpage->lock, flags); 522 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 523 if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) { 524 ASSERT(PageWriteback(page)); 525 end_page_writeback(page); 526 } 527 spin_unlock_irqrestore(&subpage->lock, flags); 528 } 529 530 void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, 531 struct page *page, u64 start, u32 len) 532 { 533 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 534 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 535 ordered, start, len); 536 unsigned long flags; 537 538 spin_lock_irqsave(&subpage->lock, flags); 539 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 540 SetPageOrdered(page); 541 spin_unlock_irqrestore(&subpage->lock, flags); 542 } 543 544 void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, 545 struct page *page, u64 start, u32 len) 546 { 547 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 548 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 549 ordered, start, len); 550 unsigned long flags; 551 552 spin_lock_irqsave(&subpage->lock, flags); 553 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 554 if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered)) 555 ClearPageOrdered(page); 556 spin_unlock_irqrestore(&subpage->lock, flags); 557 } 558 559 void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info, 560 struct page *page, u64 start, u32 len) 561 { 562 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 563 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 564 checked, start, len); 565 unsigned long flags; 566 567 spin_lock_irqsave(&subpage->lock, flags); 568 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 569 if (subpage_test_bitmap_all_set(fs_info, subpage, checked)) 570 SetPageChecked(page); 571 spin_unlock_irqrestore(&subpage->lock, flags); 572 } 573 574 void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info, 575 struct page *page, u64 start, u32 len) 576 { 577 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 578 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 579 checked, start, len); 580 unsigned long flags; 581 582 spin_lock_irqsave(&subpage->lock, flags); 583 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 584 ClearPageChecked(page); 585 spin_unlock_irqrestore(&subpage->lock, flags); 586 } 587 588 /* 589 * Unlike set/clear which is dependent on each page status, for test all bits 590 * are tested in the same way. 591 */ 592 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 593 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 594 struct page *page, u64 start, u32 len) \ 595 { \ 596 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ 597 unsigned int start_bit = subpage_calc_start_bit(fs_info, page, \ 598 name, start, len); \ 599 unsigned long flags; \ 600 bool ret; \ 601 \ 602 spin_lock_irqsave(&subpage->lock, flags); \ 603 ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit, \ 604 len >> fs_info->sectorsize_bits); \ 605 spin_unlock_irqrestore(&subpage->lock, flags); \ 606 return ret; \ 607 } 608 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 609 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); 610 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); 611 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); 612 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); 613 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked); 614 615 /* 616 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 617 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 618 * back to regular sectorsize branch. 619 */ 620 #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ 621 test_page_func) \ 622 void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ 623 struct page *page, u64 start, u32 len) \ 624 { \ 625 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) { \ 626 set_page_func(page); \ 627 return; \ 628 } \ 629 btrfs_subpage_set_##name(fs_info, page, start, len); \ 630 } \ 631 void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ 632 struct page *page, u64 start, u32 len) \ 633 { \ 634 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) { \ 635 clear_page_func(page); \ 636 return; \ 637 } \ 638 btrfs_subpage_clear_##name(fs_info, page, start, len); \ 639 } \ 640 bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ 641 struct page *page, u64 start, u32 len) \ 642 { \ 643 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) \ 644 return test_page_func(page); \ 645 return btrfs_subpage_test_##name(fs_info, page, start, len); \ 646 } \ 647 void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 648 struct page *page, u64 start, u32 len) \ 649 { \ 650 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) { \ 651 set_page_func(page); \ 652 return; \ 653 } \ 654 btrfs_subpage_clamp_range(page, &start, &len); \ 655 btrfs_subpage_set_##name(fs_info, page, start, len); \ 656 } \ 657 void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 658 struct page *page, u64 start, u32 len) \ 659 { \ 660 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) { \ 661 clear_page_func(page); \ 662 return; \ 663 } \ 664 btrfs_subpage_clamp_range(page, &start, &len); \ 665 btrfs_subpage_clear_##name(fs_info, page, start, len); \ 666 } \ 667 bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 668 struct page *page, u64 start, u32 len) \ 669 { \ 670 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) \ 671 return test_page_func(page); \ 672 btrfs_subpage_clamp_range(page, &start, &len); \ 673 return btrfs_subpage_test_##name(fs_info, page, start, len); \ 674 } 675 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, 676 PageUptodate); 677 IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); 678 IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io, 679 PageDirty); 680 IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback, 681 PageWriteback); 682 IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered, 683 PageOrdered); 684 IMPLEMENT_BTRFS_PAGE_OPS(checked, SetPageChecked, ClearPageChecked, PageChecked); 685 686 /* 687 * Make sure not only the page dirty bit is cleared, but also subpage dirty bit 688 * is cleared. 689 */ 690 void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info, 691 struct page *page) 692 { 693 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 694 695 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 696 return; 697 698 ASSERT(!PageDirty(page)); 699 if (!btrfs_is_subpage(fs_info, page)) 700 return; 701 702 ASSERT(PagePrivate(page) && page->private); 703 ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty)); 704 } 705 706 /* 707 * Handle different locked pages with different page sizes: 708 * 709 * - Page locked by plain lock_page() 710 * It should not have any subpage::writers count. 711 * Can be unlocked by unlock_page(). 712 * This is the most common locked page for __extent_writepage() called 713 * inside extent_write_cache_pages(). 714 * Rarer cases include the @locked_page from extent_write_locked_range(). 715 * 716 * - Page locked by lock_delalloc_pages() 717 * There is only one caller, all pages except @locked_page for 718 * extent_write_locked_range(). 719 * In this case, we have to call subpage helper to handle the case. 720 */ 721 void btrfs_page_unlock_writer(struct btrfs_fs_info *fs_info, struct page *page, 722 u64 start, u32 len) 723 { 724 struct btrfs_subpage *subpage; 725 726 ASSERT(PageLocked(page)); 727 /* For non-subpage case, we just unlock the page */ 728 if (!btrfs_is_subpage(fs_info, page)) 729 return unlock_page(page); 730 731 ASSERT(PagePrivate(page) && page->private); 732 subpage = (struct btrfs_subpage *)page->private; 733 734 /* 735 * For subpage case, there are two types of locked page. With or 736 * without writers number. 737 * 738 * Since we own the page lock, no one else could touch subpage::writers 739 * and we are safe to do several atomic operations without spinlock. 740 */ 741 if (atomic_read(&subpage->writers) == 0) 742 /* No writers, locked by plain lock_page() */ 743 return unlock_page(page); 744 745 /* Have writers, use proper subpage helper to end it */ 746 btrfs_page_end_writer_lock(fs_info, page, start, len); 747 } 748