1cac06d84SQu Wenruo // SPDX-License-Identifier: GPL-2.0 2cac06d84SQu Wenruo 3cac06d84SQu Wenruo #include <linux/slab.h> 4cac06d84SQu Wenruo #include "ctree.h" 5cac06d84SQu Wenruo #include "subpage.h" 6cac06d84SQu Wenruo 7894d1378SQu Wenruo /* 8894d1378SQu Wenruo * Subpage (sectorsize < PAGE_SIZE) support overview: 9894d1378SQu Wenruo * 10894d1378SQu Wenruo * Limitations: 11894d1378SQu Wenruo * 12894d1378SQu Wenruo * - Only support 64K page size for now 13894d1378SQu Wenruo * This is to make metadata handling easier, as 64K page would ensure 14894d1378SQu Wenruo * all nodesize would fit inside one page, thus we don't need to handle 15894d1378SQu Wenruo * cases where a tree block crosses several pages. 16894d1378SQu Wenruo * 17894d1378SQu Wenruo * - Only metadata read-write for now 18894d1378SQu Wenruo * The data read-write part is in development. 19894d1378SQu Wenruo * 20894d1378SQu Wenruo * - Metadata can't cross 64K page boundary 21894d1378SQu Wenruo * btrfs-progs and kernel have done that for a while, thus only ancient 22894d1378SQu Wenruo * filesystems could have such problem. For such case, do a graceful 23894d1378SQu Wenruo * rejection. 24894d1378SQu Wenruo * 25894d1378SQu Wenruo * Special behavior: 26894d1378SQu Wenruo * 27894d1378SQu Wenruo * - Metadata 28894d1378SQu Wenruo * Metadata read is fully supported. 29894d1378SQu Wenruo * Meaning when reading one tree block will only trigger the read for the 30894d1378SQu Wenruo * needed range, other unrelated range in the same page will not be touched. 31894d1378SQu Wenruo * 32894d1378SQu Wenruo * Metadata write support is partial. 33894d1378SQu Wenruo * The writeback is still for the full page, but we will only submit 34894d1378SQu Wenruo * the dirty extent buffers in the page. 35894d1378SQu Wenruo * 36894d1378SQu Wenruo * This means, if we have a metadata page like this: 37894d1378SQu Wenruo * 38894d1378SQu Wenruo * Page offset 39894d1378SQu Wenruo * 0 16K 32K 48K 64K 40894d1378SQu Wenruo * |/////////| |///////////| 41894d1378SQu Wenruo * \- Tree block A \- Tree block B 42894d1378SQu Wenruo * 43894d1378SQu Wenruo * Even if we just want to writeback tree block A, we will also writeback 44894d1378SQu Wenruo * tree block B if it's also dirty. 45894d1378SQu Wenruo * 46894d1378SQu Wenruo * This may cause extra metadata writeback which results more COW. 47894d1378SQu Wenruo * 48894d1378SQu Wenruo * Implementation: 49894d1378SQu Wenruo * 50894d1378SQu Wenruo * - Common 51894d1378SQu Wenruo * Both metadata and data will use a new structure, btrfs_subpage, to 52894d1378SQu Wenruo * record the status of each sector inside a page. This provides the extra 53894d1378SQu Wenruo * granularity needed. 54894d1378SQu Wenruo * 55894d1378SQu Wenruo * - Metadata 56894d1378SQu Wenruo * Since we have multiple tree blocks inside one page, we can't rely on page 57894d1378SQu Wenruo * locking anymore, or we will have greatly reduced concurrency or even 58894d1378SQu Wenruo * deadlocks (hold one tree lock while trying to lock another tree lock in 59894d1378SQu Wenruo * the same page). 60894d1378SQu Wenruo * 61894d1378SQu Wenruo * Thus for metadata locking, subpage support relies on io_tree locking only. 62894d1378SQu Wenruo * This means a slightly higher tree locking latency. 63894d1378SQu Wenruo */ 64894d1378SQu Wenruo 65cac06d84SQu Wenruo int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, 66cac06d84SQu Wenruo struct page *page, enum btrfs_subpage_type type) 67cac06d84SQu Wenruo { 68760f991fSQu Wenruo struct btrfs_subpage *subpage = NULL; 69760f991fSQu Wenruo int ret; 70cac06d84SQu Wenruo 71cac06d84SQu Wenruo /* 72cac06d84SQu Wenruo * We have cases like a dummy extent buffer page, which is not mappped 73cac06d84SQu Wenruo * and doesn't need to be locked. 74cac06d84SQu Wenruo */ 75cac06d84SQu Wenruo if (page->mapping) 76cac06d84SQu Wenruo ASSERT(PageLocked(page)); 77cac06d84SQu Wenruo /* Either not subpage, or the page already has private attached */ 78cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) 79cac06d84SQu Wenruo return 0; 80cac06d84SQu Wenruo 81760f991fSQu Wenruo ret = btrfs_alloc_subpage(fs_info, &subpage, type); 82760f991fSQu Wenruo if (ret < 0) 83760f991fSQu Wenruo return ret; 84cac06d84SQu Wenruo attach_page_private(page, subpage); 85cac06d84SQu Wenruo return 0; 86cac06d84SQu Wenruo } 87cac06d84SQu Wenruo 88cac06d84SQu Wenruo void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, 89cac06d84SQu Wenruo struct page *page) 90cac06d84SQu Wenruo { 91cac06d84SQu Wenruo struct btrfs_subpage *subpage; 92cac06d84SQu Wenruo 93cac06d84SQu Wenruo /* Either not subpage, or already detached */ 94cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page)) 95cac06d84SQu Wenruo return; 96cac06d84SQu Wenruo 97cac06d84SQu Wenruo subpage = (struct btrfs_subpage *)detach_page_private(page); 98cac06d84SQu Wenruo ASSERT(subpage); 99760f991fSQu Wenruo btrfs_free_subpage(subpage); 100760f991fSQu Wenruo } 101760f991fSQu Wenruo 102760f991fSQu Wenruo int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, 103760f991fSQu Wenruo struct btrfs_subpage **ret, 104760f991fSQu Wenruo enum btrfs_subpage_type type) 105760f991fSQu Wenruo { 106760f991fSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 107760f991fSQu Wenruo return 0; 108760f991fSQu Wenruo 109760f991fSQu Wenruo *ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS); 110760f991fSQu Wenruo if (!*ret) 111760f991fSQu Wenruo return -ENOMEM; 112760f991fSQu Wenruo spin_lock_init(&(*ret)->lock); 1131e1de387SQu Wenruo if (type == BTRFS_SUBPAGE_METADATA) { 1148ff8466dSQu Wenruo atomic_set(&(*ret)->eb_refs, 0); 1151e1de387SQu Wenruo } else { 11692082d40SQu Wenruo atomic_set(&(*ret)->readers, 0); 1171e1de387SQu Wenruo atomic_set(&(*ret)->writers, 0); 1181e1de387SQu Wenruo } 119760f991fSQu Wenruo return 0; 120760f991fSQu Wenruo } 121760f991fSQu Wenruo 122760f991fSQu Wenruo void btrfs_free_subpage(struct btrfs_subpage *subpage) 123760f991fSQu Wenruo { 124cac06d84SQu Wenruo kfree(subpage); 125cac06d84SQu Wenruo } 1268ff8466dSQu Wenruo 1278ff8466dSQu Wenruo /* 1288ff8466dSQu Wenruo * Increase the eb_refs of current subpage. 1298ff8466dSQu Wenruo * 1308ff8466dSQu Wenruo * This is important for eb allocation, to prevent race with last eb freeing 1318ff8466dSQu Wenruo * of the same page. 1328ff8466dSQu Wenruo * With the eb_refs increased before the eb inserted into radix tree, 1338ff8466dSQu Wenruo * detach_extent_buffer_page() won't detach the page private while we're still 1348ff8466dSQu Wenruo * allocating the extent buffer. 1358ff8466dSQu Wenruo */ 1368ff8466dSQu Wenruo void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, 1378ff8466dSQu Wenruo struct page *page) 1388ff8466dSQu Wenruo { 1398ff8466dSQu Wenruo struct btrfs_subpage *subpage; 1408ff8466dSQu Wenruo 1418ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 1428ff8466dSQu Wenruo return; 1438ff8466dSQu Wenruo 1448ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 1458ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 1468ff8466dSQu Wenruo 1478ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 1488ff8466dSQu Wenruo atomic_inc(&subpage->eb_refs); 1498ff8466dSQu Wenruo } 1508ff8466dSQu Wenruo 1518ff8466dSQu Wenruo void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, 1528ff8466dSQu Wenruo struct page *page) 1538ff8466dSQu Wenruo { 1548ff8466dSQu Wenruo struct btrfs_subpage *subpage; 1558ff8466dSQu Wenruo 1568ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 1578ff8466dSQu Wenruo return; 1588ff8466dSQu Wenruo 1598ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 1608ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 1618ff8466dSQu Wenruo 1628ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 1638ff8466dSQu Wenruo ASSERT(atomic_read(&subpage->eb_refs)); 1648ff8466dSQu Wenruo atomic_dec(&subpage->eb_refs); 1658ff8466dSQu Wenruo } 166a1d767c1SQu Wenruo 16792082d40SQu Wenruo static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 16892082d40SQu Wenruo struct page *page, u64 start, u32 len) 16992082d40SQu Wenruo { 17092082d40SQu Wenruo /* Basic checks */ 17192082d40SQu Wenruo ASSERT(PagePrivate(page) && page->private); 17292082d40SQu Wenruo ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 17392082d40SQu Wenruo IS_ALIGNED(len, fs_info->sectorsize)); 17492082d40SQu Wenruo /* 17592082d40SQu Wenruo * The range check only works for mapped page, we can still have 17692082d40SQu Wenruo * unmapped page like dummy extent buffer pages. 17792082d40SQu Wenruo */ 17892082d40SQu Wenruo if (page->mapping) 17992082d40SQu Wenruo ASSERT(page_offset(page) <= start && 18092082d40SQu Wenruo start + len <= page_offset(page) + PAGE_SIZE); 18192082d40SQu Wenruo } 18292082d40SQu Wenruo 18392082d40SQu Wenruo void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, 18492082d40SQu Wenruo struct page *page, u64 start, u32 len) 18592082d40SQu Wenruo { 18692082d40SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 18792082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 18892082d40SQu Wenruo int ret; 18992082d40SQu Wenruo 19092082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 19192082d40SQu Wenruo 19292082d40SQu Wenruo ret = atomic_add_return(nbits, &subpage->readers); 19392082d40SQu Wenruo ASSERT(ret == nbits); 19492082d40SQu Wenruo } 19592082d40SQu Wenruo 19692082d40SQu Wenruo void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, 19792082d40SQu Wenruo struct page *page, u64 start, u32 len) 19892082d40SQu Wenruo { 19992082d40SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 20092082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 20192082d40SQu Wenruo 20292082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 20392082d40SQu Wenruo ASSERT(atomic_read(&subpage->readers) >= nbits); 20492082d40SQu Wenruo if (atomic_sub_and_test(nbits, &subpage->readers)) 20592082d40SQu Wenruo unlock_page(page); 20692082d40SQu Wenruo } 20792082d40SQu Wenruo 2081e1de387SQu Wenruo static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len) 2091e1de387SQu Wenruo { 2101e1de387SQu Wenruo u64 orig_start = *start; 2111e1de387SQu Wenruo u32 orig_len = *len; 2121e1de387SQu Wenruo 2131e1de387SQu Wenruo *start = max_t(u64, page_offset(page), orig_start); 2141e1de387SQu Wenruo *len = min_t(u64, page_offset(page) + PAGE_SIZE, 2151e1de387SQu Wenruo orig_start + orig_len) - *start; 2161e1de387SQu Wenruo } 2171e1de387SQu Wenruo 2181e1de387SQu Wenruo void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info, 2191e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2201e1de387SQu Wenruo { 2211e1de387SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 2221e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits); 2231e1de387SQu Wenruo int ret; 2241e1de387SQu Wenruo 2251e1de387SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2261e1de387SQu Wenruo 2271e1de387SQu Wenruo ASSERT(atomic_read(&subpage->readers) == 0); 2281e1de387SQu Wenruo ret = atomic_add_return(nbits, &subpage->writers); 2291e1de387SQu Wenruo ASSERT(ret == nbits); 2301e1de387SQu Wenruo } 2311e1de387SQu Wenruo 2321e1de387SQu Wenruo bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, 2331e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2341e1de387SQu Wenruo { 2351e1de387SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 2361e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits); 2371e1de387SQu Wenruo 2381e1de387SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2391e1de387SQu Wenruo 2401e1de387SQu Wenruo ASSERT(atomic_read(&subpage->writers) >= nbits); 2411e1de387SQu Wenruo return atomic_sub_and_test(nbits, &subpage->writers); 2421e1de387SQu Wenruo } 2431e1de387SQu Wenruo 2441e1de387SQu Wenruo /* 2451e1de387SQu Wenruo * Lock a page for delalloc page writeback. 2461e1de387SQu Wenruo * 2471e1de387SQu Wenruo * Return -EAGAIN if the page is not properly initialized. 2481e1de387SQu Wenruo * Return 0 with the page locked, and writer counter updated. 2491e1de387SQu Wenruo * 2501e1de387SQu Wenruo * Even with 0 returned, the page still need extra check to make sure 2511e1de387SQu Wenruo * it's really the correct page, as the caller is using 2521e1de387SQu Wenruo * find_get_pages_contig(), which can race with page invalidating. 2531e1de387SQu Wenruo */ 2541e1de387SQu Wenruo int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, 2551e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2561e1de387SQu Wenruo { 2571e1de387SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { 2581e1de387SQu Wenruo lock_page(page); 2591e1de387SQu Wenruo return 0; 2601e1de387SQu Wenruo } 2611e1de387SQu Wenruo lock_page(page); 2621e1de387SQu Wenruo if (!PagePrivate(page) || !page->private) { 2631e1de387SQu Wenruo unlock_page(page); 2641e1de387SQu Wenruo return -EAGAIN; 2651e1de387SQu Wenruo } 2661e1de387SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); 2671e1de387SQu Wenruo btrfs_subpage_start_writer(fs_info, page, start, len); 2681e1de387SQu Wenruo return 0; 2691e1de387SQu Wenruo } 2701e1de387SQu Wenruo 2711e1de387SQu Wenruo void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info, 2721e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2731e1de387SQu Wenruo { 2741e1de387SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) 2751e1de387SQu Wenruo return unlock_page(page); 2761e1de387SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); 2771e1de387SQu Wenruo if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len)) 2781e1de387SQu Wenruo unlock_page(page); 2791e1de387SQu Wenruo } 2801e1de387SQu Wenruo 281a1d767c1SQu Wenruo /* 282a1d767c1SQu Wenruo * Convert the [start, start + len) range into a u16 bitmap 283a1d767c1SQu Wenruo * 284a1d767c1SQu Wenruo * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0. 285a1d767c1SQu Wenruo */ 286a1d767c1SQu Wenruo static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, 287a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 288a1d767c1SQu Wenruo { 289a1d767c1SQu Wenruo const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits; 290a1d767c1SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 291a1d767c1SQu Wenruo 29292082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 293a1d767c1SQu Wenruo 294a1d767c1SQu Wenruo /* 295a1d767c1SQu Wenruo * Here nbits can be 16, thus can go beyond u16 range. We make the 296a1d767c1SQu Wenruo * first left shift to be calculate in unsigned long (at least u32), 297a1d767c1SQu Wenruo * then truncate the result to u16. 298a1d767c1SQu Wenruo */ 299a1d767c1SQu Wenruo return (u16)(((1UL << nbits) - 1) << bit_start); 300a1d767c1SQu Wenruo } 301a1d767c1SQu Wenruo 302a1d767c1SQu Wenruo void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 303a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 304a1d767c1SQu Wenruo { 305a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 306a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 307a1d767c1SQu Wenruo unsigned long flags; 308a1d767c1SQu Wenruo 309a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 310a1d767c1SQu Wenruo subpage->uptodate_bitmap |= tmp; 311a1d767c1SQu Wenruo if (subpage->uptodate_bitmap == U16_MAX) 312a1d767c1SQu Wenruo SetPageUptodate(page); 313a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 314a1d767c1SQu Wenruo } 315a1d767c1SQu Wenruo 316a1d767c1SQu Wenruo void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 317a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 318a1d767c1SQu Wenruo { 319a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 320a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 321a1d767c1SQu Wenruo unsigned long flags; 322a1d767c1SQu Wenruo 323a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 324a1d767c1SQu Wenruo subpage->uptodate_bitmap &= ~tmp; 325a1d767c1SQu Wenruo ClearPageUptodate(page); 326a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 327a1d767c1SQu Wenruo } 328a1d767c1SQu Wenruo 32903a816b3SQu Wenruo void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, 33003a816b3SQu Wenruo struct page *page, u64 start, u32 len) 33103a816b3SQu Wenruo { 33203a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 33303a816b3SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 33403a816b3SQu Wenruo unsigned long flags; 33503a816b3SQu Wenruo 33603a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 33703a816b3SQu Wenruo subpage->error_bitmap |= tmp; 33803a816b3SQu Wenruo SetPageError(page); 33903a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 34003a816b3SQu Wenruo } 34103a816b3SQu Wenruo 34203a816b3SQu Wenruo void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, 34303a816b3SQu Wenruo struct page *page, u64 start, u32 len) 34403a816b3SQu Wenruo { 34503a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 34603a816b3SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 34703a816b3SQu Wenruo unsigned long flags; 34803a816b3SQu Wenruo 34903a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 35003a816b3SQu Wenruo subpage->error_bitmap &= ~tmp; 35103a816b3SQu Wenruo if (subpage->error_bitmap == 0) 35203a816b3SQu Wenruo ClearPageError(page); 35303a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 35403a816b3SQu Wenruo } 35503a816b3SQu Wenruo 356d8a5713eSQu Wenruo void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, 357d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 358d8a5713eSQu Wenruo { 359d8a5713eSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 360d8a5713eSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 361d8a5713eSQu Wenruo unsigned long flags; 362d8a5713eSQu Wenruo 363d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 364d8a5713eSQu Wenruo subpage->dirty_bitmap |= tmp; 365d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 366d8a5713eSQu Wenruo set_page_dirty(page); 367d8a5713eSQu Wenruo } 368d8a5713eSQu Wenruo 369d8a5713eSQu Wenruo /* 370d8a5713eSQu Wenruo * Extra clear_and_test function for subpage dirty bitmap. 371d8a5713eSQu Wenruo * 372d8a5713eSQu Wenruo * Return true if we're the last bits in the dirty_bitmap and clear the 373d8a5713eSQu Wenruo * dirty_bitmap. 374d8a5713eSQu Wenruo * Return false otherwise. 375d8a5713eSQu Wenruo * 376d8a5713eSQu Wenruo * NOTE: Callers should manually clear page dirty for true case, as we have 377d8a5713eSQu Wenruo * extra handling for tree blocks. 378d8a5713eSQu Wenruo */ 379d8a5713eSQu Wenruo bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 380d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 381d8a5713eSQu Wenruo { 382d8a5713eSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 383d8a5713eSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 384d8a5713eSQu Wenruo unsigned long flags; 385d8a5713eSQu Wenruo bool last = false; 386d8a5713eSQu Wenruo 387d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 388d8a5713eSQu Wenruo subpage->dirty_bitmap &= ~tmp; 389d8a5713eSQu Wenruo if (subpage->dirty_bitmap == 0) 390d8a5713eSQu Wenruo last = true; 391d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 392d8a5713eSQu Wenruo return last; 393d8a5713eSQu Wenruo } 394d8a5713eSQu Wenruo 395d8a5713eSQu Wenruo void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, 396d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 397d8a5713eSQu Wenruo { 398d8a5713eSQu Wenruo bool last; 399d8a5713eSQu Wenruo 400d8a5713eSQu Wenruo last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len); 401d8a5713eSQu Wenruo if (last) 402d8a5713eSQu Wenruo clear_page_dirty_for_io(page); 403d8a5713eSQu Wenruo } 404d8a5713eSQu Wenruo 4053470da3bSQu Wenruo void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, 4063470da3bSQu Wenruo struct page *page, u64 start, u32 len) 4073470da3bSQu Wenruo { 4083470da3bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 4093470da3bSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 4103470da3bSQu Wenruo unsigned long flags; 4113470da3bSQu Wenruo 4123470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 4133470da3bSQu Wenruo subpage->writeback_bitmap |= tmp; 4143470da3bSQu Wenruo set_page_writeback(page); 4153470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 4163470da3bSQu Wenruo } 4173470da3bSQu Wenruo 4183470da3bSQu Wenruo void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, 4193470da3bSQu Wenruo struct page *page, u64 start, u32 len) 4203470da3bSQu Wenruo { 4213470da3bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 4223470da3bSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 4233470da3bSQu Wenruo unsigned long flags; 4243470da3bSQu Wenruo 4253470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 4263470da3bSQu Wenruo subpage->writeback_bitmap &= ~tmp; 4273470da3bSQu Wenruo if (subpage->writeback_bitmap == 0) 4283470da3bSQu Wenruo end_page_writeback(page); 4293470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 4303470da3bSQu Wenruo } 4313470da3bSQu Wenruo 432*6f17400bSQu Wenruo void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, 433*6f17400bSQu Wenruo struct page *page, u64 start, u32 len) 434*6f17400bSQu Wenruo { 435*6f17400bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 436*6f17400bSQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 437*6f17400bSQu Wenruo unsigned long flags; 438*6f17400bSQu Wenruo 439*6f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 440*6f17400bSQu Wenruo subpage->ordered_bitmap |= tmp; 441*6f17400bSQu Wenruo SetPageOrdered(page); 442*6f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 443*6f17400bSQu Wenruo } 444*6f17400bSQu Wenruo 445*6f17400bSQu Wenruo void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, 446*6f17400bSQu Wenruo struct page *page, u64 start, u32 len) 447*6f17400bSQu Wenruo { 448*6f17400bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 449*6f17400bSQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 450*6f17400bSQu Wenruo unsigned long flags; 451*6f17400bSQu Wenruo 452*6f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 453*6f17400bSQu Wenruo subpage->ordered_bitmap &= ~tmp; 454*6f17400bSQu Wenruo if (subpage->ordered_bitmap == 0) 455*6f17400bSQu Wenruo ClearPageOrdered(page); 456*6f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 457*6f17400bSQu Wenruo } 458a1d767c1SQu Wenruo /* 459a1d767c1SQu Wenruo * Unlike set/clear which is dependent on each page status, for test all bits 460a1d767c1SQu Wenruo * are tested in the same way. 461a1d767c1SQu Wenruo */ 462a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 463a1d767c1SQu Wenruo bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 464a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 465a1d767c1SQu Wenruo { \ 466a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ 467a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \ 468a1d767c1SQu Wenruo unsigned long flags; \ 469a1d767c1SQu Wenruo bool ret; \ 470a1d767c1SQu Wenruo \ 471a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); \ 472a1d767c1SQu Wenruo ret = ((subpage->name##_bitmap & tmp) == tmp); \ 473a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); \ 474a1d767c1SQu Wenruo return ret; \ 475a1d767c1SQu Wenruo } 476a1d767c1SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 47703a816b3SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); 478d8a5713eSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); 4793470da3bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); 480*6f17400bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); 481a1d767c1SQu Wenruo 482a1d767c1SQu Wenruo /* 483a1d767c1SQu Wenruo * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 484a1d767c1SQu Wenruo * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 485a1d767c1SQu Wenruo * back to regular sectorsize branch. 486a1d767c1SQu Wenruo */ 487a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ 488a1d767c1SQu Wenruo test_page_func) \ 489a1d767c1SQu Wenruo void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ 490a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 491a1d767c1SQu Wenruo { \ 492a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 493a1d767c1SQu Wenruo set_page_func(page); \ 494a1d767c1SQu Wenruo return; \ 495a1d767c1SQu Wenruo } \ 496a1d767c1SQu Wenruo btrfs_subpage_set_##name(fs_info, page, start, len); \ 497a1d767c1SQu Wenruo } \ 498a1d767c1SQu Wenruo void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ 499a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 500a1d767c1SQu Wenruo { \ 501a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 502a1d767c1SQu Wenruo clear_page_func(page); \ 503a1d767c1SQu Wenruo return; \ 504a1d767c1SQu Wenruo } \ 505a1d767c1SQu Wenruo btrfs_subpage_clear_##name(fs_info, page, start, len); \ 506a1d767c1SQu Wenruo } \ 507a1d767c1SQu Wenruo bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ 508a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 509a1d767c1SQu Wenruo { \ 510a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 511a1d767c1SQu Wenruo return test_page_func(page); \ 512a1d767c1SQu Wenruo return btrfs_subpage_test_##name(fs_info, page, start, len); \ 51360e2d255SQu Wenruo } \ 51460e2d255SQu Wenruo void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 51560e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 51660e2d255SQu Wenruo { \ 51760e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 51860e2d255SQu Wenruo set_page_func(page); \ 51960e2d255SQu Wenruo return; \ 52060e2d255SQu Wenruo } \ 52160e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 52260e2d255SQu Wenruo btrfs_subpage_set_##name(fs_info, page, start, len); \ 52360e2d255SQu Wenruo } \ 52460e2d255SQu Wenruo void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 52560e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 52660e2d255SQu Wenruo { \ 52760e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 52860e2d255SQu Wenruo clear_page_func(page); \ 52960e2d255SQu Wenruo return; \ 53060e2d255SQu Wenruo } \ 53160e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 53260e2d255SQu Wenruo btrfs_subpage_clear_##name(fs_info, page, start, len); \ 53360e2d255SQu Wenruo } \ 53460e2d255SQu Wenruo bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 53560e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 53660e2d255SQu Wenruo { \ 53760e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 53860e2d255SQu Wenruo return test_page_func(page); \ 53960e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 54060e2d255SQu Wenruo return btrfs_subpage_test_##name(fs_info, page, start, len); \ 541a1d767c1SQu Wenruo } 542a1d767c1SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, 543a1d767c1SQu Wenruo PageUptodate); 54403a816b3SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); 545d8a5713eSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io, 546d8a5713eSQu Wenruo PageDirty); 5473470da3bSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback, 5483470da3bSQu Wenruo PageWriteback); 549*6f17400bSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered, 550*6f17400bSQu Wenruo PageOrdered); 551