1cac06d84SQu Wenruo // SPDX-License-Identifier: GPL-2.0 2cac06d84SQu Wenruo 3cac06d84SQu Wenruo #include <linux/slab.h> 4cac06d84SQu Wenruo #include "ctree.h" 5cac06d84SQu Wenruo #include "subpage.h" 63d078efaSQu Wenruo #include "btrfs_inode.h" 7cac06d84SQu Wenruo 8894d1378SQu Wenruo /* 9894d1378SQu Wenruo * Subpage (sectorsize < PAGE_SIZE) support overview: 10894d1378SQu Wenruo * 11894d1378SQu Wenruo * Limitations: 12894d1378SQu Wenruo * 13894d1378SQu Wenruo * - Only support 64K page size for now 14894d1378SQu Wenruo * This is to make metadata handling easier, as 64K page would ensure 15894d1378SQu Wenruo * all nodesize would fit inside one page, thus we don't need to handle 16894d1378SQu Wenruo * cases where a tree block crosses several pages. 17894d1378SQu Wenruo * 18894d1378SQu Wenruo * - Only metadata read-write for now 19894d1378SQu Wenruo * The data read-write part is in development. 20894d1378SQu Wenruo * 21894d1378SQu Wenruo * - Metadata can't cross 64K page boundary 22894d1378SQu Wenruo * btrfs-progs and kernel have done that for a while, thus only ancient 23894d1378SQu Wenruo * filesystems could have such problem. For such case, do a graceful 24894d1378SQu Wenruo * rejection. 25894d1378SQu Wenruo * 26894d1378SQu Wenruo * Special behavior: 27894d1378SQu Wenruo * 28894d1378SQu Wenruo * - Metadata 29894d1378SQu Wenruo * Metadata read is fully supported. 30894d1378SQu Wenruo * Meaning when reading one tree block will only trigger the read for the 31894d1378SQu Wenruo * needed range, other unrelated range in the same page will not be touched. 32894d1378SQu Wenruo * 33894d1378SQu Wenruo * Metadata write support is partial. 34894d1378SQu Wenruo * The writeback is still for the full page, but we will only submit 35894d1378SQu Wenruo * the dirty extent buffers in the page. 36894d1378SQu Wenruo * 37894d1378SQu Wenruo * This means, if we have a metadata page like this: 38894d1378SQu Wenruo * 39894d1378SQu Wenruo * Page offset 40894d1378SQu Wenruo * 0 16K 32K 48K 64K 41894d1378SQu Wenruo * |/////////| |///////////| 42894d1378SQu Wenruo * \- Tree block A \- Tree block B 43894d1378SQu Wenruo * 44894d1378SQu Wenruo * Even if we just want to writeback tree block A, we will also writeback 45894d1378SQu Wenruo * tree block B if it's also dirty. 46894d1378SQu Wenruo * 47894d1378SQu Wenruo * This may cause extra metadata writeback which results more COW. 48894d1378SQu Wenruo * 49894d1378SQu Wenruo * Implementation: 50894d1378SQu Wenruo * 51894d1378SQu Wenruo * - Common 52894d1378SQu Wenruo * Both metadata and data will use a new structure, btrfs_subpage, to 53894d1378SQu Wenruo * record the status of each sector inside a page. This provides the extra 54894d1378SQu Wenruo * granularity needed. 55894d1378SQu Wenruo * 56894d1378SQu Wenruo * - Metadata 57894d1378SQu Wenruo * Since we have multiple tree blocks inside one page, we can't rely on page 58894d1378SQu Wenruo * locking anymore, or we will have greatly reduced concurrency or even 59894d1378SQu Wenruo * deadlocks (hold one tree lock while trying to lock another tree lock in 60894d1378SQu Wenruo * the same page). 61894d1378SQu Wenruo * 62894d1378SQu Wenruo * Thus for metadata locking, subpage support relies on io_tree locking only. 63894d1378SQu Wenruo * This means a slightly higher tree locking latency. 64894d1378SQu Wenruo */ 65894d1378SQu Wenruo 66*8481dd80SQu Wenruo void btrfs_init_subpage_info(struct btrfs_subpage_info *subpage_info, u32 sectorsize) 67*8481dd80SQu Wenruo { 68*8481dd80SQu Wenruo unsigned int cur = 0; 69*8481dd80SQu Wenruo unsigned int nr_bits; 70*8481dd80SQu Wenruo 71*8481dd80SQu Wenruo ASSERT(IS_ALIGNED(PAGE_SIZE, sectorsize)); 72*8481dd80SQu Wenruo 73*8481dd80SQu Wenruo nr_bits = PAGE_SIZE / sectorsize; 74*8481dd80SQu Wenruo subpage_info->bitmap_nr_bits = nr_bits; 75*8481dd80SQu Wenruo 76*8481dd80SQu Wenruo subpage_info->uptodate_offset = cur; 77*8481dd80SQu Wenruo cur += nr_bits; 78*8481dd80SQu Wenruo 79*8481dd80SQu Wenruo subpage_info->error_offset = cur; 80*8481dd80SQu Wenruo cur += nr_bits; 81*8481dd80SQu Wenruo 82*8481dd80SQu Wenruo subpage_info->dirty_offset = cur; 83*8481dd80SQu Wenruo cur += nr_bits; 84*8481dd80SQu Wenruo 85*8481dd80SQu Wenruo subpage_info->writeback_offset = cur; 86*8481dd80SQu Wenruo cur += nr_bits; 87*8481dd80SQu Wenruo 88*8481dd80SQu Wenruo subpage_info->ordered_offset = cur; 89*8481dd80SQu Wenruo cur += nr_bits; 90*8481dd80SQu Wenruo 91*8481dd80SQu Wenruo subpage_info->total_nr_bits = cur; 92*8481dd80SQu Wenruo } 93*8481dd80SQu Wenruo 94cac06d84SQu Wenruo int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, 95cac06d84SQu Wenruo struct page *page, enum btrfs_subpage_type type) 96cac06d84SQu Wenruo { 97651fb419SQu Wenruo struct btrfs_subpage *subpage; 98cac06d84SQu Wenruo 99cac06d84SQu Wenruo /* 100cac06d84SQu Wenruo * We have cases like a dummy extent buffer page, which is not mappped 101cac06d84SQu Wenruo * and doesn't need to be locked. 102cac06d84SQu Wenruo */ 103cac06d84SQu Wenruo if (page->mapping) 104cac06d84SQu Wenruo ASSERT(PageLocked(page)); 105651fb419SQu Wenruo 106cac06d84SQu Wenruo /* Either not subpage, or the page already has private attached */ 107cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) 108cac06d84SQu Wenruo return 0; 109cac06d84SQu Wenruo 110651fb419SQu Wenruo subpage = btrfs_alloc_subpage(fs_info, type); 111651fb419SQu Wenruo if (IS_ERR(subpage)) 112651fb419SQu Wenruo return PTR_ERR(subpage); 113651fb419SQu Wenruo 114cac06d84SQu Wenruo attach_page_private(page, subpage); 115cac06d84SQu Wenruo return 0; 116cac06d84SQu Wenruo } 117cac06d84SQu Wenruo 118cac06d84SQu Wenruo void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, 119cac06d84SQu Wenruo struct page *page) 120cac06d84SQu Wenruo { 121cac06d84SQu Wenruo struct btrfs_subpage *subpage; 122cac06d84SQu Wenruo 123cac06d84SQu Wenruo /* Either not subpage, or already detached */ 124cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page)) 125cac06d84SQu Wenruo return; 126cac06d84SQu Wenruo 127cac06d84SQu Wenruo subpage = (struct btrfs_subpage *)detach_page_private(page); 128cac06d84SQu Wenruo ASSERT(subpage); 129760f991fSQu Wenruo btrfs_free_subpage(subpage); 130760f991fSQu Wenruo } 131760f991fSQu Wenruo 132651fb419SQu Wenruo struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, 133760f991fSQu Wenruo enum btrfs_subpage_type type) 134760f991fSQu Wenruo { 135651fb419SQu Wenruo struct btrfs_subpage *ret; 136651fb419SQu Wenruo 137fdf250dbSQu Wenruo ASSERT(fs_info->sectorsize < PAGE_SIZE); 138760f991fSQu Wenruo 139651fb419SQu Wenruo ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS); 140651fb419SQu Wenruo if (!ret) 141651fb419SQu Wenruo return ERR_PTR(-ENOMEM); 142651fb419SQu Wenruo 143651fb419SQu Wenruo spin_lock_init(&ret->lock); 1441e1de387SQu Wenruo if (type == BTRFS_SUBPAGE_METADATA) { 145651fb419SQu Wenruo atomic_set(&ret->eb_refs, 0); 1461e1de387SQu Wenruo } else { 147651fb419SQu Wenruo atomic_set(&ret->readers, 0); 148651fb419SQu Wenruo atomic_set(&ret->writers, 0); 1491e1de387SQu Wenruo } 150651fb419SQu Wenruo return ret; 151760f991fSQu Wenruo } 152760f991fSQu Wenruo 153760f991fSQu Wenruo void btrfs_free_subpage(struct btrfs_subpage *subpage) 154760f991fSQu Wenruo { 155cac06d84SQu Wenruo kfree(subpage); 156cac06d84SQu Wenruo } 1578ff8466dSQu Wenruo 1588ff8466dSQu Wenruo /* 1598ff8466dSQu Wenruo * Increase the eb_refs of current subpage. 1608ff8466dSQu Wenruo * 1618ff8466dSQu Wenruo * This is important for eb allocation, to prevent race with last eb freeing 1628ff8466dSQu Wenruo * of the same page. 1638ff8466dSQu Wenruo * With the eb_refs increased before the eb inserted into radix tree, 1648ff8466dSQu Wenruo * detach_extent_buffer_page() won't detach the page private while we're still 1658ff8466dSQu Wenruo * allocating the extent buffer. 1668ff8466dSQu Wenruo */ 1678ff8466dSQu Wenruo void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, 1688ff8466dSQu Wenruo struct page *page) 1698ff8466dSQu Wenruo { 1708ff8466dSQu Wenruo struct btrfs_subpage *subpage; 1718ff8466dSQu Wenruo 1728ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 1738ff8466dSQu Wenruo return; 1748ff8466dSQu Wenruo 1758ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 1768ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 1778ff8466dSQu Wenruo 1788ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 1798ff8466dSQu Wenruo atomic_inc(&subpage->eb_refs); 1808ff8466dSQu Wenruo } 1818ff8466dSQu Wenruo 1828ff8466dSQu Wenruo void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, 1838ff8466dSQu Wenruo struct page *page) 1848ff8466dSQu Wenruo { 1858ff8466dSQu Wenruo struct btrfs_subpage *subpage; 1868ff8466dSQu Wenruo 1878ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 1888ff8466dSQu Wenruo return; 1898ff8466dSQu Wenruo 1908ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 1918ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 1928ff8466dSQu Wenruo 1938ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 1948ff8466dSQu Wenruo ASSERT(atomic_read(&subpage->eb_refs)); 1958ff8466dSQu Wenruo atomic_dec(&subpage->eb_refs); 1968ff8466dSQu Wenruo } 197a1d767c1SQu Wenruo 19892082d40SQu Wenruo static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 19992082d40SQu Wenruo struct page *page, u64 start, u32 len) 20092082d40SQu Wenruo { 20192082d40SQu Wenruo /* Basic checks */ 20292082d40SQu Wenruo ASSERT(PagePrivate(page) && page->private); 20392082d40SQu Wenruo ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 20492082d40SQu Wenruo IS_ALIGNED(len, fs_info->sectorsize)); 20592082d40SQu Wenruo /* 20692082d40SQu Wenruo * The range check only works for mapped page, we can still have 20792082d40SQu Wenruo * unmapped page like dummy extent buffer pages. 20892082d40SQu Wenruo */ 20992082d40SQu Wenruo if (page->mapping) 21092082d40SQu Wenruo ASSERT(page_offset(page) <= start && 21192082d40SQu Wenruo start + len <= page_offset(page) + PAGE_SIZE); 21292082d40SQu Wenruo } 21392082d40SQu Wenruo 21492082d40SQu Wenruo void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, 21592082d40SQu Wenruo struct page *page, u64 start, u32 len) 21692082d40SQu Wenruo { 21792082d40SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 21892082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 21992082d40SQu Wenruo 22092082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 22192082d40SQu Wenruo 2223d078efaSQu Wenruo atomic_add(nbits, &subpage->readers); 22392082d40SQu Wenruo } 22492082d40SQu Wenruo 22592082d40SQu Wenruo void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, 22692082d40SQu Wenruo struct page *page, u64 start, u32 len) 22792082d40SQu Wenruo { 22892082d40SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 22992082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 2303d078efaSQu Wenruo bool is_data; 2313d078efaSQu Wenruo bool last; 23292082d40SQu Wenruo 23392082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2343d078efaSQu Wenruo is_data = is_data_inode(page->mapping->host); 23592082d40SQu Wenruo ASSERT(atomic_read(&subpage->readers) >= nbits); 2363d078efaSQu Wenruo last = atomic_sub_and_test(nbits, &subpage->readers); 2373d078efaSQu Wenruo 2383d078efaSQu Wenruo /* 2393d078efaSQu Wenruo * For data we need to unlock the page if the last read has finished. 2403d078efaSQu Wenruo * 2413d078efaSQu Wenruo * And please don't replace @last with atomic_sub_and_test() call 2423d078efaSQu Wenruo * inside if () condition. 2433d078efaSQu Wenruo * As we want the atomic_sub_and_test() to be always executed. 2443d078efaSQu Wenruo */ 2453d078efaSQu Wenruo if (is_data && last) 24692082d40SQu Wenruo unlock_page(page); 24792082d40SQu Wenruo } 24892082d40SQu Wenruo 2491e1de387SQu Wenruo static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len) 2501e1de387SQu Wenruo { 2511e1de387SQu Wenruo u64 orig_start = *start; 2521e1de387SQu Wenruo u32 orig_len = *len; 2531e1de387SQu Wenruo 2541e1de387SQu Wenruo *start = max_t(u64, page_offset(page), orig_start); 2551e1de387SQu Wenruo *len = min_t(u64, page_offset(page) + PAGE_SIZE, 2561e1de387SQu Wenruo orig_start + orig_len) - *start; 2571e1de387SQu Wenruo } 2581e1de387SQu Wenruo 2591e1de387SQu Wenruo void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info, 2601e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2611e1de387SQu Wenruo { 2621e1de387SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 2631e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits); 2641e1de387SQu Wenruo int ret; 2651e1de387SQu Wenruo 2661e1de387SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2671e1de387SQu Wenruo 2681e1de387SQu Wenruo ASSERT(atomic_read(&subpage->readers) == 0); 2691e1de387SQu Wenruo ret = atomic_add_return(nbits, &subpage->writers); 2701e1de387SQu Wenruo ASSERT(ret == nbits); 2711e1de387SQu Wenruo } 2721e1de387SQu Wenruo 2731e1de387SQu Wenruo bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, 2741e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2751e1de387SQu Wenruo { 2761e1de387SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 2771e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits); 2781e1de387SQu Wenruo 2791e1de387SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2801e1de387SQu Wenruo 2811e1de387SQu Wenruo ASSERT(atomic_read(&subpage->writers) >= nbits); 2821e1de387SQu Wenruo return atomic_sub_and_test(nbits, &subpage->writers); 2831e1de387SQu Wenruo } 2841e1de387SQu Wenruo 2851e1de387SQu Wenruo /* 2861e1de387SQu Wenruo * Lock a page for delalloc page writeback. 2871e1de387SQu Wenruo * 2881e1de387SQu Wenruo * Return -EAGAIN if the page is not properly initialized. 2891e1de387SQu Wenruo * Return 0 with the page locked, and writer counter updated. 2901e1de387SQu Wenruo * 2911e1de387SQu Wenruo * Even with 0 returned, the page still need extra check to make sure 2921e1de387SQu Wenruo * it's really the correct page, as the caller is using 2931e1de387SQu Wenruo * find_get_pages_contig(), which can race with page invalidating. 2941e1de387SQu Wenruo */ 2951e1de387SQu Wenruo int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, 2961e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2971e1de387SQu Wenruo { 2981e1de387SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { 2991e1de387SQu Wenruo lock_page(page); 3001e1de387SQu Wenruo return 0; 3011e1de387SQu Wenruo } 3021e1de387SQu Wenruo lock_page(page); 3031e1de387SQu Wenruo if (!PagePrivate(page) || !page->private) { 3041e1de387SQu Wenruo unlock_page(page); 3051e1de387SQu Wenruo return -EAGAIN; 3061e1de387SQu Wenruo } 3071e1de387SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); 3081e1de387SQu Wenruo btrfs_subpage_start_writer(fs_info, page, start, len); 3091e1de387SQu Wenruo return 0; 3101e1de387SQu Wenruo } 3111e1de387SQu Wenruo 3121e1de387SQu Wenruo void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info, 3131e1de387SQu Wenruo struct page *page, u64 start, u32 len) 3141e1de387SQu Wenruo { 3151e1de387SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) 3161e1de387SQu Wenruo return unlock_page(page); 3171e1de387SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); 3181e1de387SQu Wenruo if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len)) 3191e1de387SQu Wenruo unlock_page(page); 3201e1de387SQu Wenruo } 3211e1de387SQu Wenruo 322a1d767c1SQu Wenruo /* 323a1d767c1SQu Wenruo * Convert the [start, start + len) range into a u16 bitmap 324a1d767c1SQu Wenruo * 325a1d767c1SQu Wenruo * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0. 326a1d767c1SQu Wenruo */ 327a1d767c1SQu Wenruo static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, 328a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 329a1d767c1SQu Wenruo { 330a1d767c1SQu Wenruo const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits; 331a1d767c1SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 332a1d767c1SQu Wenruo 33392082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 334a1d767c1SQu Wenruo 335a1d767c1SQu Wenruo /* 336a1d767c1SQu Wenruo * Here nbits can be 16, thus can go beyond u16 range. We make the 337a1d767c1SQu Wenruo * first left shift to be calculate in unsigned long (at least u32), 338a1d767c1SQu Wenruo * then truncate the result to u16. 339a1d767c1SQu Wenruo */ 340a1d767c1SQu Wenruo return (u16)(((1UL << nbits) - 1) << bit_start); 341a1d767c1SQu Wenruo } 342a1d767c1SQu Wenruo 343a1d767c1SQu Wenruo void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 344a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 345a1d767c1SQu Wenruo { 346a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 347a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 348a1d767c1SQu Wenruo unsigned long flags; 349a1d767c1SQu Wenruo 350a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 351a1d767c1SQu Wenruo subpage->uptodate_bitmap |= tmp; 352a1d767c1SQu Wenruo if (subpage->uptodate_bitmap == U16_MAX) 353a1d767c1SQu Wenruo SetPageUptodate(page); 354a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 355a1d767c1SQu Wenruo } 356a1d767c1SQu Wenruo 357a1d767c1SQu Wenruo void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 358a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 359a1d767c1SQu Wenruo { 360a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 361a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 362a1d767c1SQu Wenruo unsigned long flags; 363a1d767c1SQu Wenruo 364a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 365a1d767c1SQu Wenruo subpage->uptodate_bitmap &= ~tmp; 366a1d767c1SQu Wenruo ClearPageUptodate(page); 367a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 368a1d767c1SQu Wenruo } 369a1d767c1SQu Wenruo 37003a816b3SQu Wenruo void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, 37103a816b3SQu Wenruo struct page *page, u64 start, u32 len) 37203a816b3SQu Wenruo { 37303a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 37403a816b3SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 37503a816b3SQu Wenruo unsigned long flags; 37603a816b3SQu Wenruo 37703a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 37803a816b3SQu Wenruo subpage->error_bitmap |= tmp; 37903a816b3SQu Wenruo SetPageError(page); 38003a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 38103a816b3SQu Wenruo } 38203a816b3SQu Wenruo 38303a816b3SQu Wenruo void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, 38403a816b3SQu Wenruo struct page *page, u64 start, u32 len) 38503a816b3SQu Wenruo { 38603a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 38703a816b3SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 38803a816b3SQu Wenruo unsigned long flags; 38903a816b3SQu Wenruo 39003a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 39103a816b3SQu Wenruo subpage->error_bitmap &= ~tmp; 39203a816b3SQu Wenruo if (subpage->error_bitmap == 0) 39303a816b3SQu Wenruo ClearPageError(page); 39403a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 39503a816b3SQu Wenruo } 39603a816b3SQu Wenruo 397d8a5713eSQu Wenruo void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, 398d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 399d8a5713eSQu Wenruo { 400d8a5713eSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 401d8a5713eSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 402d8a5713eSQu Wenruo unsigned long flags; 403d8a5713eSQu Wenruo 404d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 405d8a5713eSQu Wenruo subpage->dirty_bitmap |= tmp; 406d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 407d8a5713eSQu Wenruo set_page_dirty(page); 408d8a5713eSQu Wenruo } 409d8a5713eSQu Wenruo 410d8a5713eSQu Wenruo /* 411d8a5713eSQu Wenruo * Extra clear_and_test function for subpage dirty bitmap. 412d8a5713eSQu Wenruo * 413d8a5713eSQu Wenruo * Return true if we're the last bits in the dirty_bitmap and clear the 414d8a5713eSQu Wenruo * dirty_bitmap. 415d8a5713eSQu Wenruo * Return false otherwise. 416d8a5713eSQu Wenruo * 417d8a5713eSQu Wenruo * NOTE: Callers should manually clear page dirty for true case, as we have 418d8a5713eSQu Wenruo * extra handling for tree blocks. 419d8a5713eSQu Wenruo */ 420d8a5713eSQu Wenruo bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 421d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 422d8a5713eSQu Wenruo { 423d8a5713eSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 424d8a5713eSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 425d8a5713eSQu Wenruo unsigned long flags; 426d8a5713eSQu Wenruo bool last = false; 427d8a5713eSQu Wenruo 428d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 429d8a5713eSQu Wenruo subpage->dirty_bitmap &= ~tmp; 430d8a5713eSQu Wenruo if (subpage->dirty_bitmap == 0) 431d8a5713eSQu Wenruo last = true; 432d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 433d8a5713eSQu Wenruo return last; 434d8a5713eSQu Wenruo } 435d8a5713eSQu Wenruo 436d8a5713eSQu Wenruo void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, 437d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 438d8a5713eSQu Wenruo { 439d8a5713eSQu Wenruo bool last; 440d8a5713eSQu Wenruo 441d8a5713eSQu Wenruo last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len); 442d8a5713eSQu Wenruo if (last) 443d8a5713eSQu Wenruo clear_page_dirty_for_io(page); 444d8a5713eSQu Wenruo } 445d8a5713eSQu Wenruo 4463470da3bSQu Wenruo void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, 4473470da3bSQu Wenruo struct page *page, u64 start, u32 len) 4483470da3bSQu Wenruo { 4493470da3bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 4503470da3bSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 4513470da3bSQu Wenruo unsigned long flags; 4523470da3bSQu Wenruo 4533470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 4543470da3bSQu Wenruo subpage->writeback_bitmap |= tmp; 4553470da3bSQu Wenruo set_page_writeback(page); 4563470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 4573470da3bSQu Wenruo } 4583470da3bSQu Wenruo 4593470da3bSQu Wenruo void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, 4603470da3bSQu Wenruo struct page *page, u64 start, u32 len) 4613470da3bSQu Wenruo { 4623470da3bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 4633470da3bSQu Wenruo u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 4643470da3bSQu Wenruo unsigned long flags; 4653470da3bSQu Wenruo 4663470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 4673470da3bSQu Wenruo subpage->writeback_bitmap &= ~tmp; 4687c11d0aeSQu Wenruo if (subpage->writeback_bitmap == 0) { 4697c11d0aeSQu Wenruo ASSERT(PageWriteback(page)); 4703470da3bSQu Wenruo end_page_writeback(page); 4717c11d0aeSQu Wenruo } 4723470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 4733470da3bSQu Wenruo } 4743470da3bSQu Wenruo 4756f17400bSQu Wenruo void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, 4766f17400bSQu Wenruo struct page *page, u64 start, u32 len) 4776f17400bSQu Wenruo { 4786f17400bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 4796f17400bSQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 4806f17400bSQu Wenruo unsigned long flags; 4816f17400bSQu Wenruo 4826f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 4836f17400bSQu Wenruo subpage->ordered_bitmap |= tmp; 4846f17400bSQu Wenruo SetPageOrdered(page); 4856f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 4866f17400bSQu Wenruo } 4876f17400bSQu Wenruo 4886f17400bSQu Wenruo void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, 4896f17400bSQu Wenruo struct page *page, u64 start, u32 len) 4906f17400bSQu Wenruo { 4916f17400bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 4926f17400bSQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 4936f17400bSQu Wenruo unsigned long flags; 4946f17400bSQu Wenruo 4956f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 4966f17400bSQu Wenruo subpage->ordered_bitmap &= ~tmp; 4976f17400bSQu Wenruo if (subpage->ordered_bitmap == 0) 4986f17400bSQu Wenruo ClearPageOrdered(page); 4996f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 5006f17400bSQu Wenruo } 501a1d767c1SQu Wenruo /* 502a1d767c1SQu Wenruo * Unlike set/clear which is dependent on each page status, for test all bits 503a1d767c1SQu Wenruo * are tested in the same way. 504a1d767c1SQu Wenruo */ 505a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 506a1d767c1SQu Wenruo bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 507a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 508a1d767c1SQu Wenruo { \ 509a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ 510a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \ 511a1d767c1SQu Wenruo unsigned long flags; \ 512a1d767c1SQu Wenruo bool ret; \ 513a1d767c1SQu Wenruo \ 514a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); \ 515a1d767c1SQu Wenruo ret = ((subpage->name##_bitmap & tmp) == tmp); \ 516a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); \ 517a1d767c1SQu Wenruo return ret; \ 518a1d767c1SQu Wenruo } 519a1d767c1SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 52003a816b3SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); 521d8a5713eSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); 5223470da3bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); 5236f17400bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); 524a1d767c1SQu Wenruo 525a1d767c1SQu Wenruo /* 526a1d767c1SQu Wenruo * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 527a1d767c1SQu Wenruo * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 528a1d767c1SQu Wenruo * back to regular sectorsize branch. 529a1d767c1SQu Wenruo */ 530a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ 531a1d767c1SQu Wenruo test_page_func) \ 532a1d767c1SQu Wenruo void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ 533a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 534a1d767c1SQu Wenruo { \ 535a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 536a1d767c1SQu Wenruo set_page_func(page); \ 537a1d767c1SQu Wenruo return; \ 538a1d767c1SQu Wenruo } \ 539a1d767c1SQu Wenruo btrfs_subpage_set_##name(fs_info, page, start, len); \ 540a1d767c1SQu Wenruo } \ 541a1d767c1SQu Wenruo void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ 542a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 543a1d767c1SQu Wenruo { \ 544a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 545a1d767c1SQu Wenruo clear_page_func(page); \ 546a1d767c1SQu Wenruo return; \ 547a1d767c1SQu Wenruo } \ 548a1d767c1SQu Wenruo btrfs_subpage_clear_##name(fs_info, page, start, len); \ 549a1d767c1SQu Wenruo } \ 550a1d767c1SQu Wenruo bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ 551a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 552a1d767c1SQu Wenruo { \ 553a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 554a1d767c1SQu Wenruo return test_page_func(page); \ 555a1d767c1SQu Wenruo return btrfs_subpage_test_##name(fs_info, page, start, len); \ 55660e2d255SQu Wenruo } \ 55760e2d255SQu Wenruo void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 55860e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 55960e2d255SQu Wenruo { \ 56060e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 56160e2d255SQu Wenruo set_page_func(page); \ 56260e2d255SQu Wenruo return; \ 56360e2d255SQu Wenruo } \ 56460e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 56560e2d255SQu Wenruo btrfs_subpage_set_##name(fs_info, page, start, len); \ 56660e2d255SQu Wenruo } \ 56760e2d255SQu Wenruo void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 56860e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 56960e2d255SQu Wenruo { \ 57060e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 57160e2d255SQu Wenruo clear_page_func(page); \ 57260e2d255SQu Wenruo return; \ 57360e2d255SQu Wenruo } \ 57460e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 57560e2d255SQu Wenruo btrfs_subpage_clear_##name(fs_info, page, start, len); \ 57660e2d255SQu Wenruo } \ 57760e2d255SQu Wenruo bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 57860e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 57960e2d255SQu Wenruo { \ 58060e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 58160e2d255SQu Wenruo return test_page_func(page); \ 58260e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 58360e2d255SQu Wenruo return btrfs_subpage_test_##name(fs_info, page, start, len); \ 584a1d767c1SQu Wenruo } 585a1d767c1SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, 586a1d767c1SQu Wenruo PageUptodate); 58703a816b3SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); 588d8a5713eSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io, 589d8a5713eSQu Wenruo PageDirty); 5903470da3bSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback, 5913470da3bSQu Wenruo PageWriteback); 5926f17400bSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered, 5936f17400bSQu Wenruo PageOrdered); 594cc1d0d93SQu Wenruo 595cc1d0d93SQu Wenruo /* 596cc1d0d93SQu Wenruo * Make sure not only the page dirty bit is cleared, but also subpage dirty bit 597cc1d0d93SQu Wenruo * is cleared. 598cc1d0d93SQu Wenruo */ 599cc1d0d93SQu Wenruo void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info, 600cc1d0d93SQu Wenruo struct page *page) 601cc1d0d93SQu Wenruo { 602cc1d0d93SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 603cc1d0d93SQu Wenruo 604cc1d0d93SQu Wenruo if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 605cc1d0d93SQu Wenruo return; 606cc1d0d93SQu Wenruo 607cc1d0d93SQu Wenruo ASSERT(!PageDirty(page)); 608cc1d0d93SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 609cc1d0d93SQu Wenruo return; 610cc1d0d93SQu Wenruo 611cc1d0d93SQu Wenruo ASSERT(PagePrivate(page) && page->private); 612cc1d0d93SQu Wenruo ASSERT(subpage->dirty_bitmap == 0); 613cc1d0d93SQu Wenruo } 614