1cac06d84SQu Wenruo // SPDX-License-Identifier: GPL-2.0 2cac06d84SQu Wenruo 3cac06d84SQu Wenruo #include <linux/slab.h> 4cac06d84SQu Wenruo #include "ctree.h" 5cac06d84SQu Wenruo #include "subpage.h" 63d078efaSQu Wenruo #include "btrfs_inode.h" 7cac06d84SQu Wenruo 8894d1378SQu Wenruo /* 9894d1378SQu Wenruo * Subpage (sectorsize < PAGE_SIZE) support overview: 10894d1378SQu Wenruo * 11894d1378SQu Wenruo * Limitations: 12894d1378SQu Wenruo * 13894d1378SQu Wenruo * - Only support 64K page size for now 14894d1378SQu Wenruo * This is to make metadata handling easier, as 64K page would ensure 15894d1378SQu Wenruo * all nodesize would fit inside one page, thus we don't need to handle 16894d1378SQu Wenruo * cases where a tree block crosses several pages. 17894d1378SQu Wenruo * 18894d1378SQu Wenruo * - Only metadata read-write for now 19894d1378SQu Wenruo * The data read-write part is in development. 20894d1378SQu Wenruo * 21894d1378SQu Wenruo * - Metadata can't cross 64K page boundary 22894d1378SQu Wenruo * btrfs-progs and kernel have done that for a while, thus only ancient 23894d1378SQu Wenruo * filesystems could have such problem. For such case, do a graceful 24894d1378SQu Wenruo * rejection. 25894d1378SQu Wenruo * 26894d1378SQu Wenruo * Special behavior: 27894d1378SQu Wenruo * 28894d1378SQu Wenruo * - Metadata 29894d1378SQu Wenruo * Metadata read is fully supported. 30894d1378SQu Wenruo * Meaning when reading one tree block will only trigger the read for the 31894d1378SQu Wenruo * needed range, other unrelated range in the same page will not be touched. 32894d1378SQu Wenruo * 33894d1378SQu Wenruo * Metadata write support is partial. 34894d1378SQu Wenruo * The writeback is still for the full page, but we will only submit 35894d1378SQu Wenruo * the dirty extent buffers in the page. 36894d1378SQu Wenruo * 37894d1378SQu Wenruo * This means, if we have a metadata page like this: 38894d1378SQu Wenruo * 39894d1378SQu Wenruo * Page offset 40894d1378SQu Wenruo * 0 16K 32K 48K 64K 41894d1378SQu Wenruo * |/////////| |///////////| 42894d1378SQu Wenruo * \- Tree block A \- Tree block B 43894d1378SQu Wenruo * 44894d1378SQu Wenruo * Even if we just want to writeback tree block A, we will also writeback 45894d1378SQu Wenruo * tree block B if it's also dirty. 46894d1378SQu Wenruo * 47894d1378SQu Wenruo * This may cause extra metadata writeback which results more COW. 48894d1378SQu Wenruo * 49894d1378SQu Wenruo * Implementation: 50894d1378SQu Wenruo * 51894d1378SQu Wenruo * - Common 52894d1378SQu Wenruo * Both metadata and data will use a new structure, btrfs_subpage, to 53894d1378SQu Wenruo * record the status of each sector inside a page. This provides the extra 54894d1378SQu Wenruo * granularity needed. 55894d1378SQu Wenruo * 56894d1378SQu Wenruo * - Metadata 57894d1378SQu Wenruo * Since we have multiple tree blocks inside one page, we can't rely on page 58894d1378SQu Wenruo * locking anymore, or we will have greatly reduced concurrency or even 59894d1378SQu Wenruo * deadlocks (hold one tree lock while trying to lock another tree lock in 60894d1378SQu Wenruo * the same page). 61894d1378SQu Wenruo * 62894d1378SQu Wenruo * Thus for metadata locking, subpage support relies on io_tree locking only. 63894d1378SQu Wenruo * This means a slightly higher tree locking latency. 64894d1378SQu Wenruo */ 65894d1378SQu Wenruo 668481dd80SQu Wenruo void btrfs_init_subpage_info(struct btrfs_subpage_info *subpage_info, u32 sectorsize) 678481dd80SQu Wenruo { 688481dd80SQu Wenruo unsigned int cur = 0; 698481dd80SQu Wenruo unsigned int nr_bits; 708481dd80SQu Wenruo 718481dd80SQu Wenruo ASSERT(IS_ALIGNED(PAGE_SIZE, sectorsize)); 728481dd80SQu Wenruo 738481dd80SQu Wenruo nr_bits = PAGE_SIZE / sectorsize; 748481dd80SQu Wenruo subpage_info->bitmap_nr_bits = nr_bits; 758481dd80SQu Wenruo 768481dd80SQu Wenruo subpage_info->uptodate_offset = cur; 778481dd80SQu Wenruo cur += nr_bits; 788481dd80SQu Wenruo 798481dd80SQu Wenruo subpage_info->error_offset = cur; 808481dd80SQu Wenruo cur += nr_bits; 818481dd80SQu Wenruo 828481dd80SQu Wenruo subpage_info->dirty_offset = cur; 838481dd80SQu Wenruo cur += nr_bits; 848481dd80SQu Wenruo 858481dd80SQu Wenruo subpage_info->writeback_offset = cur; 868481dd80SQu Wenruo cur += nr_bits; 878481dd80SQu Wenruo 888481dd80SQu Wenruo subpage_info->ordered_offset = cur; 898481dd80SQu Wenruo cur += nr_bits; 908481dd80SQu Wenruo 918481dd80SQu Wenruo subpage_info->total_nr_bits = cur; 928481dd80SQu Wenruo } 938481dd80SQu Wenruo 94cac06d84SQu Wenruo int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, 95cac06d84SQu Wenruo struct page *page, enum btrfs_subpage_type type) 96cac06d84SQu Wenruo { 97651fb419SQu Wenruo struct btrfs_subpage *subpage; 98cac06d84SQu Wenruo 99cac06d84SQu Wenruo /* 100cac06d84SQu Wenruo * We have cases like a dummy extent buffer page, which is not mappped 101cac06d84SQu Wenruo * and doesn't need to be locked. 102cac06d84SQu Wenruo */ 103cac06d84SQu Wenruo if (page->mapping) 104cac06d84SQu Wenruo ASSERT(PageLocked(page)); 105651fb419SQu Wenruo 106cac06d84SQu Wenruo /* Either not subpage, or the page already has private attached */ 107cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) 108cac06d84SQu Wenruo return 0; 109cac06d84SQu Wenruo 110651fb419SQu Wenruo subpage = btrfs_alloc_subpage(fs_info, type); 111651fb419SQu Wenruo if (IS_ERR(subpage)) 112651fb419SQu Wenruo return PTR_ERR(subpage); 113651fb419SQu Wenruo 114cac06d84SQu Wenruo attach_page_private(page, subpage); 115cac06d84SQu Wenruo return 0; 116cac06d84SQu Wenruo } 117cac06d84SQu Wenruo 118cac06d84SQu Wenruo void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, 119cac06d84SQu Wenruo struct page *page) 120cac06d84SQu Wenruo { 121cac06d84SQu Wenruo struct btrfs_subpage *subpage; 122cac06d84SQu Wenruo 123cac06d84SQu Wenruo /* Either not subpage, or already detached */ 124cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page)) 125cac06d84SQu Wenruo return; 126cac06d84SQu Wenruo 127cac06d84SQu Wenruo subpage = (struct btrfs_subpage *)detach_page_private(page); 128cac06d84SQu Wenruo ASSERT(subpage); 129760f991fSQu Wenruo btrfs_free_subpage(subpage); 130760f991fSQu Wenruo } 131760f991fSQu Wenruo 132651fb419SQu Wenruo struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, 133760f991fSQu Wenruo enum btrfs_subpage_type type) 134760f991fSQu Wenruo { 135651fb419SQu Wenruo struct btrfs_subpage *ret; 136*72a69cd0SQu Wenruo unsigned int real_size; 137651fb419SQu Wenruo 138fdf250dbSQu Wenruo ASSERT(fs_info->sectorsize < PAGE_SIZE); 139760f991fSQu Wenruo 140*72a69cd0SQu Wenruo real_size = struct_size(ret, bitmaps, 141*72a69cd0SQu Wenruo BITS_TO_LONGS(fs_info->subpage_info->total_nr_bits)); 142*72a69cd0SQu Wenruo ret = kzalloc(real_size, GFP_NOFS); 143651fb419SQu Wenruo if (!ret) 144651fb419SQu Wenruo return ERR_PTR(-ENOMEM); 145651fb419SQu Wenruo 146651fb419SQu Wenruo spin_lock_init(&ret->lock); 1471e1de387SQu Wenruo if (type == BTRFS_SUBPAGE_METADATA) { 148651fb419SQu Wenruo atomic_set(&ret->eb_refs, 0); 1491e1de387SQu Wenruo } else { 150651fb419SQu Wenruo atomic_set(&ret->readers, 0); 151651fb419SQu Wenruo atomic_set(&ret->writers, 0); 1521e1de387SQu Wenruo } 153651fb419SQu Wenruo return ret; 154760f991fSQu Wenruo } 155760f991fSQu Wenruo 156760f991fSQu Wenruo void btrfs_free_subpage(struct btrfs_subpage *subpage) 157760f991fSQu Wenruo { 158cac06d84SQu Wenruo kfree(subpage); 159cac06d84SQu Wenruo } 1608ff8466dSQu Wenruo 1618ff8466dSQu Wenruo /* 1628ff8466dSQu Wenruo * Increase the eb_refs of current subpage. 1638ff8466dSQu Wenruo * 1648ff8466dSQu Wenruo * This is important for eb allocation, to prevent race with last eb freeing 1658ff8466dSQu Wenruo * of the same page. 1668ff8466dSQu Wenruo * With the eb_refs increased before the eb inserted into radix tree, 1678ff8466dSQu Wenruo * detach_extent_buffer_page() won't detach the page private while we're still 1688ff8466dSQu Wenruo * allocating the extent buffer. 1698ff8466dSQu Wenruo */ 1708ff8466dSQu Wenruo void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, 1718ff8466dSQu Wenruo struct page *page) 1728ff8466dSQu Wenruo { 1738ff8466dSQu Wenruo struct btrfs_subpage *subpage; 1748ff8466dSQu Wenruo 1758ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 1768ff8466dSQu Wenruo return; 1778ff8466dSQu Wenruo 1788ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 1798ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 1808ff8466dSQu Wenruo 1818ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 1828ff8466dSQu Wenruo atomic_inc(&subpage->eb_refs); 1838ff8466dSQu Wenruo } 1848ff8466dSQu Wenruo 1858ff8466dSQu Wenruo void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, 1868ff8466dSQu Wenruo struct page *page) 1878ff8466dSQu Wenruo { 1888ff8466dSQu Wenruo struct btrfs_subpage *subpage; 1898ff8466dSQu Wenruo 1908ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 1918ff8466dSQu Wenruo return; 1928ff8466dSQu Wenruo 1938ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 1948ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 1958ff8466dSQu Wenruo 1968ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 1978ff8466dSQu Wenruo ASSERT(atomic_read(&subpage->eb_refs)); 1988ff8466dSQu Wenruo atomic_dec(&subpage->eb_refs); 1998ff8466dSQu Wenruo } 200a1d767c1SQu Wenruo 20192082d40SQu Wenruo static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 20292082d40SQu Wenruo struct page *page, u64 start, u32 len) 20392082d40SQu Wenruo { 20492082d40SQu Wenruo /* Basic checks */ 20592082d40SQu Wenruo ASSERT(PagePrivate(page) && page->private); 20692082d40SQu Wenruo ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 20792082d40SQu Wenruo IS_ALIGNED(len, fs_info->sectorsize)); 20892082d40SQu Wenruo /* 20992082d40SQu Wenruo * The range check only works for mapped page, we can still have 21092082d40SQu Wenruo * unmapped page like dummy extent buffer pages. 21192082d40SQu Wenruo */ 21292082d40SQu Wenruo if (page->mapping) 21392082d40SQu Wenruo ASSERT(page_offset(page) <= start && 21492082d40SQu Wenruo start + len <= page_offset(page) + PAGE_SIZE); 21592082d40SQu Wenruo } 21692082d40SQu Wenruo 21792082d40SQu Wenruo void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, 21892082d40SQu Wenruo struct page *page, u64 start, u32 len) 21992082d40SQu Wenruo { 22092082d40SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 22192082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 22292082d40SQu Wenruo 22392082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 22492082d40SQu Wenruo 2253d078efaSQu Wenruo atomic_add(nbits, &subpage->readers); 22692082d40SQu Wenruo } 22792082d40SQu Wenruo 22892082d40SQu Wenruo void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, 22992082d40SQu Wenruo struct page *page, u64 start, u32 len) 23092082d40SQu Wenruo { 23192082d40SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 23292082d40SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 2333d078efaSQu Wenruo bool is_data; 2343d078efaSQu Wenruo bool last; 23592082d40SQu Wenruo 23692082d40SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2373d078efaSQu Wenruo is_data = is_data_inode(page->mapping->host); 23892082d40SQu Wenruo ASSERT(atomic_read(&subpage->readers) >= nbits); 2393d078efaSQu Wenruo last = atomic_sub_and_test(nbits, &subpage->readers); 2403d078efaSQu Wenruo 2413d078efaSQu Wenruo /* 2423d078efaSQu Wenruo * For data we need to unlock the page if the last read has finished. 2433d078efaSQu Wenruo * 2443d078efaSQu Wenruo * And please don't replace @last with atomic_sub_and_test() call 2453d078efaSQu Wenruo * inside if () condition. 2463d078efaSQu Wenruo * As we want the atomic_sub_and_test() to be always executed. 2473d078efaSQu Wenruo */ 2483d078efaSQu Wenruo if (is_data && last) 24992082d40SQu Wenruo unlock_page(page); 25092082d40SQu Wenruo } 25192082d40SQu Wenruo 2521e1de387SQu Wenruo static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len) 2531e1de387SQu Wenruo { 2541e1de387SQu Wenruo u64 orig_start = *start; 2551e1de387SQu Wenruo u32 orig_len = *len; 2561e1de387SQu Wenruo 2571e1de387SQu Wenruo *start = max_t(u64, page_offset(page), orig_start); 2581e1de387SQu Wenruo *len = min_t(u64, page_offset(page) + PAGE_SIZE, 2591e1de387SQu Wenruo orig_start + orig_len) - *start; 2601e1de387SQu Wenruo } 2611e1de387SQu Wenruo 2621e1de387SQu Wenruo void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info, 2631e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2641e1de387SQu Wenruo { 2651e1de387SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 2661e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits); 2671e1de387SQu Wenruo int ret; 2681e1de387SQu Wenruo 2691e1de387SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2701e1de387SQu Wenruo 2711e1de387SQu Wenruo ASSERT(atomic_read(&subpage->readers) == 0); 2721e1de387SQu Wenruo ret = atomic_add_return(nbits, &subpage->writers); 2731e1de387SQu Wenruo ASSERT(ret == nbits); 2741e1de387SQu Wenruo } 2751e1de387SQu Wenruo 2761e1de387SQu Wenruo bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, 2771e1de387SQu Wenruo struct page *page, u64 start, u32 len) 2781e1de387SQu Wenruo { 2791e1de387SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 2801e1de387SQu Wenruo const int nbits = (len >> fs_info->sectorsize_bits); 2811e1de387SQu Wenruo 2821e1de387SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); 2831e1de387SQu Wenruo 2841e1de387SQu Wenruo ASSERT(atomic_read(&subpage->writers) >= nbits); 2851e1de387SQu Wenruo return atomic_sub_and_test(nbits, &subpage->writers); 2861e1de387SQu Wenruo } 2871e1de387SQu Wenruo 2881e1de387SQu Wenruo /* 2891e1de387SQu Wenruo * Lock a page for delalloc page writeback. 2901e1de387SQu Wenruo * 2911e1de387SQu Wenruo * Return -EAGAIN if the page is not properly initialized. 2921e1de387SQu Wenruo * Return 0 with the page locked, and writer counter updated. 2931e1de387SQu Wenruo * 2941e1de387SQu Wenruo * Even with 0 returned, the page still need extra check to make sure 2951e1de387SQu Wenruo * it's really the correct page, as the caller is using 2961e1de387SQu Wenruo * find_get_pages_contig(), which can race with page invalidating. 2971e1de387SQu Wenruo */ 2981e1de387SQu Wenruo int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, 2991e1de387SQu Wenruo struct page *page, u64 start, u32 len) 3001e1de387SQu Wenruo { 3011e1de387SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { 3021e1de387SQu Wenruo lock_page(page); 3031e1de387SQu Wenruo return 0; 3041e1de387SQu Wenruo } 3051e1de387SQu Wenruo lock_page(page); 3061e1de387SQu Wenruo if (!PagePrivate(page) || !page->private) { 3071e1de387SQu Wenruo unlock_page(page); 3081e1de387SQu Wenruo return -EAGAIN; 3091e1de387SQu Wenruo } 3101e1de387SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); 3111e1de387SQu Wenruo btrfs_subpage_start_writer(fs_info, page, start, len); 3121e1de387SQu Wenruo return 0; 3131e1de387SQu Wenruo } 3141e1de387SQu Wenruo 3151e1de387SQu Wenruo void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info, 3161e1de387SQu Wenruo struct page *page, u64 start, u32 len) 3171e1de387SQu Wenruo { 3181e1de387SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) 3191e1de387SQu Wenruo return unlock_page(page); 3201e1de387SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); 3211e1de387SQu Wenruo if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len)) 3221e1de387SQu Wenruo unlock_page(page); 3231e1de387SQu Wenruo } 3241e1de387SQu Wenruo 325*72a69cd0SQu Wenruo static bool bitmap_test_range_all_set(unsigned long *addr, unsigned int start, 326*72a69cd0SQu Wenruo unsigned int nbits) 327a1d767c1SQu Wenruo { 328*72a69cd0SQu Wenruo unsigned int found_zero; 329a1d767c1SQu Wenruo 330*72a69cd0SQu Wenruo found_zero = find_next_zero_bit(addr, start + nbits, start); 331*72a69cd0SQu Wenruo if (found_zero == start + nbits) 332*72a69cd0SQu Wenruo return true; 333*72a69cd0SQu Wenruo return false; 334a1d767c1SQu Wenruo } 335a1d767c1SQu Wenruo 336*72a69cd0SQu Wenruo static bool bitmap_test_range_all_zero(unsigned long *addr, unsigned int start, 337*72a69cd0SQu Wenruo unsigned int nbits) 338*72a69cd0SQu Wenruo { 339*72a69cd0SQu Wenruo unsigned int found_set; 340*72a69cd0SQu Wenruo 341*72a69cd0SQu Wenruo found_set = find_next_bit(addr, start + nbits, start); 342*72a69cd0SQu Wenruo if (found_set == start + nbits) 343*72a69cd0SQu Wenruo return true; 344*72a69cd0SQu Wenruo return false; 345*72a69cd0SQu Wenruo } 346*72a69cd0SQu Wenruo 347*72a69cd0SQu Wenruo #define subpage_calc_start_bit(fs_info, page, name, start, len) \ 348*72a69cd0SQu Wenruo ({ \ 349*72a69cd0SQu Wenruo unsigned int start_bit; \ 350*72a69cd0SQu Wenruo \ 351*72a69cd0SQu Wenruo btrfs_subpage_assert(fs_info, page, start, len); \ 352*72a69cd0SQu Wenruo start_bit = offset_in_page(start) >> fs_info->sectorsize_bits; \ 353*72a69cd0SQu Wenruo start_bit += fs_info->subpage_info->name##_offset; \ 354*72a69cd0SQu Wenruo start_bit; \ 355*72a69cd0SQu Wenruo }) 356*72a69cd0SQu Wenruo 357*72a69cd0SQu Wenruo #define subpage_test_bitmap_all_set(fs_info, subpage, name) \ 358*72a69cd0SQu Wenruo bitmap_test_range_all_set(subpage->bitmaps, \ 359*72a69cd0SQu Wenruo fs_info->subpage_info->name##_offset, \ 360*72a69cd0SQu Wenruo fs_info->subpage_info->bitmap_nr_bits) 361*72a69cd0SQu Wenruo 362*72a69cd0SQu Wenruo #define subpage_test_bitmap_all_zero(fs_info, subpage, name) \ 363*72a69cd0SQu Wenruo bitmap_test_range_all_zero(subpage->bitmaps, \ 364*72a69cd0SQu Wenruo fs_info->subpage_info->name##_offset, \ 365*72a69cd0SQu Wenruo fs_info->subpage_info->bitmap_nr_bits) 366*72a69cd0SQu Wenruo 367a1d767c1SQu Wenruo void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 368a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 369a1d767c1SQu Wenruo { 370a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 371*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 372*72a69cd0SQu Wenruo uptodate, start, len); 373a1d767c1SQu Wenruo unsigned long flags; 374a1d767c1SQu Wenruo 375a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 376*72a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 377*72a69cd0SQu Wenruo if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate)) 378a1d767c1SQu Wenruo SetPageUptodate(page); 379a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 380a1d767c1SQu Wenruo } 381a1d767c1SQu Wenruo 382a1d767c1SQu Wenruo void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 383a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 384a1d767c1SQu Wenruo { 385a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 386*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 387*72a69cd0SQu Wenruo uptodate, start, len); 388a1d767c1SQu Wenruo unsigned long flags; 389a1d767c1SQu Wenruo 390a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 391*72a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 392a1d767c1SQu Wenruo ClearPageUptodate(page); 393a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 394a1d767c1SQu Wenruo } 395a1d767c1SQu Wenruo 39603a816b3SQu Wenruo void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, 39703a816b3SQu Wenruo struct page *page, u64 start, u32 len) 39803a816b3SQu Wenruo { 39903a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 400*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 401*72a69cd0SQu Wenruo error, start, len); 40203a816b3SQu Wenruo unsigned long flags; 40303a816b3SQu Wenruo 40403a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 405*72a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 40603a816b3SQu Wenruo SetPageError(page); 40703a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 40803a816b3SQu Wenruo } 40903a816b3SQu Wenruo 41003a816b3SQu Wenruo void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, 41103a816b3SQu Wenruo struct page *page, u64 start, u32 len) 41203a816b3SQu Wenruo { 41303a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 414*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 415*72a69cd0SQu Wenruo error, start, len); 41603a816b3SQu Wenruo unsigned long flags; 41703a816b3SQu Wenruo 41803a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 419*72a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 420*72a69cd0SQu Wenruo if (subpage_test_bitmap_all_zero(fs_info, subpage, error)) 42103a816b3SQu Wenruo ClearPageError(page); 42203a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 42303a816b3SQu Wenruo } 42403a816b3SQu Wenruo 425d8a5713eSQu Wenruo void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, 426d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 427d8a5713eSQu Wenruo { 428d8a5713eSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 429*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 430*72a69cd0SQu Wenruo dirty, start, len); 431d8a5713eSQu Wenruo unsigned long flags; 432d8a5713eSQu Wenruo 433d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 434*72a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 435d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 436d8a5713eSQu Wenruo set_page_dirty(page); 437d8a5713eSQu Wenruo } 438d8a5713eSQu Wenruo 439d8a5713eSQu Wenruo /* 440d8a5713eSQu Wenruo * Extra clear_and_test function for subpage dirty bitmap. 441d8a5713eSQu Wenruo * 442d8a5713eSQu Wenruo * Return true if we're the last bits in the dirty_bitmap and clear the 443d8a5713eSQu Wenruo * dirty_bitmap. 444d8a5713eSQu Wenruo * Return false otherwise. 445d8a5713eSQu Wenruo * 446d8a5713eSQu Wenruo * NOTE: Callers should manually clear page dirty for true case, as we have 447d8a5713eSQu Wenruo * extra handling for tree blocks. 448d8a5713eSQu Wenruo */ 449d8a5713eSQu Wenruo bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, 450d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 451d8a5713eSQu Wenruo { 452d8a5713eSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 453*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 454*72a69cd0SQu Wenruo dirty, start, len); 455d8a5713eSQu Wenruo unsigned long flags; 456d8a5713eSQu Wenruo bool last = false; 457d8a5713eSQu Wenruo 458d8a5713eSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 459*72a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 460*72a69cd0SQu Wenruo if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty)) 461d8a5713eSQu Wenruo last = true; 462d8a5713eSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 463d8a5713eSQu Wenruo return last; 464d8a5713eSQu Wenruo } 465d8a5713eSQu Wenruo 466d8a5713eSQu Wenruo void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, 467d8a5713eSQu Wenruo struct page *page, u64 start, u32 len) 468d8a5713eSQu Wenruo { 469d8a5713eSQu Wenruo bool last; 470d8a5713eSQu Wenruo 471d8a5713eSQu Wenruo last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len); 472d8a5713eSQu Wenruo if (last) 473d8a5713eSQu Wenruo clear_page_dirty_for_io(page); 474d8a5713eSQu Wenruo } 475d8a5713eSQu Wenruo 4763470da3bSQu Wenruo void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, 4773470da3bSQu Wenruo struct page *page, u64 start, u32 len) 4783470da3bSQu Wenruo { 4793470da3bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 480*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 481*72a69cd0SQu Wenruo writeback, start, len); 4823470da3bSQu Wenruo unsigned long flags; 4833470da3bSQu Wenruo 4843470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 485*72a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 4863470da3bSQu Wenruo set_page_writeback(page); 4873470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 4883470da3bSQu Wenruo } 4893470da3bSQu Wenruo 4903470da3bSQu Wenruo void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, 4913470da3bSQu Wenruo struct page *page, u64 start, u32 len) 4923470da3bSQu Wenruo { 4933470da3bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 494*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 495*72a69cd0SQu Wenruo writeback, start, len); 4963470da3bSQu Wenruo unsigned long flags; 4973470da3bSQu Wenruo 4983470da3bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 499*72a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 500*72a69cd0SQu Wenruo if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) { 5017c11d0aeSQu Wenruo ASSERT(PageWriteback(page)); 5023470da3bSQu Wenruo end_page_writeback(page); 5037c11d0aeSQu Wenruo } 5043470da3bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 5053470da3bSQu Wenruo } 5063470da3bSQu Wenruo 5076f17400bSQu Wenruo void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, 5086f17400bSQu Wenruo struct page *page, u64 start, u32 len) 5096f17400bSQu Wenruo { 5106f17400bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 511*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 512*72a69cd0SQu Wenruo ordered, start, len); 5136f17400bSQu Wenruo unsigned long flags; 5146f17400bSQu Wenruo 5156f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 516*72a69cd0SQu Wenruo bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 5176f17400bSQu Wenruo SetPageOrdered(page); 5186f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 5196f17400bSQu Wenruo } 5206f17400bSQu Wenruo 5216f17400bSQu Wenruo void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, 5226f17400bSQu Wenruo struct page *page, u64 start, u32 len) 5236f17400bSQu Wenruo { 5246f17400bSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 525*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, 526*72a69cd0SQu Wenruo ordered, start, len); 5276f17400bSQu Wenruo unsigned long flags; 5286f17400bSQu Wenruo 5296f17400bSQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 530*72a69cd0SQu Wenruo bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits); 531*72a69cd0SQu Wenruo if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered)) 5326f17400bSQu Wenruo ClearPageOrdered(page); 5336f17400bSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 5346f17400bSQu Wenruo } 535a1d767c1SQu Wenruo /* 536a1d767c1SQu Wenruo * Unlike set/clear which is dependent on each page status, for test all bits 537a1d767c1SQu Wenruo * are tested in the same way. 538a1d767c1SQu Wenruo */ 539a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 540a1d767c1SQu Wenruo bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 541a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 542a1d767c1SQu Wenruo { \ 543a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ 544*72a69cd0SQu Wenruo unsigned int start_bit = subpage_calc_start_bit(fs_info, page, \ 545*72a69cd0SQu Wenruo name, start, len); \ 546a1d767c1SQu Wenruo unsigned long flags; \ 547a1d767c1SQu Wenruo bool ret; \ 548a1d767c1SQu Wenruo \ 549a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); \ 550*72a69cd0SQu Wenruo ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit, \ 551*72a69cd0SQu Wenruo len >> fs_info->sectorsize_bits); \ 552a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); \ 553a1d767c1SQu Wenruo return ret; \ 554a1d767c1SQu Wenruo } 555a1d767c1SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 55603a816b3SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); 557d8a5713eSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); 5583470da3bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); 5596f17400bSQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); 560a1d767c1SQu Wenruo 561a1d767c1SQu Wenruo /* 562a1d767c1SQu Wenruo * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 563a1d767c1SQu Wenruo * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 564a1d767c1SQu Wenruo * back to regular sectorsize branch. 565a1d767c1SQu Wenruo */ 566a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ 567a1d767c1SQu Wenruo test_page_func) \ 568a1d767c1SQu Wenruo void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ 569a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 570a1d767c1SQu Wenruo { \ 571a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 572a1d767c1SQu Wenruo set_page_func(page); \ 573a1d767c1SQu Wenruo return; \ 574a1d767c1SQu Wenruo } \ 575a1d767c1SQu Wenruo btrfs_subpage_set_##name(fs_info, page, start, len); \ 576a1d767c1SQu Wenruo } \ 577a1d767c1SQu Wenruo void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ 578a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 579a1d767c1SQu Wenruo { \ 580a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 581a1d767c1SQu Wenruo clear_page_func(page); \ 582a1d767c1SQu Wenruo return; \ 583a1d767c1SQu Wenruo } \ 584a1d767c1SQu Wenruo btrfs_subpage_clear_##name(fs_info, page, start, len); \ 585a1d767c1SQu Wenruo } \ 586a1d767c1SQu Wenruo bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ 587a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 588a1d767c1SQu Wenruo { \ 589a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 590a1d767c1SQu Wenruo return test_page_func(page); \ 591a1d767c1SQu Wenruo return btrfs_subpage_test_##name(fs_info, page, start, len); \ 59260e2d255SQu Wenruo } \ 59360e2d255SQu Wenruo void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ 59460e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 59560e2d255SQu Wenruo { \ 59660e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 59760e2d255SQu Wenruo set_page_func(page); \ 59860e2d255SQu Wenruo return; \ 59960e2d255SQu Wenruo } \ 60060e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 60160e2d255SQu Wenruo btrfs_subpage_set_##name(fs_info, page, start, len); \ 60260e2d255SQu Wenruo } \ 60360e2d255SQu Wenruo void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ 60460e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 60560e2d255SQu Wenruo { \ 60660e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 60760e2d255SQu Wenruo clear_page_func(page); \ 60860e2d255SQu Wenruo return; \ 60960e2d255SQu Wenruo } \ 61060e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 61160e2d255SQu Wenruo btrfs_subpage_clear_##name(fs_info, page, start, len); \ 61260e2d255SQu Wenruo } \ 61360e2d255SQu Wenruo bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ 61460e2d255SQu Wenruo struct page *page, u64 start, u32 len) \ 61560e2d255SQu Wenruo { \ 61660e2d255SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 61760e2d255SQu Wenruo return test_page_func(page); \ 61860e2d255SQu Wenruo btrfs_subpage_clamp_range(page, &start, &len); \ 61960e2d255SQu Wenruo return btrfs_subpage_test_##name(fs_info, page, start, len); \ 620a1d767c1SQu Wenruo } 621a1d767c1SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, 622a1d767c1SQu Wenruo PageUptodate); 62303a816b3SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); 624d8a5713eSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io, 625d8a5713eSQu Wenruo PageDirty); 6263470da3bSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback, 6273470da3bSQu Wenruo PageWriteback); 6286f17400bSQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered, 6296f17400bSQu Wenruo PageOrdered); 630cc1d0d93SQu Wenruo 631cc1d0d93SQu Wenruo /* 632cc1d0d93SQu Wenruo * Make sure not only the page dirty bit is cleared, but also subpage dirty bit 633cc1d0d93SQu Wenruo * is cleared. 634cc1d0d93SQu Wenruo */ 635cc1d0d93SQu Wenruo void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info, 636cc1d0d93SQu Wenruo struct page *page) 637cc1d0d93SQu Wenruo { 638cc1d0d93SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 639cc1d0d93SQu Wenruo 640cc1d0d93SQu Wenruo if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 641cc1d0d93SQu Wenruo return; 642cc1d0d93SQu Wenruo 643cc1d0d93SQu Wenruo ASSERT(!PageDirty(page)); 644cc1d0d93SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 645cc1d0d93SQu Wenruo return; 646cc1d0d93SQu Wenruo 647cc1d0d93SQu Wenruo ASSERT(PagePrivate(page) && page->private); 648*72a69cd0SQu Wenruo ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty)); 649cc1d0d93SQu Wenruo } 650