1cac06d84SQu Wenruo // SPDX-License-Identifier: GPL-2.0 2cac06d84SQu Wenruo 3cac06d84SQu Wenruo #include <linux/slab.h> 4cac06d84SQu Wenruo #include "ctree.h" 5cac06d84SQu Wenruo #include "subpage.h" 6cac06d84SQu Wenruo 7cac06d84SQu Wenruo int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, 8cac06d84SQu Wenruo struct page *page, enum btrfs_subpage_type type) 9cac06d84SQu Wenruo { 10760f991fSQu Wenruo struct btrfs_subpage *subpage = NULL; 11760f991fSQu Wenruo int ret; 12cac06d84SQu Wenruo 13cac06d84SQu Wenruo /* 14cac06d84SQu Wenruo * We have cases like a dummy extent buffer page, which is not mappped 15cac06d84SQu Wenruo * and doesn't need to be locked. 16cac06d84SQu Wenruo */ 17cac06d84SQu Wenruo if (page->mapping) 18cac06d84SQu Wenruo ASSERT(PageLocked(page)); 19cac06d84SQu Wenruo /* Either not subpage, or the page already has private attached */ 20cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) 21cac06d84SQu Wenruo return 0; 22cac06d84SQu Wenruo 23760f991fSQu Wenruo ret = btrfs_alloc_subpage(fs_info, &subpage, type); 24760f991fSQu Wenruo if (ret < 0) 25760f991fSQu Wenruo return ret; 26cac06d84SQu Wenruo attach_page_private(page, subpage); 27cac06d84SQu Wenruo return 0; 28cac06d84SQu Wenruo } 29cac06d84SQu Wenruo 30cac06d84SQu Wenruo void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, 31cac06d84SQu Wenruo struct page *page) 32cac06d84SQu Wenruo { 33cac06d84SQu Wenruo struct btrfs_subpage *subpage; 34cac06d84SQu Wenruo 35cac06d84SQu Wenruo /* Either not subpage, or already detached */ 36cac06d84SQu Wenruo if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page)) 37cac06d84SQu Wenruo return; 38cac06d84SQu Wenruo 39cac06d84SQu Wenruo subpage = (struct btrfs_subpage *)detach_page_private(page); 40cac06d84SQu Wenruo ASSERT(subpage); 41760f991fSQu Wenruo btrfs_free_subpage(subpage); 42760f991fSQu Wenruo } 43760f991fSQu Wenruo 44760f991fSQu Wenruo int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, 45760f991fSQu Wenruo struct btrfs_subpage **ret, 46760f991fSQu Wenruo enum btrfs_subpage_type type) 47760f991fSQu Wenruo { 48760f991fSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 49760f991fSQu Wenruo return 0; 50760f991fSQu Wenruo 51760f991fSQu Wenruo *ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS); 52760f991fSQu Wenruo if (!*ret) 53760f991fSQu Wenruo return -ENOMEM; 54760f991fSQu Wenruo spin_lock_init(&(*ret)->lock); 558ff8466dSQu Wenruo if (type == BTRFS_SUBPAGE_METADATA) 568ff8466dSQu Wenruo atomic_set(&(*ret)->eb_refs, 0); 57760f991fSQu Wenruo return 0; 58760f991fSQu Wenruo } 59760f991fSQu Wenruo 60760f991fSQu Wenruo void btrfs_free_subpage(struct btrfs_subpage *subpage) 61760f991fSQu Wenruo { 62cac06d84SQu Wenruo kfree(subpage); 63cac06d84SQu Wenruo } 648ff8466dSQu Wenruo 658ff8466dSQu Wenruo /* 668ff8466dSQu Wenruo * Increase the eb_refs of current subpage. 678ff8466dSQu Wenruo * 688ff8466dSQu Wenruo * This is important for eb allocation, to prevent race with last eb freeing 698ff8466dSQu Wenruo * of the same page. 708ff8466dSQu Wenruo * With the eb_refs increased before the eb inserted into radix tree, 718ff8466dSQu Wenruo * detach_extent_buffer_page() won't detach the page private while we're still 728ff8466dSQu Wenruo * allocating the extent buffer. 738ff8466dSQu Wenruo */ 748ff8466dSQu Wenruo void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, 758ff8466dSQu Wenruo struct page *page) 768ff8466dSQu Wenruo { 778ff8466dSQu Wenruo struct btrfs_subpage *subpage; 788ff8466dSQu Wenruo 798ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 808ff8466dSQu Wenruo return; 818ff8466dSQu Wenruo 828ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 838ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 848ff8466dSQu Wenruo 858ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 868ff8466dSQu Wenruo atomic_inc(&subpage->eb_refs); 878ff8466dSQu Wenruo } 888ff8466dSQu Wenruo 898ff8466dSQu Wenruo void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, 908ff8466dSQu Wenruo struct page *page) 918ff8466dSQu Wenruo { 928ff8466dSQu Wenruo struct btrfs_subpage *subpage; 938ff8466dSQu Wenruo 948ff8466dSQu Wenruo if (fs_info->sectorsize == PAGE_SIZE) 958ff8466dSQu Wenruo return; 968ff8466dSQu Wenruo 978ff8466dSQu Wenruo ASSERT(PagePrivate(page) && page->mapping); 988ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock); 998ff8466dSQu Wenruo 1008ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private; 1018ff8466dSQu Wenruo ASSERT(atomic_read(&subpage->eb_refs)); 1028ff8466dSQu Wenruo atomic_dec(&subpage->eb_refs); 1038ff8466dSQu Wenruo } 104a1d767c1SQu Wenruo 105a1d767c1SQu Wenruo /* 106a1d767c1SQu Wenruo * Convert the [start, start + len) range into a u16 bitmap 107a1d767c1SQu Wenruo * 108a1d767c1SQu Wenruo * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0. 109a1d767c1SQu Wenruo */ 110a1d767c1SQu Wenruo static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, 111a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 112a1d767c1SQu Wenruo { 113a1d767c1SQu Wenruo const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits; 114a1d767c1SQu Wenruo const int nbits = len >> fs_info->sectorsize_bits; 115a1d767c1SQu Wenruo 116a1d767c1SQu Wenruo /* Basic checks */ 117a1d767c1SQu Wenruo ASSERT(PagePrivate(page) && page->private); 118a1d767c1SQu Wenruo ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 119a1d767c1SQu Wenruo IS_ALIGNED(len, fs_info->sectorsize)); 120a1d767c1SQu Wenruo 121a1d767c1SQu Wenruo /* 122a1d767c1SQu Wenruo * The range check only works for mapped page, we can still have 123a1d767c1SQu Wenruo * unmapped page like dummy extent buffer pages. 124a1d767c1SQu Wenruo */ 125a1d767c1SQu Wenruo if (page->mapping) 126a1d767c1SQu Wenruo ASSERT(page_offset(page) <= start && 127a1d767c1SQu Wenruo start + len <= page_offset(page) + PAGE_SIZE); 128a1d767c1SQu Wenruo /* 129a1d767c1SQu Wenruo * Here nbits can be 16, thus can go beyond u16 range. We make the 130a1d767c1SQu Wenruo * first left shift to be calculate in unsigned long (at least u32), 131a1d767c1SQu Wenruo * then truncate the result to u16. 132a1d767c1SQu Wenruo */ 133a1d767c1SQu Wenruo return (u16)(((1UL << nbits) - 1) << bit_start); 134a1d767c1SQu Wenruo } 135a1d767c1SQu Wenruo 136a1d767c1SQu Wenruo void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 137a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 138a1d767c1SQu Wenruo { 139a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 140a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 141a1d767c1SQu Wenruo unsigned long flags; 142a1d767c1SQu Wenruo 143a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 144a1d767c1SQu Wenruo subpage->uptodate_bitmap |= tmp; 145a1d767c1SQu Wenruo if (subpage->uptodate_bitmap == U16_MAX) 146a1d767c1SQu Wenruo SetPageUptodate(page); 147a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 148a1d767c1SQu Wenruo } 149a1d767c1SQu Wenruo 150a1d767c1SQu Wenruo void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 151a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) 152a1d767c1SQu Wenruo { 153a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 154a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 155a1d767c1SQu Wenruo unsigned long flags; 156a1d767c1SQu Wenruo 157a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 158a1d767c1SQu Wenruo subpage->uptodate_bitmap &= ~tmp; 159a1d767c1SQu Wenruo ClearPageUptodate(page); 160a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 161a1d767c1SQu Wenruo } 162a1d767c1SQu Wenruo 163*03a816b3SQu Wenruo void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, 164*03a816b3SQu Wenruo struct page *page, u64 start, u32 len) 165*03a816b3SQu Wenruo { 166*03a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 167*03a816b3SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 168*03a816b3SQu Wenruo unsigned long flags; 169*03a816b3SQu Wenruo 170*03a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 171*03a816b3SQu Wenruo subpage->error_bitmap |= tmp; 172*03a816b3SQu Wenruo SetPageError(page); 173*03a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 174*03a816b3SQu Wenruo } 175*03a816b3SQu Wenruo 176*03a816b3SQu Wenruo void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, 177*03a816b3SQu Wenruo struct page *page, u64 start, u32 len) 178*03a816b3SQu Wenruo { 179*03a816b3SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 180*03a816b3SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 181*03a816b3SQu Wenruo unsigned long flags; 182*03a816b3SQu Wenruo 183*03a816b3SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); 184*03a816b3SQu Wenruo subpage->error_bitmap &= ~tmp; 185*03a816b3SQu Wenruo if (subpage->error_bitmap == 0) 186*03a816b3SQu Wenruo ClearPageError(page); 187*03a816b3SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); 188*03a816b3SQu Wenruo } 189*03a816b3SQu Wenruo 190a1d767c1SQu Wenruo /* 191a1d767c1SQu Wenruo * Unlike set/clear which is dependent on each page status, for test all bits 192a1d767c1SQu Wenruo * are tested in the same way. 193a1d767c1SQu Wenruo */ 194a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 195a1d767c1SQu Wenruo bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 196a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 197a1d767c1SQu Wenruo { \ 198a1d767c1SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ 199a1d767c1SQu Wenruo const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \ 200a1d767c1SQu Wenruo unsigned long flags; \ 201a1d767c1SQu Wenruo bool ret; \ 202a1d767c1SQu Wenruo \ 203a1d767c1SQu Wenruo spin_lock_irqsave(&subpage->lock, flags); \ 204a1d767c1SQu Wenruo ret = ((subpage->name##_bitmap & tmp) == tmp); \ 205a1d767c1SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags); \ 206a1d767c1SQu Wenruo return ret; \ 207a1d767c1SQu Wenruo } 208a1d767c1SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 209*03a816b3SQu Wenruo IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); 210a1d767c1SQu Wenruo 211a1d767c1SQu Wenruo /* 212a1d767c1SQu Wenruo * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 213a1d767c1SQu Wenruo * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 214a1d767c1SQu Wenruo * back to regular sectorsize branch. 215a1d767c1SQu Wenruo */ 216a1d767c1SQu Wenruo #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ 217a1d767c1SQu Wenruo test_page_func) \ 218a1d767c1SQu Wenruo void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ 219a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 220a1d767c1SQu Wenruo { \ 221a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 222a1d767c1SQu Wenruo set_page_func(page); \ 223a1d767c1SQu Wenruo return; \ 224a1d767c1SQu Wenruo } \ 225a1d767c1SQu Wenruo btrfs_subpage_set_##name(fs_info, page, start, len); \ 226a1d767c1SQu Wenruo } \ 227a1d767c1SQu Wenruo void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ 228a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 229a1d767c1SQu Wenruo { \ 230a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 231a1d767c1SQu Wenruo clear_page_func(page); \ 232a1d767c1SQu Wenruo return; \ 233a1d767c1SQu Wenruo } \ 234a1d767c1SQu Wenruo btrfs_subpage_clear_##name(fs_info, page, start, len); \ 235a1d767c1SQu Wenruo } \ 236a1d767c1SQu Wenruo bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ 237a1d767c1SQu Wenruo struct page *page, u64 start, u32 len) \ 238a1d767c1SQu Wenruo { \ 239a1d767c1SQu Wenruo if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 240a1d767c1SQu Wenruo return test_page_func(page); \ 241a1d767c1SQu Wenruo return btrfs_subpage_test_##name(fs_info, page, start, len); \ 242a1d767c1SQu Wenruo } 243a1d767c1SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, 244a1d767c1SQu Wenruo PageUptodate); 245*03a816b3SQu Wenruo IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); 246