1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/slab.h> 4 #include "ctree.h" 5 #include "subpage.h" 6 7 int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, 8 struct page *page, enum btrfs_subpage_type type) 9 { 10 struct btrfs_subpage *subpage = NULL; 11 int ret; 12 13 /* 14 * We have cases like a dummy extent buffer page, which is not mappped 15 * and doesn't need to be locked. 16 */ 17 if (page->mapping) 18 ASSERT(PageLocked(page)); 19 /* Either not subpage, or the page already has private attached */ 20 if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) 21 return 0; 22 23 ret = btrfs_alloc_subpage(fs_info, &subpage, type); 24 if (ret < 0) 25 return ret; 26 attach_page_private(page, subpage); 27 return 0; 28 } 29 30 void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, 31 struct page *page) 32 { 33 struct btrfs_subpage *subpage; 34 35 /* Either not subpage, or already detached */ 36 if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page)) 37 return; 38 39 subpage = (struct btrfs_subpage *)detach_page_private(page); 40 ASSERT(subpage); 41 btrfs_free_subpage(subpage); 42 } 43 44 int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, 45 struct btrfs_subpage **ret, 46 enum btrfs_subpage_type type) 47 { 48 if (fs_info->sectorsize == PAGE_SIZE) 49 return 0; 50 51 *ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS); 52 if (!*ret) 53 return -ENOMEM; 54 spin_lock_init(&(*ret)->lock); 55 if (type == BTRFS_SUBPAGE_METADATA) 56 atomic_set(&(*ret)->eb_refs, 0); 57 else 58 atomic_set(&(*ret)->readers, 0); 59 return 0; 60 } 61 62 void btrfs_free_subpage(struct btrfs_subpage *subpage) 63 { 64 kfree(subpage); 65 } 66 67 /* 68 * Increase the eb_refs of current subpage. 69 * 70 * This is important for eb allocation, to prevent race with last eb freeing 71 * of the same page. 72 * With the eb_refs increased before the eb inserted into radix tree, 73 * detach_extent_buffer_page() won't detach the page private while we're still 74 * allocating the extent buffer. 75 */ 76 void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, 77 struct page *page) 78 { 79 struct btrfs_subpage *subpage; 80 81 if (fs_info->sectorsize == PAGE_SIZE) 82 return; 83 84 ASSERT(PagePrivate(page) && page->mapping); 85 lockdep_assert_held(&page->mapping->private_lock); 86 87 subpage = (struct btrfs_subpage *)page->private; 88 atomic_inc(&subpage->eb_refs); 89 } 90 91 void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, 92 struct page *page) 93 { 94 struct btrfs_subpage *subpage; 95 96 if (fs_info->sectorsize == PAGE_SIZE) 97 return; 98 99 ASSERT(PagePrivate(page) && page->mapping); 100 lockdep_assert_held(&page->mapping->private_lock); 101 102 subpage = (struct btrfs_subpage *)page->private; 103 ASSERT(atomic_read(&subpage->eb_refs)); 104 atomic_dec(&subpage->eb_refs); 105 } 106 107 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, 108 struct page *page, u64 start, u32 len) 109 { 110 /* Basic checks */ 111 ASSERT(PagePrivate(page) && page->private); 112 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && 113 IS_ALIGNED(len, fs_info->sectorsize)); 114 /* 115 * The range check only works for mapped page, we can still have 116 * unmapped page like dummy extent buffer pages. 117 */ 118 if (page->mapping) 119 ASSERT(page_offset(page) <= start && 120 start + len <= page_offset(page) + PAGE_SIZE); 121 } 122 123 void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, 124 struct page *page, u64 start, u32 len) 125 { 126 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 127 const int nbits = len >> fs_info->sectorsize_bits; 128 int ret; 129 130 btrfs_subpage_assert(fs_info, page, start, len); 131 132 ret = atomic_add_return(nbits, &subpage->readers); 133 ASSERT(ret == nbits); 134 } 135 136 void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, 137 struct page *page, u64 start, u32 len) 138 { 139 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 140 const int nbits = len >> fs_info->sectorsize_bits; 141 142 btrfs_subpage_assert(fs_info, page, start, len); 143 ASSERT(atomic_read(&subpage->readers) >= nbits); 144 if (atomic_sub_and_test(nbits, &subpage->readers)) 145 unlock_page(page); 146 } 147 148 /* 149 * Convert the [start, start + len) range into a u16 bitmap 150 * 151 * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0. 152 */ 153 static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, 154 struct page *page, u64 start, u32 len) 155 { 156 const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits; 157 const int nbits = len >> fs_info->sectorsize_bits; 158 159 btrfs_subpage_assert(fs_info, page, start, len); 160 161 /* 162 * Here nbits can be 16, thus can go beyond u16 range. We make the 163 * first left shift to be calculate in unsigned long (at least u32), 164 * then truncate the result to u16. 165 */ 166 return (u16)(((1UL << nbits) - 1) << bit_start); 167 } 168 169 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, 170 struct page *page, u64 start, u32 len) 171 { 172 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 173 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 174 unsigned long flags; 175 176 spin_lock_irqsave(&subpage->lock, flags); 177 subpage->uptodate_bitmap |= tmp; 178 if (subpage->uptodate_bitmap == U16_MAX) 179 SetPageUptodate(page); 180 spin_unlock_irqrestore(&subpage->lock, flags); 181 } 182 183 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, 184 struct page *page, u64 start, u32 len) 185 { 186 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 187 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 188 unsigned long flags; 189 190 spin_lock_irqsave(&subpage->lock, flags); 191 subpage->uptodate_bitmap &= ~tmp; 192 ClearPageUptodate(page); 193 spin_unlock_irqrestore(&subpage->lock, flags); 194 } 195 196 void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, 197 struct page *page, u64 start, u32 len) 198 { 199 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 200 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 201 unsigned long flags; 202 203 spin_lock_irqsave(&subpage->lock, flags); 204 subpage->error_bitmap |= tmp; 205 SetPageError(page); 206 spin_unlock_irqrestore(&subpage->lock, flags); 207 } 208 209 void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, 210 struct page *page, u64 start, u32 len) 211 { 212 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; 213 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); 214 unsigned long flags; 215 216 spin_lock_irqsave(&subpage->lock, flags); 217 subpage->error_bitmap &= ~tmp; 218 if (subpage->error_bitmap == 0) 219 ClearPageError(page); 220 spin_unlock_irqrestore(&subpage->lock, flags); 221 } 222 223 /* 224 * Unlike set/clear which is dependent on each page status, for test all bits 225 * are tested in the same way. 226 */ 227 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ 228 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ 229 struct page *page, u64 start, u32 len) \ 230 { \ 231 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ 232 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \ 233 unsigned long flags; \ 234 bool ret; \ 235 \ 236 spin_lock_irqsave(&subpage->lock, flags); \ 237 ret = ((subpage->name##_bitmap & tmp) == tmp); \ 238 spin_unlock_irqrestore(&subpage->lock, flags); \ 239 return ret; \ 240 } 241 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); 242 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); 243 244 /* 245 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed 246 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall 247 * back to regular sectorsize branch. 248 */ 249 #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ 250 test_page_func) \ 251 void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ 252 struct page *page, u64 start, u32 len) \ 253 { \ 254 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 255 set_page_func(page); \ 256 return; \ 257 } \ 258 btrfs_subpage_set_##name(fs_info, page, start, len); \ 259 } \ 260 void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ 261 struct page *page, u64 start, u32 len) \ 262 { \ 263 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ 264 clear_page_func(page); \ 265 return; \ 266 } \ 267 btrfs_subpage_clear_##name(fs_info, page, start, len); \ 268 } \ 269 bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ 270 struct page *page, u64 start, u32 len) \ 271 { \ 272 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ 273 return test_page_func(page); \ 274 return btrfs_subpage_test_##name(fs_info, page, start, len); \ 275 } 276 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, 277 PageUptodate); 278 IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); 279