1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2c1d7c514SDavid Sterba
3d1310b2eSChris Mason #include <linux/bitops.h>
4d1310b2eSChris Mason #include <linux/slab.h>
5d1310b2eSChris Mason #include <linux/bio.h>
6d1310b2eSChris Mason #include <linux/mm.h>
7d1310b2eSChris Mason #include <linux/pagemap.h>
8d1310b2eSChris Mason #include <linux/page-flags.h>
9395cb57eSSweet Tea Dorminy #include <linux/sched/mm.h>
10d1310b2eSChris Mason #include <linux/spinlock.h>
11d1310b2eSChris Mason #include <linux/blkdev.h>
12d1310b2eSChris Mason #include <linux/swap.h>
13d1310b2eSChris Mason #include <linux/writeback.h>
14d1310b2eSChris Mason #include <linux/pagevec.h>
15268bb0ceSLinus Torvalds #include <linux/prefetch.h>
1614605409SBoris Burkov #include <linux/fsverity.h>
17cea62800SJohannes Thumshirn #include "misc.h"
18d1310b2eSChris Mason #include "extent_io.h"
199c7d3a54SJosef Bacik #include "extent-io-tree.h"
20d1310b2eSChris Mason #include "extent_map.h"
21902b22f3SDavid Woodhouse #include "ctree.h"
22902b22f3SDavid Woodhouse #include "btrfs_inode.h"
23103c1972SChristoph Hellwig #include "bio.h"
2421adbd5cSStefan Behrens #include "check-integrity.h"
250b32f4bbSJosef Bacik #include "locking.h"
26606686eeSJosef Bacik #include "rcu-string.h"
27fe09e16cSLiu Bo #include "backref.h"
286af49dbdSDavid Sterba #include "disk-io.h"
29760f991fSQu Wenruo #include "subpage.h"
30d3575156SNaohiro Aota #include "zoned.h"
310bc09ca1SNaohiro Aota #include "block-group.h"
322a5232a8SDavid Sterba #include "compression.h"
33ec8eb376SJosef Bacik #include "fs.h"
3407e81dc9SJosef Bacik #include "accessors.h"
357c8ede16SJosef Bacik #include "file-item.h"
36af142b6fSJosef Bacik #include "file.h"
3777407dc0SJosef Bacik #include "dev-replace.h"
387f0add25SJosef Bacik #include "super.h"
3998c8d683SJosef Bacik #include "transaction.h"
40d1310b2eSChris Mason
41d1310b2eSChris Mason static struct kmem_cache *extent_buffer_cache;
42d1310b2eSChris Mason
436d49ba1bSEric Sandeen #ifdef CONFIG_BTRFS_DEBUG
btrfs_leak_debug_add_eb(struct extent_buffer * eb)44a40246e8SJosef Bacik static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
456d49ba1bSEric Sandeen {
46a40246e8SJosef Bacik struct btrfs_fs_info *fs_info = eb->fs_info;
476d49ba1bSEric Sandeen unsigned long flags;
486d49ba1bSEric Sandeen
49a40246e8SJosef Bacik spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
50a40246e8SJosef Bacik list_add(&eb->leak_list, &fs_info->allocated_ebs);
51a40246e8SJosef Bacik spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
526d49ba1bSEric Sandeen }
536d49ba1bSEric Sandeen
btrfs_leak_debug_del_eb(struct extent_buffer * eb)54a40246e8SJosef Bacik static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
55a40246e8SJosef Bacik {
56a40246e8SJosef Bacik struct btrfs_fs_info *fs_info = eb->fs_info;
57a40246e8SJosef Bacik unsigned long flags;
58a40246e8SJosef Bacik
59a40246e8SJosef Bacik spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
60a40246e8SJosef Bacik list_del(&eb->leak_list);
61a40246e8SJosef Bacik spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
62a40246e8SJosef Bacik }
63a40246e8SJosef Bacik
btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info * fs_info)643fd63727SJosef Bacik void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
6533ca832fSJosef Bacik {
6633ca832fSJosef Bacik struct extent_buffer *eb;
673fd63727SJosef Bacik unsigned long flags;
6833ca832fSJosef Bacik
698c38938cSJosef Bacik /*
708c38938cSJosef Bacik * If we didn't get into open_ctree our allocated_ebs will not be
718c38938cSJosef Bacik * initialized, so just skip this.
728c38938cSJosef Bacik */
738c38938cSJosef Bacik if (!fs_info->allocated_ebs.next)
748c38938cSJosef Bacik return;
758c38938cSJosef Bacik
76b95b78e6SQu Wenruo WARN_ON(!list_empty(&fs_info->allocated_ebs));
773fd63727SJosef Bacik spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
783fd63727SJosef Bacik while (!list_empty(&fs_info->allocated_ebs)) {
793fd63727SJosef Bacik eb = list_first_entry(&fs_info->allocated_ebs,
803fd63727SJosef Bacik struct extent_buffer, leak_list);
818c38938cSJosef Bacik pr_err(
828c38938cSJosef Bacik "BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
838c38938cSJosef Bacik eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
848c38938cSJosef Bacik btrfs_header_owner(eb));
8533ca832fSJosef Bacik list_del(&eb->leak_list);
8633ca832fSJosef Bacik kmem_cache_free(extent_buffer_cache, eb);
8733ca832fSJosef Bacik }
883fd63727SJosef Bacik spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
8933ca832fSJosef Bacik }
906d49ba1bSEric Sandeen #else
91a40246e8SJosef Bacik #define btrfs_leak_debug_add_eb(eb) do {} while (0)
92a40246e8SJosef Bacik #define btrfs_leak_debug_del_eb(eb) do {} while (0)
934bef0848SChris Mason #endif
94d1310b2eSChris Mason
957aab8b32SChristoph Hellwig /*
967aab8b32SChristoph Hellwig * Structure to record info about the bio being assembled, and other info like
977aab8b32SChristoph Hellwig * how many bytes are there before stripe/ordered extent boundary.
987aab8b32SChristoph Hellwig */
997aab8b32SChristoph Hellwig struct btrfs_bio_ctrl {
1009dfde1b4SChristoph Hellwig struct btrfs_bio *bbio;
1010f07003bSDavid Sterba enum btrfs_compression_type compress_type;
1027aab8b32SChristoph Hellwig u32 len_to_oe_boundary;
103c000bc04SChristoph Hellwig blk_opf_t opf;
1045467abbaSQu Wenruo btrfs_bio_end_io_t end_io_func;
10572b505dcSChristoph Hellwig struct writeback_control *wbc;
106d1310b2eSChris Mason };
107d1310b2eSChris Mason
submit_one_bio(struct btrfs_bio_ctrl * bio_ctrl)108722c82acSChristoph Hellwig static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
109bb58eb9eSQu Wenruo {
1109dfde1b4SChristoph Hellwig struct btrfs_bio *bbio = bio_ctrl->bbio;
111bb58eb9eSQu Wenruo
1129dfde1b4SChristoph Hellwig if (!bbio)
113722c82acSChristoph Hellwig return;
114722c82acSChristoph Hellwig
115e0eefe07SQu Wenruo /* Caller should ensure the bio has at least some range added */
1169dfde1b4SChristoph Hellwig ASSERT(bbio->bio.bi_iter.bi_size);
117c9583adaSQu Wenruo
1189dfde1b4SChristoph Hellwig if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ &&
11935a8d7daSChristoph Hellwig bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
120e1949310SChristoph Hellwig btrfs_submit_compressed_read(bbio);
12135a8d7daSChristoph Hellwig else
122b78b98e0SChristoph Hellwig btrfs_submit_bio(bbio, 0);
12335a8d7daSChristoph Hellwig
1249dfde1b4SChristoph Hellwig /* The bbio is owned by the end_io handler now */
1259dfde1b4SChristoph Hellwig bio_ctrl->bbio = NULL;
126bb58eb9eSQu Wenruo }
127bb58eb9eSQu Wenruo
1289845e5ddSChristoph Hellwig /*
129ee5f017dSDavid Sterba * Submit or fail the current bio in the bio_ctrl structure.
1309845e5ddSChristoph Hellwig */
submit_write_bio(struct btrfs_bio_ctrl * bio_ctrl,int ret)131ee5f017dSDavid Sterba static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
1323065976bSQu Wenruo {
1339dfde1b4SChristoph Hellwig struct btrfs_bio *bbio = bio_ctrl->bbio;
134390ed29bSQu Wenruo
1359dfde1b4SChristoph Hellwig if (!bbio)
1369845e5ddSChristoph Hellwig return;
1379845e5ddSChristoph Hellwig
1389845e5ddSChristoph Hellwig if (ret) {
1399845e5ddSChristoph Hellwig ASSERT(ret < 0);
1409dfde1b4SChristoph Hellwig btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
141917f32a2SChristoph Hellwig /* The bio is owned by the end_io handler now */
1429dfde1b4SChristoph Hellwig bio_ctrl->bbio = NULL;
143722c82acSChristoph Hellwig } else {
144ee5f017dSDavid Sterba submit_one_bio(bio_ctrl);
145bb58eb9eSQu Wenruo }
146bb58eb9eSQu Wenruo }
147e2932ee0SDavid Sterba
extent_buffer_init_cachep(void)148a62a3bd9SJosef Bacik int __init extent_buffer_init_cachep(void)
149a62a3bd9SJosef Bacik {
150837e1972SDavid Sterba extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
151d1310b2eSChris Mason sizeof(struct extent_buffer), 0,
152fba4b697SNikolay Borisov SLAB_MEM_SPREAD, NULL);
153a62a3bd9SJosef Bacik if (!extent_buffer_cache)
154d1310b2eSChris Mason return -ENOMEM;
155d1310b2eSChris Mason
156d45cfb88SChristoph Hellwig return 0;
1576f0d04f8SJosef Bacik }
1586f0d04f8SJosef Bacik
extent_buffer_free_cachep(void)159a62a3bd9SJosef Bacik void __cold extent_buffer_free_cachep(void)
160d1310b2eSChris Mason {
1618c0a8537SKirill A. Shutemov /*
1628c0a8537SKirill A. Shutemov * Make sure all delayed rcu free are flushed before we
1638c0a8537SKirill A. Shutemov * destroy caches.
1648c0a8537SKirill A. Shutemov */
1658c0a8537SKirill A. Shutemov rcu_barrier();
166d1310b2eSChris Mason kmem_cache_destroy(extent_buffer_cache);
167d1310b2eSChris Mason }
168d1310b2eSChris Mason
extent_range_clear_dirty_for_io(struct inode * inode,u64 start,u64 end)169bd1fa4f0SDavid Sterba void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1704adaa611SChris Mason {
17109cbfeafSKirill A. Shutemov unsigned long index = start >> PAGE_SHIFT;
17209cbfeafSKirill A. Shutemov unsigned long end_index = end >> PAGE_SHIFT;
1734adaa611SChris Mason struct page *page;
1744adaa611SChris Mason
1754adaa611SChris Mason while (index <= end_index) {
1764adaa611SChris Mason page = find_get_page(inode->i_mapping, index);
1774adaa611SChris Mason BUG_ON(!page); /* Pages should be in the extent_io_tree */
1784adaa611SChris Mason clear_page_dirty_for_io(page);
17909cbfeafSKirill A. Shutemov put_page(page);
1804adaa611SChris Mason index++;
1814adaa611SChris Mason }
1824adaa611SChris Mason }
1834adaa611SChris Mason
process_one_page(struct btrfs_fs_info * fs_info,struct page * page,struct page * locked_page,unsigned long page_ops,u64 start,u64 end)184ef4e88e6SChristoph Hellwig static void process_one_page(struct btrfs_fs_info *fs_info,
185ed8f13bfSQu Wenruo struct page *page, struct page *locked_page,
186e38992beSQu Wenruo unsigned long page_ops, u64 start, u64 end)
187ed8f13bfSQu Wenruo {
188e38992beSQu Wenruo u32 len;
189e38992beSQu Wenruo
190e38992beSQu Wenruo ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
191e38992beSQu Wenruo len = end + 1 - start;
192e38992beSQu Wenruo
193ed8f13bfSQu Wenruo if (page_ops & PAGE_SET_ORDERED)
194b945a463SQu Wenruo btrfs_page_clamp_set_ordered(fs_info, page, start, len);
195ed8f13bfSQu Wenruo if (page_ops & PAGE_START_WRITEBACK) {
196e38992beSQu Wenruo btrfs_page_clamp_clear_dirty(fs_info, page, start, len);
197e38992beSQu Wenruo btrfs_page_clamp_set_writeback(fs_info, page, start, len);
198ed8f13bfSQu Wenruo }
199ed8f13bfSQu Wenruo if (page_ops & PAGE_END_WRITEBACK)
200e38992beSQu Wenruo btrfs_page_clamp_clear_writeback(fs_info, page, start, len);
201a33a8e9aSQu Wenruo
202ef4e88e6SChristoph Hellwig if (page != locked_page && (page_ops & PAGE_UNLOCK))
2031e1de387SQu Wenruo btrfs_page_end_writer_lock(fs_info, page, start, len);
204ed8f13bfSQu Wenruo }
205ed8f13bfSQu Wenruo
__process_pages_contig(struct address_space * mapping,struct page * locked_page,u64 start,u64 end,unsigned long page_ops)206ef4e88e6SChristoph Hellwig static void __process_pages_contig(struct address_space *mapping,
207ef4e88e6SChristoph Hellwig struct page *locked_page, u64 start, u64 end,
208ef4e88e6SChristoph Hellwig unsigned long page_ops)
209ed8f13bfSQu Wenruo {
210e38992beSQu Wenruo struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
211ed8f13bfSQu Wenruo pgoff_t start_index = start >> PAGE_SHIFT;
212ed8f13bfSQu Wenruo pgoff_t end_index = end >> PAGE_SHIFT;
213ed8f13bfSQu Wenruo pgoff_t index = start_index;
21404c6b79aSVishal Moola (Oracle) struct folio_batch fbatch;
215ed8f13bfSQu Wenruo int i;
216ed8f13bfSQu Wenruo
21704c6b79aSVishal Moola (Oracle) folio_batch_init(&fbatch);
21804c6b79aSVishal Moola (Oracle) while (index <= end_index) {
21904c6b79aSVishal Moola (Oracle) int found_folios;
220ed8f13bfSQu Wenruo
22104c6b79aSVishal Moola (Oracle) found_folios = filemap_get_folios_contig(mapping, &index,
22204c6b79aSVishal Moola (Oracle) end_index, &fbatch);
22304c6b79aSVishal Moola (Oracle) for (i = 0; i < found_folios; i++) {
22404c6b79aSVishal Moola (Oracle) struct folio *folio = fbatch.folios[i];
225ef4e88e6SChristoph Hellwig
226ef4e88e6SChristoph Hellwig process_one_page(fs_info, &folio->page, locked_page,
227ef4e88e6SChristoph Hellwig page_ops, start, end);
228ed8f13bfSQu Wenruo }
22904c6b79aSVishal Moola (Oracle) folio_batch_release(&fbatch);
230ed8f13bfSQu Wenruo cond_resched();
231ed8f13bfSQu Wenruo }
232ed8f13bfSQu Wenruo }
233da2c7009SLiu Bo
__unlock_for_delalloc(struct inode * inode,struct page * locked_page,u64 start,u64 end)234143bede5SJeff Mahoney static noinline void __unlock_for_delalloc(struct inode *inode,
235c8b97818SChris Mason struct page *locked_page,
236c8b97818SChris Mason u64 start, u64 end)
237c8b97818SChris Mason {
23809cbfeafSKirill A. Shutemov unsigned long index = start >> PAGE_SHIFT;
23909cbfeafSKirill A. Shutemov unsigned long end_index = end >> PAGE_SHIFT;
240c8b97818SChris Mason
24176c0021dSLiu Bo ASSERT(locked_page);
242c8b97818SChris Mason if (index == locked_page->index && end_index == index)
243143bede5SJeff Mahoney return;
244c8b97818SChris Mason
24598af9ab1SQu Wenruo __process_pages_contig(inode->i_mapping, locked_page, start, end,
246ef4e88e6SChristoph Hellwig PAGE_UNLOCK);
247c8b97818SChris Mason }
248c8b97818SChris Mason
lock_delalloc_pages(struct inode * inode,struct page * locked_page,u64 start,u64 end)249c8b97818SChris Mason static noinline int lock_delalloc_pages(struct inode *inode,
250c8b97818SChris Mason struct page *locked_page,
251ef4e88e6SChristoph Hellwig u64 start,
252ef4e88e6SChristoph Hellwig u64 end)
253c8b97818SChris Mason {
254ef4e88e6SChristoph Hellwig struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
255ef4e88e6SChristoph Hellwig struct address_space *mapping = inode->i_mapping;
256ef4e88e6SChristoph Hellwig pgoff_t start_index = start >> PAGE_SHIFT;
257ef4e88e6SChristoph Hellwig pgoff_t end_index = end >> PAGE_SHIFT;
258ef4e88e6SChristoph Hellwig pgoff_t index = start_index;
259ef4e88e6SChristoph Hellwig u64 processed_end = start;
260ef4e88e6SChristoph Hellwig struct folio_batch fbatch;
261c8b97818SChris Mason
262c8b97818SChris Mason if (index == locked_page->index && index == end_index)
263c8b97818SChris Mason return 0;
264c8b97818SChris Mason
265ef4e88e6SChristoph Hellwig folio_batch_init(&fbatch);
266ef4e88e6SChristoph Hellwig while (index <= end_index) {
267ef4e88e6SChristoph Hellwig unsigned int found_folios, i;
268ef4e88e6SChristoph Hellwig
269ef4e88e6SChristoph Hellwig found_folios = filemap_get_folios_contig(mapping, &index,
270ef4e88e6SChristoph Hellwig end_index, &fbatch);
271ef4e88e6SChristoph Hellwig if (found_folios == 0)
272ef4e88e6SChristoph Hellwig goto out;
273ef4e88e6SChristoph Hellwig
274ef4e88e6SChristoph Hellwig for (i = 0; i < found_folios; i++) {
275ef4e88e6SChristoph Hellwig struct page *page = &fbatch.folios[i]->page;
276ef4e88e6SChristoph Hellwig u32 len = end + 1 - start;
277ef4e88e6SChristoph Hellwig
278ef4e88e6SChristoph Hellwig if (page == locked_page)
279ef4e88e6SChristoph Hellwig continue;
280ef4e88e6SChristoph Hellwig
281ef4e88e6SChristoph Hellwig if (btrfs_page_start_writer_lock(fs_info, page, start,
282ef4e88e6SChristoph Hellwig len))
283ef4e88e6SChristoph Hellwig goto out;
284ef4e88e6SChristoph Hellwig
285ef4e88e6SChristoph Hellwig if (!PageDirty(page) || page->mapping != mapping) {
286ef4e88e6SChristoph Hellwig btrfs_page_end_writer_lock(fs_info, page, start,
287ef4e88e6SChristoph Hellwig len);
288ef4e88e6SChristoph Hellwig goto out;
289ef4e88e6SChristoph Hellwig }
290ef4e88e6SChristoph Hellwig
291ef4e88e6SChristoph Hellwig processed_end = page_offset(page) + PAGE_SIZE - 1;
292ef4e88e6SChristoph Hellwig }
293ef4e88e6SChristoph Hellwig folio_batch_release(&fbatch);
294ef4e88e6SChristoph Hellwig cond_resched();
295ef4e88e6SChristoph Hellwig }
296ef4e88e6SChristoph Hellwig
297ef4e88e6SChristoph Hellwig return 0;
298ef4e88e6SChristoph Hellwig out:
299ef4e88e6SChristoph Hellwig folio_batch_release(&fbatch);
300ef4e88e6SChristoph Hellwig if (processed_end > start)
301ef4e88e6SChristoph Hellwig __unlock_for_delalloc(inode, locked_page, start, processed_end);
302ef4e88e6SChristoph Hellwig return -EAGAIN;
303c8b97818SChris Mason }
304c8b97818SChris Mason
305c8b97818SChris Mason /*
3063522e903SLu Fengqi * Find and lock a contiguous range of bytes in the file marked as delalloc, no
3072749f7efSQu Wenruo * more than @max_bytes.
308c8b97818SChris Mason *
3092749f7efSQu Wenruo * @start: The original start bytenr to search.
3102749f7efSQu Wenruo * Will store the extent range start bytenr.
3112749f7efSQu Wenruo * @end: The original end bytenr of the search range
3122749f7efSQu Wenruo * Will store the extent range end bytenr.
3132749f7efSQu Wenruo *
3142749f7efSQu Wenruo * Return true if we find a delalloc range which starts inside the original
3152749f7efSQu Wenruo * range, and @start/@end will store the delalloc range start/end.
3162749f7efSQu Wenruo *
3172749f7efSQu Wenruo * Return false if we can't find any delalloc range which starts inside the
3182749f7efSQu Wenruo * original range, and @start/@end will be the non-delalloc range start/end.
319c8b97818SChris Mason */
320ce9f967fSJohannes Thumshirn EXPORT_FOR_TESTS
find_lock_delalloc_range(struct inode * inode,struct page * locked_page,u64 * start,u64 * end)3213522e903SLu Fengqi noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
322294e30feSJosef Bacik struct page *locked_page, u64 *start,
323917aacecSNikolay Borisov u64 *end)
324c8b97818SChris Mason {
325f7b12a62SNaohiro Aota struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3269978059bSGoldwyn Rodrigues struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
3272749f7efSQu Wenruo const u64 orig_start = *start;
3282749f7efSQu Wenruo const u64 orig_end = *end;
329f7b12a62SNaohiro Aota /* The sanity tests may not set a valid fs_info. */
330f7b12a62SNaohiro Aota u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
331c8b97818SChris Mason u64 delalloc_start;
332c8b97818SChris Mason u64 delalloc_end;
3333522e903SLu Fengqi bool found;
3349655d298SChris Mason struct extent_state *cached_state = NULL;
335c8b97818SChris Mason int ret;
336c8b97818SChris Mason int loops = 0;
337c8b97818SChris Mason
3382749f7efSQu Wenruo /* Caller should pass a valid @end to indicate the search range end */
3392749f7efSQu Wenruo ASSERT(orig_end > orig_start);
3402749f7efSQu Wenruo
3412749f7efSQu Wenruo /* The range should at least cover part of the page */
3422749f7efSQu Wenruo ASSERT(!(orig_start >= page_offset(locked_page) + PAGE_SIZE ||
3432749f7efSQu Wenruo orig_end <= page_offset(locked_page)));
344c8b97818SChris Mason again:
345c8b97818SChris Mason /* step one, find a bunch of delalloc bytes starting at start */
346c8b97818SChris Mason delalloc_start = *start;
347c8b97818SChris Mason delalloc_end = 0;
348083e75e7SJosef Bacik found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
349c2a128d2SJosef Bacik max_bytes, &cached_state);
3502749f7efSQu Wenruo if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
351c8b97818SChris Mason *start = delalloc_start;
3522749f7efSQu Wenruo
3532749f7efSQu Wenruo /* @delalloc_end can be -1, never go beyond @orig_end */
3542749f7efSQu Wenruo *end = min(delalloc_end, orig_end);
355c2a128d2SJosef Bacik free_extent_state(cached_state);
3563522e903SLu Fengqi return false;
357c8b97818SChris Mason }
358c8b97818SChris Mason
359c8b97818SChris Mason /*
36070b99e69SChris Mason * start comes from the offset of locked_page. We have to lock
36170b99e69SChris Mason * pages in order, so we can't process delalloc bytes before
36270b99e69SChris Mason * locked_page
36370b99e69SChris Mason */
364d397712bSChris Mason if (delalloc_start < *start)
36570b99e69SChris Mason delalloc_start = *start;
36670b99e69SChris Mason
36770b99e69SChris Mason /*
368c8b97818SChris Mason * make sure to limit the number of pages we try to lock down
369c8b97818SChris Mason */
3707bf811a5SJosef Bacik if (delalloc_end + 1 - delalloc_start > max_bytes)
3717bf811a5SJosef Bacik delalloc_end = delalloc_start + max_bytes - 1;
372d397712bSChris Mason
373c8b97818SChris Mason /* step two, lock all the pages after the page that has start */
374c8b97818SChris Mason ret = lock_delalloc_pages(inode, locked_page,
375c8b97818SChris Mason delalloc_start, delalloc_end);
3769bfd61d9SNikolay Borisov ASSERT(!ret || ret == -EAGAIN);
377c8b97818SChris Mason if (ret == -EAGAIN) {
378c8b97818SChris Mason /* some of the pages are gone, lets avoid looping by
379c8b97818SChris Mason * shortening the size of the delalloc range we're searching
380c8b97818SChris Mason */
3819655d298SChris Mason free_extent_state(cached_state);
3827d788742SChris Mason cached_state = NULL;
383c8b97818SChris Mason if (!loops) {
38409cbfeafSKirill A. Shutemov max_bytes = PAGE_SIZE;
385c8b97818SChris Mason loops = 1;
386c8b97818SChris Mason goto again;
387c8b97818SChris Mason } else {
3883522e903SLu Fengqi found = false;
389c8b97818SChris Mason goto out_failed;
390c8b97818SChris Mason }
391c8b97818SChris Mason }
392c8b97818SChris Mason
393c8b97818SChris Mason /* step three, lock the state bits for the whole range */
394570eb97bSJosef Bacik lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
395c8b97818SChris Mason
396c8b97818SChris Mason /* then test to make sure it is all still delalloc */
397c8b97818SChris Mason ret = test_range_bit(tree, delalloc_start, delalloc_end,
3989655d298SChris Mason EXTENT_DELALLOC, 1, cached_state);
399c8b97818SChris Mason if (!ret) {
400570eb97bSJosef Bacik unlock_extent(tree, delalloc_start, delalloc_end,
401e43bbe5eSDavid Sterba &cached_state);
402c8b97818SChris Mason __unlock_for_delalloc(inode, locked_page,
403c8b97818SChris Mason delalloc_start, delalloc_end);
404c8b97818SChris Mason cond_resched();
405c8b97818SChris Mason goto again;
406c8b97818SChris Mason }
4079655d298SChris Mason free_extent_state(cached_state);
408c8b97818SChris Mason *start = delalloc_start;
409c8b97818SChris Mason *end = delalloc_end;
410c8b97818SChris Mason out_failed:
411c8b97818SChris Mason return found;
412c8b97818SChris Mason }
413c8b97818SChris Mason
extent_clear_unlock_delalloc(struct btrfs_inode * inode,u64 start,u64 end,struct page * locked_page,u32 clear_bits,unsigned long page_ops)414ad7ff17bSNikolay Borisov void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
41574e9194aSNikolay Borisov struct page *locked_page,
416f97e27e9SQu Wenruo u32 clear_bits, unsigned long page_ops)
417873695b3SLiu Bo {
418bd015294SJosef Bacik clear_extent_bit(&inode->io_tree, start, end, clear_bits, NULL);
419873695b3SLiu Bo
420ad7ff17bSNikolay Borisov __process_pages_contig(inode->vfs_inode.i_mapping, locked_page,
421ef4e88e6SChristoph Hellwig start, end, page_ops);
422873695b3SLiu Bo }
423873695b3SLiu Bo
btrfs_verify_page(struct page * page,u64 start)424ed9ee98eSChristoph Hellwig static bool btrfs_verify_page(struct page *page, u64 start)
425ed9ee98eSChristoph Hellwig {
426ed9ee98eSChristoph Hellwig if (!fsverity_active(page->mapping->host) ||
42757201dddSChristoph Hellwig PageUptodate(page) ||
428ed9ee98eSChristoph Hellwig start >= i_size_read(page->mapping->host))
429ed9ee98eSChristoph Hellwig return true;
430ed9ee98eSChristoph Hellwig return fsverity_verify_page(page);
431ed9ee98eSChristoph Hellwig }
432ed9ee98eSChristoph Hellwig
end_page_read(struct page * page,bool uptodate,u64 start,u32 len)433150e4b05SQu Wenruo static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
434150e4b05SQu Wenruo {
435150e4b05SQu Wenruo struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
436150e4b05SQu Wenruo
437150e4b05SQu Wenruo ASSERT(page_offset(page) <= start &&
438150e4b05SQu Wenruo start + len <= page_offset(page) + PAGE_SIZE);
439150e4b05SQu Wenruo
4402b2553f1SChristoph Hellwig if (uptodate && btrfs_verify_page(page, start))
441150e4b05SQu Wenruo btrfs_page_set_uptodate(fs_info, page, start, len);
4422b2553f1SChristoph Hellwig else
443150e4b05SQu Wenruo btrfs_page_clear_uptodate(fs_info, page, start, len);
444150e4b05SQu Wenruo
445fbca46ebSQu Wenruo if (!btrfs_is_subpage(fs_info, page))
446150e4b05SQu Wenruo unlock_page(page);
4473d078efaSQu Wenruo else
448150e4b05SQu Wenruo btrfs_subpage_end_reader(fs_info, page, start, len);
449150e4b05SQu Wenruo }
450150e4b05SQu Wenruo
451d1310b2eSChris Mason /*
452d1310b2eSChris Mason * after a writepage IO is done, we need to:
453d1310b2eSChris Mason * clear the uptodate bits on error
454d1310b2eSChris Mason * clear the writeback bits in the extent tree for this IO
455d1310b2eSChris Mason * end_page_writeback if the page has no more pending IO
456d1310b2eSChris Mason *
457d1310b2eSChris Mason * Scheduling is not allowed, so the extent state tree is expected
458d1310b2eSChris Mason * to have one and only one object corresponding to this IO.
459d1310b2eSChris Mason */
end_bio_extent_writepage(struct btrfs_bio * bbio)460917f32a2SChristoph Hellwig static void end_bio_extent_writepage(struct btrfs_bio *bbio)
461d1310b2eSChris Mason {
462917f32a2SChristoph Hellwig struct bio *bio = &bbio->bio;
4634e4cbee9SChristoph Hellwig int error = blk_status_to_errno(bio->bi_status);
4642c30c71bSKent Overstreet struct bio_vec *bvec;
4656dc4f100SMing Lei struct bvec_iter_all iter_all;
466d1310b2eSChris Mason
467c09abff8SDavid Sterba ASSERT(!bio_flagged(bio, BIO_CLONED));
4682b070cfeSChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) {
469d1310b2eSChris Mason struct page *page = bvec->bv_page;
4700b246afaSJeff Mahoney struct inode *inode = page->mapping->host;
4710b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
472321a02dbSQu Wenruo const u32 sectorsize = fs_info->sectorsize;
4734ba8223dSChristoph Hellwig u64 start = page_offset(page) + bvec->bv_offset;
4744ba8223dSChristoph Hellwig u32 len = bvec->bv_len;
475902b22f3SDavid Woodhouse
476321a02dbSQu Wenruo /* Our read/write should always be sector aligned. */
477321a02dbSQu Wenruo if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
4780b246afaSJeff Mahoney btrfs_err(fs_info,
479efe120a0SFrank Holton "partial page write in btrfs with offset %u and length %u",
48017a5adccSAlexandre Oliva bvec->bv_offset, bvec->bv_len);
481321a02dbSQu Wenruo else if (!IS_ALIGNED(bvec->bv_len, sectorsize))
4820b246afaSJeff Mahoney btrfs_info(fs_info,
483321a02dbSQu Wenruo "incomplete page write with offset %u and length %u",
484efe120a0SFrank Holton bvec->bv_offset, bvec->bv_len);
485d1310b2eSChris Mason
4860d394ccaSChristoph Hellwig btrfs_finish_ordered_extent(bbio->ordered, page, start, len, !error);
487b595d259SJosef Bacik if (error)
4884ba8223dSChristoph Hellwig mapping_set_error(page->mapping, error);
4894ba8223dSChristoph Hellwig btrfs_page_clear_writeback(fs_info, page, start, len);
4902c30c71bSKent Overstreet }
4912b1f55b0SChris Mason
492d1310b2eSChris Mason bio_put(bio);
493d1310b2eSChris Mason }
494d1310b2eSChris Mason
49594e8c95cSQu Wenruo /*
49694e8c95cSQu Wenruo * Record previously processed extent range
49794e8c95cSQu Wenruo *
49894e8c95cSQu Wenruo * For endio_readpage_release_extent() to handle a full extent range, reducing
49994e8c95cSQu Wenruo * the extent io operations.
50094e8c95cSQu Wenruo */
50194e8c95cSQu Wenruo struct processed_extent {
50294e8c95cSQu Wenruo struct btrfs_inode *inode;
50394e8c95cSQu Wenruo /* Start of the range in @inode */
50494e8c95cSQu Wenruo u64 start;
5052e626e56SNigel Christian /* End of the range in @inode */
50694e8c95cSQu Wenruo u64 end;
50794e8c95cSQu Wenruo bool uptodate;
50894e8c95cSQu Wenruo };
50994e8c95cSQu Wenruo
51094e8c95cSQu Wenruo /*
51194e8c95cSQu Wenruo * Try to release processed extent range
51294e8c95cSQu Wenruo *
51394e8c95cSQu Wenruo * May not release the extent range right now if the current range is
51494e8c95cSQu Wenruo * contiguous to processed extent.
51594e8c95cSQu Wenruo *
51694e8c95cSQu Wenruo * Will release processed extent when any of @inode, @uptodate, the range is
51794e8c95cSQu Wenruo * no longer contiguous to the processed range.
51894e8c95cSQu Wenruo *
51994e8c95cSQu Wenruo * Passing @inode == NULL will force processed extent to be released.
52094e8c95cSQu Wenruo */
endio_readpage_release_extent(struct processed_extent * processed,struct btrfs_inode * inode,u64 start,u64 end,bool uptodate)52194e8c95cSQu Wenruo static void endio_readpage_release_extent(struct processed_extent *processed,
52294e8c95cSQu Wenruo struct btrfs_inode *inode, u64 start, u64 end,
52394e8c95cSQu Wenruo bool uptodate)
524883d0de4SMiao Xie {
525883d0de4SMiao Xie struct extent_state *cached = NULL;
52694e8c95cSQu Wenruo struct extent_io_tree *tree;
527883d0de4SMiao Xie
52894e8c95cSQu Wenruo /* The first extent, initialize @processed */
52994e8c95cSQu Wenruo if (!processed->inode)
53094e8c95cSQu Wenruo goto update;
53194e8c95cSQu Wenruo
53294e8c95cSQu Wenruo /*
53394e8c95cSQu Wenruo * Contiguous to processed extent, just uptodate the end.
53494e8c95cSQu Wenruo *
53594e8c95cSQu Wenruo * Several things to notice:
53694e8c95cSQu Wenruo *
53794e8c95cSQu Wenruo * - bio can be merged as long as on-disk bytenr is contiguous
53894e8c95cSQu Wenruo * This means we can have page belonging to other inodes, thus need to
53994e8c95cSQu Wenruo * check if the inode still matches.
54094e8c95cSQu Wenruo * - bvec can contain range beyond current page for multi-page bvec
54194e8c95cSQu Wenruo * Thus we need to do processed->end + 1 >= start check
54294e8c95cSQu Wenruo */
54394e8c95cSQu Wenruo if (processed->inode == inode && processed->uptodate == uptodate &&
54494e8c95cSQu Wenruo processed->end + 1 >= start && end >= processed->end) {
54594e8c95cSQu Wenruo processed->end = end;
54694e8c95cSQu Wenruo return;
54794e8c95cSQu Wenruo }
54894e8c95cSQu Wenruo
54994e8c95cSQu Wenruo tree = &processed->inode->io_tree;
55094e8c95cSQu Wenruo /*
55194e8c95cSQu Wenruo * Now we don't have range contiguous to the processed range, release
55294e8c95cSQu Wenruo * the processed range now.
55394e8c95cSQu Wenruo */
55448acc47dSJosef Bacik unlock_extent(tree, processed->start, processed->end, &cached);
55594e8c95cSQu Wenruo
55694e8c95cSQu Wenruo update:
55794e8c95cSQu Wenruo /* Update processed to current range */
55894e8c95cSQu Wenruo processed->inode = inode;
55994e8c95cSQu Wenruo processed->start = start;
56094e8c95cSQu Wenruo processed->end = end;
56194e8c95cSQu Wenruo processed->uptodate = uptodate;
562883d0de4SMiao Xie }
563883d0de4SMiao Xie
begin_page_read(struct btrfs_fs_info * fs_info,struct page * page)56492082d40SQu Wenruo static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
56592082d40SQu Wenruo {
56692082d40SQu Wenruo ASSERT(PageLocked(page));
567fbca46ebSQu Wenruo if (!btrfs_is_subpage(fs_info, page))
56892082d40SQu Wenruo return;
56992082d40SQu Wenruo
57092082d40SQu Wenruo ASSERT(PagePrivate(page));
57192082d40SQu Wenruo btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE);
57292082d40SQu Wenruo }
57392082d40SQu Wenruo
574d1310b2eSChris Mason /*
575d1310b2eSChris Mason * after a readpage IO is done, we need to:
576d1310b2eSChris Mason * clear the uptodate bits on error
577d1310b2eSChris Mason * set the uptodate bits if things worked
578d1310b2eSChris Mason * set the page up to date if all extents in the tree are uptodate
579d1310b2eSChris Mason * clear the lock bit in the extent tree
580d1310b2eSChris Mason * unlock the page if there are no other extents locked for it
581d1310b2eSChris Mason *
582d1310b2eSChris Mason * Scheduling is not allowed, so the extent state tree is expected
583d1310b2eSChris Mason * to have one and only one object corresponding to this IO.
584d1310b2eSChris Mason */
end_bio_extent_readpage(struct btrfs_bio * bbio)585917f32a2SChristoph Hellwig static void end_bio_extent_readpage(struct btrfs_bio *bbio)
586d1310b2eSChris Mason {
587917f32a2SChristoph Hellwig struct bio *bio = &bbio->bio;
5882c30c71bSKent Overstreet struct bio_vec *bvec;
58994e8c95cSQu Wenruo struct processed_extent processed = { 0 };
5907ffd27e3SQu Wenruo /*
5917ffd27e3SQu Wenruo * The offset to the beginning of a bio, since one bio can never be
5927ffd27e3SQu Wenruo * larger than UINT_MAX, u32 here is enough.
5937ffd27e3SQu Wenruo */
5947ffd27e3SQu Wenruo u32 bio_offset = 0;
5956dc4f100SMing Lei struct bvec_iter_all iter_all;
596d1310b2eSChris Mason
597c09abff8SDavid Sterba ASSERT(!bio_flagged(bio, BIO_CLONED));
5982b070cfeSChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) {
599150e4b05SQu Wenruo bool uptodate = !bio->bi_status;
600d1310b2eSChris Mason struct page *page = bvec->bv_page;
601a71754fcSJosef Bacik struct inode *inode = page->mapping->host;
602ab8d0fc4SJeff Mahoney struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6037ffd27e3SQu Wenruo const u32 sectorsize = fs_info->sectorsize;
6047ffd27e3SQu Wenruo u64 start;
6057ffd27e3SQu Wenruo u64 end;
6067ffd27e3SQu Wenruo u32 len;
607507903b8SArne Jansen
608ab8d0fc4SJeff Mahoney btrfs_debug(fs_info,
609ab8d0fc4SJeff Mahoney "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
6101201b58bSDavid Sterba bio->bi_iter.bi_sector, bio->bi_status,
611c3a3b19bSQu Wenruo bbio->mirror_num);
612902b22f3SDavid Woodhouse
6138b8bbd46SQu Wenruo /*
6148b8bbd46SQu Wenruo * We always issue full-sector reads, but if some block in a
6158b8bbd46SQu Wenruo * page fails to read, blk_update_request() will advance
6168b8bbd46SQu Wenruo * bv_offset and adjust bv_len to compensate. Print a warning
6178b8bbd46SQu Wenruo * for unaligned offsets, and an error if they don't add up to
6188b8bbd46SQu Wenruo * a full sector.
6198b8bbd46SQu Wenruo */
6208b8bbd46SQu Wenruo if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
621ab8d0fc4SJeff Mahoney btrfs_err(fs_info,
622efe120a0SFrank Holton "partial page read in btrfs with offset %u and length %u",
62317a5adccSAlexandre Oliva bvec->bv_offset, bvec->bv_len);
6248b8bbd46SQu Wenruo else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len,
6258b8bbd46SQu Wenruo sectorsize))
626ab8d0fc4SJeff Mahoney btrfs_info(fs_info,
6278b8bbd46SQu Wenruo "incomplete page read with offset %u and length %u",
628efe120a0SFrank Holton bvec->bv_offset, bvec->bv_len);
629d1310b2eSChris Mason
6308b8bbd46SQu Wenruo start = page_offset(page) + bvec->bv_offset;
6318b8bbd46SQu Wenruo end = start + bvec->bv_len - 1;
632facc8a22SMiao Xie len = bvec->bv_len;
633d1310b2eSChris Mason
634883d0de4SMiao Xie if (likely(uptodate)) {
635a71754fcSJosef Bacik loff_t i_size = i_size_read(inode);
63609cbfeafSKirill A. Shutemov pgoff_t end_index = i_size >> PAGE_SHIFT;
637a71754fcSJosef Bacik
638c28ea613SQu Wenruo /*
639c28ea613SQu Wenruo * Zero out the remaining part if this range straddles
640c28ea613SQu Wenruo * i_size.
641c28ea613SQu Wenruo *
642c28ea613SQu Wenruo * Here we should only zero the range inside the bvec,
643c28ea613SQu Wenruo * not touch anything else.
644c28ea613SQu Wenruo *
645c28ea613SQu Wenruo * NOTE: i_size is exclusive while end is inclusive.
646c28ea613SQu Wenruo */
647c28ea613SQu Wenruo if (page->index == end_index && i_size <= end) {
648c28ea613SQu Wenruo u32 zero_start = max(offset_in_page(i_size),
649d2dcc8edSQu Wenruo offset_in_page(start));
650c28ea613SQu Wenruo
651c28ea613SQu Wenruo zero_user_segment(page, zero_start,
652c28ea613SQu Wenruo offset_in_page(end) + 1);
653c28ea613SQu Wenruo }
65497861cd1SChristoph Hellwig }
65597861cd1SChristoph Hellwig
6567609afacSChristoph Hellwig /* Update page status and unlock. */
65792082d40SQu Wenruo end_page_read(page, uptodate, start, len);
65894e8c95cSQu Wenruo endio_readpage_release_extent(&processed, BTRFS_I(inode),
65931dd8c81SQu Wenruo start, end, uptodate);
66097861cd1SChristoph Hellwig
66197861cd1SChristoph Hellwig ASSERT(bio_offset + len > bio_offset);
66297861cd1SChristoph Hellwig bio_offset += len;
66397861cd1SChristoph Hellwig
66497861cd1SChristoph Hellwig }
66594e8c95cSQu Wenruo /* Release the last extent */
66694e8c95cSQu Wenruo endio_readpage_release_extent(&processed, NULL, 0, 0, false);
667d1310b2eSChris Mason bio_put(bio);
668d1310b2eSChris Mason }
669d1310b2eSChris Mason
67043dd529aSDavid Sterba /*
671dd137dd1SSweet Tea Dorminy * Populate every free slot in a provided array with pages.
672dd137dd1SSweet Tea Dorminy *
673dd137dd1SSweet Tea Dorminy * @nr_pages: number of pages to allocate
674dd137dd1SSweet Tea Dorminy * @page_array: the array to fill with pages; any existing non-null entries in
675dd137dd1SSweet Tea Dorminy * the array will be skipped
676dd137dd1SSweet Tea Dorminy *
677dd137dd1SSweet Tea Dorminy * Return: 0 if all pages were able to be allocated;
67833357859SQu Wenruo * -ENOMEM otherwise, the partially allocated pages would be freed and
67933357859SQu Wenruo * the array slots zeroed
680dd137dd1SSweet Tea Dorminy */
btrfs_alloc_page_array(unsigned int nr_pages,struct page ** page_array)681dd137dd1SSweet Tea Dorminy int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array)
682dd137dd1SSweet Tea Dorminy {
68391d6ac1dSSweet Tea Dorminy unsigned int allocated;
684dd137dd1SSweet Tea Dorminy
68591d6ac1dSSweet Tea Dorminy for (allocated = 0; allocated < nr_pages;) {
68691d6ac1dSSweet Tea Dorminy unsigned int last = allocated;
687dd137dd1SSweet Tea Dorminy
68891d6ac1dSSweet Tea Dorminy allocated = alloc_pages_bulk_array(GFP_NOFS, nr_pages, page_array);
689845cf1c7SQu Wenruo if (unlikely(allocated == last)) {
690845cf1c7SQu Wenruo /* No progress, fail and do cleanup. */
69133357859SQu Wenruo for (int i = 0; i < allocated; i++) {
69233357859SQu Wenruo __free_page(page_array[i]);
69333357859SQu Wenruo page_array[i] = NULL;
69433357859SQu Wenruo }
695dd137dd1SSweet Tea Dorminy return -ENOMEM;
69633357859SQu Wenruo }
697dd137dd1SSweet Tea Dorminy }
698dd137dd1SSweet Tea Dorminy return 0;
699dd137dd1SSweet Tea Dorminy }
700dd137dd1SSweet Tea Dorminy
btrfs_bio_is_contig(struct btrfs_bio_ctrl * bio_ctrl,struct page * page,u64 disk_bytenr,unsigned int pg_offset)70178a2ef1bSChristoph Hellwig static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
70278a2ef1bSChristoph Hellwig struct page *page, u64 disk_bytenr,
70378a2ef1bSChristoph Hellwig unsigned int pg_offset)
70478a2ef1bSChristoph Hellwig {
7059dfde1b4SChristoph Hellwig struct bio *bio = &bio_ctrl->bbio->bio;
70678a2ef1bSChristoph Hellwig struct bio_vec *bvec = bio_last_bvec_all(bio);
70778a2ef1bSChristoph Hellwig const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
70878a2ef1bSChristoph Hellwig
70978a2ef1bSChristoph Hellwig if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
71078a2ef1bSChristoph Hellwig /*
71178a2ef1bSChristoph Hellwig * For compression, all IO should have its logical bytenr set
71278a2ef1bSChristoph Hellwig * to the starting bytenr of the compressed extent.
71378a2ef1bSChristoph Hellwig */
71478a2ef1bSChristoph Hellwig return bio->bi_iter.bi_sector == sector;
71578a2ef1bSChristoph Hellwig }
71678a2ef1bSChristoph Hellwig
71778a2ef1bSChristoph Hellwig /*
71878a2ef1bSChristoph Hellwig * The contig check requires the following conditions to be met:
71978a2ef1bSChristoph Hellwig *
72078a2ef1bSChristoph Hellwig * 1) The pages are belonging to the same inode
72178a2ef1bSChristoph Hellwig * This is implied by the call chain.
72278a2ef1bSChristoph Hellwig *
72378a2ef1bSChristoph Hellwig * 2) The range has adjacent logical bytenr
72478a2ef1bSChristoph Hellwig *
72578a2ef1bSChristoph Hellwig * 3) The range has adjacent file offset
72678a2ef1bSChristoph Hellwig * This is required for the usage of btrfs_bio->file_offset.
72778a2ef1bSChristoph Hellwig */
72878a2ef1bSChristoph Hellwig return bio_end_sector(bio) == sector &&
72978a2ef1bSChristoph Hellwig page_offset(bvec->bv_page) + bvec->bv_offset + bvec->bv_len ==
73078a2ef1bSChristoph Hellwig page_offset(page) + pg_offset;
73178a2ef1bSChristoph Hellwig }
73278a2ef1bSChristoph Hellwig
alloc_new_bio(struct btrfs_inode * inode,struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,u64 file_offset)733d5e4377dSChristoph Hellwig static void alloc_new_bio(struct btrfs_inode *inode,
734e0eefe07SQu Wenruo struct btrfs_bio_ctrl *bio_ctrl,
73524e6c808SChristoph Hellwig u64 disk_bytenr, u64 file_offset)
736e0eefe07SQu Wenruo {
737e0eefe07SQu Wenruo struct btrfs_fs_info *fs_info = inode->root->fs_info;
738b41bbd29SChristoph Hellwig struct btrfs_bio *bbio;
739e0eefe07SQu Wenruo
7404317ff00SQu Wenruo bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info,
741c000bc04SChristoph Hellwig bio_ctrl->end_io_func, NULL);
742b41bbd29SChristoph Hellwig bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
7434317ff00SQu Wenruo bbio->inode = inode;
744b41bbd29SChristoph Hellwig bbio->file_offset = file_offset;
745b41bbd29SChristoph Hellwig bio_ctrl->bbio = bbio;
746198bd49eSJohannes Thumshirn bio_ctrl->len_to_oe_boundary = U32_MAX;
747198bd49eSJohannes Thumshirn
748a39da514SChristoph Hellwig /* Limit data write bios to the ordered boundary. */
749a39da514SChristoph Hellwig if (bio_ctrl->wbc) {
750198bd49eSJohannes Thumshirn struct btrfs_ordered_extent *ordered;
751198bd49eSJohannes Thumshirn
752198bd49eSJohannes Thumshirn ordered = btrfs_lookup_ordered_extent(inode, file_offset);
753198bd49eSJohannes Thumshirn if (ordered) {
754198bd49eSJohannes Thumshirn bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
755198bd49eSJohannes Thumshirn ordered->file_offset +
756198bd49eSJohannes Thumshirn ordered->disk_num_bytes - file_offset);
757ec63b84dSChristoph Hellwig bbio->ordered = ordered;
758198bd49eSJohannes Thumshirn }
75950f1cff3SChristoph Hellwig
76050f1cff3SChristoph Hellwig /*
761d5e4377dSChristoph Hellwig * Pick the last added device to support cgroup writeback. For
762d5e4377dSChristoph Hellwig * multi-device file systems this means blk-cgroup policies have
763d5e4377dSChristoph Hellwig * to always be set on the last added/replaced device.
764d5e4377dSChristoph Hellwig * This is a bit odd but has been like that for a long time.
76550f1cff3SChristoph Hellwig */
766b41bbd29SChristoph Hellwig bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
767b41bbd29SChristoph Hellwig wbc_init_bio(bio_ctrl->wbc, &bbio->bio);
768e0eefe07SQu Wenruo }
769e0eefe07SQu Wenruo }
770e0eefe07SQu Wenruo
7714b81ba48SDavid Sterba /*
7720c64c33cSQu Wenruo * @disk_bytenr: logical bytenr where the write will be
773209ecde5SQu Wenruo * @page: page to add to the bio
7740c64c33cSQu Wenruo * @size: portion of page that we want to write to
775b8b3d625SDavid Sterba * @pg_offset: offset of the new bio or to check whether we are adding
776b8b3d625SDavid Sterba * a contiguous page to the previous one
777814b6f91SQu Wenruo *
7789dfde1b4SChristoph Hellwig * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
7799dfde1b4SChristoph Hellwig * new one in @bio_ctrl->bbio.
780814b6f91SQu Wenruo * The mirror number for this IO should already be initizlied in
781814b6f91SQu Wenruo * @bio_ctrl->mirror_num.
7824b81ba48SDavid Sterba */
submit_extent_page(struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,struct page * page,size_t size,unsigned long pg_offset)78355173337SChristoph Hellwig static void submit_extent_page(struct btrfs_bio_ctrl *bio_ctrl,
784209ecde5SQu Wenruo u64 disk_bytenr, struct page *page,
785f8ed4852SChristoph Hellwig size_t size, unsigned long pg_offset)
786d1310b2eSChris Mason {
787e1326f03SNaohiro Aota struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
788d1310b2eSChris Mason
78924e6c808SChristoph Hellwig ASSERT(pg_offset + size <= PAGE_SIZE);
7905467abbaSQu Wenruo ASSERT(bio_ctrl->end_io_func);
7915467abbaSQu Wenruo
7929dfde1b4SChristoph Hellwig if (bio_ctrl->bbio &&
79378a2ef1bSChristoph Hellwig !btrfs_bio_is_contig(bio_ctrl, page, disk_bytenr, pg_offset))
79478a2ef1bSChristoph Hellwig submit_one_bio(bio_ctrl);
79578a2ef1bSChristoph Hellwig
79624e6c808SChristoph Hellwig do {
79724e6c808SChristoph Hellwig u32 len = size;
798429aebc0SDavid Sterba
799e0eefe07SQu Wenruo /* Allocate new bio if needed */
8009dfde1b4SChristoph Hellwig if (!bio_ctrl->bbio) {
80172b505dcSChristoph Hellwig alloc_new_bio(inode, bio_ctrl, disk_bytenr,
80224e6c808SChristoph Hellwig page_offset(page) + pg_offset);
803d1310b2eSChris Mason }
804e0eefe07SQu Wenruo
80524e6c808SChristoph Hellwig /* Cap to the current ordered extent boundary if there is one. */
80624e6c808SChristoph Hellwig if (len > bio_ctrl->len_to_oe_boundary) {
80724e6c808SChristoph Hellwig ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
80824e6c808SChristoph Hellwig ASSERT(is_data_inode(&inode->vfs_inode));
80924e6c808SChristoph Hellwig len = bio_ctrl->len_to_oe_boundary;
81024e6c808SChristoph Hellwig }
811e0eefe07SQu Wenruo
8129dfde1b4SChristoph Hellwig if (bio_add_page(&bio_ctrl->bbio->bio, page, len, pg_offset) != len) {
81324e6c808SChristoph Hellwig /* bio full: move on to a new one */
814722c82acSChristoph Hellwig submit_one_bio(bio_ctrl);
81524e6c808SChristoph Hellwig continue;
816e0eefe07SQu Wenruo }
81724e6c808SChristoph Hellwig
81824e6c808SChristoph Hellwig if (bio_ctrl->wbc)
81924e6c808SChristoph Hellwig wbc_account_cgroup_owner(bio_ctrl->wbc, page, len);
82024e6c808SChristoph Hellwig
82124e6c808SChristoph Hellwig size -= len;
82224e6c808SChristoph Hellwig pg_offset += len;
82324e6c808SChristoph Hellwig disk_bytenr += len;
82409c3717cSChris Mason
82509c3717cSChris Mason /*
82609c3717cSChris Mason * len_to_oe_boundary defaults to U32_MAX, which isn't page or
82709c3717cSChris Mason * sector aligned. alloc_new_bio() then sets it to the end of
82809c3717cSChris Mason * our ordered extent for writes into zoned devices.
82909c3717cSChris Mason *
83009c3717cSChris Mason * When len_to_oe_boundary is tracking an ordered extent, we
83109c3717cSChris Mason * trust the ordered extent code to align things properly, and
83209c3717cSChris Mason * the check above to cap our write to the ordered extent
83309c3717cSChris Mason * boundary is correct.
83409c3717cSChris Mason *
83509c3717cSChris Mason * When len_to_oe_boundary is U32_MAX, the cap above would
83609c3717cSChris Mason * result in a 4095 byte IO for the last page right before
83709c3717cSChris Mason * we hit the bio limit of UINT_MAX. bio_add_page() has all
83809c3717cSChris Mason * the checks required to make sure we don't overflow the bio,
83909c3717cSChris Mason * and we should just ignore len_to_oe_boundary completely
84009c3717cSChris Mason * unless we're using it to track an ordered extent.
84109c3717cSChris Mason *
84209c3717cSChris Mason * It's pretty hard to make a bio sized U32_MAX, but it can
84309c3717cSChris Mason * happen when the page cache is able to feed us contiguous
84409c3717cSChris Mason * pages for large extents.
84509c3717cSChris Mason */
84609c3717cSChris Mason if (bio_ctrl->len_to_oe_boundary != U32_MAX)
84724e6c808SChristoph Hellwig bio_ctrl->len_to_oe_boundary -= len;
84824e6c808SChristoph Hellwig
84924e6c808SChristoph Hellwig /* Ordered extent boundary: move on to a new bio. */
85024e6c808SChristoph Hellwig if (bio_ctrl->len_to_oe_boundary == 0)
85124e6c808SChristoph Hellwig submit_one_bio(bio_ctrl);
85224e6c808SChristoph Hellwig } while (size);
853e0eefe07SQu Wenruo }
854d1310b2eSChris Mason
attach_extent_buffer_page(struct extent_buffer * eb,struct page * page,struct btrfs_subpage * prealloc)855760f991fSQu Wenruo static int attach_extent_buffer_page(struct extent_buffer *eb,
856760f991fSQu Wenruo struct page *page,
857760f991fSQu Wenruo struct btrfs_subpage *prealloc)
8584f2de97aSJosef Bacik {
859760f991fSQu Wenruo struct btrfs_fs_info *fs_info = eb->fs_info;
860760f991fSQu Wenruo int ret = 0;
861760f991fSQu Wenruo
8620d01e247SQu Wenruo /*
8630d01e247SQu Wenruo * If the page is mapped to btree inode, we should hold the private
8640d01e247SQu Wenruo * lock to prevent race.
8650d01e247SQu Wenruo * For cloned or dummy extent buffers, their pages are not mapped and
8660d01e247SQu Wenruo * will not race with any other ebs.
8670d01e247SQu Wenruo */
8680d01e247SQu Wenruo if (page->mapping)
8690d01e247SQu Wenruo lockdep_assert_held(&page->mapping->private_lock);
8700d01e247SQu Wenruo
871fbca46ebSQu Wenruo if (fs_info->nodesize >= PAGE_SIZE) {
872d1b89bc0SGuoqing Jiang if (!PagePrivate(page))
873d1b89bc0SGuoqing Jiang attach_page_private(page, eb);
874d1b89bc0SGuoqing Jiang else
8754f2de97aSJosef Bacik WARN_ON(page->private != (unsigned long)eb);
876760f991fSQu Wenruo return 0;
877760f991fSQu Wenruo }
878760f991fSQu Wenruo
879760f991fSQu Wenruo /* Already mapped, just free prealloc */
880760f991fSQu Wenruo if (PagePrivate(page)) {
881760f991fSQu Wenruo btrfs_free_subpage(prealloc);
882760f991fSQu Wenruo return 0;
883760f991fSQu Wenruo }
884760f991fSQu Wenruo
885760f991fSQu Wenruo if (prealloc)
886760f991fSQu Wenruo /* Has preallocated memory for subpage */
887760f991fSQu Wenruo attach_page_private(page, prealloc);
888760f991fSQu Wenruo else
889760f991fSQu Wenruo /* Do new allocation to attach subpage */
890760f991fSQu Wenruo ret = btrfs_attach_subpage(fs_info, page,
891760f991fSQu Wenruo BTRFS_SUBPAGE_METADATA);
892760f991fSQu Wenruo return ret;
8934f2de97aSJosef Bacik }
8944f2de97aSJosef Bacik
set_page_extent_mapped(struct page * page)89532443de3SQu Wenruo int set_page_extent_mapped(struct page *page)
896d1310b2eSChris Mason {
89732443de3SQu Wenruo struct btrfs_fs_info *fs_info;
89832443de3SQu Wenruo
89932443de3SQu Wenruo ASSERT(page->mapping);
90032443de3SQu Wenruo
90132443de3SQu Wenruo if (PagePrivate(page))
90232443de3SQu Wenruo return 0;
90332443de3SQu Wenruo
90432443de3SQu Wenruo fs_info = btrfs_sb(page->mapping->host->i_sb);
90532443de3SQu Wenruo
906fbca46ebSQu Wenruo if (btrfs_is_subpage(fs_info, page))
90732443de3SQu Wenruo return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA);
90832443de3SQu Wenruo
909d1b89bc0SGuoqing Jiang attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE);
91032443de3SQu Wenruo return 0;
91132443de3SQu Wenruo }
91232443de3SQu Wenruo
clear_page_extent_mapped(struct page * page)91332443de3SQu Wenruo void clear_page_extent_mapped(struct page *page)
91432443de3SQu Wenruo {
91532443de3SQu Wenruo struct btrfs_fs_info *fs_info;
91632443de3SQu Wenruo
91732443de3SQu Wenruo ASSERT(page->mapping);
91832443de3SQu Wenruo
91932443de3SQu Wenruo if (!PagePrivate(page))
92032443de3SQu Wenruo return;
92132443de3SQu Wenruo
92232443de3SQu Wenruo fs_info = btrfs_sb(page->mapping->host->i_sb);
923fbca46ebSQu Wenruo if (btrfs_is_subpage(fs_info, page))
92432443de3SQu Wenruo return btrfs_detach_subpage(fs_info, page);
92532443de3SQu Wenruo
92632443de3SQu Wenruo detach_page_private(page);
927d1310b2eSChris Mason }
928d1310b2eSChris Mason
929125bac01SMiao Xie static struct extent_map *
__get_extent_map(struct inode * inode,struct page * page,size_t pg_offset,u64 start,u64 len,struct extent_map ** em_cached)930125bac01SMiao Xie __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
9311a5ee1e6SNikolay Borisov u64 start, u64 len, struct extent_map **em_cached)
932125bac01SMiao Xie {
933125bac01SMiao Xie struct extent_map *em;
934125bac01SMiao Xie
935125bac01SMiao Xie if (em_cached && *em_cached) {
936125bac01SMiao Xie em = *em_cached;
937cbc0e928SFilipe Manana if (extent_map_in_tree(em) && start >= em->start &&
938125bac01SMiao Xie start < extent_map_end(em)) {
939490b54d6SElena Reshetova refcount_inc(&em->refs);
940125bac01SMiao Xie return em;
941125bac01SMiao Xie }
942125bac01SMiao Xie
943125bac01SMiao Xie free_extent_map(em);
944125bac01SMiao Xie *em_cached = NULL;
945125bac01SMiao Xie }
946125bac01SMiao Xie
9471a5ee1e6SNikolay Borisov em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len);
948c0347550SFilipe Manana if (em_cached && !IS_ERR(em)) {
949125bac01SMiao Xie BUG_ON(*em_cached);
950490b54d6SElena Reshetova refcount_inc(&em->refs);
951125bac01SMiao Xie *em_cached = em;
952125bac01SMiao Xie }
953125bac01SMiao Xie return em;
954125bac01SMiao Xie }
955d1310b2eSChris Mason /*
956d1310b2eSChris Mason * basic readpage implementation. Locked extent state structs are inserted
957d1310b2eSChris Mason * into the tree that are removed when the IO is done (by the end_io
958d1310b2eSChris Mason * handlers)
95979787eaaSJeff Mahoney * XXX JDM: This needs looking at to ensure proper page locking
960baf863b9SLiu Bo * return 0 on success, otherwise return error
961d1310b2eSChris Mason */
btrfs_do_readpage(struct page * page,struct extent_map ** em_cached,struct btrfs_bio_ctrl * bio_ctrl,u64 * prev_em_start)9627aab8b32SChristoph Hellwig static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
963c000bc04SChristoph Hellwig struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
964d1310b2eSChris Mason {
965d1310b2eSChris Mason struct inode *inode = page->mapping->host;
96692082d40SQu Wenruo struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9674eee4fa4SMiao Xie u64 start = page_offset(page);
9688eec8296SDavid Sterba const u64 end = start + PAGE_SIZE - 1;
969d1310b2eSChris Mason u64 cur = start;
970d1310b2eSChris Mason u64 extent_offset;
971d1310b2eSChris Mason u64 last_byte = i_size_read(inode);
972d1310b2eSChris Mason u64 block_start;
973d1310b2eSChris Mason struct extent_map *em;
974baf863b9SLiu Bo int ret = 0;
975306e16ceSDavid Sterba size_t pg_offset = 0;
976d1310b2eSChris Mason size_t iosize;
977d1310b2eSChris Mason size_t blocksize = inode->i_sb->s_blocksize;
978f657a31cSDavid Sterba struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
979ae6957ebSDavid Sterba
98032443de3SQu Wenruo ret = set_page_extent_mapped(page);
98132443de3SQu Wenruo if (ret < 0) {
982570eb97bSJosef Bacik unlock_extent(tree, start, end, NULL);
98392082d40SQu Wenruo unlock_page(page);
98455173337SChristoph Hellwig return ret;
98532443de3SQu Wenruo }
986d1310b2eSChris Mason
98709cbfeafSKirill A. Shutemov if (page->index == last_byte >> PAGE_SHIFT) {
9887073017aSJohannes Thumshirn size_t zero_offset = offset_in_page(last_byte);
989c8b97818SChris Mason
990c8b97818SChris Mason if (zero_offset) {
99109cbfeafSKirill A. Shutemov iosize = PAGE_SIZE - zero_offset;
992d048b9c2SIra Weiny memzero_page(page, zero_offset, iosize);
993c8b97818SChris Mason }
994c8b97818SChris Mason }
9955467abbaSQu Wenruo bio_ctrl->end_io_func = end_bio_extent_readpage;
99692082d40SQu Wenruo begin_page_read(fs_info, page);
997d1310b2eSChris Mason while (cur <= end) {
998a140453bSChristoph Hellwig enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
999005efedfSFilipe Manana bool force_bio_submit = false;
10000c64c33cSQu Wenruo u64 disk_bytenr;
1001c8f2f24bSJosef Bacik
10026a404910SQu Wenruo ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1003d1310b2eSChris Mason if (cur >= last_byte) {
100409cbfeafSKirill A. Shutemov iosize = PAGE_SIZE - pg_offset;
1005d048b9c2SIra Weiny memzero_page(page, pg_offset, iosize);
10062c8f5e8cSFilipe Manana unlock_extent(tree, cur, cur + iosize - 1, NULL);
100792082d40SQu Wenruo end_page_read(page, true, cur, iosize);
1008d1310b2eSChris Mason break;
1009d1310b2eSChris Mason }
1010125bac01SMiao Xie em = __get_extent_map(inode, page, pg_offset, cur,
10111a5ee1e6SNikolay Borisov end - cur + 1, em_cached);
1012c0347550SFilipe Manana if (IS_ERR(em)) {
1013570eb97bSJosef Bacik unlock_extent(tree, cur, end, NULL);
101492082d40SQu Wenruo end_page_read(page, false, cur, end + 1 - cur);
101555173337SChristoph Hellwig return PTR_ERR(em);
1016d1310b2eSChris Mason }
1017d1310b2eSChris Mason extent_offset = cur - em->start;
1018d1310b2eSChris Mason BUG_ON(extent_map_end(em) <= cur);
1019d1310b2eSChris Mason BUG_ON(end < cur);
1020d1310b2eSChris Mason
10217f6ca7f2SDavid Sterba if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
1022a140453bSChristoph Hellwig compress_type = em->compress_type;
1023c8b97818SChris Mason
1024d1310b2eSChris Mason iosize = min(extent_map_end(em) - cur, end - cur + 1);
1025fda2832fSQu Wenruo iosize = ALIGN(iosize, blocksize);
1026a140453bSChristoph Hellwig if (compress_type != BTRFS_COMPRESS_NONE)
10270c64c33cSQu Wenruo disk_bytenr = em->block_start;
1028949b3273SGoldwyn Rodrigues else
10290c64c33cSQu Wenruo disk_bytenr = em->block_start + extent_offset;
1030d1310b2eSChris Mason block_start = em->block_start;
1031d899e052SYan Zheng if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
1032d899e052SYan Zheng block_start = EXTENT_MAP_HOLE;
1033005efedfSFilipe Manana
1034005efedfSFilipe Manana /*
1035005efedfSFilipe Manana * If we have a file range that points to a compressed extent
1036260db43cSRandy Dunlap * and it's followed by a consecutive file range that points
1037005efedfSFilipe Manana * to the same compressed extent (possibly with a different
1038005efedfSFilipe Manana * offset and/or length, so it either points to the whole extent
1039005efedfSFilipe Manana * or only part of it), we must make sure we do not submit a
1040005efedfSFilipe Manana * single bio to populate the pages for the 2 ranges because
1041005efedfSFilipe Manana * this makes the compressed extent read zero out the pages
1042005efedfSFilipe Manana * belonging to the 2nd range. Imagine the following scenario:
1043005efedfSFilipe Manana *
1044005efedfSFilipe Manana * File layout
1045005efedfSFilipe Manana * [0 - 8K] [8K - 24K]
1046005efedfSFilipe Manana * | |
1047005efedfSFilipe Manana * | |
1048005efedfSFilipe Manana * points to extent X, points to extent X,
1049005efedfSFilipe Manana * offset 4K, length of 8K offset 0, length 16K
1050005efedfSFilipe Manana *
1051005efedfSFilipe Manana * [extent X, compressed length = 4K uncompressed length = 16K]
1052005efedfSFilipe Manana *
1053005efedfSFilipe Manana * If the bio to read the compressed extent covers both ranges,
1054005efedfSFilipe Manana * it will decompress extent X into the pages belonging to the
1055005efedfSFilipe Manana * first range and then it will stop, zeroing out the remaining
1056005efedfSFilipe Manana * pages that belong to the other range that points to extent X.
1057005efedfSFilipe Manana * So here we make sure we submit 2 bios, one for the first
1058005efedfSFilipe Manana * range and another one for the third range. Both will target
1059005efedfSFilipe Manana * the same physical extent from disk, but we can't currently
1060005efedfSFilipe Manana * make the compressed bio endio callback populate the pages
1061005efedfSFilipe Manana * for both ranges because each compressed bio is tightly
1062005efedfSFilipe Manana * coupled with a single extent map, and each range can have
1063005efedfSFilipe Manana * an extent map with a different offset value relative to the
1064005efedfSFilipe Manana * uncompressed data of our extent and different lengths. This
1065005efedfSFilipe Manana * is a corner case so we prioritize correctness over
1066005efedfSFilipe Manana * non-optimal behavior (submitting 2 bios for the same extent).
1067005efedfSFilipe Manana */
1068005efedfSFilipe Manana if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
1069005efedfSFilipe Manana prev_em_start && *prev_em_start != (u64)-1 &&
10708e928218SFilipe Manana *prev_em_start != em->start)
1071005efedfSFilipe Manana force_bio_submit = true;
1072005efedfSFilipe Manana
1073005efedfSFilipe Manana if (prev_em_start)
10748e928218SFilipe Manana *prev_em_start = em->start;
1075005efedfSFilipe Manana
1076d1310b2eSChris Mason free_extent_map(em);
1077d1310b2eSChris Mason em = NULL;
1078d1310b2eSChris Mason
1079d1310b2eSChris Mason /* we've found a hole, just zero and go on */
1080d1310b2eSChris Mason if (block_start == EXTENT_MAP_HOLE) {
1081d048b9c2SIra Weiny memzero_page(page, pg_offset, iosize);
1082d1310b2eSChris Mason
10832c8f5e8cSFilipe Manana unlock_extent(tree, cur, cur + iosize - 1, NULL);
108492082d40SQu Wenruo end_page_read(page, true, cur, iosize);
1085d1310b2eSChris Mason cur = cur + iosize;
1086306e16ceSDavid Sterba pg_offset += iosize;
1087d1310b2eSChris Mason continue;
1088d1310b2eSChris Mason }
1089d1310b2eSChris Mason /* the get_extent function already copied into the page */
109070dec807SChris Mason if (block_start == EXTENT_MAP_INLINE) {
1091570eb97bSJosef Bacik unlock_extent(tree, cur, cur + iosize - 1, NULL);
109252b029f4SEthan Lien end_page_read(page, true, cur, iosize);
109370dec807SChris Mason cur = cur + iosize;
1094306e16ceSDavid Sterba pg_offset += iosize;
109570dec807SChris Mason continue;
109670dec807SChris Mason }
1097d1310b2eSChris Mason
1098f8ed4852SChristoph Hellwig if (bio_ctrl->compress_type != compress_type) {
1099c9bc621fSChristoph Hellwig submit_one_bio(bio_ctrl);
1100f8ed4852SChristoph Hellwig bio_ctrl->compress_type = compress_type;
1101f8ed4852SChristoph Hellwig }
1102c9bc621fSChristoph Hellwig
1103eb8d0c6dSChristoph Hellwig if (force_bio_submit)
1104eb8d0c6dSChristoph Hellwig submit_one_bio(bio_ctrl);
110555173337SChristoph Hellwig submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
1106f8ed4852SChristoph Hellwig pg_offset);
1107d1310b2eSChris Mason cur = cur + iosize;
1108306e16ceSDavid Sterba pg_offset += iosize;
1109d1310b2eSChris Mason }
111055173337SChristoph Hellwig
111155173337SChristoph Hellwig return 0;
1112d1310b2eSChris Mason }
1113d1310b2eSChris Mason
btrfs_read_folio(struct file * file,struct folio * folio)1114fdaf9a58SLinus Torvalds int btrfs_read_folio(struct file *file, struct folio *folio)
11157aab8b32SChristoph Hellwig {
1116fdaf9a58SLinus Torvalds struct page *page = &folio->page;
11177aab8b32SChristoph Hellwig struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
11187aab8b32SChristoph Hellwig u64 start = page_offset(page);
11197aab8b32SChristoph Hellwig u64 end = start + PAGE_SIZE - 1;
1120c000bc04SChristoph Hellwig struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
11217aab8b32SChristoph Hellwig int ret;
11227aab8b32SChristoph Hellwig
11237aab8b32SChristoph Hellwig btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
11247aab8b32SChristoph Hellwig
1125c000bc04SChristoph Hellwig ret = btrfs_do_readpage(page, NULL, &bio_ctrl, NULL);
11267aab8b32SChristoph Hellwig /*
11277aab8b32SChristoph Hellwig * If btrfs_do_readpage() failed we will want to submit the assembled
11287aab8b32SChristoph Hellwig * bio to do the cleanup.
11297aab8b32SChristoph Hellwig */
1130722c82acSChristoph Hellwig submit_one_bio(&bio_ctrl);
11317aab8b32SChristoph Hellwig return ret;
11327aab8b32SChristoph Hellwig }
11337aab8b32SChristoph Hellwig
contiguous_readpages(struct page * pages[],int nr_pages,u64 start,u64 end,struct extent_map ** em_cached,struct btrfs_bio_ctrl * bio_ctrl,u64 * prev_em_start)1134b6660e80SDavid Sterba static inline void contiguous_readpages(struct page *pages[], int nr_pages,
11359974090bSMiao Xie u64 start, u64 end,
1136125bac01SMiao Xie struct extent_map **em_cached,
1137390ed29bSQu Wenruo struct btrfs_bio_ctrl *bio_ctrl,
1138808f80b4SFilipe Manana u64 *prev_em_start)
11399974090bSMiao Xie {
114023d31bd4SNikolay Borisov struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host);
11419974090bSMiao Xie int index;
11429974090bSMiao Xie
1143b272ae22SDavid Sterba btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
11449974090bSMiao Xie
11459974090bSMiao Xie for (index = 0; index < nr_pages; index++) {
1146390ed29bSQu Wenruo btrfs_do_readpage(pages[index], em_cached, bio_ctrl,
1147c000bc04SChristoph Hellwig prev_em_start);
114809cbfeafSKirill A. Shutemov put_page(pages[index]);
11499974090bSMiao Xie }
11509974090bSMiao Xie }
11519974090bSMiao Xie
1152d1310b2eSChris Mason /*
115340f76580SChris Mason * helper for __extent_writepage, doing all of the delayed allocation setup.
115440f76580SChris Mason *
11555eaad97aSNikolay Borisov * This returns 1 if btrfs_run_delalloc_range function did all the work required
115640f76580SChris Mason * to write the page (copy into inline extent). In this case the IO has
115740f76580SChris Mason * been started and the page is already unlocked.
115840f76580SChris Mason *
115940f76580SChris Mason * This returns 0 if all went well (page still locked)
116040f76580SChris Mason * This returns < 0 if there were errors (page still locked)
1161d1310b2eSChris Mason */
writepage_delalloc(struct btrfs_inode * inode,struct page * page,struct writeback_control * wbc)1162cd4c0bf9SNikolay Borisov static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
116383f1b680SQu Wenruo struct page *page, struct writeback_control *wbc)
1164d1310b2eSChris Mason {
11652c73162dSChristoph Hellwig const u64 page_start = page_offset(page);
11662c73162dSChristoph Hellwig const u64 page_end = page_start + PAGE_SIZE - 1;
11672c73162dSChristoph Hellwig u64 delalloc_start = page_start;
11682c73162dSChristoph Hellwig u64 delalloc_end = page_end;
1169f85d7d6cSChris Mason u64 delalloc_to_write = 0;
1170c56cbe90SChristoph Hellwig int ret = 0;
117140f76580SChris Mason
11722749f7efSQu Wenruo while (delalloc_start < page_end) {
11732c73162dSChristoph Hellwig delalloc_end = page_end;
11742c73162dSChristoph Hellwig if (!find_lock_delalloc_range(&inode->vfs_inode, page,
11752c73162dSChristoph Hellwig &delalloc_start, &delalloc_end)) {
1176d1310b2eSChris Mason delalloc_start = delalloc_end + 1;
1177d1310b2eSChris Mason continue;
1178d1310b2eSChris Mason }
1179c56cbe90SChristoph Hellwig
1180cd4c0bf9SNikolay Borisov ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
1181c56cbe90SChristoph Hellwig delalloc_end, wbc);
1182c56cbe90SChristoph Hellwig if (ret < 0)
11837361b4aeSQu Wenruo return ret;
11842b2553f1SChristoph Hellwig
11852c73162dSChristoph Hellwig delalloc_start = delalloc_end + 1;
11862c73162dSChristoph Hellwig }
11872c73162dSChristoph Hellwig
1188f85d7d6cSChris Mason /*
1189ea1754a0SKirill A. Shutemov * delalloc_end is already one less than the total length, so
1190ea1754a0SKirill A. Shutemov * we don't subtract one from PAGE_SIZE
1191f85d7d6cSChris Mason */
11922c73162dSChristoph Hellwig delalloc_to_write +=
11932c73162dSChristoph Hellwig DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1194c56cbe90SChristoph Hellwig
1195c56cbe90SChristoph Hellwig /*
1196c56cbe90SChristoph Hellwig * If btrfs_run_dealloc_range() already started I/O and unlocked
1197c56cbe90SChristoph Hellwig * the pages, we just need to account for them here.
1198c56cbe90SChristoph Hellwig */
1199c56cbe90SChristoph Hellwig if (ret == 1) {
1200c56cbe90SChristoph Hellwig wbc->nr_to_write -= delalloc_to_write;
1201c56cbe90SChristoph Hellwig return 1;
1202c56cbe90SChristoph Hellwig }
1203c56cbe90SChristoph Hellwig
1204f85d7d6cSChris Mason if (wbc->nr_to_write < delalloc_to_write) {
1205f85d7d6cSChris Mason int thresh = 8192;
1206f85d7d6cSChris Mason
1207f85d7d6cSChris Mason if (delalloc_to_write < thresh * 2)
1208f85d7d6cSChris Mason thresh = delalloc_to_write;
1209f85d7d6cSChris Mason wbc->nr_to_write = min_t(u64, delalloc_to_write,
1210f85d7d6cSChris Mason thresh);
1211f85d7d6cSChris Mason }
1212c8b97818SChris Mason
1213b69d1ee9SNikolay Borisov return 0;
1214771ed689SChris Mason }
121540f76580SChris Mason
121640f76580SChris Mason /*
1217c5ef5c6cSQu Wenruo * Find the first byte we need to write.
1218c5ef5c6cSQu Wenruo *
1219c5ef5c6cSQu Wenruo * For subpage, one page can contain several sectors, and
1220c5ef5c6cSQu Wenruo * __extent_writepage_io() will just grab all extent maps in the page
1221c5ef5c6cSQu Wenruo * range and try to submit all non-inline/non-compressed extents.
1222c5ef5c6cSQu Wenruo *
1223c5ef5c6cSQu Wenruo * This is a big problem for subpage, we shouldn't re-submit already written
1224c5ef5c6cSQu Wenruo * data at all.
1225c5ef5c6cSQu Wenruo * This function will lookup subpage dirty bit to find which range we really
1226c5ef5c6cSQu Wenruo * need to submit.
1227c5ef5c6cSQu Wenruo *
1228c5ef5c6cSQu Wenruo * Return the next dirty range in [@start, @end).
1229c5ef5c6cSQu Wenruo * If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE.
1230c5ef5c6cSQu Wenruo */
find_next_dirty_byte(struct btrfs_fs_info * fs_info,struct page * page,u64 * start,u64 * end)1231c5ef5c6cSQu Wenruo static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
1232c5ef5c6cSQu Wenruo struct page *page, u64 *start, u64 *end)
1233c5ef5c6cSQu Wenruo {
1234c5ef5c6cSQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
123572a69cd0SQu Wenruo struct btrfs_subpage_info *spi = fs_info->subpage_info;
1236c5ef5c6cSQu Wenruo u64 orig_start = *start;
1237c5ef5c6cSQu Wenruo /* Declare as unsigned long so we can use bitmap ops */
1238c5ef5c6cSQu Wenruo unsigned long flags;
123972a69cd0SQu Wenruo int range_start_bit;
1240c5ef5c6cSQu Wenruo int range_end_bit;
1241c5ef5c6cSQu Wenruo
1242c5ef5c6cSQu Wenruo /*
1243c5ef5c6cSQu Wenruo * For regular sector size == page size case, since one page only
1244c5ef5c6cSQu Wenruo * contains one sector, we return the page offset directly.
1245c5ef5c6cSQu Wenruo */
1246fbca46ebSQu Wenruo if (!btrfs_is_subpage(fs_info, page)) {
1247c5ef5c6cSQu Wenruo *start = page_offset(page);
1248c5ef5c6cSQu Wenruo *end = page_offset(page) + PAGE_SIZE;
1249c5ef5c6cSQu Wenruo return;
1250c5ef5c6cSQu Wenruo }
1251c5ef5c6cSQu Wenruo
125272a69cd0SQu Wenruo range_start_bit = spi->dirty_offset +
125372a69cd0SQu Wenruo (offset_in_page(orig_start) >> fs_info->sectorsize_bits);
125472a69cd0SQu Wenruo
1255c5ef5c6cSQu Wenruo /* We should have the page locked, but just in case */
1256c5ef5c6cSQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
125772a69cd0SQu Wenruo bitmap_next_set_region(subpage->bitmaps, &range_start_bit, &range_end_bit,
125872a69cd0SQu Wenruo spi->dirty_offset + spi->bitmap_nr_bits);
1259c5ef5c6cSQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
1260c5ef5c6cSQu Wenruo
126172a69cd0SQu Wenruo range_start_bit -= spi->dirty_offset;
126272a69cd0SQu Wenruo range_end_bit -= spi->dirty_offset;
126372a69cd0SQu Wenruo
1264c5ef5c6cSQu Wenruo *start = page_offset(page) + range_start_bit * fs_info->sectorsize;
1265c5ef5c6cSQu Wenruo *end = page_offset(page) + range_end_bit * fs_info->sectorsize;
1266c5ef5c6cSQu Wenruo }
1267c5ef5c6cSQu Wenruo
1268c5ef5c6cSQu Wenruo /*
126940f76580SChris Mason * helper for __extent_writepage. This calls the writepage start hooks,
127040f76580SChris Mason * and does the loop to map the page into extents and bios.
127140f76580SChris Mason *
127240f76580SChris Mason * We return 1 if the IO is started and the page is unlocked,
127340f76580SChris Mason * 0 if all went well (page still locked)
127440f76580SChris Mason * < 0 if there were errors (page still locked)
127540f76580SChris Mason */
__extent_writepage_io(struct btrfs_inode * inode,struct page * page,struct btrfs_bio_ctrl * bio_ctrl,loff_t i_size,int * nr_ret)1276d4580fe2SNikolay Borisov static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
127740f76580SChris Mason struct page *page,
1278ee5f017dSDavid Sterba struct btrfs_bio_ctrl *bio_ctrl,
127940f76580SChris Mason loff_t i_size,
128057e5ffebSDavid Sterba int *nr_ret)
128140f76580SChris Mason {
12826bc5636aSQu Wenruo struct btrfs_fs_info *fs_info = inode->root->fs_info;
1283a129ffb8SQu Wenruo u64 cur = page_offset(page);
1284a129ffb8SQu Wenruo u64 end = cur + PAGE_SIZE - 1;
128540f76580SChris Mason u64 extent_offset;
128640f76580SChris Mason u64 block_start;
128740f76580SChris Mason struct extent_map *em;
128840f76580SChris Mason int ret = 0;
128940f76580SChris Mason int nr = 0;
129040f76580SChris Mason
1291a129ffb8SQu Wenruo ret = btrfs_writepage_cow_fixup(page);
129287826df0SJeff Mahoney if (ret) {
129387826df0SJeff Mahoney /* Fixup worker will requeue */
129472b505dcSChristoph Hellwig redirty_page_for_writepage(bio_ctrl->wbc, page);
1295247e743cSChris Mason unlock_page(page);
1296bcf93489SLiu Bo return 1;
1297247e743cSChris Mason }
1298247e743cSChris Mason
1299ee5f017dSDavid Sterba bio_ctrl->end_io_func = end_bio_extent_writepage;
1300d1310b2eSChris Mason while (cur <= end) {
13016648ceddSChristoph Hellwig u32 len = end - cur + 1;
13020c64c33cSQu Wenruo u64 disk_bytenr;
130340f76580SChris Mason u64 em_end;
1304c5ef5c6cSQu Wenruo u64 dirty_range_start = cur;
1305c5ef5c6cSQu Wenruo u64 dirty_range_end;
13066bc5636aSQu Wenruo u32 iosize;
130758409eddSDavid Sterba
130840f76580SChris Mason if (cur >= i_size) {
13096648ceddSChristoph Hellwig btrfs_mark_ordered_io_finished(inode, page, cur, len,
13106648ceddSChristoph Hellwig true);
1311cc1d0d93SQu Wenruo /*
1312cc1d0d93SQu Wenruo * This range is beyond i_size, thus we don't need to
1313cc1d0d93SQu Wenruo * bother writing back.
1314cc1d0d93SQu Wenruo * But we still need to clear the dirty subpage bit, or
1315cc1d0d93SQu Wenruo * the next time the page gets dirtied, we will try to
1316cc1d0d93SQu Wenruo * writeback the sectors with subpage dirty bits,
1317cc1d0d93SQu Wenruo * causing writeback without ordered extent.
1318cc1d0d93SQu Wenruo */
13196648ceddSChristoph Hellwig btrfs_page_clear_dirty(fs_info, page, cur, len);
1320d1310b2eSChris Mason break;
1321d1310b2eSChris Mason }
1322c5ef5c6cSQu Wenruo
1323c5ef5c6cSQu Wenruo find_next_dirty_byte(fs_info, page, &dirty_range_start,
1324c5ef5c6cSQu Wenruo &dirty_range_end);
1325c5ef5c6cSQu Wenruo if (cur < dirty_range_start) {
1326c5ef5c6cSQu Wenruo cur = dirty_range_start;
1327c5ef5c6cSQu Wenruo continue;
1328c5ef5c6cSQu Wenruo }
1329c5ef5c6cSQu Wenruo
13306648ceddSChristoph Hellwig em = btrfs_get_extent(inode, NULL, 0, cur, len);
1331c0347550SFilipe Manana if (IS_ERR(em)) {
133261391d56SFilipe Manana ret = PTR_ERR_OR_ZERO(em);
13335380311fSChristoph Hellwig goto out_error;
1334d1310b2eSChris Mason }
1335d1310b2eSChris Mason
1336d1310b2eSChris Mason extent_offset = cur - em->start;
133740f76580SChris Mason em_end = extent_map_end(em);
13386bc5636aSQu Wenruo ASSERT(cur <= em_end);
13396bc5636aSQu Wenruo ASSERT(cur < end);
13406bc5636aSQu Wenruo ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
13416bc5636aSQu Wenruo ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
1342f22b5dcbSChristoph Hellwig
1343d1310b2eSChris Mason block_start = em->block_start;
13446bc5636aSQu Wenruo disk_bytenr = em->block_start + extent_offset;
13456bc5636aSQu Wenruo
1346f22b5dcbSChristoph Hellwig ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags));
1347f22b5dcbSChristoph Hellwig ASSERT(block_start != EXTENT_MAP_HOLE);
1348f22b5dcbSChristoph Hellwig ASSERT(block_start != EXTENT_MAP_INLINE);
1349f22b5dcbSChristoph Hellwig
1350c5ef5c6cSQu Wenruo /*
1351c5ef5c6cSQu Wenruo * Note that em_end from extent_map_end() and dirty_range_end from
1352c5ef5c6cSQu Wenruo * find_next_dirty_byte() are all exclusive
1353c5ef5c6cSQu Wenruo */
1354c5ef5c6cSQu Wenruo iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
1355d1310b2eSChris Mason free_extent_map(em);
1356d1310b2eSChris Mason em = NULL;
1357d1310b2eSChris Mason
1358d2a91064SQu Wenruo btrfs_set_range_writeback(inode, cur, cur + iosize - 1);
1359d1310b2eSChris Mason if (!PageWriteback(page)) {
1360d4580fe2SNikolay Borisov btrfs_err(inode->root->fs_info,
1361efe120a0SFrank Holton "page %lu not writeback, cur %llu end %llu",
1362c1c9ff7cSGeert Uytterhoeven page->index, cur, end);
1363d1310b2eSChris Mason }
1364d1310b2eSChris Mason
1365c5ef5c6cSQu Wenruo /*
1366c5ef5c6cSQu Wenruo * Although the PageDirty bit is cleared before entering this
1367c5ef5c6cSQu Wenruo * function, subpage dirty bit is not cleared.
1368c5ef5c6cSQu Wenruo * So clear subpage dirty bit here so next time we won't submit
1369c5ef5c6cSQu Wenruo * page for range already written to disk.
1370c5ef5c6cSQu Wenruo */
1371c5ef5c6cSQu Wenruo btrfs_page_clear_dirty(fs_info, page, cur, iosize);
1372c5ef5c6cSQu Wenruo
137355173337SChristoph Hellwig submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
137455173337SChristoph Hellwig cur - page_offset(page));
13756bc5636aSQu Wenruo cur += iosize;
1376d1310b2eSChris Mason nr++;
1377d1310b2eSChris Mason }
13785380311fSChristoph Hellwig
13795380311fSChristoph Hellwig btrfs_page_assert_not_dirty(fs_info, page);
13805380311fSChristoph Hellwig *nr_ret = nr;
13815380311fSChristoph Hellwig return 0;
13825380311fSChristoph Hellwig
13835380311fSChristoph Hellwig out_error:
1384cc1d0d93SQu Wenruo /*
1385cc1d0d93SQu Wenruo * If we finish without problem, we should not only clear page dirty,
1386cc1d0d93SQu Wenruo * but also empty subpage dirty bits
1387cc1d0d93SQu Wenruo */
138840f76580SChris Mason *nr_ret = nr;
138940f76580SChris Mason return ret;
139040f76580SChris Mason }
139140f76580SChris Mason
139240f76580SChris Mason /*
139340f76580SChris Mason * the writepage semantics are similar to regular writepage. extent
139440f76580SChris Mason * records are inserted to lock ranges in the tree, and as dirty areas
139540f76580SChris Mason * are found, they are marked writeback. Then the lock bits are removed
139640f76580SChris Mason * and the end_io handler clears the writeback ranges
13973065976bSQu Wenruo *
13983065976bSQu Wenruo * Return 0 if everything goes well.
13993065976bSQu Wenruo * Return <0 for error.
140040f76580SChris Mason */
__extent_writepage(struct page * page,struct btrfs_bio_ctrl * bio_ctrl)140172b505dcSChristoph Hellwig static int __extent_writepage(struct page *page, struct btrfs_bio_ctrl *bio_ctrl)
140240f76580SChris Mason {
14038e1dec8eSMatthew Wilcox (Oracle) struct folio *folio = page_folio(page);
140440f76580SChris Mason struct inode *inode = page->mapping->host;
1405cf3075fbSQu Wenruo const u64 page_start = page_offset(page);
140640f76580SChris Mason int ret;
140740f76580SChris Mason int nr = 0;
1408eb70d222SOmar Sandoval size_t pg_offset;
140940f76580SChris Mason loff_t i_size = i_size_read(inode);
141009cbfeafSKirill A. Shutemov unsigned long end_index = i_size >> PAGE_SHIFT;
141140f76580SChris Mason
141272b505dcSChristoph Hellwig trace___extent_writepage(page, inode, bio_ctrl->wbc);
141340f76580SChris Mason
141440f76580SChris Mason WARN_ON(!PageLocked(page));
141540f76580SChris Mason
14167073017aSJohannes Thumshirn pg_offset = offset_in_page(i_size);
141740f76580SChris Mason if (page->index > end_index ||
141840f76580SChris Mason (page->index == end_index && !pg_offset)) {
14198e1dec8eSMatthew Wilcox (Oracle) folio_invalidate(folio, 0, folio_size(folio));
14208e1dec8eSMatthew Wilcox (Oracle) folio_unlock(folio);
142140f76580SChris Mason return 0;
142240f76580SChris Mason }
142340f76580SChris Mason
142421a8935eSDavid Sterba if (page->index == end_index)
1425d048b9c2SIra Weiny memzero_page(page, pg_offset, PAGE_SIZE - pg_offset);
142640f76580SChris Mason
142732443de3SQu Wenruo ret = set_page_extent_mapped(page);
14282b2553f1SChristoph Hellwig if (ret < 0)
142932443de3SQu Wenruo goto done;
143040f76580SChris Mason
143172b505dcSChristoph Hellwig ret = writepage_delalloc(BTRFS_I(inode), page, bio_ctrl->wbc);
143240f76580SChris Mason if (ret == 1)
1433169d2c87SOmar Sandoval return 0;
143440f76580SChris Mason if (ret)
143540f76580SChris Mason goto done;
143640f76580SChris Mason
143772b505dcSChristoph Hellwig ret = __extent_writepage_io(BTRFS_I(inode), page, bio_ctrl, i_size, &nr);
143840f76580SChris Mason if (ret == 1)
1439169d2c87SOmar Sandoval return 0;
144040f76580SChris Mason
14419ecdbee8SChristoph Hellwig bio_ctrl->wbc->nr_to_write--;
14429ecdbee8SChristoph Hellwig
144340f76580SChris Mason done:
1444d1310b2eSChris Mason if (nr == 0) {
1445d1310b2eSChris Mason /* make sure the mapping tag for page dirty gets cleared */
1446d1310b2eSChris Mason set_page_writeback(page);
1447d1310b2eSChris Mason end_page_writeback(page);
1448d1310b2eSChris Mason }
14499783e4deSChristoph Hellwig if (ret) {
14509783e4deSChristoph Hellwig btrfs_mark_ordered_io_finished(BTRFS_I(inode), page, page_start,
14519783e4deSChristoph Hellwig PAGE_SIZE, !ret);
14529783e4deSChristoph Hellwig mapping_set_error(page->mapping, ret);
14539783e4deSChristoph Hellwig }
1454d1310b2eSChris Mason unlock_page(page);
14553065976bSQu Wenruo ASSERT(ret <= 0);
145640f76580SChris Mason return ret;
1457d1310b2eSChris Mason }
1458d1310b2eSChris Mason
wait_on_extent_buffer_writeback(struct extent_buffer * eb)1459fd8b2b61SJosef Bacik void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
14600b32f4bbSJosef Bacik {
146174316201SNeilBrown wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
14620b32f4bbSJosef Bacik TASK_UNINTERRUPTIBLE);
14630b32f4bbSJosef Bacik }
14640b32f4bbSJosef Bacik
14652e3c2513SQu Wenruo /*
1466a3efb2f0SQu Wenruo * Lock extent buffer status and pages for writeback.
14672e3c2513SQu Wenruo *
14689fdd1601SChristoph Hellwig * Return %false if the extent buffer doesn't need to be submitted (e.g. the
14699fdd1601SChristoph Hellwig * extent buffer is not dirty)
14709fdd1601SChristoph Hellwig * Return %true is the extent buffer is submitted to bio.
14712e3c2513SQu Wenruo */
lock_extent_buffer_for_io(struct extent_buffer * eb,struct writeback_control * wbc)14729fdd1601SChristoph Hellwig static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
147350b21d7aSChristoph Hellwig struct writeback_control *wbc)
14740b32f4bbSJosef Bacik {
14759df76fb5SDavid Sterba struct btrfs_fs_info *fs_info = eb->fs_info;
14769fdd1601SChristoph Hellwig bool ret = false;
14770b32f4bbSJosef Bacik
14780b32f4bbSJosef Bacik btrfs_tree_lock(eb);
147950b21d7aSChristoph Hellwig while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
14800b32f4bbSJosef Bacik btrfs_tree_unlock(eb);
148150b21d7aSChristoph Hellwig if (wbc->sync_mode != WB_SYNC_ALL)
14829fdd1601SChristoph Hellwig return false;
14830b32f4bbSJosef Bacik wait_on_extent_buffer_writeback(eb);
14840b32f4bbSJosef Bacik btrfs_tree_lock(eb);
14850b32f4bbSJosef Bacik }
14860b32f4bbSJosef Bacik
148751561ffeSJosef Bacik /*
148851561ffeSJosef Bacik * We need to do this to prevent races in people who check if the eb is
148951561ffeSJosef Bacik * under IO since we can end up having no IO bits set for a short period
149051561ffeSJosef Bacik * of time.
149151561ffeSJosef Bacik */
149251561ffeSJosef Bacik spin_lock(&eb->refs_lock);
14930b32f4bbSJosef Bacik if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
14940b32f4bbSJosef Bacik set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
149551561ffeSJosef Bacik spin_unlock(&eb->refs_lock);
14960b32f4bbSJosef Bacik btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1497104b4e51SNikolay Borisov percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1498e2d84521SMiao Xie -eb->len,
1499e2d84521SMiao Xie fs_info->dirty_metadata_batch);
15009fdd1601SChristoph Hellwig ret = true;
150151561ffeSJosef Bacik } else {
150251561ffeSJosef Bacik spin_unlock(&eb->refs_lock);
15030b32f4bbSJosef Bacik }
15040b32f4bbSJosef Bacik btrfs_tree_unlock(eb);
15050b32f4bbSJosef Bacik return ret;
15060b32f4bbSJosef Bacik }
15070b32f4bbSJosef Bacik
set_btree_ioerr(struct extent_buffer * eb)1508cd88a4fdSChristoph Hellwig static void set_btree_ioerr(struct extent_buffer *eb)
1509656f30dbSFilipe Manana {
15105a2c6075SQu Wenruo struct btrfs_fs_info *fs_info = eb->fs_info;
1511656f30dbSFilipe Manana
1512cd88a4fdSChristoph Hellwig set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1513656f30dbSFilipe Manana
1514656f30dbSFilipe Manana /*
1515c2e39305SJosef Bacik * A read may stumble upon this buffer later, make sure that it gets an
1516c2e39305SJosef Bacik * error and knows there was an error.
1517c2e39305SJosef Bacik */
1518c2e39305SJosef Bacik clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
1519c2e39305SJosef Bacik
1520c2e39305SJosef Bacik /*
152168b85589SJosef Bacik * We need to set the mapping with the io error as well because a write
152268b85589SJosef Bacik * error will flip the file system readonly, and then syncfs() will
152368b85589SJosef Bacik * return a 0 because we are readonly if we don't modify the err seq for
152468b85589SJosef Bacik * the superblock.
152568b85589SJosef Bacik */
1526cd88a4fdSChristoph Hellwig mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO);
152768b85589SJosef Bacik
152868b85589SJosef Bacik /*
1529656f30dbSFilipe Manana * If writeback for a btree extent that doesn't belong to a log tree
1530656f30dbSFilipe Manana * failed, increment the counter transaction->eb_write_errors.
1531656f30dbSFilipe Manana * We do this because while the transaction is running and before it's
1532656f30dbSFilipe Manana * committing (when we call filemap_fdata[write|wait]_range against
1533656f30dbSFilipe Manana * the btree inode), we might have
1534656f30dbSFilipe Manana * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1535656f30dbSFilipe Manana * returns an error or an error happens during writeback, when we're
1536656f30dbSFilipe Manana * committing the transaction we wouldn't know about it, since the pages
1537656f30dbSFilipe Manana * can be no longer dirty nor marked anymore for writeback (if a
1538656f30dbSFilipe Manana * subsequent modification to the extent buffer didn't happen before the
1539656f30dbSFilipe Manana * transaction commit), which makes filemap_fdata[write|wait]_range not
1540656f30dbSFilipe Manana * able to find the pages tagged with SetPageError at transaction
1541656f30dbSFilipe Manana * commit time. So if this happens we must abort the transaction,
1542656f30dbSFilipe Manana * otherwise we commit a super block with btree roots that point to
1543656f30dbSFilipe Manana * btree nodes/leafs whose content on disk is invalid - either garbage
1544656f30dbSFilipe Manana * or the content of some node/leaf from a past generation that got
1545656f30dbSFilipe Manana * cowed or deleted and is no longer valid.
1546656f30dbSFilipe Manana *
1547656f30dbSFilipe Manana * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1548656f30dbSFilipe Manana * not be enough - we need to distinguish between log tree extents vs
1549656f30dbSFilipe Manana * non-log tree extents, and the next filemap_fdatawait_range() call
1550656f30dbSFilipe Manana * will catch and clear such errors in the mapping - and that call might
1551656f30dbSFilipe Manana * be from a log sync and not from a transaction commit. Also, checking
1552656f30dbSFilipe Manana * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1553656f30dbSFilipe Manana * not done and would not be reliable - the eb might have been released
1554656f30dbSFilipe Manana * from memory and reading it back again means that flag would not be
1555656f30dbSFilipe Manana * set (since it's a runtime flag, not persisted on disk).
1556656f30dbSFilipe Manana *
1557656f30dbSFilipe Manana * Using the flags below in the btree inode also makes us achieve the
1558656f30dbSFilipe Manana * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1559656f30dbSFilipe Manana * writeback for all dirty pages and before filemap_fdatawait_range()
1560656f30dbSFilipe Manana * is called, the writeback for all dirty pages had already finished
1561656f30dbSFilipe Manana * with errors - because we were not using AS_EIO/AS_ENOSPC,
1562656f30dbSFilipe Manana * filemap_fdatawait_range() would return success, as it could not know
1563656f30dbSFilipe Manana * that writeback errors happened (the pages were no longer tagged for
1564656f30dbSFilipe Manana * writeback).
1565656f30dbSFilipe Manana */
1566656f30dbSFilipe Manana switch (eb->log_index) {
1567656f30dbSFilipe Manana case -1:
15685a2c6075SQu Wenruo set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
1569656f30dbSFilipe Manana break;
1570656f30dbSFilipe Manana case 0:
15715a2c6075SQu Wenruo set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
1572656f30dbSFilipe Manana break;
1573656f30dbSFilipe Manana case 1:
15745a2c6075SQu Wenruo set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
1575656f30dbSFilipe Manana break;
1576656f30dbSFilipe Manana default:
1577656f30dbSFilipe Manana BUG(); /* unexpected, logic error */
1578656f30dbSFilipe Manana }
1579656f30dbSFilipe Manana }
1580656f30dbSFilipe Manana
15812f3186d8SQu Wenruo /*
15822f3186d8SQu Wenruo * The endio specific version which won't touch any unsafe spinlock in endio
15832f3186d8SQu Wenruo * context.
15842f3186d8SQu Wenruo */
find_extent_buffer_nolock(struct btrfs_fs_info * fs_info,u64 start)15852f3186d8SQu Wenruo static struct extent_buffer *find_extent_buffer_nolock(
15862f3186d8SQu Wenruo struct btrfs_fs_info *fs_info, u64 start)
15872f3186d8SQu Wenruo {
15882f3186d8SQu Wenruo struct extent_buffer *eb;
15892f3186d8SQu Wenruo
15902f3186d8SQu Wenruo rcu_read_lock();
159101cd3909SDavid Sterba eb = radix_tree_lookup(&fs_info->buffer_radix,
15922f3186d8SQu Wenruo start >> fs_info->sectorsize_bits);
15932f3186d8SQu Wenruo if (eb && atomic_inc_not_zero(&eb->refs)) {
15942f3186d8SQu Wenruo rcu_read_unlock();
15952f3186d8SQu Wenruo return eb;
15962f3186d8SQu Wenruo }
15972f3186d8SQu Wenruo rcu_read_unlock();
15982f3186d8SQu Wenruo return NULL;
15992f3186d8SQu Wenruo }
16002f3186d8SQu Wenruo
extent_buffer_write_end_io(struct btrfs_bio * bbio)1601cd88a4fdSChristoph Hellwig static void extent_buffer_write_end_io(struct btrfs_bio *bbio)
16022f3186d8SQu Wenruo {
1603cd88a4fdSChristoph Hellwig struct extent_buffer *eb = bbio->private;
1604cd88a4fdSChristoph Hellwig struct btrfs_fs_info *fs_info = eb->fs_info;
1605cd88a4fdSChristoph Hellwig bool uptodate = !bbio->bio.bi_status;
16062f3186d8SQu Wenruo struct bvec_iter_all iter_all;
16072c30c71bSKent Overstreet struct bio_vec *bvec;
1608cd88a4fdSChristoph Hellwig u32 bio_offset = 0;
16090b32f4bbSJosef Bacik
1610cd88a4fdSChristoph Hellwig if (!uptodate)
1611cd88a4fdSChristoph Hellwig set_btree_ioerr(eb);
1612cd88a4fdSChristoph Hellwig
1613cd88a4fdSChristoph Hellwig bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
1614cd88a4fdSChristoph Hellwig u64 start = eb->start + bio_offset;
16150b32f4bbSJosef Bacik struct page *page = bvec->bv_page;
1616cd88a4fdSChristoph Hellwig u32 len = bvec->bv_len;
16170b32f4bbSJosef Bacik
1618cd88a4fdSChristoph Hellwig btrfs_page_clear_writeback(fs_info, page, start, len);
1619cd88a4fdSChristoph Hellwig bio_offset += len;
16200b32f4bbSJosef Bacik }
16210b32f4bbSJosef Bacik
1622cd88a4fdSChristoph Hellwig clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1623cd88a4fdSChristoph Hellwig smp_mb__after_atomic();
1624cd88a4fdSChristoph Hellwig wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
16250b32f4bbSJosef Bacik
1626cd88a4fdSChristoph Hellwig bio_put(&bbio->bio);
16270b32f4bbSJosef Bacik }
16280b32f4bbSJosef Bacik
prepare_eb_write(struct extent_buffer * eb)1629fa04c165SQu Wenruo static void prepare_eb_write(struct extent_buffer *eb)
1630fa04c165SQu Wenruo {
1631fa04c165SQu Wenruo u32 nritems;
1632fa04c165SQu Wenruo unsigned long start;
1633fa04c165SQu Wenruo unsigned long end;
1634fa04c165SQu Wenruo
1635fa04c165SQu Wenruo clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1636fa04c165SQu Wenruo
1637fa04c165SQu Wenruo /* Set btree blocks beyond nritems with 0 to avoid stale content */
1638fa04c165SQu Wenruo nritems = btrfs_header_nritems(eb);
1639fa04c165SQu Wenruo if (btrfs_header_level(eb) > 0) {
1640e23efd8eSJosef Bacik end = btrfs_node_key_ptr_offset(eb, nritems);
1641fa04c165SQu Wenruo memzero_extent_buffer(eb, end, eb->len - end);
1642fa04c165SQu Wenruo } else {
1643fa04c165SQu Wenruo /*
1644fa04c165SQu Wenruo * Leaf:
1645fa04c165SQu Wenruo * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
1646fa04c165SQu Wenruo */
164742c9419aSJosef Bacik start = btrfs_item_nr_offset(eb, nritems);
16488009adf3SJosef Bacik end = btrfs_item_nr_offset(eb, 0);
16493a3178c7SJosef Bacik if (nritems == 0)
16503a3178c7SJosef Bacik end += BTRFS_LEAF_DATA_SIZE(eb->fs_info);
16513a3178c7SJosef Bacik else
16523a3178c7SJosef Bacik end += btrfs_item_offset(eb, nritems - 1);
1653fa04c165SQu Wenruo memzero_extent_buffer(eb, start, end - start);
1654fa04c165SQu Wenruo }
1655fa04c165SQu Wenruo }
1656fa04c165SQu Wenruo
write_one_eb(struct extent_buffer * eb,struct writeback_control * wbc)165755173337SChristoph Hellwig static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
165850b21d7aSChristoph Hellwig struct writeback_control *wbc)
16590b32f4bbSJosef Bacik {
166046672a44SChristoph Hellwig struct btrfs_fs_info *fs_info = eb->fs_info;
1661b51e6b4bSChristoph Hellwig struct btrfs_bio *bbio;
16620b32f4bbSJosef Bacik
1663fa04c165SQu Wenruo prepare_eb_write(eb);
1664fa04c165SQu Wenruo
1665b51e6b4bSChristoph Hellwig bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
1666b51e6b4bSChristoph Hellwig REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
1667cd88a4fdSChristoph Hellwig eb->fs_info, extent_buffer_write_end_io, eb);
1668b51e6b4bSChristoph Hellwig bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
166946672a44SChristoph Hellwig bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
1670b51e6b4bSChristoph Hellwig wbc_init_bio(wbc, &bbio->bio);
1671b51e6b4bSChristoph Hellwig bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
1672b51e6b4bSChristoph Hellwig bbio->file_offset = eb->start;
167346672a44SChristoph Hellwig if (fs_info->nodesize < PAGE_SIZE) {
167446672a44SChristoph Hellwig struct page *p = eb->pages[0];
1675b51e6b4bSChristoph Hellwig
167646672a44SChristoph Hellwig lock_page(p);
167746672a44SChristoph Hellwig btrfs_subpage_set_writeback(fs_info, p, eb->start, eb->len);
167846672a44SChristoph Hellwig if (btrfs_subpage_clear_and_test_dirty(fs_info, p, eb->start,
167946672a44SChristoph Hellwig eb->len)) {
168046672a44SChristoph Hellwig clear_page_dirty_for_io(p);
168146672a44SChristoph Hellwig wbc->nr_to_write--;
168246672a44SChristoph Hellwig }
168346672a44SChristoph Hellwig __bio_add_page(&bbio->bio, p, eb->len, eb->start - page_offset(p));
168446672a44SChristoph Hellwig wbc_account_cgroup_owner(wbc, p, eb->len);
168546672a44SChristoph Hellwig unlock_page(p);
168646672a44SChristoph Hellwig } else {
168746672a44SChristoph Hellwig for (int i = 0; i < num_extent_pages(eb); i++) {
1688fb85fc9aSDavid Sterba struct page *p = eb->pages[i];
16890b32f4bbSJosef Bacik
169081a79b6aSChristoph Hellwig lock_page(p);
16910b32f4bbSJosef Bacik clear_page_dirty_for_io(p);
16920b32f4bbSJosef Bacik set_page_writeback(p);
1693b51e6b4bSChristoph Hellwig __bio_add_page(&bbio->bio, p, PAGE_SIZE, 0);
1694b51e6b4bSChristoph Hellwig wbc_account_cgroup_owner(wbc, p, PAGE_SIZE);
169550b21d7aSChristoph Hellwig wbc->nr_to_write--;
16960b32f4bbSJosef Bacik unlock_page(p);
16970b32f4bbSJosef Bacik }
169846672a44SChristoph Hellwig }
1699b51e6b4bSChristoph Hellwig btrfs_submit_bio(bbio, 0);
17000b32f4bbSJosef Bacik }
17010b32f4bbSJosef Bacik
1702f91e0d0cSQu Wenruo /*
1703c4aec299SQu Wenruo * Submit one subpage btree page.
1704c4aec299SQu Wenruo *
1705c4aec299SQu Wenruo * The main difference to submit_eb_page() is:
1706c4aec299SQu Wenruo * - Page locking
1707c4aec299SQu Wenruo * For subpage, we don't rely on page locking at all.
1708c4aec299SQu Wenruo *
1709c4aec299SQu Wenruo * - Flush write bio
1710c4aec299SQu Wenruo * We only flush bio if we may be unable to fit current extent buffers into
1711c4aec299SQu Wenruo * current bio.
1712c4aec299SQu Wenruo *
1713c4aec299SQu Wenruo * Return >=0 for the number of submitted extent buffers.
1714c4aec299SQu Wenruo * Return <0 for fatal error.
1715c4aec299SQu Wenruo */
submit_eb_subpage(struct page * page,struct writeback_control * wbc)171650b21d7aSChristoph Hellwig static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
1717c4aec299SQu Wenruo {
1718c4aec299SQu Wenruo struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
1719c4aec299SQu Wenruo int submitted = 0;
1720c4aec299SQu Wenruo u64 page_start = page_offset(page);
1721c4aec299SQu Wenruo int bit_start = 0;
1722c4aec299SQu Wenruo int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
1723c4aec299SQu Wenruo
1724c4aec299SQu Wenruo /* Lock and write each dirty extent buffers in the range */
172572a69cd0SQu Wenruo while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
1726c4aec299SQu Wenruo struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1727c4aec299SQu Wenruo struct extent_buffer *eb;
1728c4aec299SQu Wenruo unsigned long flags;
1729c4aec299SQu Wenruo u64 start;
1730c4aec299SQu Wenruo
1731c4aec299SQu Wenruo /*
1732c4aec299SQu Wenruo * Take private lock to ensure the subpage won't be detached
1733c4aec299SQu Wenruo * in the meantime.
1734c4aec299SQu Wenruo */
1735c4aec299SQu Wenruo spin_lock(&page->mapping->private_lock);
1736c4aec299SQu Wenruo if (!PagePrivate(page)) {
1737c4aec299SQu Wenruo spin_unlock(&page->mapping->private_lock);
1738c4aec299SQu Wenruo break;
1739c4aec299SQu Wenruo }
1740c4aec299SQu Wenruo spin_lock_irqsave(&subpage->lock, flags);
174172a69cd0SQu Wenruo if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
174272a69cd0SQu Wenruo subpage->bitmaps)) {
1743c4aec299SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
1744c4aec299SQu Wenruo spin_unlock(&page->mapping->private_lock);
1745c4aec299SQu Wenruo bit_start++;
1746c4aec299SQu Wenruo continue;
1747c4aec299SQu Wenruo }
1748c4aec299SQu Wenruo
1749c4aec299SQu Wenruo start = page_start + bit_start * fs_info->sectorsize;
1750c4aec299SQu Wenruo bit_start += sectors_per_node;
1751c4aec299SQu Wenruo
1752c4aec299SQu Wenruo /*
1753c4aec299SQu Wenruo * Here we just want to grab the eb without touching extra
1754c4aec299SQu Wenruo * spin locks, so call find_extent_buffer_nolock().
1755c4aec299SQu Wenruo */
1756c4aec299SQu Wenruo eb = find_extent_buffer_nolock(fs_info, start);
1757c4aec299SQu Wenruo spin_unlock_irqrestore(&subpage->lock, flags);
1758c4aec299SQu Wenruo spin_unlock(&page->mapping->private_lock);
1759c4aec299SQu Wenruo
1760c4aec299SQu Wenruo /*
1761c4aec299SQu Wenruo * The eb has already reached 0 refs thus find_extent_buffer()
1762c4aec299SQu Wenruo * doesn't return it. We don't need to write back such eb
1763c4aec299SQu Wenruo * anyway.
1764c4aec299SQu Wenruo */
1765c4aec299SQu Wenruo if (!eb)
1766c4aec299SQu Wenruo continue;
1767c4aec299SQu Wenruo
176850b21d7aSChristoph Hellwig if (lock_extent_buffer_for_io(eb, wbc)) {
176946672a44SChristoph Hellwig write_one_eb(eb, wbc);
1770c4aec299SQu Wenruo submitted++;
1771c4aec299SQu Wenruo }
17729fdd1601SChristoph Hellwig free_extent_buffer(eb);
17739fdd1601SChristoph Hellwig }
1774c4aec299SQu Wenruo return submitted;
1775c4aec299SQu Wenruo }
1776c4aec299SQu Wenruo
1777c4aec299SQu Wenruo /*
1778f91e0d0cSQu Wenruo * Submit all page(s) of one extent buffer.
1779f91e0d0cSQu Wenruo *
1780f91e0d0cSQu Wenruo * @page: the page of one extent buffer
1781f91e0d0cSQu Wenruo * @eb_context: to determine if we need to submit this page, if current page
1782f91e0d0cSQu Wenruo * belongs to this eb, we don't need to submit
1783f91e0d0cSQu Wenruo *
1784f91e0d0cSQu Wenruo * The caller should pass each page in their bytenr order, and here we use
1785f91e0d0cSQu Wenruo * @eb_context to determine if we have submitted pages of one extent buffer.
1786f91e0d0cSQu Wenruo *
1787f91e0d0cSQu Wenruo * If we have, we just skip until we hit a new page that doesn't belong to
1788f91e0d0cSQu Wenruo * current @eb_context.
1789f91e0d0cSQu Wenruo *
1790f91e0d0cSQu Wenruo * If not, we submit all the page(s) of the extent buffer.
1791f91e0d0cSQu Wenruo *
1792f91e0d0cSQu Wenruo * Return >0 if we have submitted the extent buffer successfully.
1793f91e0d0cSQu Wenruo * Return 0 if we don't need to submit the page, as it's already submitted by
1794f91e0d0cSQu Wenruo * previous call.
1795f91e0d0cSQu Wenruo * Return <0 for fatal error.
1796f91e0d0cSQu Wenruo */
submit_eb_page(struct page * page,struct btrfs_eb_write_context * ctx)1797861093efSNaohiro Aota static int submit_eb_page(struct page *page, struct btrfs_eb_write_context *ctx)
1798f91e0d0cSQu Wenruo {
1799861093efSNaohiro Aota struct writeback_control *wbc = ctx->wbc;
1800f91e0d0cSQu Wenruo struct address_space *mapping = page->mapping;
1801f91e0d0cSQu Wenruo struct extent_buffer *eb;
1802f91e0d0cSQu Wenruo int ret;
1803f91e0d0cSQu Wenruo
1804f91e0d0cSQu Wenruo if (!PagePrivate(page))
1805f91e0d0cSQu Wenruo return 0;
1806f91e0d0cSQu Wenruo
1807fbca46ebSQu Wenruo if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
180850b21d7aSChristoph Hellwig return submit_eb_subpage(page, wbc);
1809c4aec299SQu Wenruo
1810f91e0d0cSQu Wenruo spin_lock(&mapping->private_lock);
1811f91e0d0cSQu Wenruo if (!PagePrivate(page)) {
1812f91e0d0cSQu Wenruo spin_unlock(&mapping->private_lock);
1813f91e0d0cSQu Wenruo return 0;
1814f91e0d0cSQu Wenruo }
1815f91e0d0cSQu Wenruo
1816f91e0d0cSQu Wenruo eb = (struct extent_buffer *)page->private;
1817f91e0d0cSQu Wenruo
1818f91e0d0cSQu Wenruo /*
1819f91e0d0cSQu Wenruo * Shouldn't happen and normally this would be a BUG_ON but no point
1820f91e0d0cSQu Wenruo * crashing the machine for something we can survive anyway.
1821f91e0d0cSQu Wenruo */
1822f91e0d0cSQu Wenruo if (WARN_ON(!eb)) {
1823f91e0d0cSQu Wenruo spin_unlock(&mapping->private_lock);
1824f91e0d0cSQu Wenruo return 0;
1825f91e0d0cSQu Wenruo }
1826f91e0d0cSQu Wenruo
1827861093efSNaohiro Aota if (eb == ctx->eb) {
1828f91e0d0cSQu Wenruo spin_unlock(&mapping->private_lock);
1829f91e0d0cSQu Wenruo return 0;
1830f91e0d0cSQu Wenruo }
1831f91e0d0cSQu Wenruo ret = atomic_inc_not_zero(&eb->refs);
1832f91e0d0cSQu Wenruo spin_unlock(&mapping->private_lock);
1833f91e0d0cSQu Wenruo if (!ret)
1834f91e0d0cSQu Wenruo return 0;
1835f91e0d0cSQu Wenruo
1836861093efSNaohiro Aota ctx->eb = eb;
1837861093efSNaohiro Aota
18382ad8c051SNaohiro Aota ret = btrfs_check_meta_write_pointer(eb->fs_info, ctx);
18392ad8c051SNaohiro Aota if (ret) {
18402ad8c051SNaohiro Aota if (ret == -EBUSY)
18410bc09ca1SNaohiro Aota ret = 0;
18420bc09ca1SNaohiro Aota free_extent_buffer(eb);
18430bc09ca1SNaohiro Aota return ret;
18440bc09ca1SNaohiro Aota }
18450bc09ca1SNaohiro Aota
184650b21d7aSChristoph Hellwig if (!lock_extent_buffer_for_io(eb, wbc)) {
1847f91e0d0cSQu Wenruo free_extent_buffer(eb);
184850b21d7aSChristoph Hellwig return 0;
1849f91e0d0cSQu Wenruo }
18500356ad41SNaohiro Aota /* Implies write in zoned mode. */
18517db94301SNaohiro Aota if (ctx->zoned_bg) {
18520356ad41SNaohiro Aota /* Mark the last eb in the block group. */
18537db94301SNaohiro Aota btrfs_schedule_zone_finish_bg(ctx->zoned_bg, eb);
18540356ad41SNaohiro Aota ctx->zoned_bg->meta_write_pointer += eb->len;
1855be1a1d7aSNaohiro Aota }
185650b21d7aSChristoph Hellwig write_one_eb(eb, wbc);
1857f91e0d0cSQu Wenruo free_extent_buffer(eb);
1858f91e0d0cSQu Wenruo return 1;
1859f91e0d0cSQu Wenruo }
1860f91e0d0cSQu Wenruo
btree_write_cache_pages(struct address_space * mapping,struct writeback_control * wbc)18610b32f4bbSJosef Bacik int btree_write_cache_pages(struct address_space *mapping,
18620b32f4bbSJosef Bacik struct writeback_control *wbc)
18630b32f4bbSJosef Bacik {
1864861093efSNaohiro Aota struct btrfs_eb_write_context ctx = { .wbc = wbc };
1865b3ff8f1dSQu Wenruo struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
18660b32f4bbSJosef Bacik int ret = 0;
18670b32f4bbSJosef Bacik int done = 0;
18680b32f4bbSJosef Bacik int nr_to_write_done = 0;
186951c5cd3bSVishal Moola (Oracle) struct folio_batch fbatch;
187051c5cd3bSVishal Moola (Oracle) unsigned int nr_folios;
18710b32f4bbSJosef Bacik pgoff_t index;
18720b32f4bbSJosef Bacik pgoff_t end; /* Inclusive */
18730b32f4bbSJosef Bacik int scanned = 0;
187410bbd235SMatthew Wilcox xa_mark_t tag;
18750b32f4bbSJosef Bacik
187651c5cd3bSVishal Moola (Oracle) folio_batch_init(&fbatch);
18770b32f4bbSJosef Bacik if (wbc->range_cyclic) {
18780b32f4bbSJosef Bacik index = mapping->writeback_index; /* Start from prev offset */
18790b32f4bbSJosef Bacik end = -1;
1880556755a8SJosef Bacik /*
1881556755a8SJosef Bacik * Start from the beginning does not need to cycle over the
1882556755a8SJosef Bacik * range, mark it as scanned.
1883556755a8SJosef Bacik */
1884556755a8SJosef Bacik scanned = (index == 0);
18850b32f4bbSJosef Bacik } else {
188609cbfeafSKirill A. Shutemov index = wbc->range_start >> PAGE_SHIFT;
188709cbfeafSKirill A. Shutemov end = wbc->range_end >> PAGE_SHIFT;
18880b32f4bbSJosef Bacik scanned = 1;
18890b32f4bbSJosef Bacik }
18900b32f4bbSJosef Bacik if (wbc->sync_mode == WB_SYNC_ALL)
18910b32f4bbSJosef Bacik tag = PAGECACHE_TAG_TOWRITE;
18920b32f4bbSJosef Bacik else
18930b32f4bbSJosef Bacik tag = PAGECACHE_TAG_DIRTY;
18940bc09ca1SNaohiro Aota btrfs_zoned_meta_io_lock(fs_info);
18950b32f4bbSJosef Bacik retry:
18960b32f4bbSJosef Bacik if (wbc->sync_mode == WB_SYNC_ALL)
18970b32f4bbSJosef Bacik tag_pages_for_writeback(mapping, index, end);
18980b32f4bbSJosef Bacik while (!done && !nr_to_write_done && (index <= end) &&
189951c5cd3bSVishal Moola (Oracle) (nr_folios = filemap_get_folios_tag(mapping, &index, end,
190051c5cd3bSVishal Moola (Oracle) tag, &fbatch))) {
19010b32f4bbSJosef Bacik unsigned i;
19020b32f4bbSJosef Bacik
190351c5cd3bSVishal Moola (Oracle) for (i = 0; i < nr_folios; i++) {
190451c5cd3bSVishal Moola (Oracle) struct folio *folio = fbatch.folios[i];
19050b32f4bbSJosef Bacik
1906861093efSNaohiro Aota ret = submit_eb_page(&folio->page, &ctx);
1907f91e0d0cSQu Wenruo if (ret == 0)
19080b32f4bbSJosef Bacik continue;
1909f91e0d0cSQu Wenruo if (ret < 0) {
19100607eb1dSFilipe Manana done = 1;
19110607eb1dSFilipe Manana break;
19120b32f4bbSJosef Bacik }
19130b32f4bbSJosef Bacik
19140b32f4bbSJosef Bacik /*
19150b32f4bbSJosef Bacik * the filesystem may choose to bump up nr_to_write.
19160b32f4bbSJosef Bacik * We have to make sure to honor the new nr_to_write
19170b32f4bbSJosef Bacik * at any time
19180b32f4bbSJosef Bacik */
19190b32f4bbSJosef Bacik nr_to_write_done = wbc->nr_to_write <= 0;
19200b32f4bbSJosef Bacik }
192151c5cd3bSVishal Moola (Oracle) folio_batch_release(&fbatch);
19220b32f4bbSJosef Bacik cond_resched();
19230b32f4bbSJosef Bacik }
19240b32f4bbSJosef Bacik if (!scanned && !done) {
19250b32f4bbSJosef Bacik /*
19260b32f4bbSJosef Bacik * We hit the last page and there is more work to be done: wrap
19270b32f4bbSJosef Bacik * back to the start of the file
19280b32f4bbSJosef Bacik */
19290b32f4bbSJosef Bacik scanned = 1;
19300b32f4bbSJosef Bacik index = 0;
19310b32f4bbSJosef Bacik goto retry;
19320b32f4bbSJosef Bacik }
1933b3ff8f1dSQu Wenruo /*
1934b3ff8f1dSQu Wenruo * If something went wrong, don't allow any metadata write bio to be
1935b3ff8f1dSQu Wenruo * submitted.
1936b3ff8f1dSQu Wenruo *
1937b3ff8f1dSQu Wenruo * This would prevent use-after-free if we had dirty pages not
1938b3ff8f1dSQu Wenruo * cleaned up, which can still happen by fuzzed images.
1939b3ff8f1dSQu Wenruo *
1940b3ff8f1dSQu Wenruo * - Bad extent tree
1941b3ff8f1dSQu Wenruo * Allowing existing tree block to be allocated for other trees.
1942b3ff8f1dSQu Wenruo *
1943b3ff8f1dSQu Wenruo * - Log tree operations
1944b3ff8f1dSQu Wenruo * Exiting tree blocks get allocated to log tree, bumps its
1945b3ff8f1dSQu Wenruo * generation, then get cleaned in tree re-balance.
1946b3ff8f1dSQu Wenruo * Such tree block will not be written back, since it's clean,
1947b3ff8f1dSQu Wenruo * thus no WRITTEN flag set.
1948b3ff8f1dSQu Wenruo * And after log writes back, this tree block is not traced by
1949b3ff8f1dSQu Wenruo * any dirty extent_io_tree.
1950b3ff8f1dSQu Wenruo *
1951b3ff8f1dSQu Wenruo * - Offending tree block gets re-dirtied from its original owner
1952b3ff8f1dSQu Wenruo * Since it has bumped generation, no WRITTEN flag, it can be
1953b3ff8f1dSQu Wenruo * reused without COWing. This tree block will not be traced
1954b3ff8f1dSQu Wenruo * by btrfs_transaction::dirty_pages.
1955b3ff8f1dSQu Wenruo *
1956b3ff8f1dSQu Wenruo * Now such dirty tree block will not be cleaned by any dirty
1957b3ff8f1dSQu Wenruo * extent io tree. Thus we don't want to submit such wild eb
1958b3ff8f1dSQu Wenruo * if the fs already has error.
19599845e5ddSChristoph Hellwig *
1960c9583adaSQu Wenruo * We can get ret > 0 from submit_extent_page() indicating how many ebs
1961c9583adaSQu Wenruo * were submitted. Reset it to 0 to avoid false alerts for the caller.
1962c9583adaSQu Wenruo */
1963c9583adaSQu Wenruo if (ret > 0)
1964c9583adaSQu Wenruo ret = 0;
19659845e5ddSChristoph Hellwig if (!ret && BTRFS_FS_ERROR(fs_info))
19669845e5ddSChristoph Hellwig ret = -EROFS;
19677db94301SNaohiro Aota
19687db94301SNaohiro Aota if (ctx.zoned_bg)
19697db94301SNaohiro Aota btrfs_put_block_group(ctx.zoned_bg);
19709845e5ddSChristoph Hellwig btrfs_zoned_meta_io_unlock(fs_info);
19710b32f4bbSJosef Bacik return ret;
19720b32f4bbSJosef Bacik }
19730b32f4bbSJosef Bacik
197443dd529aSDavid Sterba /*
19753bed2da1SNikolay Borisov * Walk the list of dirty pages of the given address space and write all of them.
19763bed2da1SNikolay Borisov *
1977d1310b2eSChris Mason * @mapping: address space structure to write
1978d1310b2eSChris Mason * @wbc: subtract the number of written pages from *@wbc->nr_to_write
1979ee5f017dSDavid Sterba * @bio_ctrl: holds context for the write, namely the bio
1980d1310b2eSChris Mason *
1981d1310b2eSChris Mason * If a page is already under I/O, write_cache_pages() skips it, even
1982d1310b2eSChris Mason * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
1983d1310b2eSChris Mason * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
1984d1310b2eSChris Mason * and msync() need to guarantee that all the data which was dirty at the time
1985d1310b2eSChris Mason * the call was made get new I/O started against them. If wbc->sync_mode is
1986d1310b2eSChris Mason * WB_SYNC_ALL then we were called for data integrity and we must wait for
1987d1310b2eSChris Mason * existing IO to complete.
1988d1310b2eSChris Mason */
extent_write_cache_pages(struct address_space * mapping,struct btrfs_bio_ctrl * bio_ctrl)19894242b64aSDavid Sterba static int extent_write_cache_pages(struct address_space *mapping,
1990ee5f017dSDavid Sterba struct btrfs_bio_ctrl *bio_ctrl)
1991d1310b2eSChris Mason {
199272b505dcSChristoph Hellwig struct writeback_control *wbc = bio_ctrl->wbc;
19937fd1a3f7SJosef Bacik struct inode *inode = mapping->host;
1994d1310b2eSChris Mason int ret = 0;
1995d1310b2eSChris Mason int done = 0;
1996f85d7d6cSChris Mason int nr_to_write_done = 0;
19979f50fd2eSVishal Moola (Oracle) struct folio_batch fbatch;
19989f50fd2eSVishal Moola (Oracle) unsigned int nr_folios;
1999d1310b2eSChris Mason pgoff_t index;
2000d1310b2eSChris Mason pgoff_t end; /* Inclusive */
2001a9132667SLiu Bo pgoff_t done_index;
2002a9132667SLiu Bo int range_whole = 0;
2003d1310b2eSChris Mason int scanned = 0;
200410bbd235SMatthew Wilcox xa_mark_t tag;
2005d1310b2eSChris Mason
20067fd1a3f7SJosef Bacik /*
20077fd1a3f7SJosef Bacik * We have to hold onto the inode so that ordered extents can do their
20087fd1a3f7SJosef Bacik * work when the IO finishes. The alternative to this is failing to add
20097fd1a3f7SJosef Bacik * an ordered extent if the igrab() fails there and that is a huge pain
20107fd1a3f7SJosef Bacik * to deal with, so instead just hold onto the inode throughout the
20117fd1a3f7SJosef Bacik * writepages operation. If it fails here we are freeing up the inode
20127fd1a3f7SJosef Bacik * anyway and we'd rather not waste our time writing out stuff that is
20137fd1a3f7SJosef Bacik * going to be truncated anyway.
20147fd1a3f7SJosef Bacik */
20157fd1a3f7SJosef Bacik if (!igrab(inode))
20167fd1a3f7SJosef Bacik return 0;
20177fd1a3f7SJosef Bacik
20189f50fd2eSVishal Moola (Oracle) folio_batch_init(&fbatch);
2019d1310b2eSChris Mason if (wbc->range_cyclic) {
2020d1310b2eSChris Mason index = mapping->writeback_index; /* Start from prev offset */
2021d1310b2eSChris Mason end = -1;
2022556755a8SJosef Bacik /*
2023556755a8SJosef Bacik * Start from the beginning does not need to cycle over the
2024556755a8SJosef Bacik * range, mark it as scanned.
2025556755a8SJosef Bacik */
2026556755a8SJosef Bacik scanned = (index == 0);
2027d1310b2eSChris Mason } else {
202809cbfeafSKirill A. Shutemov index = wbc->range_start >> PAGE_SHIFT;
202909cbfeafSKirill A. Shutemov end = wbc->range_end >> PAGE_SHIFT;
2030a9132667SLiu Bo if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2031a9132667SLiu Bo range_whole = 1;
2032d1310b2eSChris Mason scanned = 1;
2033d1310b2eSChris Mason }
20343cd24c69SEthan Lien
20353cd24c69SEthan Lien /*
20363cd24c69SEthan Lien * We do the tagged writepage as long as the snapshot flush bit is set
20373cd24c69SEthan Lien * and we are the first one who do the filemap_flush() on this inode.
20383cd24c69SEthan Lien *
20393cd24c69SEthan Lien * The nr_to_write == LONG_MAX is needed to make sure other flushers do
20403cd24c69SEthan Lien * not race in and drop the bit.
20413cd24c69SEthan Lien */
20423cd24c69SEthan Lien if (range_whole && wbc->nr_to_write == LONG_MAX &&
20433cd24c69SEthan Lien test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
20443cd24c69SEthan Lien &BTRFS_I(inode)->runtime_flags))
20453cd24c69SEthan Lien wbc->tagged_writepages = 1;
20463cd24c69SEthan Lien
20473cd24c69SEthan Lien if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2048f7aaa06bSJosef Bacik tag = PAGECACHE_TAG_TOWRITE;
2049f7aaa06bSJosef Bacik else
2050f7aaa06bSJosef Bacik tag = PAGECACHE_TAG_DIRTY;
2051d1310b2eSChris Mason retry:
20523cd24c69SEthan Lien if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2053f7aaa06bSJosef Bacik tag_pages_for_writeback(mapping, index, end);
2054a9132667SLiu Bo done_index = index;
2055f85d7d6cSChris Mason while (!done && !nr_to_write_done && (index <= end) &&
20569f50fd2eSVishal Moola (Oracle) (nr_folios = filemap_get_folios_tag(mapping, &index,
20579f50fd2eSVishal Moola (Oracle) end, tag, &fbatch))) {
2058d1310b2eSChris Mason unsigned i;
2059d1310b2eSChris Mason
20609f50fd2eSVishal Moola (Oracle) for (i = 0; i < nr_folios; i++) {
20619f50fd2eSVishal Moola (Oracle) struct folio *folio = fbatch.folios[i];
2062d1310b2eSChris Mason
20637b365a2aSMinjie Du done_index = folio_next_index(folio);
2064d1310b2eSChris Mason /*
2065b93b0163SMatthew Wilcox * At this point we hold neither the i_pages lock nor
2066b93b0163SMatthew Wilcox * the page lock: the page may be truncated or
2067b93b0163SMatthew Wilcox * invalidated (changing page->mapping to NULL),
2068b93b0163SMatthew Wilcox * or even swizzled back from swapper_space to
2069b93b0163SMatthew Wilcox * tmpfs file mapping
2070d1310b2eSChris Mason */
20719f50fd2eSVishal Moola (Oracle) if (!folio_trylock(folio)) {
2072ee5f017dSDavid Sterba submit_write_bio(bio_ctrl, 0);
20739f50fd2eSVishal Moola (Oracle) folio_lock(folio);
207401d658f2SChris Mason }
2075d1310b2eSChris Mason
20769f50fd2eSVishal Moola (Oracle) if (unlikely(folio->mapping != mapping)) {
20779f50fd2eSVishal Moola (Oracle) folio_unlock(folio);
2078d1310b2eSChris Mason continue;
2079d1310b2eSChris Mason }
2080d1310b2eSChris Mason
20815c256998SChristoph Hellwig if (!folio_test_dirty(folio)) {
20825c256998SChristoph Hellwig /* Someone wrote it for us. */
20835c256998SChristoph Hellwig folio_unlock(folio);
20845c256998SChristoph Hellwig continue;
20855c256998SChristoph Hellwig }
20865c256998SChristoph Hellwig
2087d2c3f4f6SChris Mason if (wbc->sync_mode != WB_SYNC_NONE) {
20889f50fd2eSVishal Moola (Oracle) if (folio_test_writeback(folio))
2089ee5f017dSDavid Sterba submit_write_bio(bio_ctrl, 0);
20909f50fd2eSVishal Moola (Oracle) folio_wait_writeback(folio);
2091d2c3f4f6SChris Mason }
2092d1310b2eSChris Mason
20939f50fd2eSVishal Moola (Oracle) if (folio_test_writeback(folio) ||
20949f50fd2eSVishal Moola (Oracle) !folio_clear_dirty_for_io(folio)) {
20959f50fd2eSVishal Moola (Oracle) folio_unlock(folio);
2096d1310b2eSChris Mason continue;
2097d1310b2eSChris Mason }
2098d1310b2eSChris Mason
209972b505dcSChristoph Hellwig ret = __extent_writepage(&folio->page, bio_ctrl);
2100a9132667SLiu Bo if (ret < 0) {
2101a9132667SLiu Bo done = 1;
2102a9132667SLiu Bo break;
2103a9132667SLiu Bo }
2104f85d7d6cSChris Mason
2105f85d7d6cSChris Mason /*
2106effa24f6SChristoph Hellwig * The filesystem may choose to bump up nr_to_write.
2107f85d7d6cSChris Mason * We have to make sure to honor the new nr_to_write
2108effa24f6SChristoph Hellwig * at any time.
2109f85d7d6cSChris Mason */
2110effa24f6SChristoph Hellwig nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2111effa24f6SChristoph Hellwig wbc->nr_to_write <= 0);
2112d1310b2eSChris Mason }
21139f50fd2eSVishal Moola (Oracle) folio_batch_release(&fbatch);
2114d1310b2eSChris Mason cond_resched();
2115d1310b2eSChris Mason }
2116894b36e3SLiu Bo if (!scanned && !done) {
2117d1310b2eSChris Mason /*
2118d1310b2eSChris Mason * We hit the last page and there is more work to be done: wrap
2119d1310b2eSChris Mason * back to the start of the file
2120d1310b2eSChris Mason */
2121d1310b2eSChris Mason scanned = 1;
2122d1310b2eSChris Mason index = 0;
212342ffb0bfSJosef Bacik
212442ffb0bfSJosef Bacik /*
212542ffb0bfSJosef Bacik * If we're looping we could run into a page that is locked by a
212642ffb0bfSJosef Bacik * writer and that writer could be waiting on writeback for a
212742ffb0bfSJosef Bacik * page in our current bio, and thus deadlock, so flush the
212842ffb0bfSJosef Bacik * write bio here.
212942ffb0bfSJosef Bacik */
2130ee5f017dSDavid Sterba submit_write_bio(bio_ctrl, 0);
2131d1310b2eSChris Mason goto retry;
2132d1310b2eSChris Mason }
2133a9132667SLiu Bo
2134a9132667SLiu Bo if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2135a9132667SLiu Bo mapping->writeback_index = done_index;
2136a9132667SLiu Bo
2137e55cf7caSDavid Sterba btrfs_add_delayed_iput(BTRFS_I(inode));
2138894b36e3SLiu Bo return ret;
2139d1310b2eSChris Mason }
2140d1310b2eSChris Mason
21412bd0fc93SQu Wenruo /*
21422bd0fc93SQu Wenruo * Submit the pages in the range to bio for call sites which delalloc range has
21432bd0fc93SQu Wenruo * already been ran (aka, ordered extent inserted) and all pages are still
21442bd0fc93SQu Wenruo * locked.
21452bd0fc93SQu Wenruo */
extent_write_locked_range(struct inode * inode,struct page * locked_page,u64 start,u64 end,struct writeback_control * wbc,bool pages_dirty)2146778b8785SChristoph Hellwig void extent_write_locked_range(struct inode *inode, struct page *locked_page,
2147778b8785SChristoph Hellwig u64 start, u64 end, struct writeback_control *wbc,
2148778b8785SChristoph Hellwig bool pages_dirty)
2149771ed689SChris Mason {
21502bd0fc93SQu Wenruo bool found_error = false;
2151771ed689SChris Mason int ret = 0;
2152771ed689SChris Mason struct address_space *mapping = inode->i_mapping;
2153eb34dceaSChristoph Hellwig struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2154eb34dceaSChristoph Hellwig const u32 sectorsize = fs_info->sectorsize;
2155eb34dceaSChristoph Hellwig loff_t i_size = i_size_read(inode);
21562bd0fc93SQu Wenruo u64 cur = start;
2157c000bc04SChristoph Hellwig struct btrfs_bio_ctrl bio_ctrl = {
21587027f871SChristoph Hellwig .wbc = wbc,
21597027f871SChristoph Hellwig .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2160c000bc04SChristoph Hellwig };
2161771ed689SChris Mason
21627027f871SChristoph Hellwig if (wbc->no_cgroup_owner)
21637027f871SChristoph Hellwig bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
21647027f871SChristoph Hellwig
216566448b9dSQu Wenruo ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
216666448b9dSQu Wenruo
21672bd0fc93SQu Wenruo while (cur <= end) {
216866448b9dSQu Wenruo u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
21699783e4deSChristoph Hellwig u32 cur_len = cur_end + 1 - cur;
2170eb34dceaSChristoph Hellwig struct page *page;
2171eb34dceaSChristoph Hellwig int nr = 0;
217266448b9dSQu Wenruo
21732bd0fc93SQu Wenruo page = find_get_page(mapping, cur >> PAGE_SHIFT);
217466448b9dSQu Wenruo ASSERT(PageLocked(page));
2175778b8785SChristoph Hellwig if (pages_dirty && page != locked_page) {
21762bd0fc93SQu Wenruo ASSERT(PageDirty(page));
21772bd0fc93SQu Wenruo clear_page_dirty_for_io(page);
217844962ca3SChristoph Hellwig }
2179eb34dceaSChristoph Hellwig
2180eb34dceaSChristoph Hellwig ret = __extent_writepage_io(BTRFS_I(inode), page, &bio_ctrl,
2181eb34dceaSChristoph Hellwig i_size, &nr);
2182eb34dceaSChristoph Hellwig if (ret == 1)
2183eb34dceaSChristoph Hellwig goto next_page;
2184eb34dceaSChristoph Hellwig
2185eb34dceaSChristoph Hellwig /* Make sure the mapping tag for page dirty gets cleared. */
2186eb34dceaSChristoph Hellwig if (nr == 0) {
2187eb34dceaSChristoph Hellwig set_page_writeback(page);
2188eb34dceaSChristoph Hellwig end_page_writeback(page);
2189eb34dceaSChristoph Hellwig }
21909783e4deSChristoph Hellwig if (ret) {
21919783e4deSChristoph Hellwig btrfs_mark_ordered_io_finished(BTRFS_I(inode), page,
21929783e4deSChristoph Hellwig cur, cur_len, !ret);
21939783e4deSChristoph Hellwig mapping_set_error(page->mapping, ret);
21949783e4deSChristoph Hellwig }
21959783e4deSChristoph Hellwig btrfs_page_unlock_writer(fs_info, page, cur, cur_len);
21960835d1e6SChristoph Hellwig if (ret < 0)
21972bd0fc93SQu Wenruo found_error = true;
2198eb34dceaSChristoph Hellwig next_page:
219909cbfeafSKirill A. Shutemov put_page(page);
220066448b9dSQu Wenruo cur = cur_end + 1;
2201771ed689SChris Mason }
2202771ed689SChris Mason
2203ee5f017dSDavid Sterba submit_write_bio(&bio_ctrl, found_error ? ret : 0);
2204771ed689SChris Mason }
2205d1310b2eSChris Mason
extent_writepages(struct address_space * mapping,struct writeback_control * wbc)22068ae225a8SNikolay Borisov int extent_writepages(struct address_space *mapping,
2207d1310b2eSChris Mason struct writeback_control *wbc)
2208d1310b2eSChris Mason {
220935156d85SJohannes Thumshirn struct inode *inode = mapping->host;
2210d1310b2eSChris Mason int ret = 0;
2211ee5f017dSDavid Sterba struct btrfs_bio_ctrl bio_ctrl = {
221272b505dcSChristoph Hellwig .wbc = wbc,
2213c000bc04SChristoph Hellwig .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2214d1310b2eSChris Mason };
2215d1310b2eSChris Mason
221635156d85SJohannes Thumshirn /*
221735156d85SJohannes Thumshirn * Allow only a single thread to do the reloc work in zoned mode to
221835156d85SJohannes Thumshirn * protect the write pointer updates.
221935156d85SJohannes Thumshirn */
2220869f4cdcSJohannes Thumshirn btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
222172b505dcSChristoph Hellwig ret = extent_write_cache_pages(mapping, &bio_ctrl);
2222ee5f017dSDavid Sterba submit_write_bio(&bio_ctrl, ret);
222319ab78caSNaohiro Aota btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2224d1310b2eSChris Mason return ret;
2225d1310b2eSChris Mason }
2226d1310b2eSChris Mason
extent_readahead(struct readahead_control * rac)2227ba206a02SMatthew Wilcox (Oracle) void extent_readahead(struct readahead_control *rac)
2228d1310b2eSChris Mason {
2229c000bc04SChristoph Hellwig struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
223067c9684fSLiu Bo struct page *pagepool[16];
2231125bac01SMiao Xie struct extent_map *em_cached = NULL;
2232808f80b4SFilipe Manana u64 prev_em_start = (u64)-1;
2233ba206a02SMatthew Wilcox (Oracle) int nr;
2234d1310b2eSChris Mason
2235ba206a02SMatthew Wilcox (Oracle) while ((nr = readahead_page_batch(rac, pagepool))) {
223632c0a6bcSMatthew Wilcox (Oracle) u64 contig_start = readahead_pos(rac);
223732c0a6bcSMatthew Wilcox (Oracle) u64 contig_end = contig_start + readahead_batch_length(rac) - 1;
2238e65ef21eSNikolay Borisov
2239ba206a02SMatthew Wilcox (Oracle) contiguous_readpages(pagepool, nr, contig_start, contig_end,
2240390ed29bSQu Wenruo &em_cached, &bio_ctrl, &prev_em_start);
224161ed3a14SNikolay Borisov }
224267c9684fSLiu Bo
2243125bac01SMiao Xie if (em_cached)
2244125bac01SMiao Xie free_extent_map(em_cached);
2245722c82acSChristoph Hellwig submit_one_bio(&bio_ctrl);
2246d1310b2eSChris Mason }
2247d1310b2eSChris Mason
2248d1310b2eSChris Mason /*
2249895586ebSMatthew Wilcox (Oracle) * basic invalidate_folio code, this waits on any locked or writeback
2250895586ebSMatthew Wilcox (Oracle) * ranges corresponding to the folio, and then deletes any extent state
2251d1310b2eSChris Mason * records from the tree
2252d1310b2eSChris Mason */
extent_invalidate_folio(struct extent_io_tree * tree,struct folio * folio,size_t offset)2253895586ebSMatthew Wilcox (Oracle) int extent_invalidate_folio(struct extent_io_tree *tree,
2254895586ebSMatthew Wilcox (Oracle) struct folio *folio, size_t offset)
2255d1310b2eSChris Mason {
22562ac55d41SJosef Bacik struct extent_state *cached_state = NULL;
2257895586ebSMatthew Wilcox (Oracle) u64 start = folio_pos(folio);
2258895586ebSMatthew Wilcox (Oracle) u64 end = start + folio_size(folio) - 1;
2259895586ebSMatthew Wilcox (Oracle) size_t blocksize = folio->mapping->host->i_sb->s_blocksize;
2260d1310b2eSChris Mason
2261829ddec9SQu Wenruo /* This function is only called for the btree inode */
2262829ddec9SQu Wenruo ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2263829ddec9SQu Wenruo
2264fda2832fSQu Wenruo start += ALIGN(offset, blocksize);
2265d1310b2eSChris Mason if (start > end)
2266d1310b2eSChris Mason return 0;
2267d1310b2eSChris Mason
2268570eb97bSJosef Bacik lock_extent(tree, start, end, &cached_state);
2269895586ebSMatthew Wilcox (Oracle) folio_wait_writeback(folio);
2270829ddec9SQu Wenruo
2271829ddec9SQu Wenruo /*
2272829ddec9SQu Wenruo * Currently for btree io tree, only EXTENT_LOCKED is utilized,
2273829ddec9SQu Wenruo * so here we only need to unlock the extent range to free any
2274829ddec9SQu Wenruo * existing extent state.
2275829ddec9SQu Wenruo */
2276570eb97bSJosef Bacik unlock_extent(tree, start, end, &cached_state);
2277d1310b2eSChris Mason return 0;
2278d1310b2eSChris Mason }
2279d1310b2eSChris Mason
2280d1310b2eSChris Mason /*
2281f913cff3SMatthew Wilcox (Oracle) * a helper for release_folio, this tests for areas of the page that
22827b13b7b1SChris Mason * are locked or under IO and drops the related state bits if it is safe
22837b13b7b1SChris Mason * to drop the page.
22847b13b7b1SChris Mason */
try_release_extent_state(struct extent_io_tree * tree,struct page * page,gfp_t mask)228529c68b2dSNikolay Borisov static int try_release_extent_state(struct extent_io_tree *tree,
228648a3b636SEric Sandeen struct page *page, gfp_t mask)
22877b13b7b1SChris Mason {
22884eee4fa4SMiao Xie u64 start = page_offset(page);
228909cbfeafSKirill A. Shutemov u64 end = start + PAGE_SIZE - 1;
22907b13b7b1SChris Mason int ret = 1;
22917b13b7b1SChris Mason
22928882679eSNikolay Borisov if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) {
22937b13b7b1SChris Mason ret = 0;
22948882679eSNikolay Borisov } else {
2295b71fb16bSJosef Bacik u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
2296d3cf0243SBoris Burkov EXTENT_DELALLOC_NEW | EXTENT_CTLBITS |
2297d3cf0243SBoris Burkov EXTENT_QGROUP_RESERVED);
2298b71fb16bSJosef Bacik
229911ef160fSChris Mason /*
23002766ff61SFilipe Manana * At this point we can safely clear everything except the
23012766ff61SFilipe Manana * locked bit, the nodatasum bit and the delalloc new bit.
23022766ff61SFilipe Manana * The delalloc new bit will be cleared by ordered extent
23032766ff61SFilipe Manana * completion.
230411ef160fSChris Mason */
23051d126800SDavid Sterba ret = __clear_extent_bit(tree, start, end, clear_bits, NULL, NULL);
2306e3f24cc5SChris Mason
2307e3f24cc5SChris Mason /* if clear_extent_bit failed for enomem reasons,
2308e3f24cc5SChris Mason * we can't allow the release to continue.
2309e3f24cc5SChris Mason */
2310e3f24cc5SChris Mason if (ret < 0)
2311e3f24cc5SChris Mason ret = 0;
2312e3f24cc5SChris Mason else
2313e3f24cc5SChris Mason ret = 1;
23147b13b7b1SChris Mason }
23157b13b7b1SChris Mason return ret;
23167b13b7b1SChris Mason }
23177b13b7b1SChris Mason
23187b13b7b1SChris Mason /*
2319f913cff3SMatthew Wilcox (Oracle) * a helper for release_folio. As long as there are no locked extents
2320d1310b2eSChris Mason * in the range corresponding to the page, both state records and extent
2321d1310b2eSChris Mason * map records are removed
2322d1310b2eSChris Mason */
try_release_extent_mapping(struct page * page,gfp_t mask)2323477a30baSNikolay Borisov int try_release_extent_mapping(struct page *page, gfp_t mask)
2324d1310b2eSChris Mason {
2325d1310b2eSChris Mason struct extent_map *em;
23264eee4fa4SMiao Xie u64 start = page_offset(page);
232709cbfeafSKirill A. Shutemov u64 end = start + PAGE_SIZE - 1;
2328bd3599a0SFilipe Manana struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
2329bd3599a0SFilipe Manana struct extent_io_tree *tree = &btrfs_inode->io_tree;
2330bd3599a0SFilipe Manana struct extent_map_tree *map = &btrfs_inode->extent_tree;
23317b13b7b1SChris Mason
2332d0164adcSMel Gorman if (gfpflags_allow_blocking(mask) &&
2333ee22184bSByongho Lee page->mapping->host->i_size > SZ_16M) {
233439b5637fSYan u64 len;
2335d1310b2eSChris Mason while (start <= end) {
2336fbc2bd7eSFilipe Manana struct btrfs_fs_info *fs_info;
2337fbc2bd7eSFilipe Manana u64 cur_gen;
2338fbc2bd7eSFilipe Manana
233939b5637fSYan len = end - start + 1;
2340890871beSChris Mason write_lock(&map->lock);
234139b5637fSYan em = lookup_extent_mapping(map, start, len);
2342285190d9STsutomu Itoh if (!em) {
2343890871beSChris Mason write_unlock(&map->lock);
2344d1310b2eSChris Mason break;
2345d1310b2eSChris Mason }
23467f3c74fbSChris Mason if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
23477f3c74fbSChris Mason em->start != start) {
2348890871beSChris Mason write_unlock(&map->lock);
234970dec807SChris Mason free_extent_map(em);
235070dec807SChris Mason break;
235170dec807SChris Mason }
23523d6448e6SFilipe Manana if (test_range_bit(tree, em->start,
235370dec807SChris Mason extent_map_end(em) - 1,
23543d6448e6SFilipe Manana EXTENT_LOCKED, 0, NULL))
23553d6448e6SFilipe Manana goto next;
23563d6448e6SFilipe Manana /*
23573d6448e6SFilipe Manana * If it's not in the list of modified extents, used
23583d6448e6SFilipe Manana * by a fast fsync, we can remove it. If it's being
23593d6448e6SFilipe Manana * logged we can safely remove it since fsync took an
23603d6448e6SFilipe Manana * extra reference on the em.
23613d6448e6SFilipe Manana */
23623d6448e6SFilipe Manana if (list_empty(&em->list) ||
2363fbc2bd7eSFilipe Manana test_bit(EXTENT_FLAG_LOGGING, &em->flags))
2364fbc2bd7eSFilipe Manana goto remove_em;
2365fbc2bd7eSFilipe Manana /*
2366fbc2bd7eSFilipe Manana * If it's in the list of modified extents, remove it
2367fbc2bd7eSFilipe Manana * only if its generation is older then the current one,
2368fbc2bd7eSFilipe Manana * in which case we don't need it for a fast fsync.
2369fbc2bd7eSFilipe Manana * Otherwise don't remove it, we could be racing with an
2370fbc2bd7eSFilipe Manana * ongoing fast fsync that could miss the new extent.
2371fbc2bd7eSFilipe Manana */
2372fbc2bd7eSFilipe Manana fs_info = btrfs_inode->root->fs_info;
2373fbc2bd7eSFilipe Manana spin_lock(&fs_info->trans_lock);
2374fbc2bd7eSFilipe Manana cur_gen = fs_info->generation;
2375fbc2bd7eSFilipe Manana spin_unlock(&fs_info->trans_lock);
2376fbc2bd7eSFilipe Manana if (em->generation >= cur_gen)
2377fbc2bd7eSFilipe Manana goto next;
2378fbc2bd7eSFilipe Manana remove_em:
23795e548b32SFilipe Manana /*
23805e548b32SFilipe Manana * We only remove extent maps that are not in the list of
23815e548b32SFilipe Manana * modified extents or that are in the list but with a
23825e548b32SFilipe Manana * generation lower then the current generation, so there
23835e548b32SFilipe Manana * is no need to set the full fsync flag on the inode (it
23845e548b32SFilipe Manana * hurts the fsync performance for workloads with a data
23855e548b32SFilipe Manana * size that exceeds or is close to the system's memory).
23865e548b32SFilipe Manana */
2387d1310b2eSChris Mason remove_extent_mapping(map, em);
2388d1310b2eSChris Mason /* once for the rb tree */
2389d1310b2eSChris Mason free_extent_map(em);
23903d6448e6SFilipe Manana next:
2391d1310b2eSChris Mason start = extent_map_end(em);
2392890871beSChris Mason write_unlock(&map->lock);
2393d1310b2eSChris Mason
2394d1310b2eSChris Mason /* once for us */
2395d1310b2eSChris Mason free_extent_map(em);
23969f47eb54SPaul E. McKenney
23979f47eb54SPaul E. McKenney cond_resched(); /* Allow large-extent preemption. */
2398d1310b2eSChris Mason }
239970dec807SChris Mason }
240029c68b2dSNikolay Borisov return try_release_extent_state(tree, page, mask);
2401d1310b2eSChris Mason }
2402d1310b2eSChris Mason
240349d640d2SFilipe Manana struct btrfs_fiemap_entry {
240449d640d2SFilipe Manana u64 offset;
240549d640d2SFilipe Manana u64 phys;
240649d640d2SFilipe Manana u64 len;
240749d640d2SFilipe Manana u32 flags;
240849d640d2SFilipe Manana };
240949d640d2SFilipe Manana
2410ec29ed5bSChris Mason /*
241149d640d2SFilipe Manana * Indicate the caller of emit_fiemap_extent() that it needs to unlock the file
241249d640d2SFilipe Manana * range from the inode's io tree, unlock the subvolume tree search path, flush
241349d640d2SFilipe Manana * the fiemap cache and relock the file range and research the subvolume tree.
241449d640d2SFilipe Manana * The value here is something negative that can't be confused with a valid
241549d640d2SFilipe Manana * errno value and different from 1 because that's also a return value from
241649d640d2SFilipe Manana * fiemap_fill_next_extent() and also it's often used to mean some btree search
241749d640d2SFilipe Manana * did not find a key, so make it some distinct negative value.
241849d640d2SFilipe Manana */
241949d640d2SFilipe Manana #define BTRFS_FIEMAP_FLUSH_CACHE (-(MAX_ERRNO + 1))
242049d640d2SFilipe Manana
242149d640d2SFilipe Manana /*
242249d640d2SFilipe Manana * Used to:
24234751832dSQu Wenruo *
242449d640d2SFilipe Manana * - Cache the next entry to be emitted to the fiemap buffer, so that we can
242549d640d2SFilipe Manana * merge extents that are contiguous and can be grouped as a single one;
242649d640d2SFilipe Manana *
242749d640d2SFilipe Manana * - Store extents ready to be written to the fiemap buffer in an intermediary
242849d640d2SFilipe Manana * buffer. This intermediary buffer is to ensure that in case the fiemap
242949d640d2SFilipe Manana * buffer is memory mapped to the fiemap target file, we don't deadlock
243049d640d2SFilipe Manana * during btrfs_page_mkwrite(). This is because during fiemap we are locking
243149d640d2SFilipe Manana * an extent range in order to prevent races with delalloc flushing and
243249d640d2SFilipe Manana * ordered extent completion, which is needed in order to reliably detect
243349d640d2SFilipe Manana * delalloc in holes and prealloc extents. And this can lead to a deadlock
243449d640d2SFilipe Manana * if the fiemap buffer is memory mapped to the file we are running fiemap
243549d640d2SFilipe Manana * against (a silly, useless in practice scenario, but possible) because
243649d640d2SFilipe Manana * btrfs_page_mkwrite() will try to lock the same extent range.
24374751832dSQu Wenruo */
24384751832dSQu Wenruo struct fiemap_cache {
243949d640d2SFilipe Manana /* An array of ready fiemap entries. */
244049d640d2SFilipe Manana struct btrfs_fiemap_entry *entries;
244149d640d2SFilipe Manana /* Number of entries in the entries array. */
244249d640d2SFilipe Manana int entries_size;
244349d640d2SFilipe Manana /* Index of the next entry in the entries array to write to. */
244449d640d2SFilipe Manana int entries_pos;
244549d640d2SFilipe Manana /*
244649d640d2SFilipe Manana * Once the entries array is full, this indicates what's the offset for
244749d640d2SFilipe Manana * the next file extent item we must search for in the inode's subvolume
244849d640d2SFilipe Manana * tree after unlocking the extent range in the inode's io tree and
244949d640d2SFilipe Manana * releasing the search path.
245049d640d2SFilipe Manana */
245149d640d2SFilipe Manana u64 next_search_offset;
245249d640d2SFilipe Manana /*
245349d640d2SFilipe Manana * This matches struct fiemap_extent_info::fi_mapped_extents, we use it
245449d640d2SFilipe Manana * to count ourselves emitted extents and stop instead of relying on
245549d640d2SFilipe Manana * fiemap_fill_next_extent() because we buffer ready fiemap entries at
245649d640d2SFilipe Manana * the @entries array, and we want to stop as soon as we hit the max
245749d640d2SFilipe Manana * amount of extents to map, not just to save time but also to make the
245849d640d2SFilipe Manana * logic at extent_fiemap() simpler.
245949d640d2SFilipe Manana */
246049d640d2SFilipe Manana unsigned int extents_mapped;
246149d640d2SFilipe Manana /* Fields for the cached extent (unsubmitted, not ready, extent). */
24624751832dSQu Wenruo u64 offset;
24634751832dSQu Wenruo u64 phys;
24644751832dSQu Wenruo u64 len;
24654751832dSQu Wenruo u32 flags;
24664751832dSQu Wenruo bool cached;
24674751832dSQu Wenruo };
24684751832dSQu Wenruo
flush_fiemap_cache(struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache)246949d640d2SFilipe Manana static int flush_fiemap_cache(struct fiemap_extent_info *fieinfo,
247049d640d2SFilipe Manana struct fiemap_cache *cache)
247149d640d2SFilipe Manana {
247249d640d2SFilipe Manana for (int i = 0; i < cache->entries_pos; i++) {
247349d640d2SFilipe Manana struct btrfs_fiemap_entry *entry = &cache->entries[i];
247449d640d2SFilipe Manana int ret;
247549d640d2SFilipe Manana
247649d640d2SFilipe Manana ret = fiemap_fill_next_extent(fieinfo, entry->offset,
247749d640d2SFilipe Manana entry->phys, entry->len,
247849d640d2SFilipe Manana entry->flags);
247949d640d2SFilipe Manana /*
248049d640d2SFilipe Manana * Ignore 1 (reached max entries) because we keep track of that
248149d640d2SFilipe Manana * ourselves in emit_fiemap_extent().
248249d640d2SFilipe Manana */
248349d640d2SFilipe Manana if (ret < 0)
248449d640d2SFilipe Manana return ret;
248549d640d2SFilipe Manana }
248649d640d2SFilipe Manana cache->entries_pos = 0;
248749d640d2SFilipe Manana
248849d640d2SFilipe Manana return 0;
248949d640d2SFilipe Manana }
249049d640d2SFilipe Manana
24914751832dSQu Wenruo /*
24924751832dSQu Wenruo * Helper to submit fiemap extent.
24934751832dSQu Wenruo *
24944751832dSQu Wenruo * Will try to merge current fiemap extent specified by @offset, @phys,
24954751832dSQu Wenruo * @len and @flags with cached one.
24964751832dSQu Wenruo * And only when we fails to merge, cached one will be submitted as
24974751832dSQu Wenruo * fiemap extent.
24984751832dSQu Wenruo *
24994751832dSQu Wenruo * Return value is the same as fiemap_fill_next_extent().
25004751832dSQu Wenruo */
emit_fiemap_extent(struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache,u64 offset,u64 phys,u64 len,u32 flags)25014751832dSQu Wenruo static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
25024751832dSQu Wenruo struct fiemap_cache *cache,
25034751832dSQu Wenruo u64 offset, u64 phys, u64 len, u32 flags)
25044751832dSQu Wenruo {
250549d640d2SFilipe Manana struct btrfs_fiemap_entry *entry;
2506d43f8e58SFilipe Manana u64 cache_end;
25074751832dSQu Wenruo
2508ac3c0d36SFilipe Manana /* Set at the end of extent_fiemap(). */
2509ac3c0d36SFilipe Manana ASSERT((flags & FIEMAP_EXTENT_LAST) == 0);
2510ac3c0d36SFilipe Manana
25114751832dSQu Wenruo if (!cache->cached)
25124751832dSQu Wenruo goto assign;
25134751832dSQu Wenruo
25144751832dSQu Wenruo /*
2515d43f8e58SFilipe Manana * When iterating the extents of the inode, at extent_fiemap(), we may
2516d43f8e58SFilipe Manana * find an extent that starts at an offset behind the end offset of the
2517d43f8e58SFilipe Manana * previous extent we processed. This happens if fiemap is called
2518d43f8e58SFilipe Manana * without FIEMAP_FLAG_SYNC and there are ordered extents completing
251949d640d2SFilipe Manana * after we had to unlock the file range, release the search path, emit
252049d640d2SFilipe Manana * the fiemap extents stored in the buffer (cache->entries array) and
252149d640d2SFilipe Manana * the lock the remainder of the range and re-search the btree.
25224751832dSQu Wenruo *
2523d43f8e58SFilipe Manana * For example we are in leaf X processing its last item, which is the
2524d43f8e58SFilipe Manana * file extent item for file range [512K, 1M[, and after
2525d43f8e58SFilipe Manana * btrfs_next_leaf() releases the path, there's an ordered extent that
2526d43f8e58SFilipe Manana * completes for the file range [768K, 2M[, and that results in trimming
2527d43f8e58SFilipe Manana * the file extent item so that it now corresponds to the file range
2528d43f8e58SFilipe Manana * [512K, 768K[ and a new file extent item is inserted for the file
2529d43f8e58SFilipe Manana * range [768K, 2M[, which may end up as the last item of leaf X or as
2530d43f8e58SFilipe Manana * the first item of the next leaf - in either case btrfs_next_leaf()
2531d43f8e58SFilipe Manana * will leave us with a path pointing to the new extent item, for the
2532d43f8e58SFilipe Manana * file range [768K, 2M[, since that's the first key that follows the
2533d43f8e58SFilipe Manana * last one we processed. So in order not to report overlapping extents
2534d43f8e58SFilipe Manana * to user space, we trim the length of the previously cached extent and
2535d43f8e58SFilipe Manana * emit it.
2536d43f8e58SFilipe Manana *
2537d43f8e58SFilipe Manana * Upon calling btrfs_next_leaf() we may also find an extent with an
2538d43f8e58SFilipe Manana * offset smaller than or equals to cache->offset, and this happens
2539d43f8e58SFilipe Manana * when we had a hole or prealloc extent with several delalloc ranges in
2540d43f8e58SFilipe Manana * it, but after btrfs_next_leaf() released the path, delalloc was
2541d43f8e58SFilipe Manana * flushed and the resulting ordered extents were completed, so we can
2542d43f8e58SFilipe Manana * now have found a file extent item for an offset that is smaller than
2543d43f8e58SFilipe Manana * or equals to what we have in cache->offset. We deal with this as
2544d43f8e58SFilipe Manana * described below.
25454751832dSQu Wenruo */
2546d43f8e58SFilipe Manana cache_end = cache->offset + cache->len;
2547d43f8e58SFilipe Manana if (cache_end > offset) {
2548d43f8e58SFilipe Manana if (offset == cache->offset) {
2549d43f8e58SFilipe Manana /*
2550d43f8e58SFilipe Manana * We cached a dealloc range (found in the io tree) for
2551d43f8e58SFilipe Manana * a hole or prealloc extent and we have now found a
2552d43f8e58SFilipe Manana * file extent item for the same offset. What we have
2553d43f8e58SFilipe Manana * now is more recent and up to date, so discard what
2554d43f8e58SFilipe Manana * we had in the cache and use what we have just found.
2555d43f8e58SFilipe Manana */
2556d43f8e58SFilipe Manana goto assign;
2557d43f8e58SFilipe Manana } else if (offset > cache->offset) {
2558d43f8e58SFilipe Manana /*
2559d43f8e58SFilipe Manana * The extent range we previously found ends after the
2560d43f8e58SFilipe Manana * offset of the file extent item we found and that
2561d43f8e58SFilipe Manana * offset falls somewhere in the middle of that previous
2562d43f8e58SFilipe Manana * extent range. So adjust the range we previously found
2563d43f8e58SFilipe Manana * to end at the offset of the file extent item we have
2564d43f8e58SFilipe Manana * just found, since this extent is more up to date.
2565d43f8e58SFilipe Manana * Emit that adjusted range and cache the file extent
2566d43f8e58SFilipe Manana * item we have just found. This corresponds to the case
2567d43f8e58SFilipe Manana * where a previously found file extent item was split
2568d43f8e58SFilipe Manana * due to an ordered extent completing.
2569d43f8e58SFilipe Manana */
2570d43f8e58SFilipe Manana cache->len = offset - cache->offset;
2571d43f8e58SFilipe Manana goto emit;
2572d43f8e58SFilipe Manana } else {
2573d43f8e58SFilipe Manana const u64 range_end = offset + len;
2574d43f8e58SFilipe Manana
2575d43f8e58SFilipe Manana /*
2576d43f8e58SFilipe Manana * The offset of the file extent item we have just found
2577d43f8e58SFilipe Manana * is behind the cached offset. This means we were
2578d43f8e58SFilipe Manana * processing a hole or prealloc extent for which we
2579d43f8e58SFilipe Manana * have found delalloc ranges (in the io tree), so what
2580d43f8e58SFilipe Manana * we have in the cache is the last delalloc range we
2581d43f8e58SFilipe Manana * found while the file extent item we found can be
2582d43f8e58SFilipe Manana * either for a whole delalloc range we previously
2583d43f8e58SFilipe Manana * emmitted or only a part of that range.
2584d43f8e58SFilipe Manana *
2585d43f8e58SFilipe Manana * We have two cases here:
2586d43f8e58SFilipe Manana *
2587d43f8e58SFilipe Manana * 1) The file extent item's range ends at or behind the
2588d43f8e58SFilipe Manana * cached extent's end. In this case just ignore the
2589d43f8e58SFilipe Manana * current file extent item because we don't want to
2590d43f8e58SFilipe Manana * overlap with previous ranges that may have been
2591d43f8e58SFilipe Manana * emmitted already;
2592d43f8e58SFilipe Manana *
2593d43f8e58SFilipe Manana * 2) The file extent item starts behind the currently
2594d43f8e58SFilipe Manana * cached extent but its end offset goes beyond the
2595d43f8e58SFilipe Manana * end offset of the cached extent. We don't want to
2596d43f8e58SFilipe Manana * overlap with a previous range that may have been
2597d43f8e58SFilipe Manana * emmitted already, so we emit the currently cached
2598d43f8e58SFilipe Manana * extent and then partially store the current file
2599d43f8e58SFilipe Manana * extent item's range in the cache, for the subrange
2600d43f8e58SFilipe Manana * going the cached extent's end to the end of the
2601d43f8e58SFilipe Manana * file extent item.
2602d43f8e58SFilipe Manana */
2603d43f8e58SFilipe Manana if (range_end <= cache_end)
2604d43f8e58SFilipe Manana return 0;
2605d43f8e58SFilipe Manana
2606d43f8e58SFilipe Manana if (!(flags & (FIEMAP_EXTENT_ENCODED | FIEMAP_EXTENT_DELALLOC)))
2607d43f8e58SFilipe Manana phys += cache_end - offset;
2608d43f8e58SFilipe Manana
2609d43f8e58SFilipe Manana offset = cache_end;
2610d43f8e58SFilipe Manana len = range_end - cache_end;
2611d43f8e58SFilipe Manana goto emit;
2612d43f8e58SFilipe Manana }
26134751832dSQu Wenruo }
26144751832dSQu Wenruo
26154751832dSQu Wenruo /*
26164751832dSQu Wenruo * Only merges fiemap extents if
26174751832dSQu Wenruo * 1) Their logical addresses are continuous
26184751832dSQu Wenruo *
26194751832dSQu Wenruo * 2) Their physical addresses are continuous
26204751832dSQu Wenruo * So truly compressed (physical size smaller than logical size)
26214751832dSQu Wenruo * extents won't get merged with each other
26224751832dSQu Wenruo *
2623ac3c0d36SFilipe Manana * 3) Share same flags
26244751832dSQu Wenruo */
26254751832dSQu Wenruo if (cache->offset + cache->len == offset &&
26264751832dSQu Wenruo cache->phys + cache->len == phys &&
2627ac3c0d36SFilipe Manana cache->flags == flags) {
26284751832dSQu Wenruo cache->len += len;
2629ac3c0d36SFilipe Manana return 0;
26304751832dSQu Wenruo }
26314751832dSQu Wenruo
2632d43f8e58SFilipe Manana emit:
26334751832dSQu Wenruo /* Not mergeable, need to submit cached one */
263449d640d2SFilipe Manana
263549d640d2SFilipe Manana if (cache->entries_pos == cache->entries_size) {
263649d640d2SFilipe Manana /*
263749d640d2SFilipe Manana * We will need to research for the end offset of the last
263849d640d2SFilipe Manana * stored extent and not from the current offset, because after
263949d640d2SFilipe Manana * unlocking the range and releasing the path, if there's a hole
264049d640d2SFilipe Manana * between that end offset and this current offset, a new extent
264149d640d2SFilipe Manana * may have been inserted due to a new write, so we don't want
264249d640d2SFilipe Manana * to miss it.
264349d640d2SFilipe Manana */
264449d640d2SFilipe Manana entry = &cache->entries[cache->entries_size - 1];
264549d640d2SFilipe Manana cache->next_search_offset = entry->offset + entry->len;
26464751832dSQu Wenruo cache->cached = false;
264749d640d2SFilipe Manana
264849d640d2SFilipe Manana return BTRFS_FIEMAP_FLUSH_CACHE;
264949d640d2SFilipe Manana }
265049d640d2SFilipe Manana
265149d640d2SFilipe Manana entry = &cache->entries[cache->entries_pos];
265249d640d2SFilipe Manana entry->offset = cache->offset;
265349d640d2SFilipe Manana entry->phys = cache->phys;
265449d640d2SFilipe Manana entry->len = cache->len;
265549d640d2SFilipe Manana entry->flags = cache->flags;
265649d640d2SFilipe Manana cache->entries_pos++;
265749d640d2SFilipe Manana cache->extents_mapped++;
265849d640d2SFilipe Manana
265949d640d2SFilipe Manana if (cache->extents_mapped == fieinfo->fi_extents_max) {
266049d640d2SFilipe Manana cache->cached = false;
266149d640d2SFilipe Manana return 1;
266249d640d2SFilipe Manana }
26634751832dSQu Wenruo assign:
26644751832dSQu Wenruo cache->cached = true;
26654751832dSQu Wenruo cache->offset = offset;
26664751832dSQu Wenruo cache->phys = phys;
26674751832dSQu Wenruo cache->len = len;
26684751832dSQu Wenruo cache->flags = flags;
2669ac3c0d36SFilipe Manana
2670ac3c0d36SFilipe Manana return 0;
26714751832dSQu Wenruo }
26724751832dSQu Wenruo
26734751832dSQu Wenruo /*
2674848c23b7SQu Wenruo * Emit last fiemap cache
26754751832dSQu Wenruo *
2676848c23b7SQu Wenruo * The last fiemap cache may still be cached in the following case:
2677848c23b7SQu Wenruo * 0 4k 8k
2678848c23b7SQu Wenruo * |<- Fiemap range ->|
2679848c23b7SQu Wenruo * |<------------ First extent ----------->|
2680848c23b7SQu Wenruo *
2681848c23b7SQu Wenruo * In this case, the first extent range will be cached but not emitted.
2682848c23b7SQu Wenruo * So we must emit it before ending extent_fiemap().
26834751832dSQu Wenruo */
emit_last_fiemap_cache(struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache)26845c5aff98SDavid Sterba static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
26854751832dSQu Wenruo struct fiemap_cache *cache)
26864751832dSQu Wenruo {
26874751832dSQu Wenruo int ret;
26884751832dSQu Wenruo
26894751832dSQu Wenruo if (!cache->cached)
26904751832dSQu Wenruo return 0;
26914751832dSQu Wenruo
26924751832dSQu Wenruo ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
26934751832dSQu Wenruo cache->len, cache->flags);
26944751832dSQu Wenruo cache->cached = false;
26954751832dSQu Wenruo if (ret > 0)
26964751832dSQu Wenruo ret = 0;
26974751832dSQu Wenruo return ret;
26984751832dSQu Wenruo }
26994751832dSQu Wenruo
fiemap_next_leaf_item(struct btrfs_inode * inode,struct btrfs_path * path)2700ac3c0d36SFilipe Manana static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path)
2701ac3c0d36SFilipe Manana {
2702ac3c0d36SFilipe Manana struct extent_buffer *clone;
2703ac3c0d36SFilipe Manana struct btrfs_key key;
2704ac3c0d36SFilipe Manana int slot;
2705ac3c0d36SFilipe Manana int ret;
2706ac3c0d36SFilipe Manana
2707ac3c0d36SFilipe Manana path->slots[0]++;
2708ac3c0d36SFilipe Manana if (path->slots[0] < btrfs_header_nritems(path->nodes[0]))
2709ac3c0d36SFilipe Manana return 0;
2710ac3c0d36SFilipe Manana
2711ac3c0d36SFilipe Manana ret = btrfs_next_leaf(inode->root, path);
2712ac3c0d36SFilipe Manana if (ret != 0)
2713ac3c0d36SFilipe Manana return ret;
2714ac3c0d36SFilipe Manana
2715ac3c0d36SFilipe Manana /*
2716ac3c0d36SFilipe Manana * Don't bother with cloning if there are no more file extent items for
2717ac3c0d36SFilipe Manana * our inode.
2718ac3c0d36SFilipe Manana */
2719ac3c0d36SFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2720ac3c0d36SFilipe Manana if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY)
2721ac3c0d36SFilipe Manana return 1;
2722ac3c0d36SFilipe Manana
2723ac3c0d36SFilipe Manana /* See the comment at fiemap_search_slot() about why we clone. */
2724ac3c0d36SFilipe Manana clone = btrfs_clone_extent_buffer(path->nodes[0]);
2725ac3c0d36SFilipe Manana if (!clone)
2726ac3c0d36SFilipe Manana return -ENOMEM;
2727ac3c0d36SFilipe Manana
2728ac3c0d36SFilipe Manana slot = path->slots[0];
2729ac3c0d36SFilipe Manana btrfs_release_path(path);
2730ac3c0d36SFilipe Manana path->nodes[0] = clone;
2731ac3c0d36SFilipe Manana path->slots[0] = slot;
2732ac3c0d36SFilipe Manana
2733ac3c0d36SFilipe Manana return 0;
2734ac3c0d36SFilipe Manana }
2735ac3c0d36SFilipe Manana
2736ac3c0d36SFilipe Manana /*
2737ac3c0d36SFilipe Manana * Search for the first file extent item that starts at a given file offset or
2738ac3c0d36SFilipe Manana * the one that starts immediately before that offset.
2739ac3c0d36SFilipe Manana * Returns: 0 on success, < 0 on error, 1 if not found.
2740ac3c0d36SFilipe Manana */
fiemap_search_slot(struct btrfs_inode * inode,struct btrfs_path * path,u64 file_offset)2741ac3c0d36SFilipe Manana static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path,
2742ac3c0d36SFilipe Manana u64 file_offset)
2743ac3c0d36SFilipe Manana {
2744ac3c0d36SFilipe Manana const u64 ino = btrfs_ino(inode);
2745ac3c0d36SFilipe Manana struct btrfs_root *root = inode->root;
2746ac3c0d36SFilipe Manana struct extent_buffer *clone;
2747ac3c0d36SFilipe Manana struct btrfs_key key;
2748ac3c0d36SFilipe Manana int slot;
2749ac3c0d36SFilipe Manana int ret;
2750ac3c0d36SFilipe Manana
2751ac3c0d36SFilipe Manana key.objectid = ino;
2752ac3c0d36SFilipe Manana key.type = BTRFS_EXTENT_DATA_KEY;
2753ac3c0d36SFilipe Manana key.offset = file_offset;
2754ac3c0d36SFilipe Manana
2755ac3c0d36SFilipe Manana ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2756ac3c0d36SFilipe Manana if (ret < 0)
2757ac3c0d36SFilipe Manana return ret;
2758ac3c0d36SFilipe Manana
2759ac3c0d36SFilipe Manana if (ret > 0 && path->slots[0] > 0) {
2760ac3c0d36SFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
2761ac3c0d36SFilipe Manana if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
2762ac3c0d36SFilipe Manana path->slots[0]--;
2763ac3c0d36SFilipe Manana }
2764ac3c0d36SFilipe Manana
2765ac3c0d36SFilipe Manana if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2766ac3c0d36SFilipe Manana ret = btrfs_next_leaf(root, path);
2767ac3c0d36SFilipe Manana if (ret != 0)
2768ac3c0d36SFilipe Manana return ret;
2769ac3c0d36SFilipe Manana
2770ac3c0d36SFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2771ac3c0d36SFilipe Manana if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2772ac3c0d36SFilipe Manana return 1;
2773ac3c0d36SFilipe Manana }
2774ac3c0d36SFilipe Manana
2775ac3c0d36SFilipe Manana /*
2776ac3c0d36SFilipe Manana * We clone the leaf and use it during fiemap. This is because while
2777ac3c0d36SFilipe Manana * using the leaf we do expensive things like checking if an extent is
2778ac3c0d36SFilipe Manana * shared, which can take a long time. In order to prevent blocking
2779ac3c0d36SFilipe Manana * other tasks for too long, we use a clone of the leaf. We have locked
2780ac3c0d36SFilipe Manana * the file range in the inode's io tree, so we know none of our file
2781ac3c0d36SFilipe Manana * extent items can change. This way we avoid blocking other tasks that
2782ac3c0d36SFilipe Manana * want to insert items for other inodes in the same leaf or b+tree
2783ac3c0d36SFilipe Manana * rebalance operations (triggered for example when someone is trying
2784ac3c0d36SFilipe Manana * to push items into this leaf when trying to insert an item in a
2785ac3c0d36SFilipe Manana * neighbour leaf).
2786ac3c0d36SFilipe Manana * We also need the private clone because holding a read lock on an
2787ac3c0d36SFilipe Manana * extent buffer of the subvolume's b+tree will make lockdep unhappy
278849d640d2SFilipe Manana * when we check if extents are shared, as backref walking may need to
278949d640d2SFilipe Manana * lock the same leaf we are processing.
2790ac3c0d36SFilipe Manana */
2791ac3c0d36SFilipe Manana clone = btrfs_clone_extent_buffer(path->nodes[0]);
2792ac3c0d36SFilipe Manana if (!clone)
2793ac3c0d36SFilipe Manana return -ENOMEM;
2794ac3c0d36SFilipe Manana
2795ac3c0d36SFilipe Manana slot = path->slots[0];
2796ac3c0d36SFilipe Manana btrfs_release_path(path);
2797ac3c0d36SFilipe Manana path->nodes[0] = clone;
2798ac3c0d36SFilipe Manana path->slots[0] = slot;
2799ac3c0d36SFilipe Manana
2800ac3c0d36SFilipe Manana return 0;
2801ac3c0d36SFilipe Manana }
2802ac3c0d36SFilipe Manana
2803ac3c0d36SFilipe Manana /*
2804ac3c0d36SFilipe Manana * Process a range which is a hole or a prealloc extent in the inode's subvolume
2805ac3c0d36SFilipe Manana * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc
2806ac3c0d36SFilipe Manana * extent. The end offset (@end) is inclusive.
2807ac3c0d36SFilipe Manana */
fiemap_process_hole(struct btrfs_inode * inode,struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache,struct extent_state ** delalloc_cached_state,struct btrfs_backref_share_check_ctx * backref_ctx,u64 disk_bytenr,u64 extent_offset,u64 extent_gen,u64 start,u64 end)2808ac3c0d36SFilipe Manana static int fiemap_process_hole(struct btrfs_inode *inode,
2809ac3c0d36SFilipe Manana struct fiemap_extent_info *fieinfo,
2810ac3c0d36SFilipe Manana struct fiemap_cache *cache,
2811b3e744feSFilipe Manana struct extent_state **delalloc_cached_state,
281261dbb952SFilipe Manana struct btrfs_backref_share_check_ctx *backref_ctx,
2813ac3c0d36SFilipe Manana u64 disk_bytenr, u64 extent_offset,
2814ac3c0d36SFilipe Manana u64 extent_gen,
2815ac3c0d36SFilipe Manana u64 start, u64 end)
2816ac3c0d36SFilipe Manana {
2817ac3c0d36SFilipe Manana const u64 i_size = i_size_read(&inode->vfs_inode);
2818ac3c0d36SFilipe Manana u64 cur_offset = start;
2819ac3c0d36SFilipe Manana u64 last_delalloc_end = 0;
2820ac3c0d36SFilipe Manana u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN;
2821ac3c0d36SFilipe Manana bool checked_extent_shared = false;
2822ac3c0d36SFilipe Manana int ret;
2823ac3c0d36SFilipe Manana
2824ac3c0d36SFilipe Manana /*
2825ac3c0d36SFilipe Manana * There can be no delalloc past i_size, so don't waste time looking for
2826ac3c0d36SFilipe Manana * it beyond i_size.
2827ac3c0d36SFilipe Manana */
2828ac3c0d36SFilipe Manana while (cur_offset < end && cur_offset < i_size) {
2829ac3c0d36SFilipe Manana u64 delalloc_start;
2830ac3c0d36SFilipe Manana u64 delalloc_end;
2831ac3c0d36SFilipe Manana u64 prealloc_start;
2832ac3c0d36SFilipe Manana u64 prealloc_len = 0;
2833ac3c0d36SFilipe Manana bool delalloc;
2834ac3c0d36SFilipe Manana
2835ac3c0d36SFilipe Manana delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end,
2836b3e744feSFilipe Manana delalloc_cached_state,
2837ac3c0d36SFilipe Manana &delalloc_start,
2838ac3c0d36SFilipe Manana &delalloc_end);
2839ac3c0d36SFilipe Manana if (!delalloc)
2840ac3c0d36SFilipe Manana break;
2841ac3c0d36SFilipe Manana
2842ac3c0d36SFilipe Manana /*
2843ac3c0d36SFilipe Manana * If this is a prealloc extent we have to report every section
2844ac3c0d36SFilipe Manana * of it that has no delalloc.
2845ac3c0d36SFilipe Manana */
2846ac3c0d36SFilipe Manana if (disk_bytenr != 0) {
2847ac3c0d36SFilipe Manana if (last_delalloc_end == 0) {
2848ac3c0d36SFilipe Manana prealloc_start = start;
2849ac3c0d36SFilipe Manana prealloc_len = delalloc_start - start;
2850ac3c0d36SFilipe Manana } else {
2851ac3c0d36SFilipe Manana prealloc_start = last_delalloc_end + 1;
2852ac3c0d36SFilipe Manana prealloc_len = delalloc_start - prealloc_start;
2853ac3c0d36SFilipe Manana }
2854ac3c0d36SFilipe Manana }
2855ac3c0d36SFilipe Manana
2856ac3c0d36SFilipe Manana if (prealloc_len > 0) {
2857ac3c0d36SFilipe Manana if (!checked_extent_shared && fieinfo->fi_extents_max) {
2858ceb707daSFilipe Manana ret = btrfs_is_data_extent_shared(inode,
2859ceb707daSFilipe Manana disk_bytenr,
286084a7949dSFilipe Manana extent_gen,
286161dbb952SFilipe Manana backref_ctx);
2862ac3c0d36SFilipe Manana if (ret < 0)
2863ac3c0d36SFilipe Manana return ret;
2864ac3c0d36SFilipe Manana else if (ret > 0)
2865ac3c0d36SFilipe Manana prealloc_flags |= FIEMAP_EXTENT_SHARED;
2866ac3c0d36SFilipe Manana
2867ac3c0d36SFilipe Manana checked_extent_shared = true;
2868ac3c0d36SFilipe Manana }
2869ac3c0d36SFilipe Manana ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2870ac3c0d36SFilipe Manana disk_bytenr + extent_offset,
2871ac3c0d36SFilipe Manana prealloc_len, prealloc_flags);
2872ac3c0d36SFilipe Manana if (ret)
2873ac3c0d36SFilipe Manana return ret;
2874ac3c0d36SFilipe Manana extent_offset += prealloc_len;
2875ac3c0d36SFilipe Manana }
2876ac3c0d36SFilipe Manana
2877ac3c0d36SFilipe Manana ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0,
2878ac3c0d36SFilipe Manana delalloc_end + 1 - delalloc_start,
2879ac3c0d36SFilipe Manana FIEMAP_EXTENT_DELALLOC |
2880ac3c0d36SFilipe Manana FIEMAP_EXTENT_UNKNOWN);
2881ac3c0d36SFilipe Manana if (ret)
2882ac3c0d36SFilipe Manana return ret;
2883ac3c0d36SFilipe Manana
2884ac3c0d36SFilipe Manana last_delalloc_end = delalloc_end;
2885ac3c0d36SFilipe Manana cur_offset = delalloc_end + 1;
2886ac3c0d36SFilipe Manana extent_offset += cur_offset - delalloc_start;
2887ac3c0d36SFilipe Manana cond_resched();
2888ac3c0d36SFilipe Manana }
2889ac3c0d36SFilipe Manana
2890ac3c0d36SFilipe Manana /*
2891ac3c0d36SFilipe Manana * Either we found no delalloc for the whole prealloc extent or we have
2892ac3c0d36SFilipe Manana * a prealloc extent that spans i_size or starts at or after i_size.
2893ac3c0d36SFilipe Manana */
2894ac3c0d36SFilipe Manana if (disk_bytenr != 0 && last_delalloc_end < end) {
2895ac3c0d36SFilipe Manana u64 prealloc_start;
2896ac3c0d36SFilipe Manana u64 prealloc_len;
2897ac3c0d36SFilipe Manana
2898ac3c0d36SFilipe Manana if (last_delalloc_end == 0) {
2899ac3c0d36SFilipe Manana prealloc_start = start;
2900ac3c0d36SFilipe Manana prealloc_len = end + 1 - start;
2901ac3c0d36SFilipe Manana } else {
2902ac3c0d36SFilipe Manana prealloc_start = last_delalloc_end + 1;
2903ac3c0d36SFilipe Manana prealloc_len = end + 1 - prealloc_start;
2904ac3c0d36SFilipe Manana }
2905ac3c0d36SFilipe Manana
2906ac3c0d36SFilipe Manana if (!checked_extent_shared && fieinfo->fi_extents_max) {
2907ceb707daSFilipe Manana ret = btrfs_is_data_extent_shared(inode,
2908ceb707daSFilipe Manana disk_bytenr,
290984a7949dSFilipe Manana extent_gen,
291061dbb952SFilipe Manana backref_ctx);
2911ac3c0d36SFilipe Manana if (ret < 0)
2912ac3c0d36SFilipe Manana return ret;
2913ac3c0d36SFilipe Manana else if (ret > 0)
2914ac3c0d36SFilipe Manana prealloc_flags |= FIEMAP_EXTENT_SHARED;
2915ac3c0d36SFilipe Manana }
2916ac3c0d36SFilipe Manana ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2917ac3c0d36SFilipe Manana disk_bytenr + extent_offset,
2918ac3c0d36SFilipe Manana prealloc_len, prealloc_flags);
2919ac3c0d36SFilipe Manana if (ret)
2920ac3c0d36SFilipe Manana return ret;
2921ac3c0d36SFilipe Manana }
2922ac3c0d36SFilipe Manana
2923ac3c0d36SFilipe Manana return 0;
2924ac3c0d36SFilipe Manana }
2925ac3c0d36SFilipe Manana
fiemap_find_last_extent_offset(struct btrfs_inode * inode,struct btrfs_path * path,u64 * last_extent_end_ret)2926ac3c0d36SFilipe Manana static int fiemap_find_last_extent_offset(struct btrfs_inode *inode,
2927ac3c0d36SFilipe Manana struct btrfs_path *path,
2928ac3c0d36SFilipe Manana u64 *last_extent_end_ret)
2929ac3c0d36SFilipe Manana {
2930ac3c0d36SFilipe Manana const u64 ino = btrfs_ino(inode);
2931ac3c0d36SFilipe Manana struct btrfs_root *root = inode->root;
2932ac3c0d36SFilipe Manana struct extent_buffer *leaf;
2933ac3c0d36SFilipe Manana struct btrfs_file_extent_item *ei;
2934ac3c0d36SFilipe Manana struct btrfs_key key;
2935ac3c0d36SFilipe Manana u64 disk_bytenr;
2936ac3c0d36SFilipe Manana int ret;
2937ac3c0d36SFilipe Manana
2938ac3c0d36SFilipe Manana /*
2939ac3c0d36SFilipe Manana * Lookup the last file extent. We're not using i_size here because
2940ac3c0d36SFilipe Manana * there might be preallocation past i_size.
2941ac3c0d36SFilipe Manana */
2942ac3c0d36SFilipe Manana ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0);
2943ac3c0d36SFilipe Manana /* There can't be a file extent item at offset (u64)-1 */
2944ac3c0d36SFilipe Manana ASSERT(ret != 0);
2945ac3c0d36SFilipe Manana if (ret < 0)
2946ac3c0d36SFilipe Manana return ret;
2947ac3c0d36SFilipe Manana
2948ac3c0d36SFilipe Manana /*
2949ac3c0d36SFilipe Manana * For a non-existing key, btrfs_search_slot() always leaves us at a
2950ac3c0d36SFilipe Manana * slot > 0, except if the btree is empty, which is impossible because
2951ac3c0d36SFilipe Manana * at least it has the inode item for this inode and all the items for
2952ac3c0d36SFilipe Manana * the root inode 256.
2953ac3c0d36SFilipe Manana */
2954ac3c0d36SFilipe Manana ASSERT(path->slots[0] > 0);
2955ac3c0d36SFilipe Manana path->slots[0]--;
2956ac3c0d36SFilipe Manana leaf = path->nodes[0];
2957ac3c0d36SFilipe Manana btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2958ac3c0d36SFilipe Manana if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) {
2959ac3c0d36SFilipe Manana /* No file extent items in the subvolume tree. */
2960ac3c0d36SFilipe Manana *last_extent_end_ret = 0;
2961ac3c0d36SFilipe Manana return 0;
2962ac3c0d36SFilipe Manana }
2963ac3c0d36SFilipe Manana
2964ac3c0d36SFilipe Manana /*
2965ac3c0d36SFilipe Manana * For an inline extent, the disk_bytenr is where inline data starts at,
2966ac3c0d36SFilipe Manana * so first check if we have an inline extent item before checking if we
2967ac3c0d36SFilipe Manana * have an implicit hole (disk_bytenr == 0).
2968ac3c0d36SFilipe Manana */
2969ac3c0d36SFilipe Manana ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
2970ac3c0d36SFilipe Manana if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) {
2971ac3c0d36SFilipe Manana *last_extent_end_ret = btrfs_file_extent_end(path);
2972ac3c0d36SFilipe Manana return 0;
2973ac3c0d36SFilipe Manana }
2974ac3c0d36SFilipe Manana
2975ac3c0d36SFilipe Manana /*
2976ac3c0d36SFilipe Manana * Find the last file extent item that is not a hole (when NO_HOLES is
2977ac3c0d36SFilipe Manana * not enabled). This should take at most 2 iterations in the worst
2978ac3c0d36SFilipe Manana * case: we have one hole file extent item at slot 0 of a leaf and
2979ac3c0d36SFilipe Manana * another hole file extent item as the last item in the previous leaf.
2980ac3c0d36SFilipe Manana * This is because we merge file extent items that represent holes.
2981ac3c0d36SFilipe Manana */
2982ac3c0d36SFilipe Manana disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2983ac3c0d36SFilipe Manana while (disk_bytenr == 0) {
2984ac3c0d36SFilipe Manana ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY);
2985ac3c0d36SFilipe Manana if (ret < 0) {
2986ac3c0d36SFilipe Manana return ret;
2987ac3c0d36SFilipe Manana } else if (ret > 0) {
2988ac3c0d36SFilipe Manana /* No file extent items that are not holes. */
2989ac3c0d36SFilipe Manana *last_extent_end_ret = 0;
2990ac3c0d36SFilipe Manana return 0;
2991ac3c0d36SFilipe Manana }
2992ac3c0d36SFilipe Manana leaf = path->nodes[0];
2993ac3c0d36SFilipe Manana ei = btrfs_item_ptr(leaf, path->slots[0],
2994ac3c0d36SFilipe Manana struct btrfs_file_extent_item);
2995ac3c0d36SFilipe Manana disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2996ac3c0d36SFilipe Manana }
2997ac3c0d36SFilipe Manana
2998ac3c0d36SFilipe Manana *last_extent_end_ret = btrfs_file_extent_end(path);
2999ac3c0d36SFilipe Manana return 0;
3000ac3c0d36SFilipe Manana }
3001ac3c0d36SFilipe Manana
extent_fiemap(struct btrfs_inode * inode,struct fiemap_extent_info * fieinfo,u64 start,u64 len)3002facee0a0SNikolay Borisov int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
3003bab16e21SDavid Sterba u64 start, u64 len)
30041506fcc8SYehuda Sadeh {
3005ac3c0d36SFilipe Manana const u64 ino = btrfs_ino(inode);
300649d640d2SFilipe Manana struct extent_state *cached_state = NULL;
3007b3e744feSFilipe Manana struct extent_state *delalloc_cached_state = NULL;
3008975f84feSJosef Bacik struct btrfs_path *path;
30094751832dSQu Wenruo struct fiemap_cache cache = { 0 };
301061dbb952SFilipe Manana struct btrfs_backref_share_check_ctx *backref_ctx;
3011ac3c0d36SFilipe Manana u64 last_extent_end;
3012ac3c0d36SFilipe Manana u64 prev_extent_end;
3013ded566b4SJosef Bacik u64 range_start;
3014ded566b4SJosef Bacik u64 range_end;
3015ded566b4SJosef Bacik const u64 sectorsize = inode->root->fs_info->sectorsize;
3016ac3c0d36SFilipe Manana bool stopped = false;
3017ac3c0d36SFilipe Manana int ret;
30181506fcc8SYehuda Sadeh
301949d640d2SFilipe Manana cache.entries_size = PAGE_SIZE / sizeof(struct btrfs_fiemap_entry);
302049d640d2SFilipe Manana cache.entries = kmalloc_array(cache.entries_size,
302149d640d2SFilipe Manana sizeof(struct btrfs_fiemap_entry),
302249d640d2SFilipe Manana GFP_KERNEL);
302384a7949dSFilipe Manana backref_ctx = btrfs_alloc_backref_share_check_ctx();
3024975f84feSJosef Bacik path = btrfs_alloc_path();
302549d640d2SFilipe Manana if (!cache.entries || !backref_ctx || !path) {
30265911c8feSDavid Sterba ret = -ENOMEM;
3027ac3c0d36SFilipe Manana goto out;
30285911c8feSDavid Sterba }
30295911c8feSDavid Sterba
303049d640d2SFilipe Manana restart:
3031ded566b4SJosef Bacik range_start = round_down(start, sectorsize);
3032ded566b4SJosef Bacik range_end = round_up(start + len, sectorsize);
3033ded566b4SJosef Bacik prev_extent_end = range_start;
30344d479cf0SJosef Bacik
303549d640d2SFilipe Manana lock_extent(&inode->io_tree, range_start, range_end, &cached_state);
303649d640d2SFilipe Manana
3037ac3c0d36SFilipe Manana ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end);
3038ac3c0d36SFilipe Manana if (ret < 0)
303949d640d2SFilipe Manana goto out_unlock;
3040fe09e16cSLiu Bo btrfs_release_path(path);
3041975f84feSJosef Bacik
3042ac3c0d36SFilipe Manana path->reada = READA_FORWARD;
3043ded566b4SJosef Bacik ret = fiemap_search_slot(inode, path, range_start);
3044ac3c0d36SFilipe Manana if (ret < 0) {
304549d640d2SFilipe Manana goto out_unlock;
3046ac3c0d36SFilipe Manana } else if (ret > 0) {
3047ec29ed5bSChris Mason /*
3048ac3c0d36SFilipe Manana * No file extent item found, but we may have delalloc between
3049ac3c0d36SFilipe Manana * the current offset and i_size. So check for that.
3050ec29ed5bSChris Mason */
3051ac3c0d36SFilipe Manana ret = 0;
3052ac3c0d36SFilipe Manana goto check_eof_delalloc;
3053ec29ed5bSChris Mason }
3054ec29ed5bSChris Mason
3055ded566b4SJosef Bacik while (prev_extent_end < range_end) {
3056ac3c0d36SFilipe Manana struct extent_buffer *leaf = path->nodes[0];
3057ac3c0d36SFilipe Manana struct btrfs_file_extent_item *ei;
3058ac3c0d36SFilipe Manana struct btrfs_key key;
3059ac3c0d36SFilipe Manana u64 extent_end;
3060ac3c0d36SFilipe Manana u64 extent_len;
3061ac3c0d36SFilipe Manana u64 extent_offset = 0;
3062ac3c0d36SFilipe Manana u64 extent_gen;
3063ac3c0d36SFilipe Manana u64 disk_bytenr = 0;
3064ac3c0d36SFilipe Manana u64 flags = 0;
3065ac3c0d36SFilipe Manana int extent_type;
3066ac3c0d36SFilipe Manana u8 compression;
3067ec29ed5bSChris Mason
3068ac3c0d36SFilipe Manana btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
3069ac3c0d36SFilipe Manana if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
3070ea8efc74SChris Mason break;
3071ea8efc74SChris Mason
3072ac3c0d36SFilipe Manana extent_end = btrfs_file_extent_end(path);
3073ea8efc74SChris Mason
3074ea8efc74SChris Mason /*
3075ac3c0d36SFilipe Manana * The first iteration can leave us at an extent item that ends
3076ac3c0d36SFilipe Manana * before our range's start. Move to the next item.
3077ea8efc74SChris Mason */
3078ded566b4SJosef Bacik if (extent_end <= range_start)
3079ac3c0d36SFilipe Manana goto next_item;
30801506fcc8SYehuda Sadeh
3081877c1476SFilipe Manana backref_ctx->curr_leaf_bytenr = leaf->start;
3082877c1476SFilipe Manana
3083ac3c0d36SFilipe Manana /* We have in implicit hole (NO_HOLES feature enabled). */
3084ac3c0d36SFilipe Manana if (prev_extent_end < key.offset) {
3085ded566b4SJosef Bacik const u64 hole_end = min(key.offset, range_end) - 1;
3086ea8efc74SChris Mason
3087ac3c0d36SFilipe Manana ret = fiemap_process_hole(inode, fieinfo, &cache,
3088b3e744feSFilipe Manana &delalloc_cached_state,
308961dbb952SFilipe Manana backref_ctx, 0, 0, 0,
3090ded566b4SJosef Bacik prev_extent_end, hole_end);
3091ac3c0d36SFilipe Manana if (ret < 0) {
309249d640d2SFilipe Manana goto out_unlock;
3093ac3c0d36SFilipe Manana } else if (ret > 0) {
3094ac3c0d36SFilipe Manana /* fiemap_fill_next_extent() told us to stop. */
3095ac3c0d36SFilipe Manana stopped = true;
3096ac3c0d36SFilipe Manana break;
3097ac3c0d36SFilipe Manana }
3098ac3c0d36SFilipe Manana
3099ac3c0d36SFilipe Manana /* We've reached the end of the fiemap range, stop. */
3100ded566b4SJosef Bacik if (key.offset >= range_end) {
3101ac3c0d36SFilipe Manana stopped = true;
3102ac3c0d36SFilipe Manana break;
3103ac3c0d36SFilipe Manana }
3104ac3c0d36SFilipe Manana }
3105ac3c0d36SFilipe Manana
3106ac3c0d36SFilipe Manana extent_len = extent_end - key.offset;
3107ac3c0d36SFilipe Manana ei = btrfs_item_ptr(leaf, path->slots[0],
3108ac3c0d36SFilipe Manana struct btrfs_file_extent_item);
3109ac3c0d36SFilipe Manana compression = btrfs_file_extent_compression(leaf, ei);
3110ac3c0d36SFilipe Manana extent_type = btrfs_file_extent_type(leaf, ei);
3111ac3c0d36SFilipe Manana extent_gen = btrfs_file_extent_generation(leaf, ei);
3112ac3c0d36SFilipe Manana
3113ac3c0d36SFilipe Manana if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
3114ac3c0d36SFilipe Manana disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
3115ac3c0d36SFilipe Manana if (compression == BTRFS_COMPRESS_NONE)
3116ac3c0d36SFilipe Manana extent_offset = btrfs_file_extent_offset(leaf, ei);
3117ac3c0d36SFilipe Manana }
3118ac3c0d36SFilipe Manana
3119ac3c0d36SFilipe Manana if (compression != BTRFS_COMPRESS_NONE)
3120ac3c0d36SFilipe Manana flags |= FIEMAP_EXTENT_ENCODED;
3121ac3c0d36SFilipe Manana
3122ac3c0d36SFilipe Manana if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
3123ac3c0d36SFilipe Manana flags |= FIEMAP_EXTENT_DATA_INLINE;
3124ac3c0d36SFilipe Manana flags |= FIEMAP_EXTENT_NOT_ALIGNED;
3125ac3c0d36SFilipe Manana ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0,
3126ac3c0d36SFilipe Manana extent_len, flags);
3127ac3c0d36SFilipe Manana } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
3128ac3c0d36SFilipe Manana ret = fiemap_process_hole(inode, fieinfo, &cache,
3129b3e744feSFilipe Manana &delalloc_cached_state,
313061dbb952SFilipe Manana backref_ctx,
3131ac3c0d36SFilipe Manana disk_bytenr, extent_offset,
313284a7949dSFilipe Manana extent_gen, key.offset,
313384a7949dSFilipe Manana extent_end - 1);
3134ac3c0d36SFilipe Manana } else if (disk_bytenr == 0) {
3135ac3c0d36SFilipe Manana /* We have an explicit hole. */
3136ac3c0d36SFilipe Manana ret = fiemap_process_hole(inode, fieinfo, &cache,
3137b3e744feSFilipe Manana &delalloc_cached_state,
313861dbb952SFilipe Manana backref_ctx, 0, 0, 0,
3139ac3c0d36SFilipe Manana key.offset, extent_end - 1);
3140ac3c0d36SFilipe Manana } else {
3141ac3c0d36SFilipe Manana /* We have a regular extent. */
3142ac3c0d36SFilipe Manana if (fieinfo->fi_extents_max) {
3143ceb707daSFilipe Manana ret = btrfs_is_data_extent_shared(inode,
3144ac3c0d36SFilipe Manana disk_bytenr,
3145ac3c0d36SFilipe Manana extent_gen,
314661dbb952SFilipe Manana backref_ctx);
3147dc046b10SJosef Bacik if (ret < 0)
314849d640d2SFilipe Manana goto out_unlock;
3149ac3c0d36SFilipe Manana else if (ret > 0)
3150fe09e16cSLiu Bo flags |= FIEMAP_EXTENT_SHARED;
3151ec29ed5bSChris Mason }
3152ec29ed5bSChris Mason
3153ac3c0d36SFilipe Manana ret = emit_fiemap_extent(fieinfo, &cache, key.offset,
3154ac3c0d36SFilipe Manana disk_bytenr + extent_offset,
3155ac3c0d36SFilipe Manana extent_len, flags);
31561506fcc8SYehuda Sadeh }
315709fbc1c8SFilipe Manana
3158ac3c0d36SFilipe Manana if (ret < 0) {
315949d640d2SFilipe Manana goto out_unlock;
3160ac3c0d36SFilipe Manana } else if (ret > 0) {
316149d640d2SFilipe Manana /* emit_fiemap_extent() told us to stop. */
3162ac3c0d36SFilipe Manana stopped = true;
3163ac3c0d36SFilipe Manana break;
3164ac3c0d36SFilipe Manana }
3165ac3c0d36SFilipe Manana
3166ac3c0d36SFilipe Manana prev_extent_end = extent_end;
3167ac3c0d36SFilipe Manana next_item:
316809fbc1c8SFilipe Manana if (fatal_signal_pending(current)) {
316909fbc1c8SFilipe Manana ret = -EINTR;
317049d640d2SFilipe Manana goto out_unlock;
317109fbc1c8SFilipe Manana }
31725911c8feSDavid Sterba
3173ac3c0d36SFilipe Manana ret = fiemap_next_leaf_item(inode, path);
3174ac3c0d36SFilipe Manana if (ret < 0) {
317549d640d2SFilipe Manana goto out_unlock;
3176ac3c0d36SFilipe Manana } else if (ret > 0) {
3177ac3c0d36SFilipe Manana /* No more file extent items for this inode. */
3178ac3c0d36SFilipe Manana break;
3179ac3c0d36SFilipe Manana }
3180ac3c0d36SFilipe Manana cond_resched();
3181ac3c0d36SFilipe Manana }
3182ac3c0d36SFilipe Manana
3183ac3c0d36SFilipe Manana check_eof_delalloc:
3184ded566b4SJosef Bacik if (!stopped && prev_extent_end < range_end) {
3185b3e744feSFilipe Manana ret = fiemap_process_hole(inode, fieinfo, &cache,
3186b3e744feSFilipe Manana &delalloc_cached_state, backref_ctx,
3187ded566b4SJosef Bacik 0, 0, 0, prev_extent_end, range_end - 1);
3188ac3c0d36SFilipe Manana if (ret < 0)
318949d640d2SFilipe Manana goto out_unlock;
3190ded566b4SJosef Bacik prev_extent_end = range_end;
3191ac3c0d36SFilipe Manana }
3192ac3c0d36SFilipe Manana
3193ac3c0d36SFilipe Manana if (cache.cached && cache.offset + cache.len >= last_extent_end) {
3194ac3c0d36SFilipe Manana const u64 i_size = i_size_read(&inode->vfs_inode);
3195ac3c0d36SFilipe Manana
3196ac3c0d36SFilipe Manana if (prev_extent_end < i_size) {
3197ac3c0d36SFilipe Manana u64 delalloc_start;
3198ac3c0d36SFilipe Manana u64 delalloc_end;
3199ac3c0d36SFilipe Manana bool delalloc;
3200ac3c0d36SFilipe Manana
3201ac3c0d36SFilipe Manana delalloc = btrfs_find_delalloc_in_range(inode,
3202ac3c0d36SFilipe Manana prev_extent_end,
3203ac3c0d36SFilipe Manana i_size - 1,
3204b3e744feSFilipe Manana &delalloc_cached_state,
3205ac3c0d36SFilipe Manana &delalloc_start,
3206ac3c0d36SFilipe Manana &delalloc_end);
3207ac3c0d36SFilipe Manana if (!delalloc)
3208ac3c0d36SFilipe Manana cache.flags |= FIEMAP_EXTENT_LAST;
3209ac3c0d36SFilipe Manana } else {
3210ac3c0d36SFilipe Manana cache.flags |= FIEMAP_EXTENT_LAST;
3211ac3c0d36SFilipe Manana }
3212ac3c0d36SFilipe Manana }
3213ac3c0d36SFilipe Manana
321449d640d2SFilipe Manana out_unlock:
321549d640d2SFilipe Manana unlock_extent(&inode->io_tree, range_start, range_end, &cached_state);
321649d640d2SFilipe Manana
321749d640d2SFilipe Manana if (ret == BTRFS_FIEMAP_FLUSH_CACHE) {
321849d640d2SFilipe Manana btrfs_release_path(path);
321949d640d2SFilipe Manana ret = flush_fiemap_cache(fieinfo, &cache);
322049d640d2SFilipe Manana if (ret)
322149d640d2SFilipe Manana goto out;
322249d640d2SFilipe Manana len -= cache.next_search_offset - start;
322349d640d2SFilipe Manana start = cache.next_search_offset;
322449d640d2SFilipe Manana goto restart;
322549d640d2SFilipe Manana } else if (ret < 0) {
322649d640d2SFilipe Manana goto out;
322749d640d2SFilipe Manana }
322849d640d2SFilipe Manana
322949d640d2SFilipe Manana /*
323049d640d2SFilipe Manana * Must free the path before emitting to the fiemap buffer because we
323149d640d2SFilipe Manana * may have a non-cloned leaf and if the fiemap buffer is memory mapped
323249d640d2SFilipe Manana * to a file, a write into it (through btrfs_page_mkwrite()) may trigger
323349d640d2SFilipe Manana * waiting for an ordered extent that in order to complete needs to
323449d640d2SFilipe Manana * modify that leaf, therefore leading to a deadlock.
323549d640d2SFilipe Manana */
323649d640d2SFilipe Manana btrfs_free_path(path);
323749d640d2SFilipe Manana path = NULL;
323849d640d2SFilipe Manana
323949d640d2SFilipe Manana ret = flush_fiemap_cache(fieinfo, &cache);
324049d640d2SFilipe Manana if (ret)
324149d640d2SFilipe Manana goto out;
324249d640d2SFilipe Manana
3243ac3c0d36SFilipe Manana ret = emit_last_fiemap_cache(fieinfo, &cache);
3244ac3c0d36SFilipe Manana out:
3245b3e744feSFilipe Manana free_extent_state(delalloc_cached_state);
324649d640d2SFilipe Manana kfree(cache.entries);
324784a7949dSFilipe Manana btrfs_free_backref_share_ctx(backref_ctx);
3248e02d48eaSColin Ian King btrfs_free_path(path);
32491506fcc8SYehuda Sadeh return ret;
32501506fcc8SYehuda Sadeh }
32511506fcc8SYehuda Sadeh
__free_extent_buffer(struct extent_buffer * eb)3252727011e0SChris Mason static void __free_extent_buffer(struct extent_buffer *eb)
3253727011e0SChris Mason {
3254727011e0SChris Mason kmem_cache_free(extent_buffer_cache, eb);
3255727011e0SChris Mason }
3256727011e0SChris Mason
extent_buffer_under_io(const struct extent_buffer * eb)32577f26fb1cSChristoph Hellwig static int extent_buffer_under_io(const struct extent_buffer *eb)
3258d1310b2eSChris Mason {
3259113fa05cSChristoph Hellwig return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
32600b32f4bbSJosef Bacik test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3261d1310b2eSChris Mason }
3262d1310b2eSChris Mason
page_range_has_eb(struct btrfs_fs_info * fs_info,struct page * page)32638ff8466dSQu Wenruo static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
3264897ca6e9SMiao Xie {
32658ff8466dSQu Wenruo struct btrfs_subpage *subpage;
3266897ca6e9SMiao Xie
32678ff8466dSQu Wenruo lockdep_assert_held(&page->mapping->private_lock);
3268897ca6e9SMiao Xie
32698ff8466dSQu Wenruo if (PagePrivate(page)) {
32708ff8466dSQu Wenruo subpage = (struct btrfs_subpage *)page->private;
32718ff8466dSQu Wenruo if (atomic_read(&subpage->eb_refs))
32728ff8466dSQu Wenruo return true;
32733d078efaSQu Wenruo /*
32743d078efaSQu Wenruo * Even there is no eb refs here, we may still have
32753d078efaSQu Wenruo * end_page_read() call relying on page::private.
32763d078efaSQu Wenruo */
32773d078efaSQu Wenruo if (atomic_read(&subpage->readers))
32783d078efaSQu Wenruo return true;
32798ff8466dSQu Wenruo }
32808ff8466dSQu Wenruo return false;
32818ff8466dSQu Wenruo }
3282897ca6e9SMiao Xie
detach_extent_buffer_page(struct extent_buffer * eb,struct page * page)32838ff8466dSQu Wenruo static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
32848ff8466dSQu Wenruo {
32858ff8466dSQu Wenruo struct btrfs_fs_info *fs_info = eb->fs_info;
32868ff8466dSQu Wenruo const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
32878ff8466dSQu Wenruo
32888ff8466dSQu Wenruo /*
32898ff8466dSQu Wenruo * For mapped eb, we're going to change the page private, which should
32908ff8466dSQu Wenruo * be done under the private_lock.
32918ff8466dSQu Wenruo */
32925d2361dbSForrest Liu if (mapped)
32934f2de97aSJosef Bacik spin_lock(&page->mapping->private_lock);
32948ff8466dSQu Wenruo
32958ff8466dSQu Wenruo if (!PagePrivate(page)) {
32968ff8466dSQu Wenruo if (mapped)
32978ff8466dSQu Wenruo spin_unlock(&page->mapping->private_lock);
32988ff8466dSQu Wenruo return;
32998ff8466dSQu Wenruo }
33008ff8466dSQu Wenruo
3301fbca46ebSQu Wenruo if (fs_info->nodesize >= PAGE_SIZE) {
33024f2de97aSJosef Bacik /*
33034f2de97aSJosef Bacik * We do this since we'll remove the pages after we've
33044f2de97aSJosef Bacik * removed the eb from the radix tree, so we could race
33054f2de97aSJosef Bacik * and have this page now attached to the new eb. So
33064f2de97aSJosef Bacik * only clear page_private if it's still connected to
33074f2de97aSJosef Bacik * this eb.
33084f2de97aSJosef Bacik */
33094f2de97aSJosef Bacik if (PagePrivate(page) &&
33104f2de97aSJosef Bacik page->private == (unsigned long)eb) {
33110b32f4bbSJosef Bacik BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
33123083ee2eSJosef Bacik BUG_ON(PageDirty(page));
33133083ee2eSJosef Bacik BUG_ON(PageWriteback(page));
33144f2de97aSJosef Bacik /*
33154f2de97aSJosef Bacik * We need to make sure we haven't be attached
33164f2de97aSJosef Bacik * to a new eb.
33174f2de97aSJosef Bacik */
3318d1b89bc0SGuoqing Jiang detach_page_private(page);
33194f2de97aSJosef Bacik }
33205d2361dbSForrest Liu if (mapped)
33214f2de97aSJosef Bacik spin_unlock(&page->mapping->private_lock);
33228ff8466dSQu Wenruo return;
33238ff8466dSQu Wenruo }
33248ff8466dSQu Wenruo
33258ff8466dSQu Wenruo /*
33268ff8466dSQu Wenruo * For subpage, we can have dummy eb with page private. In this case,
33278ff8466dSQu Wenruo * we can directly detach the private as such page is only attached to
33288ff8466dSQu Wenruo * one dummy eb, no sharing.
33298ff8466dSQu Wenruo */
33308ff8466dSQu Wenruo if (!mapped) {
33318ff8466dSQu Wenruo btrfs_detach_subpage(fs_info, page);
33328ff8466dSQu Wenruo return;
33338ff8466dSQu Wenruo }
33348ff8466dSQu Wenruo
33358ff8466dSQu Wenruo btrfs_page_dec_eb_refs(fs_info, page);
33368ff8466dSQu Wenruo
33378ff8466dSQu Wenruo /*
33388ff8466dSQu Wenruo * We can only detach the page private if there are no other ebs in the
33393d078efaSQu Wenruo * page range and no unfinished IO.
33408ff8466dSQu Wenruo */
33418ff8466dSQu Wenruo if (!page_range_has_eb(fs_info, page))
33428ff8466dSQu Wenruo btrfs_detach_subpage(fs_info, page);
33438ff8466dSQu Wenruo
33448ff8466dSQu Wenruo spin_unlock(&page->mapping->private_lock);
33458ff8466dSQu Wenruo }
33468ff8466dSQu Wenruo
33478ff8466dSQu Wenruo /* Release all pages attached to the extent buffer */
btrfs_release_extent_buffer_pages(struct extent_buffer * eb)33488ff8466dSQu Wenruo static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
33498ff8466dSQu Wenruo {
33508ff8466dSQu Wenruo int i;
33518ff8466dSQu Wenruo int num_pages;
33528ff8466dSQu Wenruo
33538ff8466dSQu Wenruo ASSERT(!extent_buffer_under_io(eb));
33548ff8466dSQu Wenruo
33558ff8466dSQu Wenruo num_pages = num_extent_pages(eb);
33568ff8466dSQu Wenruo for (i = 0; i < num_pages; i++) {
33578ff8466dSQu Wenruo struct page *page = eb->pages[i];
33588ff8466dSQu Wenruo
33598ff8466dSQu Wenruo if (!page)
33608ff8466dSQu Wenruo continue;
33618ff8466dSQu Wenruo
33628ff8466dSQu Wenruo detach_extent_buffer_page(eb, page);
33634f2de97aSJosef Bacik
336401327610SNicholas D Steeves /* One for when we allocated the page */
336509cbfeafSKirill A. Shutemov put_page(page);
3366d64766fdSNikolay Borisov }
3367897ca6e9SMiao Xie }
3368897ca6e9SMiao Xie
3369897ca6e9SMiao Xie /*
3370897ca6e9SMiao Xie * Helper for releasing the extent buffer.
3371897ca6e9SMiao Xie */
btrfs_release_extent_buffer(struct extent_buffer * eb)3372897ca6e9SMiao Xie static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3373897ca6e9SMiao Xie {
337455ac0139SDavid Sterba btrfs_release_extent_buffer_pages(eb);
3375a40246e8SJosef Bacik btrfs_leak_debug_del_eb(eb);
3376897ca6e9SMiao Xie __free_extent_buffer(eb);
3377897ca6e9SMiao Xie }
3378897ca6e9SMiao Xie
3379f28491e0SJosef Bacik static struct extent_buffer *
__alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,unsigned long len)3380f28491e0SJosef Bacik __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
338123d79d81SDavid Sterba unsigned long len)
3382db7f3436SJosef Bacik {
3383db7f3436SJosef Bacik struct extent_buffer *eb = NULL;
3384db7f3436SJosef Bacik
3385d1b5c567SMichal Hocko eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
3386db7f3436SJosef Bacik eb->start = start;
3387db7f3436SJosef Bacik eb->len = len;
3388f28491e0SJosef Bacik eb->fs_info = fs_info;
3389196d59abSJosef Bacik init_rwsem(&eb->lock);
3390db7f3436SJosef Bacik
3391a40246e8SJosef Bacik btrfs_leak_debug_add_eb(eb);
3392db7f3436SJosef Bacik
3393db7f3436SJosef Bacik spin_lock_init(&eb->refs_lock);
3394db7f3436SJosef Bacik atomic_set(&eb->refs, 1);
3395db7f3436SJosef Bacik
3396deb67895SQu Wenruo ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
3397db7f3436SJosef Bacik
3398db7f3436SJosef Bacik return eb;
3399db7f3436SJosef Bacik }
3400db7f3436SJosef Bacik
btrfs_clone_extent_buffer(const struct extent_buffer * src)34012b48966aSDavid Sterba struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3402db7f3436SJosef Bacik {
3403cc5e31a4SDavid Sterba int i;
3404db7f3436SJosef Bacik struct extent_buffer *new;
3405cc5e31a4SDavid Sterba int num_pages = num_extent_pages(src);
3406dd137dd1SSweet Tea Dorminy int ret;
3407db7f3436SJosef Bacik
34083f556f78SDavid Sterba new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
3409db7f3436SJosef Bacik if (new == NULL)
3410db7f3436SJosef Bacik return NULL;
3411db7f3436SJosef Bacik
341262c053fbSQu Wenruo /*
341362c053fbSQu Wenruo * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
341462c053fbSQu Wenruo * btrfs_release_extent_buffer() have different behavior for
341562c053fbSQu Wenruo * UNMAPPED subpage extent buffer.
341662c053fbSQu Wenruo */
341762c053fbSQu Wenruo set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
341862c053fbSQu Wenruo
3419dd137dd1SSweet Tea Dorminy ret = btrfs_alloc_page_array(num_pages, new->pages);
3420dd137dd1SSweet Tea Dorminy if (ret) {
3421db7f3436SJosef Bacik btrfs_release_extent_buffer(new);
3422db7f3436SJosef Bacik return NULL;
3423db7f3436SJosef Bacik }
3424dd137dd1SSweet Tea Dorminy
3425dd137dd1SSweet Tea Dorminy for (i = 0; i < num_pages; i++) {
3426dd137dd1SSweet Tea Dorminy int ret;
3427dd137dd1SSweet Tea Dorminy struct page *p = new->pages[i];
3428dd137dd1SSweet Tea Dorminy
3429760f991fSQu Wenruo ret = attach_extent_buffer_page(new, p, NULL);
3430760f991fSQu Wenruo if (ret < 0) {
3431760f991fSQu Wenruo btrfs_release_extent_buffer(new);
3432760f991fSQu Wenruo return NULL;
3433760f991fSQu Wenruo }
3434db7f3436SJosef Bacik WARN_ON(PageDirty(p));
3435db7f3436SJosef Bacik }
3436682a0bc5SQu Wenruo copy_extent_buffer_full(new, src);
343792d83e94SQu Wenruo set_extent_buffer_uptodate(new);
3438db7f3436SJosef Bacik
3439db7f3436SJosef Bacik return new;
3440db7f3436SJosef Bacik }
3441db7f3436SJosef Bacik
__alloc_dummy_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,unsigned long len)34420f331229SOmar Sandoval struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
34430f331229SOmar Sandoval u64 start, unsigned long len)
3444db7f3436SJosef Bacik {
3445db7f3436SJosef Bacik struct extent_buffer *eb;
3446cc5e31a4SDavid Sterba int num_pages;
3447cc5e31a4SDavid Sterba int i;
3448dd137dd1SSweet Tea Dorminy int ret;
3449db7f3436SJosef Bacik
34503f556f78SDavid Sterba eb = __alloc_extent_buffer(fs_info, start, len);
3451db7f3436SJosef Bacik if (!eb)
3452db7f3436SJosef Bacik return NULL;
3453db7f3436SJosef Bacik
345465ad0104SDavid Sterba num_pages = num_extent_pages(eb);
3455dd137dd1SSweet Tea Dorminy ret = btrfs_alloc_page_array(num_pages, eb->pages);
3456dd137dd1SSweet Tea Dorminy if (ret)
3457db7f3436SJosef Bacik goto err;
3458dd137dd1SSweet Tea Dorminy
3459dd137dd1SSweet Tea Dorminy for (i = 0; i < num_pages; i++) {
3460dd137dd1SSweet Tea Dorminy struct page *p = eb->pages[i];
3461dd137dd1SSweet Tea Dorminy
3462dd137dd1SSweet Tea Dorminy ret = attach_extent_buffer_page(eb, p, NULL);
346309bc1f0fSQu Wenruo if (ret < 0)
346409bc1f0fSQu Wenruo goto err;
3465db7f3436SJosef Bacik }
3466dd137dd1SSweet Tea Dorminy
3467db7f3436SJosef Bacik set_extent_buffer_uptodate(eb);
3468db7f3436SJosef Bacik btrfs_set_header_nritems(eb, 0);
3469b0132a3bSNikolay Borisov set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3470db7f3436SJosef Bacik
3471db7f3436SJosef Bacik return eb;
3472db7f3436SJosef Bacik err:
3473dd137dd1SSweet Tea Dorminy for (i = 0; i < num_pages; i++) {
3474dd137dd1SSweet Tea Dorminy if (eb->pages[i]) {
3475dd137dd1SSweet Tea Dorminy detach_extent_buffer_page(eb, eb->pages[i]);
3476dd137dd1SSweet Tea Dorminy __free_page(eb->pages[i]);
3477dd137dd1SSweet Tea Dorminy }
347809bc1f0fSQu Wenruo }
3479db7f3436SJosef Bacik __free_extent_buffer(eb);
3480db7f3436SJosef Bacik return NULL;
3481db7f3436SJosef Bacik }
3482db7f3436SJosef Bacik
alloc_dummy_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)34830f331229SOmar Sandoval struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3484da17066cSJeff Mahoney u64 start)
34850f331229SOmar Sandoval {
3486da17066cSJeff Mahoney return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
34870f331229SOmar Sandoval }
34880f331229SOmar Sandoval
check_buffer_tree_ref(struct extent_buffer * eb)34890b32f4bbSJosef Bacik static void check_buffer_tree_ref(struct extent_buffer *eb)
34900b32f4bbSJosef Bacik {
3491242e18c7SChris Mason int refs;
34926bf9cd2eSBoris Burkov /*
34936bf9cd2eSBoris Burkov * The TREE_REF bit is first set when the extent_buffer is added
34946bf9cd2eSBoris Burkov * to the radix tree. It is also reset, if unset, when a new reference
34956bf9cd2eSBoris Burkov * is created by find_extent_buffer.
34960b32f4bbSJosef Bacik *
34976bf9cd2eSBoris Burkov * It is only cleared in two cases: freeing the last non-tree
34986bf9cd2eSBoris Burkov * reference to the extent_buffer when its STALE bit is set or
3499f913cff3SMatthew Wilcox (Oracle) * calling release_folio when the tree reference is the only reference.
35000b32f4bbSJosef Bacik *
35016bf9cd2eSBoris Burkov * In both cases, care is taken to ensure that the extent_buffer's
3502f913cff3SMatthew Wilcox (Oracle) * pages are not under io. However, release_folio can be concurrently
35036bf9cd2eSBoris Burkov * called with creating new references, which is prone to race
35046bf9cd2eSBoris Burkov * conditions between the calls to check_buffer_tree_ref in those
35056bf9cd2eSBoris Burkov * codepaths and clearing TREE_REF in try_release_extent_buffer.
35060b32f4bbSJosef Bacik *
35076bf9cd2eSBoris Burkov * The actual lifetime of the extent_buffer in the radix tree is
35086bf9cd2eSBoris Burkov * adequately protected by the refcount, but the TREE_REF bit and
35096bf9cd2eSBoris Burkov * its corresponding reference are not. To protect against this
35106bf9cd2eSBoris Burkov * class of races, we call check_buffer_tree_ref from the codepaths
3511113fa05cSChristoph Hellwig * which trigger io. Note that once io is initiated, TREE_REF can no
3512113fa05cSChristoph Hellwig * longer be cleared, so that is the moment at which any such race is
3513113fa05cSChristoph Hellwig * best fixed.
35140b32f4bbSJosef Bacik */
3515242e18c7SChris Mason refs = atomic_read(&eb->refs);
3516242e18c7SChris Mason if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3517242e18c7SChris Mason return;
3518242e18c7SChris Mason
3519594831c4SJosef Bacik spin_lock(&eb->refs_lock);
3520594831c4SJosef Bacik if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
35210b32f4bbSJosef Bacik atomic_inc(&eb->refs);
3522594831c4SJosef Bacik spin_unlock(&eb->refs_lock);
35230b32f4bbSJosef Bacik }
35240b32f4bbSJosef Bacik
mark_extent_buffer_accessed(struct extent_buffer * eb,struct page * accessed)35252457aec6SMel Gorman static void mark_extent_buffer_accessed(struct extent_buffer *eb,
35262457aec6SMel Gorman struct page *accessed)
35275df4235eSJosef Bacik {
3528cc5e31a4SDavid Sterba int num_pages, i;
35295df4235eSJosef Bacik
35300b32f4bbSJosef Bacik check_buffer_tree_ref(eb);
35310b32f4bbSJosef Bacik
353265ad0104SDavid Sterba num_pages = num_extent_pages(eb);
35335df4235eSJosef Bacik for (i = 0; i < num_pages; i++) {
3534fb85fc9aSDavid Sterba struct page *p = eb->pages[i];
3535fb85fc9aSDavid Sterba
35362457aec6SMel Gorman if (p != accessed)
35375df4235eSJosef Bacik mark_page_accessed(p);
35385df4235eSJosef Bacik }
35395df4235eSJosef Bacik }
35405df4235eSJosef Bacik
find_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)3541f28491e0SJosef Bacik struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3542452c75c3SChandra Seetharaman u64 start)
3543452c75c3SChandra Seetharaman {
3544452c75c3SChandra Seetharaman struct extent_buffer *eb;
3545452c75c3SChandra Seetharaman
35462f3186d8SQu Wenruo eb = find_extent_buffer_nolock(fs_info, start);
35472f3186d8SQu Wenruo if (!eb)
35482f3186d8SQu Wenruo return NULL;
3549062c19e9SFilipe Manana /*
35502f3186d8SQu Wenruo * Lock our eb's refs_lock to avoid races with free_extent_buffer().
35512f3186d8SQu Wenruo * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
35522f3186d8SQu Wenruo * another task running free_extent_buffer() might have seen that flag
35532f3186d8SQu Wenruo * set, eb->refs == 2, that the buffer isn't under IO (dirty and
3554062c19e9SFilipe Manana * writeback flags not set) and it's still in the tree (flag
35552f3186d8SQu Wenruo * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
35562f3186d8SQu Wenruo * decrementing the extent buffer's reference count twice. So here we
35572f3186d8SQu Wenruo * could race and increment the eb's reference count, clear its stale
35582f3186d8SQu Wenruo * flag, mark it as dirty and drop our reference before the other task
35592f3186d8SQu Wenruo * finishes executing free_extent_buffer, which would later result in
35602f3186d8SQu Wenruo * an attempt to free an extent buffer that is dirty.
3561062c19e9SFilipe Manana */
3562062c19e9SFilipe Manana if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3563062c19e9SFilipe Manana spin_lock(&eb->refs_lock);
3564062c19e9SFilipe Manana spin_unlock(&eb->refs_lock);
3565062c19e9SFilipe Manana }
35662457aec6SMel Gorman mark_extent_buffer_accessed(eb, NULL);
3567452c75c3SChandra Seetharaman return eb;
3568452c75c3SChandra Seetharaman }
3569452c75c3SChandra Seetharaman
3570faa2dbf0SJosef Bacik #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
alloc_test_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)3571faa2dbf0SJosef Bacik struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3572da17066cSJeff Mahoney u64 start)
3573faa2dbf0SJosef Bacik {
3574faa2dbf0SJosef Bacik struct extent_buffer *eb, *exists = NULL;
3575faa2dbf0SJosef Bacik int ret;
3576faa2dbf0SJosef Bacik
3577faa2dbf0SJosef Bacik eb = find_extent_buffer(fs_info, start);
3578faa2dbf0SJosef Bacik if (eb)
3579faa2dbf0SJosef Bacik return eb;
3580da17066cSJeff Mahoney eb = alloc_dummy_extent_buffer(fs_info, start);
3581faa2dbf0SJosef Bacik if (!eb)
3582b6293c82SDan Carpenter return ERR_PTR(-ENOMEM);
3583faa2dbf0SJosef Bacik eb->fs_info = fs_info;
358401cd3909SDavid Sterba again:
358501cd3909SDavid Sterba ret = radix_tree_preload(GFP_NOFS);
358601cd3909SDavid Sterba if (ret) {
3587b6293c82SDan Carpenter exists = ERR_PTR(ret);
3588faa2dbf0SJosef Bacik goto free_eb;
3589b6293c82SDan Carpenter }
359001cd3909SDavid Sterba spin_lock(&fs_info->buffer_lock);
359101cd3909SDavid Sterba ret = radix_tree_insert(&fs_info->buffer_radix,
359201cd3909SDavid Sterba start >> fs_info->sectorsize_bits, eb);
359301cd3909SDavid Sterba spin_unlock(&fs_info->buffer_lock);
359401cd3909SDavid Sterba radix_tree_preload_end();
359501cd3909SDavid Sterba if (ret == -EEXIST) {
3596faa2dbf0SJosef Bacik exists = find_extent_buffer(fs_info, start);
3597faa2dbf0SJosef Bacik if (exists)
3598faa2dbf0SJosef Bacik goto free_eb;
359901cd3909SDavid Sterba else
360001cd3909SDavid Sterba goto again;
3601faa2dbf0SJosef Bacik }
3602faa2dbf0SJosef Bacik check_buffer_tree_ref(eb);
3603faa2dbf0SJosef Bacik set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3604faa2dbf0SJosef Bacik
3605faa2dbf0SJosef Bacik return eb;
3606faa2dbf0SJosef Bacik free_eb:
3607faa2dbf0SJosef Bacik btrfs_release_extent_buffer(eb);
3608faa2dbf0SJosef Bacik return exists;
3609faa2dbf0SJosef Bacik }
3610faa2dbf0SJosef Bacik #endif
3611faa2dbf0SJosef Bacik
grab_extent_buffer(struct btrfs_fs_info * fs_info,struct page * page)361281982210SQu Wenruo static struct extent_buffer *grab_extent_buffer(
361381982210SQu Wenruo struct btrfs_fs_info *fs_info, struct page *page)
3614c0f0a9e7SQu Wenruo {
3615c0f0a9e7SQu Wenruo struct extent_buffer *exists;
3616c0f0a9e7SQu Wenruo
361781982210SQu Wenruo /*
361881982210SQu Wenruo * For subpage case, we completely rely on radix tree to ensure we
361981982210SQu Wenruo * don't try to insert two ebs for the same bytenr. So here we always
362081982210SQu Wenruo * return NULL and just continue.
362181982210SQu Wenruo */
3622fbca46ebSQu Wenruo if (fs_info->nodesize < PAGE_SIZE)
362381982210SQu Wenruo return NULL;
362481982210SQu Wenruo
3625c0f0a9e7SQu Wenruo /* Page not yet attached to an extent buffer */
3626c0f0a9e7SQu Wenruo if (!PagePrivate(page))
3627c0f0a9e7SQu Wenruo return NULL;
3628c0f0a9e7SQu Wenruo
3629c0f0a9e7SQu Wenruo /*
3630c0f0a9e7SQu Wenruo * We could have already allocated an eb for this page and attached one
3631c0f0a9e7SQu Wenruo * so lets see if we can get a ref on the existing eb, and if we can we
3632c0f0a9e7SQu Wenruo * know it's good and we can just return that one, else we know we can
3633c0f0a9e7SQu Wenruo * just overwrite page->private.
3634c0f0a9e7SQu Wenruo */
3635c0f0a9e7SQu Wenruo exists = (struct extent_buffer *)page->private;
3636c0f0a9e7SQu Wenruo if (atomic_inc_not_zero(&exists->refs))
3637c0f0a9e7SQu Wenruo return exists;
3638c0f0a9e7SQu Wenruo
3639c0f0a9e7SQu Wenruo WARN_ON(PageDirty(page));
3640c0f0a9e7SQu Wenruo detach_page_private(page);
3641c0f0a9e7SQu Wenruo return NULL;
3642c0f0a9e7SQu Wenruo }
3643c0f0a9e7SQu Wenruo
check_eb_alignment(struct btrfs_fs_info * fs_info,u64 start)3644fbca46ebSQu Wenruo static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3645fbca46ebSQu Wenruo {
3646fbca46ebSQu Wenruo if (!IS_ALIGNED(start, fs_info->sectorsize)) {
3647fbca46ebSQu Wenruo btrfs_err(fs_info, "bad tree block start %llu", start);
3648fbca46ebSQu Wenruo return -EINVAL;
3649fbca46ebSQu Wenruo }
3650fbca46ebSQu Wenruo
3651fbca46ebSQu Wenruo if (fs_info->nodesize < PAGE_SIZE &&
3652fbca46ebSQu Wenruo offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
3653fbca46ebSQu Wenruo btrfs_err(fs_info,
3654fbca46ebSQu Wenruo "tree block crosses page boundary, start %llu nodesize %u",
3655fbca46ebSQu Wenruo start, fs_info->nodesize);
3656fbca46ebSQu Wenruo return -EINVAL;
3657fbca46ebSQu Wenruo }
3658fbca46ebSQu Wenruo if (fs_info->nodesize >= PAGE_SIZE &&
36591280d2d1SFanjun Kong !PAGE_ALIGNED(start)) {
3660fbca46ebSQu Wenruo btrfs_err(fs_info,
3661fbca46ebSQu Wenruo "tree block is not page aligned, start %llu nodesize %u",
3662fbca46ebSQu Wenruo start, fs_info->nodesize);
3663fbca46ebSQu Wenruo return -EINVAL;
3664fbca46ebSQu Wenruo }
3665fbca46ebSQu Wenruo return 0;
3666fbca46ebSQu Wenruo }
3667fbca46ebSQu Wenruo
alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,u64 owner_root,int level)3668f28491e0SJosef Bacik struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
36693fbaf258SJosef Bacik u64 start, u64 owner_root, int level)
3670d1310b2eSChris Mason {
3671da17066cSJeff Mahoney unsigned long len = fs_info->nodesize;
3672cc5e31a4SDavid Sterba int num_pages;
3673cc5e31a4SDavid Sterba int i;
367409cbfeafSKirill A. Shutemov unsigned long index = start >> PAGE_SHIFT;
3675d1310b2eSChris Mason struct extent_buffer *eb;
36766af118ceSChris Mason struct extent_buffer *exists = NULL;
3677d1310b2eSChris Mason struct page *p;
3678f28491e0SJosef Bacik struct address_space *mapping = fs_info->btree_inode->i_mapping;
367952ea5bfbSQu Wenruo struct btrfs_subpage *prealloc = NULL;
3680b40130b2SJosef Bacik u64 lockdep_owner = owner_root;
3681d1310b2eSChris Mason int uptodate = 1;
368219fe0a8bSMiao Xie int ret;
3683d1310b2eSChris Mason
3684fbca46ebSQu Wenruo if (check_eb_alignment(fs_info, start))
3685c871b0f2SLiu Bo return ERR_PTR(-EINVAL);
3686c871b0f2SLiu Bo
3687e9306ad4SQu Wenruo #if BITS_PER_LONG == 32
3688e9306ad4SQu Wenruo if (start >= MAX_LFS_FILESIZE) {
3689e9306ad4SQu Wenruo btrfs_err_rl(fs_info,
3690e9306ad4SQu Wenruo "extent buffer %llu is beyond 32bit page cache limit", start);
3691e9306ad4SQu Wenruo btrfs_err_32bit_limit(fs_info);
3692e9306ad4SQu Wenruo return ERR_PTR(-EOVERFLOW);
3693e9306ad4SQu Wenruo }
3694e9306ad4SQu Wenruo if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3695e9306ad4SQu Wenruo btrfs_warn_32bit_limit(fs_info);
3696e9306ad4SQu Wenruo #endif
3697e9306ad4SQu Wenruo
3698f28491e0SJosef Bacik eb = find_extent_buffer(fs_info, start);
3699452c75c3SChandra Seetharaman if (eb)
37006af118ceSChris Mason return eb;
37016af118ceSChris Mason
370223d79d81SDavid Sterba eb = __alloc_extent_buffer(fs_info, start, len);
37032b114d1dSPeter if (!eb)
3704c871b0f2SLiu Bo return ERR_PTR(-ENOMEM);
3705b40130b2SJosef Bacik
3706b40130b2SJosef Bacik /*
3707b40130b2SJosef Bacik * The reloc trees are just snapshots, so we need them to appear to be
3708b40130b2SJosef Bacik * just like any other fs tree WRT lockdep.
3709b40130b2SJosef Bacik */
3710b40130b2SJosef Bacik if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3711b40130b2SJosef Bacik lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3712b40130b2SJosef Bacik
3713b40130b2SJosef Bacik btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
3714d1310b2eSChris Mason
371565ad0104SDavid Sterba num_pages = num_extent_pages(eb);
37164f2de97aSJosef Bacik
3717760f991fSQu Wenruo /*
3718760f991fSQu Wenruo * Preallocate page->private for subpage case, so that we won't
371952ea5bfbSQu Wenruo * allocate memory with private_lock nor page lock hold.
3720760f991fSQu Wenruo *
372152ea5bfbSQu Wenruo * The memory will be freed by attach_extent_buffer_page() or freed
372252ea5bfbSQu Wenruo * manually if we exit earlier.
3723760f991fSQu Wenruo */
3724fbca46ebSQu Wenruo if (fs_info->nodesize < PAGE_SIZE) {
3725651fb419SQu Wenruo prealloc = btrfs_alloc_subpage(fs_info, BTRFS_SUBPAGE_METADATA);
3726651fb419SQu Wenruo if (IS_ERR(prealloc)) {
372752ea5bfbSQu Wenruo exists = ERR_CAST(prealloc);
3728760f991fSQu Wenruo goto free_eb;
3729760f991fSQu Wenruo }
3730fdf250dbSQu Wenruo }
3731760f991fSQu Wenruo
373252ea5bfbSQu Wenruo for (i = 0; i < num_pages; i++, index++) {
373352ea5bfbSQu Wenruo p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
373452ea5bfbSQu Wenruo if (!p) {
373552ea5bfbSQu Wenruo exists = ERR_PTR(-ENOMEM);
373652ea5bfbSQu Wenruo btrfs_free_subpage(prealloc);
373752ea5bfbSQu Wenruo goto free_eb;
373852ea5bfbSQu Wenruo }
373952ea5bfbSQu Wenruo
37404f2de97aSJosef Bacik spin_lock(&mapping->private_lock);
374181982210SQu Wenruo exists = grab_extent_buffer(fs_info, p);
3742c0f0a9e7SQu Wenruo if (exists) {
37434f2de97aSJosef Bacik spin_unlock(&mapping->private_lock);
37444f2de97aSJosef Bacik unlock_page(p);
374509cbfeafSKirill A. Shutemov put_page(p);
37462457aec6SMel Gorman mark_extent_buffer_accessed(exists, p);
3747760f991fSQu Wenruo btrfs_free_subpage(prealloc);
37484f2de97aSJosef Bacik goto free_eb;
3749d1310b2eSChris Mason }
3750760f991fSQu Wenruo /* Should not fail, as we have preallocated the memory */
3751760f991fSQu Wenruo ret = attach_extent_buffer_page(eb, p, prealloc);
3752760f991fSQu Wenruo ASSERT(!ret);
37538ff8466dSQu Wenruo /*
37548ff8466dSQu Wenruo * To inform we have extra eb under allocation, so that
37558ff8466dSQu Wenruo * detach_extent_buffer_page() won't release the page private
37568ff8466dSQu Wenruo * when the eb hasn't yet been inserted into radix tree.
37578ff8466dSQu Wenruo *
37588ff8466dSQu Wenruo * The ref will be decreased when the eb released the page, in
37598ff8466dSQu Wenruo * detach_extent_buffer_page().
37608ff8466dSQu Wenruo * Thus needs no special handling in error path.
37618ff8466dSQu Wenruo */
37628ff8466dSQu Wenruo btrfs_page_inc_eb_refs(fs_info, p);
37634f2de97aSJosef Bacik spin_unlock(&mapping->private_lock);
3764760f991fSQu Wenruo
37651e5eb3d6SQu Wenruo WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len));
3766727011e0SChris Mason eb->pages[i] = p;
37675a963419SQu Wenruo if (!btrfs_page_test_uptodate(fs_info, p, eb->start, eb->len))
3768d1310b2eSChris Mason uptodate = 0;
3769eb14ab8eSChris Mason
3770eb14ab8eSChris Mason /*
3771b16d011eSNikolay Borisov * We can't unlock the pages just yet since the extent buffer
3772b16d011eSNikolay Borisov * hasn't been properly inserted in the radix tree, this
3773f913cff3SMatthew Wilcox (Oracle) * opens a race with btree_release_folio which can free a page
3774b16d011eSNikolay Borisov * while we are still filling in all pages for the buffer and
3775b16d011eSNikolay Borisov * we could crash.
3776eb14ab8eSChris Mason */
3777d1310b2eSChris Mason }
3778d1310b2eSChris Mason if (uptodate)
3779b4ce94deSChris Mason set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
378001cd3909SDavid Sterba again:
378101cd3909SDavid Sterba ret = radix_tree_preload(GFP_NOFS);
378201cd3909SDavid Sterba if (ret) {
3783c871b0f2SLiu Bo exists = ERR_PTR(ret);
378419fe0a8bSMiao Xie goto free_eb;
3785c871b0f2SLiu Bo }
378601cd3909SDavid Sterba
378701cd3909SDavid Sterba spin_lock(&fs_info->buffer_lock);
378801cd3909SDavid Sterba ret = radix_tree_insert(&fs_info->buffer_radix,
378901cd3909SDavid Sterba start >> fs_info->sectorsize_bits, eb);
379001cd3909SDavid Sterba spin_unlock(&fs_info->buffer_lock);
379101cd3909SDavid Sterba radix_tree_preload_end();
379201cd3909SDavid Sterba if (ret == -EEXIST) {
3793f28491e0SJosef Bacik exists = find_extent_buffer(fs_info, start);
3794452c75c3SChandra Seetharaman if (exists)
37956af118ceSChris Mason goto free_eb;
379601cd3909SDavid Sterba else
379701cd3909SDavid Sterba goto again;
37986af118ceSChris Mason }
37996af118ceSChris Mason /* add one reference for the tree */
38000b32f4bbSJosef Bacik check_buffer_tree_ref(eb);
380134b41aceSJosef Bacik set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3802eb14ab8eSChris Mason
3803eb14ab8eSChris Mason /*
3804b16d011eSNikolay Borisov * Now it's safe to unlock the pages because any calls to
3805f913cff3SMatthew Wilcox (Oracle) * btree_release_folio will correctly detect that a page belongs to a
3806b16d011eSNikolay Borisov * live buffer and won't free them prematurely.
3807eb14ab8eSChris Mason */
380828187ae5SNikolay Borisov for (i = 0; i < num_pages; i++)
380928187ae5SNikolay Borisov unlock_page(eb->pages[i]);
3810d1310b2eSChris Mason return eb;
3811d1310b2eSChris Mason
38126af118ceSChris Mason free_eb:
38135ca64f45SOmar Sandoval WARN_ON(!atomic_dec_and_test(&eb->refs));
3814727011e0SChris Mason for (i = 0; i < num_pages; i++) {
3815727011e0SChris Mason if (eb->pages[i])
3816727011e0SChris Mason unlock_page(eb->pages[i]);
3817727011e0SChris Mason }
3818eb14ab8eSChris Mason
3819897ca6e9SMiao Xie btrfs_release_extent_buffer(eb);
38206af118ceSChris Mason return exists;
3821d1310b2eSChris Mason }
3822d1310b2eSChris Mason
btrfs_release_extent_buffer_rcu(struct rcu_head * head)38233083ee2eSJosef Bacik static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
38243083ee2eSJosef Bacik {
38253083ee2eSJosef Bacik struct extent_buffer *eb =
38263083ee2eSJosef Bacik container_of(head, struct extent_buffer, rcu_head);
38273083ee2eSJosef Bacik
38283083ee2eSJosef Bacik __free_extent_buffer(eb);
38293083ee2eSJosef Bacik }
38303083ee2eSJosef Bacik
release_extent_buffer(struct extent_buffer * eb)3831f7a52a40SDavid Sterba static int release_extent_buffer(struct extent_buffer *eb)
38325ce48d0fSJules Irenge __releases(&eb->refs_lock)
38333083ee2eSJosef Bacik {
383407e21c4dSNikolay Borisov lockdep_assert_held(&eb->refs_lock);
383507e21c4dSNikolay Borisov
38363083ee2eSJosef Bacik WARN_ON(atomic_read(&eb->refs) == 0);
38373083ee2eSJosef Bacik if (atomic_dec_and_test(&eb->refs)) {
383834b41aceSJosef Bacik if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
3839f28491e0SJosef Bacik struct btrfs_fs_info *fs_info = eb->fs_info;
38403083ee2eSJosef Bacik
38413083ee2eSJosef Bacik spin_unlock(&eb->refs_lock);
38423083ee2eSJosef Bacik
384301cd3909SDavid Sterba spin_lock(&fs_info->buffer_lock);
384401cd3909SDavid Sterba radix_tree_delete(&fs_info->buffer_radix,
3845478ef886SQu Wenruo eb->start >> fs_info->sectorsize_bits);
384601cd3909SDavid Sterba spin_unlock(&fs_info->buffer_lock);
384734b41aceSJosef Bacik } else {
384834b41aceSJosef Bacik spin_unlock(&eb->refs_lock);
3849815a51c7SJan Schmidt }
38503083ee2eSJosef Bacik
3851a40246e8SJosef Bacik btrfs_leak_debug_del_eb(eb);
38523083ee2eSJosef Bacik /* Should be safe to release our pages at this point */
385355ac0139SDavid Sterba btrfs_release_extent_buffer_pages(eb);
3854bcb7e449SJosef Bacik #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3855b0132a3bSNikolay Borisov if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3856bcb7e449SJosef Bacik __free_extent_buffer(eb);
3857bcb7e449SJosef Bacik return 1;
3858bcb7e449SJosef Bacik }
3859bcb7e449SJosef Bacik #endif
38603083ee2eSJosef Bacik call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3861e64860aaSJosef Bacik return 1;
38623083ee2eSJosef Bacik }
38633083ee2eSJosef Bacik spin_unlock(&eb->refs_lock);
3864e64860aaSJosef Bacik
3865e64860aaSJosef Bacik return 0;
38663083ee2eSJosef Bacik }
38673083ee2eSJosef Bacik
free_extent_buffer(struct extent_buffer * eb)3868d1310b2eSChris Mason void free_extent_buffer(struct extent_buffer *eb)
3869d1310b2eSChris Mason {
3870242e18c7SChris Mason int refs;
3871d1310b2eSChris Mason if (!eb)
3872d1310b2eSChris Mason return;
3873d1310b2eSChris Mason
3874242e18c7SChris Mason refs = atomic_read(&eb->refs);
3875e5677f05SUros Bizjak while (1) {
387646cc775eSNikolay Borisov if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
387746cc775eSNikolay Borisov || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
387846cc775eSNikolay Borisov refs == 1))
3879242e18c7SChris Mason break;
3880e5677f05SUros Bizjak if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1))
3881242e18c7SChris Mason return;
3882242e18c7SChris Mason }
3883242e18c7SChris Mason
38843083ee2eSJosef Bacik spin_lock(&eb->refs_lock);
38853083ee2eSJosef Bacik if (atomic_read(&eb->refs) == 2 &&
38863083ee2eSJosef Bacik test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
38870b32f4bbSJosef Bacik !extent_buffer_under_io(eb) &&
38883083ee2eSJosef Bacik test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
38893083ee2eSJosef Bacik atomic_dec(&eb->refs);
3890d1310b2eSChris Mason
38913083ee2eSJosef Bacik /*
38923083ee2eSJosef Bacik * I know this is terrible, but it's temporary until we stop tracking
38933083ee2eSJosef Bacik * the uptodate bits and such for the extent buffers.
38943083ee2eSJosef Bacik */
3895f7a52a40SDavid Sterba release_extent_buffer(eb);
3896d1310b2eSChris Mason }
3897d1310b2eSChris Mason
free_extent_buffer_stale(struct extent_buffer * eb)38983083ee2eSJosef Bacik void free_extent_buffer_stale(struct extent_buffer *eb)
38993083ee2eSJosef Bacik {
39003083ee2eSJosef Bacik if (!eb)
3901d1310b2eSChris Mason return;
3902d1310b2eSChris Mason
39033083ee2eSJosef Bacik spin_lock(&eb->refs_lock);
39043083ee2eSJosef Bacik set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
39053083ee2eSJosef Bacik
39060b32f4bbSJosef Bacik if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
39073083ee2eSJosef Bacik test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
39083083ee2eSJosef Bacik atomic_dec(&eb->refs);
3909f7a52a40SDavid Sterba release_extent_buffer(eb);
3910d1310b2eSChris Mason }
3911d1310b2eSChris Mason
btree_clear_page_dirty(struct page * page)39120d27797eSQu Wenruo static void btree_clear_page_dirty(struct page *page)
39130d27797eSQu Wenruo {
39140d27797eSQu Wenruo ASSERT(PageDirty(page));
39150d27797eSQu Wenruo ASSERT(PageLocked(page));
39160d27797eSQu Wenruo clear_page_dirty_for_io(page);
39170d27797eSQu Wenruo xa_lock_irq(&page->mapping->i_pages);
39180d27797eSQu Wenruo if (!PageDirty(page))
39190d27797eSQu Wenruo __xa_clear_mark(&page->mapping->i_pages,
39200d27797eSQu Wenruo page_index(page), PAGECACHE_TAG_DIRTY);
39210d27797eSQu Wenruo xa_unlock_irq(&page->mapping->i_pages);
39220d27797eSQu Wenruo }
39230d27797eSQu Wenruo
clear_subpage_extent_buffer_dirty(const struct extent_buffer * eb)39240d27797eSQu Wenruo static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
39250d27797eSQu Wenruo {
39260d27797eSQu Wenruo struct btrfs_fs_info *fs_info = eb->fs_info;
39270d27797eSQu Wenruo struct page *page = eb->pages[0];
39280d27797eSQu Wenruo bool last;
39290d27797eSQu Wenruo
39300d27797eSQu Wenruo /* btree_clear_page_dirty() needs page locked */
39310d27797eSQu Wenruo lock_page(page);
39320d27797eSQu Wenruo last = btrfs_subpage_clear_and_test_dirty(fs_info, page, eb->start,
39330d27797eSQu Wenruo eb->len);
39340d27797eSQu Wenruo if (last)
39350d27797eSQu Wenruo btree_clear_page_dirty(page);
39360d27797eSQu Wenruo unlock_page(page);
39370d27797eSQu Wenruo WARN_ON(atomic_read(&eb->refs) == 0);
39380d27797eSQu Wenruo }
39390d27797eSQu Wenruo
btrfs_clear_buffer_dirty(struct btrfs_trans_handle * trans,struct extent_buffer * eb)394098c8d683SJosef Bacik void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
394198c8d683SJosef Bacik struct extent_buffer *eb)
3942d1310b2eSChris Mason {
394398c8d683SJosef Bacik struct btrfs_fs_info *fs_info = eb->fs_info;
3944cc5e31a4SDavid Sterba int i;
3945cc5e31a4SDavid Sterba int num_pages;
3946d1310b2eSChris Mason struct page *page;
3947d1310b2eSChris Mason
394898c8d683SJosef Bacik btrfs_assert_tree_write_locked(eb);
394998c8d683SJosef Bacik
395098c8d683SJosef Bacik if (trans && btrfs_header_generation(eb) != trans->transid)
395198c8d683SJosef Bacik return;
395298c8d683SJosef Bacik
395398c8d683SJosef Bacik if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
395498c8d683SJosef Bacik return;
395598c8d683SJosef Bacik
395698c8d683SJosef Bacik percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
395798c8d683SJosef Bacik fs_info->dirty_metadata_batch);
395898c8d683SJosef Bacik
3959fbca46ebSQu Wenruo if (eb->fs_info->nodesize < PAGE_SIZE)
39600d27797eSQu Wenruo return clear_subpage_extent_buffer_dirty(eb);
39610d27797eSQu Wenruo
396265ad0104SDavid Sterba num_pages = num_extent_pages(eb);
3963d1310b2eSChris Mason
3964d1310b2eSChris Mason for (i = 0; i < num_pages; i++) {
3965fb85fc9aSDavid Sterba page = eb->pages[i];
3966b9473439SChris Mason if (!PageDirty(page))
3967d2c3f4f6SChris Mason continue;
3968a61e6f29SChris Mason lock_page(page);
39690d27797eSQu Wenruo btree_clear_page_dirty(page);
3970a61e6f29SChris Mason unlock_page(page);
3971d1310b2eSChris Mason }
39720b32f4bbSJosef Bacik WARN_ON(atomic_read(&eb->refs) == 0);
3973d1310b2eSChris Mason }
3974d1310b2eSChris Mason
set_extent_buffer_dirty(struct extent_buffer * eb)3975f18cc978SChristoph Hellwig void set_extent_buffer_dirty(struct extent_buffer *eb)
3976d1310b2eSChris Mason {
3977cc5e31a4SDavid Sterba int i;
3978cc5e31a4SDavid Sterba int num_pages;
3979abb57ef3SLiu Bo bool was_dirty;
3980d1310b2eSChris Mason
39810b32f4bbSJosef Bacik check_buffer_tree_ref(eb);
39820b32f4bbSJosef Bacik
3983b9473439SChris Mason was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
39840b32f4bbSJosef Bacik
398565ad0104SDavid Sterba num_pages = num_extent_pages(eb);
39863083ee2eSJosef Bacik WARN_ON(atomic_read(&eb->refs) == 0);
39870b32f4bbSJosef Bacik WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
39880b32f4bbSJosef Bacik
39890d27797eSQu Wenruo if (!was_dirty) {
3990fbca46ebSQu Wenruo bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
399151995c39SLiu Bo
39920d27797eSQu Wenruo /*
39930d27797eSQu Wenruo * For subpage case, we can have other extent buffers in the
39940d27797eSQu Wenruo * same page, and in clear_subpage_extent_buffer_dirty() we
39950d27797eSQu Wenruo * have to clear page dirty without subpage lock held.
39960d27797eSQu Wenruo * This can cause race where our page gets dirty cleared after
39970d27797eSQu Wenruo * we just set it.
39980d27797eSQu Wenruo *
39990d27797eSQu Wenruo * Thankfully, clear_subpage_extent_buffer_dirty() has locked
40000d27797eSQu Wenruo * its page for other reasons, we can use page lock to prevent
40010d27797eSQu Wenruo * the above race.
40020d27797eSQu Wenruo */
40030d27797eSQu Wenruo if (subpage)
40040d27797eSQu Wenruo lock_page(eb->pages[0]);
40050d27797eSQu Wenruo for (i = 0; i < num_pages; i++)
40060d27797eSQu Wenruo btrfs_page_set_dirty(eb->fs_info, eb->pages[i],
40070d27797eSQu Wenruo eb->start, eb->len);
40080d27797eSQu Wenruo if (subpage)
40090d27797eSQu Wenruo unlock_page(eb->pages[0]);
4010f18cc978SChristoph Hellwig percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes,
4011f18cc978SChristoph Hellwig eb->len,
4012f18cc978SChristoph Hellwig eb->fs_info->dirty_metadata_batch);
40130d27797eSQu Wenruo }
401451995c39SLiu Bo #ifdef CONFIG_BTRFS_DEBUG
401551995c39SLiu Bo for (i = 0; i < num_pages; i++)
401651995c39SLiu Bo ASSERT(PageDirty(eb->pages[i]));
401751995c39SLiu Bo #endif
4018d1310b2eSChris Mason }
4019d1310b2eSChris Mason
clear_extent_buffer_uptodate(struct extent_buffer * eb)402069ba3927SDavid Sterba void clear_extent_buffer_uptodate(struct extent_buffer *eb)
40211259ab75SChris Mason {
4022251f2accSQu Wenruo struct btrfs_fs_info *fs_info = eb->fs_info;
40231259ab75SChris Mason struct page *page;
4024cc5e31a4SDavid Sterba int num_pages;
4025251f2accSQu Wenruo int i;
40261259ab75SChris Mason
4027b4ce94deSChris Mason clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
402865ad0104SDavid Sterba num_pages = num_extent_pages(eb);
40291259ab75SChris Mason for (i = 0; i < num_pages; i++) {
4030fb85fc9aSDavid Sterba page = eb->pages[i];
4031fbca46ebSQu Wenruo if (!page)
4032fbca46ebSQu Wenruo continue;
4033fbca46ebSQu Wenruo
4034fbca46ebSQu Wenruo /*
4035fbca46ebSQu Wenruo * This is special handling for metadata subpage, as regular
4036fbca46ebSQu Wenruo * btrfs_is_subpage() can not handle cloned/dummy metadata.
4037fbca46ebSQu Wenruo */
4038fbca46ebSQu Wenruo if (fs_info->nodesize >= PAGE_SIZE)
4039fbca46ebSQu Wenruo ClearPageUptodate(page);
4040fbca46ebSQu Wenruo else
4041fbca46ebSQu Wenruo btrfs_subpage_clear_uptodate(fs_info, page, eb->start,
4042fbca46ebSQu Wenruo eb->len);
40431259ab75SChris Mason }
40441259ab75SChris Mason }
40451259ab75SChris Mason
set_extent_buffer_uptodate(struct extent_buffer * eb)404609c25a8cSDavid Sterba void set_extent_buffer_uptodate(struct extent_buffer *eb)
4047d1310b2eSChris Mason {
4048251f2accSQu Wenruo struct btrfs_fs_info *fs_info = eb->fs_info;
4049d1310b2eSChris Mason struct page *page;
4050cc5e31a4SDavid Sterba int num_pages;
4051251f2accSQu Wenruo int i;
4052d1310b2eSChris Mason
40530b32f4bbSJosef Bacik set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
405465ad0104SDavid Sterba num_pages = num_extent_pages(eb);
4055d1310b2eSChris Mason for (i = 0; i < num_pages; i++) {
4056fb85fc9aSDavid Sterba page = eb->pages[i];
4057fbca46ebSQu Wenruo
4058fbca46ebSQu Wenruo /*
4059fbca46ebSQu Wenruo * This is special handling for metadata subpage, as regular
4060fbca46ebSQu Wenruo * btrfs_is_subpage() can not handle cloned/dummy metadata.
4061fbca46ebSQu Wenruo */
4062fbca46ebSQu Wenruo if (fs_info->nodesize >= PAGE_SIZE)
4063fbca46ebSQu Wenruo SetPageUptodate(page);
4064fbca46ebSQu Wenruo else
4065fbca46ebSQu Wenruo btrfs_subpage_set_uptodate(fs_info, page, eb->start,
4066fbca46ebSQu Wenruo eb->len);
4067d1310b2eSChris Mason }
4068d1310b2eSChris Mason }
4069d1310b2eSChris Mason
extent_buffer_read_end_io(struct btrfs_bio * bbio)4070046b562bSChristoph Hellwig static void extent_buffer_read_end_io(struct btrfs_bio *bbio)
4071046b562bSChristoph Hellwig {
4072046b562bSChristoph Hellwig struct extent_buffer *eb = bbio->private;
4073d7172f52SChristoph Hellwig struct btrfs_fs_info *fs_info = eb->fs_info;
4074046b562bSChristoph Hellwig bool uptodate = !bbio->bio.bi_status;
4075046b562bSChristoph Hellwig struct bvec_iter_all iter_all;
4076046b562bSChristoph Hellwig struct bio_vec *bvec;
4077046b562bSChristoph Hellwig u32 bio_offset = 0;
4078046b562bSChristoph Hellwig
4079046b562bSChristoph Hellwig eb->read_mirror = bbio->mirror_num;
4080046b562bSChristoph Hellwig
4081046b562bSChristoph Hellwig if (uptodate &&
4082046b562bSChristoph Hellwig btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0)
4083046b562bSChristoph Hellwig uptodate = false;
4084046b562bSChristoph Hellwig
4085046b562bSChristoph Hellwig if (uptodate) {
4086046b562bSChristoph Hellwig set_extent_buffer_uptodate(eb);
4087046b562bSChristoph Hellwig } else {
4088046b562bSChristoph Hellwig clear_extent_buffer_uptodate(eb);
4089046b562bSChristoph Hellwig set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
4090046b562bSChristoph Hellwig }
4091046b562bSChristoph Hellwig
4092046b562bSChristoph Hellwig bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
4093d7172f52SChristoph Hellwig u64 start = eb->start + bio_offset;
4094d7172f52SChristoph Hellwig struct page *page = bvec->bv_page;
4095d7172f52SChristoph Hellwig u32 len = bvec->bv_len;
4096d7172f52SChristoph Hellwig
4097d7172f52SChristoph Hellwig if (uptodate)
4098d7172f52SChristoph Hellwig btrfs_page_set_uptodate(fs_info, page, start, len);
4099d7172f52SChristoph Hellwig else
4100d7172f52SChristoph Hellwig btrfs_page_clear_uptodate(fs_info, page, start, len);
4101d7172f52SChristoph Hellwig
4102d7172f52SChristoph Hellwig bio_offset += len;
4103046b562bSChristoph Hellwig }
4104046b562bSChristoph Hellwig
4105d7172f52SChristoph Hellwig clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
4106d7172f52SChristoph Hellwig smp_mb__after_atomic();
4107d7172f52SChristoph Hellwig wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
4108046b562bSChristoph Hellwig free_extent_buffer(eb);
4109046b562bSChristoph Hellwig
4110046b562bSChristoph Hellwig bio_put(&bbio->bio);
4111046b562bSChristoph Hellwig }
4112046b562bSChristoph Hellwig
read_extent_buffer_pages(struct extent_buffer * eb,int wait,int mirror_num,struct btrfs_tree_parent_check * check)4113d7172f52SChristoph Hellwig int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
4114b78b98e0SChristoph Hellwig struct btrfs_tree_parent_check *check)
4115b78b98e0SChristoph Hellwig {
4116b78b98e0SChristoph Hellwig int num_pages = num_extent_pages(eb), i;
4117b78b98e0SChristoph Hellwig struct btrfs_bio *bbio;
4118b78b98e0SChristoph Hellwig
4119d7172f52SChristoph Hellwig if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4120d7172f52SChristoph Hellwig return 0;
4121d7172f52SChristoph Hellwig
4122d7172f52SChristoph Hellwig /*
4123d7172f52SChristoph Hellwig * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
4124d7172f52SChristoph Hellwig * operation, which could potentially still be in flight. In this case
4125d7172f52SChristoph Hellwig * we simply want to return an error.
4126d7172f52SChristoph Hellwig */
4127d7172f52SChristoph Hellwig if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
4128d7172f52SChristoph Hellwig return -EIO;
4129d7172f52SChristoph Hellwig
4130d7172f52SChristoph Hellwig /* Someone else is already reading the buffer, just wait for it. */
4131d7172f52SChristoph Hellwig if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags))
4132d7172f52SChristoph Hellwig goto done;
4133d7172f52SChristoph Hellwig
41340427c8efSTavian Barnes /*
41350427c8efSTavian Barnes * Between the initial test_bit(EXTENT_BUFFER_UPTODATE) and the above
41360427c8efSTavian Barnes * test_and_set_bit(EXTENT_BUFFER_READING), someone else could have
41370427c8efSTavian Barnes * started and finished reading the same eb. In this case, UPTODATE
41380427c8efSTavian Barnes * will now be set, and we shouldn't read it in again.
41390427c8efSTavian Barnes */
41400427c8efSTavian Barnes if (unlikely(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))) {
41410427c8efSTavian Barnes clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
41420427c8efSTavian Barnes smp_mb__after_atomic();
41430427c8efSTavian Barnes wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
41440427c8efSTavian Barnes return 0;
41450427c8efSTavian Barnes }
41460427c8efSTavian Barnes
4147b78b98e0SChristoph Hellwig clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
4148b78b98e0SChristoph Hellwig eb->read_mirror = 0;
4149b78b98e0SChristoph Hellwig check_buffer_tree_ref(eb);
4150113fa05cSChristoph Hellwig atomic_inc(&eb->refs);
4151b78b98e0SChristoph Hellwig
4152b78b98e0SChristoph Hellwig bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
4153b78b98e0SChristoph Hellwig REQ_OP_READ | REQ_META, eb->fs_info,
4154046b562bSChristoph Hellwig extent_buffer_read_end_io, eb);
4155b78b98e0SChristoph Hellwig bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
4156b78b98e0SChristoph Hellwig bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
4157b78b98e0SChristoph Hellwig bbio->file_offset = eb->start;
4158b78b98e0SChristoph Hellwig memcpy(&bbio->parent_check, check, sizeof(*check));
4159b78b98e0SChristoph Hellwig if (eb->fs_info->nodesize < PAGE_SIZE) {
4160b78b98e0SChristoph Hellwig __bio_add_page(&bbio->bio, eb->pages[0], eb->len,
4161b78b98e0SChristoph Hellwig eb->start - page_offset(eb->pages[0]));
4162b78b98e0SChristoph Hellwig } else {
4163011134f4SChristoph Hellwig for (i = 0; i < num_pages; i++)
4164b78b98e0SChristoph Hellwig __bio_add_page(&bbio->bio, eb->pages[i], PAGE_SIZE, 0);
4165b78b98e0SChristoph Hellwig }
4166b78b98e0SChristoph Hellwig btrfs_submit_bio(bbio, mirror_num);
4167b78b98e0SChristoph Hellwig
4168d7172f52SChristoph Hellwig done:
4169d7172f52SChristoph Hellwig if (wait == WAIT_COMPLETE) {
4170d7172f52SChristoph Hellwig wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
41714012daf7SQu Wenruo if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
417255173337SChristoph Hellwig return -EIO;
41734012daf7SQu Wenruo }
41744012daf7SQu Wenruo
417555173337SChristoph Hellwig return 0;
4176d1310b2eSChris Mason }
4177d1310b2eSChris Mason
report_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)4178f98b6215SQu Wenruo static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
4179f98b6215SQu Wenruo unsigned long len)
4180f98b6215SQu Wenruo {
4181f98b6215SQu Wenruo btrfs_warn(eb->fs_info,
4182f98b6215SQu Wenruo "access to eb bytenr %llu len %lu out of range start %lu len %lu",
4183f98b6215SQu Wenruo eb->start, eb->len, start, len);
4184f98b6215SQu Wenruo WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4185f98b6215SQu Wenruo
4186f98b6215SQu Wenruo return true;
4187f98b6215SQu Wenruo }
4188f98b6215SQu Wenruo
4189f98b6215SQu Wenruo /*
4190f98b6215SQu Wenruo * Check if the [start, start + len) range is valid before reading/writing
4191f98b6215SQu Wenruo * the eb.
4192f98b6215SQu Wenruo * NOTE: @start and @len are offset inside the eb, not logical address.
4193f98b6215SQu Wenruo *
4194f98b6215SQu Wenruo * Caller should not touch the dst/src memory if this function returns error.
4195f98b6215SQu Wenruo */
check_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)4196f98b6215SQu Wenruo static inline int check_eb_range(const struct extent_buffer *eb,
4197f98b6215SQu Wenruo unsigned long start, unsigned long len)
4198f98b6215SQu Wenruo {
4199f98b6215SQu Wenruo unsigned long offset;
4200f98b6215SQu Wenruo
4201f98b6215SQu Wenruo /* start, start + len should not go beyond eb->len nor overflow */
4202f98b6215SQu Wenruo if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
4203f98b6215SQu Wenruo return report_eb_range(eb, start, len);
4204f98b6215SQu Wenruo
4205f98b6215SQu Wenruo return false;
4206f98b6215SQu Wenruo }
4207f98b6215SQu Wenruo
read_extent_buffer(const struct extent_buffer * eb,void * dstv,unsigned long start,unsigned long len)42081cbb1f45SJeff Mahoney void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
42091cbb1f45SJeff Mahoney unsigned long start, unsigned long len)
4210d1310b2eSChris Mason {
4211d1310b2eSChris Mason size_t cur;
4212d1310b2eSChris Mason size_t offset;
4213d1310b2eSChris Mason struct page *page;
4214d1310b2eSChris Mason char *kaddr;
4215d1310b2eSChris Mason char *dst = (char *)dstv;
4216884b07d0SQu Wenruo unsigned long i = get_eb_page_index(start);
4217d1310b2eSChris Mason
421874ee7914SQu Wenruo if (check_eb_range(eb, start, len)) {
421974ee7914SQu Wenruo /*
422074ee7914SQu Wenruo * Invalid range hit, reset the memory, so callers won't get
422174ee7914SQu Wenruo * some random garbage for their uninitialzed memory.
422274ee7914SQu Wenruo */
422374ee7914SQu Wenruo memset(dstv, 0, len);
4224f716abd5SLiu Bo return;
422574ee7914SQu Wenruo }
4226d1310b2eSChris Mason
4227884b07d0SQu Wenruo offset = get_eb_offset_in_page(eb, start);
4228d1310b2eSChris Mason
4229d1310b2eSChris Mason while (len > 0) {
4230fb85fc9aSDavid Sterba page = eb->pages[i];
4231d1310b2eSChris Mason
423209cbfeafSKirill A. Shutemov cur = min(len, (PAGE_SIZE - offset));
4233a6591715SChris Mason kaddr = page_address(page);
4234d1310b2eSChris Mason memcpy(dst, kaddr + offset, cur);
4235d1310b2eSChris Mason
4236d1310b2eSChris Mason dst += cur;
4237d1310b2eSChris Mason len -= cur;
4238d1310b2eSChris Mason offset = 0;
4239d1310b2eSChris Mason i++;
4240d1310b2eSChris Mason }
4241d1310b2eSChris Mason }
4242d1310b2eSChris Mason
read_extent_buffer_to_user_nofault(const struct extent_buffer * eb,void __user * dstv,unsigned long start,unsigned long len)4243a48b73ecSJosef Bacik int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
42441cbb1f45SJeff Mahoney void __user *dstv,
42451cbb1f45SJeff Mahoney unsigned long start, unsigned long len)
4246550ac1d8SGerhard Heift {
4247550ac1d8SGerhard Heift size_t cur;
4248550ac1d8SGerhard Heift size_t offset;
4249550ac1d8SGerhard Heift struct page *page;
4250550ac1d8SGerhard Heift char *kaddr;
4251550ac1d8SGerhard Heift char __user *dst = (char __user *)dstv;
4252884b07d0SQu Wenruo unsigned long i = get_eb_page_index(start);
4253550ac1d8SGerhard Heift int ret = 0;
4254550ac1d8SGerhard Heift
4255550ac1d8SGerhard Heift WARN_ON(start > eb->len);
4256550ac1d8SGerhard Heift WARN_ON(start + len > eb->start + eb->len);
4257550ac1d8SGerhard Heift
4258884b07d0SQu Wenruo offset = get_eb_offset_in_page(eb, start);
4259550ac1d8SGerhard Heift
4260550ac1d8SGerhard Heift while (len > 0) {
4261fb85fc9aSDavid Sterba page = eb->pages[i];
4262550ac1d8SGerhard Heift
426309cbfeafSKirill A. Shutemov cur = min(len, (PAGE_SIZE - offset));
4264550ac1d8SGerhard Heift kaddr = page_address(page);
4265a48b73ecSJosef Bacik if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
4266550ac1d8SGerhard Heift ret = -EFAULT;
4267550ac1d8SGerhard Heift break;
4268550ac1d8SGerhard Heift }
4269550ac1d8SGerhard Heift
4270550ac1d8SGerhard Heift dst += cur;
4271550ac1d8SGerhard Heift len -= cur;
4272550ac1d8SGerhard Heift offset = 0;
4273550ac1d8SGerhard Heift i++;
4274550ac1d8SGerhard Heift }
4275550ac1d8SGerhard Heift
4276550ac1d8SGerhard Heift return ret;
4277550ac1d8SGerhard Heift }
4278550ac1d8SGerhard Heift
memcmp_extent_buffer(const struct extent_buffer * eb,const void * ptrv,unsigned long start,unsigned long len)42791cbb1f45SJeff Mahoney int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
42801cbb1f45SJeff Mahoney unsigned long start, unsigned long len)
4281d1310b2eSChris Mason {
4282d1310b2eSChris Mason size_t cur;
4283d1310b2eSChris Mason size_t offset;
4284d1310b2eSChris Mason struct page *page;
4285d1310b2eSChris Mason char *kaddr;
4286d1310b2eSChris Mason char *ptr = (char *)ptrv;
4287884b07d0SQu Wenruo unsigned long i = get_eb_page_index(start);
4288d1310b2eSChris Mason int ret = 0;
4289d1310b2eSChris Mason
4290f98b6215SQu Wenruo if (check_eb_range(eb, start, len))
4291f98b6215SQu Wenruo return -EINVAL;
4292d1310b2eSChris Mason
4293884b07d0SQu Wenruo offset = get_eb_offset_in_page(eb, start);
4294d1310b2eSChris Mason
4295d1310b2eSChris Mason while (len > 0) {
4296fb85fc9aSDavid Sterba page = eb->pages[i];
4297d1310b2eSChris Mason
429809cbfeafSKirill A. Shutemov cur = min(len, (PAGE_SIZE - offset));
4299d1310b2eSChris Mason
4300a6591715SChris Mason kaddr = page_address(page);
4301d1310b2eSChris Mason ret = memcmp(ptr, kaddr + offset, cur);
4302d1310b2eSChris Mason if (ret)
4303d1310b2eSChris Mason break;
4304d1310b2eSChris Mason
4305d1310b2eSChris Mason ptr += cur;
4306d1310b2eSChris Mason len -= cur;
4307d1310b2eSChris Mason offset = 0;
4308d1310b2eSChris Mason i++;
4309d1310b2eSChris Mason }
4310d1310b2eSChris Mason return ret;
4311d1310b2eSChris Mason }
4312d1310b2eSChris Mason
4313b8f95771SQu Wenruo /*
4314b8f95771SQu Wenruo * Check that the extent buffer is uptodate.
4315b8f95771SQu Wenruo *
4316b8f95771SQu Wenruo * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4317b8f95771SQu Wenruo * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4318b8f95771SQu Wenruo */
assert_eb_page_uptodate(const struct extent_buffer * eb,struct page * page)4319b8f95771SQu Wenruo static void assert_eb_page_uptodate(const struct extent_buffer *eb,
4320b8f95771SQu Wenruo struct page *page)
4321b8f95771SQu Wenruo {
4322b8f95771SQu Wenruo struct btrfs_fs_info *fs_info = eb->fs_info;
4323b8f95771SQu Wenruo
4324a50e1fcbSJosef Bacik /*
4325a50e1fcbSJosef Bacik * If we are using the commit root we could potentially clear a page
4326a50e1fcbSJosef Bacik * Uptodate while we're using the extent buffer that we've previously
4327a50e1fcbSJosef Bacik * looked up. We don't want to complain in this case, as the page was
4328a50e1fcbSJosef Bacik * valid before, we just didn't write it out. Instead we want to catch
4329a50e1fcbSJosef Bacik * the case where we didn't actually read the block properly, which
4330011134f4SChristoph Hellwig * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4331a50e1fcbSJosef Bacik */
4332011134f4SChristoph Hellwig if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4333011134f4SChristoph Hellwig return;
4334b8f95771SQu Wenruo
4335011134f4SChristoph Hellwig if (fs_info->nodesize < PAGE_SIZE) {
433675258f20SQu Wenruo if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, page,
433775258f20SQu Wenruo eb->start, eb->len)))
433875258f20SQu Wenruo btrfs_subpage_dump_bitmap(fs_info, page, eb->start, eb->len);
4339b8f95771SQu Wenruo } else {
4340011134f4SChristoph Hellwig WARN_ON(!PageUptodate(page));
4341b8f95771SQu Wenruo }
4342b8f95771SQu Wenruo }
4343b8f95771SQu Wenruo
__write_extent_buffer(const struct extent_buffer * eb,const void * srcv,unsigned long start,unsigned long len,bool use_memmove)434413840f3fSQu Wenruo static void __write_extent_buffer(const struct extent_buffer *eb,
434513840f3fSQu Wenruo const void *srcv, unsigned long start,
434613840f3fSQu Wenruo unsigned long len, bool use_memmove)
4347d1310b2eSChris Mason {
4348d1310b2eSChris Mason size_t cur;
4349d1310b2eSChris Mason size_t offset;
4350d1310b2eSChris Mason struct page *page;
4351d1310b2eSChris Mason char *kaddr;
4352d1310b2eSChris Mason char *src = (char *)srcv;
4353884b07d0SQu Wenruo unsigned long i = get_eb_page_index(start);
435413840f3fSQu Wenruo /* For unmapped (dummy) ebs, no need to check their uptodate status. */
435513840f3fSQu Wenruo const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4356d1310b2eSChris Mason
4357d3575156SNaohiro Aota WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags));
4358d3575156SNaohiro Aota
4359f98b6215SQu Wenruo if (check_eb_range(eb, start, len))
4360f98b6215SQu Wenruo return;
4361d1310b2eSChris Mason
4362884b07d0SQu Wenruo offset = get_eb_offset_in_page(eb, start);
4363d1310b2eSChris Mason
4364d1310b2eSChris Mason while (len > 0) {
4365fb85fc9aSDavid Sterba page = eb->pages[i];
436613840f3fSQu Wenruo if (check_uptodate)
4367b8f95771SQu Wenruo assert_eb_page_uptodate(eb, page);
4368d1310b2eSChris Mason
436909cbfeafSKirill A. Shutemov cur = min(len, PAGE_SIZE - offset);
4370a6591715SChris Mason kaddr = page_address(page);
437113840f3fSQu Wenruo if (use_memmove)
437213840f3fSQu Wenruo memmove(kaddr + offset, src, cur);
437313840f3fSQu Wenruo else
4374d1310b2eSChris Mason memcpy(kaddr + offset, src, cur);
4375d1310b2eSChris Mason
4376d1310b2eSChris Mason src += cur;
4377d1310b2eSChris Mason len -= cur;
4378d1310b2eSChris Mason offset = 0;
4379d1310b2eSChris Mason i++;
4380d1310b2eSChris Mason }
4381d1310b2eSChris Mason }
4382d1310b2eSChris Mason
write_extent_buffer(const struct extent_buffer * eb,const void * srcv,unsigned long start,unsigned long len)438313840f3fSQu Wenruo void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
438413840f3fSQu Wenruo unsigned long start, unsigned long len)
438513840f3fSQu Wenruo {
438613840f3fSQu Wenruo return __write_extent_buffer(eb, srcv, start, len, false);
438713840f3fSQu Wenruo }
438813840f3fSQu Wenruo
memset_extent_buffer(const struct extent_buffer * eb,int c,unsigned long start,unsigned long len)4389cb22964fSQu Wenruo static void memset_extent_buffer(const struct extent_buffer *eb, int c,
4390cb22964fSQu Wenruo unsigned long start, unsigned long len)
4391cb22964fSQu Wenruo {
4392cb22964fSQu Wenruo unsigned long cur = start;
4393cb22964fSQu Wenruo
4394cb22964fSQu Wenruo while (cur < start + len) {
4395cb22964fSQu Wenruo unsigned long index = get_eb_page_index(cur);
4396cb22964fSQu Wenruo unsigned int offset = get_eb_offset_in_page(eb, cur);
4397cb22964fSQu Wenruo unsigned int cur_len = min(start + len - cur, PAGE_SIZE - offset);
4398cb22964fSQu Wenruo struct page *page = eb->pages[index];
4399cb22964fSQu Wenruo
4400cb22964fSQu Wenruo assert_eb_page_uptodate(eb, page);
4401cb22964fSQu Wenruo memset(page_address(page) + offset, c, cur_len);
4402cb22964fSQu Wenruo
4403cb22964fSQu Wenruo cur += cur_len;
4404cb22964fSQu Wenruo }
4405cb22964fSQu Wenruo }
4406cb22964fSQu Wenruo
memzero_extent_buffer(const struct extent_buffer * eb,unsigned long start,unsigned long len)44072b48966aSDavid Sterba void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4408b159fa28SDavid Sterba unsigned long len)
4409d1310b2eSChris Mason {
4410f98b6215SQu Wenruo if (check_eb_range(eb, start, len))
4411f98b6215SQu Wenruo return;
4412cb22964fSQu Wenruo return memset_extent_buffer(eb, 0, start, len);
4413d1310b2eSChris Mason }
4414d1310b2eSChris Mason
copy_extent_buffer_full(const struct extent_buffer * dst,const struct extent_buffer * src)44152b48966aSDavid Sterba void copy_extent_buffer_full(const struct extent_buffer *dst,
44162b48966aSDavid Sterba const struct extent_buffer *src)
441758e8012cSDavid Sterba {
441854948681SQu Wenruo unsigned long cur = 0;
441958e8012cSDavid Sterba
442058e8012cSDavid Sterba ASSERT(dst->len == src->len);
442158e8012cSDavid Sterba
442254948681SQu Wenruo while (cur < src->len) {
442354948681SQu Wenruo unsigned long index = get_eb_page_index(cur);
442454948681SQu Wenruo unsigned long offset = get_eb_offset_in_page(src, cur);
442554948681SQu Wenruo unsigned long cur_len = min(src->len, PAGE_SIZE - offset);
442654948681SQu Wenruo void *addr = page_address(src->pages[index]) + offset;
4427884b07d0SQu Wenruo
442854948681SQu Wenruo write_extent_buffer(dst, addr, cur, cur_len);
442954948681SQu Wenruo
443054948681SQu Wenruo cur += cur_len;
4431884b07d0SQu Wenruo }
443258e8012cSDavid Sterba }
443358e8012cSDavid Sterba
copy_extent_buffer(const struct extent_buffer * dst,const struct extent_buffer * src,unsigned long dst_offset,unsigned long src_offset,unsigned long len)44342b48966aSDavid Sterba void copy_extent_buffer(const struct extent_buffer *dst,
44352b48966aSDavid Sterba const struct extent_buffer *src,
4436d1310b2eSChris Mason unsigned long dst_offset, unsigned long src_offset,
4437d1310b2eSChris Mason unsigned long len)
4438d1310b2eSChris Mason {
4439d1310b2eSChris Mason u64 dst_len = dst->len;
4440d1310b2eSChris Mason size_t cur;
4441d1310b2eSChris Mason size_t offset;
4442d1310b2eSChris Mason struct page *page;
4443d1310b2eSChris Mason char *kaddr;
4444884b07d0SQu Wenruo unsigned long i = get_eb_page_index(dst_offset);
4445d1310b2eSChris Mason
4446f98b6215SQu Wenruo if (check_eb_range(dst, dst_offset, len) ||
4447f98b6215SQu Wenruo check_eb_range(src, src_offset, len))
4448f98b6215SQu Wenruo return;
4449f98b6215SQu Wenruo
4450d1310b2eSChris Mason WARN_ON(src->len != dst_len);
4451d1310b2eSChris Mason
4452884b07d0SQu Wenruo offset = get_eb_offset_in_page(dst, dst_offset);
4453d1310b2eSChris Mason
4454d1310b2eSChris Mason while (len > 0) {
4455fb85fc9aSDavid Sterba page = dst->pages[i];
4456b8f95771SQu Wenruo assert_eb_page_uptodate(dst, page);
4457d1310b2eSChris Mason
445809cbfeafSKirill A. Shutemov cur = min(len, (unsigned long)(PAGE_SIZE - offset));
4459d1310b2eSChris Mason
4460a6591715SChris Mason kaddr = page_address(page);
4461d1310b2eSChris Mason read_extent_buffer(src, kaddr + offset, src_offset, cur);
4462d1310b2eSChris Mason
4463d1310b2eSChris Mason src_offset += cur;
4464d1310b2eSChris Mason len -= cur;
4465d1310b2eSChris Mason offset = 0;
4466d1310b2eSChris Mason i++;
4467d1310b2eSChris Mason }
4468d1310b2eSChris Mason }
4469d1310b2eSChris Mason
44703e1e8bb7SOmar Sandoval /*
44713e1e8bb7SOmar Sandoval * eb_bitmap_offset() - calculate the page and offset of the byte containing the
44723e1e8bb7SOmar Sandoval * given bit number
44733e1e8bb7SOmar Sandoval * @eb: the extent buffer
44743e1e8bb7SOmar Sandoval * @start: offset of the bitmap item in the extent buffer
44753e1e8bb7SOmar Sandoval * @nr: bit number
44763e1e8bb7SOmar Sandoval * @page_index: return index of the page in the extent buffer that contains the
44773e1e8bb7SOmar Sandoval * given bit number
44783e1e8bb7SOmar Sandoval * @page_offset: return offset into the page given by page_index
44793e1e8bb7SOmar Sandoval *
44803e1e8bb7SOmar Sandoval * This helper hides the ugliness of finding the byte in an extent buffer which
44813e1e8bb7SOmar Sandoval * contains a given bit.
44823e1e8bb7SOmar Sandoval */
eb_bitmap_offset(const struct extent_buffer * eb,unsigned long start,unsigned long nr,unsigned long * page_index,size_t * page_offset)44832b48966aSDavid Sterba static inline void eb_bitmap_offset(const struct extent_buffer *eb,
44843e1e8bb7SOmar Sandoval unsigned long start, unsigned long nr,
44853e1e8bb7SOmar Sandoval unsigned long *page_index,
44863e1e8bb7SOmar Sandoval size_t *page_offset)
44873e1e8bb7SOmar Sandoval {
44883e1e8bb7SOmar Sandoval size_t byte_offset = BIT_BYTE(nr);
44893e1e8bb7SOmar Sandoval size_t offset;
44903e1e8bb7SOmar Sandoval
44913e1e8bb7SOmar Sandoval /*
44923e1e8bb7SOmar Sandoval * The byte we want is the offset of the extent buffer + the offset of
44933e1e8bb7SOmar Sandoval * the bitmap item in the extent buffer + the offset of the byte in the
44943e1e8bb7SOmar Sandoval * bitmap item.
44953e1e8bb7SOmar Sandoval */
4496884b07d0SQu Wenruo offset = start + offset_in_page(eb->start) + byte_offset;
44973e1e8bb7SOmar Sandoval
449809cbfeafSKirill A. Shutemov *page_index = offset >> PAGE_SHIFT;
44997073017aSJohannes Thumshirn *page_offset = offset_in_page(offset);
45003e1e8bb7SOmar Sandoval }
45013e1e8bb7SOmar Sandoval
450243dd529aSDavid Sterba /*
450343dd529aSDavid Sterba * Determine whether a bit in a bitmap item is set.
450443dd529aSDavid Sterba *
45053e1e8bb7SOmar Sandoval * @eb: the extent buffer
45063e1e8bb7SOmar Sandoval * @start: offset of the bitmap item in the extent buffer
45073e1e8bb7SOmar Sandoval * @nr: bit number to test
45083e1e8bb7SOmar Sandoval */
extent_buffer_test_bit(const struct extent_buffer * eb,unsigned long start,unsigned long nr)45092b48966aSDavid Sterba int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
45103e1e8bb7SOmar Sandoval unsigned long nr)
45113e1e8bb7SOmar Sandoval {
45122fe1d551SOmar Sandoval u8 *kaddr;
45133e1e8bb7SOmar Sandoval struct page *page;
45143e1e8bb7SOmar Sandoval unsigned long i;
45153e1e8bb7SOmar Sandoval size_t offset;
45163e1e8bb7SOmar Sandoval
45173e1e8bb7SOmar Sandoval eb_bitmap_offset(eb, start, nr, &i, &offset);
45183e1e8bb7SOmar Sandoval page = eb->pages[i];
4519b8f95771SQu Wenruo assert_eb_page_uptodate(eb, page);
45203e1e8bb7SOmar Sandoval kaddr = page_address(page);
45213e1e8bb7SOmar Sandoval return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
45223e1e8bb7SOmar Sandoval }
45233e1e8bb7SOmar Sandoval
extent_buffer_get_byte(const struct extent_buffer * eb,unsigned long bytenr)4524cb22964fSQu Wenruo static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
4525cb22964fSQu Wenruo {
4526cb22964fSQu Wenruo unsigned long index = get_eb_page_index(bytenr);
4527cb22964fSQu Wenruo
4528cb22964fSQu Wenruo if (check_eb_range(eb, bytenr, 1))
4529cb22964fSQu Wenruo return NULL;
4530cb22964fSQu Wenruo return page_address(eb->pages[index]) + get_eb_offset_in_page(eb, bytenr);
4531cb22964fSQu Wenruo }
4532cb22964fSQu Wenruo
453343dd529aSDavid Sterba /*
453443dd529aSDavid Sterba * Set an area of a bitmap to 1.
453543dd529aSDavid Sterba *
45363e1e8bb7SOmar Sandoval * @eb: the extent buffer
45373e1e8bb7SOmar Sandoval * @start: offset of the bitmap item in the extent buffer
45383e1e8bb7SOmar Sandoval * @pos: bit number of the first bit
45393e1e8bb7SOmar Sandoval * @len: number of bits to set
45403e1e8bb7SOmar Sandoval */
extent_buffer_bitmap_set(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)45412b48966aSDavid Sterba void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
45423e1e8bb7SOmar Sandoval unsigned long pos, unsigned long len)
45433e1e8bb7SOmar Sandoval {
4544cb22964fSQu Wenruo unsigned int first_byte = start + BIT_BYTE(pos);
4545cb22964fSQu Wenruo unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4546cb22964fSQu Wenruo const bool same_byte = (first_byte == last_byte);
4547cb22964fSQu Wenruo u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
45482fe1d551SOmar Sandoval u8 *kaddr;
45493e1e8bb7SOmar Sandoval
4550cb22964fSQu Wenruo if (same_byte)
4551cb22964fSQu Wenruo mask &= BITMAP_LAST_BYTE_MASK(pos + len);
45523e1e8bb7SOmar Sandoval
4553cb22964fSQu Wenruo /* Handle the first byte. */
4554cb22964fSQu Wenruo kaddr = extent_buffer_get_byte(eb, first_byte);
4555cb22964fSQu Wenruo *kaddr |= mask;
4556cb22964fSQu Wenruo if (same_byte)
4557cb22964fSQu Wenruo return;
4558cb22964fSQu Wenruo
4559cb22964fSQu Wenruo /* Handle the byte aligned part. */
4560cb22964fSQu Wenruo ASSERT(first_byte + 1 <= last_byte);
4561cb22964fSQu Wenruo memset_extent_buffer(eb, 0xff, first_byte + 1, last_byte - first_byte - 1);
4562cb22964fSQu Wenruo
4563cb22964fSQu Wenruo /* Handle the last byte. */
4564cb22964fSQu Wenruo kaddr = extent_buffer_get_byte(eb, last_byte);
4565cb22964fSQu Wenruo *kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
45663e1e8bb7SOmar Sandoval }
45673e1e8bb7SOmar Sandoval
45683e1e8bb7SOmar Sandoval
456943dd529aSDavid Sterba /*
457043dd529aSDavid Sterba * Clear an area of a bitmap.
457143dd529aSDavid Sterba *
45723e1e8bb7SOmar Sandoval * @eb: the extent buffer
45733e1e8bb7SOmar Sandoval * @start: offset of the bitmap item in the extent buffer
45743e1e8bb7SOmar Sandoval * @pos: bit number of the first bit
45753e1e8bb7SOmar Sandoval * @len: number of bits to clear
45763e1e8bb7SOmar Sandoval */
extent_buffer_bitmap_clear(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)45772b48966aSDavid Sterba void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
45782b48966aSDavid Sterba unsigned long start, unsigned long pos,
45792b48966aSDavid Sterba unsigned long len)
45803e1e8bb7SOmar Sandoval {
4581cb22964fSQu Wenruo unsigned int first_byte = start + BIT_BYTE(pos);
4582cb22964fSQu Wenruo unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4583cb22964fSQu Wenruo const bool same_byte = (first_byte == last_byte);
4584cb22964fSQu Wenruo u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
45852fe1d551SOmar Sandoval u8 *kaddr;
45863e1e8bb7SOmar Sandoval
4587cb22964fSQu Wenruo if (same_byte)
4588cb22964fSQu Wenruo mask &= BITMAP_LAST_BYTE_MASK(pos + len);
45893e1e8bb7SOmar Sandoval
4590cb22964fSQu Wenruo /* Handle the first byte. */
4591cb22964fSQu Wenruo kaddr = extent_buffer_get_byte(eb, first_byte);
4592cb22964fSQu Wenruo *kaddr &= ~mask;
4593cb22964fSQu Wenruo if (same_byte)
4594cb22964fSQu Wenruo return;
4595cb22964fSQu Wenruo
4596cb22964fSQu Wenruo /* Handle the byte aligned part. */
4597cb22964fSQu Wenruo ASSERT(first_byte + 1 <= last_byte);
4598cb22964fSQu Wenruo memset_extent_buffer(eb, 0, first_byte + 1, last_byte - first_byte - 1);
4599cb22964fSQu Wenruo
4600cb22964fSQu Wenruo /* Handle the last byte. */
4601cb22964fSQu Wenruo kaddr = extent_buffer_get_byte(eb, last_byte);
4602cb22964fSQu Wenruo *kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
46033e1e8bb7SOmar Sandoval }
46043e1e8bb7SOmar Sandoval
areas_overlap(unsigned long src,unsigned long dst,unsigned long len)46053387206fSSergei Trofimovich static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
46063387206fSSergei Trofimovich {
46073387206fSSergei Trofimovich unsigned long distance = (src > dst) ? src - dst : dst - src;
46083387206fSSergei Trofimovich return distance < len;
46093387206fSSergei Trofimovich }
46103387206fSSergei Trofimovich
memcpy_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)46112b48966aSDavid Sterba void memcpy_extent_buffer(const struct extent_buffer *dst,
46122b48966aSDavid Sterba unsigned long dst_offset, unsigned long src_offset,
46132b48966aSDavid Sterba unsigned long len)
4614d1310b2eSChris Mason {
461513840f3fSQu Wenruo unsigned long cur_off = 0;
4616d1310b2eSChris Mason
4617f98b6215SQu Wenruo if (check_eb_range(dst, dst_offset, len) ||
4618f98b6215SQu Wenruo check_eb_range(dst, src_offset, len))
4619f98b6215SQu Wenruo return;
4620d1310b2eSChris Mason
462113840f3fSQu Wenruo while (cur_off < len) {
462213840f3fSQu Wenruo unsigned long cur_src = cur_off + src_offset;
462313840f3fSQu Wenruo unsigned long pg_index = get_eb_page_index(cur_src);
462413840f3fSQu Wenruo unsigned long pg_off = get_eb_offset_in_page(dst, cur_src);
462513840f3fSQu Wenruo unsigned long cur_len = min(src_offset + len - cur_src,
462613840f3fSQu Wenruo PAGE_SIZE - pg_off);
462713840f3fSQu Wenruo void *src_addr = page_address(dst->pages[pg_index]) + pg_off;
462813840f3fSQu Wenruo const bool use_memmove = areas_overlap(src_offset + cur_off,
462913840f3fSQu Wenruo dst_offset + cur_off, cur_len);
4630d1310b2eSChris Mason
463113840f3fSQu Wenruo __write_extent_buffer(dst, src_addr, dst_offset + cur_off, cur_len,
463213840f3fSQu Wenruo use_memmove);
463313840f3fSQu Wenruo cur_off += cur_len;
4634d1310b2eSChris Mason }
4635d1310b2eSChris Mason }
4636d1310b2eSChris Mason
memmove_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)46372b48966aSDavid Sterba void memmove_extent_buffer(const struct extent_buffer *dst,
46382b48966aSDavid Sterba unsigned long dst_offset, unsigned long src_offset,
46392b48966aSDavid Sterba unsigned long len)
4640d1310b2eSChris Mason {
4641d1310b2eSChris Mason unsigned long dst_end = dst_offset + len - 1;
4642d1310b2eSChris Mason unsigned long src_end = src_offset + len - 1;
4643d1310b2eSChris Mason
4644f98b6215SQu Wenruo if (check_eb_range(dst, dst_offset, len) ||
4645f98b6215SQu Wenruo check_eb_range(dst, src_offset, len))
4646f98b6215SQu Wenruo return;
4647096d2301SQu Wenruo
4648727011e0SChris Mason if (dst_offset < src_offset) {
4649d1310b2eSChris Mason memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4650d1310b2eSChris Mason return;
4651d1310b2eSChris Mason }
4652096d2301SQu Wenruo
4653d1310b2eSChris Mason while (len > 0) {
4654096d2301SQu Wenruo unsigned long src_i;
4655096d2301SQu Wenruo size_t cur;
4656096d2301SQu Wenruo size_t dst_off_in_page;
4657096d2301SQu Wenruo size_t src_off_in_page;
4658096d2301SQu Wenruo void *src_addr;
4659096d2301SQu Wenruo bool use_memmove;
4660096d2301SQu Wenruo
4661884b07d0SQu Wenruo src_i = get_eb_page_index(src_end);
4662d1310b2eSChris Mason
4663884b07d0SQu Wenruo dst_off_in_page = get_eb_offset_in_page(dst, dst_end);
4664884b07d0SQu Wenruo src_off_in_page = get_eb_offset_in_page(dst, src_end);
4665d1310b2eSChris Mason
4666d1310b2eSChris Mason cur = min_t(unsigned long, len, src_off_in_page + 1);
4667d1310b2eSChris Mason cur = min(cur, dst_off_in_page + 1);
4668096d2301SQu Wenruo
4669096d2301SQu Wenruo src_addr = page_address(dst->pages[src_i]) + src_off_in_page -
4670096d2301SQu Wenruo cur + 1;
4671096d2301SQu Wenruo use_memmove = areas_overlap(src_end - cur + 1, dst_end - cur + 1,
4672096d2301SQu Wenruo cur);
4673096d2301SQu Wenruo
4674096d2301SQu Wenruo __write_extent_buffer(dst, src_addr, dst_end - cur + 1, cur,
4675096d2301SQu Wenruo use_memmove);
4676d1310b2eSChris Mason
4677d1310b2eSChris Mason dst_end -= cur;
4678d1310b2eSChris Mason src_end -= cur;
4679d1310b2eSChris Mason len -= cur;
4680d1310b2eSChris Mason }
4681d1310b2eSChris Mason }
46826af118ceSChris Mason
468301cd3909SDavid Sterba #define GANG_LOOKUP_SIZE 16
get_next_extent_buffer(struct btrfs_fs_info * fs_info,struct page * page,u64 bytenr)4684d1e86e3fSQu Wenruo static struct extent_buffer *get_next_extent_buffer(
4685d1e86e3fSQu Wenruo struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
4686d1e86e3fSQu Wenruo {
468701cd3909SDavid Sterba struct extent_buffer *gang[GANG_LOOKUP_SIZE];
468801cd3909SDavid Sterba struct extent_buffer *found = NULL;
4689d1e86e3fSQu Wenruo u64 page_start = page_offset(page);
469001cd3909SDavid Sterba u64 cur = page_start;
469172a69cd0SQu Wenruo
469272a69cd0SQu Wenruo ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
469372a69cd0SQu Wenruo lockdep_assert_held(&fs_info->buffer_lock);
469472a69cd0SQu Wenruo
469501cd3909SDavid Sterba while (cur < page_start + PAGE_SIZE) {
469601cd3909SDavid Sterba int ret;
469701cd3909SDavid Sterba int i;
469801cd3909SDavid Sterba
469901cd3909SDavid Sterba ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
470001cd3909SDavid Sterba (void **)gang, cur >> fs_info->sectorsize_bits,
470101cd3909SDavid Sterba min_t(unsigned int, GANG_LOOKUP_SIZE,
470201cd3909SDavid Sterba PAGE_SIZE / fs_info->nodesize));
470301cd3909SDavid Sterba if (ret == 0)
470401cd3909SDavid Sterba goto out;
470501cd3909SDavid Sterba for (i = 0; i < ret; i++) {
4706d1e86e3fSQu Wenruo /* Already beyond page end */
470701cd3909SDavid Sterba if (gang[i]->start >= page_start + PAGE_SIZE)
470801cd3909SDavid Sterba goto out;
470901cd3909SDavid Sterba /* Found one */
471001cd3909SDavid Sterba if (gang[i]->start >= bytenr) {
471101cd3909SDavid Sterba found = gang[i];
471201cd3909SDavid Sterba goto out;
4713d1e86e3fSQu Wenruo }
471401cd3909SDavid Sterba }
471501cd3909SDavid Sterba cur = gang[ret - 1]->start + gang[ret - 1]->len;
471601cd3909SDavid Sterba }
471701cd3909SDavid Sterba out:
471801cd3909SDavid Sterba return found;
4719d1e86e3fSQu Wenruo }
4720d1e86e3fSQu Wenruo
try_release_subpage_extent_buffer(struct page * page)4721d1e86e3fSQu Wenruo static int try_release_subpage_extent_buffer(struct page *page)
4722d1e86e3fSQu Wenruo {
4723d1e86e3fSQu Wenruo struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
4724d1e86e3fSQu Wenruo u64 cur = page_offset(page);
4725d1e86e3fSQu Wenruo const u64 end = page_offset(page) + PAGE_SIZE;
4726d1e86e3fSQu Wenruo int ret;
4727d1e86e3fSQu Wenruo
4728d1e86e3fSQu Wenruo while (cur < end) {
4729d1e86e3fSQu Wenruo struct extent_buffer *eb = NULL;
4730d1e86e3fSQu Wenruo
4731d1e86e3fSQu Wenruo /*
4732d1e86e3fSQu Wenruo * Unlike try_release_extent_buffer() which uses page->private
4733d1e86e3fSQu Wenruo * to grab buffer, for subpage case we rely on radix tree, thus
4734d1e86e3fSQu Wenruo * we need to ensure radix tree consistency.
4735d1e86e3fSQu Wenruo *
4736d1e86e3fSQu Wenruo * We also want an atomic snapshot of the radix tree, thus go
4737d1e86e3fSQu Wenruo * with spinlock rather than RCU.
4738d1e86e3fSQu Wenruo */
4739d1e86e3fSQu Wenruo spin_lock(&fs_info->buffer_lock);
4740d1e86e3fSQu Wenruo eb = get_next_extent_buffer(fs_info, page, cur);
4741d1e86e3fSQu Wenruo if (!eb) {
4742d1e86e3fSQu Wenruo /* No more eb in the page range after or at cur */
4743d1e86e3fSQu Wenruo spin_unlock(&fs_info->buffer_lock);
4744d1e86e3fSQu Wenruo break;
4745d1e86e3fSQu Wenruo }
4746d1e86e3fSQu Wenruo cur = eb->start + eb->len;
4747d1e86e3fSQu Wenruo
4748d1e86e3fSQu Wenruo /*
4749d1e86e3fSQu Wenruo * The same as try_release_extent_buffer(), to ensure the eb
4750d1e86e3fSQu Wenruo * won't disappear out from under us.
4751d1e86e3fSQu Wenruo */
4752d1e86e3fSQu Wenruo spin_lock(&eb->refs_lock);
4753d1e86e3fSQu Wenruo if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4754d1e86e3fSQu Wenruo spin_unlock(&eb->refs_lock);
4755d1e86e3fSQu Wenruo spin_unlock(&fs_info->buffer_lock);
4756d1e86e3fSQu Wenruo break;
4757d1e86e3fSQu Wenruo }
4758d1e86e3fSQu Wenruo spin_unlock(&fs_info->buffer_lock);
4759d1e86e3fSQu Wenruo
4760d1e86e3fSQu Wenruo /*
4761d1e86e3fSQu Wenruo * If tree ref isn't set then we know the ref on this eb is a
4762d1e86e3fSQu Wenruo * real ref, so just return, this eb will likely be freed soon
4763d1e86e3fSQu Wenruo * anyway.
4764d1e86e3fSQu Wenruo */
4765d1e86e3fSQu Wenruo if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4766d1e86e3fSQu Wenruo spin_unlock(&eb->refs_lock);
4767d1e86e3fSQu Wenruo break;
4768d1e86e3fSQu Wenruo }
4769d1e86e3fSQu Wenruo
4770d1e86e3fSQu Wenruo /*
4771d1e86e3fSQu Wenruo * Here we don't care about the return value, we will always
4772d1e86e3fSQu Wenruo * check the page private at the end. And
4773d1e86e3fSQu Wenruo * release_extent_buffer() will release the refs_lock.
4774d1e86e3fSQu Wenruo */
4775d1e86e3fSQu Wenruo release_extent_buffer(eb);
4776d1e86e3fSQu Wenruo }
4777d1e86e3fSQu Wenruo /*
4778d1e86e3fSQu Wenruo * Finally to check if we have cleared page private, as if we have
4779d1e86e3fSQu Wenruo * released all ebs in the page, the page private should be cleared now.
4780d1e86e3fSQu Wenruo */
4781d1e86e3fSQu Wenruo spin_lock(&page->mapping->private_lock);
4782d1e86e3fSQu Wenruo if (!PagePrivate(page))
4783d1e86e3fSQu Wenruo ret = 1;
4784d1e86e3fSQu Wenruo else
4785d1e86e3fSQu Wenruo ret = 0;
4786d1e86e3fSQu Wenruo spin_unlock(&page->mapping->private_lock);
4787d1e86e3fSQu Wenruo return ret;
4788d1e86e3fSQu Wenruo
4789d1e86e3fSQu Wenruo }
4790d1e86e3fSQu Wenruo
try_release_extent_buffer(struct page * page)4791f7a52a40SDavid Sterba int try_release_extent_buffer(struct page *page)
479219fe0a8bSMiao Xie {
47936af118ceSChris Mason struct extent_buffer *eb;
4794897ca6e9SMiao Xie
4795fbca46ebSQu Wenruo if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
4796d1e86e3fSQu Wenruo return try_release_subpage_extent_buffer(page);
4797d1e86e3fSQu Wenruo
479819fe0a8bSMiao Xie /*
4799d1e86e3fSQu Wenruo * We need to make sure nobody is changing page->private, as we rely on
4800d1e86e3fSQu Wenruo * page->private as the pointer to extent buffer.
480119fe0a8bSMiao Xie */
48023083ee2eSJosef Bacik spin_lock(&page->mapping->private_lock);
48033083ee2eSJosef Bacik if (!PagePrivate(page)) {
48043083ee2eSJosef Bacik spin_unlock(&page->mapping->private_lock);
48053083ee2eSJosef Bacik return 1;
480619fe0a8bSMiao Xie }
480719fe0a8bSMiao Xie
48083083ee2eSJosef Bacik eb = (struct extent_buffer *)page->private;
48093083ee2eSJosef Bacik BUG_ON(!eb);
481019fe0a8bSMiao Xie
48113083ee2eSJosef Bacik /*
48123083ee2eSJosef Bacik * This is a little awful but should be ok, we need to make sure that
48133083ee2eSJosef Bacik * the eb doesn't disappear out from under us while we're looking at
48143083ee2eSJosef Bacik * this page.
48153083ee2eSJosef Bacik */
48163083ee2eSJosef Bacik spin_lock(&eb->refs_lock);
48170b32f4bbSJosef Bacik if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
48183083ee2eSJosef Bacik spin_unlock(&eb->refs_lock);
48193083ee2eSJosef Bacik spin_unlock(&page->mapping->private_lock);
48203083ee2eSJosef Bacik return 0;
48213083ee2eSJosef Bacik }
48223083ee2eSJosef Bacik spin_unlock(&page->mapping->private_lock);
48233083ee2eSJosef Bacik
48243083ee2eSJosef Bacik /*
48253083ee2eSJosef Bacik * If tree ref isn't set then we know the ref on this eb is a real ref,
48263083ee2eSJosef Bacik * so just return, this page will likely be freed soon anyway.
48273083ee2eSJosef Bacik */
48283083ee2eSJosef Bacik if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
48293083ee2eSJosef Bacik spin_unlock(&eb->refs_lock);
48303083ee2eSJosef Bacik return 0;
48313083ee2eSJosef Bacik }
48323083ee2eSJosef Bacik
4833f7a52a40SDavid Sterba return release_extent_buffer(eb);
48346af118ceSChris Mason }
4835bfb484d9SJosef Bacik
4836bfb484d9SJosef Bacik /*
4837bfb484d9SJosef Bacik * btrfs_readahead_tree_block - attempt to readahead a child block
4838bfb484d9SJosef Bacik * @fs_info: the fs_info
4839bfb484d9SJosef Bacik * @bytenr: bytenr to read
48403fbaf258SJosef Bacik * @owner_root: objectid of the root that owns this eb
4841bfb484d9SJosef Bacik * @gen: generation for the uptodate check, can be 0
48423fbaf258SJosef Bacik * @level: level for the eb
4843bfb484d9SJosef Bacik *
4844bfb484d9SJosef Bacik * Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a
4845bfb484d9SJosef Bacik * normal uptodate check of the eb, without checking the generation. If we have
4846bfb484d9SJosef Bacik * to read the block we will not block on anything.
4847bfb484d9SJosef Bacik */
btrfs_readahead_tree_block(struct btrfs_fs_info * fs_info,u64 bytenr,u64 owner_root,u64 gen,int level)4848bfb484d9SJosef Bacik void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
48493fbaf258SJosef Bacik u64 bytenr, u64 owner_root, u64 gen, int level)
4850bfb484d9SJosef Bacik {
4851947a6299SQu Wenruo struct btrfs_tree_parent_check check = {
4852947a6299SQu Wenruo .has_first_key = 0,
4853947a6299SQu Wenruo .level = level,
4854947a6299SQu Wenruo .transid = gen
4855947a6299SQu Wenruo };
4856bfb484d9SJosef Bacik struct extent_buffer *eb;
4857bfb484d9SJosef Bacik int ret;
4858bfb484d9SJosef Bacik
48593fbaf258SJosef Bacik eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4860bfb484d9SJosef Bacik if (IS_ERR(eb))
4861bfb484d9SJosef Bacik return;
4862bfb484d9SJosef Bacik
4863bfb484d9SJosef Bacik if (btrfs_buffer_uptodate(eb, gen, 1)) {
4864bfb484d9SJosef Bacik free_extent_buffer(eb);
4865bfb484d9SJosef Bacik return;
4866bfb484d9SJosef Bacik }
4867bfb484d9SJosef Bacik
4868947a6299SQu Wenruo ret = read_extent_buffer_pages(eb, WAIT_NONE, 0, &check);
4869bfb484d9SJosef Bacik if (ret < 0)
4870bfb484d9SJosef Bacik free_extent_buffer_stale(eb);
4871bfb484d9SJosef Bacik else
4872bfb484d9SJosef Bacik free_extent_buffer(eb);
4873bfb484d9SJosef Bacik }
4874bfb484d9SJosef Bacik
4875bfb484d9SJosef Bacik /*
4876bfb484d9SJosef Bacik * btrfs_readahead_node_child - readahead a node's child block
4877bfb484d9SJosef Bacik * @node: parent node we're reading from
4878bfb484d9SJosef Bacik * @slot: slot in the parent node for the child we want to read
4879bfb484d9SJosef Bacik *
4880bfb484d9SJosef Bacik * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4881bfb484d9SJosef Bacik * the slot in the node provided.
4882bfb484d9SJosef Bacik */
btrfs_readahead_node_child(struct extent_buffer * node,int slot)4883bfb484d9SJosef Bacik void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4884bfb484d9SJosef Bacik {
4885bfb484d9SJosef Bacik btrfs_readahead_tree_block(node->fs_info,
4886bfb484d9SJosef Bacik btrfs_node_blockptr(node, slot),
48873fbaf258SJosef Bacik btrfs_header_owner(node),
48883fbaf258SJosef Bacik btrfs_node_ptr_generation(node, slot),
48893fbaf258SJosef Bacik btrfs_header_level(node) - 1);
4890bfb484d9SJosef Bacik }
4891