46 static inline bool ifs_is_fully_uptodate(struct folio *folio,  in ifs_is_fully_uptodate()  argument
49 	struct inode *inode = folio->mapping->host;  in ifs_is_fully_uptodate()
51 	return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio));  in ifs_is_fully_uptodate()
60 static void ifs_set_range_uptodate(struct folio *folio,  in ifs_set_range_uptodate()  argument
63 	struct inode *inode = folio->mapping->host;  in ifs_set_range_uptodate()
71 	if (ifs_is_fully_uptodate(folio, ifs))  in ifs_set_range_uptodate()
72 		folio_mark_uptodate(folio);  in ifs_set_range_uptodate()
76 static void iomap_set_range_uptodate(struct folio *folio, size_t off,  in iomap_set_range_uptodate()  argument
79 	struct iomap_folio_state *ifs = folio->private;  in iomap_set_range_uptodate()
82 		ifs_set_range_uptodate(folio, ifs, off, len);  in iomap_set_range_uptodate()
84 		folio_mark_uptodate(folio);  in iomap_set_range_uptodate()
87 static inline bool ifs_block_is_dirty(struct folio *folio,  in ifs_block_is_dirty()  argument
90 	struct inode *inode = folio->mapping->host;  in ifs_block_is_dirty()
91 	unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);  in ifs_block_is_dirty()
96 static void ifs_clear_range_dirty(struct folio *folio,  in ifs_clear_range_dirty()  argument
99 	struct inode *inode = folio->mapping->host;  in ifs_clear_range_dirty()
100 	unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);  in ifs_clear_range_dirty()
111 static void iomap_clear_range_dirty(struct folio *folio, size_t off, size_t len)  in iomap_clear_range_dirty()  argument
113 	struct iomap_folio_state *ifs = folio->private;  in iomap_clear_range_dirty()
116 		ifs_clear_range_dirty(folio, ifs, off, len);  in iomap_clear_range_dirty()
119 static void ifs_set_range_dirty(struct folio *folio,  in ifs_set_range_dirty()  argument
122 	struct inode *inode = folio->mapping->host;  in ifs_set_range_dirty()
123 	unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);  in ifs_set_range_dirty()
134 static void iomap_set_range_dirty(struct folio *folio, size_t off, size_t len)  in iomap_set_range_dirty()  argument
136 	struct iomap_folio_state *ifs = folio->private;  in iomap_set_range_dirty()
139 		ifs_set_range_dirty(folio, ifs, off, len);  in iomap_set_range_dirty()
143 		struct folio *folio, unsigned int flags)  in ifs_alloc()  argument
145 	struct iomap_folio_state *ifs = folio->private;  in ifs_alloc()
146 	unsigned int nr_blocks = i_blocks_per_folio(inode, folio);  in ifs_alloc()
169 	if (folio_test_uptodate(folio))  in ifs_alloc()
171 	if (folio_test_dirty(folio))  in ifs_alloc()
173 	folio_attach_private(folio, ifs);  in ifs_alloc()
178 static void ifs_free(struct folio *folio)  in ifs_free()  argument
180 	struct iomap_folio_state *ifs = folio_detach_private(folio);  in ifs_free()
186 	WARN_ON_ONCE(ifs_is_fully_uptodate(folio, ifs) !=  in ifs_free()
187 			folio_test_uptodate(folio));  in ifs_free()
194 static void iomap_adjust_read_range(struct inode *inode, struct folio *folio,  in iomap_adjust_read_range()  argument
197 	struct iomap_folio_state *ifs = folio->private;  in iomap_adjust_read_range()
202 	size_t poff = offset_in_folio(folio, *pos);  in iomap_adjust_read_range()
203 	size_t plen = min_t(loff_t, folio_size(folio) - poff, length);  in iomap_adjust_read_range()
242 		unsigned end = offset_in_folio(folio, isize - 1) >> block_bits;  in iomap_adjust_read_range()
252 static void iomap_finish_folio_read(struct folio *folio, size_t offset,  in iomap_finish_folio_read()  argument
255 	struct iomap_folio_state *ifs = folio->private;  in iomap_finish_folio_read()
258 		folio_clear_uptodate(folio);  in iomap_finish_folio_read()
259 		folio_set_error(folio);  in iomap_finish_folio_read()
261 		iomap_set_range_uptodate(folio, offset, len);  in iomap_finish_folio_read()
265 		folio_unlock(folio);  in iomap_finish_folio_read()
274 		iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);  in iomap_read_end_io()
279 	struct folio		*cur_folio;
295 		struct folio *folio)  in iomap_read_inline_data()  argument
300 	size_t offset = offset_in_folio(folio, iomap->offset);  in iomap_read_inline_data()
303 	if (folio_test_uptodate(folio))  in iomap_read_inline_data()
314 		ifs_alloc(iter->inode, folio, iter->flags);  in iomap_read_inline_data()
316 	addr = kmap_local_folio(folio, offset);  in iomap_read_inline_data()
320 	iomap_set_range_uptodate(folio, offset, PAGE_SIZE - poff);  in iomap_read_inline_data()
340 	struct folio *folio = ctx->cur_folio;  in iomap_readpage_iter()  local
347 		return iomap_read_inline_data(iter, folio);  in iomap_readpage_iter()
350 	ifs = ifs_alloc(iter->inode, folio, iter->flags);  in iomap_readpage_iter()
351 	iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen);  in iomap_readpage_iter()
356 		folio_zero_range(folio, poff, plen);  in iomap_readpage_iter()
357 		iomap_set_range_uptodate(folio, poff, plen);  in iomap_readpage_iter()
368 	    !bio_add_folio(ctx->bio, folio, plen, poff)) {  in iomap_readpage_iter()
369 		gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);  in iomap_readpage_iter()
393 		bio_add_folio_nofail(ctx->bio, folio, plen, poff);  in iomap_readpage_iter()
406 int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)  in iomap_read_folio()  argument
409 		.inode		= folio->mapping->host,  in iomap_read_folio()
410 		.pos		= folio_pos(folio),  in iomap_read_folio()
411 		.len		= folio_size(folio),  in iomap_read_folio()
414 		.cur_folio	= folio,  in iomap_read_folio()
424 		folio_set_error(folio);  in iomap_read_folio()
431 		folio_unlock(folio);  in iomap_read_folio()
515 bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count)  in iomap_is_partially_uptodate()  argument
517 	struct iomap_folio_state *ifs = folio->private;  in iomap_is_partially_uptodate()
518 	struct inode *inode = folio->mapping->host;  in iomap_is_partially_uptodate()
525 	count = min(folio_size(folio) - from, count);  in iomap_is_partially_uptodate()
547 struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len)  in iomap_get_folio()
560 bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags)  in iomap_release_folio()  argument
562 	trace_iomap_release_folio(folio->mapping->host, folio_pos(folio),  in iomap_release_folio()
563 			folio_size(folio));  in iomap_release_folio()
570 	if (folio_test_dirty(folio))  in iomap_release_folio()
572 	ifs_free(folio);  in iomap_release_folio()
577 void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len)  in iomap_invalidate_folio()  argument
579 	trace_iomap_invalidate_folio(folio->mapping->host,  in iomap_invalidate_folio()
580 					folio_pos(folio) + offset, len);  in iomap_invalidate_folio()
586 	if (offset == 0 && len == folio_size(folio)) {  in iomap_invalidate_folio()
587 		WARN_ON_ONCE(folio_test_writeback(folio));  in iomap_invalidate_folio()
588 		folio_cancel_dirty(folio);  in iomap_invalidate_folio()
589 		ifs_free(folio);  in iomap_invalidate_folio()
594 bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio)  in iomap_dirty_folio()  argument
597 	size_t len = folio_size(folio);  in iomap_dirty_folio()
599 	ifs_alloc(inode, folio, 0);  in iomap_dirty_folio()
600 	iomap_set_range_dirty(folio, 0, len);  in iomap_dirty_folio()
601 	return filemap_dirty_folio(mapping, folio);  in iomap_dirty_folio()
619 static int iomap_read_folio_sync(loff_t block_start, struct folio *folio,  in iomap_read_folio_sync()  argument
627 	bio_add_folio_nofail(&bio, folio, plen, poff);  in iomap_read_folio_sync()
632 		size_t len, struct folio *folio)  in __iomap_write_begin()  argument
639 	unsigned int nr_blocks = i_blocks_per_folio(iter->inode, folio);  in __iomap_write_begin()
640 	size_t from = offset_in_folio(folio, pos), to = from + len;  in __iomap_write_begin()
650 	if (!(iter->flags & IOMAP_UNSHARE) && pos <= folio_pos(folio) &&  in __iomap_write_begin()
651 	    pos + len >= folio_pos(folio) + folio_size(folio))  in __iomap_write_begin()
654 	ifs = ifs_alloc(iter->inode, folio, iter->flags);  in __iomap_write_begin()
658 	if (folio_test_uptodate(folio))  in __iomap_write_begin()
660 	folio_clear_error(folio);  in __iomap_write_begin()
663 		iomap_adjust_read_range(iter->inode, folio, &block_start,  in __iomap_write_begin()
676 			folio_zero_segments(folio, poff, from, to, poff + plen);  in __iomap_write_begin()
683 			status = iomap_read_folio_sync(block_start, folio,  in __iomap_write_begin()
688 		iomap_set_range_uptodate(folio, poff, plen);  in __iomap_write_begin()
694 static struct folio *__iomap_get_folio(struct iomap_iter *iter, loff_t pos,  in __iomap_get_folio()
706 		struct folio *folio)  in __iomap_put_folio()  argument
711 		folio_ops->put_folio(iter->inode, pos, ret, folio);  in __iomap_put_folio()
713 		folio_unlock(folio);  in __iomap_put_folio()
714 		folio_put(folio);  in __iomap_put_folio()
719 		struct folio *folio)  in iomap_write_begin_inline()  argument
724 	return iomap_read_inline_data(iter, folio);  in iomap_write_begin_inline()
728 		size_t len, struct folio **foliop)  in iomap_write_begin()
732 	struct folio *folio;  in iomap_write_begin()  local
745 	folio = __iomap_get_folio(iter, pos, len);  in iomap_write_begin()
746 	if (IS_ERR(folio))  in iomap_write_begin()
747 		return PTR_ERR(folio);  in iomap_write_begin()
769 	if (pos + len > folio_pos(folio) + folio_size(folio))  in iomap_write_begin()
770 		len = folio_pos(folio) + folio_size(folio) - pos;  in iomap_write_begin()
773 		status = iomap_write_begin_inline(iter, folio);  in iomap_write_begin()
775 		status = __block_write_begin_int(folio, pos, len, NULL, srcmap);  in iomap_write_begin()
777 		status = __iomap_write_begin(iter, pos, len, folio);  in iomap_write_begin()
782 	*foliop = folio;  in iomap_write_begin()
786 	__iomap_put_folio(iter, pos, 0, folio);  in iomap_write_begin()
793 		size_t copied, struct folio *folio)  in __iomap_write_end()  argument
795 	flush_dcache_folio(folio);  in __iomap_write_end()
808 	if (unlikely(copied < len && !folio_test_uptodate(folio)))  in __iomap_write_end()
810 	iomap_set_range_uptodate(folio, offset_in_folio(folio, pos), len);  in __iomap_write_end()
811 	iomap_set_range_dirty(folio, offset_in_folio(folio, pos), copied);  in __iomap_write_end()
812 	filemap_dirty_folio(inode->i_mapping, folio);  in __iomap_write_end()
817 		struct folio *folio, loff_t pos, size_t copied)  in iomap_write_end_inline()  argument
822 	WARN_ON_ONCE(!folio_test_uptodate(folio));  in iomap_write_end_inline()
825 	flush_dcache_folio(folio);  in iomap_write_end_inline()
826 	addr = kmap_local_folio(folio, pos);  in iomap_write_end_inline()
836 		size_t copied, struct folio *folio)  in iomap_write_end()  argument
843 		ret = iomap_write_end_inline(iter, folio, pos, copied);  in iomap_write_end()
846 				copied, &folio->page, NULL);  in iomap_write_end()
848 		ret = __iomap_write_end(iter->inode, pos, len, copied, folio);  in iomap_write_end()
860 	__iomap_put_folio(iter, pos, ret, folio);  in iomap_write_end()
880 		struct folio *folio;  in iomap_write_iter()  local
912 		status = iomap_write_begin(iter, pos, bytes, &folio);  in iomap_write_iter()
918 		offset = offset_in_folio(folio, pos);  in iomap_write_iter()
919 		if (bytes > folio_size(folio) - offset)  in iomap_write_iter()
920 			bytes = folio_size(folio) - offset;  in iomap_write_iter()
923 			flush_dcache_folio(folio);  in iomap_write_iter()
925 		copied = copy_folio_from_iter_atomic(folio, offset, bytes, i);  in iomap_write_iter()
926 		status = iomap_write_end(iter, pos, bytes, copied, folio);  in iomap_write_iter()
986 		struct folio *folio, loff_t start_byte, loff_t end_byte,  in iomap_write_delalloc_ifs_punch()  argument
1001 	ifs = folio->private;  in iomap_write_delalloc_ifs_punch()
1006 			folio_pos(folio) + folio_size(folio) - 1);  in iomap_write_delalloc_ifs_punch()
1007 	first_blk = offset_in_folio(folio, start_byte) >> blkbits;  in iomap_write_delalloc_ifs_punch()
1008 	last_blk = offset_in_folio(folio, last_byte) >> blkbits;  in iomap_write_delalloc_ifs_punch()
1010 		if (!ifs_block_is_dirty(folio, ifs, i)) {  in iomap_write_delalloc_ifs_punch()
1011 			ret = punch(inode, folio_pos(folio) + (i << blkbits),  in iomap_write_delalloc_ifs_punch()
1022 static int iomap_write_delalloc_punch(struct inode *inode, struct folio *folio,  in iomap_write_delalloc_punch()  argument
1028 	if (!folio_test_dirty(folio))  in iomap_write_delalloc_punch()
1040 	ret = iomap_write_delalloc_ifs_punch(inode, folio, start_byte,  in iomap_write_delalloc_punch()
1050 				folio_pos(folio) + folio_size(folio));  in iomap_write_delalloc_punch()
1077 		struct folio	*folio;  in iomap_write_delalloc_scan()  local
1081 		folio = filemap_lock_folio(inode->i_mapping,  in iomap_write_delalloc_scan()
1083 		if (IS_ERR(folio)) {  in iomap_write_delalloc_scan()
1089 		ret = iomap_write_delalloc_punch(inode, folio, punch_start_byte,  in iomap_write_delalloc_scan()
1092 			folio_unlock(folio);  in iomap_write_delalloc_scan()
1093 			folio_put(folio);  in iomap_write_delalloc_scan()
1098 		start_byte = folio_next_index(folio) << PAGE_SHIFT;  in iomap_write_delalloc_scan()
1099 		folio_unlock(folio);  in iomap_write_delalloc_scan()
1100 		folio_put(folio);  in iomap_write_delalloc_scan()
1289 		struct folio *folio;  in iomap_unshare_iter()  local
1294 		status = iomap_write_begin(iter, pos, bytes, &folio);  in iomap_unshare_iter()
1300 		offset = offset_in_folio(folio, pos);  in iomap_unshare_iter()
1301 		if (bytes > folio_size(folio) - offset)  in iomap_unshare_iter()
1302 			bytes = folio_size(folio) - offset;  in iomap_unshare_iter()
1304 		bytes = iomap_write_end(iter, pos, bytes, bytes, folio);  in iomap_unshare_iter()
1354 		struct folio *folio;  in iomap_zero_iter()  local
1359 		status = iomap_write_begin(iter, pos, bytes, &folio);  in iomap_zero_iter()
1365 		offset = offset_in_folio(folio, pos);  in iomap_zero_iter()
1366 		if (bytes > folio_size(folio) - offset)  in iomap_zero_iter()
1367 			bytes = folio_size(folio) - offset;  in iomap_zero_iter()
1369 		folio_zero_range(folio, offset, bytes);  in iomap_zero_iter()
1370 		folio_mark_accessed(folio);  in iomap_zero_iter()
1372 		bytes = iomap_write_end(iter, pos, bytes, bytes, folio);  in iomap_zero_iter()
1419 		struct folio *folio)  in iomap_folio_mkwrite_iter()  argument
1425 		ret = __block_write_begin_int(folio, iter->pos, length, NULL,  in iomap_folio_mkwrite_iter()
1429 		block_commit_write(&folio->page, 0, length);  in iomap_folio_mkwrite_iter()
1431 		WARN_ON_ONCE(!folio_test_uptodate(folio));  in iomap_folio_mkwrite_iter()
1432 		folio_mark_dirty(folio);  in iomap_folio_mkwrite_iter()
1444 	struct folio *folio = page_folio(vmf->page);  in iomap_page_mkwrite()  local
1447 	folio_lock(folio);  in iomap_page_mkwrite()
1448 	ret = folio_mkwrite_check_truncate(folio, iter.inode);  in iomap_page_mkwrite()
1451 	iter.pos = folio_pos(folio);  in iomap_page_mkwrite()
1454 		iter.processed = iomap_folio_mkwrite_iter(&iter, folio);  in iomap_page_mkwrite()
1458 	folio_wait_stable(folio);  in iomap_page_mkwrite()
1461 	folio_unlock(folio);  in iomap_page_mkwrite()
1466 static void iomap_finish_folio_write(struct inode *inode, struct folio *folio,  in iomap_finish_folio_write()  argument
1469 	struct iomap_folio_state *ifs = folio->private;  in iomap_finish_folio_write()
1472 		folio_set_error(folio);  in iomap_finish_folio_write()
1476 	WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs);  in iomap_finish_folio_write()
1480 		folio_end_writeback(folio);  in iomap_finish_folio_write()
1513 			iomap_finish_folio_write(inode, fi.folio, fi.length,  in iomap_finish_ioend()
1745 iomap_add_to_ioend(struct inode *inode, loff_t pos, struct folio *folio,  in iomap_add_to_ioend()  argument
1751 	size_t poff = offset_in_folio(folio, pos);  in iomap_add_to_ioend()
1759 	if (!bio_add_folio(wpc->ioend->io_bio, folio, len, poff)) {  in iomap_add_to_ioend()
1761 		bio_add_folio_nofail(wpc->ioend->io_bio, folio, len, poff);  in iomap_add_to_ioend()
1767 	wbc_account_cgroup_owner(wbc, &folio->page, len);  in iomap_add_to_ioend()
1789 		struct folio *folio, u64 end_pos)  in iomap_writepage_map()  argument
1791 	struct iomap_folio_state *ifs = folio->private;  in iomap_writepage_map()
1794 	unsigned nblocks = i_blocks_per_folio(inode, folio);  in iomap_writepage_map()
1795 	u64 pos = folio_pos(folio);  in iomap_writepage_map()
1802 		ifs = ifs_alloc(inode, folio, 0);  in iomap_writepage_map()
1803 		iomap_set_range_dirty(folio, 0, end_pos - pos);  in iomap_writepage_map()
1814 		if (ifs && !ifs_block_is_dirty(folio, ifs, i))  in iomap_writepage_map()
1825 		iomap_add_to_ioend(inode, pos, folio, ifs, wpc, wbc,  in iomap_writepage_map()
1833 	WARN_ON_ONCE(!folio_test_locked(folio));  in iomap_writepage_map()
1834 	WARN_ON_ONCE(folio_test_writeback(folio));  in iomap_writepage_map()
1835 	WARN_ON_ONCE(folio_test_dirty(folio));  in iomap_writepage_map()
1849 			wpc->ops->discard_folio(folio, pos);  in iomap_writepage_map()
1857 	iomap_clear_range_dirty(folio, 0, folio_size(folio));  in iomap_writepage_map()
1864 		folio_unlock(folio);  in iomap_writepage_map()
1868 	folio_start_writeback(folio);  in iomap_writepage_map()
1869 	folio_unlock(folio);  in iomap_writepage_map()
1890 		folio_end_writeback(folio);  in iomap_writepage_map()
1903 static int iomap_do_writepage(struct folio *folio,  in iomap_do_writepage()  argument
1907 	struct inode *inode = folio->mapping->host;  in iomap_do_writepage()
1910 	trace_iomap_writepage(inode, folio_pos(folio), folio_size(folio));  in iomap_do_writepage()
1940 	end_pos = folio_pos(folio) + folio_size(folio);  in iomap_do_writepage()
1953 		size_t poff = offset_in_folio(folio, isize);  in iomap_do_writepage()
1973 		if (folio->index > end_index ||  in iomap_do_writepage()
1974 		    (folio->index == end_index && poff == 0))  in iomap_do_writepage()
1985 		folio_zero_segment(folio, poff, folio_size(folio));  in iomap_do_writepage()
1989 	return iomap_writepage_map(wpc, wbc, inode, folio, end_pos);  in iomap_do_writepage()
1992 	folio_redirty_for_writepage(wbc, folio);  in iomap_do_writepage()
1994 	folio_unlock(folio);  in iomap_do_writepage()