47 		struct iomap_folio_state *ifs)  in ifs_is_fully_uptodate()  argument
51 	return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio));  in ifs_is_fully_uptodate()
54 static inline bool ifs_block_is_uptodate(struct iomap_folio_state *ifs,  in ifs_block_is_uptodate()  argument
57 	return test_bit(block, ifs->state);  in ifs_block_is_uptodate()
61 		struct iomap_folio_state *ifs, size_t off, size_t len)  in ifs_set_range_uptodate()  argument
69 	spin_lock_irqsave(&ifs->state_lock, flags);  in ifs_set_range_uptodate()
70 	bitmap_set(ifs->state, first_blk, nr_blks);  in ifs_set_range_uptodate()
71 	if (ifs_is_fully_uptodate(folio, ifs))  in ifs_set_range_uptodate()
73 	spin_unlock_irqrestore(&ifs->state_lock, flags);  in ifs_set_range_uptodate()
79 	struct iomap_folio_state *ifs = folio->private;  in iomap_set_range_uptodate()  local
81 	if (ifs)  in iomap_set_range_uptodate()
82 		ifs_set_range_uptodate(folio, ifs, off, len);  in iomap_set_range_uptodate()
88 		struct iomap_folio_state *ifs, int block)  in ifs_block_is_dirty()  argument
93 	return test_bit(block + blks_per_folio, ifs->state);  in ifs_block_is_dirty()
97 		struct iomap_folio_state *ifs, size_t off, size_t len)  in ifs_clear_range_dirty()  argument
106 	spin_lock_irqsave(&ifs->state_lock, flags);  in ifs_clear_range_dirty()
107 	bitmap_clear(ifs->state, first_blk + blks_per_folio, nr_blks);  in ifs_clear_range_dirty()
108 	spin_unlock_irqrestore(&ifs->state_lock, flags);  in ifs_clear_range_dirty()
113 	struct iomap_folio_state *ifs = folio->private;  in iomap_clear_range_dirty()  local
115 	if (ifs)  in iomap_clear_range_dirty()
116 		ifs_clear_range_dirty(folio, ifs, off, len);  in iomap_clear_range_dirty()
120 		struct iomap_folio_state *ifs, size_t off, size_t len)  in ifs_set_range_dirty()  argument
129 	spin_lock_irqsave(&ifs->state_lock, flags);  in ifs_set_range_dirty()
130 	bitmap_set(ifs->state, first_blk + blks_per_folio, nr_blks);  in ifs_set_range_dirty()
131 	spin_unlock_irqrestore(&ifs->state_lock, flags);  in ifs_set_range_dirty()
136 	struct iomap_folio_state *ifs = folio->private;  in iomap_set_range_dirty()  local
138 	if (ifs)  in iomap_set_range_dirty()
139 		ifs_set_range_dirty(folio, ifs, off, len);  in iomap_set_range_dirty()
145 	struct iomap_folio_state *ifs = folio->private;  in ifs_alloc()  local
149 	if (ifs || nr_blocks <= 1)  in ifs_alloc()
150 		return ifs;  in ifs_alloc()
163 	ifs = kzalloc(struct_size(ifs, state,  in ifs_alloc()
165 	if (!ifs)  in ifs_alloc()
166 		return ifs;  in ifs_alloc()
168 	spin_lock_init(&ifs->state_lock);  in ifs_alloc()
170 		bitmap_set(ifs->state, 0, nr_blocks);  in ifs_alloc()
172 		bitmap_set(ifs->state, nr_blocks, nr_blocks);  in ifs_alloc()
173 	folio_attach_private(folio, ifs);  in ifs_alloc()
175 	return ifs;  in ifs_alloc()
180 	struct iomap_folio_state *ifs = folio_detach_private(folio);  in ifs_free()  local
182 	if (!ifs)  in ifs_free()
184 	WARN_ON_ONCE(atomic_read(&ifs->read_bytes_pending));  in ifs_free()
185 	WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending));  in ifs_free()
186 	WARN_ON_ONCE(ifs_is_fully_uptodate(folio, ifs) !=  in ifs_free()
188 	kfree(ifs);  in ifs_free()
197 	struct iomap_folio_state *ifs = folio->private;  in iomap_adjust_read_range()  local
213 	if (ifs) {  in iomap_adjust_read_range()
218 			if (!ifs_block_is_uptodate(ifs, i))  in iomap_adjust_read_range()
228 			if (ifs_block_is_uptodate(ifs, i)) {  in iomap_adjust_read_range()
255 	struct iomap_folio_state *ifs = folio->private;  in iomap_finish_folio_read()  local
264 	if (!ifs || atomic_sub_and_test(len, &ifs->read_bytes_pending))  in iomap_finish_folio_read()
341 	struct iomap_folio_state *ifs;  in iomap_readpage_iter()  local
350 	ifs = ifs_alloc(iter->inode, folio, iter->flags);  in iomap_readpage_iter()
362 	if (ifs)  in iomap_readpage_iter()
363 		atomic_add(plen, &ifs->read_bytes_pending);  in iomap_readpage_iter()
517 	struct iomap_folio_state *ifs = folio->private;  in iomap_is_partially_uptodate()  local
521 	if (!ifs)  in iomap_is_partially_uptodate()
532 		if (!ifs_block_is_uptodate(ifs, i))  in iomap_is_partially_uptodate()
635 	struct iomap_folio_state *ifs;  in __iomap_write_begin()  local
654 	ifs = ifs_alloc(iter->inode, folio, iter->flags);  in __iomap_write_begin()
655 	if ((iter->flags & IOMAP_NOWAIT) && !ifs && nr_blocks > 1)  in __iomap_write_begin()
992 	struct iomap_folio_state *ifs;  in iomap_write_delalloc_ifs_punch()  local
1001 	ifs = folio->private;  in iomap_write_delalloc_ifs_punch()
1002 	if (!ifs)  in iomap_write_delalloc_ifs_punch()
1010 		if (!ifs_block_is_dirty(folio, ifs, i)) {  in iomap_write_delalloc_ifs_punch()
1464 	struct iomap_folio_state *ifs = folio->private;  in iomap_finish_folio_write()  local
1471 	WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs);  in iomap_finish_folio_write()
1472 	WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) <= 0);  in iomap_finish_folio_write()
1474 	if (!ifs || atomic_sub_and_test(len, &ifs->write_bytes_pending))  in iomap_finish_folio_write()
1741 		struct iomap_folio_state *ifs, struct iomap_writepage_ctx *wpc,  in iomap_add_to_ioend()  argument
1759 	if (ifs)  in iomap_add_to_ioend()
1760 		atomic_add(len, &ifs->write_bytes_pending);  in iomap_add_to_ioend()
1786 	struct iomap_folio_state *ifs = folio->private;  in iomap_writepage_map()  local
1796 	if (!ifs && nblocks > 1) {  in iomap_writepage_map()
1797 		ifs = ifs_alloc(inode, folio, 0);  in iomap_writepage_map()
1801 	WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) != 0);  in iomap_writepage_map()
1809 		if (ifs && !ifs_block_is_dirty(folio, ifs, i))  in iomap_writepage_map()
1820 		iomap_add_to_ioend(inode, pos, folio, ifs, wpc, wbc,  in iomap_writepage_map()