1afc51aaaSDarrick J. Wong // SPDX-License-Identifier: GPL-2.0 2afc51aaaSDarrick J. Wong /* 3afc51aaaSDarrick J. Wong * Copyright (C) 2010 Red Hat, Inc. 4598ecfbaSChristoph Hellwig * Copyright (C) 2016-2019 Christoph Hellwig. 5afc51aaaSDarrick J. Wong */ 6afc51aaaSDarrick J. Wong #include <linux/module.h> 7afc51aaaSDarrick J. Wong #include <linux/compiler.h> 8afc51aaaSDarrick J. Wong #include <linux/fs.h> 9afc51aaaSDarrick J. Wong #include <linux/iomap.h> 10afc51aaaSDarrick J. Wong #include <linux/pagemap.h> 11afc51aaaSDarrick J. Wong #include <linux/uio.h> 12afc51aaaSDarrick J. Wong #include <linux/buffer_head.h> 13afc51aaaSDarrick J. Wong #include <linux/dax.h> 14afc51aaaSDarrick J. Wong #include <linux/writeback.h> 15598ecfbaSChristoph Hellwig #include <linux/list_sort.h> 16afc51aaaSDarrick J. Wong #include <linux/swap.h> 17afc51aaaSDarrick J. Wong #include <linux/bio.h> 18afc51aaaSDarrick J. Wong #include <linux/sched/signal.h> 19afc51aaaSDarrick J. Wong #include <linux/migrate.h> 209e91c572SChristoph Hellwig #include "trace.h" 21afc51aaaSDarrick J. Wong 22afc51aaaSDarrick J. Wong #include "../internal.h" 23afc51aaaSDarrick J. Wong 24ebb7fb15SDave Chinner #define IOEND_BATCH_SIZE 4096 25ebb7fb15SDave Chinner 26ab08b01eSChristoph Hellwig /* 2795c4cd05SMatthew Wilcox (Oracle) * Structure allocated for each folio when block size < folio size 2895c4cd05SMatthew Wilcox (Oracle) * to track sub-folio uptodate status and I/O completions. 29ab08b01eSChristoph Hellwig */ 30ab08b01eSChristoph Hellwig struct iomap_page { 317d636676SMatthew Wilcox (Oracle) atomic_t read_bytes_pending; 320fb2d720SMatthew Wilcox (Oracle) atomic_t write_bytes_pending; 331cea335dSChristoph Hellwig spinlock_t uptodate_lock; 340a195b91SMatthew Wilcox (Oracle) unsigned long uptodate[]; 35ab08b01eSChristoph Hellwig }; 36ab08b01eSChristoph Hellwig 3795c4cd05SMatthew Wilcox (Oracle) static inline struct iomap_page *to_iomap_page(struct folio *folio) 38ab08b01eSChristoph Hellwig { 3995c4cd05SMatthew Wilcox (Oracle) if (folio_test_private(folio)) 4095c4cd05SMatthew Wilcox (Oracle) return folio_get_private(folio); 41ab08b01eSChristoph Hellwig return NULL; 42ab08b01eSChristoph Hellwig } 43ab08b01eSChristoph Hellwig 44598ecfbaSChristoph Hellwig static struct bio_set iomap_ioend_bioset; 45598ecfbaSChristoph Hellwig 46afc51aaaSDarrick J. Wong static struct iomap_page * 47435d44b3SMatthew Wilcox (Oracle) iomap_page_create(struct inode *inode, struct folio *folio) 48afc51aaaSDarrick J. Wong { 4995c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 50435d44b3SMatthew Wilcox (Oracle) unsigned int nr_blocks = i_blocks_per_folio(inode, folio); 51afc51aaaSDarrick J. Wong 520a195b91SMatthew Wilcox (Oracle) if (iop || nr_blocks <= 1) 53afc51aaaSDarrick J. Wong return iop; 54afc51aaaSDarrick J. Wong 550a195b91SMatthew Wilcox (Oracle) iop = kzalloc(struct_size(iop, uptodate, BITS_TO_LONGS(nr_blocks)), 560a195b91SMatthew Wilcox (Oracle) GFP_NOFS | __GFP_NOFAIL); 571cea335dSChristoph Hellwig spin_lock_init(&iop->uptodate_lock); 58435d44b3SMatthew Wilcox (Oracle) if (folio_test_uptodate(folio)) 594595a298SMatthew Wilcox (Oracle) bitmap_fill(iop->uptodate, nr_blocks); 60435d44b3SMatthew Wilcox (Oracle) folio_attach_private(folio, iop); 61afc51aaaSDarrick J. Wong return iop; 62afc51aaaSDarrick J. Wong } 63afc51aaaSDarrick J. Wong 64c46e8324SMatthew Wilcox (Oracle) static void iomap_page_release(struct folio *folio) 65afc51aaaSDarrick J. Wong { 66c46e8324SMatthew Wilcox (Oracle) struct iomap_page *iop = folio_detach_private(folio); 67c46e8324SMatthew Wilcox (Oracle) struct inode *inode = folio->mapping->host; 68c46e8324SMatthew Wilcox (Oracle) unsigned int nr_blocks = i_blocks_per_folio(inode, folio); 69afc51aaaSDarrick J. Wong 70afc51aaaSDarrick J. Wong if (!iop) 71afc51aaaSDarrick J. Wong return; 727d636676SMatthew Wilcox (Oracle) WARN_ON_ONCE(atomic_read(&iop->read_bytes_pending)); 730fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(atomic_read(&iop->write_bytes_pending)); 740a195b91SMatthew Wilcox (Oracle) WARN_ON_ONCE(bitmap_full(iop->uptodate, nr_blocks) != 75c46e8324SMatthew Wilcox (Oracle) folio_test_uptodate(folio)); 76afc51aaaSDarrick J. Wong kfree(iop); 77afc51aaaSDarrick J. Wong } 78afc51aaaSDarrick J. Wong 79afc51aaaSDarrick J. Wong /* 80431c0566SMatthew Wilcox (Oracle) * Calculate the range inside the folio that we actually need to read. 81afc51aaaSDarrick J. Wong */ 82431c0566SMatthew Wilcox (Oracle) static void iomap_adjust_read_range(struct inode *inode, struct folio *folio, 83431c0566SMatthew Wilcox (Oracle) loff_t *pos, loff_t length, size_t *offp, size_t *lenp) 84afc51aaaSDarrick J. Wong { 85431c0566SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 86afc51aaaSDarrick J. Wong loff_t orig_pos = *pos; 87afc51aaaSDarrick J. Wong loff_t isize = i_size_read(inode); 88afc51aaaSDarrick J. Wong unsigned block_bits = inode->i_blkbits; 89afc51aaaSDarrick J. Wong unsigned block_size = (1 << block_bits); 90431c0566SMatthew Wilcox (Oracle) size_t poff = offset_in_folio(folio, *pos); 91431c0566SMatthew Wilcox (Oracle) size_t plen = min_t(loff_t, folio_size(folio) - poff, length); 92afc51aaaSDarrick J. Wong unsigned first = poff >> block_bits; 93afc51aaaSDarrick J. Wong unsigned last = (poff + plen - 1) >> block_bits; 94afc51aaaSDarrick J. Wong 95afc51aaaSDarrick J. Wong /* 96f1f264b4SAndreas Gruenbacher * If the block size is smaller than the page size, we need to check the 97afc51aaaSDarrick J. Wong * per-block uptodate status and adjust the offset and length if needed 98afc51aaaSDarrick J. Wong * to avoid reading in already uptodate ranges. 99afc51aaaSDarrick J. Wong */ 100afc51aaaSDarrick J. Wong if (iop) { 101afc51aaaSDarrick J. Wong unsigned int i; 102afc51aaaSDarrick J. Wong 103afc51aaaSDarrick J. Wong /* move forward for each leading block marked uptodate */ 104afc51aaaSDarrick J. Wong for (i = first; i <= last; i++) { 105afc51aaaSDarrick J. Wong if (!test_bit(i, iop->uptodate)) 106afc51aaaSDarrick J. Wong break; 107afc51aaaSDarrick J. Wong *pos += block_size; 108afc51aaaSDarrick J. Wong poff += block_size; 109afc51aaaSDarrick J. Wong plen -= block_size; 110afc51aaaSDarrick J. Wong first++; 111afc51aaaSDarrick J. Wong } 112afc51aaaSDarrick J. Wong 113afc51aaaSDarrick J. Wong /* truncate len if we find any trailing uptodate block(s) */ 114afc51aaaSDarrick J. Wong for ( ; i <= last; i++) { 115afc51aaaSDarrick J. Wong if (test_bit(i, iop->uptodate)) { 116afc51aaaSDarrick J. Wong plen -= (last - i + 1) * block_size; 117afc51aaaSDarrick J. Wong last = i - 1; 118afc51aaaSDarrick J. Wong break; 119afc51aaaSDarrick J. Wong } 120afc51aaaSDarrick J. Wong } 121afc51aaaSDarrick J. Wong } 122afc51aaaSDarrick J. Wong 123afc51aaaSDarrick J. Wong /* 124f1f264b4SAndreas Gruenbacher * If the extent spans the block that contains the i_size, we need to 125afc51aaaSDarrick J. Wong * handle both halves separately so that we properly zero data in the 126afc51aaaSDarrick J. Wong * page cache for blocks that are entirely outside of i_size. 127afc51aaaSDarrick J. Wong */ 128afc51aaaSDarrick J. Wong if (orig_pos <= isize && orig_pos + length > isize) { 129431c0566SMatthew Wilcox (Oracle) unsigned end = offset_in_folio(folio, isize - 1) >> block_bits; 130afc51aaaSDarrick J. Wong 131afc51aaaSDarrick J. Wong if (first <= end && last > end) 132afc51aaaSDarrick J. Wong plen -= (last - end) * block_size; 133afc51aaaSDarrick J. Wong } 134afc51aaaSDarrick J. Wong 135afc51aaaSDarrick J. Wong *offp = poff; 136afc51aaaSDarrick J. Wong *lenp = plen; 137afc51aaaSDarrick J. Wong } 138afc51aaaSDarrick J. Wong 139431c0566SMatthew Wilcox (Oracle) static void iomap_iop_set_range_uptodate(struct folio *folio, 140431c0566SMatthew Wilcox (Oracle) struct iomap_page *iop, size_t off, size_t len) 141afc51aaaSDarrick J. Wong { 142431c0566SMatthew Wilcox (Oracle) struct inode *inode = folio->mapping->host; 143afc51aaaSDarrick J. Wong unsigned first = off >> inode->i_blkbits; 144afc51aaaSDarrick J. Wong unsigned last = (off + len - 1) >> inode->i_blkbits; 1451cea335dSChristoph Hellwig unsigned long flags; 146afc51aaaSDarrick J. Wong 1471cea335dSChristoph Hellwig spin_lock_irqsave(&iop->uptodate_lock, flags); 148b21866f5SMatthew Wilcox (Oracle) bitmap_set(iop->uptodate, first, last - first + 1); 149431c0566SMatthew Wilcox (Oracle) if (bitmap_full(iop->uptodate, i_blocks_per_folio(inode, folio))) 150431c0566SMatthew Wilcox (Oracle) folio_mark_uptodate(folio); 1511cea335dSChristoph Hellwig spin_unlock_irqrestore(&iop->uptodate_lock, flags); 152afc51aaaSDarrick J. Wong } 153afc51aaaSDarrick J. Wong 154431c0566SMatthew Wilcox (Oracle) static void iomap_set_range_uptodate(struct folio *folio, 155431c0566SMatthew Wilcox (Oracle) struct iomap_page *iop, size_t off, size_t len) 1561cea335dSChristoph Hellwig { 157431c0566SMatthew Wilcox (Oracle) if (folio_test_error(folio)) 1581cea335dSChristoph Hellwig return; 1591cea335dSChristoph Hellwig 160cd1e5afeSMatthew Wilcox (Oracle) if (iop) 161431c0566SMatthew Wilcox (Oracle) iomap_iop_set_range_uptodate(folio, iop, off, len); 1621cea335dSChristoph Hellwig else 163431c0566SMatthew Wilcox (Oracle) folio_mark_uptodate(folio); 164afc51aaaSDarrick J. Wong } 165afc51aaaSDarrick J. Wong 1668ffd74e9SMatthew Wilcox (Oracle) static void iomap_finish_folio_read(struct folio *folio, size_t offset, 1678ffd74e9SMatthew Wilcox (Oracle) size_t len, int error) 168afc51aaaSDarrick J. Wong { 16995c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 170afc51aaaSDarrick J. Wong 171afc51aaaSDarrick J. Wong if (unlikely(error)) { 1728ffd74e9SMatthew Wilcox (Oracle) folio_clear_uptodate(folio); 1738ffd74e9SMatthew Wilcox (Oracle) folio_set_error(folio); 174afc51aaaSDarrick J. Wong } else { 175431c0566SMatthew Wilcox (Oracle) iomap_set_range_uptodate(folio, iop, offset, len); 176afc51aaaSDarrick J. Wong } 177afc51aaaSDarrick J. Wong 1788ffd74e9SMatthew Wilcox (Oracle) if (!iop || atomic_sub_and_test(len, &iop->read_bytes_pending)) 1798ffd74e9SMatthew Wilcox (Oracle) folio_unlock(folio); 180afc51aaaSDarrick J. Wong } 181afc51aaaSDarrick J. Wong 1828ffd74e9SMatthew Wilcox (Oracle) static void iomap_read_end_io(struct bio *bio) 183afc51aaaSDarrick J. Wong { 184afc51aaaSDarrick J. Wong int error = blk_status_to_errno(bio->bi_status); 1858ffd74e9SMatthew Wilcox (Oracle) struct folio_iter fi; 186afc51aaaSDarrick J. Wong 1878ffd74e9SMatthew Wilcox (Oracle) bio_for_each_folio_all(fi, bio) 1888ffd74e9SMatthew Wilcox (Oracle) iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error); 189afc51aaaSDarrick J. Wong bio_put(bio); 190afc51aaaSDarrick J. Wong } 191afc51aaaSDarrick J. Wong 192afc51aaaSDarrick J. Wong struct iomap_readpage_ctx { 1933aa9c659SMatthew Wilcox (Oracle) struct folio *cur_folio; 1943aa9c659SMatthew Wilcox (Oracle) bool cur_folio_in_bio; 195afc51aaaSDarrick J. Wong struct bio *bio; 1969d24a13aSMatthew Wilcox (Oracle) struct readahead_control *rac; 197afc51aaaSDarrick J. Wong }; 198afc51aaaSDarrick J. Wong 1995ad448ceSAndreas Gruenbacher /** 2005ad448ceSAndreas Gruenbacher * iomap_read_inline_data - copy inline data into the page cache 2015ad448ceSAndreas Gruenbacher * @iter: iteration structure 202874628a2SMatthew Wilcox (Oracle) * @folio: folio to copy to 2035ad448ceSAndreas Gruenbacher * 204874628a2SMatthew Wilcox (Oracle) * Copy the inline data in @iter into @folio and zero out the rest of the folio. 2055ad448ceSAndreas Gruenbacher * Only a single IOMAP_INLINE extent is allowed at the end of each file. 2065ad448ceSAndreas Gruenbacher * Returns zero for success to complete the read, or the usual negative errno. 2075ad448ceSAndreas Gruenbacher */ 2085ad448ceSAndreas Gruenbacher static int iomap_read_inline_data(const struct iomap_iter *iter, 209874628a2SMatthew Wilcox (Oracle) struct folio *folio) 210afc51aaaSDarrick J. Wong { 211cd1e5afeSMatthew Wilcox (Oracle) struct iomap_page *iop; 212fad0a1abSChristoph Hellwig const struct iomap *iomap = iomap_iter_srcmap(iter); 2131b5c1e36SChristoph Hellwig size_t size = i_size_read(iter->inode) - iomap->offset; 214b405435bSMatthew Wilcox (Oracle) size_t poff = offset_in_page(iomap->offset); 215431c0566SMatthew Wilcox (Oracle) size_t offset = offset_in_folio(folio, iomap->offset); 216afc51aaaSDarrick J. Wong void *addr; 217afc51aaaSDarrick J. Wong 218874628a2SMatthew Wilcox (Oracle) if (folio_test_uptodate(folio)) 2195ad448ceSAndreas Gruenbacher return 0; 220afc51aaaSDarrick J. Wong 221ae44f9c2SMatthew Wilcox (Oracle) if (WARN_ON_ONCE(size > PAGE_SIZE - poff)) 222ae44f9c2SMatthew Wilcox (Oracle) return -EIO; 22369f4a26cSGao Xiang if (WARN_ON_ONCE(size > PAGE_SIZE - 22469f4a26cSGao Xiang offset_in_page(iomap->inline_data))) 22569f4a26cSGao Xiang return -EIO; 22669f4a26cSGao Xiang if (WARN_ON_ONCE(size > iomap->length)) 22769f4a26cSGao Xiang return -EIO; 228431c0566SMatthew Wilcox (Oracle) if (offset > 0) 229cd1e5afeSMatthew Wilcox (Oracle) iop = iomap_page_create(iter->inode, folio); 230cd1e5afeSMatthew Wilcox (Oracle) else 231cd1e5afeSMatthew Wilcox (Oracle) iop = to_iomap_page(folio); 232afc51aaaSDarrick J. Wong 233874628a2SMatthew Wilcox (Oracle) addr = kmap_local_folio(folio, offset); 234afc51aaaSDarrick J. Wong memcpy(addr, iomap->inline_data, size); 235b405435bSMatthew Wilcox (Oracle) memset(addr + size, 0, PAGE_SIZE - poff - size); 236ab069d5fSMatthew Wilcox (Oracle) kunmap_local(addr); 237431c0566SMatthew Wilcox (Oracle) iomap_set_range_uptodate(folio, iop, offset, PAGE_SIZE - poff); 2385ad448ceSAndreas Gruenbacher return 0; 239afc51aaaSDarrick J. Wong } 240afc51aaaSDarrick J. Wong 241fad0a1abSChristoph Hellwig static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter, 2421b5c1e36SChristoph Hellwig loff_t pos) 243009d8d84SChristoph Hellwig { 244fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 2451b5c1e36SChristoph Hellwig 2461b5c1e36SChristoph Hellwig return srcmap->type != IOMAP_MAPPED || 2471b5c1e36SChristoph Hellwig (srcmap->flags & IOMAP_F_NEW) || 2481b5c1e36SChristoph Hellwig pos >= i_size_read(iter->inode); 249009d8d84SChristoph Hellwig } 250009d8d84SChristoph Hellwig 251fad0a1abSChristoph Hellwig static loff_t iomap_readpage_iter(const struct iomap_iter *iter, 252f6d48000SChristoph Hellwig struct iomap_readpage_ctx *ctx, loff_t offset) 253afc51aaaSDarrick J. Wong { 254fad0a1abSChristoph Hellwig const struct iomap *iomap = &iter->iomap; 255f6d48000SChristoph Hellwig loff_t pos = iter->pos + offset; 256f6d48000SChristoph Hellwig loff_t length = iomap_length(iter) - offset; 2573aa9c659SMatthew Wilcox (Oracle) struct folio *folio = ctx->cur_folio; 258637d3375SAndreas Gruenbacher struct iomap_page *iop; 259afc51aaaSDarrick J. Wong loff_t orig_pos = pos; 260431c0566SMatthew Wilcox (Oracle) size_t poff, plen; 261afc51aaaSDarrick J. Wong sector_t sector; 262afc51aaaSDarrick J. Wong 2635ad448ceSAndreas Gruenbacher if (iomap->type == IOMAP_INLINE) 264874628a2SMatthew Wilcox (Oracle) return iomap_read_inline_data(iter, folio); 265afc51aaaSDarrick J. Wong 266afc51aaaSDarrick J. Wong /* zero post-eof blocks as the page may be mapped */ 267435d44b3SMatthew Wilcox (Oracle) iop = iomap_page_create(iter->inode, folio); 268431c0566SMatthew Wilcox (Oracle) iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen); 269afc51aaaSDarrick J. Wong if (plen == 0) 270afc51aaaSDarrick J. Wong goto done; 271afc51aaaSDarrick J. Wong 2721b5c1e36SChristoph Hellwig if (iomap_block_needs_zeroing(iter, pos)) { 273431c0566SMatthew Wilcox (Oracle) folio_zero_range(folio, poff, plen); 274431c0566SMatthew Wilcox (Oracle) iomap_set_range_uptodate(folio, iop, poff, plen); 275afc51aaaSDarrick J. Wong goto done; 276afc51aaaSDarrick J. Wong } 277afc51aaaSDarrick J. Wong 2783aa9c659SMatthew Wilcox (Oracle) ctx->cur_folio_in_bio = true; 2797d636676SMatthew Wilcox (Oracle) if (iop) 2807d636676SMatthew Wilcox (Oracle) atomic_add(plen, &iop->read_bytes_pending); 281afc51aaaSDarrick J. Wong 282afc51aaaSDarrick J. Wong sector = iomap_sector(iomap, pos); 283d0364f94SChristoph Hellwig if (!ctx->bio || 284d0364f94SChristoph Hellwig bio_end_sector(ctx->bio) != sector || 285431c0566SMatthew Wilcox (Oracle) !bio_add_folio(ctx->bio, folio, plen, poff)) { 2863aa9c659SMatthew Wilcox (Oracle) gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL); 287457df33eSMatthew Wilcox (Oracle) gfp_t orig_gfp = gfp; 2885f7136dbSMatthew Wilcox (Oracle) unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE); 289afc51aaaSDarrick J. Wong 290afc51aaaSDarrick J. Wong if (ctx->bio) 291afc51aaaSDarrick J. Wong submit_bio(ctx->bio); 292afc51aaaSDarrick J. Wong 2939d24a13aSMatthew Wilcox (Oracle) if (ctx->rac) /* same as readahead_gfp_mask */ 294afc51aaaSDarrick J. Wong gfp |= __GFP_NORETRY | __GFP_NOWARN; 29507888c66SChristoph Hellwig ctx->bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs), 29607888c66SChristoph Hellwig REQ_OP_READ, gfp); 297457df33eSMatthew Wilcox (Oracle) /* 298457df33eSMatthew Wilcox (Oracle) * If the bio_alloc fails, try it again for a single page to 299457df33eSMatthew Wilcox (Oracle) * avoid having to deal with partial page reads. This emulates 300457df33eSMatthew Wilcox (Oracle) * what do_mpage_readpage does. 301457df33eSMatthew Wilcox (Oracle) */ 30207888c66SChristoph Hellwig if (!ctx->bio) { 30307888c66SChristoph Hellwig ctx->bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ, 30407888c66SChristoph Hellwig orig_gfp); 30507888c66SChristoph Hellwig } 3069d24a13aSMatthew Wilcox (Oracle) if (ctx->rac) 307afc51aaaSDarrick J. Wong ctx->bio->bi_opf |= REQ_RAHEAD; 308afc51aaaSDarrick J. Wong ctx->bio->bi_iter.bi_sector = sector; 309afc51aaaSDarrick J. Wong ctx->bio->bi_end_io = iomap_read_end_io; 310431c0566SMatthew Wilcox (Oracle) bio_add_folio(ctx->bio, folio, plen, poff); 311afc51aaaSDarrick J. Wong } 312431c0566SMatthew Wilcox (Oracle) 313afc51aaaSDarrick J. Wong done: 314afc51aaaSDarrick J. Wong /* 315afc51aaaSDarrick J. Wong * Move the caller beyond our range so that it keeps making progress. 316f1f264b4SAndreas Gruenbacher * For that, we have to include any leading non-uptodate ranges, but 317afc51aaaSDarrick J. Wong * we can skip trailing ones as they will be handled in the next 318afc51aaaSDarrick J. Wong * iteration. 319afc51aaaSDarrick J. Wong */ 320afc51aaaSDarrick J. Wong return pos - orig_pos + plen; 321afc51aaaSDarrick J. Wong } 322afc51aaaSDarrick J. Wong 323afc51aaaSDarrick J. Wong int 324afc51aaaSDarrick J. Wong iomap_readpage(struct page *page, const struct iomap_ops *ops) 325afc51aaaSDarrick J. Wong { 3263aa9c659SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 327f6d48000SChristoph Hellwig struct iomap_iter iter = { 3283aa9c659SMatthew Wilcox (Oracle) .inode = folio->mapping->host, 3293aa9c659SMatthew Wilcox (Oracle) .pos = folio_pos(folio), 3303aa9c659SMatthew Wilcox (Oracle) .len = folio_size(folio), 331f6d48000SChristoph Hellwig }; 332f6d48000SChristoph Hellwig struct iomap_readpage_ctx ctx = { 3333aa9c659SMatthew Wilcox (Oracle) .cur_folio = folio, 334f6d48000SChristoph Hellwig }; 335f6d48000SChristoph Hellwig int ret; 336afc51aaaSDarrick J. Wong 3373aa9c659SMatthew Wilcox (Oracle) trace_iomap_readpage(iter.inode, 1); 3389e91c572SChristoph Hellwig 339f6d48000SChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 340f6d48000SChristoph Hellwig iter.processed = iomap_readpage_iter(&iter, &ctx, 0); 341f6d48000SChristoph Hellwig 342f6d48000SChristoph Hellwig if (ret < 0) 3433aa9c659SMatthew Wilcox (Oracle) folio_set_error(folio); 344afc51aaaSDarrick J. Wong 345afc51aaaSDarrick J. Wong if (ctx.bio) { 346afc51aaaSDarrick J. Wong submit_bio(ctx.bio); 3473aa9c659SMatthew Wilcox (Oracle) WARN_ON_ONCE(!ctx.cur_folio_in_bio); 348afc51aaaSDarrick J. Wong } else { 3493aa9c659SMatthew Wilcox (Oracle) WARN_ON_ONCE(ctx.cur_folio_in_bio); 3503aa9c659SMatthew Wilcox (Oracle) folio_unlock(folio); 351afc51aaaSDarrick J. Wong } 352afc51aaaSDarrick J. Wong 353afc51aaaSDarrick J. Wong /* 354f1f264b4SAndreas Gruenbacher * Just like mpage_readahead and block_read_full_page, we always 355afc51aaaSDarrick J. Wong * return 0 and just mark the page as PageError on errors. This 356f1f264b4SAndreas Gruenbacher * should be cleaned up throughout the stack eventually. 357afc51aaaSDarrick J. Wong */ 358afc51aaaSDarrick J. Wong return 0; 359afc51aaaSDarrick J. Wong } 360afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_readpage); 361afc51aaaSDarrick J. Wong 362fad0a1abSChristoph Hellwig static loff_t iomap_readahead_iter(const struct iomap_iter *iter, 363f6d48000SChristoph Hellwig struct iomap_readpage_ctx *ctx) 364afc51aaaSDarrick J. Wong { 365f6d48000SChristoph Hellwig loff_t length = iomap_length(iter); 366afc51aaaSDarrick J. Wong loff_t done, ret; 367afc51aaaSDarrick J. Wong 368afc51aaaSDarrick J. Wong for (done = 0; done < length; done += ret) { 3693aa9c659SMatthew Wilcox (Oracle) if (ctx->cur_folio && 3703aa9c659SMatthew Wilcox (Oracle) offset_in_folio(ctx->cur_folio, iter->pos + done) == 0) { 3713aa9c659SMatthew Wilcox (Oracle) if (!ctx->cur_folio_in_bio) 3723aa9c659SMatthew Wilcox (Oracle) folio_unlock(ctx->cur_folio); 3733aa9c659SMatthew Wilcox (Oracle) ctx->cur_folio = NULL; 374afc51aaaSDarrick J. Wong } 3753aa9c659SMatthew Wilcox (Oracle) if (!ctx->cur_folio) { 3763aa9c659SMatthew Wilcox (Oracle) ctx->cur_folio = readahead_folio(ctx->rac); 3773aa9c659SMatthew Wilcox (Oracle) ctx->cur_folio_in_bio = false; 378afc51aaaSDarrick J. Wong } 379f6d48000SChristoph Hellwig ret = iomap_readpage_iter(iter, ctx, done); 380d8af404fSAndreas Gruenbacher if (ret <= 0) 381d8af404fSAndreas Gruenbacher return ret; 382afc51aaaSDarrick J. Wong } 383afc51aaaSDarrick J. Wong 384afc51aaaSDarrick J. Wong return done; 385afc51aaaSDarrick J. Wong } 386afc51aaaSDarrick J. Wong 3879d24a13aSMatthew Wilcox (Oracle) /** 3889d24a13aSMatthew Wilcox (Oracle) * iomap_readahead - Attempt to read pages from a file. 3899d24a13aSMatthew Wilcox (Oracle) * @rac: Describes the pages to be read. 3909d24a13aSMatthew Wilcox (Oracle) * @ops: The operations vector for the filesystem. 3919d24a13aSMatthew Wilcox (Oracle) * 3929d24a13aSMatthew Wilcox (Oracle) * This function is for filesystems to call to implement their readahead 3939d24a13aSMatthew Wilcox (Oracle) * address_space operation. 3949d24a13aSMatthew Wilcox (Oracle) * 3959d24a13aSMatthew Wilcox (Oracle) * Context: The @ops callbacks may submit I/O (eg to read the addresses of 3969d24a13aSMatthew Wilcox (Oracle) * blocks from disc), and may wait for it. The caller may be trying to 3979d24a13aSMatthew Wilcox (Oracle) * access a different page, and so sleeping excessively should be avoided. 3989d24a13aSMatthew Wilcox (Oracle) * It may allocate memory, but should avoid costly allocations. This 3999d24a13aSMatthew Wilcox (Oracle) * function is called with memalloc_nofs set, so allocations will not cause 4009d24a13aSMatthew Wilcox (Oracle) * the filesystem to be reentered. 4019d24a13aSMatthew Wilcox (Oracle) */ 4029d24a13aSMatthew Wilcox (Oracle) void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops) 403afc51aaaSDarrick J. Wong { 404f6d48000SChristoph Hellwig struct iomap_iter iter = { 405f6d48000SChristoph Hellwig .inode = rac->mapping->host, 406f6d48000SChristoph Hellwig .pos = readahead_pos(rac), 407f6d48000SChristoph Hellwig .len = readahead_length(rac), 408f6d48000SChristoph Hellwig }; 409afc51aaaSDarrick J. Wong struct iomap_readpage_ctx ctx = { 4109d24a13aSMatthew Wilcox (Oracle) .rac = rac, 411afc51aaaSDarrick J. Wong }; 412afc51aaaSDarrick J. Wong 413f6d48000SChristoph Hellwig trace_iomap_readahead(rac->mapping->host, readahead_count(rac)); 4149e91c572SChristoph Hellwig 415f6d48000SChristoph Hellwig while (iomap_iter(&iter, ops) > 0) 416f6d48000SChristoph Hellwig iter.processed = iomap_readahead_iter(&iter, &ctx); 4179d24a13aSMatthew Wilcox (Oracle) 418afc51aaaSDarrick J. Wong if (ctx.bio) 419afc51aaaSDarrick J. Wong submit_bio(ctx.bio); 4203aa9c659SMatthew Wilcox (Oracle) if (ctx.cur_folio) { 4213aa9c659SMatthew Wilcox (Oracle) if (!ctx.cur_folio_in_bio) 4223aa9c659SMatthew Wilcox (Oracle) folio_unlock(ctx.cur_folio); 423afc51aaaSDarrick J. Wong } 424afc51aaaSDarrick J. Wong } 4259d24a13aSMatthew Wilcox (Oracle) EXPORT_SYMBOL_GPL(iomap_readahead); 426afc51aaaSDarrick J. Wong 427afc51aaaSDarrick J. Wong /* 4282e7e80f7SMatthew Wilcox (Oracle) * iomap_is_partially_uptodate checks whether blocks within a folio are 429afc51aaaSDarrick J. Wong * uptodate or not. 430afc51aaaSDarrick J. Wong * 4312e7e80f7SMatthew Wilcox (Oracle) * Returns true if all blocks which correspond to the specified part 4322e7e80f7SMatthew Wilcox (Oracle) * of the folio are uptodate. 433afc51aaaSDarrick J. Wong */ 4342e7e80f7SMatthew Wilcox (Oracle) bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count) 435afc51aaaSDarrick J. Wong { 43695c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 4372e7e80f7SMatthew Wilcox (Oracle) struct inode *inode = folio->mapping->host; 4382e7e80f7SMatthew Wilcox (Oracle) unsigned first, last, i; 439afc51aaaSDarrick J. Wong 4402e7e80f7SMatthew Wilcox (Oracle) if (!iop) 4412e7e80f7SMatthew Wilcox (Oracle) return false; 4422e7e80f7SMatthew Wilcox (Oracle) 443*2756c818SMatthew Wilcox (Oracle) /* Caller's range may extend past the end of this folio */ 444*2756c818SMatthew Wilcox (Oracle) count = min(folio_size(folio) - from, count); 445afc51aaaSDarrick J. Wong 446*2756c818SMatthew Wilcox (Oracle) /* First and last blocks in range within folio */ 447afc51aaaSDarrick J. Wong first = from >> inode->i_blkbits; 448*2756c818SMatthew Wilcox (Oracle) last = (from + count - 1) >> inode->i_blkbits; 449afc51aaaSDarrick J. Wong 450afc51aaaSDarrick J. Wong for (i = first; i <= last; i++) 451afc51aaaSDarrick J. Wong if (!test_bit(i, iop->uptodate)) 4522e7e80f7SMatthew Wilcox (Oracle) return false; 4532e7e80f7SMatthew Wilcox (Oracle) return true; 454afc51aaaSDarrick J. Wong } 455afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); 456afc51aaaSDarrick J. Wong 457afc51aaaSDarrick J. Wong int 458afc51aaaSDarrick J. Wong iomap_releasepage(struct page *page, gfp_t gfp_mask) 459afc51aaaSDarrick J. Wong { 460c46e8324SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 461c46e8324SMatthew Wilcox (Oracle) 46239f16c83SMatthew Wilcox (Oracle) trace_iomap_releasepage(folio->mapping->host, folio_pos(folio), 46339f16c83SMatthew Wilcox (Oracle) folio_size(folio)); 4649e91c572SChristoph Hellwig 465afc51aaaSDarrick J. Wong /* 466afc51aaaSDarrick J. Wong * mm accommodates an old ext3 case where clean pages might not have had 467afc51aaaSDarrick J. Wong * the dirty bit cleared. Thus, it can send actual dirty pages to 468f1f264b4SAndreas Gruenbacher * ->releasepage() via shrink_active_list(); skip those here. 469afc51aaaSDarrick J. Wong */ 47039f16c83SMatthew Wilcox (Oracle) if (folio_test_dirty(folio) || folio_test_writeback(folio)) 471afc51aaaSDarrick J. Wong return 0; 472c46e8324SMatthew Wilcox (Oracle) iomap_page_release(folio); 473afc51aaaSDarrick J. Wong return 1; 474afc51aaaSDarrick J. Wong } 475afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_releasepage); 476afc51aaaSDarrick J. Wong 4778306a5f5SMatthew Wilcox (Oracle) void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len) 478afc51aaaSDarrick J. Wong { 479d82354f6SMatthew Wilcox (Oracle) trace_iomap_invalidate_folio(folio->mapping->host, 4801241ebecSMatthew Wilcox (Oracle) folio_pos(folio) + offset, len); 4819e91c572SChristoph Hellwig 482afc51aaaSDarrick J. Wong /* 48360d82310SMatthew Wilcox (Oracle) * If we're invalidating the entire folio, clear the dirty state 48460d82310SMatthew Wilcox (Oracle) * from it and release it to avoid unnecessary buildup of the LRU. 485afc51aaaSDarrick J. Wong */ 4868306a5f5SMatthew Wilcox (Oracle) if (offset == 0 && len == folio_size(folio)) { 4878306a5f5SMatthew Wilcox (Oracle) WARN_ON_ONCE(folio_test_writeback(folio)); 4888306a5f5SMatthew Wilcox (Oracle) folio_cancel_dirty(folio); 489c46e8324SMatthew Wilcox (Oracle) iomap_page_release(folio); 49060d82310SMatthew Wilcox (Oracle) } else if (folio_test_large(folio)) { 49160d82310SMatthew Wilcox (Oracle) /* Must release the iop so the page can be split */ 49260d82310SMatthew Wilcox (Oracle) WARN_ON_ONCE(!folio_test_uptodate(folio) && 49360d82310SMatthew Wilcox (Oracle) folio_test_dirty(folio)); 49460d82310SMatthew Wilcox (Oracle) iomap_page_release(folio); 495afc51aaaSDarrick J. Wong } 496afc51aaaSDarrick J. Wong } 4978306a5f5SMatthew Wilcox (Oracle) EXPORT_SYMBOL_GPL(iomap_invalidate_folio); 4988306a5f5SMatthew Wilcox (Oracle) 499afc51aaaSDarrick J. Wong #ifdef CONFIG_MIGRATION 500afc51aaaSDarrick J. Wong int 501afc51aaaSDarrick J. Wong iomap_migrate_page(struct address_space *mapping, struct page *newpage, 502afc51aaaSDarrick J. Wong struct page *page, enum migrate_mode mode) 503afc51aaaSDarrick J. Wong { 504589110e8SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 505589110e8SMatthew Wilcox (Oracle) struct folio *newfolio = page_folio(newpage); 506afc51aaaSDarrick J. Wong int ret; 507afc51aaaSDarrick J. Wong 508589110e8SMatthew Wilcox (Oracle) ret = folio_migrate_mapping(mapping, newfolio, folio, 0); 509afc51aaaSDarrick J. Wong if (ret != MIGRATEPAGE_SUCCESS) 510afc51aaaSDarrick J. Wong return ret; 511afc51aaaSDarrick J. Wong 512589110e8SMatthew Wilcox (Oracle) if (folio_test_private(folio)) 513589110e8SMatthew Wilcox (Oracle) folio_attach_private(newfolio, folio_detach_private(folio)); 514afc51aaaSDarrick J. Wong 515afc51aaaSDarrick J. Wong if (mode != MIGRATE_SYNC_NO_COPY) 516589110e8SMatthew Wilcox (Oracle) folio_migrate_copy(newfolio, folio); 517afc51aaaSDarrick J. Wong else 518589110e8SMatthew Wilcox (Oracle) folio_migrate_flags(newfolio, folio); 519afc51aaaSDarrick J. Wong return MIGRATEPAGE_SUCCESS; 520afc51aaaSDarrick J. Wong } 521afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_migrate_page); 522afc51aaaSDarrick J. Wong #endif /* CONFIG_MIGRATION */ 523afc51aaaSDarrick J. Wong 524afc51aaaSDarrick J. Wong static void 525afc51aaaSDarrick J. Wong iomap_write_failed(struct inode *inode, loff_t pos, unsigned len) 526afc51aaaSDarrick J. Wong { 527afc51aaaSDarrick J. Wong loff_t i_size = i_size_read(inode); 528afc51aaaSDarrick J. Wong 529afc51aaaSDarrick J. Wong /* 530afc51aaaSDarrick J. Wong * Only truncate newly allocated pages beyoned EOF, even if the 531afc51aaaSDarrick J. Wong * write started inside the existing inode size. 532afc51aaaSDarrick J. Wong */ 533afc51aaaSDarrick J. Wong if (pos + len > i_size) 534afc51aaaSDarrick J. Wong truncate_pagecache_range(inode, max(pos, i_size), pos + len); 535afc51aaaSDarrick J. Wong } 536afc51aaaSDarrick J. Wong 537431c0566SMatthew Wilcox (Oracle) static int iomap_read_folio_sync(loff_t block_start, struct folio *folio, 538431c0566SMatthew Wilcox (Oracle) size_t poff, size_t plen, const struct iomap *iomap) 539afc51aaaSDarrick J. Wong { 540afc51aaaSDarrick J. Wong struct bio_vec bvec; 541afc51aaaSDarrick J. Wong struct bio bio; 542afc51aaaSDarrick J. Wong 54349add496SChristoph Hellwig bio_init(&bio, iomap->bdev, &bvec, 1, REQ_OP_READ); 544afc51aaaSDarrick J. Wong bio.bi_iter.bi_sector = iomap_sector(iomap, block_start); 545431c0566SMatthew Wilcox (Oracle) bio_add_folio(&bio, folio, plen, poff); 546afc51aaaSDarrick J. Wong return submit_bio_wait(&bio); 547afc51aaaSDarrick J. Wong } 548afc51aaaSDarrick J. Wong 549fad0a1abSChristoph Hellwig static int __iomap_write_begin(const struct iomap_iter *iter, loff_t pos, 550bc6123a8SMatthew Wilcox (Oracle) size_t len, struct folio *folio) 551afc51aaaSDarrick J. Wong { 552fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 553435d44b3SMatthew Wilcox (Oracle) struct iomap_page *iop = iomap_page_create(iter->inode, folio); 5541b5c1e36SChristoph Hellwig loff_t block_size = i_blocksize(iter->inode); 5556cc19c5fSNikolay Borisov loff_t block_start = round_down(pos, block_size); 5566cc19c5fSNikolay Borisov loff_t block_end = round_up(pos + len, block_size); 557431c0566SMatthew Wilcox (Oracle) size_t from = offset_in_folio(folio, pos), to = from + len; 558431c0566SMatthew Wilcox (Oracle) size_t poff, plen; 559afc51aaaSDarrick J. Wong 560431c0566SMatthew Wilcox (Oracle) if (folio_test_uptodate(folio)) 561afc51aaaSDarrick J. Wong return 0; 562431c0566SMatthew Wilcox (Oracle) folio_clear_error(folio); 563afc51aaaSDarrick J. Wong 564afc51aaaSDarrick J. Wong do { 565431c0566SMatthew Wilcox (Oracle) iomap_adjust_read_range(iter->inode, folio, &block_start, 566afc51aaaSDarrick J. Wong block_end - block_start, &poff, &plen); 567afc51aaaSDarrick J. Wong if (plen == 0) 568afc51aaaSDarrick J. Wong break; 569afc51aaaSDarrick J. Wong 570b74b1293SChristoph Hellwig if (!(iter->flags & IOMAP_UNSHARE) && 57132a38a49SChristoph Hellwig (from <= poff || from >= poff + plen) && 572d3b40439SChristoph Hellwig (to <= poff || to >= poff + plen)) 573d3b40439SChristoph Hellwig continue; 574d3b40439SChristoph Hellwig 5751b5c1e36SChristoph Hellwig if (iomap_block_needs_zeroing(iter, block_start)) { 576b74b1293SChristoph Hellwig if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE)) 57732a38a49SChristoph Hellwig return -EIO; 578431c0566SMatthew Wilcox (Oracle) folio_zero_segments(folio, poff, from, to, poff + plen); 57914284fedSMatthew Wilcox (Oracle) } else { 580431c0566SMatthew Wilcox (Oracle) int status = iomap_read_folio_sync(block_start, folio, 58114284fedSMatthew Wilcox (Oracle) poff, plen, srcmap); 582d3b40439SChristoph Hellwig if (status) 583d3b40439SChristoph Hellwig return status; 58414284fedSMatthew Wilcox (Oracle) } 585431c0566SMatthew Wilcox (Oracle) iomap_set_range_uptodate(folio, iop, poff, plen); 586afc51aaaSDarrick J. Wong } while ((block_start += plen) < block_end); 587afc51aaaSDarrick J. Wong 588d3b40439SChristoph Hellwig return 0; 589afc51aaaSDarrick J. Wong } 590afc51aaaSDarrick J. Wong 591fad0a1abSChristoph Hellwig static int iomap_write_begin_inline(const struct iomap_iter *iter, 592bc6123a8SMatthew Wilcox (Oracle) struct folio *folio) 59369f4a26cSGao Xiang { 59469f4a26cSGao Xiang /* needs more work for the tailpacking case; disable for now */ 5951b5c1e36SChristoph Hellwig if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0)) 59669f4a26cSGao Xiang return -EIO; 597874628a2SMatthew Wilcox (Oracle) return iomap_read_inline_data(iter, folio); 59869f4a26cSGao Xiang } 59969f4a26cSGao Xiang 600fad0a1abSChristoph Hellwig static int iomap_write_begin(const struct iomap_iter *iter, loff_t pos, 601bc6123a8SMatthew Wilcox (Oracle) size_t len, struct folio **foliop) 602afc51aaaSDarrick J. Wong { 6031b5c1e36SChristoph Hellwig const struct iomap_page_ops *page_ops = iter->iomap.page_ops; 604fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 605d1bd0b4eSMatthew Wilcox (Oracle) struct folio *folio; 606bc6123a8SMatthew Wilcox (Oracle) unsigned fgp = FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE | FGP_NOFS; 607afc51aaaSDarrick J. Wong int status = 0; 608afc51aaaSDarrick J. Wong 6091b5c1e36SChristoph Hellwig BUG_ON(pos + len > iter->iomap.offset + iter->iomap.length); 6101b5c1e36SChristoph Hellwig if (srcmap != &iter->iomap) 611c039b997SGoldwyn Rodrigues BUG_ON(pos + len > srcmap->offset + srcmap->length); 612afc51aaaSDarrick J. Wong 613afc51aaaSDarrick J. Wong if (fatal_signal_pending(current)) 614afc51aaaSDarrick J. Wong return -EINTR; 615afc51aaaSDarrick J. Wong 616d454ab82SMatthew Wilcox (Oracle) if (!mapping_large_folio_support(iter->inode->i_mapping)) 617d454ab82SMatthew Wilcox (Oracle) len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos)); 618d454ab82SMatthew Wilcox (Oracle) 619afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_prepare) { 6201b5c1e36SChristoph Hellwig status = page_ops->page_prepare(iter->inode, pos, len); 621afc51aaaSDarrick J. Wong if (status) 622afc51aaaSDarrick J. Wong return status; 623afc51aaaSDarrick J. Wong } 624afc51aaaSDarrick J. Wong 625bc6123a8SMatthew Wilcox (Oracle) folio = __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT, 626bc6123a8SMatthew Wilcox (Oracle) fgp, mapping_gfp_mask(iter->inode->i_mapping)); 627bc6123a8SMatthew Wilcox (Oracle) if (!folio) { 628afc51aaaSDarrick J. Wong status = -ENOMEM; 629afc51aaaSDarrick J. Wong goto out_no_page; 630afc51aaaSDarrick J. Wong } 631d454ab82SMatthew Wilcox (Oracle) if (pos + len > folio_pos(folio) + folio_size(folio)) 632d454ab82SMatthew Wilcox (Oracle) len = folio_pos(folio) + folio_size(folio) - pos; 633afc51aaaSDarrick J. Wong 634c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_INLINE) 635bc6123a8SMatthew Wilcox (Oracle) status = iomap_write_begin_inline(iter, folio); 6361b5c1e36SChristoph Hellwig else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) 637d1bd0b4eSMatthew Wilcox (Oracle) status = __block_write_begin_int(folio, pos, len, NULL, srcmap); 638afc51aaaSDarrick J. Wong else 639bc6123a8SMatthew Wilcox (Oracle) status = __iomap_write_begin(iter, pos, len, folio); 640afc51aaaSDarrick J. Wong 641afc51aaaSDarrick J. Wong if (unlikely(status)) 642afc51aaaSDarrick J. Wong goto out_unlock; 643afc51aaaSDarrick J. Wong 644bc6123a8SMatthew Wilcox (Oracle) *foliop = folio; 645afc51aaaSDarrick J. Wong return 0; 646afc51aaaSDarrick J. Wong 647afc51aaaSDarrick J. Wong out_unlock: 648bc6123a8SMatthew Wilcox (Oracle) folio_unlock(folio); 649bc6123a8SMatthew Wilcox (Oracle) folio_put(folio); 6501b5c1e36SChristoph Hellwig iomap_write_failed(iter->inode, pos, len); 651afc51aaaSDarrick J. Wong 652afc51aaaSDarrick J. Wong out_no_page: 653afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_done) 6541b5c1e36SChristoph Hellwig page_ops->page_done(iter->inode, pos, 0, NULL); 655afc51aaaSDarrick J. Wong return status; 656afc51aaaSDarrick J. Wong } 657afc51aaaSDarrick J. Wong 658e25ba8cbSMatthew Wilcox (Oracle) static size_t __iomap_write_end(struct inode *inode, loff_t pos, size_t len, 659bc6123a8SMatthew Wilcox (Oracle) size_t copied, struct folio *folio) 660afc51aaaSDarrick J. Wong { 661cd1e5afeSMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 662bc6123a8SMatthew Wilcox (Oracle) flush_dcache_folio(folio); 663afc51aaaSDarrick J. Wong 664afc51aaaSDarrick J. Wong /* 665afc51aaaSDarrick J. Wong * The blocks that were entirely written will now be uptodate, so we 666afc51aaaSDarrick J. Wong * don't have to worry about a readpage reading them and overwriting a 667f1f264b4SAndreas Gruenbacher * partial write. However, if we've encountered a short write and only 668afc51aaaSDarrick J. Wong * partially written into a block, it will not be marked uptodate, so a 669afc51aaaSDarrick J. Wong * readpage might come in and destroy our partial write. 670afc51aaaSDarrick J. Wong * 671f1f264b4SAndreas Gruenbacher * Do the simplest thing and just treat any short write to a 672f1f264b4SAndreas Gruenbacher * non-uptodate page as a zero-length write, and force the caller to 673f1f264b4SAndreas Gruenbacher * redo the whole thing. 674afc51aaaSDarrick J. Wong */ 675bc6123a8SMatthew Wilcox (Oracle) if (unlikely(copied < len && !folio_test_uptodate(folio))) 676afc51aaaSDarrick J. Wong return 0; 677431c0566SMatthew Wilcox (Oracle) iomap_set_range_uptodate(folio, iop, offset_in_folio(folio, pos), len); 678bc6123a8SMatthew Wilcox (Oracle) filemap_dirty_folio(inode->i_mapping, folio); 679afc51aaaSDarrick J. Wong return copied; 680afc51aaaSDarrick J. Wong } 681afc51aaaSDarrick J. Wong 682fad0a1abSChristoph Hellwig static size_t iomap_write_end_inline(const struct iomap_iter *iter, 6839c4ce08dSMatthew Wilcox (Oracle) struct folio *folio, loff_t pos, size_t copied) 684afc51aaaSDarrick J. Wong { 685fad0a1abSChristoph Hellwig const struct iomap *iomap = &iter->iomap; 686afc51aaaSDarrick J. Wong void *addr; 687afc51aaaSDarrick J. Wong 6889c4ce08dSMatthew Wilcox (Oracle) WARN_ON_ONCE(!folio_test_uptodate(folio)); 68969f4a26cSGao Xiang BUG_ON(!iomap_inline_data_valid(iomap)); 690afc51aaaSDarrick J. Wong 6919c4ce08dSMatthew Wilcox (Oracle) flush_dcache_folio(folio); 6929c4ce08dSMatthew Wilcox (Oracle) addr = kmap_local_folio(folio, pos); 693ab069d5fSMatthew Wilcox (Oracle) memcpy(iomap_inline_data(iomap, pos), addr, copied); 694ab069d5fSMatthew Wilcox (Oracle) kunmap_local(addr); 695afc51aaaSDarrick J. Wong 6961b5c1e36SChristoph Hellwig mark_inode_dirty(iter->inode); 697afc51aaaSDarrick J. Wong return copied; 698afc51aaaSDarrick J. Wong } 699afc51aaaSDarrick J. Wong 700e25ba8cbSMatthew Wilcox (Oracle) /* Returns the number of bytes copied. May be 0. Cannot be an errno. */ 7011b5c1e36SChristoph Hellwig static size_t iomap_write_end(struct iomap_iter *iter, loff_t pos, size_t len, 702bc6123a8SMatthew Wilcox (Oracle) size_t copied, struct folio *folio) 703afc51aaaSDarrick J. Wong { 7041b5c1e36SChristoph Hellwig const struct iomap_page_ops *page_ops = iter->iomap.page_ops; 705fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 7061b5c1e36SChristoph Hellwig loff_t old_size = iter->inode->i_size; 707e25ba8cbSMatthew Wilcox (Oracle) size_t ret; 708afc51aaaSDarrick J. Wong 709c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_INLINE) { 7109c4ce08dSMatthew Wilcox (Oracle) ret = iomap_write_end_inline(iter, folio, pos, copied); 711c039b997SGoldwyn Rodrigues } else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) { 7121b5c1e36SChristoph Hellwig ret = block_write_end(NULL, iter->inode->i_mapping, pos, len, 713bc6123a8SMatthew Wilcox (Oracle) copied, &folio->page, NULL); 714afc51aaaSDarrick J. Wong } else { 715bc6123a8SMatthew Wilcox (Oracle) ret = __iomap_write_end(iter->inode, pos, len, copied, folio); 716afc51aaaSDarrick J. Wong } 717afc51aaaSDarrick J. Wong 718afc51aaaSDarrick J. Wong /* 719afc51aaaSDarrick J. Wong * Update the in-memory inode size after copying the data into the page 720afc51aaaSDarrick J. Wong * cache. It's up to the file system to write the updated size to disk, 721afc51aaaSDarrick J. Wong * preferably after I/O completion so that no stale data is exposed. 722afc51aaaSDarrick J. Wong */ 723afc51aaaSDarrick J. Wong if (pos + ret > old_size) { 7241b5c1e36SChristoph Hellwig i_size_write(iter->inode, pos + ret); 7251b5c1e36SChristoph Hellwig iter->iomap.flags |= IOMAP_F_SIZE_CHANGED; 726afc51aaaSDarrick J. Wong } 727bc6123a8SMatthew Wilcox (Oracle) folio_unlock(folio); 728afc51aaaSDarrick J. Wong 729afc51aaaSDarrick J. Wong if (old_size < pos) 7301b5c1e36SChristoph Hellwig pagecache_isize_extended(iter->inode, old_size, pos); 731afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_done) 732bc6123a8SMatthew Wilcox (Oracle) page_ops->page_done(iter->inode, pos, ret, &folio->page); 733bc6123a8SMatthew Wilcox (Oracle) folio_put(folio); 734afc51aaaSDarrick J. Wong 735afc51aaaSDarrick J. Wong if (ret < len) 7361b5c1e36SChristoph Hellwig iomap_write_failed(iter->inode, pos, len); 737afc51aaaSDarrick J. Wong return ret; 738afc51aaaSDarrick J. Wong } 739afc51aaaSDarrick J. Wong 740ce83a025SChristoph Hellwig static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i) 741afc51aaaSDarrick J. Wong { 742ce83a025SChristoph Hellwig loff_t length = iomap_length(iter); 743ce83a025SChristoph Hellwig loff_t pos = iter->pos; 744afc51aaaSDarrick J. Wong ssize_t written = 0; 745ce83a025SChristoph Hellwig long status = 0; 746afc51aaaSDarrick J. Wong 747afc51aaaSDarrick J. Wong do { 748bc6123a8SMatthew Wilcox (Oracle) struct folio *folio; 749afc51aaaSDarrick J. Wong struct page *page; 750afc51aaaSDarrick J. Wong unsigned long offset; /* Offset into pagecache page */ 751afc51aaaSDarrick J. Wong unsigned long bytes; /* Bytes to write to page */ 752afc51aaaSDarrick J. Wong size_t copied; /* Bytes copied from user */ 753afc51aaaSDarrick J. Wong 754afc51aaaSDarrick J. Wong offset = offset_in_page(pos); 755afc51aaaSDarrick J. Wong bytes = min_t(unsigned long, PAGE_SIZE - offset, 756afc51aaaSDarrick J. Wong iov_iter_count(i)); 757afc51aaaSDarrick J. Wong again: 758afc51aaaSDarrick J. Wong if (bytes > length) 759afc51aaaSDarrick J. Wong bytes = length; 760afc51aaaSDarrick J. Wong 761afc51aaaSDarrick J. Wong /* 762f1f264b4SAndreas Gruenbacher * Bring in the user page that we'll copy from _first_. 763afc51aaaSDarrick J. Wong * Otherwise there's a nasty deadlock on copying from the 764afc51aaaSDarrick J. Wong * same page as we're writing to, without it being marked 765afc51aaaSDarrick J. Wong * up-to-date. 766afc51aaaSDarrick J. Wong */ 767631f871fSAndreas Gruenbacher if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) { 768afc51aaaSDarrick J. Wong status = -EFAULT; 769afc51aaaSDarrick J. Wong break; 770afc51aaaSDarrick J. Wong } 771afc51aaaSDarrick J. Wong 772bc6123a8SMatthew Wilcox (Oracle) status = iomap_write_begin(iter, pos, bytes, &folio); 773afc51aaaSDarrick J. Wong if (unlikely(status)) 774afc51aaaSDarrick J. Wong break; 775afc51aaaSDarrick J. Wong 776bc6123a8SMatthew Wilcox (Oracle) page = folio_file_page(folio, pos >> PAGE_SHIFT); 777ce83a025SChristoph Hellwig if (mapping_writably_mapped(iter->inode->i_mapping)) 778afc51aaaSDarrick J. Wong flush_dcache_page(page); 779afc51aaaSDarrick J. Wong 780f0b65f39SAl Viro copied = copy_page_from_iter_atomic(page, offset, bytes, i); 781afc51aaaSDarrick J. Wong 782bc6123a8SMatthew Wilcox (Oracle) status = iomap_write_end(iter, pos, bytes, copied, folio); 783afc51aaaSDarrick J. Wong 784f0b65f39SAl Viro if (unlikely(copied != status)) 785f0b65f39SAl Viro iov_iter_revert(i, copied - status); 786afc51aaaSDarrick J. Wong 787f0b65f39SAl Viro cond_resched(); 788bc1bb416SAl Viro if (unlikely(status == 0)) { 789afc51aaaSDarrick J. Wong /* 790bc1bb416SAl Viro * A short copy made iomap_write_end() reject the 791bc1bb416SAl Viro * thing entirely. Might be memory poisoning 792bc1bb416SAl Viro * halfway through, might be a race with munmap, 793bc1bb416SAl Viro * might be severe memory pressure. 794afc51aaaSDarrick J. Wong */ 795bc1bb416SAl Viro if (copied) 796bc1bb416SAl Viro bytes = copied; 797afc51aaaSDarrick J. Wong goto again; 798afc51aaaSDarrick J. Wong } 799f0b65f39SAl Viro pos += status; 800f0b65f39SAl Viro written += status; 801f0b65f39SAl Viro length -= status; 802afc51aaaSDarrick J. Wong 803ce83a025SChristoph Hellwig balance_dirty_pages_ratelimited(iter->inode->i_mapping); 804afc51aaaSDarrick J. Wong } while (iov_iter_count(i) && length); 805afc51aaaSDarrick J. Wong 806afc51aaaSDarrick J. Wong return written ? written : status; 807afc51aaaSDarrick J. Wong } 808afc51aaaSDarrick J. Wong 809afc51aaaSDarrick J. Wong ssize_t 810ce83a025SChristoph Hellwig iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, 811afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 812afc51aaaSDarrick J. Wong { 813ce83a025SChristoph Hellwig struct iomap_iter iter = { 814ce83a025SChristoph Hellwig .inode = iocb->ki_filp->f_mapping->host, 815ce83a025SChristoph Hellwig .pos = iocb->ki_pos, 816ce83a025SChristoph Hellwig .len = iov_iter_count(i), 817ce83a025SChristoph Hellwig .flags = IOMAP_WRITE, 818ce83a025SChristoph Hellwig }; 819ce83a025SChristoph Hellwig int ret; 820afc51aaaSDarrick J. Wong 821ce83a025SChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 822ce83a025SChristoph Hellwig iter.processed = iomap_write_iter(&iter, i); 823ce83a025SChristoph Hellwig if (iter.pos == iocb->ki_pos) 824ce83a025SChristoph Hellwig return ret; 825ce83a025SChristoph Hellwig return iter.pos - iocb->ki_pos; 826afc51aaaSDarrick J. Wong } 827afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_file_buffered_write); 828afc51aaaSDarrick J. Wong 8298fc274d1SChristoph Hellwig static loff_t iomap_unshare_iter(struct iomap_iter *iter) 830afc51aaaSDarrick J. Wong { 8318fc274d1SChristoph Hellwig struct iomap *iomap = &iter->iomap; 832fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 8338fc274d1SChristoph Hellwig loff_t pos = iter->pos; 8348fc274d1SChristoph Hellwig loff_t length = iomap_length(iter); 835afc51aaaSDarrick J. Wong long status = 0; 836d4ff3b2eSMatthew Wilcox (Oracle) loff_t written = 0; 837afc51aaaSDarrick J. Wong 8383590c4d8SChristoph Hellwig /* don't bother with blocks that are not shared to start with */ 8393590c4d8SChristoph Hellwig if (!(iomap->flags & IOMAP_F_SHARED)) 8403590c4d8SChristoph Hellwig return length; 8413590c4d8SChristoph Hellwig /* don't bother with holes or unwritten extents */ 842c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) 8433590c4d8SChristoph Hellwig return length; 8443590c4d8SChristoph Hellwig 845afc51aaaSDarrick J. Wong do { 84632a38a49SChristoph Hellwig unsigned long offset = offset_in_page(pos); 84732a38a49SChristoph Hellwig unsigned long bytes = min_t(loff_t, PAGE_SIZE - offset, length); 848bc6123a8SMatthew Wilcox (Oracle) struct folio *folio; 849afc51aaaSDarrick J. Wong 850bc6123a8SMatthew Wilcox (Oracle) status = iomap_write_begin(iter, pos, bytes, &folio); 851afc51aaaSDarrick J. Wong if (unlikely(status)) 852afc51aaaSDarrick J. Wong return status; 853afc51aaaSDarrick J. Wong 854bc6123a8SMatthew Wilcox (Oracle) status = iomap_write_end(iter, pos, bytes, bytes, folio); 855afc51aaaSDarrick J. Wong if (WARN_ON_ONCE(status == 0)) 856afc51aaaSDarrick J. Wong return -EIO; 857afc51aaaSDarrick J. Wong 858afc51aaaSDarrick J. Wong cond_resched(); 859afc51aaaSDarrick J. Wong 860afc51aaaSDarrick J. Wong pos += status; 861afc51aaaSDarrick J. Wong written += status; 862afc51aaaSDarrick J. Wong length -= status; 863afc51aaaSDarrick J. Wong 8648fc274d1SChristoph Hellwig balance_dirty_pages_ratelimited(iter->inode->i_mapping); 865afc51aaaSDarrick J. Wong } while (length); 866afc51aaaSDarrick J. Wong 867afc51aaaSDarrick J. Wong return written; 868afc51aaaSDarrick J. Wong } 869afc51aaaSDarrick J. Wong 870afc51aaaSDarrick J. Wong int 8713590c4d8SChristoph Hellwig iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, 872afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 873afc51aaaSDarrick J. Wong { 8748fc274d1SChristoph Hellwig struct iomap_iter iter = { 8758fc274d1SChristoph Hellwig .inode = inode, 8768fc274d1SChristoph Hellwig .pos = pos, 8778fc274d1SChristoph Hellwig .len = len, 878b74b1293SChristoph Hellwig .flags = IOMAP_WRITE | IOMAP_UNSHARE, 8798fc274d1SChristoph Hellwig }; 8808fc274d1SChristoph Hellwig int ret; 881afc51aaaSDarrick J. Wong 8828fc274d1SChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 8838fc274d1SChristoph Hellwig iter.processed = iomap_unshare_iter(&iter); 884afc51aaaSDarrick J. Wong return ret; 885afc51aaaSDarrick J. Wong } 8863590c4d8SChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_file_unshare); 887afc51aaaSDarrick J. Wong 8882aa3048eSChristoph Hellwig static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero) 889afc51aaaSDarrick J. Wong { 890fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 8912aa3048eSChristoph Hellwig loff_t pos = iter->pos; 8922aa3048eSChristoph Hellwig loff_t length = iomap_length(iter); 893afc51aaaSDarrick J. Wong loff_t written = 0; 894afc51aaaSDarrick J. Wong 895afc51aaaSDarrick J. Wong /* already zeroed? we're done. */ 896c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) 89781ee8e52SMatthew Wilcox (Oracle) return length; 898afc51aaaSDarrick J. Wong 899afc51aaaSDarrick J. Wong do { 9004d7bd0ebSMatthew Wilcox (Oracle) struct folio *folio; 9014d7bd0ebSMatthew Wilcox (Oracle) int status; 9024d7bd0ebSMatthew Wilcox (Oracle) size_t offset; 9034d7bd0ebSMatthew Wilcox (Oracle) size_t bytes = min_t(u64, SIZE_MAX, length); 904afc51aaaSDarrick J. Wong 9054d7bd0ebSMatthew Wilcox (Oracle) status = iomap_write_begin(iter, pos, bytes, &folio); 9064d7bd0ebSMatthew Wilcox (Oracle) if (status) 9074d7bd0ebSMatthew Wilcox (Oracle) return status; 9084d7bd0ebSMatthew Wilcox (Oracle) 9094d7bd0ebSMatthew Wilcox (Oracle) offset = offset_in_folio(folio, pos); 9104d7bd0ebSMatthew Wilcox (Oracle) if (bytes > folio_size(folio) - offset) 9114d7bd0ebSMatthew Wilcox (Oracle) bytes = folio_size(folio) - offset; 9124d7bd0ebSMatthew Wilcox (Oracle) 9134d7bd0ebSMatthew Wilcox (Oracle) folio_zero_range(folio, offset, bytes); 9144d7bd0ebSMatthew Wilcox (Oracle) folio_mark_accessed(folio); 9154d7bd0ebSMatthew Wilcox (Oracle) 9164d7bd0ebSMatthew Wilcox (Oracle) bytes = iomap_write_end(iter, pos, bytes, bytes, folio); 9174d7bd0ebSMatthew Wilcox (Oracle) if (WARN_ON_ONCE(bytes == 0)) 9184d7bd0ebSMatthew Wilcox (Oracle) return -EIO; 919afc51aaaSDarrick J. Wong 920afc51aaaSDarrick J. Wong pos += bytes; 92181ee8e52SMatthew Wilcox (Oracle) length -= bytes; 922afc51aaaSDarrick J. Wong written += bytes; 923afc51aaaSDarrick J. Wong if (did_zero) 924afc51aaaSDarrick J. Wong *did_zero = true; 92581ee8e52SMatthew Wilcox (Oracle) } while (length > 0); 926afc51aaaSDarrick J. Wong 927afc51aaaSDarrick J. Wong return written; 928afc51aaaSDarrick J. Wong } 929afc51aaaSDarrick J. Wong 930afc51aaaSDarrick J. Wong int 931afc51aaaSDarrick J. Wong iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, 932afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 933afc51aaaSDarrick J. Wong { 9342aa3048eSChristoph Hellwig struct iomap_iter iter = { 9352aa3048eSChristoph Hellwig .inode = inode, 9362aa3048eSChristoph Hellwig .pos = pos, 9372aa3048eSChristoph Hellwig .len = len, 9382aa3048eSChristoph Hellwig .flags = IOMAP_ZERO, 9392aa3048eSChristoph Hellwig }; 9402aa3048eSChristoph Hellwig int ret; 941afc51aaaSDarrick J. Wong 9422aa3048eSChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 9432aa3048eSChristoph Hellwig iter.processed = iomap_zero_iter(&iter, did_zero); 944afc51aaaSDarrick J. Wong return ret; 945afc51aaaSDarrick J. Wong } 946afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_zero_range); 947afc51aaaSDarrick J. Wong 948afc51aaaSDarrick J. Wong int 949afc51aaaSDarrick J. Wong iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, 950afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 951afc51aaaSDarrick J. Wong { 952afc51aaaSDarrick J. Wong unsigned int blocksize = i_blocksize(inode); 953afc51aaaSDarrick J. Wong unsigned int off = pos & (blocksize - 1); 954afc51aaaSDarrick J. Wong 955afc51aaaSDarrick J. Wong /* Block boundary? Nothing to do */ 956afc51aaaSDarrick J. Wong if (!off) 957afc51aaaSDarrick J. Wong return 0; 958afc51aaaSDarrick J. Wong return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops); 959afc51aaaSDarrick J. Wong } 960afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_truncate_page); 961afc51aaaSDarrick J. Wong 962ea0f843aSMatthew Wilcox (Oracle) static loff_t iomap_folio_mkwrite_iter(struct iomap_iter *iter, 963ea0f843aSMatthew Wilcox (Oracle) struct folio *folio) 964afc51aaaSDarrick J. Wong { 965253564baSChristoph Hellwig loff_t length = iomap_length(iter); 966afc51aaaSDarrick J. Wong int ret; 967afc51aaaSDarrick J. Wong 968253564baSChristoph Hellwig if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) { 969d1bd0b4eSMatthew Wilcox (Oracle) ret = __block_write_begin_int(folio, iter->pos, length, NULL, 970253564baSChristoph Hellwig &iter->iomap); 971afc51aaaSDarrick J. Wong if (ret) 972afc51aaaSDarrick J. Wong return ret; 973ea0f843aSMatthew Wilcox (Oracle) block_commit_write(&folio->page, 0, length); 974afc51aaaSDarrick J. Wong } else { 975ea0f843aSMatthew Wilcox (Oracle) WARN_ON_ONCE(!folio_test_uptodate(folio)); 976ea0f843aSMatthew Wilcox (Oracle) folio_mark_dirty(folio); 977afc51aaaSDarrick J. Wong } 978afc51aaaSDarrick J. Wong 979afc51aaaSDarrick J. Wong return length; 980afc51aaaSDarrick J. Wong } 981afc51aaaSDarrick J. Wong 982afc51aaaSDarrick J. Wong vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops) 983afc51aaaSDarrick J. Wong { 984253564baSChristoph Hellwig struct iomap_iter iter = { 985253564baSChristoph Hellwig .inode = file_inode(vmf->vma->vm_file), 986253564baSChristoph Hellwig .flags = IOMAP_WRITE | IOMAP_FAULT, 987253564baSChristoph Hellwig }; 988ea0f843aSMatthew Wilcox (Oracle) struct folio *folio = page_folio(vmf->page); 989afc51aaaSDarrick J. Wong ssize_t ret; 990afc51aaaSDarrick J. Wong 991ea0f843aSMatthew Wilcox (Oracle) folio_lock(folio); 992ea0f843aSMatthew Wilcox (Oracle) ret = folio_mkwrite_check_truncate(folio, iter.inode); 993243145bcSAndreas Gruenbacher if (ret < 0) 994afc51aaaSDarrick J. Wong goto out_unlock; 995ea0f843aSMatthew Wilcox (Oracle) iter.pos = folio_pos(folio); 996253564baSChristoph Hellwig iter.len = ret; 997253564baSChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 998ea0f843aSMatthew Wilcox (Oracle) iter.processed = iomap_folio_mkwrite_iter(&iter, folio); 999afc51aaaSDarrick J. Wong 1000253564baSChristoph Hellwig if (ret < 0) 1001afc51aaaSDarrick J. Wong goto out_unlock; 1002ea0f843aSMatthew Wilcox (Oracle) folio_wait_stable(folio); 1003afc51aaaSDarrick J. Wong return VM_FAULT_LOCKED; 1004afc51aaaSDarrick J. Wong out_unlock: 1005ea0f843aSMatthew Wilcox (Oracle) folio_unlock(folio); 1006afc51aaaSDarrick J. Wong return block_page_mkwrite_return(ret); 1007afc51aaaSDarrick J. Wong } 1008afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_page_mkwrite); 1009598ecfbaSChristoph Hellwig 10108ffd74e9SMatthew Wilcox (Oracle) static void iomap_finish_folio_write(struct inode *inode, struct folio *folio, 10118ffd74e9SMatthew Wilcox (Oracle) size_t len, int error) 1012598ecfbaSChristoph Hellwig { 101395c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 1014598ecfbaSChristoph Hellwig 1015598ecfbaSChristoph Hellwig if (error) { 10168ffd74e9SMatthew Wilcox (Oracle) folio_set_error(folio); 1017b69eea82SDarrick J. Wong mapping_set_error(inode->i_mapping, error); 1018598ecfbaSChristoph Hellwig } 1019598ecfbaSChristoph Hellwig 10208ffd74e9SMatthew Wilcox (Oracle) WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !iop); 10210fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) <= 0); 1022598ecfbaSChristoph Hellwig 10230fb2d720SMatthew Wilcox (Oracle) if (!iop || atomic_sub_and_test(len, &iop->write_bytes_pending)) 10248ffd74e9SMatthew Wilcox (Oracle) folio_end_writeback(folio); 1025598ecfbaSChristoph Hellwig } 1026598ecfbaSChristoph Hellwig 1027598ecfbaSChristoph Hellwig /* 1028598ecfbaSChristoph Hellwig * We're now finished for good with this ioend structure. Update the page 1029598ecfbaSChristoph Hellwig * state, release holds on bios, and finally free up memory. Do not use the 1030598ecfbaSChristoph Hellwig * ioend after this. 1031598ecfbaSChristoph Hellwig */ 1032ebb7fb15SDave Chinner static u32 1033598ecfbaSChristoph Hellwig iomap_finish_ioend(struct iomap_ioend *ioend, int error) 1034598ecfbaSChristoph Hellwig { 1035598ecfbaSChristoph Hellwig struct inode *inode = ioend->io_inode; 1036598ecfbaSChristoph Hellwig struct bio *bio = &ioend->io_inline_bio; 1037598ecfbaSChristoph Hellwig struct bio *last = ioend->io_bio, *next; 1038598ecfbaSChristoph Hellwig u64 start = bio->bi_iter.bi_sector; 1039c275779fSZorro Lang loff_t offset = ioend->io_offset; 1040598ecfbaSChristoph Hellwig bool quiet = bio_flagged(bio, BIO_QUIET); 1041ebb7fb15SDave Chinner u32 folio_count = 0; 1042598ecfbaSChristoph Hellwig 1043598ecfbaSChristoph Hellwig for (bio = &ioend->io_inline_bio; bio; bio = next) { 10448ffd74e9SMatthew Wilcox (Oracle) struct folio_iter fi; 1045598ecfbaSChristoph Hellwig 1046598ecfbaSChristoph Hellwig /* 1047598ecfbaSChristoph Hellwig * For the last bio, bi_private points to the ioend, so we 1048598ecfbaSChristoph Hellwig * need to explicitly end the iteration here. 1049598ecfbaSChristoph Hellwig */ 1050598ecfbaSChristoph Hellwig if (bio == last) 1051598ecfbaSChristoph Hellwig next = NULL; 1052598ecfbaSChristoph Hellwig else 1053598ecfbaSChristoph Hellwig next = bio->bi_private; 1054598ecfbaSChristoph Hellwig 10558ffd74e9SMatthew Wilcox (Oracle) /* walk all folios in bio, ending page IO on them */ 1056ebb7fb15SDave Chinner bio_for_each_folio_all(fi, bio) { 10578ffd74e9SMatthew Wilcox (Oracle) iomap_finish_folio_write(inode, fi.folio, fi.length, 10588ffd74e9SMatthew Wilcox (Oracle) error); 1059ebb7fb15SDave Chinner folio_count++; 1060ebb7fb15SDave Chinner } 1061598ecfbaSChristoph Hellwig bio_put(bio); 1062598ecfbaSChristoph Hellwig } 1063c275779fSZorro Lang /* The ioend has been freed by bio_put() */ 1064598ecfbaSChristoph Hellwig 1065598ecfbaSChristoph Hellwig if (unlikely(error && !quiet)) { 1066598ecfbaSChristoph Hellwig printk_ratelimited(KERN_ERR 10679cd0ed63SDarrick J. Wong "%s: writeback error on inode %lu, offset %lld, sector %llu", 1068c275779fSZorro Lang inode->i_sb->s_id, inode->i_ino, offset, start); 1069598ecfbaSChristoph Hellwig } 1070ebb7fb15SDave Chinner return folio_count; 1071598ecfbaSChristoph Hellwig } 1072598ecfbaSChristoph Hellwig 1073ebb7fb15SDave Chinner /* 1074ebb7fb15SDave Chinner * Ioend completion routine for merged bios. This can only be called from task 1075ebb7fb15SDave Chinner * contexts as merged ioends can be of unbound length. Hence we have to break up 1076ebb7fb15SDave Chinner * the writeback completions into manageable chunks to avoid long scheduler 1077ebb7fb15SDave Chinner * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get 1078ebb7fb15SDave Chinner * good batch processing throughput without creating adverse scheduler latency 1079ebb7fb15SDave Chinner * conditions. 1080ebb7fb15SDave Chinner */ 1081598ecfbaSChristoph Hellwig void 1082598ecfbaSChristoph Hellwig iomap_finish_ioends(struct iomap_ioend *ioend, int error) 1083598ecfbaSChristoph Hellwig { 1084598ecfbaSChristoph Hellwig struct list_head tmp; 1085ebb7fb15SDave Chinner u32 completions; 1086ebb7fb15SDave Chinner 1087ebb7fb15SDave Chinner might_sleep(); 1088598ecfbaSChristoph Hellwig 1089598ecfbaSChristoph Hellwig list_replace_init(&ioend->io_list, &tmp); 1090ebb7fb15SDave Chinner completions = iomap_finish_ioend(ioend, error); 1091598ecfbaSChristoph Hellwig 1092598ecfbaSChristoph Hellwig while (!list_empty(&tmp)) { 1093ebb7fb15SDave Chinner if (completions > IOEND_BATCH_SIZE * 8) { 1094ebb7fb15SDave Chinner cond_resched(); 1095ebb7fb15SDave Chinner completions = 0; 1096ebb7fb15SDave Chinner } 1097598ecfbaSChristoph Hellwig ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); 1098598ecfbaSChristoph Hellwig list_del_init(&ioend->io_list); 1099ebb7fb15SDave Chinner completions += iomap_finish_ioend(ioend, error); 1100598ecfbaSChristoph Hellwig } 1101598ecfbaSChristoph Hellwig } 1102598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_finish_ioends); 1103598ecfbaSChristoph Hellwig 1104598ecfbaSChristoph Hellwig /* 1105598ecfbaSChristoph Hellwig * We can merge two adjacent ioends if they have the same set of work to do. 1106598ecfbaSChristoph Hellwig */ 1107598ecfbaSChristoph Hellwig static bool 1108598ecfbaSChristoph Hellwig iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next) 1109598ecfbaSChristoph Hellwig { 1110598ecfbaSChristoph Hellwig if (ioend->io_bio->bi_status != next->io_bio->bi_status) 1111598ecfbaSChristoph Hellwig return false; 1112598ecfbaSChristoph Hellwig if ((ioend->io_flags & IOMAP_F_SHARED) ^ 1113598ecfbaSChristoph Hellwig (next->io_flags & IOMAP_F_SHARED)) 1114598ecfbaSChristoph Hellwig return false; 1115598ecfbaSChristoph Hellwig if ((ioend->io_type == IOMAP_UNWRITTEN) ^ 1116598ecfbaSChristoph Hellwig (next->io_type == IOMAP_UNWRITTEN)) 1117598ecfbaSChristoph Hellwig return false; 1118598ecfbaSChristoph Hellwig if (ioend->io_offset + ioend->io_size != next->io_offset) 1119598ecfbaSChristoph Hellwig return false; 1120ebb7fb15SDave Chinner /* 1121ebb7fb15SDave Chinner * Do not merge physically discontiguous ioends. The filesystem 1122ebb7fb15SDave Chinner * completion functions will have to iterate the physical 1123ebb7fb15SDave Chinner * discontiguities even if we merge the ioends at a logical level, so 1124ebb7fb15SDave Chinner * we don't gain anything by merging physical discontiguities here. 1125ebb7fb15SDave Chinner * 1126ebb7fb15SDave Chinner * We cannot use bio->bi_iter.bi_sector here as it is modified during 1127ebb7fb15SDave Chinner * submission so does not point to the start sector of the bio at 1128ebb7fb15SDave Chinner * completion. 1129ebb7fb15SDave Chinner */ 1130ebb7fb15SDave Chinner if (ioend->io_sector + (ioend->io_size >> 9) != next->io_sector) 1131ebb7fb15SDave Chinner return false; 1132598ecfbaSChristoph Hellwig return true; 1133598ecfbaSChristoph Hellwig } 1134598ecfbaSChristoph Hellwig 1135598ecfbaSChristoph Hellwig void 11366e552494SBrian Foster iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends) 1137598ecfbaSChristoph Hellwig { 1138598ecfbaSChristoph Hellwig struct iomap_ioend *next; 1139598ecfbaSChristoph Hellwig 1140598ecfbaSChristoph Hellwig INIT_LIST_HEAD(&ioend->io_list); 1141598ecfbaSChristoph Hellwig 1142598ecfbaSChristoph Hellwig while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, 1143598ecfbaSChristoph Hellwig io_list))) { 1144598ecfbaSChristoph Hellwig if (!iomap_ioend_can_merge(ioend, next)) 1145598ecfbaSChristoph Hellwig break; 1146598ecfbaSChristoph Hellwig list_move_tail(&next->io_list, &ioend->io_list); 1147598ecfbaSChristoph Hellwig ioend->io_size += next->io_size; 1148598ecfbaSChristoph Hellwig } 1149598ecfbaSChristoph Hellwig } 1150598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); 1151598ecfbaSChristoph Hellwig 1152598ecfbaSChristoph Hellwig static int 11534f0f586bSSami Tolvanen iomap_ioend_compare(void *priv, const struct list_head *a, 11544f0f586bSSami Tolvanen const struct list_head *b) 1155598ecfbaSChristoph Hellwig { 1156b3d423ecSChristoph Hellwig struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); 1157b3d423ecSChristoph Hellwig struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); 1158598ecfbaSChristoph Hellwig 1159598ecfbaSChristoph Hellwig if (ia->io_offset < ib->io_offset) 1160598ecfbaSChristoph Hellwig return -1; 1161b3d423ecSChristoph Hellwig if (ia->io_offset > ib->io_offset) 1162598ecfbaSChristoph Hellwig return 1; 1163598ecfbaSChristoph Hellwig return 0; 1164598ecfbaSChristoph Hellwig } 1165598ecfbaSChristoph Hellwig 1166598ecfbaSChristoph Hellwig void 1167598ecfbaSChristoph Hellwig iomap_sort_ioends(struct list_head *ioend_list) 1168598ecfbaSChristoph Hellwig { 1169598ecfbaSChristoph Hellwig list_sort(NULL, ioend_list, iomap_ioend_compare); 1170598ecfbaSChristoph Hellwig } 1171598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_sort_ioends); 1172598ecfbaSChristoph Hellwig 1173598ecfbaSChristoph Hellwig static void iomap_writepage_end_bio(struct bio *bio) 1174598ecfbaSChristoph Hellwig { 1175598ecfbaSChristoph Hellwig struct iomap_ioend *ioend = bio->bi_private; 1176598ecfbaSChristoph Hellwig 1177598ecfbaSChristoph Hellwig iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status)); 1178598ecfbaSChristoph Hellwig } 1179598ecfbaSChristoph Hellwig 1180598ecfbaSChristoph Hellwig /* 1181598ecfbaSChristoph Hellwig * Submit the final bio for an ioend. 1182598ecfbaSChristoph Hellwig * 1183598ecfbaSChristoph Hellwig * If @error is non-zero, it means that we have a situation where some part of 1184f1f264b4SAndreas Gruenbacher * the submission process has failed after we've marked pages for writeback 1185598ecfbaSChristoph Hellwig * and unlocked them. In this situation, we need to fail the bio instead of 1186598ecfbaSChristoph Hellwig * submitting it. This typically only happens on a filesystem shutdown. 1187598ecfbaSChristoph Hellwig */ 1188598ecfbaSChristoph Hellwig static int 1189598ecfbaSChristoph Hellwig iomap_submit_ioend(struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend, 1190598ecfbaSChristoph Hellwig int error) 1191598ecfbaSChristoph Hellwig { 1192598ecfbaSChristoph Hellwig ioend->io_bio->bi_private = ioend; 1193598ecfbaSChristoph Hellwig ioend->io_bio->bi_end_io = iomap_writepage_end_bio; 1194598ecfbaSChristoph Hellwig 1195598ecfbaSChristoph Hellwig if (wpc->ops->prepare_ioend) 1196598ecfbaSChristoph Hellwig error = wpc->ops->prepare_ioend(ioend, error); 1197598ecfbaSChristoph Hellwig if (error) { 1198598ecfbaSChristoph Hellwig /* 1199f1f264b4SAndreas Gruenbacher * If we're failing the IO now, just mark the ioend with an 1200598ecfbaSChristoph Hellwig * error and finish it. This will run IO completion immediately 1201598ecfbaSChristoph Hellwig * as there is only one reference to the ioend at this point in 1202598ecfbaSChristoph Hellwig * time. 1203598ecfbaSChristoph Hellwig */ 1204598ecfbaSChristoph Hellwig ioend->io_bio->bi_status = errno_to_blk_status(error); 1205598ecfbaSChristoph Hellwig bio_endio(ioend->io_bio); 1206598ecfbaSChristoph Hellwig return error; 1207598ecfbaSChristoph Hellwig } 1208598ecfbaSChristoph Hellwig 1209598ecfbaSChristoph Hellwig submit_bio(ioend->io_bio); 1210598ecfbaSChristoph Hellwig return 0; 1211598ecfbaSChristoph Hellwig } 1212598ecfbaSChristoph Hellwig 1213598ecfbaSChristoph Hellwig static struct iomap_ioend * 1214598ecfbaSChristoph Hellwig iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc, 1215598ecfbaSChristoph Hellwig loff_t offset, sector_t sector, struct writeback_control *wbc) 1216598ecfbaSChristoph Hellwig { 1217598ecfbaSChristoph Hellwig struct iomap_ioend *ioend; 1218598ecfbaSChristoph Hellwig struct bio *bio; 1219598ecfbaSChristoph Hellwig 1220609be106SChristoph Hellwig bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS, 1221609be106SChristoph Hellwig REQ_OP_WRITE | wbc_to_write_flags(wbc), 1222609be106SChristoph Hellwig GFP_NOFS, &iomap_ioend_bioset); 1223598ecfbaSChristoph Hellwig bio->bi_iter.bi_sector = sector; 1224598ecfbaSChristoph Hellwig wbc_init_bio(wbc, bio); 1225598ecfbaSChristoph Hellwig 1226598ecfbaSChristoph Hellwig ioend = container_of(bio, struct iomap_ioend, io_inline_bio); 1227598ecfbaSChristoph Hellwig INIT_LIST_HEAD(&ioend->io_list); 1228598ecfbaSChristoph Hellwig ioend->io_type = wpc->iomap.type; 1229598ecfbaSChristoph Hellwig ioend->io_flags = wpc->iomap.flags; 1230598ecfbaSChristoph Hellwig ioend->io_inode = inode; 1231598ecfbaSChristoph Hellwig ioend->io_size = 0; 1232ebb7fb15SDave Chinner ioend->io_folios = 0; 1233598ecfbaSChristoph Hellwig ioend->io_offset = offset; 1234598ecfbaSChristoph Hellwig ioend->io_bio = bio; 1235ebb7fb15SDave Chinner ioend->io_sector = sector; 1236598ecfbaSChristoph Hellwig return ioend; 1237598ecfbaSChristoph Hellwig } 1238598ecfbaSChristoph Hellwig 1239598ecfbaSChristoph Hellwig /* 1240598ecfbaSChristoph Hellwig * Allocate a new bio, and chain the old bio to the new one. 1241598ecfbaSChristoph Hellwig * 1242f1f264b4SAndreas Gruenbacher * Note that we have to perform the chaining in this unintuitive order 1243598ecfbaSChristoph Hellwig * so that the bi_private linkage is set up in the right direction for the 1244598ecfbaSChristoph Hellwig * traversal in iomap_finish_ioend(). 1245598ecfbaSChristoph Hellwig */ 1246598ecfbaSChristoph Hellwig static struct bio * 1247598ecfbaSChristoph Hellwig iomap_chain_bio(struct bio *prev) 1248598ecfbaSChristoph Hellwig { 1249598ecfbaSChristoph Hellwig struct bio *new; 1250598ecfbaSChristoph Hellwig 125107888c66SChristoph Hellwig new = bio_alloc(prev->bi_bdev, BIO_MAX_VECS, prev->bi_opf, GFP_NOFS); 125207888c66SChristoph Hellwig bio_clone_blkg_association(new, prev); 1253598ecfbaSChristoph Hellwig new->bi_iter.bi_sector = bio_end_sector(prev); 1254598ecfbaSChristoph Hellwig 1255598ecfbaSChristoph Hellwig bio_chain(prev, new); 1256598ecfbaSChristoph Hellwig bio_get(prev); /* for iomap_finish_ioend */ 1257598ecfbaSChristoph Hellwig submit_bio(prev); 1258598ecfbaSChristoph Hellwig return new; 1259598ecfbaSChristoph Hellwig } 1260598ecfbaSChristoph Hellwig 1261598ecfbaSChristoph Hellwig static bool 1262598ecfbaSChristoph Hellwig iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset, 1263598ecfbaSChristoph Hellwig sector_t sector) 1264598ecfbaSChristoph Hellwig { 1265598ecfbaSChristoph Hellwig if ((wpc->iomap.flags & IOMAP_F_SHARED) != 1266598ecfbaSChristoph Hellwig (wpc->ioend->io_flags & IOMAP_F_SHARED)) 1267598ecfbaSChristoph Hellwig return false; 1268598ecfbaSChristoph Hellwig if (wpc->iomap.type != wpc->ioend->io_type) 1269598ecfbaSChristoph Hellwig return false; 1270598ecfbaSChristoph Hellwig if (offset != wpc->ioend->io_offset + wpc->ioend->io_size) 1271598ecfbaSChristoph Hellwig return false; 1272598ecfbaSChristoph Hellwig if (sector != bio_end_sector(wpc->ioend->io_bio)) 1273598ecfbaSChristoph Hellwig return false; 1274ebb7fb15SDave Chinner /* 1275ebb7fb15SDave Chinner * Limit ioend bio chain lengths to minimise IO completion latency. This 1276ebb7fb15SDave Chinner * also prevents long tight loops ending page writeback on all the 1277ebb7fb15SDave Chinner * folios in the ioend. 1278ebb7fb15SDave Chinner */ 1279ebb7fb15SDave Chinner if (wpc->ioend->io_folios >= IOEND_BATCH_SIZE) 1280ebb7fb15SDave Chinner return false; 1281598ecfbaSChristoph Hellwig return true; 1282598ecfbaSChristoph Hellwig } 1283598ecfbaSChristoph Hellwig 1284598ecfbaSChristoph Hellwig /* 1285598ecfbaSChristoph Hellwig * Test to see if we have an existing ioend structure that we could append to 1286f1f264b4SAndreas Gruenbacher * first; otherwise finish off the current ioend and start another. 1287598ecfbaSChristoph Hellwig */ 1288598ecfbaSChristoph Hellwig static void 1289e735c007SMatthew Wilcox (Oracle) iomap_add_to_ioend(struct inode *inode, loff_t pos, struct folio *folio, 1290598ecfbaSChristoph Hellwig struct iomap_page *iop, struct iomap_writepage_ctx *wpc, 1291598ecfbaSChristoph Hellwig struct writeback_control *wbc, struct list_head *iolist) 1292598ecfbaSChristoph Hellwig { 1293e735c007SMatthew Wilcox (Oracle) sector_t sector = iomap_sector(&wpc->iomap, pos); 1294598ecfbaSChristoph Hellwig unsigned len = i_blocksize(inode); 1295e735c007SMatthew Wilcox (Oracle) size_t poff = offset_in_folio(folio, pos); 1296598ecfbaSChristoph Hellwig 1297e735c007SMatthew Wilcox (Oracle) if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, pos, sector)) { 1298598ecfbaSChristoph Hellwig if (wpc->ioend) 1299598ecfbaSChristoph Hellwig list_add(&wpc->ioend->io_list, iolist); 1300e735c007SMatthew Wilcox (Oracle) wpc->ioend = iomap_alloc_ioend(inode, wpc, pos, sector, wbc); 1301598ecfbaSChristoph Hellwig } 1302598ecfbaSChristoph Hellwig 1303e735c007SMatthew Wilcox (Oracle) if (!bio_add_folio(wpc->ioend->io_bio, folio, len, poff)) { 1304c1b79f11SChristoph Hellwig wpc->ioend->io_bio = iomap_chain_bio(wpc->ioend->io_bio); 1305e735c007SMatthew Wilcox (Oracle) bio_add_folio(wpc->ioend->io_bio, folio, len, poff); 1306c1b79f11SChristoph Hellwig } 1307c1b79f11SChristoph Hellwig 13080fb2d720SMatthew Wilcox (Oracle) if (iop) 13090fb2d720SMatthew Wilcox (Oracle) atomic_add(len, &iop->write_bytes_pending); 1310598ecfbaSChristoph Hellwig wpc->ioend->io_size += len; 1311e735c007SMatthew Wilcox (Oracle) wbc_account_cgroup_owner(wbc, &folio->page, len); 1312598ecfbaSChristoph Hellwig } 1313598ecfbaSChristoph Hellwig 1314598ecfbaSChristoph Hellwig /* 1315598ecfbaSChristoph Hellwig * We implement an immediate ioend submission policy here to avoid needing to 1316598ecfbaSChristoph Hellwig * chain multiple ioends and hence nest mempool allocations which can violate 1317f1f264b4SAndreas Gruenbacher * the forward progress guarantees we need to provide. The current ioend we're 1318f1f264b4SAndreas Gruenbacher * adding blocks to is cached in the writepage context, and if the new block 1319f1f264b4SAndreas Gruenbacher * doesn't append to the cached ioend, it will create a new ioend and cache that 1320598ecfbaSChristoph Hellwig * instead. 1321598ecfbaSChristoph Hellwig * 1322598ecfbaSChristoph Hellwig * If a new ioend is created and cached, the old ioend is returned and queued 1323598ecfbaSChristoph Hellwig * locally for submission once the entire page is processed or an error has been 1324598ecfbaSChristoph Hellwig * detected. While ioends are submitted immediately after they are completed, 1325598ecfbaSChristoph Hellwig * batching optimisations are provided by higher level block plugging. 1326598ecfbaSChristoph Hellwig * 1327598ecfbaSChristoph Hellwig * At the end of a writeback pass, there will be a cached ioend remaining on the 1328598ecfbaSChristoph Hellwig * writepage context that the caller will need to submit. 1329598ecfbaSChristoph Hellwig */ 1330598ecfbaSChristoph Hellwig static int 1331598ecfbaSChristoph Hellwig iomap_writepage_map(struct iomap_writepage_ctx *wpc, 1332598ecfbaSChristoph Hellwig struct writeback_control *wbc, struct inode *inode, 1333e735c007SMatthew Wilcox (Oracle) struct folio *folio, u64 end_pos) 1334598ecfbaSChristoph Hellwig { 1335435d44b3SMatthew Wilcox (Oracle) struct iomap_page *iop = iomap_page_create(inode, folio); 1336598ecfbaSChristoph Hellwig struct iomap_ioend *ioend, *next; 1337598ecfbaSChristoph Hellwig unsigned len = i_blocksize(inode); 133892655036SMatthew Wilcox (Oracle) unsigned nblocks = i_blocks_per_folio(inode, folio); 133992655036SMatthew Wilcox (Oracle) u64 pos = folio_pos(folio); 1340598ecfbaSChristoph Hellwig int error = 0, count = 0, i; 1341598ecfbaSChristoph Hellwig LIST_HEAD(submit_list); 1342598ecfbaSChristoph Hellwig 13430fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) != 0); 1344598ecfbaSChristoph Hellwig 1345598ecfbaSChristoph Hellwig /* 134692655036SMatthew Wilcox (Oracle) * Walk through the folio to find areas to write back. If we 134792655036SMatthew Wilcox (Oracle) * run off the end of the current map or find the current map 134892655036SMatthew Wilcox (Oracle) * invalid, grab a new one. 1349598ecfbaSChristoph Hellwig */ 135092655036SMatthew Wilcox (Oracle) for (i = 0; i < nblocks && pos < end_pos; i++, pos += len) { 1351598ecfbaSChristoph Hellwig if (iop && !test_bit(i, iop->uptodate)) 1352598ecfbaSChristoph Hellwig continue; 1353598ecfbaSChristoph Hellwig 135492655036SMatthew Wilcox (Oracle) error = wpc->ops->map_blocks(wpc, inode, pos); 1355598ecfbaSChristoph Hellwig if (error) 1356598ecfbaSChristoph Hellwig break; 13573e19e6f3SChristoph Hellwig if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE)) 13583e19e6f3SChristoph Hellwig continue; 1359598ecfbaSChristoph Hellwig if (wpc->iomap.type == IOMAP_HOLE) 1360598ecfbaSChristoph Hellwig continue; 1361e735c007SMatthew Wilcox (Oracle) iomap_add_to_ioend(inode, pos, folio, iop, wpc, wbc, 1362598ecfbaSChristoph Hellwig &submit_list); 1363598ecfbaSChristoph Hellwig count++; 1364598ecfbaSChristoph Hellwig } 1365ebb7fb15SDave Chinner if (count) 1366ebb7fb15SDave Chinner wpc->ioend->io_folios++; 1367598ecfbaSChristoph Hellwig 1368598ecfbaSChristoph Hellwig WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list)); 1369e735c007SMatthew Wilcox (Oracle) WARN_ON_ONCE(!folio_test_locked(folio)); 1370e735c007SMatthew Wilcox (Oracle) WARN_ON_ONCE(folio_test_writeback(folio)); 1371e735c007SMatthew Wilcox (Oracle) WARN_ON_ONCE(folio_test_dirty(folio)); 1372598ecfbaSChristoph Hellwig 1373598ecfbaSChristoph Hellwig /* 1374598ecfbaSChristoph Hellwig * We cannot cancel the ioend directly here on error. We may have 1375598ecfbaSChristoph Hellwig * already set other pages under writeback and hence we have to run I/O 1376598ecfbaSChristoph Hellwig * completion to mark the error state of the pages under writeback 1377598ecfbaSChristoph Hellwig * appropriately. 1378598ecfbaSChristoph Hellwig */ 1379598ecfbaSChristoph Hellwig if (unlikely(error)) { 1380598ecfbaSChristoph Hellwig /* 1381763e4cdcSBrian Foster * Let the filesystem know what portion of the current page 1382f1f264b4SAndreas Gruenbacher * failed to map. If the page hasn't been added to ioend, it 1383763e4cdcSBrian Foster * won't be affected by I/O completion and we must unlock it 1384763e4cdcSBrian Foster * now. 1385598ecfbaSChristoph Hellwig */ 13866e478521SMatthew Wilcox (Oracle) if (wpc->ops->discard_folio) 138792655036SMatthew Wilcox (Oracle) wpc->ops->discard_folio(folio, pos); 1388763e4cdcSBrian Foster if (!count) { 1389e735c007SMatthew Wilcox (Oracle) folio_clear_uptodate(folio); 1390e735c007SMatthew Wilcox (Oracle) folio_unlock(folio); 1391598ecfbaSChristoph Hellwig goto done; 1392598ecfbaSChristoph Hellwig } 1393598ecfbaSChristoph Hellwig } 1394598ecfbaSChristoph Hellwig 1395e735c007SMatthew Wilcox (Oracle) folio_start_writeback(folio); 1396e735c007SMatthew Wilcox (Oracle) folio_unlock(folio); 1397598ecfbaSChristoph Hellwig 1398598ecfbaSChristoph Hellwig /* 1399f1f264b4SAndreas Gruenbacher * Preserve the original error if there was one; catch 1400598ecfbaSChristoph Hellwig * submission errors here and propagate into subsequent ioend 1401598ecfbaSChristoph Hellwig * submissions. 1402598ecfbaSChristoph Hellwig */ 1403598ecfbaSChristoph Hellwig list_for_each_entry_safe(ioend, next, &submit_list, io_list) { 1404598ecfbaSChristoph Hellwig int error2; 1405598ecfbaSChristoph Hellwig 1406598ecfbaSChristoph Hellwig list_del_init(&ioend->io_list); 1407598ecfbaSChristoph Hellwig error2 = iomap_submit_ioend(wpc, ioend, error); 1408598ecfbaSChristoph Hellwig if (error2 && !error) 1409598ecfbaSChristoph Hellwig error = error2; 1410598ecfbaSChristoph Hellwig } 1411598ecfbaSChristoph Hellwig 1412598ecfbaSChristoph Hellwig /* 1413598ecfbaSChristoph Hellwig * We can end up here with no error and nothing to write only if we race 1414598ecfbaSChristoph Hellwig * with a partial page truncate on a sub-page block sized filesystem. 1415598ecfbaSChristoph Hellwig */ 1416598ecfbaSChristoph Hellwig if (!count) 1417e735c007SMatthew Wilcox (Oracle) folio_end_writeback(folio); 1418598ecfbaSChristoph Hellwig done: 1419e735c007SMatthew Wilcox (Oracle) mapping_set_error(folio->mapping, error); 1420598ecfbaSChristoph Hellwig return error; 1421598ecfbaSChristoph Hellwig } 1422598ecfbaSChristoph Hellwig 1423598ecfbaSChristoph Hellwig /* 1424598ecfbaSChristoph Hellwig * Write out a dirty page. 1425598ecfbaSChristoph Hellwig * 1426f1f264b4SAndreas Gruenbacher * For delalloc space on the page, we need to allocate space and flush it. 1427f1f264b4SAndreas Gruenbacher * For unwritten space on the page, we need to start the conversion to 1428598ecfbaSChristoph Hellwig * regular allocated space. 1429598ecfbaSChristoph Hellwig */ 1430598ecfbaSChristoph Hellwig static int 1431598ecfbaSChristoph Hellwig iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data) 1432598ecfbaSChristoph Hellwig { 1433e735c007SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 1434598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc = data; 1435e735c007SMatthew Wilcox (Oracle) struct inode *inode = folio->mapping->host; 143681d4782aSMatthew Wilcox (Oracle) u64 end_pos, isize; 1437598ecfbaSChristoph Hellwig 1438e735c007SMatthew Wilcox (Oracle) trace_iomap_writepage(inode, folio_pos(folio), folio_size(folio)); 1439598ecfbaSChristoph Hellwig 1440598ecfbaSChristoph Hellwig /* 1441e735c007SMatthew Wilcox (Oracle) * Refuse to write the folio out if we're called from reclaim context. 1442598ecfbaSChristoph Hellwig * 1443598ecfbaSChristoph Hellwig * This avoids stack overflows when called from deeply used stacks in 1444598ecfbaSChristoph Hellwig * random callers for direct reclaim or memcg reclaim. We explicitly 1445598ecfbaSChristoph Hellwig * allow reclaim from kswapd as the stack usage there is relatively low. 1446598ecfbaSChristoph Hellwig * 1447598ecfbaSChristoph Hellwig * This should never happen except in the case of a VM regression so 1448598ecfbaSChristoph Hellwig * warn about it. 1449598ecfbaSChristoph Hellwig */ 1450598ecfbaSChristoph Hellwig if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == 1451598ecfbaSChristoph Hellwig PF_MEMALLOC)) 1452598ecfbaSChristoph Hellwig goto redirty; 1453598ecfbaSChristoph Hellwig 1454598ecfbaSChristoph Hellwig /* 1455e735c007SMatthew Wilcox (Oracle) * Is this folio beyond the end of the file? 1456598ecfbaSChristoph Hellwig * 1457e735c007SMatthew Wilcox (Oracle) * The folio index is less than the end_index, adjust the end_pos 1458e735c007SMatthew Wilcox (Oracle) * to the highest offset that this folio should represent. 1459598ecfbaSChristoph Hellwig * ----------------------------------------------------- 1460598ecfbaSChristoph Hellwig * | file mapping | <EOF> | 1461598ecfbaSChristoph Hellwig * ----------------------------------------------------- 1462598ecfbaSChristoph Hellwig * | Page ... | Page N-2 | Page N-1 | Page N | | 1463598ecfbaSChristoph Hellwig * ^--------------------------------^----------|-------- 1464598ecfbaSChristoph Hellwig * | desired writeback range | see else | 1465598ecfbaSChristoph Hellwig * ---------------------------------^------------------| 1466598ecfbaSChristoph Hellwig */ 146781d4782aSMatthew Wilcox (Oracle) isize = i_size_read(inode); 1468e735c007SMatthew Wilcox (Oracle) end_pos = folio_pos(folio) + folio_size(folio); 146981d4782aSMatthew Wilcox (Oracle) if (end_pos > isize) { 1470598ecfbaSChristoph Hellwig /* 1471598ecfbaSChristoph Hellwig * Check whether the page to write out is beyond or straddles 1472598ecfbaSChristoph Hellwig * i_size or not. 1473598ecfbaSChristoph Hellwig * ------------------------------------------------------- 1474598ecfbaSChristoph Hellwig * | file mapping | <EOF> | 1475598ecfbaSChristoph Hellwig * ------------------------------------------------------- 1476598ecfbaSChristoph Hellwig * | Page ... | Page N-2 | Page N-1 | Page N | Beyond | 1477598ecfbaSChristoph Hellwig * ^--------------------------------^-----------|--------- 1478598ecfbaSChristoph Hellwig * | | Straddles | 1479598ecfbaSChristoph Hellwig * ---------------------------------^-----------|--------| 1480598ecfbaSChristoph Hellwig */ 1481e735c007SMatthew Wilcox (Oracle) size_t poff = offset_in_folio(folio, isize); 148281d4782aSMatthew Wilcox (Oracle) pgoff_t end_index = isize >> PAGE_SHIFT; 1483598ecfbaSChristoph Hellwig 1484598ecfbaSChristoph Hellwig /* 1485f1f264b4SAndreas Gruenbacher * Skip the page if it's fully outside i_size, e.g. due to a 1486f1f264b4SAndreas Gruenbacher * truncate operation that's in progress. We must redirty the 1487598ecfbaSChristoph Hellwig * page so that reclaim stops reclaiming it. Otherwise 1488598ecfbaSChristoph Hellwig * iomap_vm_releasepage() is called on it and gets confused. 1489598ecfbaSChristoph Hellwig * 1490f1f264b4SAndreas Gruenbacher * Note that the end_index is unsigned long. If the given 1491f1f264b4SAndreas Gruenbacher * offset is greater than 16TB on a 32-bit system then if we 1492f1f264b4SAndreas Gruenbacher * checked if the page is fully outside i_size with 1493f1f264b4SAndreas Gruenbacher * "if (page->index >= end_index + 1)", "end_index + 1" would 1494f1f264b4SAndreas Gruenbacher * overflow and evaluate to 0. Hence this page would be 1495f1f264b4SAndreas Gruenbacher * redirtied and written out repeatedly, which would result in 1496f1f264b4SAndreas Gruenbacher * an infinite loop; the user program performing this operation 1497f1f264b4SAndreas Gruenbacher * would hang. Instead, we can detect this situation by 1498f1f264b4SAndreas Gruenbacher * checking if the page is totally beyond i_size or if its 1499598ecfbaSChristoph Hellwig * offset is just equal to the EOF. 1500598ecfbaSChristoph Hellwig */ 1501e735c007SMatthew Wilcox (Oracle) if (folio->index > end_index || 1502e735c007SMatthew Wilcox (Oracle) (folio->index == end_index && poff == 0)) 1503598ecfbaSChristoph Hellwig goto redirty; 1504598ecfbaSChristoph Hellwig 1505598ecfbaSChristoph Hellwig /* 1506598ecfbaSChristoph Hellwig * The page straddles i_size. It must be zeroed out on each 1507598ecfbaSChristoph Hellwig * and every writepage invocation because it may be mmapped. 1508598ecfbaSChristoph Hellwig * "A file is mapped in multiples of the page size. For a file 1509598ecfbaSChristoph Hellwig * that is not a multiple of the page size, the remaining 1510598ecfbaSChristoph Hellwig * memory is zeroed when mapped, and writes to that region are 1511598ecfbaSChristoph Hellwig * not written out to the file." 1512598ecfbaSChristoph Hellwig */ 1513e735c007SMatthew Wilcox (Oracle) folio_zero_segment(folio, poff, folio_size(folio)); 151481d4782aSMatthew Wilcox (Oracle) end_pos = isize; 1515598ecfbaSChristoph Hellwig } 1516598ecfbaSChristoph Hellwig 1517e735c007SMatthew Wilcox (Oracle) return iomap_writepage_map(wpc, wbc, inode, folio, end_pos); 1518598ecfbaSChristoph Hellwig 1519598ecfbaSChristoph Hellwig redirty: 1520e735c007SMatthew Wilcox (Oracle) folio_redirty_for_writepage(wbc, folio); 1521e735c007SMatthew Wilcox (Oracle) folio_unlock(folio); 1522598ecfbaSChristoph Hellwig return 0; 1523598ecfbaSChristoph Hellwig } 1524598ecfbaSChristoph Hellwig 1525598ecfbaSChristoph Hellwig int 1526598ecfbaSChristoph Hellwig iomap_writepage(struct page *page, struct writeback_control *wbc, 1527598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc, 1528598ecfbaSChristoph Hellwig const struct iomap_writeback_ops *ops) 1529598ecfbaSChristoph Hellwig { 1530598ecfbaSChristoph Hellwig int ret; 1531598ecfbaSChristoph Hellwig 1532598ecfbaSChristoph Hellwig wpc->ops = ops; 1533598ecfbaSChristoph Hellwig ret = iomap_do_writepage(page, wbc, wpc); 1534598ecfbaSChristoph Hellwig if (!wpc->ioend) 1535598ecfbaSChristoph Hellwig return ret; 1536598ecfbaSChristoph Hellwig return iomap_submit_ioend(wpc, wpc->ioend, ret); 1537598ecfbaSChristoph Hellwig } 1538598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_writepage); 1539598ecfbaSChristoph Hellwig 1540598ecfbaSChristoph Hellwig int 1541598ecfbaSChristoph Hellwig iomap_writepages(struct address_space *mapping, struct writeback_control *wbc, 1542598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc, 1543598ecfbaSChristoph Hellwig const struct iomap_writeback_ops *ops) 1544598ecfbaSChristoph Hellwig { 1545598ecfbaSChristoph Hellwig int ret; 1546598ecfbaSChristoph Hellwig 1547598ecfbaSChristoph Hellwig wpc->ops = ops; 1548598ecfbaSChristoph Hellwig ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc); 1549598ecfbaSChristoph Hellwig if (!wpc->ioend) 1550598ecfbaSChristoph Hellwig return ret; 1551598ecfbaSChristoph Hellwig return iomap_submit_ioend(wpc, wpc->ioend, ret); 1552598ecfbaSChristoph Hellwig } 1553598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_writepages); 1554598ecfbaSChristoph Hellwig 1555598ecfbaSChristoph Hellwig static int __init iomap_init(void) 1556598ecfbaSChristoph Hellwig { 1557598ecfbaSChristoph Hellwig return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), 1558598ecfbaSChristoph Hellwig offsetof(struct iomap_ioend, io_inline_bio), 1559598ecfbaSChristoph Hellwig BIOSET_NEED_BVECS); 1560598ecfbaSChristoph Hellwig } 1561598ecfbaSChristoph Hellwig fs_initcall(iomap_init); 1562