1afc51aaaSDarrick J. Wong // SPDX-License-Identifier: GPL-2.0 2afc51aaaSDarrick J. Wong /* 3afc51aaaSDarrick J. Wong * Copyright (C) 2010 Red Hat, Inc. 4598ecfbaSChristoph Hellwig * Copyright (C) 2016-2019 Christoph Hellwig. 5afc51aaaSDarrick J. Wong */ 6afc51aaaSDarrick J. Wong #include <linux/module.h> 7afc51aaaSDarrick J. Wong #include <linux/compiler.h> 8afc51aaaSDarrick J. Wong #include <linux/fs.h> 9afc51aaaSDarrick J. Wong #include <linux/iomap.h> 10afc51aaaSDarrick J. Wong #include <linux/pagemap.h> 11afc51aaaSDarrick J. Wong #include <linux/uio.h> 12afc51aaaSDarrick J. Wong #include <linux/buffer_head.h> 13afc51aaaSDarrick J. Wong #include <linux/dax.h> 14afc51aaaSDarrick J. Wong #include <linux/writeback.h> 15598ecfbaSChristoph Hellwig #include <linux/list_sort.h> 16afc51aaaSDarrick J. Wong #include <linux/swap.h> 17afc51aaaSDarrick J. Wong #include <linux/bio.h> 18afc51aaaSDarrick J. Wong #include <linux/sched/signal.h> 19afc51aaaSDarrick J. Wong #include <linux/migrate.h> 209e91c572SChristoph Hellwig #include "trace.h" 21afc51aaaSDarrick J. Wong 22afc51aaaSDarrick J. Wong #include "../internal.h" 23afc51aaaSDarrick J. Wong 24ab08b01eSChristoph Hellwig /* 2595c4cd05SMatthew Wilcox (Oracle) * Structure allocated for each folio when block size < folio size 2695c4cd05SMatthew Wilcox (Oracle) * to track sub-folio uptodate status and I/O completions. 27ab08b01eSChristoph Hellwig */ 28ab08b01eSChristoph Hellwig struct iomap_page { 297d636676SMatthew Wilcox (Oracle) atomic_t read_bytes_pending; 300fb2d720SMatthew Wilcox (Oracle) atomic_t write_bytes_pending; 311cea335dSChristoph Hellwig spinlock_t uptodate_lock; 320a195b91SMatthew Wilcox (Oracle) unsigned long uptodate[]; 33ab08b01eSChristoph Hellwig }; 34ab08b01eSChristoph Hellwig 3595c4cd05SMatthew Wilcox (Oracle) static inline struct iomap_page *to_iomap_page(struct folio *folio) 36ab08b01eSChristoph Hellwig { 3795c4cd05SMatthew Wilcox (Oracle) if (folio_test_private(folio)) 3895c4cd05SMatthew Wilcox (Oracle) return folio_get_private(folio); 39ab08b01eSChristoph Hellwig return NULL; 40ab08b01eSChristoph Hellwig } 41ab08b01eSChristoph Hellwig 42598ecfbaSChristoph Hellwig static struct bio_set iomap_ioend_bioset; 43598ecfbaSChristoph Hellwig 44afc51aaaSDarrick J. Wong static struct iomap_page * 45435d44b3SMatthew Wilcox (Oracle) iomap_page_create(struct inode *inode, struct folio *folio) 46afc51aaaSDarrick J. Wong { 4795c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 48435d44b3SMatthew Wilcox (Oracle) unsigned int nr_blocks = i_blocks_per_folio(inode, folio); 49afc51aaaSDarrick J. Wong 500a195b91SMatthew Wilcox (Oracle) if (iop || nr_blocks <= 1) 51afc51aaaSDarrick J. Wong return iop; 52afc51aaaSDarrick J. Wong 530a195b91SMatthew Wilcox (Oracle) iop = kzalloc(struct_size(iop, uptodate, BITS_TO_LONGS(nr_blocks)), 540a195b91SMatthew Wilcox (Oracle) GFP_NOFS | __GFP_NOFAIL); 551cea335dSChristoph Hellwig spin_lock_init(&iop->uptodate_lock); 56435d44b3SMatthew Wilcox (Oracle) if (folio_test_uptodate(folio)) 574595a298SMatthew Wilcox (Oracle) bitmap_fill(iop->uptodate, nr_blocks); 58435d44b3SMatthew Wilcox (Oracle) folio_attach_private(folio, iop); 59afc51aaaSDarrick J. Wong return iop; 60afc51aaaSDarrick J. Wong } 61afc51aaaSDarrick J. Wong 62c46e8324SMatthew Wilcox (Oracle) static void iomap_page_release(struct folio *folio) 63afc51aaaSDarrick J. Wong { 64c46e8324SMatthew Wilcox (Oracle) struct iomap_page *iop = folio_detach_private(folio); 65c46e8324SMatthew Wilcox (Oracle) struct inode *inode = folio->mapping->host; 66c46e8324SMatthew Wilcox (Oracle) unsigned int nr_blocks = i_blocks_per_folio(inode, folio); 67afc51aaaSDarrick J. Wong 68afc51aaaSDarrick J. Wong if (!iop) 69afc51aaaSDarrick J. Wong return; 707d636676SMatthew Wilcox (Oracle) WARN_ON_ONCE(atomic_read(&iop->read_bytes_pending)); 710fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(atomic_read(&iop->write_bytes_pending)); 720a195b91SMatthew Wilcox (Oracle) WARN_ON_ONCE(bitmap_full(iop->uptodate, nr_blocks) != 73c46e8324SMatthew Wilcox (Oracle) folio_test_uptodate(folio)); 74afc51aaaSDarrick J. Wong kfree(iop); 75afc51aaaSDarrick J. Wong } 76afc51aaaSDarrick J. Wong 77afc51aaaSDarrick J. Wong /* 78afc51aaaSDarrick J. Wong * Calculate the range inside the page that we actually need to read. 79afc51aaaSDarrick J. Wong */ 80afc51aaaSDarrick J. Wong static void 81afc51aaaSDarrick J. Wong iomap_adjust_read_range(struct inode *inode, struct iomap_page *iop, 82afc51aaaSDarrick J. Wong loff_t *pos, loff_t length, unsigned *offp, unsigned *lenp) 83afc51aaaSDarrick J. Wong { 84afc51aaaSDarrick J. Wong loff_t orig_pos = *pos; 85afc51aaaSDarrick J. Wong loff_t isize = i_size_read(inode); 86afc51aaaSDarrick J. Wong unsigned block_bits = inode->i_blkbits; 87afc51aaaSDarrick J. Wong unsigned block_size = (1 << block_bits); 88afc51aaaSDarrick J. Wong unsigned poff = offset_in_page(*pos); 89afc51aaaSDarrick J. Wong unsigned plen = min_t(loff_t, PAGE_SIZE - poff, length); 90afc51aaaSDarrick J. Wong unsigned first = poff >> block_bits; 91afc51aaaSDarrick J. Wong unsigned last = (poff + plen - 1) >> block_bits; 92afc51aaaSDarrick J. Wong 93afc51aaaSDarrick J. Wong /* 94f1f264b4SAndreas Gruenbacher * If the block size is smaller than the page size, we need to check the 95afc51aaaSDarrick J. Wong * per-block uptodate status and adjust the offset and length if needed 96afc51aaaSDarrick J. Wong * to avoid reading in already uptodate ranges. 97afc51aaaSDarrick J. Wong */ 98afc51aaaSDarrick J. Wong if (iop) { 99afc51aaaSDarrick J. Wong unsigned int i; 100afc51aaaSDarrick J. Wong 101afc51aaaSDarrick J. Wong /* move forward for each leading block marked uptodate */ 102afc51aaaSDarrick J. Wong for (i = first; i <= last; i++) { 103afc51aaaSDarrick J. Wong if (!test_bit(i, iop->uptodate)) 104afc51aaaSDarrick J. Wong break; 105afc51aaaSDarrick J. Wong *pos += block_size; 106afc51aaaSDarrick J. Wong poff += block_size; 107afc51aaaSDarrick J. Wong plen -= block_size; 108afc51aaaSDarrick J. Wong first++; 109afc51aaaSDarrick J. Wong } 110afc51aaaSDarrick J. Wong 111afc51aaaSDarrick J. Wong /* truncate len if we find any trailing uptodate block(s) */ 112afc51aaaSDarrick J. Wong for ( ; i <= last; i++) { 113afc51aaaSDarrick J. Wong if (test_bit(i, iop->uptodate)) { 114afc51aaaSDarrick J. Wong plen -= (last - i + 1) * block_size; 115afc51aaaSDarrick J. Wong last = i - 1; 116afc51aaaSDarrick J. Wong break; 117afc51aaaSDarrick J. Wong } 118afc51aaaSDarrick J. Wong } 119afc51aaaSDarrick J. Wong } 120afc51aaaSDarrick J. Wong 121afc51aaaSDarrick J. Wong /* 122f1f264b4SAndreas Gruenbacher * If the extent spans the block that contains the i_size, we need to 123afc51aaaSDarrick J. Wong * handle both halves separately so that we properly zero data in the 124afc51aaaSDarrick J. Wong * page cache for blocks that are entirely outside of i_size. 125afc51aaaSDarrick J. Wong */ 126afc51aaaSDarrick J. Wong if (orig_pos <= isize && orig_pos + length > isize) { 127afc51aaaSDarrick J. Wong unsigned end = offset_in_page(isize - 1) >> block_bits; 128afc51aaaSDarrick J. Wong 129afc51aaaSDarrick J. Wong if (first <= end && last > end) 130afc51aaaSDarrick J. Wong plen -= (last - end) * block_size; 131afc51aaaSDarrick J. Wong } 132afc51aaaSDarrick J. Wong 133afc51aaaSDarrick J. Wong *offp = poff; 134afc51aaaSDarrick J. Wong *lenp = plen; 135afc51aaaSDarrick J. Wong } 136afc51aaaSDarrick J. Wong 137cd1e5afeSMatthew Wilcox (Oracle) static void iomap_iop_set_range_uptodate(struct page *page, 138cd1e5afeSMatthew Wilcox (Oracle) struct iomap_page *iop, unsigned off, unsigned len) 139afc51aaaSDarrick J. Wong { 140afc51aaaSDarrick J. Wong struct inode *inode = page->mapping->host; 141afc51aaaSDarrick J. Wong unsigned first = off >> inode->i_blkbits; 142afc51aaaSDarrick J. Wong unsigned last = (off + len - 1) >> inode->i_blkbits; 1431cea335dSChristoph Hellwig unsigned long flags; 144afc51aaaSDarrick J. Wong 1451cea335dSChristoph Hellwig spin_lock_irqsave(&iop->uptodate_lock, flags); 146b21866f5SMatthew Wilcox (Oracle) bitmap_set(iop->uptodate, first, last - first + 1); 147b21866f5SMatthew Wilcox (Oracle) if (bitmap_full(iop->uptodate, i_blocks_per_page(inode, page))) 1481cea335dSChristoph Hellwig SetPageUptodate(page); 1491cea335dSChristoph Hellwig spin_unlock_irqrestore(&iop->uptodate_lock, flags); 150afc51aaaSDarrick J. Wong } 151afc51aaaSDarrick J. Wong 152cd1e5afeSMatthew Wilcox (Oracle) static void iomap_set_range_uptodate(struct page *page, 153cd1e5afeSMatthew Wilcox (Oracle) struct iomap_page *iop, unsigned off, unsigned len) 1541cea335dSChristoph Hellwig { 1551cea335dSChristoph Hellwig if (PageError(page)) 1561cea335dSChristoph Hellwig return; 1571cea335dSChristoph Hellwig 158cd1e5afeSMatthew Wilcox (Oracle) if (iop) 159cd1e5afeSMatthew Wilcox (Oracle) iomap_iop_set_range_uptodate(page, iop, off, len); 1601cea335dSChristoph Hellwig else 161afc51aaaSDarrick J. Wong SetPageUptodate(page); 162afc51aaaSDarrick J. Wong } 163afc51aaaSDarrick J. Wong 164*8ffd74e9SMatthew Wilcox (Oracle) static void iomap_finish_folio_read(struct folio *folio, size_t offset, 165*8ffd74e9SMatthew Wilcox (Oracle) size_t len, int error) 166afc51aaaSDarrick J. Wong { 16795c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 168afc51aaaSDarrick J. Wong 169afc51aaaSDarrick J. Wong if (unlikely(error)) { 170*8ffd74e9SMatthew Wilcox (Oracle) folio_clear_uptodate(folio); 171*8ffd74e9SMatthew Wilcox (Oracle) folio_set_error(folio); 172afc51aaaSDarrick J. Wong } else { 173*8ffd74e9SMatthew Wilcox (Oracle) iomap_set_range_uptodate(&folio->page, iop, offset, len); 174afc51aaaSDarrick J. Wong } 175afc51aaaSDarrick J. Wong 176*8ffd74e9SMatthew Wilcox (Oracle) if (!iop || atomic_sub_and_test(len, &iop->read_bytes_pending)) 177*8ffd74e9SMatthew Wilcox (Oracle) folio_unlock(folio); 178afc51aaaSDarrick J. Wong } 179afc51aaaSDarrick J. Wong 180*8ffd74e9SMatthew Wilcox (Oracle) static void iomap_read_end_io(struct bio *bio) 181afc51aaaSDarrick J. Wong { 182afc51aaaSDarrick J. Wong int error = blk_status_to_errno(bio->bi_status); 183*8ffd74e9SMatthew Wilcox (Oracle) struct folio_iter fi; 184afc51aaaSDarrick J. Wong 185*8ffd74e9SMatthew Wilcox (Oracle) bio_for_each_folio_all(fi, bio) 186*8ffd74e9SMatthew Wilcox (Oracle) iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error); 187afc51aaaSDarrick J. Wong bio_put(bio); 188afc51aaaSDarrick J. Wong } 189afc51aaaSDarrick J. Wong 190afc51aaaSDarrick J. Wong struct iomap_readpage_ctx { 191afc51aaaSDarrick J. Wong struct page *cur_page; 192afc51aaaSDarrick J. Wong bool cur_page_in_bio; 193afc51aaaSDarrick J. Wong struct bio *bio; 1949d24a13aSMatthew Wilcox (Oracle) struct readahead_control *rac; 195afc51aaaSDarrick J. Wong }; 196afc51aaaSDarrick J. Wong 1975ad448ceSAndreas Gruenbacher /** 1985ad448ceSAndreas Gruenbacher * iomap_read_inline_data - copy inline data into the page cache 1995ad448ceSAndreas Gruenbacher * @iter: iteration structure 2005ad448ceSAndreas Gruenbacher * @page: page to copy to 2015ad448ceSAndreas Gruenbacher * 2025ad448ceSAndreas Gruenbacher * Copy the inline data in @iter into @page and zero out the rest of the page. 2035ad448ceSAndreas Gruenbacher * Only a single IOMAP_INLINE extent is allowed at the end of each file. 2045ad448ceSAndreas Gruenbacher * Returns zero for success to complete the read, or the usual negative errno. 2055ad448ceSAndreas Gruenbacher */ 2065ad448ceSAndreas Gruenbacher static int iomap_read_inline_data(const struct iomap_iter *iter, 2071b5c1e36SChristoph Hellwig struct page *page) 208afc51aaaSDarrick J. Wong { 209435d44b3SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 210cd1e5afeSMatthew Wilcox (Oracle) struct iomap_page *iop; 211fad0a1abSChristoph Hellwig const struct iomap *iomap = iomap_iter_srcmap(iter); 2121b5c1e36SChristoph Hellwig size_t size = i_size_read(iter->inode) - iomap->offset; 213b405435bSMatthew Wilcox (Oracle) size_t poff = offset_in_page(iomap->offset); 214afc51aaaSDarrick J. Wong void *addr; 215afc51aaaSDarrick J. Wong 216afc51aaaSDarrick J. Wong if (PageUptodate(page)) 2175ad448ceSAndreas Gruenbacher return 0; 218afc51aaaSDarrick J. Wong 219ae44f9c2SMatthew Wilcox (Oracle) if (WARN_ON_ONCE(size > PAGE_SIZE - poff)) 220ae44f9c2SMatthew Wilcox (Oracle) return -EIO; 22169f4a26cSGao Xiang if (WARN_ON_ONCE(size > PAGE_SIZE - 22269f4a26cSGao Xiang offset_in_page(iomap->inline_data))) 22369f4a26cSGao Xiang return -EIO; 22469f4a26cSGao Xiang if (WARN_ON_ONCE(size > iomap->length)) 22569f4a26cSGao Xiang return -EIO; 226b405435bSMatthew Wilcox (Oracle) if (poff > 0) 227cd1e5afeSMatthew Wilcox (Oracle) iop = iomap_page_create(iter->inode, folio); 228cd1e5afeSMatthew Wilcox (Oracle) else 229cd1e5afeSMatthew Wilcox (Oracle) iop = to_iomap_page(folio); 230afc51aaaSDarrick J. Wong 231ab069d5fSMatthew Wilcox (Oracle) addr = kmap_local_page(page) + poff; 232afc51aaaSDarrick J. Wong memcpy(addr, iomap->inline_data, size); 233b405435bSMatthew Wilcox (Oracle) memset(addr + size, 0, PAGE_SIZE - poff - size); 234ab069d5fSMatthew Wilcox (Oracle) kunmap_local(addr); 235cd1e5afeSMatthew Wilcox (Oracle) iomap_set_range_uptodate(page, iop, poff, PAGE_SIZE - poff); 2365ad448ceSAndreas Gruenbacher return 0; 237afc51aaaSDarrick J. Wong } 238afc51aaaSDarrick J. Wong 239fad0a1abSChristoph Hellwig static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter, 2401b5c1e36SChristoph Hellwig loff_t pos) 241009d8d84SChristoph Hellwig { 242fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 2431b5c1e36SChristoph Hellwig 2441b5c1e36SChristoph Hellwig return srcmap->type != IOMAP_MAPPED || 2451b5c1e36SChristoph Hellwig (srcmap->flags & IOMAP_F_NEW) || 2461b5c1e36SChristoph Hellwig pos >= i_size_read(iter->inode); 247009d8d84SChristoph Hellwig } 248009d8d84SChristoph Hellwig 249fad0a1abSChristoph Hellwig static loff_t iomap_readpage_iter(const struct iomap_iter *iter, 250f6d48000SChristoph Hellwig struct iomap_readpage_ctx *ctx, loff_t offset) 251afc51aaaSDarrick J. Wong { 252fad0a1abSChristoph Hellwig const struct iomap *iomap = &iter->iomap; 253f6d48000SChristoph Hellwig loff_t pos = iter->pos + offset; 254f6d48000SChristoph Hellwig loff_t length = iomap_length(iter) - offset; 255afc51aaaSDarrick J. Wong struct page *page = ctx->cur_page; 256435d44b3SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 257637d3375SAndreas Gruenbacher struct iomap_page *iop; 258afc51aaaSDarrick J. Wong loff_t orig_pos = pos; 259afc51aaaSDarrick J. Wong unsigned poff, plen; 260afc51aaaSDarrick J. Wong sector_t sector; 261afc51aaaSDarrick J. Wong 2625ad448ceSAndreas Gruenbacher if (iomap->type == IOMAP_INLINE) 2635ad448ceSAndreas Gruenbacher return iomap_read_inline_data(iter, page); 264afc51aaaSDarrick J. Wong 265afc51aaaSDarrick J. Wong /* zero post-eof blocks as the page may be mapped */ 266435d44b3SMatthew Wilcox (Oracle) iop = iomap_page_create(iter->inode, folio); 267f6d48000SChristoph Hellwig iomap_adjust_read_range(iter->inode, iop, &pos, length, &poff, &plen); 268afc51aaaSDarrick J. Wong if (plen == 0) 269afc51aaaSDarrick J. Wong goto done; 270afc51aaaSDarrick J. Wong 2711b5c1e36SChristoph Hellwig if (iomap_block_needs_zeroing(iter, pos)) { 272afc51aaaSDarrick J. Wong zero_user(page, poff, plen); 273cd1e5afeSMatthew Wilcox (Oracle) iomap_set_range_uptodate(page, iop, poff, plen); 274afc51aaaSDarrick J. Wong goto done; 275afc51aaaSDarrick J. Wong } 276afc51aaaSDarrick J. Wong 277afc51aaaSDarrick J. Wong ctx->cur_page_in_bio = true; 2787d636676SMatthew Wilcox (Oracle) if (iop) 2797d636676SMatthew Wilcox (Oracle) atomic_add(plen, &iop->read_bytes_pending); 280afc51aaaSDarrick J. Wong 281afc51aaaSDarrick J. Wong sector = iomap_sector(iomap, pos); 282d0364f94SChristoph Hellwig if (!ctx->bio || 283d0364f94SChristoph Hellwig bio_end_sector(ctx->bio) != sector || 284d0364f94SChristoph Hellwig bio_add_page(ctx->bio, page, plen, poff) != plen) { 285afc51aaaSDarrick J. Wong gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL); 286457df33eSMatthew Wilcox (Oracle) gfp_t orig_gfp = gfp; 2875f7136dbSMatthew Wilcox (Oracle) unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE); 288afc51aaaSDarrick J. Wong 289afc51aaaSDarrick J. Wong if (ctx->bio) 290afc51aaaSDarrick J. Wong submit_bio(ctx->bio); 291afc51aaaSDarrick J. Wong 2929d24a13aSMatthew Wilcox (Oracle) if (ctx->rac) /* same as readahead_gfp_mask */ 293afc51aaaSDarrick J. Wong gfp |= __GFP_NORETRY | __GFP_NOWARN; 2945f7136dbSMatthew Wilcox (Oracle) ctx->bio = bio_alloc(gfp, bio_max_segs(nr_vecs)); 295457df33eSMatthew Wilcox (Oracle) /* 296457df33eSMatthew Wilcox (Oracle) * If the bio_alloc fails, try it again for a single page to 297457df33eSMatthew Wilcox (Oracle) * avoid having to deal with partial page reads. This emulates 298457df33eSMatthew Wilcox (Oracle) * what do_mpage_readpage does. 299457df33eSMatthew Wilcox (Oracle) */ 300457df33eSMatthew Wilcox (Oracle) if (!ctx->bio) 301457df33eSMatthew Wilcox (Oracle) ctx->bio = bio_alloc(orig_gfp, 1); 302afc51aaaSDarrick J. Wong ctx->bio->bi_opf = REQ_OP_READ; 3039d24a13aSMatthew Wilcox (Oracle) if (ctx->rac) 304afc51aaaSDarrick J. Wong ctx->bio->bi_opf |= REQ_RAHEAD; 305afc51aaaSDarrick J. Wong ctx->bio->bi_iter.bi_sector = sector; 306afc51aaaSDarrick J. Wong bio_set_dev(ctx->bio, iomap->bdev); 307afc51aaaSDarrick J. Wong ctx->bio->bi_end_io = iomap_read_end_io; 308d0364f94SChristoph Hellwig __bio_add_page(ctx->bio, page, plen, poff); 309afc51aaaSDarrick J. Wong } 310afc51aaaSDarrick J. Wong done: 311afc51aaaSDarrick J. Wong /* 312afc51aaaSDarrick J. Wong * Move the caller beyond our range so that it keeps making progress. 313f1f264b4SAndreas Gruenbacher * For that, we have to include any leading non-uptodate ranges, but 314afc51aaaSDarrick J. Wong * we can skip trailing ones as they will be handled in the next 315afc51aaaSDarrick J. Wong * iteration. 316afc51aaaSDarrick J. Wong */ 317afc51aaaSDarrick J. Wong return pos - orig_pos + plen; 318afc51aaaSDarrick J. Wong } 319afc51aaaSDarrick J. Wong 320afc51aaaSDarrick J. Wong int 321afc51aaaSDarrick J. Wong iomap_readpage(struct page *page, const struct iomap_ops *ops) 322afc51aaaSDarrick J. Wong { 323f6d48000SChristoph Hellwig struct iomap_iter iter = { 324f6d48000SChristoph Hellwig .inode = page->mapping->host, 325f6d48000SChristoph Hellwig .pos = page_offset(page), 326f6d48000SChristoph Hellwig .len = PAGE_SIZE, 327f6d48000SChristoph Hellwig }; 328f6d48000SChristoph Hellwig struct iomap_readpage_ctx ctx = { 329f6d48000SChristoph Hellwig .cur_page = page, 330f6d48000SChristoph Hellwig }; 331f6d48000SChristoph Hellwig int ret; 332afc51aaaSDarrick J. Wong 3339e91c572SChristoph Hellwig trace_iomap_readpage(page->mapping->host, 1); 3349e91c572SChristoph Hellwig 335f6d48000SChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 336f6d48000SChristoph Hellwig iter.processed = iomap_readpage_iter(&iter, &ctx, 0); 337f6d48000SChristoph Hellwig 338f6d48000SChristoph Hellwig if (ret < 0) 339afc51aaaSDarrick J. Wong SetPageError(page); 340afc51aaaSDarrick J. Wong 341afc51aaaSDarrick J. Wong if (ctx.bio) { 342afc51aaaSDarrick J. Wong submit_bio(ctx.bio); 343afc51aaaSDarrick J. Wong WARN_ON_ONCE(!ctx.cur_page_in_bio); 344afc51aaaSDarrick J. Wong } else { 345afc51aaaSDarrick J. Wong WARN_ON_ONCE(ctx.cur_page_in_bio); 346afc51aaaSDarrick J. Wong unlock_page(page); 347afc51aaaSDarrick J. Wong } 348afc51aaaSDarrick J. Wong 349afc51aaaSDarrick J. Wong /* 350f1f264b4SAndreas Gruenbacher * Just like mpage_readahead and block_read_full_page, we always 351afc51aaaSDarrick J. Wong * return 0 and just mark the page as PageError on errors. This 352f1f264b4SAndreas Gruenbacher * should be cleaned up throughout the stack eventually. 353afc51aaaSDarrick J. Wong */ 354afc51aaaSDarrick J. Wong return 0; 355afc51aaaSDarrick J. Wong } 356afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_readpage); 357afc51aaaSDarrick J. Wong 358fad0a1abSChristoph Hellwig static loff_t iomap_readahead_iter(const struct iomap_iter *iter, 359f6d48000SChristoph Hellwig struct iomap_readpage_ctx *ctx) 360afc51aaaSDarrick J. Wong { 361f6d48000SChristoph Hellwig loff_t length = iomap_length(iter); 362afc51aaaSDarrick J. Wong loff_t done, ret; 363afc51aaaSDarrick J. Wong 364afc51aaaSDarrick J. Wong for (done = 0; done < length; done += ret) { 365f6d48000SChristoph Hellwig if (ctx->cur_page && offset_in_page(iter->pos + done) == 0) { 366afc51aaaSDarrick J. Wong if (!ctx->cur_page_in_bio) 367afc51aaaSDarrick J. Wong unlock_page(ctx->cur_page); 368afc51aaaSDarrick J. Wong put_page(ctx->cur_page); 369afc51aaaSDarrick J. Wong ctx->cur_page = NULL; 370afc51aaaSDarrick J. Wong } 371afc51aaaSDarrick J. Wong if (!ctx->cur_page) { 3729d24a13aSMatthew Wilcox (Oracle) ctx->cur_page = readahead_page(ctx->rac); 373afc51aaaSDarrick J. Wong ctx->cur_page_in_bio = false; 374afc51aaaSDarrick J. Wong } 375f6d48000SChristoph Hellwig ret = iomap_readpage_iter(iter, ctx, done); 376d8af404fSAndreas Gruenbacher if (ret <= 0) 377d8af404fSAndreas Gruenbacher return ret; 378afc51aaaSDarrick J. Wong } 379afc51aaaSDarrick J. Wong 380afc51aaaSDarrick J. Wong return done; 381afc51aaaSDarrick J. Wong } 382afc51aaaSDarrick J. Wong 3839d24a13aSMatthew Wilcox (Oracle) /** 3849d24a13aSMatthew Wilcox (Oracle) * iomap_readahead - Attempt to read pages from a file. 3859d24a13aSMatthew Wilcox (Oracle) * @rac: Describes the pages to be read. 3869d24a13aSMatthew Wilcox (Oracle) * @ops: The operations vector for the filesystem. 3879d24a13aSMatthew Wilcox (Oracle) * 3889d24a13aSMatthew Wilcox (Oracle) * This function is for filesystems to call to implement their readahead 3899d24a13aSMatthew Wilcox (Oracle) * address_space operation. 3909d24a13aSMatthew Wilcox (Oracle) * 3919d24a13aSMatthew Wilcox (Oracle) * Context: The @ops callbacks may submit I/O (eg to read the addresses of 3929d24a13aSMatthew Wilcox (Oracle) * blocks from disc), and may wait for it. The caller may be trying to 3939d24a13aSMatthew Wilcox (Oracle) * access a different page, and so sleeping excessively should be avoided. 3949d24a13aSMatthew Wilcox (Oracle) * It may allocate memory, but should avoid costly allocations. This 3959d24a13aSMatthew Wilcox (Oracle) * function is called with memalloc_nofs set, so allocations will not cause 3969d24a13aSMatthew Wilcox (Oracle) * the filesystem to be reentered. 3979d24a13aSMatthew Wilcox (Oracle) */ 3989d24a13aSMatthew Wilcox (Oracle) void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops) 399afc51aaaSDarrick J. Wong { 400f6d48000SChristoph Hellwig struct iomap_iter iter = { 401f6d48000SChristoph Hellwig .inode = rac->mapping->host, 402f6d48000SChristoph Hellwig .pos = readahead_pos(rac), 403f6d48000SChristoph Hellwig .len = readahead_length(rac), 404f6d48000SChristoph Hellwig }; 405afc51aaaSDarrick J. Wong struct iomap_readpage_ctx ctx = { 4069d24a13aSMatthew Wilcox (Oracle) .rac = rac, 407afc51aaaSDarrick J. Wong }; 408afc51aaaSDarrick J. Wong 409f6d48000SChristoph Hellwig trace_iomap_readahead(rac->mapping->host, readahead_count(rac)); 4109e91c572SChristoph Hellwig 411f6d48000SChristoph Hellwig while (iomap_iter(&iter, ops) > 0) 412f6d48000SChristoph Hellwig iter.processed = iomap_readahead_iter(&iter, &ctx); 4139d24a13aSMatthew Wilcox (Oracle) 414afc51aaaSDarrick J. Wong if (ctx.bio) 415afc51aaaSDarrick J. Wong submit_bio(ctx.bio); 416afc51aaaSDarrick J. Wong if (ctx.cur_page) { 417afc51aaaSDarrick J. Wong if (!ctx.cur_page_in_bio) 418afc51aaaSDarrick J. Wong unlock_page(ctx.cur_page); 419afc51aaaSDarrick J. Wong put_page(ctx.cur_page); 420afc51aaaSDarrick J. Wong } 421afc51aaaSDarrick J. Wong } 4229d24a13aSMatthew Wilcox (Oracle) EXPORT_SYMBOL_GPL(iomap_readahead); 423afc51aaaSDarrick J. Wong 424afc51aaaSDarrick J. Wong /* 425afc51aaaSDarrick J. Wong * iomap_is_partially_uptodate checks whether blocks within a page are 426afc51aaaSDarrick J. Wong * uptodate or not. 427afc51aaaSDarrick J. Wong * 428afc51aaaSDarrick J. Wong * Returns true if all blocks which correspond to a file portion 429afc51aaaSDarrick J. Wong * we want to read within the page are uptodate. 430afc51aaaSDarrick J. Wong */ 431afc51aaaSDarrick J. Wong int 432afc51aaaSDarrick J. Wong iomap_is_partially_uptodate(struct page *page, unsigned long from, 433afc51aaaSDarrick J. Wong unsigned long count) 434afc51aaaSDarrick J. Wong { 43595c4cd05SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 43695c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 437afc51aaaSDarrick J. Wong struct inode *inode = page->mapping->host; 438afc51aaaSDarrick J. Wong unsigned len, first, last; 439afc51aaaSDarrick J. Wong unsigned i; 440afc51aaaSDarrick J. Wong 441afc51aaaSDarrick J. Wong /* Limit range to one page */ 442afc51aaaSDarrick J. Wong len = min_t(unsigned, PAGE_SIZE - from, count); 443afc51aaaSDarrick J. Wong 444afc51aaaSDarrick J. Wong /* First and last blocks in range within page */ 445afc51aaaSDarrick J. Wong first = from >> inode->i_blkbits; 446afc51aaaSDarrick J. Wong last = (from + len - 1) >> inode->i_blkbits; 447afc51aaaSDarrick J. Wong 448afc51aaaSDarrick J. Wong if (iop) { 449afc51aaaSDarrick J. Wong for (i = first; i <= last; i++) 450afc51aaaSDarrick J. Wong if (!test_bit(i, iop->uptodate)) 451afc51aaaSDarrick J. Wong return 0; 452afc51aaaSDarrick J. Wong return 1; 453afc51aaaSDarrick J. Wong } 454afc51aaaSDarrick J. Wong 455afc51aaaSDarrick J. Wong return 0; 456afc51aaaSDarrick J. Wong } 457afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); 458afc51aaaSDarrick J. Wong 459afc51aaaSDarrick J. Wong int 460afc51aaaSDarrick J. Wong iomap_releasepage(struct page *page, gfp_t gfp_mask) 461afc51aaaSDarrick J. Wong { 462c46e8324SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 463c46e8324SMatthew Wilcox (Oracle) 46439f16c83SMatthew Wilcox (Oracle) trace_iomap_releasepage(folio->mapping->host, folio_pos(folio), 46539f16c83SMatthew Wilcox (Oracle) folio_size(folio)); 4669e91c572SChristoph Hellwig 467afc51aaaSDarrick J. Wong /* 468afc51aaaSDarrick J. Wong * mm accommodates an old ext3 case where clean pages might not have had 469afc51aaaSDarrick J. Wong * the dirty bit cleared. Thus, it can send actual dirty pages to 470f1f264b4SAndreas Gruenbacher * ->releasepage() via shrink_active_list(); skip those here. 471afc51aaaSDarrick J. Wong */ 47239f16c83SMatthew Wilcox (Oracle) if (folio_test_dirty(folio) || folio_test_writeback(folio)) 473afc51aaaSDarrick J. Wong return 0; 474c46e8324SMatthew Wilcox (Oracle) iomap_page_release(folio); 475afc51aaaSDarrick J. Wong return 1; 476afc51aaaSDarrick J. Wong } 477afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_releasepage); 478afc51aaaSDarrick J. Wong 4798306a5f5SMatthew Wilcox (Oracle) void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len) 480afc51aaaSDarrick J. Wong { 4818306a5f5SMatthew Wilcox (Oracle) trace_iomap_invalidatepage(folio->mapping->host, offset, len); 4829e91c572SChristoph Hellwig 483afc51aaaSDarrick J. Wong /* 484f1f264b4SAndreas Gruenbacher * If we're invalidating the entire page, clear the dirty state from it 485afc51aaaSDarrick J. Wong * and release it to avoid unnecessary buildup of the LRU. 486afc51aaaSDarrick J. Wong */ 4878306a5f5SMatthew Wilcox (Oracle) if (offset == 0 && len == folio_size(folio)) { 4888306a5f5SMatthew Wilcox (Oracle) WARN_ON_ONCE(folio_test_writeback(folio)); 4898306a5f5SMatthew Wilcox (Oracle) folio_cancel_dirty(folio); 490c46e8324SMatthew Wilcox (Oracle) iomap_page_release(folio); 491afc51aaaSDarrick J. Wong } 492afc51aaaSDarrick J. Wong } 4938306a5f5SMatthew Wilcox (Oracle) EXPORT_SYMBOL_GPL(iomap_invalidate_folio); 4948306a5f5SMatthew Wilcox (Oracle) 4958306a5f5SMatthew Wilcox (Oracle) void iomap_invalidatepage(struct page *page, unsigned int offset, 4968306a5f5SMatthew Wilcox (Oracle) unsigned int len) 4978306a5f5SMatthew Wilcox (Oracle) { 4988306a5f5SMatthew Wilcox (Oracle) iomap_invalidate_folio(page_folio(page), offset, len); 4998306a5f5SMatthew Wilcox (Oracle) } 500afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_invalidatepage); 501afc51aaaSDarrick J. Wong 502afc51aaaSDarrick J. Wong #ifdef CONFIG_MIGRATION 503afc51aaaSDarrick J. Wong int 504afc51aaaSDarrick J. Wong iomap_migrate_page(struct address_space *mapping, struct page *newpage, 505afc51aaaSDarrick J. Wong struct page *page, enum migrate_mode mode) 506afc51aaaSDarrick J. Wong { 507afc51aaaSDarrick J. Wong int ret; 508afc51aaaSDarrick J. Wong 50926473f83SLinus Torvalds ret = migrate_page_move_mapping(mapping, newpage, page, 0); 510afc51aaaSDarrick J. Wong if (ret != MIGRATEPAGE_SUCCESS) 511afc51aaaSDarrick J. Wong return ret; 512afc51aaaSDarrick J. Wong 51358aeb731SGuoqing Jiang if (page_has_private(page)) 51458aeb731SGuoqing Jiang attach_page_private(newpage, detach_page_private(page)); 515afc51aaaSDarrick J. Wong 516afc51aaaSDarrick J. Wong if (mode != MIGRATE_SYNC_NO_COPY) 517afc51aaaSDarrick J. Wong migrate_page_copy(newpage, page); 518afc51aaaSDarrick J. Wong else 519afc51aaaSDarrick J. Wong migrate_page_states(newpage, page); 520afc51aaaSDarrick J. Wong return MIGRATEPAGE_SUCCESS; 521afc51aaaSDarrick J. Wong } 522afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_migrate_page); 523afc51aaaSDarrick J. Wong #endif /* CONFIG_MIGRATION */ 524afc51aaaSDarrick J. Wong 525afc51aaaSDarrick J. Wong static void 526afc51aaaSDarrick J. Wong iomap_write_failed(struct inode *inode, loff_t pos, unsigned len) 527afc51aaaSDarrick J. Wong { 528afc51aaaSDarrick J. Wong loff_t i_size = i_size_read(inode); 529afc51aaaSDarrick J. Wong 530afc51aaaSDarrick J. Wong /* 531afc51aaaSDarrick J. Wong * Only truncate newly allocated pages beyoned EOF, even if the 532afc51aaaSDarrick J. Wong * write started inside the existing inode size. 533afc51aaaSDarrick J. Wong */ 534afc51aaaSDarrick J. Wong if (pos + len > i_size) 535afc51aaaSDarrick J. Wong truncate_pagecache_range(inode, max(pos, i_size), pos + len); 536afc51aaaSDarrick J. Wong } 537afc51aaaSDarrick J. Wong 538afc51aaaSDarrick J. Wong static int 539d3b40439SChristoph Hellwig iomap_read_page_sync(loff_t block_start, struct page *page, unsigned poff, 5401acd9e9cSChristoph Hellwig unsigned plen, const struct iomap *iomap) 541afc51aaaSDarrick J. Wong { 542afc51aaaSDarrick J. Wong struct bio_vec bvec; 543afc51aaaSDarrick J. Wong struct bio bio; 544afc51aaaSDarrick J. Wong 545afc51aaaSDarrick J. Wong bio_init(&bio, &bvec, 1); 546afc51aaaSDarrick J. Wong bio.bi_opf = REQ_OP_READ; 547afc51aaaSDarrick J. Wong bio.bi_iter.bi_sector = iomap_sector(iomap, block_start); 548afc51aaaSDarrick J. Wong bio_set_dev(&bio, iomap->bdev); 549afc51aaaSDarrick J. Wong __bio_add_page(&bio, page, plen, poff); 550afc51aaaSDarrick J. Wong return submit_bio_wait(&bio); 551afc51aaaSDarrick J. Wong } 552afc51aaaSDarrick J. Wong 553fad0a1abSChristoph Hellwig static int __iomap_write_begin(const struct iomap_iter *iter, loff_t pos, 554b74b1293SChristoph Hellwig unsigned len, struct page *page) 555afc51aaaSDarrick J. Wong { 556435d44b3SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 557fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 558435d44b3SMatthew Wilcox (Oracle) struct iomap_page *iop = iomap_page_create(iter->inode, folio); 5591b5c1e36SChristoph Hellwig loff_t block_size = i_blocksize(iter->inode); 5606cc19c5fSNikolay Borisov loff_t block_start = round_down(pos, block_size); 5616cc19c5fSNikolay Borisov loff_t block_end = round_up(pos + len, block_size); 562afc51aaaSDarrick J. Wong unsigned from = offset_in_page(pos), to = from + len, poff, plen; 563afc51aaaSDarrick J. Wong 564afc51aaaSDarrick J. Wong if (PageUptodate(page)) 565afc51aaaSDarrick J. Wong return 0; 566e6e7ca92SMatthew Wilcox (Oracle) ClearPageError(page); 567afc51aaaSDarrick J. Wong 568afc51aaaSDarrick J. Wong do { 5691b5c1e36SChristoph Hellwig iomap_adjust_read_range(iter->inode, iop, &block_start, 570afc51aaaSDarrick J. Wong block_end - block_start, &poff, &plen); 571afc51aaaSDarrick J. Wong if (plen == 0) 572afc51aaaSDarrick J. Wong break; 573afc51aaaSDarrick J. Wong 574b74b1293SChristoph Hellwig if (!(iter->flags & IOMAP_UNSHARE) && 57532a38a49SChristoph Hellwig (from <= poff || from >= poff + plen) && 576d3b40439SChristoph Hellwig (to <= poff || to >= poff + plen)) 577d3b40439SChristoph Hellwig continue; 578d3b40439SChristoph Hellwig 5791b5c1e36SChristoph Hellwig if (iomap_block_needs_zeroing(iter, block_start)) { 580b74b1293SChristoph Hellwig if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE)) 58132a38a49SChristoph Hellwig return -EIO; 582d3b40439SChristoph Hellwig zero_user_segments(page, poff, from, to, poff + plen); 58314284fedSMatthew Wilcox (Oracle) } else { 58414284fedSMatthew Wilcox (Oracle) int status = iomap_read_page_sync(block_start, page, 58514284fedSMatthew Wilcox (Oracle) poff, plen, srcmap); 586d3b40439SChristoph Hellwig if (status) 587d3b40439SChristoph Hellwig return status; 58814284fedSMatthew Wilcox (Oracle) } 589cd1e5afeSMatthew Wilcox (Oracle) iomap_set_range_uptodate(page, iop, poff, plen); 590afc51aaaSDarrick J. Wong } while ((block_start += plen) < block_end); 591afc51aaaSDarrick J. Wong 592d3b40439SChristoph Hellwig return 0; 593afc51aaaSDarrick J. Wong } 594afc51aaaSDarrick J. Wong 595fad0a1abSChristoph Hellwig static int iomap_write_begin_inline(const struct iomap_iter *iter, 5961b5c1e36SChristoph Hellwig struct page *page) 59769f4a26cSGao Xiang { 59869f4a26cSGao Xiang /* needs more work for the tailpacking case; disable for now */ 5991b5c1e36SChristoph Hellwig if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0)) 60069f4a26cSGao Xiang return -EIO; 6015ad448ceSAndreas Gruenbacher return iomap_read_inline_data(iter, page); 60269f4a26cSGao Xiang } 60369f4a26cSGao Xiang 604fad0a1abSChristoph Hellwig static int iomap_write_begin(const struct iomap_iter *iter, loff_t pos, 605fad0a1abSChristoph Hellwig unsigned len, struct page **pagep) 606afc51aaaSDarrick J. Wong { 6071b5c1e36SChristoph Hellwig const struct iomap_page_ops *page_ops = iter->iomap.page_ops; 608fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 609afc51aaaSDarrick J. Wong struct page *page; 610d1bd0b4eSMatthew Wilcox (Oracle) struct folio *folio; 611afc51aaaSDarrick J. Wong int status = 0; 612afc51aaaSDarrick J. Wong 6131b5c1e36SChristoph Hellwig BUG_ON(pos + len > iter->iomap.offset + iter->iomap.length); 6141b5c1e36SChristoph Hellwig if (srcmap != &iter->iomap) 615c039b997SGoldwyn Rodrigues BUG_ON(pos + len > srcmap->offset + srcmap->length); 616afc51aaaSDarrick J. Wong 617afc51aaaSDarrick J. Wong if (fatal_signal_pending(current)) 618afc51aaaSDarrick J. Wong return -EINTR; 619afc51aaaSDarrick J. Wong 620afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_prepare) { 6211b5c1e36SChristoph Hellwig status = page_ops->page_prepare(iter->inode, pos, len); 622afc51aaaSDarrick J. Wong if (status) 623afc51aaaSDarrick J. Wong return status; 624afc51aaaSDarrick J. Wong } 625afc51aaaSDarrick J. Wong 6261b5c1e36SChristoph Hellwig page = grab_cache_page_write_begin(iter->inode->i_mapping, 6271b5c1e36SChristoph Hellwig pos >> PAGE_SHIFT, AOP_FLAG_NOFS); 628afc51aaaSDarrick J. Wong if (!page) { 629afc51aaaSDarrick J. Wong status = -ENOMEM; 630afc51aaaSDarrick J. Wong goto out_no_page; 631afc51aaaSDarrick J. Wong } 632d1bd0b4eSMatthew Wilcox (Oracle) folio = page_folio(page); 633afc51aaaSDarrick J. Wong 634c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_INLINE) 6351b5c1e36SChristoph Hellwig status = iomap_write_begin_inline(iter, page); 6361b5c1e36SChristoph Hellwig else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) 637d1bd0b4eSMatthew Wilcox (Oracle) status = __block_write_begin_int(folio, pos, len, NULL, srcmap); 638afc51aaaSDarrick J. Wong else 639b74b1293SChristoph Hellwig status = __iomap_write_begin(iter, pos, len, page); 640afc51aaaSDarrick J. Wong 641afc51aaaSDarrick J. Wong if (unlikely(status)) 642afc51aaaSDarrick J. Wong goto out_unlock; 643afc51aaaSDarrick J. Wong 644afc51aaaSDarrick J. Wong *pagep = page; 645afc51aaaSDarrick J. Wong return 0; 646afc51aaaSDarrick J. Wong 647afc51aaaSDarrick J. Wong out_unlock: 648afc51aaaSDarrick J. Wong unlock_page(page); 649afc51aaaSDarrick J. Wong put_page(page); 6501b5c1e36SChristoph Hellwig iomap_write_failed(iter->inode, pos, len); 651afc51aaaSDarrick J. Wong 652afc51aaaSDarrick J. Wong out_no_page: 653afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_done) 6541b5c1e36SChristoph Hellwig page_ops->page_done(iter->inode, pos, 0, NULL); 655afc51aaaSDarrick J. Wong return status; 656afc51aaaSDarrick J. Wong } 657afc51aaaSDarrick J. Wong 658e25ba8cbSMatthew Wilcox (Oracle) static size_t __iomap_write_end(struct inode *inode, loff_t pos, size_t len, 659e25ba8cbSMatthew Wilcox (Oracle) size_t copied, struct page *page) 660afc51aaaSDarrick J. Wong { 661cd1e5afeSMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 662cd1e5afeSMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 663afc51aaaSDarrick J. Wong flush_dcache_page(page); 664afc51aaaSDarrick J. Wong 665afc51aaaSDarrick J. Wong /* 666afc51aaaSDarrick J. Wong * The blocks that were entirely written will now be uptodate, so we 667afc51aaaSDarrick J. Wong * don't have to worry about a readpage reading them and overwriting a 668f1f264b4SAndreas Gruenbacher * partial write. However, if we've encountered a short write and only 669afc51aaaSDarrick J. Wong * partially written into a block, it will not be marked uptodate, so a 670afc51aaaSDarrick J. Wong * readpage might come in and destroy our partial write. 671afc51aaaSDarrick J. Wong * 672f1f264b4SAndreas Gruenbacher * Do the simplest thing and just treat any short write to a 673f1f264b4SAndreas Gruenbacher * non-uptodate page as a zero-length write, and force the caller to 674f1f264b4SAndreas Gruenbacher * redo the whole thing. 675afc51aaaSDarrick J. Wong */ 676afc51aaaSDarrick J. Wong if (unlikely(copied < len && !PageUptodate(page))) 677afc51aaaSDarrick J. Wong return 0; 678cd1e5afeSMatthew Wilcox (Oracle) iomap_set_range_uptodate(page, iop, offset_in_page(pos), len); 679fd7353f8SMatthew Wilcox (Oracle) __set_page_dirty_nobuffers(page); 680afc51aaaSDarrick J. Wong return copied; 681afc51aaaSDarrick J. Wong } 682afc51aaaSDarrick J. Wong 683fad0a1abSChristoph Hellwig static size_t iomap_write_end_inline(const struct iomap_iter *iter, 684fad0a1abSChristoph Hellwig struct page *page, loff_t pos, size_t copied) 685afc51aaaSDarrick J. Wong { 686fad0a1abSChristoph Hellwig const struct iomap *iomap = &iter->iomap; 687afc51aaaSDarrick J. Wong void *addr; 688afc51aaaSDarrick J. Wong 689afc51aaaSDarrick J. Wong WARN_ON_ONCE(!PageUptodate(page)); 69069f4a26cSGao Xiang BUG_ON(!iomap_inline_data_valid(iomap)); 691afc51aaaSDarrick J. Wong 6927ed3cd1aSMatthew Wilcox (Oracle) flush_dcache_page(page); 693ab069d5fSMatthew Wilcox (Oracle) addr = kmap_local_page(page) + pos; 694ab069d5fSMatthew Wilcox (Oracle) memcpy(iomap_inline_data(iomap, pos), addr, copied); 695ab069d5fSMatthew Wilcox (Oracle) kunmap_local(addr); 696afc51aaaSDarrick J. Wong 6971b5c1e36SChristoph Hellwig mark_inode_dirty(iter->inode); 698afc51aaaSDarrick J. Wong return copied; 699afc51aaaSDarrick J. Wong } 700afc51aaaSDarrick J. Wong 701e25ba8cbSMatthew Wilcox (Oracle) /* Returns the number of bytes copied. May be 0. Cannot be an errno. */ 7021b5c1e36SChristoph Hellwig static size_t iomap_write_end(struct iomap_iter *iter, loff_t pos, size_t len, 7031b5c1e36SChristoph Hellwig size_t copied, struct page *page) 704afc51aaaSDarrick J. Wong { 7051b5c1e36SChristoph Hellwig const struct iomap_page_ops *page_ops = iter->iomap.page_ops; 706fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 7071b5c1e36SChristoph Hellwig loff_t old_size = iter->inode->i_size; 708e25ba8cbSMatthew Wilcox (Oracle) size_t ret; 709afc51aaaSDarrick J. Wong 710c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_INLINE) { 7111b5c1e36SChristoph Hellwig ret = iomap_write_end_inline(iter, page, pos, copied); 712c039b997SGoldwyn Rodrigues } else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) { 7131b5c1e36SChristoph Hellwig ret = block_write_end(NULL, iter->inode->i_mapping, pos, len, 7141b5c1e36SChristoph Hellwig copied, page, NULL); 715afc51aaaSDarrick J. Wong } else { 7161b5c1e36SChristoph Hellwig ret = __iomap_write_end(iter->inode, pos, len, copied, page); 717afc51aaaSDarrick J. Wong } 718afc51aaaSDarrick J. Wong 719afc51aaaSDarrick J. Wong /* 720afc51aaaSDarrick J. Wong * Update the in-memory inode size after copying the data into the page 721afc51aaaSDarrick J. Wong * cache. It's up to the file system to write the updated size to disk, 722afc51aaaSDarrick J. Wong * preferably after I/O completion so that no stale data is exposed. 723afc51aaaSDarrick J. Wong */ 724afc51aaaSDarrick J. Wong if (pos + ret > old_size) { 7251b5c1e36SChristoph Hellwig i_size_write(iter->inode, pos + ret); 7261b5c1e36SChristoph Hellwig iter->iomap.flags |= IOMAP_F_SIZE_CHANGED; 727afc51aaaSDarrick J. Wong } 728afc51aaaSDarrick J. Wong unlock_page(page); 729afc51aaaSDarrick J. Wong 730afc51aaaSDarrick J. Wong if (old_size < pos) 7311b5c1e36SChristoph Hellwig pagecache_isize_extended(iter->inode, old_size, pos); 732afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_done) 7331b5c1e36SChristoph Hellwig page_ops->page_done(iter->inode, pos, ret, page); 734afc51aaaSDarrick J. Wong put_page(page); 735afc51aaaSDarrick J. Wong 736afc51aaaSDarrick J. Wong if (ret < len) 7371b5c1e36SChristoph Hellwig iomap_write_failed(iter->inode, pos, len); 738afc51aaaSDarrick J. Wong return ret; 739afc51aaaSDarrick J. Wong } 740afc51aaaSDarrick J. Wong 741ce83a025SChristoph Hellwig static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i) 742afc51aaaSDarrick J. Wong { 743ce83a025SChristoph Hellwig loff_t length = iomap_length(iter); 744ce83a025SChristoph Hellwig loff_t pos = iter->pos; 745afc51aaaSDarrick J. Wong ssize_t written = 0; 746ce83a025SChristoph Hellwig long status = 0; 747afc51aaaSDarrick J. Wong 748afc51aaaSDarrick J. Wong do { 749afc51aaaSDarrick J. Wong struct page *page; 750afc51aaaSDarrick J. Wong unsigned long offset; /* Offset into pagecache page */ 751afc51aaaSDarrick J. Wong unsigned long bytes; /* Bytes to write to page */ 752afc51aaaSDarrick J. Wong size_t copied; /* Bytes copied from user */ 753afc51aaaSDarrick J. Wong 754afc51aaaSDarrick J. Wong offset = offset_in_page(pos); 755afc51aaaSDarrick J. Wong bytes = min_t(unsigned long, PAGE_SIZE - offset, 756afc51aaaSDarrick J. Wong iov_iter_count(i)); 757afc51aaaSDarrick J. Wong again: 758afc51aaaSDarrick J. Wong if (bytes > length) 759afc51aaaSDarrick J. Wong bytes = length; 760afc51aaaSDarrick J. Wong 761afc51aaaSDarrick J. Wong /* 762f1f264b4SAndreas Gruenbacher * Bring in the user page that we'll copy from _first_. 763afc51aaaSDarrick J. Wong * Otherwise there's a nasty deadlock on copying from the 764afc51aaaSDarrick J. Wong * same page as we're writing to, without it being marked 765afc51aaaSDarrick J. Wong * up-to-date. 766afc51aaaSDarrick J. Wong */ 767a6294593SAndreas Gruenbacher if (unlikely(fault_in_iov_iter_readable(i, bytes))) { 768afc51aaaSDarrick J. Wong status = -EFAULT; 769afc51aaaSDarrick J. Wong break; 770afc51aaaSDarrick J. Wong } 771afc51aaaSDarrick J. Wong 772b74b1293SChristoph Hellwig status = iomap_write_begin(iter, pos, bytes, &page); 773afc51aaaSDarrick J. Wong if (unlikely(status)) 774afc51aaaSDarrick J. Wong break; 775afc51aaaSDarrick J. Wong 776ce83a025SChristoph Hellwig if (mapping_writably_mapped(iter->inode->i_mapping)) 777afc51aaaSDarrick J. Wong flush_dcache_page(page); 778afc51aaaSDarrick J. Wong 779f0b65f39SAl Viro copied = copy_page_from_iter_atomic(page, offset, bytes, i); 780afc51aaaSDarrick J. Wong 7811b5c1e36SChristoph Hellwig status = iomap_write_end(iter, pos, bytes, copied, page); 782afc51aaaSDarrick J. Wong 783f0b65f39SAl Viro if (unlikely(copied != status)) 784f0b65f39SAl Viro iov_iter_revert(i, copied - status); 785afc51aaaSDarrick J. Wong 786f0b65f39SAl Viro cond_resched(); 787bc1bb416SAl Viro if (unlikely(status == 0)) { 788afc51aaaSDarrick J. Wong /* 789bc1bb416SAl Viro * A short copy made iomap_write_end() reject the 790bc1bb416SAl Viro * thing entirely. Might be memory poisoning 791bc1bb416SAl Viro * halfway through, might be a race with munmap, 792bc1bb416SAl Viro * might be severe memory pressure. 793afc51aaaSDarrick J. Wong */ 794bc1bb416SAl Viro if (copied) 795bc1bb416SAl Viro bytes = copied; 796afc51aaaSDarrick J. Wong goto again; 797afc51aaaSDarrick J. Wong } 798f0b65f39SAl Viro pos += status; 799f0b65f39SAl Viro written += status; 800f0b65f39SAl Viro length -= status; 801afc51aaaSDarrick J. Wong 802ce83a025SChristoph Hellwig balance_dirty_pages_ratelimited(iter->inode->i_mapping); 803afc51aaaSDarrick J. Wong } while (iov_iter_count(i) && length); 804afc51aaaSDarrick J. Wong 805afc51aaaSDarrick J. Wong return written ? written : status; 806afc51aaaSDarrick J. Wong } 807afc51aaaSDarrick J. Wong 808afc51aaaSDarrick J. Wong ssize_t 809ce83a025SChristoph Hellwig iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, 810afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 811afc51aaaSDarrick J. Wong { 812ce83a025SChristoph Hellwig struct iomap_iter iter = { 813ce83a025SChristoph Hellwig .inode = iocb->ki_filp->f_mapping->host, 814ce83a025SChristoph Hellwig .pos = iocb->ki_pos, 815ce83a025SChristoph Hellwig .len = iov_iter_count(i), 816ce83a025SChristoph Hellwig .flags = IOMAP_WRITE, 817ce83a025SChristoph Hellwig }; 818ce83a025SChristoph Hellwig int ret; 819afc51aaaSDarrick J. Wong 820ce83a025SChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 821ce83a025SChristoph Hellwig iter.processed = iomap_write_iter(&iter, i); 822ce83a025SChristoph Hellwig if (iter.pos == iocb->ki_pos) 823ce83a025SChristoph Hellwig return ret; 824ce83a025SChristoph Hellwig return iter.pos - iocb->ki_pos; 825afc51aaaSDarrick J. Wong } 826afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_file_buffered_write); 827afc51aaaSDarrick J. Wong 8288fc274d1SChristoph Hellwig static loff_t iomap_unshare_iter(struct iomap_iter *iter) 829afc51aaaSDarrick J. Wong { 8308fc274d1SChristoph Hellwig struct iomap *iomap = &iter->iomap; 831fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 8328fc274d1SChristoph Hellwig loff_t pos = iter->pos; 8338fc274d1SChristoph Hellwig loff_t length = iomap_length(iter); 834afc51aaaSDarrick J. Wong long status = 0; 835d4ff3b2eSMatthew Wilcox (Oracle) loff_t written = 0; 836afc51aaaSDarrick J. Wong 8373590c4d8SChristoph Hellwig /* don't bother with blocks that are not shared to start with */ 8383590c4d8SChristoph Hellwig if (!(iomap->flags & IOMAP_F_SHARED)) 8393590c4d8SChristoph Hellwig return length; 8403590c4d8SChristoph Hellwig /* don't bother with holes or unwritten extents */ 841c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) 8423590c4d8SChristoph Hellwig return length; 8433590c4d8SChristoph Hellwig 844afc51aaaSDarrick J. Wong do { 84532a38a49SChristoph Hellwig unsigned long offset = offset_in_page(pos); 84632a38a49SChristoph Hellwig unsigned long bytes = min_t(loff_t, PAGE_SIZE - offset, length); 84732a38a49SChristoph Hellwig struct page *page; 848afc51aaaSDarrick J. Wong 849b74b1293SChristoph Hellwig status = iomap_write_begin(iter, pos, bytes, &page); 850afc51aaaSDarrick J. Wong if (unlikely(status)) 851afc51aaaSDarrick J. Wong return status; 852afc51aaaSDarrick J. Wong 8531b5c1e36SChristoph Hellwig status = iomap_write_end(iter, pos, bytes, bytes, page); 854afc51aaaSDarrick J. Wong if (WARN_ON_ONCE(status == 0)) 855afc51aaaSDarrick J. Wong return -EIO; 856afc51aaaSDarrick J. Wong 857afc51aaaSDarrick J. Wong cond_resched(); 858afc51aaaSDarrick J. Wong 859afc51aaaSDarrick J. Wong pos += status; 860afc51aaaSDarrick J. Wong written += status; 861afc51aaaSDarrick J. Wong length -= status; 862afc51aaaSDarrick J. Wong 8638fc274d1SChristoph Hellwig balance_dirty_pages_ratelimited(iter->inode->i_mapping); 864afc51aaaSDarrick J. Wong } while (length); 865afc51aaaSDarrick J. Wong 866afc51aaaSDarrick J. Wong return written; 867afc51aaaSDarrick J. Wong } 868afc51aaaSDarrick J. Wong 869afc51aaaSDarrick J. Wong int 8703590c4d8SChristoph Hellwig iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, 871afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 872afc51aaaSDarrick J. Wong { 8738fc274d1SChristoph Hellwig struct iomap_iter iter = { 8748fc274d1SChristoph Hellwig .inode = inode, 8758fc274d1SChristoph Hellwig .pos = pos, 8768fc274d1SChristoph Hellwig .len = len, 877b74b1293SChristoph Hellwig .flags = IOMAP_WRITE | IOMAP_UNSHARE, 8788fc274d1SChristoph Hellwig }; 8798fc274d1SChristoph Hellwig int ret; 880afc51aaaSDarrick J. Wong 8818fc274d1SChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 8828fc274d1SChristoph Hellwig iter.processed = iomap_unshare_iter(&iter); 883afc51aaaSDarrick J. Wong return ret; 884afc51aaaSDarrick J. Wong } 8853590c4d8SChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_file_unshare); 886afc51aaaSDarrick J. Wong 8871b5c1e36SChristoph Hellwig static s64 __iomap_zero_iter(struct iomap_iter *iter, loff_t pos, u64 length) 888afc51aaaSDarrick J. Wong { 889afc51aaaSDarrick J. Wong struct page *page; 890afc51aaaSDarrick J. Wong int status; 89181ee8e52SMatthew Wilcox (Oracle) unsigned offset = offset_in_page(pos); 89281ee8e52SMatthew Wilcox (Oracle) unsigned bytes = min_t(u64, PAGE_SIZE - offset, length); 893afc51aaaSDarrick J. Wong 894b74b1293SChristoph Hellwig status = iomap_write_begin(iter, pos, bytes, &page); 895afc51aaaSDarrick J. Wong if (status) 896afc51aaaSDarrick J. Wong return status; 897afc51aaaSDarrick J. Wong 898afc51aaaSDarrick J. Wong zero_user(page, offset, bytes); 899afc51aaaSDarrick J. Wong mark_page_accessed(page); 900afc51aaaSDarrick J. Wong 9011b5c1e36SChristoph Hellwig return iomap_write_end(iter, pos, bytes, bytes, page); 902afc51aaaSDarrick J. Wong } 903afc51aaaSDarrick J. Wong 9042aa3048eSChristoph Hellwig static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero) 905afc51aaaSDarrick J. Wong { 9062aa3048eSChristoph Hellwig struct iomap *iomap = &iter->iomap; 907fad0a1abSChristoph Hellwig const struct iomap *srcmap = iomap_iter_srcmap(iter); 9082aa3048eSChristoph Hellwig loff_t pos = iter->pos; 9092aa3048eSChristoph Hellwig loff_t length = iomap_length(iter); 910afc51aaaSDarrick J. Wong loff_t written = 0; 911afc51aaaSDarrick J. Wong 912afc51aaaSDarrick J. Wong /* already zeroed? we're done. */ 913c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) 91481ee8e52SMatthew Wilcox (Oracle) return length; 915afc51aaaSDarrick J. Wong 916afc51aaaSDarrick J. Wong do { 91781ee8e52SMatthew Wilcox (Oracle) s64 bytes; 918afc51aaaSDarrick J. Wong 9192aa3048eSChristoph Hellwig if (IS_DAX(iter->inode)) 92081ee8e52SMatthew Wilcox (Oracle) bytes = dax_iomap_zero(pos, length, iomap); 921afc51aaaSDarrick J. Wong else 9221b5c1e36SChristoph Hellwig bytes = __iomap_zero_iter(iter, pos, length); 92381ee8e52SMatthew Wilcox (Oracle) if (bytes < 0) 92481ee8e52SMatthew Wilcox (Oracle) return bytes; 925afc51aaaSDarrick J. Wong 926afc51aaaSDarrick J. Wong pos += bytes; 92781ee8e52SMatthew Wilcox (Oracle) length -= bytes; 928afc51aaaSDarrick J. Wong written += bytes; 929afc51aaaSDarrick J. Wong if (did_zero) 930afc51aaaSDarrick J. Wong *did_zero = true; 93181ee8e52SMatthew Wilcox (Oracle) } while (length > 0); 932afc51aaaSDarrick J. Wong 933afc51aaaSDarrick J. Wong return written; 934afc51aaaSDarrick J. Wong } 935afc51aaaSDarrick J. Wong 936afc51aaaSDarrick J. Wong int 937afc51aaaSDarrick J. Wong iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, 938afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 939afc51aaaSDarrick J. Wong { 9402aa3048eSChristoph Hellwig struct iomap_iter iter = { 9412aa3048eSChristoph Hellwig .inode = inode, 9422aa3048eSChristoph Hellwig .pos = pos, 9432aa3048eSChristoph Hellwig .len = len, 9442aa3048eSChristoph Hellwig .flags = IOMAP_ZERO, 9452aa3048eSChristoph Hellwig }; 9462aa3048eSChristoph Hellwig int ret; 947afc51aaaSDarrick J. Wong 9482aa3048eSChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 9492aa3048eSChristoph Hellwig iter.processed = iomap_zero_iter(&iter, did_zero); 950afc51aaaSDarrick J. Wong return ret; 951afc51aaaSDarrick J. Wong } 952afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_zero_range); 953afc51aaaSDarrick J. Wong 954afc51aaaSDarrick J. Wong int 955afc51aaaSDarrick J. Wong iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, 956afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 957afc51aaaSDarrick J. Wong { 958afc51aaaSDarrick J. Wong unsigned int blocksize = i_blocksize(inode); 959afc51aaaSDarrick J. Wong unsigned int off = pos & (blocksize - 1); 960afc51aaaSDarrick J. Wong 961afc51aaaSDarrick J. Wong /* Block boundary? Nothing to do */ 962afc51aaaSDarrick J. Wong if (!off) 963afc51aaaSDarrick J. Wong return 0; 964afc51aaaSDarrick J. Wong return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops); 965afc51aaaSDarrick J. Wong } 966afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_truncate_page); 967afc51aaaSDarrick J. Wong 968253564baSChristoph Hellwig static loff_t iomap_page_mkwrite_iter(struct iomap_iter *iter, 969253564baSChristoph Hellwig struct page *page) 970afc51aaaSDarrick J. Wong { 971d1bd0b4eSMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 972253564baSChristoph Hellwig loff_t length = iomap_length(iter); 973afc51aaaSDarrick J. Wong int ret; 974afc51aaaSDarrick J. Wong 975253564baSChristoph Hellwig if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) { 976d1bd0b4eSMatthew Wilcox (Oracle) ret = __block_write_begin_int(folio, iter->pos, length, NULL, 977253564baSChristoph Hellwig &iter->iomap); 978afc51aaaSDarrick J. Wong if (ret) 979afc51aaaSDarrick J. Wong return ret; 980afc51aaaSDarrick J. Wong block_commit_write(page, 0, length); 981afc51aaaSDarrick J. Wong } else { 982afc51aaaSDarrick J. Wong WARN_ON_ONCE(!PageUptodate(page)); 983afc51aaaSDarrick J. Wong set_page_dirty(page); 984afc51aaaSDarrick J. Wong } 985afc51aaaSDarrick J. Wong 986afc51aaaSDarrick J. Wong return length; 987afc51aaaSDarrick J. Wong } 988afc51aaaSDarrick J. Wong 989afc51aaaSDarrick J. Wong vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops) 990afc51aaaSDarrick J. Wong { 991253564baSChristoph Hellwig struct iomap_iter iter = { 992253564baSChristoph Hellwig .inode = file_inode(vmf->vma->vm_file), 993253564baSChristoph Hellwig .flags = IOMAP_WRITE | IOMAP_FAULT, 994253564baSChristoph Hellwig }; 995afc51aaaSDarrick J. Wong struct page *page = vmf->page; 996afc51aaaSDarrick J. Wong ssize_t ret; 997afc51aaaSDarrick J. Wong 998afc51aaaSDarrick J. Wong lock_page(page); 999253564baSChristoph Hellwig ret = page_mkwrite_check_truncate(page, iter.inode); 1000243145bcSAndreas Gruenbacher if (ret < 0) 1001afc51aaaSDarrick J. Wong goto out_unlock; 1002253564baSChristoph Hellwig iter.pos = page_offset(page); 1003253564baSChristoph Hellwig iter.len = ret; 1004253564baSChristoph Hellwig while ((ret = iomap_iter(&iter, ops)) > 0) 1005253564baSChristoph Hellwig iter.processed = iomap_page_mkwrite_iter(&iter, page); 1006afc51aaaSDarrick J. Wong 1007253564baSChristoph Hellwig if (ret < 0) 1008afc51aaaSDarrick J. Wong goto out_unlock; 1009afc51aaaSDarrick J. Wong wait_for_stable_page(page); 1010afc51aaaSDarrick J. Wong return VM_FAULT_LOCKED; 1011afc51aaaSDarrick J. Wong out_unlock: 1012afc51aaaSDarrick J. Wong unlock_page(page); 1013afc51aaaSDarrick J. Wong return block_page_mkwrite_return(ret); 1014afc51aaaSDarrick J. Wong } 1015afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_page_mkwrite); 1016598ecfbaSChristoph Hellwig 1017*8ffd74e9SMatthew Wilcox (Oracle) static void iomap_finish_folio_write(struct inode *inode, struct folio *folio, 1018*8ffd74e9SMatthew Wilcox (Oracle) size_t len, int error) 1019598ecfbaSChristoph Hellwig { 102095c4cd05SMatthew Wilcox (Oracle) struct iomap_page *iop = to_iomap_page(folio); 1021598ecfbaSChristoph Hellwig 1022598ecfbaSChristoph Hellwig if (error) { 1023*8ffd74e9SMatthew Wilcox (Oracle) folio_set_error(folio); 1024b69eea82SDarrick J. Wong mapping_set_error(inode->i_mapping, error); 1025598ecfbaSChristoph Hellwig } 1026598ecfbaSChristoph Hellwig 1027*8ffd74e9SMatthew Wilcox (Oracle) WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !iop); 10280fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) <= 0); 1029598ecfbaSChristoph Hellwig 10300fb2d720SMatthew Wilcox (Oracle) if (!iop || atomic_sub_and_test(len, &iop->write_bytes_pending)) 1031*8ffd74e9SMatthew Wilcox (Oracle) folio_end_writeback(folio); 1032598ecfbaSChristoph Hellwig } 1033598ecfbaSChristoph Hellwig 1034598ecfbaSChristoph Hellwig /* 1035598ecfbaSChristoph Hellwig * We're now finished for good with this ioend structure. Update the page 1036598ecfbaSChristoph Hellwig * state, release holds on bios, and finally free up memory. Do not use the 1037598ecfbaSChristoph Hellwig * ioend after this. 1038598ecfbaSChristoph Hellwig */ 1039598ecfbaSChristoph Hellwig static void 1040598ecfbaSChristoph Hellwig iomap_finish_ioend(struct iomap_ioend *ioend, int error) 1041598ecfbaSChristoph Hellwig { 1042598ecfbaSChristoph Hellwig struct inode *inode = ioend->io_inode; 1043598ecfbaSChristoph Hellwig struct bio *bio = &ioend->io_inline_bio; 1044598ecfbaSChristoph Hellwig struct bio *last = ioend->io_bio, *next; 1045598ecfbaSChristoph Hellwig u64 start = bio->bi_iter.bi_sector; 1046c275779fSZorro Lang loff_t offset = ioend->io_offset; 1047598ecfbaSChristoph Hellwig bool quiet = bio_flagged(bio, BIO_QUIET); 1048598ecfbaSChristoph Hellwig 1049598ecfbaSChristoph Hellwig for (bio = &ioend->io_inline_bio; bio; bio = next) { 1050*8ffd74e9SMatthew Wilcox (Oracle) struct folio_iter fi; 1051598ecfbaSChristoph Hellwig 1052598ecfbaSChristoph Hellwig /* 1053598ecfbaSChristoph Hellwig * For the last bio, bi_private points to the ioend, so we 1054598ecfbaSChristoph Hellwig * need to explicitly end the iteration here. 1055598ecfbaSChristoph Hellwig */ 1056598ecfbaSChristoph Hellwig if (bio == last) 1057598ecfbaSChristoph Hellwig next = NULL; 1058598ecfbaSChristoph Hellwig else 1059598ecfbaSChristoph Hellwig next = bio->bi_private; 1060598ecfbaSChristoph Hellwig 1061*8ffd74e9SMatthew Wilcox (Oracle) /* walk all folios in bio, ending page IO on them */ 1062*8ffd74e9SMatthew Wilcox (Oracle) bio_for_each_folio_all(fi, bio) 1063*8ffd74e9SMatthew Wilcox (Oracle) iomap_finish_folio_write(inode, fi.folio, fi.length, 1064*8ffd74e9SMatthew Wilcox (Oracle) error); 1065598ecfbaSChristoph Hellwig bio_put(bio); 1066598ecfbaSChristoph Hellwig } 1067c275779fSZorro Lang /* The ioend has been freed by bio_put() */ 1068598ecfbaSChristoph Hellwig 1069598ecfbaSChristoph Hellwig if (unlikely(error && !quiet)) { 1070598ecfbaSChristoph Hellwig printk_ratelimited(KERN_ERR 10719cd0ed63SDarrick J. Wong "%s: writeback error on inode %lu, offset %lld, sector %llu", 1072c275779fSZorro Lang inode->i_sb->s_id, inode->i_ino, offset, start); 1073598ecfbaSChristoph Hellwig } 1074598ecfbaSChristoph Hellwig } 1075598ecfbaSChristoph Hellwig 1076598ecfbaSChristoph Hellwig void 1077598ecfbaSChristoph Hellwig iomap_finish_ioends(struct iomap_ioend *ioend, int error) 1078598ecfbaSChristoph Hellwig { 1079598ecfbaSChristoph Hellwig struct list_head tmp; 1080598ecfbaSChristoph Hellwig 1081598ecfbaSChristoph Hellwig list_replace_init(&ioend->io_list, &tmp); 1082598ecfbaSChristoph Hellwig iomap_finish_ioend(ioend, error); 1083598ecfbaSChristoph Hellwig 1084598ecfbaSChristoph Hellwig while (!list_empty(&tmp)) { 1085598ecfbaSChristoph Hellwig ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); 1086598ecfbaSChristoph Hellwig list_del_init(&ioend->io_list); 1087598ecfbaSChristoph Hellwig iomap_finish_ioend(ioend, error); 1088598ecfbaSChristoph Hellwig } 1089598ecfbaSChristoph Hellwig } 1090598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_finish_ioends); 1091598ecfbaSChristoph Hellwig 1092598ecfbaSChristoph Hellwig /* 1093598ecfbaSChristoph Hellwig * We can merge two adjacent ioends if they have the same set of work to do. 1094598ecfbaSChristoph Hellwig */ 1095598ecfbaSChristoph Hellwig static bool 1096598ecfbaSChristoph Hellwig iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next) 1097598ecfbaSChristoph Hellwig { 1098598ecfbaSChristoph Hellwig if (ioend->io_bio->bi_status != next->io_bio->bi_status) 1099598ecfbaSChristoph Hellwig return false; 1100598ecfbaSChristoph Hellwig if ((ioend->io_flags & IOMAP_F_SHARED) ^ 1101598ecfbaSChristoph Hellwig (next->io_flags & IOMAP_F_SHARED)) 1102598ecfbaSChristoph Hellwig return false; 1103598ecfbaSChristoph Hellwig if ((ioend->io_type == IOMAP_UNWRITTEN) ^ 1104598ecfbaSChristoph Hellwig (next->io_type == IOMAP_UNWRITTEN)) 1105598ecfbaSChristoph Hellwig return false; 1106598ecfbaSChristoph Hellwig if (ioend->io_offset + ioend->io_size != next->io_offset) 1107598ecfbaSChristoph Hellwig return false; 1108598ecfbaSChristoph Hellwig return true; 1109598ecfbaSChristoph Hellwig } 1110598ecfbaSChristoph Hellwig 1111598ecfbaSChristoph Hellwig void 11126e552494SBrian Foster iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends) 1113598ecfbaSChristoph Hellwig { 1114598ecfbaSChristoph Hellwig struct iomap_ioend *next; 1115598ecfbaSChristoph Hellwig 1116598ecfbaSChristoph Hellwig INIT_LIST_HEAD(&ioend->io_list); 1117598ecfbaSChristoph Hellwig 1118598ecfbaSChristoph Hellwig while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, 1119598ecfbaSChristoph Hellwig io_list))) { 1120598ecfbaSChristoph Hellwig if (!iomap_ioend_can_merge(ioend, next)) 1121598ecfbaSChristoph Hellwig break; 1122598ecfbaSChristoph Hellwig list_move_tail(&next->io_list, &ioend->io_list); 1123598ecfbaSChristoph Hellwig ioend->io_size += next->io_size; 1124598ecfbaSChristoph Hellwig } 1125598ecfbaSChristoph Hellwig } 1126598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); 1127598ecfbaSChristoph Hellwig 1128598ecfbaSChristoph Hellwig static int 11294f0f586bSSami Tolvanen iomap_ioend_compare(void *priv, const struct list_head *a, 11304f0f586bSSami Tolvanen const struct list_head *b) 1131598ecfbaSChristoph Hellwig { 1132b3d423ecSChristoph Hellwig struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); 1133b3d423ecSChristoph Hellwig struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); 1134598ecfbaSChristoph Hellwig 1135598ecfbaSChristoph Hellwig if (ia->io_offset < ib->io_offset) 1136598ecfbaSChristoph Hellwig return -1; 1137b3d423ecSChristoph Hellwig if (ia->io_offset > ib->io_offset) 1138598ecfbaSChristoph Hellwig return 1; 1139598ecfbaSChristoph Hellwig return 0; 1140598ecfbaSChristoph Hellwig } 1141598ecfbaSChristoph Hellwig 1142598ecfbaSChristoph Hellwig void 1143598ecfbaSChristoph Hellwig iomap_sort_ioends(struct list_head *ioend_list) 1144598ecfbaSChristoph Hellwig { 1145598ecfbaSChristoph Hellwig list_sort(NULL, ioend_list, iomap_ioend_compare); 1146598ecfbaSChristoph Hellwig } 1147598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_sort_ioends); 1148598ecfbaSChristoph Hellwig 1149598ecfbaSChristoph Hellwig static void iomap_writepage_end_bio(struct bio *bio) 1150598ecfbaSChristoph Hellwig { 1151598ecfbaSChristoph Hellwig struct iomap_ioend *ioend = bio->bi_private; 1152598ecfbaSChristoph Hellwig 1153598ecfbaSChristoph Hellwig iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status)); 1154598ecfbaSChristoph Hellwig } 1155598ecfbaSChristoph Hellwig 1156598ecfbaSChristoph Hellwig /* 1157598ecfbaSChristoph Hellwig * Submit the final bio for an ioend. 1158598ecfbaSChristoph Hellwig * 1159598ecfbaSChristoph Hellwig * If @error is non-zero, it means that we have a situation where some part of 1160f1f264b4SAndreas Gruenbacher * the submission process has failed after we've marked pages for writeback 1161598ecfbaSChristoph Hellwig * and unlocked them. In this situation, we need to fail the bio instead of 1162598ecfbaSChristoph Hellwig * submitting it. This typically only happens on a filesystem shutdown. 1163598ecfbaSChristoph Hellwig */ 1164598ecfbaSChristoph Hellwig static int 1165598ecfbaSChristoph Hellwig iomap_submit_ioend(struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend, 1166598ecfbaSChristoph Hellwig int error) 1167598ecfbaSChristoph Hellwig { 1168598ecfbaSChristoph Hellwig ioend->io_bio->bi_private = ioend; 1169598ecfbaSChristoph Hellwig ioend->io_bio->bi_end_io = iomap_writepage_end_bio; 1170598ecfbaSChristoph Hellwig 1171598ecfbaSChristoph Hellwig if (wpc->ops->prepare_ioend) 1172598ecfbaSChristoph Hellwig error = wpc->ops->prepare_ioend(ioend, error); 1173598ecfbaSChristoph Hellwig if (error) { 1174598ecfbaSChristoph Hellwig /* 1175f1f264b4SAndreas Gruenbacher * If we're failing the IO now, just mark the ioend with an 1176598ecfbaSChristoph Hellwig * error and finish it. This will run IO completion immediately 1177598ecfbaSChristoph Hellwig * as there is only one reference to the ioend at this point in 1178598ecfbaSChristoph Hellwig * time. 1179598ecfbaSChristoph Hellwig */ 1180598ecfbaSChristoph Hellwig ioend->io_bio->bi_status = errno_to_blk_status(error); 1181598ecfbaSChristoph Hellwig bio_endio(ioend->io_bio); 1182598ecfbaSChristoph Hellwig return error; 1183598ecfbaSChristoph Hellwig } 1184598ecfbaSChristoph Hellwig 1185598ecfbaSChristoph Hellwig submit_bio(ioend->io_bio); 1186598ecfbaSChristoph Hellwig return 0; 1187598ecfbaSChristoph Hellwig } 1188598ecfbaSChristoph Hellwig 1189598ecfbaSChristoph Hellwig static struct iomap_ioend * 1190598ecfbaSChristoph Hellwig iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc, 1191598ecfbaSChristoph Hellwig loff_t offset, sector_t sector, struct writeback_control *wbc) 1192598ecfbaSChristoph Hellwig { 1193598ecfbaSChristoph Hellwig struct iomap_ioend *ioend; 1194598ecfbaSChristoph Hellwig struct bio *bio; 1195598ecfbaSChristoph Hellwig 1196a8affc03SChristoph Hellwig bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_VECS, &iomap_ioend_bioset); 1197598ecfbaSChristoph Hellwig bio_set_dev(bio, wpc->iomap.bdev); 1198598ecfbaSChristoph Hellwig bio->bi_iter.bi_sector = sector; 1199598ecfbaSChristoph Hellwig bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc); 1200598ecfbaSChristoph Hellwig bio->bi_write_hint = inode->i_write_hint; 1201598ecfbaSChristoph Hellwig wbc_init_bio(wbc, bio); 1202598ecfbaSChristoph Hellwig 1203598ecfbaSChristoph Hellwig ioend = container_of(bio, struct iomap_ioend, io_inline_bio); 1204598ecfbaSChristoph Hellwig INIT_LIST_HEAD(&ioend->io_list); 1205598ecfbaSChristoph Hellwig ioend->io_type = wpc->iomap.type; 1206598ecfbaSChristoph Hellwig ioend->io_flags = wpc->iomap.flags; 1207598ecfbaSChristoph Hellwig ioend->io_inode = inode; 1208598ecfbaSChristoph Hellwig ioend->io_size = 0; 1209598ecfbaSChristoph Hellwig ioend->io_offset = offset; 1210598ecfbaSChristoph Hellwig ioend->io_bio = bio; 1211598ecfbaSChristoph Hellwig return ioend; 1212598ecfbaSChristoph Hellwig } 1213598ecfbaSChristoph Hellwig 1214598ecfbaSChristoph Hellwig /* 1215598ecfbaSChristoph Hellwig * Allocate a new bio, and chain the old bio to the new one. 1216598ecfbaSChristoph Hellwig * 1217f1f264b4SAndreas Gruenbacher * Note that we have to perform the chaining in this unintuitive order 1218598ecfbaSChristoph Hellwig * so that the bi_private linkage is set up in the right direction for the 1219598ecfbaSChristoph Hellwig * traversal in iomap_finish_ioend(). 1220598ecfbaSChristoph Hellwig */ 1221598ecfbaSChristoph Hellwig static struct bio * 1222598ecfbaSChristoph Hellwig iomap_chain_bio(struct bio *prev) 1223598ecfbaSChristoph Hellwig { 1224598ecfbaSChristoph Hellwig struct bio *new; 1225598ecfbaSChristoph Hellwig 1226a8affc03SChristoph Hellwig new = bio_alloc(GFP_NOFS, BIO_MAX_VECS); 1227598ecfbaSChristoph Hellwig bio_copy_dev(new, prev);/* also copies over blkcg information */ 1228598ecfbaSChristoph Hellwig new->bi_iter.bi_sector = bio_end_sector(prev); 1229598ecfbaSChristoph Hellwig new->bi_opf = prev->bi_opf; 1230598ecfbaSChristoph Hellwig new->bi_write_hint = prev->bi_write_hint; 1231598ecfbaSChristoph Hellwig 1232598ecfbaSChristoph Hellwig bio_chain(prev, new); 1233598ecfbaSChristoph Hellwig bio_get(prev); /* for iomap_finish_ioend */ 1234598ecfbaSChristoph Hellwig submit_bio(prev); 1235598ecfbaSChristoph Hellwig return new; 1236598ecfbaSChristoph Hellwig } 1237598ecfbaSChristoph Hellwig 1238598ecfbaSChristoph Hellwig static bool 1239598ecfbaSChristoph Hellwig iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset, 1240598ecfbaSChristoph Hellwig sector_t sector) 1241598ecfbaSChristoph Hellwig { 1242598ecfbaSChristoph Hellwig if ((wpc->iomap.flags & IOMAP_F_SHARED) != 1243598ecfbaSChristoph Hellwig (wpc->ioend->io_flags & IOMAP_F_SHARED)) 1244598ecfbaSChristoph Hellwig return false; 1245598ecfbaSChristoph Hellwig if (wpc->iomap.type != wpc->ioend->io_type) 1246598ecfbaSChristoph Hellwig return false; 1247598ecfbaSChristoph Hellwig if (offset != wpc->ioend->io_offset + wpc->ioend->io_size) 1248598ecfbaSChristoph Hellwig return false; 1249598ecfbaSChristoph Hellwig if (sector != bio_end_sector(wpc->ioend->io_bio)) 1250598ecfbaSChristoph Hellwig return false; 1251598ecfbaSChristoph Hellwig return true; 1252598ecfbaSChristoph Hellwig } 1253598ecfbaSChristoph Hellwig 1254598ecfbaSChristoph Hellwig /* 1255598ecfbaSChristoph Hellwig * Test to see if we have an existing ioend structure that we could append to 1256f1f264b4SAndreas Gruenbacher * first; otherwise finish off the current ioend and start another. 1257598ecfbaSChristoph Hellwig */ 1258598ecfbaSChristoph Hellwig static void 1259598ecfbaSChristoph Hellwig iomap_add_to_ioend(struct inode *inode, loff_t offset, struct page *page, 1260598ecfbaSChristoph Hellwig struct iomap_page *iop, struct iomap_writepage_ctx *wpc, 1261598ecfbaSChristoph Hellwig struct writeback_control *wbc, struct list_head *iolist) 1262598ecfbaSChristoph Hellwig { 1263598ecfbaSChristoph Hellwig sector_t sector = iomap_sector(&wpc->iomap, offset); 1264598ecfbaSChristoph Hellwig unsigned len = i_blocksize(inode); 1265598ecfbaSChristoph Hellwig unsigned poff = offset & (PAGE_SIZE - 1); 1266598ecfbaSChristoph Hellwig 1267598ecfbaSChristoph Hellwig if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, offset, sector)) { 1268598ecfbaSChristoph Hellwig if (wpc->ioend) 1269598ecfbaSChristoph Hellwig list_add(&wpc->ioend->io_list, iolist); 1270598ecfbaSChristoph Hellwig wpc->ioend = iomap_alloc_ioend(inode, wpc, offset, sector, wbc); 1271598ecfbaSChristoph Hellwig } 1272598ecfbaSChristoph Hellwig 1273c1b79f11SChristoph Hellwig if (bio_add_page(wpc->ioend->io_bio, page, len, poff) != len) { 1274c1b79f11SChristoph Hellwig wpc->ioend->io_bio = iomap_chain_bio(wpc->ioend->io_bio); 1275c1b79f11SChristoph Hellwig __bio_add_page(wpc->ioend->io_bio, page, len, poff); 1276c1b79f11SChristoph Hellwig } 1277c1b79f11SChristoph Hellwig 12780fb2d720SMatthew Wilcox (Oracle) if (iop) 12790fb2d720SMatthew Wilcox (Oracle) atomic_add(len, &iop->write_bytes_pending); 1280598ecfbaSChristoph Hellwig wpc->ioend->io_size += len; 1281598ecfbaSChristoph Hellwig wbc_account_cgroup_owner(wbc, page, len); 1282598ecfbaSChristoph Hellwig } 1283598ecfbaSChristoph Hellwig 1284598ecfbaSChristoph Hellwig /* 1285598ecfbaSChristoph Hellwig * We implement an immediate ioend submission policy here to avoid needing to 1286598ecfbaSChristoph Hellwig * chain multiple ioends and hence nest mempool allocations which can violate 1287f1f264b4SAndreas Gruenbacher * the forward progress guarantees we need to provide. The current ioend we're 1288f1f264b4SAndreas Gruenbacher * adding blocks to is cached in the writepage context, and if the new block 1289f1f264b4SAndreas Gruenbacher * doesn't append to the cached ioend, it will create a new ioend and cache that 1290598ecfbaSChristoph Hellwig * instead. 1291598ecfbaSChristoph Hellwig * 1292598ecfbaSChristoph Hellwig * If a new ioend is created and cached, the old ioend is returned and queued 1293598ecfbaSChristoph Hellwig * locally for submission once the entire page is processed or an error has been 1294598ecfbaSChristoph Hellwig * detected. While ioends are submitted immediately after they are completed, 1295598ecfbaSChristoph Hellwig * batching optimisations are provided by higher level block plugging. 1296598ecfbaSChristoph Hellwig * 1297598ecfbaSChristoph Hellwig * At the end of a writeback pass, there will be a cached ioend remaining on the 1298598ecfbaSChristoph Hellwig * writepage context that the caller will need to submit. 1299598ecfbaSChristoph Hellwig */ 1300598ecfbaSChristoph Hellwig static int 1301598ecfbaSChristoph Hellwig iomap_writepage_map(struct iomap_writepage_ctx *wpc, 1302598ecfbaSChristoph Hellwig struct writeback_control *wbc, struct inode *inode, 1303598ecfbaSChristoph Hellwig struct page *page, u64 end_offset) 1304598ecfbaSChristoph Hellwig { 1305435d44b3SMatthew Wilcox (Oracle) struct folio *folio = page_folio(page); 1306435d44b3SMatthew Wilcox (Oracle) struct iomap_page *iop = iomap_page_create(inode, folio); 1307598ecfbaSChristoph Hellwig struct iomap_ioend *ioend, *next; 1308598ecfbaSChristoph Hellwig unsigned len = i_blocksize(inode); 1309598ecfbaSChristoph Hellwig u64 file_offset; /* file offset of page */ 1310598ecfbaSChristoph Hellwig int error = 0, count = 0, i; 1311598ecfbaSChristoph Hellwig LIST_HEAD(submit_list); 1312598ecfbaSChristoph Hellwig 13130fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) != 0); 1314598ecfbaSChristoph Hellwig 1315598ecfbaSChristoph Hellwig /* 1316598ecfbaSChristoph Hellwig * Walk through the page to find areas to write back. If we run off the 1317598ecfbaSChristoph Hellwig * end of the current map or find the current map invalid, grab a new 1318598ecfbaSChristoph Hellwig * one. 1319598ecfbaSChristoph Hellwig */ 1320598ecfbaSChristoph Hellwig for (i = 0, file_offset = page_offset(page); 1321598ecfbaSChristoph Hellwig i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset; 1322598ecfbaSChristoph Hellwig i++, file_offset += len) { 1323598ecfbaSChristoph Hellwig if (iop && !test_bit(i, iop->uptodate)) 1324598ecfbaSChristoph Hellwig continue; 1325598ecfbaSChristoph Hellwig 1326598ecfbaSChristoph Hellwig error = wpc->ops->map_blocks(wpc, inode, file_offset); 1327598ecfbaSChristoph Hellwig if (error) 1328598ecfbaSChristoph Hellwig break; 13293e19e6f3SChristoph Hellwig if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE)) 13303e19e6f3SChristoph Hellwig continue; 1331598ecfbaSChristoph Hellwig if (wpc->iomap.type == IOMAP_HOLE) 1332598ecfbaSChristoph Hellwig continue; 1333598ecfbaSChristoph Hellwig iomap_add_to_ioend(inode, file_offset, page, iop, wpc, wbc, 1334598ecfbaSChristoph Hellwig &submit_list); 1335598ecfbaSChristoph Hellwig count++; 1336598ecfbaSChristoph Hellwig } 1337598ecfbaSChristoph Hellwig 1338598ecfbaSChristoph Hellwig WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list)); 1339598ecfbaSChristoph Hellwig WARN_ON_ONCE(!PageLocked(page)); 1340598ecfbaSChristoph Hellwig WARN_ON_ONCE(PageWriteback(page)); 134150e7d6c7SBrian Foster WARN_ON_ONCE(PageDirty(page)); 1342598ecfbaSChristoph Hellwig 1343598ecfbaSChristoph Hellwig /* 1344598ecfbaSChristoph Hellwig * We cannot cancel the ioend directly here on error. We may have 1345598ecfbaSChristoph Hellwig * already set other pages under writeback and hence we have to run I/O 1346598ecfbaSChristoph Hellwig * completion to mark the error state of the pages under writeback 1347598ecfbaSChristoph Hellwig * appropriately. 1348598ecfbaSChristoph Hellwig */ 1349598ecfbaSChristoph Hellwig if (unlikely(error)) { 1350598ecfbaSChristoph Hellwig /* 1351763e4cdcSBrian Foster * Let the filesystem know what portion of the current page 1352f1f264b4SAndreas Gruenbacher * failed to map. If the page hasn't been added to ioend, it 1353763e4cdcSBrian Foster * won't be affected by I/O completion and we must unlock it 1354763e4cdcSBrian Foster * now. 1355598ecfbaSChristoph Hellwig */ 1356598ecfbaSChristoph Hellwig if (wpc->ops->discard_page) 1357763e4cdcSBrian Foster wpc->ops->discard_page(page, file_offset); 1358763e4cdcSBrian Foster if (!count) { 1359598ecfbaSChristoph Hellwig ClearPageUptodate(page); 1360598ecfbaSChristoph Hellwig unlock_page(page); 1361598ecfbaSChristoph Hellwig goto done; 1362598ecfbaSChristoph Hellwig } 1363598ecfbaSChristoph Hellwig } 1364598ecfbaSChristoph Hellwig 136550e7d6c7SBrian Foster set_page_writeback(page); 1366598ecfbaSChristoph Hellwig unlock_page(page); 1367598ecfbaSChristoph Hellwig 1368598ecfbaSChristoph Hellwig /* 1369f1f264b4SAndreas Gruenbacher * Preserve the original error if there was one; catch 1370598ecfbaSChristoph Hellwig * submission errors here and propagate into subsequent ioend 1371598ecfbaSChristoph Hellwig * submissions. 1372598ecfbaSChristoph Hellwig */ 1373598ecfbaSChristoph Hellwig list_for_each_entry_safe(ioend, next, &submit_list, io_list) { 1374598ecfbaSChristoph Hellwig int error2; 1375598ecfbaSChristoph Hellwig 1376598ecfbaSChristoph Hellwig list_del_init(&ioend->io_list); 1377598ecfbaSChristoph Hellwig error2 = iomap_submit_ioend(wpc, ioend, error); 1378598ecfbaSChristoph Hellwig if (error2 && !error) 1379598ecfbaSChristoph Hellwig error = error2; 1380598ecfbaSChristoph Hellwig } 1381598ecfbaSChristoph Hellwig 1382598ecfbaSChristoph Hellwig /* 1383598ecfbaSChristoph Hellwig * We can end up here with no error and nothing to write only if we race 1384598ecfbaSChristoph Hellwig * with a partial page truncate on a sub-page block sized filesystem. 1385598ecfbaSChristoph Hellwig */ 1386598ecfbaSChristoph Hellwig if (!count) 1387598ecfbaSChristoph Hellwig end_page_writeback(page); 1388598ecfbaSChristoph Hellwig done: 1389598ecfbaSChristoph Hellwig mapping_set_error(page->mapping, error); 1390598ecfbaSChristoph Hellwig return error; 1391598ecfbaSChristoph Hellwig } 1392598ecfbaSChristoph Hellwig 1393598ecfbaSChristoph Hellwig /* 1394598ecfbaSChristoph Hellwig * Write out a dirty page. 1395598ecfbaSChristoph Hellwig * 1396f1f264b4SAndreas Gruenbacher * For delalloc space on the page, we need to allocate space and flush it. 1397f1f264b4SAndreas Gruenbacher * For unwritten space on the page, we need to start the conversion to 1398598ecfbaSChristoph Hellwig * regular allocated space. 1399598ecfbaSChristoph Hellwig */ 1400598ecfbaSChristoph Hellwig static int 1401598ecfbaSChristoph Hellwig iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data) 1402598ecfbaSChristoph Hellwig { 1403598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc = data; 1404598ecfbaSChristoph Hellwig struct inode *inode = page->mapping->host; 1405598ecfbaSChristoph Hellwig pgoff_t end_index; 1406598ecfbaSChristoph Hellwig u64 end_offset; 1407598ecfbaSChristoph Hellwig loff_t offset; 1408598ecfbaSChristoph Hellwig 14091ac99452SMatthew Wilcox (Oracle) trace_iomap_writepage(inode, page_offset(page), PAGE_SIZE); 1410598ecfbaSChristoph Hellwig 1411598ecfbaSChristoph Hellwig /* 1412f1f264b4SAndreas Gruenbacher * Refuse to write the page out if we're called from reclaim context. 1413598ecfbaSChristoph Hellwig * 1414598ecfbaSChristoph Hellwig * This avoids stack overflows when called from deeply used stacks in 1415598ecfbaSChristoph Hellwig * random callers for direct reclaim or memcg reclaim. We explicitly 1416598ecfbaSChristoph Hellwig * allow reclaim from kswapd as the stack usage there is relatively low. 1417598ecfbaSChristoph Hellwig * 1418598ecfbaSChristoph Hellwig * This should never happen except in the case of a VM regression so 1419598ecfbaSChristoph Hellwig * warn about it. 1420598ecfbaSChristoph Hellwig */ 1421598ecfbaSChristoph Hellwig if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == 1422598ecfbaSChristoph Hellwig PF_MEMALLOC)) 1423598ecfbaSChristoph Hellwig goto redirty; 1424598ecfbaSChristoph Hellwig 1425598ecfbaSChristoph Hellwig /* 1426598ecfbaSChristoph Hellwig * Is this page beyond the end of the file? 1427598ecfbaSChristoph Hellwig * 1428598ecfbaSChristoph Hellwig * The page index is less than the end_index, adjust the end_offset 1429598ecfbaSChristoph Hellwig * to the highest offset that this page should represent. 1430598ecfbaSChristoph Hellwig * ----------------------------------------------------- 1431598ecfbaSChristoph Hellwig * | file mapping | <EOF> | 1432598ecfbaSChristoph Hellwig * ----------------------------------------------------- 1433598ecfbaSChristoph Hellwig * | Page ... | Page N-2 | Page N-1 | Page N | | 1434598ecfbaSChristoph Hellwig * ^--------------------------------^----------|-------- 1435598ecfbaSChristoph Hellwig * | desired writeback range | see else | 1436598ecfbaSChristoph Hellwig * ---------------------------------^------------------| 1437598ecfbaSChristoph Hellwig */ 1438598ecfbaSChristoph Hellwig offset = i_size_read(inode); 1439598ecfbaSChristoph Hellwig end_index = offset >> PAGE_SHIFT; 1440598ecfbaSChristoph Hellwig if (page->index < end_index) 1441598ecfbaSChristoph Hellwig end_offset = (loff_t)(page->index + 1) << PAGE_SHIFT; 1442598ecfbaSChristoph Hellwig else { 1443598ecfbaSChristoph Hellwig /* 1444598ecfbaSChristoph Hellwig * Check whether the page to write out is beyond or straddles 1445598ecfbaSChristoph Hellwig * i_size or not. 1446598ecfbaSChristoph Hellwig * ------------------------------------------------------- 1447598ecfbaSChristoph Hellwig * | file mapping | <EOF> | 1448598ecfbaSChristoph Hellwig * ------------------------------------------------------- 1449598ecfbaSChristoph Hellwig * | Page ... | Page N-2 | Page N-1 | Page N | Beyond | 1450598ecfbaSChristoph Hellwig * ^--------------------------------^-----------|--------- 1451598ecfbaSChristoph Hellwig * | | Straddles | 1452598ecfbaSChristoph Hellwig * ---------------------------------^-----------|--------| 1453598ecfbaSChristoph Hellwig */ 1454598ecfbaSChristoph Hellwig unsigned offset_into_page = offset & (PAGE_SIZE - 1); 1455598ecfbaSChristoph Hellwig 1456598ecfbaSChristoph Hellwig /* 1457f1f264b4SAndreas Gruenbacher * Skip the page if it's fully outside i_size, e.g. due to a 1458f1f264b4SAndreas Gruenbacher * truncate operation that's in progress. We must redirty the 1459598ecfbaSChristoph Hellwig * page so that reclaim stops reclaiming it. Otherwise 1460598ecfbaSChristoph Hellwig * iomap_vm_releasepage() is called on it and gets confused. 1461598ecfbaSChristoph Hellwig * 1462f1f264b4SAndreas Gruenbacher * Note that the end_index is unsigned long. If the given 1463f1f264b4SAndreas Gruenbacher * offset is greater than 16TB on a 32-bit system then if we 1464f1f264b4SAndreas Gruenbacher * checked if the page is fully outside i_size with 1465f1f264b4SAndreas Gruenbacher * "if (page->index >= end_index + 1)", "end_index + 1" would 1466f1f264b4SAndreas Gruenbacher * overflow and evaluate to 0. Hence this page would be 1467f1f264b4SAndreas Gruenbacher * redirtied and written out repeatedly, which would result in 1468f1f264b4SAndreas Gruenbacher * an infinite loop; the user program performing this operation 1469f1f264b4SAndreas Gruenbacher * would hang. Instead, we can detect this situation by 1470f1f264b4SAndreas Gruenbacher * checking if the page is totally beyond i_size or if its 1471598ecfbaSChristoph Hellwig * offset is just equal to the EOF. 1472598ecfbaSChristoph Hellwig */ 1473598ecfbaSChristoph Hellwig if (page->index > end_index || 1474598ecfbaSChristoph Hellwig (page->index == end_index && offset_into_page == 0)) 1475598ecfbaSChristoph Hellwig goto redirty; 1476598ecfbaSChristoph Hellwig 1477598ecfbaSChristoph Hellwig /* 1478598ecfbaSChristoph Hellwig * The page straddles i_size. It must be zeroed out on each 1479598ecfbaSChristoph Hellwig * and every writepage invocation because it may be mmapped. 1480598ecfbaSChristoph Hellwig * "A file is mapped in multiples of the page size. For a file 1481598ecfbaSChristoph Hellwig * that is not a multiple of the page size, the remaining 1482598ecfbaSChristoph Hellwig * memory is zeroed when mapped, and writes to that region are 1483598ecfbaSChristoph Hellwig * not written out to the file." 1484598ecfbaSChristoph Hellwig */ 1485598ecfbaSChristoph Hellwig zero_user_segment(page, offset_into_page, PAGE_SIZE); 1486598ecfbaSChristoph Hellwig 1487598ecfbaSChristoph Hellwig /* Adjust the end_offset to the end of file */ 1488598ecfbaSChristoph Hellwig end_offset = offset; 1489598ecfbaSChristoph Hellwig } 1490598ecfbaSChristoph Hellwig 1491598ecfbaSChristoph Hellwig return iomap_writepage_map(wpc, wbc, inode, page, end_offset); 1492598ecfbaSChristoph Hellwig 1493598ecfbaSChristoph Hellwig redirty: 1494598ecfbaSChristoph Hellwig redirty_page_for_writepage(wbc, page); 1495598ecfbaSChristoph Hellwig unlock_page(page); 1496598ecfbaSChristoph Hellwig return 0; 1497598ecfbaSChristoph Hellwig } 1498598ecfbaSChristoph Hellwig 1499598ecfbaSChristoph Hellwig int 1500598ecfbaSChristoph Hellwig iomap_writepage(struct page *page, struct writeback_control *wbc, 1501598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc, 1502598ecfbaSChristoph Hellwig const struct iomap_writeback_ops *ops) 1503598ecfbaSChristoph Hellwig { 1504598ecfbaSChristoph Hellwig int ret; 1505598ecfbaSChristoph Hellwig 1506598ecfbaSChristoph Hellwig wpc->ops = ops; 1507598ecfbaSChristoph Hellwig ret = iomap_do_writepage(page, wbc, wpc); 1508598ecfbaSChristoph Hellwig if (!wpc->ioend) 1509598ecfbaSChristoph Hellwig return ret; 1510598ecfbaSChristoph Hellwig return iomap_submit_ioend(wpc, wpc->ioend, ret); 1511598ecfbaSChristoph Hellwig } 1512598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_writepage); 1513598ecfbaSChristoph Hellwig 1514598ecfbaSChristoph Hellwig int 1515598ecfbaSChristoph Hellwig iomap_writepages(struct address_space *mapping, struct writeback_control *wbc, 1516598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc, 1517598ecfbaSChristoph Hellwig const struct iomap_writeback_ops *ops) 1518598ecfbaSChristoph Hellwig { 1519598ecfbaSChristoph Hellwig int ret; 1520598ecfbaSChristoph Hellwig 1521598ecfbaSChristoph Hellwig wpc->ops = ops; 1522598ecfbaSChristoph Hellwig ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc); 1523598ecfbaSChristoph Hellwig if (!wpc->ioend) 1524598ecfbaSChristoph Hellwig return ret; 1525598ecfbaSChristoph Hellwig return iomap_submit_ioend(wpc, wpc->ioend, ret); 1526598ecfbaSChristoph Hellwig } 1527598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_writepages); 1528598ecfbaSChristoph Hellwig 1529598ecfbaSChristoph Hellwig static int __init iomap_init(void) 1530598ecfbaSChristoph Hellwig { 1531598ecfbaSChristoph Hellwig return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), 1532598ecfbaSChristoph Hellwig offsetof(struct iomap_ioend, io_inline_bio), 1533598ecfbaSChristoph Hellwig BIOSET_NEED_BVECS); 1534598ecfbaSChristoph Hellwig } 1535598ecfbaSChristoph Hellwig fs_initcall(iomap_init); 1536