1afc51aaaSDarrick J. Wong // SPDX-License-Identifier: GPL-2.0 2afc51aaaSDarrick J. Wong /* 3afc51aaaSDarrick J. Wong * Copyright (C) 2010 Red Hat, Inc. 4598ecfbaSChristoph Hellwig * Copyright (C) 2016-2019 Christoph Hellwig. 5afc51aaaSDarrick J. Wong */ 6afc51aaaSDarrick J. Wong #include <linux/module.h> 7afc51aaaSDarrick J. Wong #include <linux/compiler.h> 8afc51aaaSDarrick J. Wong #include <linux/fs.h> 9afc51aaaSDarrick J. Wong #include <linux/iomap.h> 10afc51aaaSDarrick J. Wong #include <linux/pagemap.h> 11afc51aaaSDarrick J. Wong #include <linux/uio.h> 12afc51aaaSDarrick J. Wong #include <linux/buffer_head.h> 13afc51aaaSDarrick J. Wong #include <linux/dax.h> 14afc51aaaSDarrick J. Wong #include <linux/writeback.h> 15598ecfbaSChristoph Hellwig #include <linux/list_sort.h> 16afc51aaaSDarrick J. Wong #include <linux/swap.h> 17afc51aaaSDarrick J. Wong #include <linux/bio.h> 18afc51aaaSDarrick J. Wong #include <linux/sched/signal.h> 19afc51aaaSDarrick J. Wong #include <linux/migrate.h> 209e91c572SChristoph Hellwig #include "trace.h" 21afc51aaaSDarrick J. Wong 22afc51aaaSDarrick J. Wong #include "../internal.h" 23afc51aaaSDarrick J. Wong 24ab08b01eSChristoph Hellwig /* 250a195b91SMatthew Wilcox (Oracle) * Structure allocated for each page or THP when block size < page size 260a195b91SMatthew Wilcox (Oracle) * to track sub-page uptodate status and I/O completions. 27ab08b01eSChristoph Hellwig */ 28ab08b01eSChristoph Hellwig struct iomap_page { 297d636676SMatthew Wilcox (Oracle) atomic_t read_bytes_pending; 300fb2d720SMatthew Wilcox (Oracle) atomic_t write_bytes_pending; 311cea335dSChristoph Hellwig spinlock_t uptodate_lock; 320a195b91SMatthew Wilcox (Oracle) unsigned long uptodate[]; 33ab08b01eSChristoph Hellwig }; 34ab08b01eSChristoph Hellwig 35ab08b01eSChristoph Hellwig static inline struct iomap_page *to_iomap_page(struct page *page) 36ab08b01eSChristoph Hellwig { 370a195b91SMatthew Wilcox (Oracle) /* 380a195b91SMatthew Wilcox (Oracle) * per-block data is stored in the head page. Callers should 390a195b91SMatthew Wilcox (Oracle) * not be dealing with tail pages (and if they are, they can 400a195b91SMatthew Wilcox (Oracle) * call thp_head() first. 410a195b91SMatthew Wilcox (Oracle) */ 420a195b91SMatthew Wilcox (Oracle) VM_BUG_ON_PGFLAGS(PageTail(page), page); 430a195b91SMatthew Wilcox (Oracle) 44ab08b01eSChristoph Hellwig if (page_has_private(page)) 45ab08b01eSChristoph Hellwig return (struct iomap_page *)page_private(page); 46ab08b01eSChristoph Hellwig return NULL; 47ab08b01eSChristoph Hellwig } 48ab08b01eSChristoph Hellwig 49598ecfbaSChristoph Hellwig static struct bio_set iomap_ioend_bioset; 50598ecfbaSChristoph Hellwig 51afc51aaaSDarrick J. Wong static struct iomap_page * 52afc51aaaSDarrick J. Wong iomap_page_create(struct inode *inode, struct page *page) 53afc51aaaSDarrick J. Wong { 54afc51aaaSDarrick J. Wong struct iomap_page *iop = to_iomap_page(page); 550a195b91SMatthew Wilcox (Oracle) unsigned int nr_blocks = i_blocks_per_page(inode, page); 56afc51aaaSDarrick J. Wong 570a195b91SMatthew Wilcox (Oracle) if (iop || nr_blocks <= 1) 58afc51aaaSDarrick J. Wong return iop; 59afc51aaaSDarrick J. Wong 600a195b91SMatthew Wilcox (Oracle) iop = kzalloc(struct_size(iop, uptodate, BITS_TO_LONGS(nr_blocks)), 610a195b91SMatthew Wilcox (Oracle) GFP_NOFS | __GFP_NOFAIL); 621cea335dSChristoph Hellwig spin_lock_init(&iop->uptodate_lock); 634595a298SMatthew Wilcox (Oracle) if (PageUptodate(page)) 644595a298SMatthew Wilcox (Oracle) bitmap_fill(iop->uptodate, nr_blocks); 6558aeb731SGuoqing Jiang attach_page_private(page, iop); 66afc51aaaSDarrick J. Wong return iop; 67afc51aaaSDarrick J. Wong } 68afc51aaaSDarrick J. Wong 69afc51aaaSDarrick J. Wong static void 70afc51aaaSDarrick J. Wong iomap_page_release(struct page *page) 71afc51aaaSDarrick J. Wong { 7258aeb731SGuoqing Jiang struct iomap_page *iop = detach_page_private(page); 730a195b91SMatthew Wilcox (Oracle) unsigned int nr_blocks = i_blocks_per_page(page->mapping->host, page); 74afc51aaaSDarrick J. Wong 75afc51aaaSDarrick J. Wong if (!iop) 76afc51aaaSDarrick J. Wong return; 777d636676SMatthew Wilcox (Oracle) WARN_ON_ONCE(atomic_read(&iop->read_bytes_pending)); 780fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(atomic_read(&iop->write_bytes_pending)); 790a195b91SMatthew Wilcox (Oracle) WARN_ON_ONCE(bitmap_full(iop->uptodate, nr_blocks) != 800a195b91SMatthew Wilcox (Oracle) PageUptodate(page)); 81afc51aaaSDarrick J. Wong kfree(iop); 82afc51aaaSDarrick J. Wong } 83afc51aaaSDarrick J. Wong 84afc51aaaSDarrick J. Wong /* 85afc51aaaSDarrick J. Wong * Calculate the range inside the page that we actually need to read. 86afc51aaaSDarrick J. Wong */ 87afc51aaaSDarrick J. Wong static void 88afc51aaaSDarrick J. Wong iomap_adjust_read_range(struct inode *inode, struct iomap_page *iop, 89afc51aaaSDarrick J. Wong loff_t *pos, loff_t length, unsigned *offp, unsigned *lenp) 90afc51aaaSDarrick J. Wong { 91afc51aaaSDarrick J. Wong loff_t orig_pos = *pos; 92afc51aaaSDarrick J. Wong loff_t isize = i_size_read(inode); 93afc51aaaSDarrick J. Wong unsigned block_bits = inode->i_blkbits; 94afc51aaaSDarrick J. Wong unsigned block_size = (1 << block_bits); 95afc51aaaSDarrick J. Wong unsigned poff = offset_in_page(*pos); 96afc51aaaSDarrick J. Wong unsigned plen = min_t(loff_t, PAGE_SIZE - poff, length); 97afc51aaaSDarrick J. Wong unsigned first = poff >> block_bits; 98afc51aaaSDarrick J. Wong unsigned last = (poff + plen - 1) >> block_bits; 99afc51aaaSDarrick J. Wong 100afc51aaaSDarrick J. Wong /* 101afc51aaaSDarrick J. Wong * If the block size is smaller than the page size we need to check the 102afc51aaaSDarrick J. Wong * per-block uptodate status and adjust the offset and length if needed 103afc51aaaSDarrick J. Wong * to avoid reading in already uptodate ranges. 104afc51aaaSDarrick J. Wong */ 105afc51aaaSDarrick J. Wong if (iop) { 106afc51aaaSDarrick J. Wong unsigned int i; 107afc51aaaSDarrick J. Wong 108afc51aaaSDarrick J. Wong /* move forward for each leading block marked uptodate */ 109afc51aaaSDarrick J. Wong for (i = first; i <= last; i++) { 110afc51aaaSDarrick J. Wong if (!test_bit(i, iop->uptodate)) 111afc51aaaSDarrick J. Wong break; 112afc51aaaSDarrick J. Wong *pos += block_size; 113afc51aaaSDarrick J. Wong poff += block_size; 114afc51aaaSDarrick J. Wong plen -= block_size; 115afc51aaaSDarrick J. Wong first++; 116afc51aaaSDarrick J. Wong } 117afc51aaaSDarrick J. Wong 118afc51aaaSDarrick J. Wong /* truncate len if we find any trailing uptodate block(s) */ 119afc51aaaSDarrick J. Wong for ( ; i <= last; i++) { 120afc51aaaSDarrick J. Wong if (test_bit(i, iop->uptodate)) { 121afc51aaaSDarrick J. Wong plen -= (last - i + 1) * block_size; 122afc51aaaSDarrick J. Wong last = i - 1; 123afc51aaaSDarrick J. Wong break; 124afc51aaaSDarrick J. Wong } 125afc51aaaSDarrick J. Wong } 126afc51aaaSDarrick J. Wong } 127afc51aaaSDarrick J. Wong 128afc51aaaSDarrick J. Wong /* 129afc51aaaSDarrick J. Wong * If the extent spans the block that contains the i_size we need to 130afc51aaaSDarrick J. Wong * handle both halves separately so that we properly zero data in the 131afc51aaaSDarrick J. Wong * page cache for blocks that are entirely outside of i_size. 132afc51aaaSDarrick J. Wong */ 133afc51aaaSDarrick J. Wong if (orig_pos <= isize && orig_pos + length > isize) { 134afc51aaaSDarrick J. Wong unsigned end = offset_in_page(isize - 1) >> block_bits; 135afc51aaaSDarrick J. Wong 136afc51aaaSDarrick J. Wong if (first <= end && last > end) 137afc51aaaSDarrick J. Wong plen -= (last - end) * block_size; 138afc51aaaSDarrick J. Wong } 139afc51aaaSDarrick J. Wong 140afc51aaaSDarrick J. Wong *offp = poff; 141afc51aaaSDarrick J. Wong *lenp = plen; 142afc51aaaSDarrick J. Wong } 143afc51aaaSDarrick J. Wong 144afc51aaaSDarrick J. Wong static void 1451cea335dSChristoph Hellwig iomap_iop_set_range_uptodate(struct page *page, unsigned off, unsigned len) 146afc51aaaSDarrick J. Wong { 147afc51aaaSDarrick J. Wong struct iomap_page *iop = to_iomap_page(page); 148afc51aaaSDarrick J. Wong struct inode *inode = page->mapping->host; 149afc51aaaSDarrick J. Wong unsigned first = off >> inode->i_blkbits; 150afc51aaaSDarrick J. Wong unsigned last = (off + len - 1) >> inode->i_blkbits; 1511cea335dSChristoph Hellwig unsigned long flags; 152afc51aaaSDarrick J. Wong 1531cea335dSChristoph Hellwig spin_lock_irqsave(&iop->uptodate_lock, flags); 154b21866f5SMatthew Wilcox (Oracle) bitmap_set(iop->uptodate, first, last - first + 1); 155b21866f5SMatthew Wilcox (Oracle) if (bitmap_full(iop->uptodate, i_blocks_per_page(inode, page))) 1561cea335dSChristoph Hellwig SetPageUptodate(page); 1571cea335dSChristoph Hellwig spin_unlock_irqrestore(&iop->uptodate_lock, flags); 158afc51aaaSDarrick J. Wong } 159afc51aaaSDarrick J. Wong 1601cea335dSChristoph Hellwig static void 1611cea335dSChristoph Hellwig iomap_set_range_uptodate(struct page *page, unsigned off, unsigned len) 1621cea335dSChristoph Hellwig { 1631cea335dSChristoph Hellwig if (PageError(page)) 1641cea335dSChristoph Hellwig return; 1651cea335dSChristoph Hellwig 1661cea335dSChristoph Hellwig if (page_has_private(page)) 1671cea335dSChristoph Hellwig iomap_iop_set_range_uptodate(page, off, len); 1681cea335dSChristoph Hellwig else 169afc51aaaSDarrick J. Wong SetPageUptodate(page); 170afc51aaaSDarrick J. Wong } 171afc51aaaSDarrick J. Wong 172afc51aaaSDarrick J. Wong static void 173afc51aaaSDarrick J. Wong iomap_read_page_end_io(struct bio_vec *bvec, int error) 174afc51aaaSDarrick J. Wong { 175afc51aaaSDarrick J. Wong struct page *page = bvec->bv_page; 176afc51aaaSDarrick J. Wong struct iomap_page *iop = to_iomap_page(page); 177afc51aaaSDarrick J. Wong 178afc51aaaSDarrick J. Wong if (unlikely(error)) { 179afc51aaaSDarrick J. Wong ClearPageUptodate(page); 180afc51aaaSDarrick J. Wong SetPageError(page); 181afc51aaaSDarrick J. Wong } else { 182afc51aaaSDarrick J. Wong iomap_set_range_uptodate(page, bvec->bv_offset, bvec->bv_len); 183afc51aaaSDarrick J. Wong } 184afc51aaaSDarrick J. Wong 1857d636676SMatthew Wilcox (Oracle) if (!iop || atomic_sub_and_test(bvec->bv_len, &iop->read_bytes_pending)) 1867d636676SMatthew Wilcox (Oracle) unlock_page(page); 187afc51aaaSDarrick J. Wong } 188afc51aaaSDarrick J. Wong 189afc51aaaSDarrick J. Wong static void 190afc51aaaSDarrick J. Wong iomap_read_end_io(struct bio *bio) 191afc51aaaSDarrick J. Wong { 192afc51aaaSDarrick J. Wong int error = blk_status_to_errno(bio->bi_status); 193afc51aaaSDarrick J. Wong struct bio_vec *bvec; 194afc51aaaSDarrick J. Wong struct bvec_iter_all iter_all; 195afc51aaaSDarrick J. Wong 196afc51aaaSDarrick J. Wong bio_for_each_segment_all(bvec, bio, iter_all) 197afc51aaaSDarrick J. Wong iomap_read_page_end_io(bvec, error); 198afc51aaaSDarrick J. Wong bio_put(bio); 199afc51aaaSDarrick J. Wong } 200afc51aaaSDarrick J. Wong 201afc51aaaSDarrick J. Wong struct iomap_readpage_ctx { 202afc51aaaSDarrick J. Wong struct page *cur_page; 203afc51aaaSDarrick J. Wong bool cur_page_in_bio; 204afc51aaaSDarrick J. Wong struct bio *bio; 2059d24a13aSMatthew Wilcox (Oracle) struct readahead_control *rac; 206afc51aaaSDarrick J. Wong }; 207afc51aaaSDarrick J. Wong 208*69f4a26cSGao Xiang static int iomap_read_inline_data(struct inode *inode, struct page *page, 209afc51aaaSDarrick J. Wong struct iomap *iomap) 210afc51aaaSDarrick J. Wong { 211*69f4a26cSGao Xiang size_t size = i_size_read(inode) - iomap->offset; 212afc51aaaSDarrick J. Wong void *addr; 213afc51aaaSDarrick J. Wong 214afc51aaaSDarrick J. Wong if (PageUptodate(page)) 215*69f4a26cSGao Xiang return 0; 216afc51aaaSDarrick J. Wong 217*69f4a26cSGao Xiang /* inline data must start page aligned in the file */ 218*69f4a26cSGao Xiang if (WARN_ON_ONCE(offset_in_page(iomap->offset))) 219*69f4a26cSGao Xiang return -EIO; 220*69f4a26cSGao Xiang if (WARN_ON_ONCE(size > PAGE_SIZE - 221*69f4a26cSGao Xiang offset_in_page(iomap->inline_data))) 222*69f4a26cSGao Xiang return -EIO; 223*69f4a26cSGao Xiang if (WARN_ON_ONCE(size > iomap->length)) 224*69f4a26cSGao Xiang return -EIO; 225*69f4a26cSGao Xiang if (WARN_ON_ONCE(page_has_private(page))) 226*69f4a26cSGao Xiang return -EIO; 227afc51aaaSDarrick J. Wong 228afc51aaaSDarrick J. Wong addr = kmap_atomic(page); 229afc51aaaSDarrick J. Wong memcpy(addr, iomap->inline_data, size); 230afc51aaaSDarrick J. Wong memset(addr + size, 0, PAGE_SIZE - size); 231afc51aaaSDarrick J. Wong kunmap_atomic(addr); 232afc51aaaSDarrick J. Wong SetPageUptodate(page); 233*69f4a26cSGao Xiang return 0; 234afc51aaaSDarrick J. Wong } 235afc51aaaSDarrick J. Wong 236009d8d84SChristoph Hellwig static inline bool iomap_block_needs_zeroing(struct inode *inode, 237009d8d84SChristoph Hellwig struct iomap *iomap, loff_t pos) 238009d8d84SChristoph Hellwig { 239009d8d84SChristoph Hellwig return iomap->type != IOMAP_MAPPED || 240009d8d84SChristoph Hellwig (iomap->flags & IOMAP_F_NEW) || 241009d8d84SChristoph Hellwig pos >= i_size_read(inode); 242009d8d84SChristoph Hellwig } 243009d8d84SChristoph Hellwig 244afc51aaaSDarrick J. Wong static loff_t 245afc51aaaSDarrick J. Wong iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data, 246c039b997SGoldwyn Rodrigues struct iomap *iomap, struct iomap *srcmap) 247afc51aaaSDarrick J. Wong { 248afc51aaaSDarrick J. Wong struct iomap_readpage_ctx *ctx = data; 249afc51aaaSDarrick J. Wong struct page *page = ctx->cur_page; 250637d3375SAndreas Gruenbacher struct iomap_page *iop; 251afc51aaaSDarrick J. Wong loff_t orig_pos = pos; 252afc51aaaSDarrick J. Wong unsigned poff, plen; 253afc51aaaSDarrick J. Wong sector_t sector; 254afc51aaaSDarrick J. Wong 255afc51aaaSDarrick J. Wong if (iomap->type == IOMAP_INLINE) { 256*69f4a26cSGao Xiang int ret = iomap_read_inline_data(inode, page, iomap); 257*69f4a26cSGao Xiang 258*69f4a26cSGao Xiang if (ret) 259*69f4a26cSGao Xiang return ret; 260afc51aaaSDarrick J. Wong return PAGE_SIZE; 261afc51aaaSDarrick J. Wong } 262afc51aaaSDarrick J. Wong 263afc51aaaSDarrick J. Wong /* zero post-eof blocks as the page may be mapped */ 264637d3375SAndreas Gruenbacher iop = iomap_page_create(inode, page); 265afc51aaaSDarrick J. Wong iomap_adjust_read_range(inode, iop, &pos, length, &poff, &plen); 266afc51aaaSDarrick J. Wong if (plen == 0) 267afc51aaaSDarrick J. Wong goto done; 268afc51aaaSDarrick J. Wong 269009d8d84SChristoph Hellwig if (iomap_block_needs_zeroing(inode, iomap, pos)) { 270afc51aaaSDarrick J. Wong zero_user(page, poff, plen); 271afc51aaaSDarrick J. Wong iomap_set_range_uptodate(page, poff, plen); 272afc51aaaSDarrick J. Wong goto done; 273afc51aaaSDarrick J. Wong } 274afc51aaaSDarrick J. Wong 275afc51aaaSDarrick J. Wong ctx->cur_page_in_bio = true; 2767d636676SMatthew Wilcox (Oracle) if (iop) 2777d636676SMatthew Wilcox (Oracle) atomic_add(plen, &iop->read_bytes_pending); 278afc51aaaSDarrick J. Wong 279afc51aaaSDarrick J. Wong sector = iomap_sector(iomap, pos); 280d0364f94SChristoph Hellwig if (!ctx->bio || 281d0364f94SChristoph Hellwig bio_end_sector(ctx->bio) != sector || 282d0364f94SChristoph Hellwig bio_add_page(ctx->bio, page, plen, poff) != plen) { 283afc51aaaSDarrick J. Wong gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL); 284457df33eSMatthew Wilcox (Oracle) gfp_t orig_gfp = gfp; 2855f7136dbSMatthew Wilcox (Oracle) unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE); 286afc51aaaSDarrick J. Wong 287afc51aaaSDarrick J. Wong if (ctx->bio) 288afc51aaaSDarrick J. Wong submit_bio(ctx->bio); 289afc51aaaSDarrick J. Wong 2909d24a13aSMatthew Wilcox (Oracle) if (ctx->rac) /* same as readahead_gfp_mask */ 291afc51aaaSDarrick J. Wong gfp |= __GFP_NORETRY | __GFP_NOWARN; 2925f7136dbSMatthew Wilcox (Oracle) ctx->bio = bio_alloc(gfp, bio_max_segs(nr_vecs)); 293457df33eSMatthew Wilcox (Oracle) /* 294457df33eSMatthew Wilcox (Oracle) * If the bio_alloc fails, try it again for a single page to 295457df33eSMatthew Wilcox (Oracle) * avoid having to deal with partial page reads. This emulates 296457df33eSMatthew Wilcox (Oracle) * what do_mpage_readpage does. 297457df33eSMatthew Wilcox (Oracle) */ 298457df33eSMatthew Wilcox (Oracle) if (!ctx->bio) 299457df33eSMatthew Wilcox (Oracle) ctx->bio = bio_alloc(orig_gfp, 1); 300afc51aaaSDarrick J. Wong ctx->bio->bi_opf = REQ_OP_READ; 3019d24a13aSMatthew Wilcox (Oracle) if (ctx->rac) 302afc51aaaSDarrick J. Wong ctx->bio->bi_opf |= REQ_RAHEAD; 303afc51aaaSDarrick J. Wong ctx->bio->bi_iter.bi_sector = sector; 304afc51aaaSDarrick J. Wong bio_set_dev(ctx->bio, iomap->bdev); 305afc51aaaSDarrick J. Wong ctx->bio->bi_end_io = iomap_read_end_io; 306d0364f94SChristoph Hellwig __bio_add_page(ctx->bio, page, plen, poff); 307afc51aaaSDarrick J. Wong } 308afc51aaaSDarrick J. Wong done: 309afc51aaaSDarrick J. Wong /* 310afc51aaaSDarrick J. Wong * Move the caller beyond our range so that it keeps making progress. 311afc51aaaSDarrick J. Wong * For that we have to include any leading non-uptodate ranges, but 312afc51aaaSDarrick J. Wong * we can skip trailing ones as they will be handled in the next 313afc51aaaSDarrick J. Wong * iteration. 314afc51aaaSDarrick J. Wong */ 315afc51aaaSDarrick J. Wong return pos - orig_pos + plen; 316afc51aaaSDarrick J. Wong } 317afc51aaaSDarrick J. Wong 318afc51aaaSDarrick J. Wong int 319afc51aaaSDarrick J. Wong iomap_readpage(struct page *page, const struct iomap_ops *ops) 320afc51aaaSDarrick J. Wong { 321afc51aaaSDarrick J. Wong struct iomap_readpage_ctx ctx = { .cur_page = page }; 322afc51aaaSDarrick J. Wong struct inode *inode = page->mapping->host; 323afc51aaaSDarrick J. Wong unsigned poff; 324afc51aaaSDarrick J. Wong loff_t ret; 325afc51aaaSDarrick J. Wong 3269e91c572SChristoph Hellwig trace_iomap_readpage(page->mapping->host, 1); 3279e91c572SChristoph Hellwig 328afc51aaaSDarrick J. Wong for (poff = 0; poff < PAGE_SIZE; poff += ret) { 329afc51aaaSDarrick J. Wong ret = iomap_apply(inode, page_offset(page) + poff, 330afc51aaaSDarrick J. Wong PAGE_SIZE - poff, 0, ops, &ctx, 331afc51aaaSDarrick J. Wong iomap_readpage_actor); 332afc51aaaSDarrick J. Wong if (ret <= 0) { 333afc51aaaSDarrick J. Wong WARN_ON_ONCE(ret == 0); 334afc51aaaSDarrick J. Wong SetPageError(page); 335afc51aaaSDarrick J. Wong break; 336afc51aaaSDarrick J. Wong } 337afc51aaaSDarrick J. Wong } 338afc51aaaSDarrick J. Wong 339afc51aaaSDarrick J. Wong if (ctx.bio) { 340afc51aaaSDarrick J. Wong submit_bio(ctx.bio); 341afc51aaaSDarrick J. Wong WARN_ON_ONCE(!ctx.cur_page_in_bio); 342afc51aaaSDarrick J. Wong } else { 343afc51aaaSDarrick J. Wong WARN_ON_ONCE(ctx.cur_page_in_bio); 344afc51aaaSDarrick J. Wong unlock_page(page); 345afc51aaaSDarrick J. Wong } 346afc51aaaSDarrick J. Wong 347afc51aaaSDarrick J. Wong /* 348d4388340SMatthew Wilcox (Oracle) * Just like mpage_readahead and block_read_full_page we always 349afc51aaaSDarrick J. Wong * return 0 and just mark the page as PageError on errors. This 350afc51aaaSDarrick J. Wong * should be cleaned up all through the stack eventually. 351afc51aaaSDarrick J. Wong */ 352afc51aaaSDarrick J. Wong return 0; 353afc51aaaSDarrick J. Wong } 354afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_readpage); 355afc51aaaSDarrick J. Wong 356afc51aaaSDarrick J. Wong static loff_t 3579d24a13aSMatthew Wilcox (Oracle) iomap_readahead_actor(struct inode *inode, loff_t pos, loff_t length, 358c039b997SGoldwyn Rodrigues void *data, struct iomap *iomap, struct iomap *srcmap) 359afc51aaaSDarrick J. Wong { 360afc51aaaSDarrick J. Wong struct iomap_readpage_ctx *ctx = data; 361afc51aaaSDarrick J. Wong loff_t done, ret; 362afc51aaaSDarrick J. Wong 363afc51aaaSDarrick J. Wong for (done = 0; done < length; done += ret) { 364afc51aaaSDarrick J. Wong if (ctx->cur_page && offset_in_page(pos + done) == 0) { 365afc51aaaSDarrick J. Wong if (!ctx->cur_page_in_bio) 366afc51aaaSDarrick J. Wong unlock_page(ctx->cur_page); 367afc51aaaSDarrick J. Wong put_page(ctx->cur_page); 368afc51aaaSDarrick J. Wong ctx->cur_page = NULL; 369afc51aaaSDarrick J. Wong } 370afc51aaaSDarrick J. Wong if (!ctx->cur_page) { 3719d24a13aSMatthew Wilcox (Oracle) ctx->cur_page = readahead_page(ctx->rac); 372afc51aaaSDarrick J. Wong ctx->cur_page_in_bio = false; 373afc51aaaSDarrick J. Wong } 374afc51aaaSDarrick J. Wong ret = iomap_readpage_actor(inode, pos + done, length - done, 375c039b997SGoldwyn Rodrigues ctx, iomap, srcmap); 376afc51aaaSDarrick J. Wong } 377afc51aaaSDarrick J. Wong 378afc51aaaSDarrick J. Wong return done; 379afc51aaaSDarrick J. Wong } 380afc51aaaSDarrick J. Wong 3819d24a13aSMatthew Wilcox (Oracle) /** 3829d24a13aSMatthew Wilcox (Oracle) * iomap_readahead - Attempt to read pages from a file. 3839d24a13aSMatthew Wilcox (Oracle) * @rac: Describes the pages to be read. 3849d24a13aSMatthew Wilcox (Oracle) * @ops: The operations vector for the filesystem. 3859d24a13aSMatthew Wilcox (Oracle) * 3869d24a13aSMatthew Wilcox (Oracle) * This function is for filesystems to call to implement their readahead 3879d24a13aSMatthew Wilcox (Oracle) * address_space operation. 3889d24a13aSMatthew Wilcox (Oracle) * 3899d24a13aSMatthew Wilcox (Oracle) * Context: The @ops callbacks may submit I/O (eg to read the addresses of 3909d24a13aSMatthew Wilcox (Oracle) * blocks from disc), and may wait for it. The caller may be trying to 3919d24a13aSMatthew Wilcox (Oracle) * access a different page, and so sleeping excessively should be avoided. 3929d24a13aSMatthew Wilcox (Oracle) * It may allocate memory, but should avoid costly allocations. This 3939d24a13aSMatthew Wilcox (Oracle) * function is called with memalloc_nofs set, so allocations will not cause 3949d24a13aSMatthew Wilcox (Oracle) * the filesystem to be reentered. 3959d24a13aSMatthew Wilcox (Oracle) */ 3969d24a13aSMatthew Wilcox (Oracle) void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops) 397afc51aaaSDarrick J. Wong { 3989d24a13aSMatthew Wilcox (Oracle) struct inode *inode = rac->mapping->host; 3999d24a13aSMatthew Wilcox (Oracle) loff_t pos = readahead_pos(rac); 400076171a6SMatthew Wilcox (Oracle) size_t length = readahead_length(rac); 401afc51aaaSDarrick J. Wong struct iomap_readpage_ctx ctx = { 4029d24a13aSMatthew Wilcox (Oracle) .rac = rac, 403afc51aaaSDarrick J. Wong }; 404afc51aaaSDarrick J. Wong 4059d24a13aSMatthew Wilcox (Oracle) trace_iomap_readahead(inode, readahead_count(rac)); 4069e91c572SChristoph Hellwig 407afc51aaaSDarrick J. Wong while (length > 0) { 408076171a6SMatthew Wilcox (Oracle) ssize_t ret = iomap_apply(inode, pos, length, 0, ops, 4099d24a13aSMatthew Wilcox (Oracle) &ctx, iomap_readahead_actor); 410afc51aaaSDarrick J. Wong if (ret <= 0) { 411afc51aaaSDarrick J. Wong WARN_ON_ONCE(ret == 0); 4129d24a13aSMatthew Wilcox (Oracle) break; 413afc51aaaSDarrick J. Wong } 414afc51aaaSDarrick J. Wong pos += ret; 415afc51aaaSDarrick J. Wong length -= ret; 416afc51aaaSDarrick J. Wong } 4179d24a13aSMatthew Wilcox (Oracle) 418afc51aaaSDarrick J. Wong if (ctx.bio) 419afc51aaaSDarrick J. Wong submit_bio(ctx.bio); 420afc51aaaSDarrick J. Wong if (ctx.cur_page) { 421afc51aaaSDarrick J. Wong if (!ctx.cur_page_in_bio) 422afc51aaaSDarrick J. Wong unlock_page(ctx.cur_page); 423afc51aaaSDarrick J. Wong put_page(ctx.cur_page); 424afc51aaaSDarrick J. Wong } 425afc51aaaSDarrick J. Wong } 4269d24a13aSMatthew Wilcox (Oracle) EXPORT_SYMBOL_GPL(iomap_readahead); 427afc51aaaSDarrick J. Wong 428afc51aaaSDarrick J. Wong /* 429afc51aaaSDarrick J. Wong * iomap_is_partially_uptodate checks whether blocks within a page are 430afc51aaaSDarrick J. Wong * uptodate or not. 431afc51aaaSDarrick J. Wong * 432afc51aaaSDarrick J. Wong * Returns true if all blocks which correspond to a file portion 433afc51aaaSDarrick J. Wong * we want to read within the page are uptodate. 434afc51aaaSDarrick J. Wong */ 435afc51aaaSDarrick J. Wong int 436afc51aaaSDarrick J. Wong iomap_is_partially_uptodate(struct page *page, unsigned long from, 437afc51aaaSDarrick J. Wong unsigned long count) 438afc51aaaSDarrick J. Wong { 439afc51aaaSDarrick J. Wong struct iomap_page *iop = to_iomap_page(page); 440afc51aaaSDarrick J. Wong struct inode *inode = page->mapping->host; 441afc51aaaSDarrick J. Wong unsigned len, first, last; 442afc51aaaSDarrick J. Wong unsigned i; 443afc51aaaSDarrick J. Wong 444afc51aaaSDarrick J. Wong /* Limit range to one page */ 445afc51aaaSDarrick J. Wong len = min_t(unsigned, PAGE_SIZE - from, count); 446afc51aaaSDarrick J. Wong 447afc51aaaSDarrick J. Wong /* First and last blocks in range within page */ 448afc51aaaSDarrick J. Wong first = from >> inode->i_blkbits; 449afc51aaaSDarrick J. Wong last = (from + len - 1) >> inode->i_blkbits; 450afc51aaaSDarrick J. Wong 451afc51aaaSDarrick J. Wong if (iop) { 452afc51aaaSDarrick J. Wong for (i = first; i <= last; i++) 453afc51aaaSDarrick J. Wong if (!test_bit(i, iop->uptodate)) 454afc51aaaSDarrick J. Wong return 0; 455afc51aaaSDarrick J. Wong return 1; 456afc51aaaSDarrick J. Wong } 457afc51aaaSDarrick J. Wong 458afc51aaaSDarrick J. Wong return 0; 459afc51aaaSDarrick J. Wong } 460afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); 461afc51aaaSDarrick J. Wong 462afc51aaaSDarrick J. Wong int 463afc51aaaSDarrick J. Wong iomap_releasepage(struct page *page, gfp_t gfp_mask) 464afc51aaaSDarrick J. Wong { 4651ac99452SMatthew Wilcox (Oracle) trace_iomap_releasepage(page->mapping->host, page_offset(page), 4661ac99452SMatthew Wilcox (Oracle) PAGE_SIZE); 4679e91c572SChristoph Hellwig 468afc51aaaSDarrick J. Wong /* 469afc51aaaSDarrick J. Wong * mm accommodates an old ext3 case where clean pages might not have had 470afc51aaaSDarrick J. Wong * the dirty bit cleared. Thus, it can send actual dirty pages to 471afc51aaaSDarrick J. Wong * ->releasepage() via shrink_active_list(), skip those here. 472afc51aaaSDarrick J. Wong */ 473afc51aaaSDarrick J. Wong if (PageDirty(page) || PageWriteback(page)) 474afc51aaaSDarrick J. Wong return 0; 475afc51aaaSDarrick J. Wong iomap_page_release(page); 476afc51aaaSDarrick J. Wong return 1; 477afc51aaaSDarrick J. Wong } 478afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_releasepage); 479afc51aaaSDarrick J. Wong 480afc51aaaSDarrick J. Wong void 481afc51aaaSDarrick J. Wong iomap_invalidatepage(struct page *page, unsigned int offset, unsigned int len) 482afc51aaaSDarrick J. Wong { 4831ac99452SMatthew Wilcox (Oracle) trace_iomap_invalidatepage(page->mapping->host, offset, len); 4849e91c572SChristoph Hellwig 485afc51aaaSDarrick J. Wong /* 486afc51aaaSDarrick J. Wong * If we are invalidating the entire page, clear the dirty state from it 487afc51aaaSDarrick J. Wong * and release it to avoid unnecessary buildup of the LRU. 488afc51aaaSDarrick J. Wong */ 489afc51aaaSDarrick J. Wong if (offset == 0 && len == PAGE_SIZE) { 490afc51aaaSDarrick J. Wong WARN_ON_ONCE(PageWriteback(page)); 491afc51aaaSDarrick J. Wong cancel_dirty_page(page); 492afc51aaaSDarrick J. Wong iomap_page_release(page); 493afc51aaaSDarrick J. Wong } 494afc51aaaSDarrick J. Wong } 495afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_invalidatepage); 496afc51aaaSDarrick J. Wong 497afc51aaaSDarrick J. Wong #ifdef CONFIG_MIGRATION 498afc51aaaSDarrick J. Wong int 499afc51aaaSDarrick J. Wong iomap_migrate_page(struct address_space *mapping, struct page *newpage, 500afc51aaaSDarrick J. Wong struct page *page, enum migrate_mode mode) 501afc51aaaSDarrick J. Wong { 502afc51aaaSDarrick J. Wong int ret; 503afc51aaaSDarrick J. Wong 50426473f83SLinus Torvalds ret = migrate_page_move_mapping(mapping, newpage, page, 0); 505afc51aaaSDarrick J. Wong if (ret != MIGRATEPAGE_SUCCESS) 506afc51aaaSDarrick J. Wong return ret; 507afc51aaaSDarrick J. Wong 50858aeb731SGuoqing Jiang if (page_has_private(page)) 50958aeb731SGuoqing Jiang attach_page_private(newpage, detach_page_private(page)); 510afc51aaaSDarrick J. Wong 511afc51aaaSDarrick J. Wong if (mode != MIGRATE_SYNC_NO_COPY) 512afc51aaaSDarrick J. Wong migrate_page_copy(newpage, page); 513afc51aaaSDarrick J. Wong else 514afc51aaaSDarrick J. Wong migrate_page_states(newpage, page); 515afc51aaaSDarrick J. Wong return MIGRATEPAGE_SUCCESS; 516afc51aaaSDarrick J. Wong } 517afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_migrate_page); 518afc51aaaSDarrick J. Wong #endif /* CONFIG_MIGRATION */ 519afc51aaaSDarrick J. Wong 52032a38a49SChristoph Hellwig enum { 52132a38a49SChristoph Hellwig IOMAP_WRITE_F_UNSHARE = (1 << 0), 52232a38a49SChristoph Hellwig }; 52332a38a49SChristoph Hellwig 524afc51aaaSDarrick J. Wong static void 525afc51aaaSDarrick J. Wong iomap_write_failed(struct inode *inode, loff_t pos, unsigned len) 526afc51aaaSDarrick J. Wong { 527afc51aaaSDarrick J. Wong loff_t i_size = i_size_read(inode); 528afc51aaaSDarrick J. Wong 529afc51aaaSDarrick J. Wong /* 530afc51aaaSDarrick J. Wong * Only truncate newly allocated pages beyoned EOF, even if the 531afc51aaaSDarrick J. Wong * write started inside the existing inode size. 532afc51aaaSDarrick J. Wong */ 533afc51aaaSDarrick J. Wong if (pos + len > i_size) 534afc51aaaSDarrick J. Wong truncate_pagecache_range(inode, max(pos, i_size), pos + len); 535afc51aaaSDarrick J. Wong } 536afc51aaaSDarrick J. Wong 537afc51aaaSDarrick J. Wong static int 538d3b40439SChristoph Hellwig iomap_read_page_sync(loff_t block_start, struct page *page, unsigned poff, 539d3b40439SChristoph Hellwig unsigned plen, struct iomap *iomap) 540afc51aaaSDarrick J. Wong { 541afc51aaaSDarrick J. Wong struct bio_vec bvec; 542afc51aaaSDarrick J. Wong struct bio bio; 543afc51aaaSDarrick J. Wong 544afc51aaaSDarrick J. Wong bio_init(&bio, &bvec, 1); 545afc51aaaSDarrick J. Wong bio.bi_opf = REQ_OP_READ; 546afc51aaaSDarrick J. Wong bio.bi_iter.bi_sector = iomap_sector(iomap, block_start); 547afc51aaaSDarrick J. Wong bio_set_dev(&bio, iomap->bdev); 548afc51aaaSDarrick J. Wong __bio_add_page(&bio, page, plen, poff); 549afc51aaaSDarrick J. Wong return submit_bio_wait(&bio); 550afc51aaaSDarrick J. Wong } 551afc51aaaSDarrick J. Wong 552afc51aaaSDarrick J. Wong static int 55332a38a49SChristoph Hellwig __iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, int flags, 554c039b997SGoldwyn Rodrigues struct page *page, struct iomap *srcmap) 555afc51aaaSDarrick J. Wong { 556afc51aaaSDarrick J. Wong struct iomap_page *iop = iomap_page_create(inode, page); 557afc51aaaSDarrick J. Wong loff_t block_size = i_blocksize(inode); 5586cc19c5fSNikolay Borisov loff_t block_start = round_down(pos, block_size); 5596cc19c5fSNikolay Borisov loff_t block_end = round_up(pos + len, block_size); 560afc51aaaSDarrick J. Wong unsigned from = offset_in_page(pos), to = from + len, poff, plen; 561afc51aaaSDarrick J. Wong 562afc51aaaSDarrick J. Wong if (PageUptodate(page)) 563afc51aaaSDarrick J. Wong return 0; 564e6e7ca92SMatthew Wilcox (Oracle) ClearPageError(page); 565afc51aaaSDarrick J. Wong 566afc51aaaSDarrick J. Wong do { 567afc51aaaSDarrick J. Wong iomap_adjust_read_range(inode, iop, &block_start, 568afc51aaaSDarrick J. Wong block_end - block_start, &poff, &plen); 569afc51aaaSDarrick J. Wong if (plen == 0) 570afc51aaaSDarrick J. Wong break; 571afc51aaaSDarrick J. Wong 57232a38a49SChristoph Hellwig if (!(flags & IOMAP_WRITE_F_UNSHARE) && 57332a38a49SChristoph Hellwig (from <= poff || from >= poff + plen) && 574d3b40439SChristoph Hellwig (to <= poff || to >= poff + plen)) 575d3b40439SChristoph Hellwig continue; 576d3b40439SChristoph Hellwig 577c039b997SGoldwyn Rodrigues if (iomap_block_needs_zeroing(inode, srcmap, block_start)) { 57832a38a49SChristoph Hellwig if (WARN_ON_ONCE(flags & IOMAP_WRITE_F_UNSHARE)) 57932a38a49SChristoph Hellwig return -EIO; 580d3b40439SChristoph Hellwig zero_user_segments(page, poff, from, to, poff + plen); 58114284fedSMatthew Wilcox (Oracle) } else { 58214284fedSMatthew Wilcox (Oracle) int status = iomap_read_page_sync(block_start, page, 58314284fedSMatthew Wilcox (Oracle) poff, plen, srcmap); 584d3b40439SChristoph Hellwig if (status) 585d3b40439SChristoph Hellwig return status; 58614284fedSMatthew Wilcox (Oracle) } 58714284fedSMatthew Wilcox (Oracle) iomap_set_range_uptodate(page, poff, plen); 588afc51aaaSDarrick J. Wong } while ((block_start += plen) < block_end); 589afc51aaaSDarrick J. Wong 590d3b40439SChristoph Hellwig return 0; 591afc51aaaSDarrick J. Wong } 592afc51aaaSDarrick J. Wong 593*69f4a26cSGao Xiang static int iomap_write_begin_inline(struct inode *inode, 594*69f4a26cSGao Xiang struct page *page, struct iomap *srcmap) 595*69f4a26cSGao Xiang { 596*69f4a26cSGao Xiang /* needs more work for the tailpacking case; disable for now */ 597*69f4a26cSGao Xiang if (WARN_ON_ONCE(srcmap->offset != 0)) 598*69f4a26cSGao Xiang return -EIO; 599*69f4a26cSGao Xiang return iomap_read_inline_data(inode, page, srcmap); 600*69f4a26cSGao Xiang } 601*69f4a26cSGao Xiang 602afc51aaaSDarrick J. Wong static int 603afc51aaaSDarrick J. Wong iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags, 604c039b997SGoldwyn Rodrigues struct page **pagep, struct iomap *iomap, struct iomap *srcmap) 605afc51aaaSDarrick J. Wong { 606afc51aaaSDarrick J. Wong const struct iomap_page_ops *page_ops = iomap->page_ops; 607afc51aaaSDarrick J. Wong struct page *page; 608afc51aaaSDarrick J. Wong int status = 0; 609afc51aaaSDarrick J. Wong 610afc51aaaSDarrick J. Wong BUG_ON(pos + len > iomap->offset + iomap->length); 611c039b997SGoldwyn Rodrigues if (srcmap != iomap) 612c039b997SGoldwyn Rodrigues BUG_ON(pos + len > srcmap->offset + srcmap->length); 613afc51aaaSDarrick J. Wong 614afc51aaaSDarrick J. Wong if (fatal_signal_pending(current)) 615afc51aaaSDarrick J. Wong return -EINTR; 616afc51aaaSDarrick J. Wong 617afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_prepare) { 618afc51aaaSDarrick J. Wong status = page_ops->page_prepare(inode, pos, len, iomap); 619afc51aaaSDarrick J. Wong if (status) 620afc51aaaSDarrick J. Wong return status; 621afc51aaaSDarrick J. Wong } 622afc51aaaSDarrick J. Wong 623dcd6158dSChristoph Hellwig page = grab_cache_page_write_begin(inode->i_mapping, pos >> PAGE_SHIFT, 624dcd6158dSChristoph Hellwig AOP_FLAG_NOFS); 625afc51aaaSDarrick J. Wong if (!page) { 626afc51aaaSDarrick J. Wong status = -ENOMEM; 627afc51aaaSDarrick J. Wong goto out_no_page; 628afc51aaaSDarrick J. Wong } 629afc51aaaSDarrick J. Wong 630c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_INLINE) 631*69f4a26cSGao Xiang status = iomap_write_begin_inline(inode, page, srcmap); 632afc51aaaSDarrick J. Wong else if (iomap->flags & IOMAP_F_BUFFER_HEAD) 633c039b997SGoldwyn Rodrigues status = __block_write_begin_int(page, pos, len, NULL, srcmap); 634afc51aaaSDarrick J. Wong else 63532a38a49SChristoph Hellwig status = __iomap_write_begin(inode, pos, len, flags, page, 636c039b997SGoldwyn Rodrigues srcmap); 637afc51aaaSDarrick J. Wong 638afc51aaaSDarrick J. Wong if (unlikely(status)) 639afc51aaaSDarrick J. Wong goto out_unlock; 640afc51aaaSDarrick J. Wong 641afc51aaaSDarrick J. Wong *pagep = page; 642afc51aaaSDarrick J. Wong return 0; 643afc51aaaSDarrick J. Wong 644afc51aaaSDarrick J. Wong out_unlock: 645afc51aaaSDarrick J. Wong unlock_page(page); 646afc51aaaSDarrick J. Wong put_page(page); 647afc51aaaSDarrick J. Wong iomap_write_failed(inode, pos, len); 648afc51aaaSDarrick J. Wong 649afc51aaaSDarrick J. Wong out_no_page: 650afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_done) 651afc51aaaSDarrick J. Wong page_ops->page_done(inode, pos, 0, NULL, iomap); 652afc51aaaSDarrick J. Wong return status; 653afc51aaaSDarrick J. Wong } 654afc51aaaSDarrick J. Wong 655e25ba8cbSMatthew Wilcox (Oracle) static size_t __iomap_write_end(struct inode *inode, loff_t pos, size_t len, 656e25ba8cbSMatthew Wilcox (Oracle) size_t copied, struct page *page) 657afc51aaaSDarrick J. Wong { 658afc51aaaSDarrick J. Wong flush_dcache_page(page); 659afc51aaaSDarrick J. Wong 660afc51aaaSDarrick J. Wong /* 661afc51aaaSDarrick J. Wong * The blocks that were entirely written will now be uptodate, so we 662afc51aaaSDarrick J. Wong * don't have to worry about a readpage reading them and overwriting a 663afc51aaaSDarrick J. Wong * partial write. However if we have encountered a short write and only 664afc51aaaSDarrick J. Wong * partially written into a block, it will not be marked uptodate, so a 665afc51aaaSDarrick J. Wong * readpage might come in and destroy our partial write. 666afc51aaaSDarrick J. Wong * 667afc51aaaSDarrick J. Wong * Do the simplest thing, and just treat any short write to a non 668afc51aaaSDarrick J. Wong * uptodate page as a zero-length write, and force the caller to redo 669afc51aaaSDarrick J. Wong * the whole thing. 670afc51aaaSDarrick J. Wong */ 671afc51aaaSDarrick J. Wong if (unlikely(copied < len && !PageUptodate(page))) 672afc51aaaSDarrick J. Wong return 0; 673afc51aaaSDarrick J. Wong iomap_set_range_uptodate(page, offset_in_page(pos), len); 674fd7353f8SMatthew Wilcox (Oracle) __set_page_dirty_nobuffers(page); 675afc51aaaSDarrick J. Wong return copied; 676afc51aaaSDarrick J. Wong } 677afc51aaaSDarrick J. Wong 678e25ba8cbSMatthew Wilcox (Oracle) static size_t iomap_write_end_inline(struct inode *inode, struct page *page, 679e25ba8cbSMatthew Wilcox (Oracle) struct iomap *iomap, loff_t pos, size_t copied) 680afc51aaaSDarrick J. Wong { 681afc51aaaSDarrick J. Wong void *addr; 682afc51aaaSDarrick J. Wong 683afc51aaaSDarrick J. Wong WARN_ON_ONCE(!PageUptodate(page)); 684*69f4a26cSGao Xiang BUG_ON(!iomap_inline_data_valid(iomap)); 685afc51aaaSDarrick J. Wong 6867ed3cd1aSMatthew Wilcox (Oracle) flush_dcache_page(page); 687afc51aaaSDarrick J. Wong addr = kmap_atomic(page); 688*69f4a26cSGao Xiang memcpy(iomap_inline_data(iomap, pos), addr + pos, copied); 689afc51aaaSDarrick J. Wong kunmap_atomic(addr); 690afc51aaaSDarrick J. Wong 691afc51aaaSDarrick J. Wong mark_inode_dirty(inode); 692afc51aaaSDarrick J. Wong return copied; 693afc51aaaSDarrick J. Wong } 694afc51aaaSDarrick J. Wong 695e25ba8cbSMatthew Wilcox (Oracle) /* Returns the number of bytes copied. May be 0. Cannot be an errno. */ 696e25ba8cbSMatthew Wilcox (Oracle) static size_t iomap_write_end(struct inode *inode, loff_t pos, size_t len, 697e25ba8cbSMatthew Wilcox (Oracle) size_t copied, struct page *page, struct iomap *iomap, 698e25ba8cbSMatthew Wilcox (Oracle) struct iomap *srcmap) 699afc51aaaSDarrick J. Wong { 700afc51aaaSDarrick J. Wong const struct iomap_page_ops *page_ops = iomap->page_ops; 701afc51aaaSDarrick J. Wong loff_t old_size = inode->i_size; 702e25ba8cbSMatthew Wilcox (Oracle) size_t ret; 703afc51aaaSDarrick J. Wong 704c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_INLINE) { 705afc51aaaSDarrick J. Wong ret = iomap_write_end_inline(inode, page, iomap, pos, copied); 706c039b997SGoldwyn Rodrigues } else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) { 707afc51aaaSDarrick J. Wong ret = block_write_end(NULL, inode->i_mapping, pos, len, copied, 708afc51aaaSDarrick J. Wong page, NULL); 709afc51aaaSDarrick J. Wong } else { 710c12d6fa8SChristoph Hellwig ret = __iomap_write_end(inode, pos, len, copied, page); 711afc51aaaSDarrick J. Wong } 712afc51aaaSDarrick J. Wong 713afc51aaaSDarrick J. Wong /* 714afc51aaaSDarrick J. Wong * Update the in-memory inode size after copying the data into the page 715afc51aaaSDarrick J. Wong * cache. It's up to the file system to write the updated size to disk, 716afc51aaaSDarrick J. Wong * preferably after I/O completion so that no stale data is exposed. 717afc51aaaSDarrick J. Wong */ 718afc51aaaSDarrick J. Wong if (pos + ret > old_size) { 719afc51aaaSDarrick J. Wong i_size_write(inode, pos + ret); 720afc51aaaSDarrick J. Wong iomap->flags |= IOMAP_F_SIZE_CHANGED; 721afc51aaaSDarrick J. Wong } 722afc51aaaSDarrick J. Wong unlock_page(page); 723afc51aaaSDarrick J. Wong 724afc51aaaSDarrick J. Wong if (old_size < pos) 725afc51aaaSDarrick J. Wong pagecache_isize_extended(inode, old_size, pos); 726afc51aaaSDarrick J. Wong if (page_ops && page_ops->page_done) 727afc51aaaSDarrick J. Wong page_ops->page_done(inode, pos, ret, page, iomap); 728afc51aaaSDarrick J. Wong put_page(page); 729afc51aaaSDarrick J. Wong 730afc51aaaSDarrick J. Wong if (ret < len) 731afc51aaaSDarrick J. Wong iomap_write_failed(inode, pos, len); 732afc51aaaSDarrick J. Wong return ret; 733afc51aaaSDarrick J. Wong } 734afc51aaaSDarrick J. Wong 735afc51aaaSDarrick J. Wong static loff_t 736afc51aaaSDarrick J. Wong iomap_write_actor(struct inode *inode, loff_t pos, loff_t length, void *data, 737c039b997SGoldwyn Rodrigues struct iomap *iomap, struct iomap *srcmap) 738afc51aaaSDarrick J. Wong { 739afc51aaaSDarrick J. Wong struct iov_iter *i = data; 740afc51aaaSDarrick J. Wong long status = 0; 741afc51aaaSDarrick J. Wong ssize_t written = 0; 742afc51aaaSDarrick J. Wong 743afc51aaaSDarrick J. Wong do { 744afc51aaaSDarrick J. Wong struct page *page; 745afc51aaaSDarrick J. Wong unsigned long offset; /* Offset into pagecache page */ 746afc51aaaSDarrick J. Wong unsigned long bytes; /* Bytes to write to page */ 747afc51aaaSDarrick J. Wong size_t copied; /* Bytes copied from user */ 748afc51aaaSDarrick J. Wong 749afc51aaaSDarrick J. Wong offset = offset_in_page(pos); 750afc51aaaSDarrick J. Wong bytes = min_t(unsigned long, PAGE_SIZE - offset, 751afc51aaaSDarrick J. Wong iov_iter_count(i)); 752afc51aaaSDarrick J. Wong again: 753afc51aaaSDarrick J. Wong if (bytes > length) 754afc51aaaSDarrick J. Wong bytes = length; 755afc51aaaSDarrick J. Wong 756afc51aaaSDarrick J. Wong /* 757afc51aaaSDarrick J. Wong * Bring in the user page that we will copy from _first_. 758afc51aaaSDarrick J. Wong * Otherwise there's a nasty deadlock on copying from the 759afc51aaaSDarrick J. Wong * same page as we're writing to, without it being marked 760afc51aaaSDarrick J. Wong * up-to-date. 761afc51aaaSDarrick J. Wong */ 762afc51aaaSDarrick J. Wong if (unlikely(iov_iter_fault_in_readable(i, bytes))) { 763afc51aaaSDarrick J. Wong status = -EFAULT; 764afc51aaaSDarrick J. Wong break; 765afc51aaaSDarrick J. Wong } 766afc51aaaSDarrick J. Wong 767c039b997SGoldwyn Rodrigues status = iomap_write_begin(inode, pos, bytes, 0, &page, iomap, 768c039b997SGoldwyn Rodrigues srcmap); 769afc51aaaSDarrick J. Wong if (unlikely(status)) 770afc51aaaSDarrick J. Wong break; 771afc51aaaSDarrick J. Wong 772afc51aaaSDarrick J. Wong if (mapping_writably_mapped(inode->i_mapping)) 773afc51aaaSDarrick J. Wong flush_dcache_page(page); 774afc51aaaSDarrick J. Wong 775f0b65f39SAl Viro copied = copy_page_from_iter_atomic(page, offset, bytes, i); 776afc51aaaSDarrick J. Wong 777bc1bb416SAl Viro status = iomap_write_end(inode, pos, bytes, copied, page, iomap, 778c039b997SGoldwyn Rodrigues srcmap); 779afc51aaaSDarrick J. Wong 780f0b65f39SAl Viro if (unlikely(copied != status)) 781f0b65f39SAl Viro iov_iter_revert(i, copied - status); 782afc51aaaSDarrick J. Wong 783f0b65f39SAl Viro cond_resched(); 784bc1bb416SAl Viro if (unlikely(status == 0)) { 785afc51aaaSDarrick J. Wong /* 786bc1bb416SAl Viro * A short copy made iomap_write_end() reject the 787bc1bb416SAl Viro * thing entirely. Might be memory poisoning 788bc1bb416SAl Viro * halfway through, might be a race with munmap, 789bc1bb416SAl Viro * might be severe memory pressure. 790afc51aaaSDarrick J. Wong */ 791bc1bb416SAl Viro if (copied) 792bc1bb416SAl Viro bytes = copied; 793afc51aaaSDarrick J. Wong goto again; 794afc51aaaSDarrick J. Wong } 795f0b65f39SAl Viro pos += status; 796f0b65f39SAl Viro written += status; 797f0b65f39SAl Viro length -= status; 798afc51aaaSDarrick J. Wong 799afc51aaaSDarrick J. Wong balance_dirty_pages_ratelimited(inode->i_mapping); 800afc51aaaSDarrick J. Wong } while (iov_iter_count(i) && length); 801afc51aaaSDarrick J. Wong 802afc51aaaSDarrick J. Wong return written ? written : status; 803afc51aaaSDarrick J. Wong } 804afc51aaaSDarrick J. Wong 805afc51aaaSDarrick J. Wong ssize_t 806afc51aaaSDarrick J. Wong iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *iter, 807afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 808afc51aaaSDarrick J. Wong { 809afc51aaaSDarrick J. Wong struct inode *inode = iocb->ki_filp->f_mapping->host; 810afc51aaaSDarrick J. Wong loff_t pos = iocb->ki_pos, ret = 0, written = 0; 811afc51aaaSDarrick J. Wong 812afc51aaaSDarrick J. Wong while (iov_iter_count(iter)) { 813afc51aaaSDarrick J. Wong ret = iomap_apply(inode, pos, iov_iter_count(iter), 814afc51aaaSDarrick J. Wong IOMAP_WRITE, ops, iter, iomap_write_actor); 815afc51aaaSDarrick J. Wong if (ret <= 0) 816afc51aaaSDarrick J. Wong break; 817afc51aaaSDarrick J. Wong pos += ret; 818afc51aaaSDarrick J. Wong written += ret; 819afc51aaaSDarrick J. Wong } 820afc51aaaSDarrick J. Wong 821afc51aaaSDarrick J. Wong return written ? written : ret; 822afc51aaaSDarrick J. Wong } 823afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_file_buffered_write); 824afc51aaaSDarrick J. Wong 825afc51aaaSDarrick J. Wong static loff_t 8263590c4d8SChristoph Hellwig iomap_unshare_actor(struct inode *inode, loff_t pos, loff_t length, void *data, 827c039b997SGoldwyn Rodrigues struct iomap *iomap, struct iomap *srcmap) 828afc51aaaSDarrick J. Wong { 829afc51aaaSDarrick J. Wong long status = 0; 830d4ff3b2eSMatthew Wilcox (Oracle) loff_t written = 0; 831afc51aaaSDarrick J. Wong 8323590c4d8SChristoph Hellwig /* don't bother with blocks that are not shared to start with */ 8333590c4d8SChristoph Hellwig if (!(iomap->flags & IOMAP_F_SHARED)) 8343590c4d8SChristoph Hellwig return length; 8353590c4d8SChristoph Hellwig /* don't bother with holes or unwritten extents */ 836c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) 8373590c4d8SChristoph Hellwig return length; 8383590c4d8SChristoph Hellwig 839afc51aaaSDarrick J. Wong do { 84032a38a49SChristoph Hellwig unsigned long offset = offset_in_page(pos); 84132a38a49SChristoph Hellwig unsigned long bytes = min_t(loff_t, PAGE_SIZE - offset, length); 84232a38a49SChristoph Hellwig struct page *page; 843afc51aaaSDarrick J. Wong 84432a38a49SChristoph Hellwig status = iomap_write_begin(inode, pos, bytes, 845c039b997SGoldwyn Rodrigues IOMAP_WRITE_F_UNSHARE, &page, iomap, srcmap); 846afc51aaaSDarrick J. Wong if (unlikely(status)) 847afc51aaaSDarrick J. Wong return status; 848afc51aaaSDarrick J. Wong 849c039b997SGoldwyn Rodrigues status = iomap_write_end(inode, pos, bytes, bytes, page, iomap, 850c039b997SGoldwyn Rodrigues srcmap); 851afc51aaaSDarrick J. Wong if (WARN_ON_ONCE(status == 0)) 852afc51aaaSDarrick J. Wong return -EIO; 853afc51aaaSDarrick J. Wong 854afc51aaaSDarrick J. Wong cond_resched(); 855afc51aaaSDarrick J. Wong 856afc51aaaSDarrick J. Wong pos += status; 857afc51aaaSDarrick J. Wong written += status; 858afc51aaaSDarrick J. Wong length -= status; 859afc51aaaSDarrick J. Wong 860afc51aaaSDarrick J. Wong balance_dirty_pages_ratelimited(inode->i_mapping); 861afc51aaaSDarrick J. Wong } while (length); 862afc51aaaSDarrick J. Wong 863afc51aaaSDarrick J. Wong return written; 864afc51aaaSDarrick J. Wong } 865afc51aaaSDarrick J. Wong 866afc51aaaSDarrick J. Wong int 8673590c4d8SChristoph Hellwig iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, 868afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 869afc51aaaSDarrick J. Wong { 870afc51aaaSDarrick J. Wong loff_t ret; 871afc51aaaSDarrick J. Wong 872afc51aaaSDarrick J. Wong while (len) { 873afc51aaaSDarrick J. Wong ret = iomap_apply(inode, pos, len, IOMAP_WRITE, ops, NULL, 8743590c4d8SChristoph Hellwig iomap_unshare_actor); 875afc51aaaSDarrick J. Wong if (ret <= 0) 876afc51aaaSDarrick J. Wong return ret; 877afc51aaaSDarrick J. Wong pos += ret; 878afc51aaaSDarrick J. Wong len -= ret; 879afc51aaaSDarrick J. Wong } 880afc51aaaSDarrick J. Wong 881afc51aaaSDarrick J. Wong return 0; 882afc51aaaSDarrick J. Wong } 8833590c4d8SChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_file_unshare); 884afc51aaaSDarrick J. Wong 88581ee8e52SMatthew Wilcox (Oracle) static s64 iomap_zero(struct inode *inode, loff_t pos, u64 length, 88681ee8e52SMatthew Wilcox (Oracle) struct iomap *iomap, struct iomap *srcmap) 887afc51aaaSDarrick J. Wong { 888afc51aaaSDarrick J. Wong struct page *page; 889afc51aaaSDarrick J. Wong int status; 89081ee8e52SMatthew Wilcox (Oracle) unsigned offset = offset_in_page(pos); 89181ee8e52SMatthew Wilcox (Oracle) unsigned bytes = min_t(u64, PAGE_SIZE - offset, length); 892afc51aaaSDarrick J. Wong 893c039b997SGoldwyn Rodrigues status = iomap_write_begin(inode, pos, bytes, 0, &page, iomap, srcmap); 894afc51aaaSDarrick J. Wong if (status) 895afc51aaaSDarrick J. Wong return status; 896afc51aaaSDarrick J. Wong 897afc51aaaSDarrick J. Wong zero_user(page, offset, bytes); 898afc51aaaSDarrick J. Wong mark_page_accessed(page); 899afc51aaaSDarrick J. Wong 900c039b997SGoldwyn Rodrigues return iomap_write_end(inode, pos, bytes, bytes, page, iomap, srcmap); 901afc51aaaSDarrick J. Wong } 902afc51aaaSDarrick J. Wong 90381ee8e52SMatthew Wilcox (Oracle) static loff_t iomap_zero_range_actor(struct inode *inode, loff_t pos, 90481ee8e52SMatthew Wilcox (Oracle) loff_t length, void *data, struct iomap *iomap, 90581ee8e52SMatthew Wilcox (Oracle) struct iomap *srcmap) 906afc51aaaSDarrick J. Wong { 907afc51aaaSDarrick J. Wong bool *did_zero = data; 908afc51aaaSDarrick J. Wong loff_t written = 0; 909afc51aaaSDarrick J. Wong 910afc51aaaSDarrick J. Wong /* already zeroed? we're done. */ 911c039b997SGoldwyn Rodrigues if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) 91281ee8e52SMatthew Wilcox (Oracle) return length; 913afc51aaaSDarrick J. Wong 914afc51aaaSDarrick J. Wong do { 91581ee8e52SMatthew Wilcox (Oracle) s64 bytes; 916afc51aaaSDarrick J. Wong 917afc51aaaSDarrick J. Wong if (IS_DAX(inode)) 91881ee8e52SMatthew Wilcox (Oracle) bytes = dax_iomap_zero(pos, length, iomap); 919afc51aaaSDarrick J. Wong else 92081ee8e52SMatthew Wilcox (Oracle) bytes = iomap_zero(inode, pos, length, iomap, srcmap); 92181ee8e52SMatthew Wilcox (Oracle) if (bytes < 0) 92281ee8e52SMatthew Wilcox (Oracle) return bytes; 923afc51aaaSDarrick J. Wong 924afc51aaaSDarrick J. Wong pos += bytes; 92581ee8e52SMatthew Wilcox (Oracle) length -= bytes; 926afc51aaaSDarrick J. Wong written += bytes; 927afc51aaaSDarrick J. Wong if (did_zero) 928afc51aaaSDarrick J. Wong *did_zero = true; 92981ee8e52SMatthew Wilcox (Oracle) } while (length > 0); 930afc51aaaSDarrick J. Wong 931afc51aaaSDarrick J. Wong return written; 932afc51aaaSDarrick J. Wong } 933afc51aaaSDarrick J. Wong 934afc51aaaSDarrick J. Wong int 935afc51aaaSDarrick J. Wong iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, 936afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 937afc51aaaSDarrick J. Wong { 938afc51aaaSDarrick J. Wong loff_t ret; 939afc51aaaSDarrick J. Wong 940afc51aaaSDarrick J. Wong while (len > 0) { 941afc51aaaSDarrick J. Wong ret = iomap_apply(inode, pos, len, IOMAP_ZERO, 942afc51aaaSDarrick J. Wong ops, did_zero, iomap_zero_range_actor); 943afc51aaaSDarrick J. Wong if (ret <= 0) 944afc51aaaSDarrick J. Wong return ret; 945afc51aaaSDarrick J. Wong 946afc51aaaSDarrick J. Wong pos += ret; 947afc51aaaSDarrick J. Wong len -= ret; 948afc51aaaSDarrick J. Wong } 949afc51aaaSDarrick J. Wong 950afc51aaaSDarrick J. Wong return 0; 951afc51aaaSDarrick J. Wong } 952afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_zero_range); 953afc51aaaSDarrick J. Wong 954afc51aaaSDarrick J. Wong int 955afc51aaaSDarrick J. Wong iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, 956afc51aaaSDarrick J. Wong const struct iomap_ops *ops) 957afc51aaaSDarrick J. Wong { 958afc51aaaSDarrick J. Wong unsigned int blocksize = i_blocksize(inode); 959afc51aaaSDarrick J. Wong unsigned int off = pos & (blocksize - 1); 960afc51aaaSDarrick J. Wong 961afc51aaaSDarrick J. Wong /* Block boundary? Nothing to do */ 962afc51aaaSDarrick J. Wong if (!off) 963afc51aaaSDarrick J. Wong return 0; 964afc51aaaSDarrick J. Wong return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops); 965afc51aaaSDarrick J. Wong } 966afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_truncate_page); 967afc51aaaSDarrick J. Wong 968afc51aaaSDarrick J. Wong static loff_t 969afc51aaaSDarrick J. Wong iomap_page_mkwrite_actor(struct inode *inode, loff_t pos, loff_t length, 970c039b997SGoldwyn Rodrigues void *data, struct iomap *iomap, struct iomap *srcmap) 971afc51aaaSDarrick J. Wong { 972afc51aaaSDarrick J. Wong struct page *page = data; 973afc51aaaSDarrick J. Wong int ret; 974afc51aaaSDarrick J. Wong 975afc51aaaSDarrick J. Wong if (iomap->flags & IOMAP_F_BUFFER_HEAD) { 976afc51aaaSDarrick J. Wong ret = __block_write_begin_int(page, pos, length, NULL, iomap); 977afc51aaaSDarrick J. Wong if (ret) 978afc51aaaSDarrick J. Wong return ret; 979afc51aaaSDarrick J. Wong block_commit_write(page, 0, length); 980afc51aaaSDarrick J. Wong } else { 981afc51aaaSDarrick J. Wong WARN_ON_ONCE(!PageUptodate(page)); 982afc51aaaSDarrick J. Wong set_page_dirty(page); 983afc51aaaSDarrick J. Wong } 984afc51aaaSDarrick J. Wong 985afc51aaaSDarrick J. Wong return length; 986afc51aaaSDarrick J. Wong } 987afc51aaaSDarrick J. Wong 988afc51aaaSDarrick J. Wong vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops) 989afc51aaaSDarrick J. Wong { 990afc51aaaSDarrick J. Wong struct page *page = vmf->page; 991afc51aaaSDarrick J. Wong struct inode *inode = file_inode(vmf->vma->vm_file); 992afc51aaaSDarrick J. Wong unsigned long length; 993243145bcSAndreas Gruenbacher loff_t offset; 994afc51aaaSDarrick J. Wong ssize_t ret; 995afc51aaaSDarrick J. Wong 996afc51aaaSDarrick J. Wong lock_page(page); 997243145bcSAndreas Gruenbacher ret = page_mkwrite_check_truncate(page, inode); 998243145bcSAndreas Gruenbacher if (ret < 0) 999afc51aaaSDarrick J. Wong goto out_unlock; 1000243145bcSAndreas Gruenbacher length = ret; 1001afc51aaaSDarrick J. Wong 1002243145bcSAndreas Gruenbacher offset = page_offset(page); 1003afc51aaaSDarrick J. Wong while (length > 0) { 1004afc51aaaSDarrick J. Wong ret = iomap_apply(inode, offset, length, 1005afc51aaaSDarrick J. Wong IOMAP_WRITE | IOMAP_FAULT, ops, page, 1006afc51aaaSDarrick J. Wong iomap_page_mkwrite_actor); 1007afc51aaaSDarrick J. Wong if (unlikely(ret <= 0)) 1008afc51aaaSDarrick J. Wong goto out_unlock; 1009afc51aaaSDarrick J. Wong offset += ret; 1010afc51aaaSDarrick J. Wong length -= ret; 1011afc51aaaSDarrick J. Wong } 1012afc51aaaSDarrick J. Wong 1013afc51aaaSDarrick J. Wong wait_for_stable_page(page); 1014afc51aaaSDarrick J. Wong return VM_FAULT_LOCKED; 1015afc51aaaSDarrick J. Wong out_unlock: 1016afc51aaaSDarrick J. Wong unlock_page(page); 1017afc51aaaSDarrick J. Wong return block_page_mkwrite_return(ret); 1018afc51aaaSDarrick J. Wong } 1019afc51aaaSDarrick J. Wong EXPORT_SYMBOL_GPL(iomap_page_mkwrite); 1020598ecfbaSChristoph Hellwig 1021598ecfbaSChristoph Hellwig static void 102248d64cd1SChristoph Hellwig iomap_finish_page_writeback(struct inode *inode, struct page *page, 10230fb2d720SMatthew Wilcox (Oracle) int error, unsigned int len) 1024598ecfbaSChristoph Hellwig { 102548d64cd1SChristoph Hellwig struct iomap_page *iop = to_iomap_page(page); 1026598ecfbaSChristoph Hellwig 1027598ecfbaSChristoph Hellwig if (error) { 102848d64cd1SChristoph Hellwig SetPageError(page); 1029598ecfbaSChristoph Hellwig mapping_set_error(inode->i_mapping, -EIO); 1030598ecfbaSChristoph Hellwig } 1031598ecfbaSChristoph Hellwig 103224addd84SMatthew Wilcox (Oracle) WARN_ON_ONCE(i_blocks_per_page(inode, page) > 1 && !iop); 10330fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) <= 0); 1034598ecfbaSChristoph Hellwig 10350fb2d720SMatthew Wilcox (Oracle) if (!iop || atomic_sub_and_test(len, &iop->write_bytes_pending)) 103648d64cd1SChristoph Hellwig end_page_writeback(page); 1037598ecfbaSChristoph Hellwig } 1038598ecfbaSChristoph Hellwig 1039598ecfbaSChristoph Hellwig /* 1040598ecfbaSChristoph Hellwig * We're now finished for good with this ioend structure. Update the page 1041598ecfbaSChristoph Hellwig * state, release holds on bios, and finally free up memory. Do not use the 1042598ecfbaSChristoph Hellwig * ioend after this. 1043598ecfbaSChristoph Hellwig */ 1044598ecfbaSChristoph Hellwig static void 1045598ecfbaSChristoph Hellwig iomap_finish_ioend(struct iomap_ioend *ioend, int error) 1046598ecfbaSChristoph Hellwig { 1047598ecfbaSChristoph Hellwig struct inode *inode = ioend->io_inode; 1048598ecfbaSChristoph Hellwig struct bio *bio = &ioend->io_inline_bio; 1049598ecfbaSChristoph Hellwig struct bio *last = ioend->io_bio, *next; 1050598ecfbaSChristoph Hellwig u64 start = bio->bi_iter.bi_sector; 1051c275779fSZorro Lang loff_t offset = ioend->io_offset; 1052598ecfbaSChristoph Hellwig bool quiet = bio_flagged(bio, BIO_QUIET); 1053598ecfbaSChristoph Hellwig 1054598ecfbaSChristoph Hellwig for (bio = &ioend->io_inline_bio; bio; bio = next) { 1055598ecfbaSChristoph Hellwig struct bio_vec *bv; 1056598ecfbaSChristoph Hellwig struct bvec_iter_all iter_all; 1057598ecfbaSChristoph Hellwig 1058598ecfbaSChristoph Hellwig /* 1059598ecfbaSChristoph Hellwig * For the last bio, bi_private points to the ioend, so we 1060598ecfbaSChristoph Hellwig * need to explicitly end the iteration here. 1061598ecfbaSChristoph Hellwig */ 1062598ecfbaSChristoph Hellwig if (bio == last) 1063598ecfbaSChristoph Hellwig next = NULL; 1064598ecfbaSChristoph Hellwig else 1065598ecfbaSChristoph Hellwig next = bio->bi_private; 1066598ecfbaSChristoph Hellwig 1067598ecfbaSChristoph Hellwig /* walk each page on bio, ending page IO on them */ 1068598ecfbaSChristoph Hellwig bio_for_each_segment_all(bv, bio, iter_all) 10690fb2d720SMatthew Wilcox (Oracle) iomap_finish_page_writeback(inode, bv->bv_page, error, 10700fb2d720SMatthew Wilcox (Oracle) bv->bv_len); 1071598ecfbaSChristoph Hellwig bio_put(bio); 1072598ecfbaSChristoph Hellwig } 1073c275779fSZorro Lang /* The ioend has been freed by bio_put() */ 1074598ecfbaSChristoph Hellwig 1075598ecfbaSChristoph Hellwig if (unlikely(error && !quiet)) { 1076598ecfbaSChristoph Hellwig printk_ratelimited(KERN_ERR 10779cd0ed63SDarrick J. Wong "%s: writeback error on inode %lu, offset %lld, sector %llu", 1078c275779fSZorro Lang inode->i_sb->s_id, inode->i_ino, offset, start); 1079598ecfbaSChristoph Hellwig } 1080598ecfbaSChristoph Hellwig } 1081598ecfbaSChristoph Hellwig 1082598ecfbaSChristoph Hellwig void 1083598ecfbaSChristoph Hellwig iomap_finish_ioends(struct iomap_ioend *ioend, int error) 1084598ecfbaSChristoph Hellwig { 1085598ecfbaSChristoph Hellwig struct list_head tmp; 1086598ecfbaSChristoph Hellwig 1087598ecfbaSChristoph Hellwig list_replace_init(&ioend->io_list, &tmp); 1088598ecfbaSChristoph Hellwig iomap_finish_ioend(ioend, error); 1089598ecfbaSChristoph Hellwig 1090598ecfbaSChristoph Hellwig while (!list_empty(&tmp)) { 1091598ecfbaSChristoph Hellwig ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); 1092598ecfbaSChristoph Hellwig list_del_init(&ioend->io_list); 1093598ecfbaSChristoph Hellwig iomap_finish_ioend(ioend, error); 1094598ecfbaSChristoph Hellwig } 1095598ecfbaSChristoph Hellwig } 1096598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_finish_ioends); 1097598ecfbaSChristoph Hellwig 1098598ecfbaSChristoph Hellwig /* 1099598ecfbaSChristoph Hellwig * We can merge two adjacent ioends if they have the same set of work to do. 1100598ecfbaSChristoph Hellwig */ 1101598ecfbaSChristoph Hellwig static bool 1102598ecfbaSChristoph Hellwig iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next) 1103598ecfbaSChristoph Hellwig { 1104598ecfbaSChristoph Hellwig if (ioend->io_bio->bi_status != next->io_bio->bi_status) 1105598ecfbaSChristoph Hellwig return false; 1106598ecfbaSChristoph Hellwig if ((ioend->io_flags & IOMAP_F_SHARED) ^ 1107598ecfbaSChristoph Hellwig (next->io_flags & IOMAP_F_SHARED)) 1108598ecfbaSChristoph Hellwig return false; 1109598ecfbaSChristoph Hellwig if ((ioend->io_type == IOMAP_UNWRITTEN) ^ 1110598ecfbaSChristoph Hellwig (next->io_type == IOMAP_UNWRITTEN)) 1111598ecfbaSChristoph Hellwig return false; 1112598ecfbaSChristoph Hellwig if (ioend->io_offset + ioend->io_size != next->io_offset) 1113598ecfbaSChristoph Hellwig return false; 1114598ecfbaSChristoph Hellwig return true; 1115598ecfbaSChristoph Hellwig } 1116598ecfbaSChristoph Hellwig 1117598ecfbaSChristoph Hellwig void 11186e552494SBrian Foster iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends) 1119598ecfbaSChristoph Hellwig { 1120598ecfbaSChristoph Hellwig struct iomap_ioend *next; 1121598ecfbaSChristoph Hellwig 1122598ecfbaSChristoph Hellwig INIT_LIST_HEAD(&ioend->io_list); 1123598ecfbaSChristoph Hellwig 1124598ecfbaSChristoph Hellwig while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, 1125598ecfbaSChristoph Hellwig io_list))) { 1126598ecfbaSChristoph Hellwig if (!iomap_ioend_can_merge(ioend, next)) 1127598ecfbaSChristoph Hellwig break; 1128598ecfbaSChristoph Hellwig list_move_tail(&next->io_list, &ioend->io_list); 1129598ecfbaSChristoph Hellwig ioend->io_size += next->io_size; 1130598ecfbaSChristoph Hellwig } 1131598ecfbaSChristoph Hellwig } 1132598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); 1133598ecfbaSChristoph Hellwig 1134598ecfbaSChristoph Hellwig static int 11354f0f586bSSami Tolvanen iomap_ioend_compare(void *priv, const struct list_head *a, 11364f0f586bSSami Tolvanen const struct list_head *b) 1137598ecfbaSChristoph Hellwig { 1138b3d423ecSChristoph Hellwig struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); 1139b3d423ecSChristoph Hellwig struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); 1140598ecfbaSChristoph Hellwig 1141598ecfbaSChristoph Hellwig if (ia->io_offset < ib->io_offset) 1142598ecfbaSChristoph Hellwig return -1; 1143b3d423ecSChristoph Hellwig if (ia->io_offset > ib->io_offset) 1144598ecfbaSChristoph Hellwig return 1; 1145598ecfbaSChristoph Hellwig return 0; 1146598ecfbaSChristoph Hellwig } 1147598ecfbaSChristoph Hellwig 1148598ecfbaSChristoph Hellwig void 1149598ecfbaSChristoph Hellwig iomap_sort_ioends(struct list_head *ioend_list) 1150598ecfbaSChristoph Hellwig { 1151598ecfbaSChristoph Hellwig list_sort(NULL, ioend_list, iomap_ioend_compare); 1152598ecfbaSChristoph Hellwig } 1153598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_sort_ioends); 1154598ecfbaSChristoph Hellwig 1155598ecfbaSChristoph Hellwig static void iomap_writepage_end_bio(struct bio *bio) 1156598ecfbaSChristoph Hellwig { 1157598ecfbaSChristoph Hellwig struct iomap_ioend *ioend = bio->bi_private; 1158598ecfbaSChristoph Hellwig 1159598ecfbaSChristoph Hellwig iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status)); 1160598ecfbaSChristoph Hellwig } 1161598ecfbaSChristoph Hellwig 1162598ecfbaSChristoph Hellwig /* 1163598ecfbaSChristoph Hellwig * Submit the final bio for an ioend. 1164598ecfbaSChristoph Hellwig * 1165598ecfbaSChristoph Hellwig * If @error is non-zero, it means that we have a situation where some part of 1166598ecfbaSChristoph Hellwig * the submission process has failed after we have marked paged for writeback 1167598ecfbaSChristoph Hellwig * and unlocked them. In this situation, we need to fail the bio instead of 1168598ecfbaSChristoph Hellwig * submitting it. This typically only happens on a filesystem shutdown. 1169598ecfbaSChristoph Hellwig */ 1170598ecfbaSChristoph Hellwig static int 1171598ecfbaSChristoph Hellwig iomap_submit_ioend(struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend, 1172598ecfbaSChristoph Hellwig int error) 1173598ecfbaSChristoph Hellwig { 1174598ecfbaSChristoph Hellwig ioend->io_bio->bi_private = ioend; 1175598ecfbaSChristoph Hellwig ioend->io_bio->bi_end_io = iomap_writepage_end_bio; 1176598ecfbaSChristoph Hellwig 1177598ecfbaSChristoph Hellwig if (wpc->ops->prepare_ioend) 1178598ecfbaSChristoph Hellwig error = wpc->ops->prepare_ioend(ioend, error); 1179598ecfbaSChristoph Hellwig if (error) { 1180598ecfbaSChristoph Hellwig /* 1181598ecfbaSChristoph Hellwig * If we are failing the IO now, just mark the ioend with an 1182598ecfbaSChristoph Hellwig * error and finish it. This will run IO completion immediately 1183598ecfbaSChristoph Hellwig * as there is only one reference to the ioend at this point in 1184598ecfbaSChristoph Hellwig * time. 1185598ecfbaSChristoph Hellwig */ 1186598ecfbaSChristoph Hellwig ioend->io_bio->bi_status = errno_to_blk_status(error); 1187598ecfbaSChristoph Hellwig bio_endio(ioend->io_bio); 1188598ecfbaSChristoph Hellwig return error; 1189598ecfbaSChristoph Hellwig } 1190598ecfbaSChristoph Hellwig 1191598ecfbaSChristoph Hellwig submit_bio(ioend->io_bio); 1192598ecfbaSChristoph Hellwig return 0; 1193598ecfbaSChristoph Hellwig } 1194598ecfbaSChristoph Hellwig 1195598ecfbaSChristoph Hellwig static struct iomap_ioend * 1196598ecfbaSChristoph Hellwig iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc, 1197598ecfbaSChristoph Hellwig loff_t offset, sector_t sector, struct writeback_control *wbc) 1198598ecfbaSChristoph Hellwig { 1199598ecfbaSChristoph Hellwig struct iomap_ioend *ioend; 1200598ecfbaSChristoph Hellwig struct bio *bio; 1201598ecfbaSChristoph Hellwig 1202a8affc03SChristoph Hellwig bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_VECS, &iomap_ioend_bioset); 1203598ecfbaSChristoph Hellwig bio_set_dev(bio, wpc->iomap.bdev); 1204598ecfbaSChristoph Hellwig bio->bi_iter.bi_sector = sector; 1205598ecfbaSChristoph Hellwig bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc); 1206598ecfbaSChristoph Hellwig bio->bi_write_hint = inode->i_write_hint; 1207598ecfbaSChristoph Hellwig wbc_init_bio(wbc, bio); 1208598ecfbaSChristoph Hellwig 1209598ecfbaSChristoph Hellwig ioend = container_of(bio, struct iomap_ioend, io_inline_bio); 1210598ecfbaSChristoph Hellwig INIT_LIST_HEAD(&ioend->io_list); 1211598ecfbaSChristoph Hellwig ioend->io_type = wpc->iomap.type; 1212598ecfbaSChristoph Hellwig ioend->io_flags = wpc->iomap.flags; 1213598ecfbaSChristoph Hellwig ioend->io_inode = inode; 1214598ecfbaSChristoph Hellwig ioend->io_size = 0; 1215598ecfbaSChristoph Hellwig ioend->io_offset = offset; 1216598ecfbaSChristoph Hellwig ioend->io_bio = bio; 1217598ecfbaSChristoph Hellwig return ioend; 1218598ecfbaSChristoph Hellwig } 1219598ecfbaSChristoph Hellwig 1220598ecfbaSChristoph Hellwig /* 1221598ecfbaSChristoph Hellwig * Allocate a new bio, and chain the old bio to the new one. 1222598ecfbaSChristoph Hellwig * 1223598ecfbaSChristoph Hellwig * Note that we have to do perform the chaining in this unintuitive order 1224598ecfbaSChristoph Hellwig * so that the bi_private linkage is set up in the right direction for the 1225598ecfbaSChristoph Hellwig * traversal in iomap_finish_ioend(). 1226598ecfbaSChristoph Hellwig */ 1227598ecfbaSChristoph Hellwig static struct bio * 1228598ecfbaSChristoph Hellwig iomap_chain_bio(struct bio *prev) 1229598ecfbaSChristoph Hellwig { 1230598ecfbaSChristoph Hellwig struct bio *new; 1231598ecfbaSChristoph Hellwig 1232a8affc03SChristoph Hellwig new = bio_alloc(GFP_NOFS, BIO_MAX_VECS); 1233598ecfbaSChristoph Hellwig bio_copy_dev(new, prev);/* also copies over blkcg information */ 1234598ecfbaSChristoph Hellwig new->bi_iter.bi_sector = bio_end_sector(prev); 1235598ecfbaSChristoph Hellwig new->bi_opf = prev->bi_opf; 1236598ecfbaSChristoph Hellwig new->bi_write_hint = prev->bi_write_hint; 1237598ecfbaSChristoph Hellwig 1238598ecfbaSChristoph Hellwig bio_chain(prev, new); 1239598ecfbaSChristoph Hellwig bio_get(prev); /* for iomap_finish_ioend */ 1240598ecfbaSChristoph Hellwig submit_bio(prev); 1241598ecfbaSChristoph Hellwig return new; 1242598ecfbaSChristoph Hellwig } 1243598ecfbaSChristoph Hellwig 1244598ecfbaSChristoph Hellwig static bool 1245598ecfbaSChristoph Hellwig iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset, 1246598ecfbaSChristoph Hellwig sector_t sector) 1247598ecfbaSChristoph Hellwig { 1248598ecfbaSChristoph Hellwig if ((wpc->iomap.flags & IOMAP_F_SHARED) != 1249598ecfbaSChristoph Hellwig (wpc->ioend->io_flags & IOMAP_F_SHARED)) 1250598ecfbaSChristoph Hellwig return false; 1251598ecfbaSChristoph Hellwig if (wpc->iomap.type != wpc->ioend->io_type) 1252598ecfbaSChristoph Hellwig return false; 1253598ecfbaSChristoph Hellwig if (offset != wpc->ioend->io_offset + wpc->ioend->io_size) 1254598ecfbaSChristoph Hellwig return false; 1255598ecfbaSChristoph Hellwig if (sector != bio_end_sector(wpc->ioend->io_bio)) 1256598ecfbaSChristoph Hellwig return false; 1257598ecfbaSChristoph Hellwig return true; 1258598ecfbaSChristoph Hellwig } 1259598ecfbaSChristoph Hellwig 1260598ecfbaSChristoph Hellwig /* 1261598ecfbaSChristoph Hellwig * Test to see if we have an existing ioend structure that we could append to 1262598ecfbaSChristoph Hellwig * first, otherwise finish off the current ioend and start another. 1263598ecfbaSChristoph Hellwig */ 1264598ecfbaSChristoph Hellwig static void 1265598ecfbaSChristoph Hellwig iomap_add_to_ioend(struct inode *inode, loff_t offset, struct page *page, 1266598ecfbaSChristoph Hellwig struct iomap_page *iop, struct iomap_writepage_ctx *wpc, 1267598ecfbaSChristoph Hellwig struct writeback_control *wbc, struct list_head *iolist) 1268598ecfbaSChristoph Hellwig { 1269598ecfbaSChristoph Hellwig sector_t sector = iomap_sector(&wpc->iomap, offset); 1270598ecfbaSChristoph Hellwig unsigned len = i_blocksize(inode); 1271598ecfbaSChristoph Hellwig unsigned poff = offset & (PAGE_SIZE - 1); 1272598ecfbaSChristoph Hellwig 1273598ecfbaSChristoph Hellwig if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, offset, sector)) { 1274598ecfbaSChristoph Hellwig if (wpc->ioend) 1275598ecfbaSChristoph Hellwig list_add(&wpc->ioend->io_list, iolist); 1276598ecfbaSChristoph Hellwig wpc->ioend = iomap_alloc_ioend(inode, wpc, offset, sector, wbc); 1277598ecfbaSChristoph Hellwig } 1278598ecfbaSChristoph Hellwig 1279c1b79f11SChristoph Hellwig if (bio_add_page(wpc->ioend->io_bio, page, len, poff) != len) { 1280c1b79f11SChristoph Hellwig wpc->ioend->io_bio = iomap_chain_bio(wpc->ioend->io_bio); 1281c1b79f11SChristoph Hellwig __bio_add_page(wpc->ioend->io_bio, page, len, poff); 1282c1b79f11SChristoph Hellwig } 1283c1b79f11SChristoph Hellwig 12840fb2d720SMatthew Wilcox (Oracle) if (iop) 12850fb2d720SMatthew Wilcox (Oracle) atomic_add(len, &iop->write_bytes_pending); 1286598ecfbaSChristoph Hellwig wpc->ioend->io_size += len; 1287598ecfbaSChristoph Hellwig wbc_account_cgroup_owner(wbc, page, len); 1288598ecfbaSChristoph Hellwig } 1289598ecfbaSChristoph Hellwig 1290598ecfbaSChristoph Hellwig /* 1291598ecfbaSChristoph Hellwig * We implement an immediate ioend submission policy here to avoid needing to 1292598ecfbaSChristoph Hellwig * chain multiple ioends and hence nest mempool allocations which can violate 1293598ecfbaSChristoph Hellwig * forward progress guarantees we need to provide. The current ioend we are 1294598ecfbaSChristoph Hellwig * adding blocks to is cached on the writepage context, and if the new block 1295598ecfbaSChristoph Hellwig * does not append to the cached ioend it will create a new ioend and cache that 1296598ecfbaSChristoph Hellwig * instead. 1297598ecfbaSChristoph Hellwig * 1298598ecfbaSChristoph Hellwig * If a new ioend is created and cached, the old ioend is returned and queued 1299598ecfbaSChristoph Hellwig * locally for submission once the entire page is processed or an error has been 1300598ecfbaSChristoph Hellwig * detected. While ioends are submitted immediately after they are completed, 1301598ecfbaSChristoph Hellwig * batching optimisations are provided by higher level block plugging. 1302598ecfbaSChristoph Hellwig * 1303598ecfbaSChristoph Hellwig * At the end of a writeback pass, there will be a cached ioend remaining on the 1304598ecfbaSChristoph Hellwig * writepage context that the caller will need to submit. 1305598ecfbaSChristoph Hellwig */ 1306598ecfbaSChristoph Hellwig static int 1307598ecfbaSChristoph Hellwig iomap_writepage_map(struct iomap_writepage_ctx *wpc, 1308598ecfbaSChristoph Hellwig struct writeback_control *wbc, struct inode *inode, 1309598ecfbaSChristoph Hellwig struct page *page, u64 end_offset) 1310598ecfbaSChristoph Hellwig { 13118e1bcef8SAndreas Gruenbacher struct iomap_page *iop = iomap_page_create(inode, page); 1312598ecfbaSChristoph Hellwig struct iomap_ioend *ioend, *next; 1313598ecfbaSChristoph Hellwig unsigned len = i_blocksize(inode); 1314598ecfbaSChristoph Hellwig u64 file_offset; /* file offset of page */ 1315598ecfbaSChristoph Hellwig int error = 0, count = 0, i; 1316598ecfbaSChristoph Hellwig LIST_HEAD(submit_list); 1317598ecfbaSChristoph Hellwig 13180fb2d720SMatthew Wilcox (Oracle) WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) != 0); 1319598ecfbaSChristoph Hellwig 1320598ecfbaSChristoph Hellwig /* 1321598ecfbaSChristoph Hellwig * Walk through the page to find areas to write back. If we run off the 1322598ecfbaSChristoph Hellwig * end of the current map or find the current map invalid, grab a new 1323598ecfbaSChristoph Hellwig * one. 1324598ecfbaSChristoph Hellwig */ 1325598ecfbaSChristoph Hellwig for (i = 0, file_offset = page_offset(page); 1326598ecfbaSChristoph Hellwig i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset; 1327598ecfbaSChristoph Hellwig i++, file_offset += len) { 1328598ecfbaSChristoph Hellwig if (iop && !test_bit(i, iop->uptodate)) 1329598ecfbaSChristoph Hellwig continue; 1330598ecfbaSChristoph Hellwig 1331598ecfbaSChristoph Hellwig error = wpc->ops->map_blocks(wpc, inode, file_offset); 1332598ecfbaSChristoph Hellwig if (error) 1333598ecfbaSChristoph Hellwig break; 13343e19e6f3SChristoph Hellwig if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE)) 13353e19e6f3SChristoph Hellwig continue; 1336598ecfbaSChristoph Hellwig if (wpc->iomap.type == IOMAP_HOLE) 1337598ecfbaSChristoph Hellwig continue; 1338598ecfbaSChristoph Hellwig iomap_add_to_ioend(inode, file_offset, page, iop, wpc, wbc, 1339598ecfbaSChristoph Hellwig &submit_list); 1340598ecfbaSChristoph Hellwig count++; 1341598ecfbaSChristoph Hellwig } 1342598ecfbaSChristoph Hellwig 1343598ecfbaSChristoph Hellwig WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list)); 1344598ecfbaSChristoph Hellwig WARN_ON_ONCE(!PageLocked(page)); 1345598ecfbaSChristoph Hellwig WARN_ON_ONCE(PageWriteback(page)); 134650e7d6c7SBrian Foster WARN_ON_ONCE(PageDirty(page)); 1347598ecfbaSChristoph Hellwig 1348598ecfbaSChristoph Hellwig /* 1349598ecfbaSChristoph Hellwig * We cannot cancel the ioend directly here on error. We may have 1350598ecfbaSChristoph Hellwig * already set other pages under writeback and hence we have to run I/O 1351598ecfbaSChristoph Hellwig * completion to mark the error state of the pages under writeback 1352598ecfbaSChristoph Hellwig * appropriately. 1353598ecfbaSChristoph Hellwig */ 1354598ecfbaSChristoph Hellwig if (unlikely(error)) { 1355598ecfbaSChristoph Hellwig /* 1356763e4cdcSBrian Foster * Let the filesystem know what portion of the current page 1357763e4cdcSBrian Foster * failed to map. If the page wasn't been added to ioend, it 1358763e4cdcSBrian Foster * won't be affected by I/O completion and we must unlock it 1359763e4cdcSBrian Foster * now. 1360598ecfbaSChristoph Hellwig */ 1361598ecfbaSChristoph Hellwig if (wpc->ops->discard_page) 1362763e4cdcSBrian Foster wpc->ops->discard_page(page, file_offset); 1363763e4cdcSBrian Foster if (!count) { 1364598ecfbaSChristoph Hellwig ClearPageUptodate(page); 1365598ecfbaSChristoph Hellwig unlock_page(page); 1366598ecfbaSChristoph Hellwig goto done; 1367598ecfbaSChristoph Hellwig } 1368598ecfbaSChristoph Hellwig } 1369598ecfbaSChristoph Hellwig 137050e7d6c7SBrian Foster set_page_writeback(page); 1371598ecfbaSChristoph Hellwig unlock_page(page); 1372598ecfbaSChristoph Hellwig 1373598ecfbaSChristoph Hellwig /* 1374598ecfbaSChristoph Hellwig * Preserve the original error if there was one, otherwise catch 1375598ecfbaSChristoph Hellwig * submission errors here and propagate into subsequent ioend 1376598ecfbaSChristoph Hellwig * submissions. 1377598ecfbaSChristoph Hellwig */ 1378598ecfbaSChristoph Hellwig list_for_each_entry_safe(ioend, next, &submit_list, io_list) { 1379598ecfbaSChristoph Hellwig int error2; 1380598ecfbaSChristoph Hellwig 1381598ecfbaSChristoph Hellwig list_del_init(&ioend->io_list); 1382598ecfbaSChristoph Hellwig error2 = iomap_submit_ioend(wpc, ioend, error); 1383598ecfbaSChristoph Hellwig if (error2 && !error) 1384598ecfbaSChristoph Hellwig error = error2; 1385598ecfbaSChristoph Hellwig } 1386598ecfbaSChristoph Hellwig 1387598ecfbaSChristoph Hellwig /* 1388598ecfbaSChristoph Hellwig * We can end up here with no error and nothing to write only if we race 1389598ecfbaSChristoph Hellwig * with a partial page truncate on a sub-page block sized filesystem. 1390598ecfbaSChristoph Hellwig */ 1391598ecfbaSChristoph Hellwig if (!count) 1392598ecfbaSChristoph Hellwig end_page_writeback(page); 1393598ecfbaSChristoph Hellwig done: 1394598ecfbaSChristoph Hellwig mapping_set_error(page->mapping, error); 1395598ecfbaSChristoph Hellwig return error; 1396598ecfbaSChristoph Hellwig } 1397598ecfbaSChristoph Hellwig 1398598ecfbaSChristoph Hellwig /* 1399598ecfbaSChristoph Hellwig * Write out a dirty page. 1400598ecfbaSChristoph Hellwig * 1401598ecfbaSChristoph Hellwig * For delalloc space on the page we need to allocate space and flush it. 1402598ecfbaSChristoph Hellwig * For unwritten space on the page we need to start the conversion to 1403598ecfbaSChristoph Hellwig * regular allocated space. 1404598ecfbaSChristoph Hellwig */ 1405598ecfbaSChristoph Hellwig static int 1406598ecfbaSChristoph Hellwig iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data) 1407598ecfbaSChristoph Hellwig { 1408598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc = data; 1409598ecfbaSChristoph Hellwig struct inode *inode = page->mapping->host; 1410598ecfbaSChristoph Hellwig pgoff_t end_index; 1411598ecfbaSChristoph Hellwig u64 end_offset; 1412598ecfbaSChristoph Hellwig loff_t offset; 1413598ecfbaSChristoph Hellwig 14141ac99452SMatthew Wilcox (Oracle) trace_iomap_writepage(inode, page_offset(page), PAGE_SIZE); 1415598ecfbaSChristoph Hellwig 1416598ecfbaSChristoph Hellwig /* 1417598ecfbaSChristoph Hellwig * Refuse to write the page out if we are called from reclaim context. 1418598ecfbaSChristoph Hellwig * 1419598ecfbaSChristoph Hellwig * This avoids stack overflows when called from deeply used stacks in 1420598ecfbaSChristoph Hellwig * random callers for direct reclaim or memcg reclaim. We explicitly 1421598ecfbaSChristoph Hellwig * allow reclaim from kswapd as the stack usage there is relatively low. 1422598ecfbaSChristoph Hellwig * 1423598ecfbaSChristoph Hellwig * This should never happen except in the case of a VM regression so 1424598ecfbaSChristoph Hellwig * warn about it. 1425598ecfbaSChristoph Hellwig */ 1426598ecfbaSChristoph Hellwig if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == 1427598ecfbaSChristoph Hellwig PF_MEMALLOC)) 1428598ecfbaSChristoph Hellwig goto redirty; 1429598ecfbaSChristoph Hellwig 1430598ecfbaSChristoph Hellwig /* 1431598ecfbaSChristoph Hellwig * Is this page beyond the end of the file? 1432598ecfbaSChristoph Hellwig * 1433598ecfbaSChristoph Hellwig * The page index is less than the end_index, adjust the end_offset 1434598ecfbaSChristoph Hellwig * to the highest offset that this page should represent. 1435598ecfbaSChristoph Hellwig * ----------------------------------------------------- 1436598ecfbaSChristoph Hellwig * | file mapping | <EOF> | 1437598ecfbaSChristoph Hellwig * ----------------------------------------------------- 1438598ecfbaSChristoph Hellwig * | Page ... | Page N-2 | Page N-1 | Page N | | 1439598ecfbaSChristoph Hellwig * ^--------------------------------^----------|-------- 1440598ecfbaSChristoph Hellwig * | desired writeback range | see else | 1441598ecfbaSChristoph Hellwig * ---------------------------------^------------------| 1442598ecfbaSChristoph Hellwig */ 1443598ecfbaSChristoph Hellwig offset = i_size_read(inode); 1444598ecfbaSChristoph Hellwig end_index = offset >> PAGE_SHIFT; 1445598ecfbaSChristoph Hellwig if (page->index < end_index) 1446598ecfbaSChristoph Hellwig end_offset = (loff_t)(page->index + 1) << PAGE_SHIFT; 1447598ecfbaSChristoph Hellwig else { 1448598ecfbaSChristoph Hellwig /* 1449598ecfbaSChristoph Hellwig * Check whether the page to write out is beyond or straddles 1450598ecfbaSChristoph Hellwig * i_size or not. 1451598ecfbaSChristoph Hellwig * ------------------------------------------------------- 1452598ecfbaSChristoph Hellwig * | file mapping | <EOF> | 1453598ecfbaSChristoph Hellwig * ------------------------------------------------------- 1454598ecfbaSChristoph Hellwig * | Page ... | Page N-2 | Page N-1 | Page N | Beyond | 1455598ecfbaSChristoph Hellwig * ^--------------------------------^-----------|--------- 1456598ecfbaSChristoph Hellwig * | | Straddles | 1457598ecfbaSChristoph Hellwig * ---------------------------------^-----------|--------| 1458598ecfbaSChristoph Hellwig */ 1459598ecfbaSChristoph Hellwig unsigned offset_into_page = offset & (PAGE_SIZE - 1); 1460598ecfbaSChristoph Hellwig 1461598ecfbaSChristoph Hellwig /* 1462598ecfbaSChristoph Hellwig * Skip the page if it is fully outside i_size, e.g. due to a 1463598ecfbaSChristoph Hellwig * truncate operation that is in progress. We must redirty the 1464598ecfbaSChristoph Hellwig * page so that reclaim stops reclaiming it. Otherwise 1465598ecfbaSChristoph Hellwig * iomap_vm_releasepage() is called on it and gets confused. 1466598ecfbaSChristoph Hellwig * 1467598ecfbaSChristoph Hellwig * Note that the end_index is unsigned long, it would overflow 1468598ecfbaSChristoph Hellwig * if the given offset is greater than 16TB on 32-bit system 1469598ecfbaSChristoph Hellwig * and if we do check the page is fully outside i_size or not 1470598ecfbaSChristoph Hellwig * via "if (page->index >= end_index + 1)" as "end_index + 1" 1471598ecfbaSChristoph Hellwig * will be evaluated to 0. Hence this page will be redirtied 1472598ecfbaSChristoph Hellwig * and be written out repeatedly which would result in an 1473598ecfbaSChristoph Hellwig * infinite loop, the user program that perform this operation 1474598ecfbaSChristoph Hellwig * will hang. Instead, we can verify this situation by checking 1475598ecfbaSChristoph Hellwig * if the page to write is totally beyond the i_size or if it's 1476598ecfbaSChristoph Hellwig * offset is just equal to the EOF. 1477598ecfbaSChristoph Hellwig */ 1478598ecfbaSChristoph Hellwig if (page->index > end_index || 1479598ecfbaSChristoph Hellwig (page->index == end_index && offset_into_page == 0)) 1480598ecfbaSChristoph Hellwig goto redirty; 1481598ecfbaSChristoph Hellwig 1482598ecfbaSChristoph Hellwig /* 1483598ecfbaSChristoph Hellwig * The page straddles i_size. It must be zeroed out on each 1484598ecfbaSChristoph Hellwig * and every writepage invocation because it may be mmapped. 1485598ecfbaSChristoph Hellwig * "A file is mapped in multiples of the page size. For a file 1486598ecfbaSChristoph Hellwig * that is not a multiple of the page size, the remaining 1487598ecfbaSChristoph Hellwig * memory is zeroed when mapped, and writes to that region are 1488598ecfbaSChristoph Hellwig * not written out to the file." 1489598ecfbaSChristoph Hellwig */ 1490598ecfbaSChristoph Hellwig zero_user_segment(page, offset_into_page, PAGE_SIZE); 1491598ecfbaSChristoph Hellwig 1492598ecfbaSChristoph Hellwig /* Adjust the end_offset to the end of file */ 1493598ecfbaSChristoph Hellwig end_offset = offset; 1494598ecfbaSChristoph Hellwig } 1495598ecfbaSChristoph Hellwig 1496598ecfbaSChristoph Hellwig return iomap_writepage_map(wpc, wbc, inode, page, end_offset); 1497598ecfbaSChristoph Hellwig 1498598ecfbaSChristoph Hellwig redirty: 1499598ecfbaSChristoph Hellwig redirty_page_for_writepage(wbc, page); 1500598ecfbaSChristoph Hellwig unlock_page(page); 1501598ecfbaSChristoph Hellwig return 0; 1502598ecfbaSChristoph Hellwig } 1503598ecfbaSChristoph Hellwig 1504598ecfbaSChristoph Hellwig int 1505598ecfbaSChristoph Hellwig iomap_writepage(struct page *page, struct writeback_control *wbc, 1506598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc, 1507598ecfbaSChristoph Hellwig const struct iomap_writeback_ops *ops) 1508598ecfbaSChristoph Hellwig { 1509598ecfbaSChristoph Hellwig int ret; 1510598ecfbaSChristoph Hellwig 1511598ecfbaSChristoph Hellwig wpc->ops = ops; 1512598ecfbaSChristoph Hellwig ret = iomap_do_writepage(page, wbc, wpc); 1513598ecfbaSChristoph Hellwig if (!wpc->ioend) 1514598ecfbaSChristoph Hellwig return ret; 1515598ecfbaSChristoph Hellwig return iomap_submit_ioend(wpc, wpc->ioend, ret); 1516598ecfbaSChristoph Hellwig } 1517598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_writepage); 1518598ecfbaSChristoph Hellwig 1519598ecfbaSChristoph Hellwig int 1520598ecfbaSChristoph Hellwig iomap_writepages(struct address_space *mapping, struct writeback_control *wbc, 1521598ecfbaSChristoph Hellwig struct iomap_writepage_ctx *wpc, 1522598ecfbaSChristoph Hellwig const struct iomap_writeback_ops *ops) 1523598ecfbaSChristoph Hellwig { 1524598ecfbaSChristoph Hellwig int ret; 1525598ecfbaSChristoph Hellwig 1526598ecfbaSChristoph Hellwig wpc->ops = ops; 1527598ecfbaSChristoph Hellwig ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc); 1528598ecfbaSChristoph Hellwig if (!wpc->ioend) 1529598ecfbaSChristoph Hellwig return ret; 1530598ecfbaSChristoph Hellwig return iomap_submit_ioend(wpc, wpc->ioend, ret); 1531598ecfbaSChristoph Hellwig } 1532598ecfbaSChristoph Hellwig EXPORT_SYMBOL_GPL(iomap_writepages); 1533598ecfbaSChristoph Hellwig 1534598ecfbaSChristoph Hellwig static int __init iomap_init(void) 1535598ecfbaSChristoph Hellwig { 1536598ecfbaSChristoph Hellwig return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), 1537598ecfbaSChristoph Hellwig offsetof(struct iomap_ioend, io_inline_bio), 1538598ecfbaSChristoph Hellwig BIOSET_NEED_BVECS); 1539598ecfbaSChristoph Hellwig } 1540598ecfbaSChristoph Hellwig fs_initcall(iomap_init); 1541