147e4937aSGao Xiang // SPDX-License-Identifier: GPL-2.0-only 247e4937aSGao Xiang /* 347e4937aSGao Xiang * Copyright (C) 2017-2018 HUAWEI, Inc. 447e4937aSGao Xiang * http://www.huawei.com/ 547e4937aSGao Xiang * Created by Gao Xiang <gaoxiang25@huawei.com> 647e4937aSGao Xiang */ 747e4937aSGao Xiang #include "internal.h" 847e4937aSGao Xiang #include <linux/prefetch.h> 947e4937aSGao Xiang 1047e4937aSGao Xiang #include <trace/events/erofs.h> 1147e4937aSGao Xiang 1299634bf3SGao Xiang static void erofs_readendio(struct bio *bio) 1347e4937aSGao Xiang { 1447e4937aSGao Xiang struct bio_vec *bvec; 1547e4937aSGao Xiang blk_status_t err = bio->bi_status; 1647e4937aSGao Xiang struct bvec_iter_all iter_all; 1747e4937aSGao Xiang 1847e4937aSGao Xiang bio_for_each_segment_all(bvec, bio, iter_all) { 1947e4937aSGao Xiang struct page *page = bvec->bv_page; 2047e4937aSGao Xiang 2147e4937aSGao Xiang /* page is already locked */ 2247e4937aSGao Xiang DBG_BUGON(PageUptodate(page)); 2347e4937aSGao Xiang 248d8a09b0SGao Xiang if (err) 2547e4937aSGao Xiang SetPageError(page); 2647e4937aSGao Xiang else 2747e4937aSGao Xiang SetPageUptodate(page); 2847e4937aSGao Xiang 2947e4937aSGao Xiang unlock_page(page); 3047e4937aSGao Xiang /* page could be reclaimed now */ 3147e4937aSGao Xiang } 3247e4937aSGao Xiang bio_put(bio); 3347e4937aSGao Xiang } 3447e4937aSGao Xiang 35e655b5b3SGao Xiang struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr) 3647e4937aSGao Xiang { 3755252ab7SGao Xiang struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping; 3855252ab7SGao Xiang struct page *page; 3947e4937aSGao Xiang 4055252ab7SGao Xiang page = read_cache_page_gfp(mapping, blkaddr, 41618f40eaSGao Xiang mapping_gfp_constraint(mapping, ~__GFP_FS)); 4255252ab7SGao Xiang /* should already be PageUptodate */ 4355252ab7SGao Xiang if (!IS_ERR(page)) 4455252ab7SGao Xiang lock_page(page); 4555252ab7SGao Xiang return page; 4647e4937aSGao Xiang } 4747e4937aSGao Xiang 4847e4937aSGao Xiang static int erofs_map_blocks_flatmode(struct inode *inode, 4947e4937aSGao Xiang struct erofs_map_blocks *map, 5047e4937aSGao Xiang int flags) 5147e4937aSGao Xiang { 5247e4937aSGao Xiang int err = 0; 5347e4937aSGao Xiang erofs_blk_t nblocks, lastblk; 5447e4937aSGao Xiang u64 offset = map->m_la; 55a5876e24SGao Xiang struct erofs_inode *vi = EROFS_I(inode); 568a765682SGao Xiang bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE); 5747e4937aSGao Xiang 5847e4937aSGao Xiang trace_erofs_map_blocks_flatmode_enter(inode, map, flags); 5947e4937aSGao Xiang 6047e4937aSGao Xiang nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE); 618a765682SGao Xiang lastblk = nblocks - tailendpacking; 6247e4937aSGao Xiang 638d8a09b0SGao Xiang if (offset >= inode->i_size) { 6447e4937aSGao Xiang /* leave out-of-bound access unmapped */ 6547e4937aSGao Xiang map->m_flags = 0; 6647e4937aSGao Xiang map->m_plen = 0; 6747e4937aSGao Xiang goto out; 6847e4937aSGao Xiang } 6947e4937aSGao Xiang 7047e4937aSGao Xiang /* there is no hole in flatmode */ 7147e4937aSGao Xiang map->m_flags = EROFS_MAP_MAPPED; 7247e4937aSGao Xiang 7347e4937aSGao Xiang if (offset < blknr_to_addr(lastblk)) { 7447e4937aSGao Xiang map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la; 7547e4937aSGao Xiang map->m_plen = blknr_to_addr(lastblk) - offset; 768a765682SGao Xiang } else if (tailendpacking) { 7747e4937aSGao Xiang /* 2 - inode inline B: inode, [xattrs], inline last blk... */ 7847e4937aSGao Xiang struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); 7947e4937aSGao Xiang 8047e4937aSGao Xiang map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize + 8147e4937aSGao Xiang vi->xattr_isize + erofs_blkoff(map->m_la); 8247e4937aSGao Xiang map->m_plen = inode->i_size - offset; 8347e4937aSGao Xiang 8447e4937aSGao Xiang /* inline data should be located in one meta block */ 8547e4937aSGao Xiang if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) { 864f761fa2SGao Xiang erofs_err(inode->i_sb, 874f761fa2SGao Xiang "inline data cross block boundary @ nid %llu", 8847e4937aSGao Xiang vi->nid); 8947e4937aSGao Xiang DBG_BUGON(1); 9047e4937aSGao Xiang err = -EFSCORRUPTED; 9147e4937aSGao Xiang goto err_out; 9247e4937aSGao Xiang } 9347e4937aSGao Xiang 9447e4937aSGao Xiang map->m_flags |= EROFS_MAP_META; 9547e4937aSGao Xiang } else { 964f761fa2SGao Xiang erofs_err(inode->i_sb, 974f761fa2SGao Xiang "internal error @ nid: %llu (size %llu), m_la 0x%llx", 9847e4937aSGao Xiang vi->nid, inode->i_size, map->m_la); 9947e4937aSGao Xiang DBG_BUGON(1); 10047e4937aSGao Xiang err = -EIO; 10147e4937aSGao Xiang goto err_out; 10247e4937aSGao Xiang } 10347e4937aSGao Xiang 10447e4937aSGao Xiang out: 10547e4937aSGao Xiang map->m_llen = map->m_plen; 10647e4937aSGao Xiang 10747e4937aSGao Xiang err_out: 10847e4937aSGao Xiang trace_erofs_map_blocks_flatmode_exit(inode, map, flags, 0); 10947e4937aSGao Xiang return err; 11047e4937aSGao Xiang } 11147e4937aSGao Xiang 11247e4937aSGao Xiang int erofs_map_blocks(struct inode *inode, 11347e4937aSGao Xiang struct erofs_map_blocks *map, int flags) 11447e4937aSGao Xiang { 115a5876e24SGao Xiang if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) { 11647e4937aSGao Xiang int err = z_erofs_map_blocks_iter(inode, map, flags); 11747e4937aSGao Xiang 11847e4937aSGao Xiang if (map->mpage) { 11947e4937aSGao Xiang put_page(map->mpage); 12047e4937aSGao Xiang map->mpage = NULL; 12147e4937aSGao Xiang } 12247e4937aSGao Xiang return err; 12347e4937aSGao Xiang } 12447e4937aSGao Xiang return erofs_map_blocks_flatmode(inode, map, flags); 12547e4937aSGao Xiang } 12647e4937aSGao Xiang 12747e4937aSGao Xiang static inline struct bio *erofs_read_raw_page(struct bio *bio, 12847e4937aSGao Xiang struct address_space *mapping, 12947e4937aSGao Xiang struct page *page, 13047e4937aSGao Xiang erofs_off_t *last_block, 13147e4937aSGao Xiang unsigned int nblocks, 13247e4937aSGao Xiang bool ra) 13347e4937aSGao Xiang { 13447e4937aSGao Xiang struct inode *const inode = mapping->host; 13547e4937aSGao Xiang struct super_block *const sb = inode->i_sb; 13647e4937aSGao Xiang erofs_off_t current_block = (erofs_off_t)page->index; 13747e4937aSGao Xiang int err; 13847e4937aSGao Xiang 13947e4937aSGao Xiang DBG_BUGON(!nblocks); 14047e4937aSGao Xiang 14147e4937aSGao Xiang if (PageUptodate(page)) { 14247e4937aSGao Xiang err = 0; 14347e4937aSGao Xiang goto has_updated; 14447e4937aSGao Xiang } 14547e4937aSGao Xiang 14647e4937aSGao Xiang /* note that for readpage case, bio also equals to NULL */ 14747e4937aSGao Xiang if (bio && 14847e4937aSGao Xiang /* not continuous */ 14947e4937aSGao Xiang *last_block + 1 != current_block) { 15047e4937aSGao Xiang submit_bio_retry: 15194e4e153SGao Xiang submit_bio(bio); 15247e4937aSGao Xiang bio = NULL; 15347e4937aSGao Xiang } 15447e4937aSGao Xiang 15547e4937aSGao Xiang if (!bio) { 15647e4937aSGao Xiang struct erofs_map_blocks map = { 15747e4937aSGao Xiang .m_la = blknr_to_addr(current_block), 15847e4937aSGao Xiang }; 15947e4937aSGao Xiang erofs_blk_t blknr; 16047e4937aSGao Xiang unsigned int blkoff; 16147e4937aSGao Xiang 16247e4937aSGao Xiang err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); 1638d8a09b0SGao Xiang if (err) 16447e4937aSGao Xiang goto err_out; 16547e4937aSGao Xiang 16647e4937aSGao Xiang /* zero out the holed page */ 1678d8a09b0SGao Xiang if (!(map.m_flags & EROFS_MAP_MAPPED)) { 16847e4937aSGao Xiang zero_user_segment(page, 0, PAGE_SIZE); 16947e4937aSGao Xiang SetPageUptodate(page); 17047e4937aSGao Xiang 17147e4937aSGao Xiang /* imply err = 0, see erofs_map_blocks */ 17247e4937aSGao Xiang goto has_updated; 17347e4937aSGao Xiang } 17447e4937aSGao Xiang 17547e4937aSGao Xiang /* for RAW access mode, m_plen must be equal to m_llen */ 17647e4937aSGao Xiang DBG_BUGON(map.m_plen != map.m_llen); 17747e4937aSGao Xiang 17847e4937aSGao Xiang blknr = erofs_blknr(map.m_pa); 17947e4937aSGao Xiang blkoff = erofs_blkoff(map.m_pa); 18047e4937aSGao Xiang 18147e4937aSGao Xiang /* deal with inline page */ 18247e4937aSGao Xiang if (map.m_flags & EROFS_MAP_META) { 18347e4937aSGao Xiang void *vsrc, *vto; 18447e4937aSGao Xiang struct page *ipage; 18547e4937aSGao Xiang 18647e4937aSGao Xiang DBG_BUGON(map.m_plen > PAGE_SIZE); 18747e4937aSGao Xiang 188e655b5b3SGao Xiang ipage = erofs_get_meta_page(inode->i_sb, blknr); 18947e4937aSGao Xiang 19047e4937aSGao Xiang if (IS_ERR(ipage)) { 19147e4937aSGao Xiang err = PTR_ERR(ipage); 19247e4937aSGao Xiang goto err_out; 19347e4937aSGao Xiang } 19447e4937aSGao Xiang 19547e4937aSGao Xiang vsrc = kmap_atomic(ipage); 19647e4937aSGao Xiang vto = kmap_atomic(page); 19747e4937aSGao Xiang memcpy(vto, vsrc + blkoff, map.m_plen); 19847e4937aSGao Xiang memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen); 19947e4937aSGao Xiang kunmap_atomic(vto); 20047e4937aSGao Xiang kunmap_atomic(vsrc); 20147e4937aSGao Xiang flush_dcache_page(page); 20247e4937aSGao Xiang 20347e4937aSGao Xiang SetPageUptodate(page); 20447e4937aSGao Xiang /* TODO: could we unlock the page earlier? */ 20547e4937aSGao Xiang unlock_page(ipage); 20647e4937aSGao Xiang put_page(ipage); 20747e4937aSGao Xiang 20847e4937aSGao Xiang /* imply err = 0, see erofs_map_blocks */ 20947e4937aSGao Xiang goto has_updated; 21047e4937aSGao Xiang } 21147e4937aSGao Xiang 21247e4937aSGao Xiang /* pa must be block-aligned for raw reading */ 21347e4937aSGao Xiang DBG_BUGON(erofs_blkoff(map.m_pa)); 21447e4937aSGao Xiang 21547e4937aSGao Xiang /* max # of continuous pages */ 21647e4937aSGao Xiang if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE)) 21747e4937aSGao Xiang nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE); 21847e4937aSGao Xiang if (nblocks > BIO_MAX_PAGES) 21947e4937aSGao Xiang nblocks = BIO_MAX_PAGES; 22047e4937aSGao Xiang 221618f40eaSGao Xiang bio = bio_alloc(GFP_NOIO, nblocks); 222618f40eaSGao Xiang 223618f40eaSGao Xiang bio->bi_end_io = erofs_readendio; 224618f40eaSGao Xiang bio_set_dev(bio, sb->s_bdev); 225618f40eaSGao Xiang bio->bi_iter.bi_sector = (sector_t)blknr << 226618f40eaSGao Xiang LOG_SECTORS_PER_BLOCK; 227618f40eaSGao Xiang bio->bi_opf = REQ_OP_READ; 22847e4937aSGao Xiang } 22947e4937aSGao Xiang 23047e4937aSGao Xiang err = bio_add_page(bio, page, PAGE_SIZE, 0); 23147e4937aSGao Xiang /* out of the extent or bio is full */ 23247e4937aSGao Xiang if (err < PAGE_SIZE) 23347e4937aSGao Xiang goto submit_bio_retry; 23447e4937aSGao Xiang 23547e4937aSGao Xiang *last_block = current_block; 23647e4937aSGao Xiang 23747e4937aSGao Xiang /* shift in advance in case of it followed by too many gaps */ 23847e4937aSGao Xiang if (bio->bi_iter.bi_size >= bio->bi_max_vecs * PAGE_SIZE) { 23947e4937aSGao Xiang /* err should reassign to 0 after submitting */ 24047e4937aSGao Xiang err = 0; 24147e4937aSGao Xiang goto submit_bio_out; 24247e4937aSGao Xiang } 24347e4937aSGao Xiang 24447e4937aSGao Xiang return bio; 24547e4937aSGao Xiang 24647e4937aSGao Xiang err_out: 24747e4937aSGao Xiang /* for sync reading, set page error immediately */ 24847e4937aSGao Xiang if (!ra) { 24947e4937aSGao Xiang SetPageError(page); 25047e4937aSGao Xiang ClearPageUptodate(page); 25147e4937aSGao Xiang } 25247e4937aSGao Xiang has_updated: 25347e4937aSGao Xiang unlock_page(page); 25447e4937aSGao Xiang 25547e4937aSGao Xiang /* if updated manually, continuous pages has a gap */ 25647e4937aSGao Xiang if (bio) 25747e4937aSGao Xiang submit_bio_out: 25894e4e153SGao Xiang submit_bio(bio); 2598d8a09b0SGao Xiang return err ? ERR_PTR(err) : NULL; 26047e4937aSGao Xiang } 26147e4937aSGao Xiang 26247e4937aSGao Xiang /* 26347e4937aSGao Xiang * since we dont have write or truncate flows, so no inode 26447e4937aSGao Xiang * locking needs to be held at the moment. 26547e4937aSGao Xiang */ 26647e4937aSGao Xiang static int erofs_raw_access_readpage(struct file *file, struct page *page) 26747e4937aSGao Xiang { 26847e4937aSGao Xiang erofs_off_t last_block; 26947e4937aSGao Xiang struct bio *bio; 27047e4937aSGao Xiang 27147e4937aSGao Xiang trace_erofs_readpage(page, true); 27247e4937aSGao Xiang 27347e4937aSGao Xiang bio = erofs_read_raw_page(NULL, page->mapping, 27447e4937aSGao Xiang page, &last_block, 1, false); 27547e4937aSGao Xiang 27647e4937aSGao Xiang if (IS_ERR(bio)) 27747e4937aSGao Xiang return PTR_ERR(bio); 27847e4937aSGao Xiang 27947e4937aSGao Xiang DBG_BUGON(bio); /* since we have only one bio -- must be NULL */ 28047e4937aSGao Xiang return 0; 28147e4937aSGao Xiang } 28247e4937aSGao Xiang 283*0c07a9f9SMatthew Wilcox (Oracle) static void erofs_raw_access_readahead(struct readahead_control *rac) 28447e4937aSGao Xiang { 28547e4937aSGao Xiang erofs_off_t last_block; 28647e4937aSGao Xiang struct bio *bio = NULL; 287*0c07a9f9SMatthew Wilcox (Oracle) struct page *page; 28847e4937aSGao Xiang 289*0c07a9f9SMatthew Wilcox (Oracle) trace_erofs_readpages(rac->mapping->host, readahead_index(rac), 290*0c07a9f9SMatthew Wilcox (Oracle) readahead_count(rac), true); 29147e4937aSGao Xiang 292*0c07a9f9SMatthew Wilcox (Oracle) while ((page = readahead_page(rac))) { 29347e4937aSGao Xiang prefetchw(&page->flags); 29447e4937aSGao Xiang 295*0c07a9f9SMatthew Wilcox (Oracle) bio = erofs_read_raw_page(bio, rac->mapping, page, &last_block, 296*0c07a9f9SMatthew Wilcox (Oracle) readahead_count(rac), true); 29747e4937aSGao Xiang 29847e4937aSGao Xiang /* all the page errors are ignored when readahead */ 29947e4937aSGao Xiang if (IS_ERR(bio)) { 30047e4937aSGao Xiang pr_err("%s, readahead error at page %lu of nid %llu\n", 30147e4937aSGao Xiang __func__, page->index, 302*0c07a9f9SMatthew Wilcox (Oracle) EROFS_I(rac->mapping->host)->nid); 30347e4937aSGao Xiang 30447e4937aSGao Xiang bio = NULL; 30547e4937aSGao Xiang } 30647e4937aSGao Xiang 30747e4937aSGao Xiang put_page(page); 30847e4937aSGao Xiang } 30947e4937aSGao Xiang 31047e4937aSGao Xiang /* the rare case (end in gaps) */ 3118d8a09b0SGao Xiang if (bio) 31294e4e153SGao Xiang submit_bio(bio); 31347e4937aSGao Xiang } 31447e4937aSGao Xiang 31547e4937aSGao Xiang static int erofs_get_block(struct inode *inode, sector_t iblock, 31647e4937aSGao Xiang struct buffer_head *bh, int create) 31747e4937aSGao Xiang { 31847e4937aSGao Xiang struct erofs_map_blocks map = { 31947e4937aSGao Xiang .m_la = iblock << 9, 32047e4937aSGao Xiang }; 32147e4937aSGao Xiang int err; 32247e4937aSGao Xiang 32347e4937aSGao Xiang err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); 32447e4937aSGao Xiang if (err) 32547e4937aSGao Xiang return err; 32647e4937aSGao Xiang 32747e4937aSGao Xiang if (map.m_flags & EROFS_MAP_MAPPED) 32847e4937aSGao Xiang bh->b_blocknr = erofs_blknr(map.m_pa); 32947e4937aSGao Xiang 33047e4937aSGao Xiang return err; 33147e4937aSGao Xiang } 33247e4937aSGao Xiang 33347e4937aSGao Xiang static sector_t erofs_bmap(struct address_space *mapping, sector_t block) 33447e4937aSGao Xiang { 33547e4937aSGao Xiang struct inode *inode = mapping->host; 33647e4937aSGao Xiang 337a5876e24SGao Xiang if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) { 33847e4937aSGao Xiang erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE; 33947e4937aSGao Xiang 34047e4937aSGao Xiang if (block >> LOG_SECTORS_PER_BLOCK >= blks) 34147e4937aSGao Xiang return 0; 34247e4937aSGao Xiang } 34347e4937aSGao Xiang 34447e4937aSGao Xiang return generic_block_bmap(mapping, block, erofs_get_block); 34547e4937aSGao Xiang } 34647e4937aSGao Xiang 34747e4937aSGao Xiang /* for uncompressed (aligned) files and raw access for other files */ 34847e4937aSGao Xiang const struct address_space_operations erofs_raw_access_aops = { 34947e4937aSGao Xiang .readpage = erofs_raw_access_readpage, 350*0c07a9f9SMatthew Wilcox (Oracle) .readahead = erofs_raw_access_readahead, 35147e4937aSGao Xiang .bmap = erofs_bmap, 35247e4937aSGao Xiang }; 35347e4937aSGao Xiang 354