1 /* 2 * linux/fs/ext4/readpage.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * Copyright (C) 2015, Google, Inc. 6 * 7 * This was originally taken from fs/mpage.c 8 * 9 * The intent is the ext4_mpage_readpages() function here is intended 10 * to replace mpage_readpages() in the general case, not just for 11 * encrypted files. It has some limitations (see below), where it 12 * will fall back to read_block_full_page(), but these limitations 13 * should only be hit when page_size != block_size. 14 * 15 * This will allow us to attach a callback function to support ext4 16 * encryption. 17 * 18 * If anything unusual happens, such as: 19 * 20 * - encountering a page which has buffers 21 * - encountering a page which has a non-hole after a hole 22 * - encountering a page with non-contiguous blocks 23 * 24 * then this code just gives up and calls the buffer_head-based read function. 25 * It does handle a page which has holes at the end - that is a common case: 26 * the end-of-file on blocksize < PAGE_SIZE setups. 27 * 28 */ 29 30 #include <linux/kernel.h> 31 #include <linux/export.h> 32 #include <linux/mm.h> 33 #include <linux/kdev_t.h> 34 #include <linux/gfp.h> 35 #include <linux/bio.h> 36 #include <linux/fs.h> 37 #include <linux/buffer_head.h> 38 #include <linux/blkdev.h> 39 #include <linux/highmem.h> 40 #include <linux/prefetch.h> 41 #include <linux/mpage.h> 42 #include <linux/writeback.h> 43 #include <linux/backing-dev.h> 44 #include <linux/pagevec.h> 45 #include <linux/cleancache.h> 46 47 #include "ext4.h" 48 49 static inline bool ext4_bio_encrypted(struct bio *bio) 50 { 51 #ifdef CONFIG_EXT4_FS_ENCRYPTION 52 return unlikely(bio->bi_private != NULL); 53 #else 54 return false; 55 #endif 56 } 57 58 /* 59 * I/O completion handler for multipage BIOs. 60 * 61 * The mpage code never puts partial pages into a BIO (except for end-of-file). 62 * If a page does not map to a contiguous run of blocks then it simply falls 63 * back to block_read_full_page(). 64 * 65 * Why is this? If a page's completion depends on a number of different BIOs 66 * which can complete in any order (or at the same time) then determining the 67 * status of that page is hard. See end_buffer_async_read() for the details. 68 * There is no point in duplicating all that complexity. 69 */ 70 static void mpage_end_io(struct bio *bio) 71 { 72 struct bio_vec *bv; 73 int i; 74 75 if (ext4_bio_encrypted(bio)) { 76 if (bio->bi_error) { 77 fscrypt_release_ctx(bio->bi_private); 78 } else { 79 fscrypt_decrypt_bio_pages(bio->bi_private, bio); 80 return; 81 } 82 } 83 bio_for_each_segment_all(bv, bio, i) { 84 struct page *page = bv->bv_page; 85 86 if (!bio->bi_error) { 87 SetPageUptodate(page); 88 } else { 89 ClearPageUptodate(page); 90 SetPageError(page); 91 } 92 unlock_page(page); 93 } 94 95 bio_put(bio); 96 } 97 98 int ext4_mpage_readpages(struct address_space *mapping, 99 struct list_head *pages, struct page *page, 100 unsigned nr_pages) 101 { 102 struct bio *bio = NULL; 103 sector_t last_block_in_bio = 0; 104 105 struct inode *inode = mapping->host; 106 const unsigned blkbits = inode->i_blkbits; 107 const unsigned blocks_per_page = PAGE_SIZE >> blkbits; 108 const unsigned blocksize = 1 << blkbits; 109 sector_t block_in_file; 110 sector_t last_block; 111 sector_t last_block_in_file; 112 sector_t blocks[MAX_BUF_PER_PAGE]; 113 unsigned page_block; 114 struct block_device *bdev = inode->i_sb->s_bdev; 115 int length; 116 unsigned relative_block = 0; 117 struct ext4_map_blocks map; 118 119 map.m_pblk = 0; 120 map.m_lblk = 0; 121 map.m_len = 0; 122 map.m_flags = 0; 123 124 for (; nr_pages; nr_pages--) { 125 int fully_mapped = 1; 126 unsigned first_hole = blocks_per_page; 127 128 prefetchw(&page->flags); 129 if (pages) { 130 page = list_entry(pages->prev, struct page, lru); 131 list_del(&page->lru); 132 if (add_to_page_cache_lru(page, mapping, page->index, 133 readahead_gfp_mask(mapping))) 134 goto next_page; 135 } 136 137 if (page_has_buffers(page)) 138 goto confused; 139 140 block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits); 141 last_block = block_in_file + nr_pages * blocks_per_page; 142 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits; 143 if (last_block > last_block_in_file) 144 last_block = last_block_in_file; 145 page_block = 0; 146 147 /* 148 * Map blocks using the previous result first. 149 */ 150 if ((map.m_flags & EXT4_MAP_MAPPED) && 151 block_in_file > map.m_lblk && 152 block_in_file < (map.m_lblk + map.m_len)) { 153 unsigned map_offset = block_in_file - map.m_lblk; 154 unsigned last = map.m_len - map_offset; 155 156 for (relative_block = 0; ; relative_block++) { 157 if (relative_block == last) { 158 /* needed? */ 159 map.m_flags &= ~EXT4_MAP_MAPPED; 160 break; 161 } 162 if (page_block == blocks_per_page) 163 break; 164 blocks[page_block] = map.m_pblk + map_offset + 165 relative_block; 166 page_block++; 167 block_in_file++; 168 } 169 } 170 171 /* 172 * Then do more ext4_map_blocks() calls until we are 173 * done with this page. 174 */ 175 while (page_block < blocks_per_page) { 176 if (block_in_file < last_block) { 177 map.m_lblk = block_in_file; 178 map.m_len = last_block - block_in_file; 179 180 if (ext4_map_blocks(NULL, inode, &map, 0) < 0) { 181 set_error_page: 182 SetPageError(page); 183 zero_user_segment(page, 0, 184 PAGE_SIZE); 185 unlock_page(page); 186 goto next_page; 187 } 188 } 189 if ((map.m_flags & EXT4_MAP_MAPPED) == 0) { 190 fully_mapped = 0; 191 if (first_hole == blocks_per_page) 192 first_hole = page_block; 193 page_block++; 194 block_in_file++; 195 continue; 196 } 197 if (first_hole != blocks_per_page) 198 goto confused; /* hole -> non-hole */ 199 200 /* Contiguous blocks? */ 201 if (page_block && blocks[page_block-1] != map.m_pblk-1) 202 goto confused; 203 for (relative_block = 0; ; relative_block++) { 204 if (relative_block == map.m_len) { 205 /* needed? */ 206 map.m_flags &= ~EXT4_MAP_MAPPED; 207 break; 208 } else if (page_block == blocks_per_page) 209 break; 210 blocks[page_block] = map.m_pblk+relative_block; 211 page_block++; 212 block_in_file++; 213 } 214 } 215 if (first_hole != blocks_per_page) { 216 zero_user_segment(page, first_hole << blkbits, 217 PAGE_SIZE); 218 if (first_hole == 0) { 219 SetPageUptodate(page); 220 unlock_page(page); 221 goto next_page; 222 } 223 } else if (fully_mapped) { 224 SetPageMappedToDisk(page); 225 } 226 if (fully_mapped && blocks_per_page == 1 && 227 !PageUptodate(page) && cleancache_get_page(page) == 0) { 228 SetPageUptodate(page); 229 goto confused; 230 } 231 232 /* 233 * This page will go to BIO. Do we need to send this 234 * BIO off first? 235 */ 236 if (bio && (last_block_in_bio != blocks[0] - 1)) { 237 submit_and_realloc: 238 submit_bio(bio); 239 bio = NULL; 240 } 241 if (bio == NULL) { 242 struct fscrypt_ctx *ctx = NULL; 243 244 if (ext4_encrypted_inode(inode) && 245 S_ISREG(inode->i_mode)) { 246 ctx = fscrypt_get_ctx(inode, GFP_NOFS); 247 if (IS_ERR(ctx)) 248 goto set_error_page; 249 } 250 bio = bio_alloc(GFP_KERNEL, 251 min_t(int, nr_pages, BIO_MAX_PAGES)); 252 if (!bio) { 253 if (ctx) 254 fscrypt_release_ctx(ctx); 255 goto set_error_page; 256 } 257 bio->bi_bdev = bdev; 258 bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9); 259 bio->bi_end_io = mpage_end_io; 260 bio->bi_private = ctx; 261 bio_set_op_attrs(bio, REQ_OP_READ, 0); 262 } 263 264 length = first_hole << blkbits; 265 if (bio_add_page(bio, page, length, 0) < length) 266 goto submit_and_realloc; 267 268 if (((map.m_flags & EXT4_MAP_BOUNDARY) && 269 (relative_block == map.m_len)) || 270 (first_hole != blocks_per_page)) { 271 submit_bio(bio); 272 bio = NULL; 273 } else 274 last_block_in_bio = blocks[blocks_per_page - 1]; 275 goto next_page; 276 confused: 277 if (bio) { 278 submit_bio(bio); 279 bio = NULL; 280 } 281 if (!PageUptodate(page)) 282 block_read_full_page(page, ext4_get_block); 283 else 284 unlock_page(page); 285 next_page: 286 if (pages) 287 put_page(page); 288 } 289 BUG_ON(pages && !list_empty(pages)); 290 if (bio) 291 submit_bio(bio); 292 return 0; 293 } 294