1 /* 2 * linux/fs/ext4/page-io.c 3 * 4 * This contains the new page_io functions for ext4 5 * 6 * Written by Theodore Ts'o, 2010. 7 */ 8 9 #include <linux/fs.h> 10 #include <linux/time.h> 11 #include <linux/highuid.h> 12 #include <linux/pagemap.h> 13 #include <linux/quotaops.h> 14 #include <linux/string.h> 15 #include <linux/buffer_head.h> 16 #include <linux/writeback.h> 17 #include <linux/pagevec.h> 18 #include <linux/mpage.h> 19 #include <linux/namei.h> 20 #include <linux/uio.h> 21 #include <linux/bio.h> 22 #include <linux/workqueue.h> 23 #include <linux/kernel.h> 24 #include <linux/slab.h> 25 #include <linux/mm.h> 26 27 #include "ext4_jbd2.h" 28 #include "xattr.h" 29 #include "acl.h" 30 31 static struct kmem_cache *io_end_cachep; 32 33 int __init ext4_init_pageio(void) 34 { 35 io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT); 36 if (io_end_cachep == NULL) 37 return -ENOMEM; 38 return 0; 39 } 40 41 void ext4_exit_pageio(void) 42 { 43 kmem_cache_destroy(io_end_cachep); 44 } 45 46 /* 47 * Print an buffer I/O error compatible with the fs/buffer.c. This 48 * provides compatibility with dmesg scrapers that look for a specific 49 * buffer I/O error message. We really need a unified error reporting 50 * structure to userspace ala Digital Unix's uerf system, but it's 51 * probably not going to happen in my lifetime, due to LKML politics... 52 */ 53 static void buffer_io_error(struct buffer_head *bh) 54 { 55 char b[BDEVNAME_SIZE]; 56 printk_ratelimited(KERN_ERR "Buffer I/O error on device %s, logical block %llu\n", 57 bdevname(bh->b_bdev, b), 58 (unsigned long long)bh->b_blocknr); 59 } 60 61 static void ext4_finish_bio(struct bio *bio) 62 { 63 int i; 64 int error = !test_bit(BIO_UPTODATE, &bio->bi_flags); 65 struct bio_vec *bvec; 66 67 bio_for_each_segment_all(bvec, bio, i) { 68 struct page *page = bvec->bv_page; 69 #ifdef CONFIG_EXT4_FS_ENCRYPTION 70 struct page *data_page = NULL; 71 struct ext4_crypto_ctx *ctx = NULL; 72 #endif 73 struct buffer_head *bh, *head; 74 unsigned bio_start = bvec->bv_offset; 75 unsigned bio_end = bio_start + bvec->bv_len; 76 unsigned under_io = 0; 77 unsigned long flags; 78 79 if (!page) 80 continue; 81 82 #ifdef CONFIG_EXT4_FS_ENCRYPTION 83 if (!page->mapping) { 84 /* The bounce data pages are unmapped. */ 85 data_page = page; 86 ctx = (struct ext4_crypto_ctx *)page_private(data_page); 87 page = ctx->control_page; 88 } 89 #endif 90 91 if (error) { 92 SetPageError(page); 93 set_bit(AS_EIO, &page->mapping->flags); 94 } 95 bh = head = page_buffers(page); 96 /* 97 * We check all buffers in the page under BH_Uptodate_Lock 98 * to avoid races with other end io clearing async_write flags 99 */ 100 local_irq_save(flags); 101 bit_spin_lock(BH_Uptodate_Lock, &head->b_state); 102 do { 103 if (bh_offset(bh) < bio_start || 104 bh_offset(bh) + bh->b_size > bio_end) { 105 if (buffer_async_write(bh)) 106 under_io++; 107 continue; 108 } 109 clear_buffer_async_write(bh); 110 if (error) 111 buffer_io_error(bh); 112 } while ((bh = bh->b_this_page) != head); 113 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); 114 local_irq_restore(flags); 115 if (!under_io) { 116 #ifdef CONFIG_EXT4_FS_ENCRYPTION 117 if (ctx) 118 ext4_restore_control_page(data_page); 119 #endif 120 end_page_writeback(page); 121 } 122 } 123 } 124 125 static void ext4_release_io_end(ext4_io_end_t *io_end) 126 { 127 struct bio *bio, *next_bio; 128 129 BUG_ON(!list_empty(&io_end->list)); 130 BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN); 131 WARN_ON(io_end->handle); 132 133 if (atomic_dec_and_test(&EXT4_I(io_end->inode)->i_ioend_count)) 134 wake_up_all(ext4_ioend_wq(io_end->inode)); 135 136 for (bio = io_end->bio; bio; bio = next_bio) { 137 next_bio = bio->bi_private; 138 ext4_finish_bio(bio); 139 bio_put(bio); 140 } 141 kmem_cache_free(io_end_cachep, io_end); 142 } 143 144 static void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) 145 { 146 struct inode *inode = io_end->inode; 147 148 io_end->flag &= ~EXT4_IO_END_UNWRITTEN; 149 /* Wake up anyone waiting on unwritten extent conversion */ 150 if (atomic_dec_and_test(&EXT4_I(inode)->i_unwritten)) 151 wake_up_all(ext4_ioend_wq(inode)); 152 } 153 154 /* 155 * Check a range of space and convert unwritten extents to written. Note that 156 * we are protected from truncate touching same part of extent tree by the 157 * fact that truncate code waits for all DIO to finish (thus exclusion from 158 * direct IO is achieved) and also waits for PageWriteback bits. Thus we 159 * cannot get to ext4_ext_truncate() before all IOs overlapping that range are 160 * completed (happens from ext4_free_ioend()). 161 */ 162 static int ext4_end_io(ext4_io_end_t *io) 163 { 164 struct inode *inode = io->inode; 165 loff_t offset = io->offset; 166 ssize_t size = io->size; 167 handle_t *handle = io->handle; 168 int ret = 0; 169 170 ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," 171 "list->prev 0x%p\n", 172 io, inode->i_ino, io->list.next, io->list.prev); 173 174 io->handle = NULL; /* Following call will use up the handle */ 175 ret = ext4_convert_unwritten_extents(handle, inode, offset, size); 176 if (ret < 0) { 177 ext4_msg(inode->i_sb, KERN_EMERG, 178 "failed to convert unwritten extents to written " 179 "extents -- potential data loss! " 180 "(inode %lu, offset %llu, size %zd, error %d)", 181 inode->i_ino, offset, size, ret); 182 } 183 ext4_clear_io_unwritten_flag(io); 184 ext4_release_io_end(io); 185 return ret; 186 } 187 188 static void dump_completed_IO(struct inode *inode, struct list_head *head) 189 { 190 #ifdef EXT4FS_DEBUG 191 struct list_head *cur, *before, *after; 192 ext4_io_end_t *io, *io0, *io1; 193 194 if (list_empty(head)) 195 return; 196 197 ext4_debug("Dump inode %lu completed io list\n", inode->i_ino); 198 list_for_each_entry(io, head, list) { 199 cur = &io->list; 200 before = cur->prev; 201 io0 = container_of(before, ext4_io_end_t, list); 202 after = cur->next; 203 io1 = container_of(after, ext4_io_end_t, list); 204 205 ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n", 206 io, inode->i_ino, io0, io1); 207 } 208 #endif 209 } 210 211 /* Add the io_end to per-inode completed end_io list. */ 212 static void ext4_add_complete_io(ext4_io_end_t *io_end) 213 { 214 struct ext4_inode_info *ei = EXT4_I(io_end->inode); 215 struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb); 216 struct workqueue_struct *wq; 217 unsigned long flags; 218 219 /* Only reserved conversions from writeback should enter here */ 220 WARN_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN)); 221 WARN_ON(!io_end->handle && sbi->s_journal); 222 spin_lock_irqsave(&ei->i_completed_io_lock, flags); 223 wq = sbi->rsv_conversion_wq; 224 if (list_empty(&ei->i_rsv_conversion_list)) 225 queue_work(wq, &ei->i_rsv_conversion_work); 226 list_add_tail(&io_end->list, &ei->i_rsv_conversion_list); 227 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 228 } 229 230 static int ext4_do_flush_completed_IO(struct inode *inode, 231 struct list_head *head) 232 { 233 ext4_io_end_t *io; 234 struct list_head unwritten; 235 unsigned long flags; 236 struct ext4_inode_info *ei = EXT4_I(inode); 237 int err, ret = 0; 238 239 spin_lock_irqsave(&ei->i_completed_io_lock, flags); 240 dump_completed_IO(inode, head); 241 list_replace_init(head, &unwritten); 242 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 243 244 while (!list_empty(&unwritten)) { 245 io = list_entry(unwritten.next, ext4_io_end_t, list); 246 BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN)); 247 list_del_init(&io->list); 248 249 err = ext4_end_io(io); 250 if (unlikely(!ret && err)) 251 ret = err; 252 } 253 return ret; 254 } 255 256 /* 257 * work on completed IO, to convert unwritten extents to extents 258 */ 259 void ext4_end_io_rsv_work(struct work_struct *work) 260 { 261 struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info, 262 i_rsv_conversion_work); 263 ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list); 264 } 265 266 ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags) 267 { 268 ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags); 269 if (io) { 270 atomic_inc(&EXT4_I(inode)->i_ioend_count); 271 io->inode = inode; 272 INIT_LIST_HEAD(&io->list); 273 atomic_set(&io->count, 1); 274 } 275 return io; 276 } 277 278 void ext4_put_io_end_defer(ext4_io_end_t *io_end) 279 { 280 if (atomic_dec_and_test(&io_end->count)) { 281 if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) || !io_end->size) { 282 ext4_release_io_end(io_end); 283 return; 284 } 285 ext4_add_complete_io(io_end); 286 } 287 } 288 289 int ext4_put_io_end(ext4_io_end_t *io_end) 290 { 291 int err = 0; 292 293 if (atomic_dec_and_test(&io_end->count)) { 294 if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 295 err = ext4_convert_unwritten_extents(io_end->handle, 296 io_end->inode, io_end->offset, 297 io_end->size); 298 io_end->handle = NULL; 299 ext4_clear_io_unwritten_flag(io_end); 300 } 301 ext4_release_io_end(io_end); 302 } 303 return err; 304 } 305 306 ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end) 307 { 308 atomic_inc(&io_end->count); 309 return io_end; 310 } 311 312 /* BIO completion function for page writeback */ 313 static void ext4_end_bio(struct bio *bio, int error) 314 { 315 ext4_io_end_t *io_end = bio->bi_private; 316 sector_t bi_sector = bio->bi_iter.bi_sector; 317 318 BUG_ON(!io_end); 319 bio->bi_end_io = NULL; 320 if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 321 error = 0; 322 323 if (error) { 324 struct inode *inode = io_end->inode; 325 326 ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu " 327 "(offset %llu size %ld starting block %llu)", 328 error, inode->i_ino, 329 (unsigned long long) io_end->offset, 330 (long) io_end->size, 331 (unsigned long long) 332 bi_sector >> (inode->i_blkbits - 9)); 333 mapping_set_error(inode->i_mapping, error); 334 } 335 336 if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 337 /* 338 * Link bio into list hanging from io_end. We have to do it 339 * atomically as bio completions can be racing against each 340 * other. 341 */ 342 bio->bi_private = xchg(&io_end->bio, bio); 343 ext4_put_io_end_defer(io_end); 344 } else { 345 /* 346 * Drop io_end reference early. Inode can get freed once 347 * we finish the bio. 348 */ 349 ext4_put_io_end_defer(io_end); 350 ext4_finish_bio(bio); 351 bio_put(bio); 352 } 353 } 354 355 void ext4_io_submit(struct ext4_io_submit *io) 356 { 357 struct bio *bio = io->io_bio; 358 359 if (bio) { 360 bio_get(io->io_bio); 361 submit_bio(io->io_op, io->io_bio); 362 BUG_ON(bio_flagged(io->io_bio, BIO_EOPNOTSUPP)); 363 bio_put(io->io_bio); 364 } 365 io->io_bio = NULL; 366 } 367 368 void ext4_io_submit_init(struct ext4_io_submit *io, 369 struct writeback_control *wbc) 370 { 371 io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE); 372 io->io_bio = NULL; 373 io->io_end = NULL; 374 } 375 376 static int io_submit_init_bio(struct ext4_io_submit *io, 377 struct buffer_head *bh) 378 { 379 int nvecs = bio_get_nr_vecs(bh->b_bdev); 380 struct bio *bio; 381 382 bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES)); 383 if (!bio) 384 return -ENOMEM; 385 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 386 bio->bi_bdev = bh->b_bdev; 387 bio->bi_end_io = ext4_end_bio; 388 bio->bi_private = ext4_get_io_end(io->io_end); 389 io->io_bio = bio; 390 io->io_next_block = bh->b_blocknr; 391 return 0; 392 } 393 394 static int io_submit_add_bh(struct ext4_io_submit *io, 395 struct inode *inode, 396 struct page *page, 397 struct buffer_head *bh) 398 { 399 int ret; 400 401 if (io->io_bio && bh->b_blocknr != io->io_next_block) { 402 submit_and_retry: 403 ext4_io_submit(io); 404 } 405 if (io->io_bio == NULL) { 406 ret = io_submit_init_bio(io, bh); 407 if (ret) 408 return ret; 409 } 410 ret = bio_add_page(io->io_bio, page, bh->b_size, bh_offset(bh)); 411 if (ret != bh->b_size) 412 goto submit_and_retry; 413 io->io_next_block++; 414 return 0; 415 } 416 417 int ext4_bio_write_page(struct ext4_io_submit *io, 418 struct page *page, 419 int len, 420 struct writeback_control *wbc, 421 bool keep_towrite) 422 { 423 struct page *data_page = NULL; 424 struct inode *inode = page->mapping->host; 425 unsigned block_start, blocksize; 426 struct buffer_head *bh, *head; 427 int ret = 0; 428 int nr_submitted = 0; 429 430 blocksize = 1 << inode->i_blkbits; 431 432 BUG_ON(!PageLocked(page)); 433 BUG_ON(PageWriteback(page)); 434 435 if (keep_towrite) 436 set_page_writeback_keepwrite(page); 437 else 438 set_page_writeback(page); 439 ClearPageError(page); 440 441 /* 442 * Comments copied from block_write_full_page: 443 * 444 * The page straddles i_size. It must be zeroed out on each and every 445 * writepage invocation because it may be mmapped. "A file is mapped 446 * in multiples of the page size. For a file that is not a multiple of 447 * the page size, the remaining memory is zeroed when mapped, and 448 * writes to that region are not written out to the file." 449 */ 450 if (len < PAGE_CACHE_SIZE) 451 zero_user_segment(page, len, PAGE_CACHE_SIZE); 452 /* 453 * In the first loop we prepare and mark buffers to submit. We have to 454 * mark all buffers in the page before submitting so that 455 * end_page_writeback() cannot be called from ext4_bio_end_io() when IO 456 * on the first buffer finishes and we are still working on submitting 457 * the second buffer. 458 */ 459 bh = head = page_buffers(page); 460 do { 461 block_start = bh_offset(bh); 462 if (block_start >= len) { 463 clear_buffer_dirty(bh); 464 set_buffer_uptodate(bh); 465 continue; 466 } 467 if (!buffer_dirty(bh) || buffer_delay(bh) || 468 !buffer_mapped(bh) || buffer_unwritten(bh)) { 469 /* A hole? We can safely clear the dirty bit */ 470 if (!buffer_mapped(bh)) 471 clear_buffer_dirty(bh); 472 if (io->io_bio) 473 ext4_io_submit(io); 474 continue; 475 } 476 if (buffer_new(bh)) { 477 clear_buffer_new(bh); 478 unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr); 479 } 480 set_buffer_async_write(bh); 481 } while ((bh = bh->b_this_page) != head); 482 483 bh = head = page_buffers(page); 484 485 if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { 486 data_page = ext4_encrypt(inode, page); 487 if (IS_ERR(data_page)) { 488 ret = PTR_ERR(data_page); 489 data_page = NULL; 490 goto out; 491 } 492 } 493 494 /* Now submit buffers to write */ 495 do { 496 if (!buffer_async_write(bh)) 497 continue; 498 ret = io_submit_add_bh(io, inode, 499 data_page ? data_page : page, bh); 500 if (ret) { 501 /* 502 * We only get here on ENOMEM. Not much else 503 * we can do but mark the page as dirty, and 504 * better luck next time. 505 */ 506 break; 507 } 508 nr_submitted++; 509 clear_buffer_dirty(bh); 510 } while ((bh = bh->b_this_page) != head); 511 512 /* Error stopped previous loop? Clean up buffers... */ 513 if (ret) { 514 out: 515 if (data_page) 516 ext4_restore_control_page(data_page); 517 printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret); 518 redirty_page_for_writepage(wbc, page); 519 do { 520 clear_buffer_async_write(bh); 521 bh = bh->b_this_page; 522 } while (bh != head); 523 } 524 unlock_page(page); 525 /* Nothing submitted - we have to end page writeback */ 526 if (!nr_submitted) 527 end_page_writeback(page); 528 return ret; 529 } 530