1 /* 2 * linux/fs/ext4/page-io.c 3 * 4 * This contains the new page_io functions for ext4 5 * 6 * Written by Theodore Ts'o, 2010. 7 */ 8 9 #include <linux/module.h> 10 #include <linux/fs.h> 11 #include <linux/time.h> 12 #include <linux/jbd2.h> 13 #include <linux/highuid.h> 14 #include <linux/pagemap.h> 15 #include <linux/quotaops.h> 16 #include <linux/string.h> 17 #include <linux/buffer_head.h> 18 #include <linux/writeback.h> 19 #include <linux/pagevec.h> 20 #include <linux/mpage.h> 21 #include <linux/namei.h> 22 #include <linux/uio.h> 23 #include <linux/bio.h> 24 #include <linux/workqueue.h> 25 #include <linux/kernel.h> 26 #include <linux/slab.h> 27 28 #include "ext4_jbd2.h" 29 #include "xattr.h" 30 #include "acl.h" 31 #include "ext4_extents.h" 32 33 static struct kmem_cache *io_page_cachep, *io_end_cachep; 34 35 #define WQ_HASH_SZ 37 36 #define to_ioend_wq(v) (&ioend_wq[((unsigned long)v) % WQ_HASH_SZ]) 37 static wait_queue_head_t ioend_wq[WQ_HASH_SZ]; 38 39 int __init ext4_init_pageio(void) 40 { 41 int i; 42 43 io_page_cachep = KMEM_CACHE(ext4_io_page, SLAB_RECLAIM_ACCOUNT); 44 if (io_page_cachep == NULL) 45 return -ENOMEM; 46 io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT); 47 if (io_page_cachep == NULL) { 48 kmem_cache_destroy(io_page_cachep); 49 return -ENOMEM; 50 } 51 for (i = 0; i < WQ_HASH_SZ; i++) 52 init_waitqueue_head(&ioend_wq[i]); 53 54 return 0; 55 } 56 57 void ext4_exit_pageio(void) 58 { 59 kmem_cache_destroy(io_end_cachep); 60 kmem_cache_destroy(io_page_cachep); 61 } 62 63 void ext4_ioend_wait(struct inode *inode) 64 { 65 wait_queue_head_t *wq = to_ioend_wq(inode); 66 67 wait_event(*wq, (atomic_read(&EXT4_I(inode)->i_ioend_count) == 0)); 68 } 69 70 static void put_io_page(struct ext4_io_page *io_page) 71 { 72 if (atomic_dec_and_test(&io_page->p_count)) { 73 end_page_writeback(io_page->p_page); 74 put_page(io_page->p_page); 75 kmem_cache_free(io_page_cachep, io_page); 76 } 77 } 78 79 void ext4_free_io_end(ext4_io_end_t *io) 80 { 81 int i; 82 wait_queue_head_t *wq; 83 84 BUG_ON(!io); 85 if (io->page) 86 put_page(io->page); 87 for (i = 0; i < io->num_io_pages; i++) 88 put_io_page(io->pages[i]); 89 io->num_io_pages = 0; 90 wq = to_ioend_wq(io->inode); 91 if (atomic_dec_and_test(&EXT4_I(io->inode)->i_ioend_count) && 92 waitqueue_active(wq)) 93 wake_up_all(wq); 94 kmem_cache_free(io_end_cachep, io); 95 } 96 97 /* 98 * check a range of space and convert unwritten extents to written. 99 */ 100 int ext4_end_io_nolock(ext4_io_end_t *io) 101 { 102 struct inode *inode = io->inode; 103 loff_t offset = io->offset; 104 ssize_t size = io->size; 105 int ret = 0; 106 107 ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," 108 "list->prev 0x%p\n", 109 io, inode->i_ino, io->list.next, io->list.prev); 110 111 if (list_empty(&io->list)) 112 return ret; 113 114 if (!(io->flag & EXT4_IO_END_UNWRITTEN)) 115 return ret; 116 117 ret = ext4_convert_unwritten_extents(inode, offset, size); 118 if (ret < 0) { 119 printk(KERN_EMERG "%s: failed to convert unwritten " 120 "extents to written extents, error is %d " 121 "io is still on inode %lu aio dio list\n", 122 __func__, ret, inode->i_ino); 123 return ret; 124 } 125 126 if (io->iocb) 127 aio_complete(io->iocb, io->result, 0); 128 /* clear the DIO AIO unwritten flag */ 129 io->flag &= ~EXT4_IO_END_UNWRITTEN; 130 return ret; 131 } 132 133 /* 134 * work on completed aio dio IO, to convert unwritten extents to extents 135 */ 136 static void ext4_end_io_work(struct work_struct *work) 137 { 138 ext4_io_end_t *io = container_of(work, ext4_io_end_t, work); 139 struct inode *inode = io->inode; 140 struct ext4_inode_info *ei = EXT4_I(inode); 141 unsigned long flags; 142 int ret; 143 144 mutex_lock(&inode->i_mutex); 145 ret = ext4_end_io_nolock(io); 146 if (ret < 0) { 147 mutex_unlock(&inode->i_mutex); 148 return; 149 } 150 151 spin_lock_irqsave(&ei->i_completed_io_lock, flags); 152 if (!list_empty(&io->list)) 153 list_del_init(&io->list); 154 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); 155 mutex_unlock(&inode->i_mutex); 156 ext4_free_io_end(io); 157 } 158 159 ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags) 160 { 161 ext4_io_end_t *io = NULL; 162 163 io = kmem_cache_alloc(io_end_cachep, flags); 164 if (io) { 165 memset(io, 0, sizeof(*io)); 166 atomic_inc(&EXT4_I(inode)->i_ioend_count); 167 io->inode = inode; 168 INIT_WORK(&io->work, ext4_end_io_work); 169 INIT_LIST_HEAD(&io->list); 170 } 171 return io; 172 } 173 174 /* 175 * Print an buffer I/O error compatible with the fs/buffer.c. This 176 * provides compatibility with dmesg scrapers that look for a specific 177 * buffer I/O error message. We really need a unified error reporting 178 * structure to userspace ala Digital Unix's uerf system, but it's 179 * probably not going to happen in my lifetime, due to LKML politics... 180 */ 181 static void buffer_io_error(struct buffer_head *bh) 182 { 183 char b[BDEVNAME_SIZE]; 184 printk(KERN_ERR "Buffer I/O error on device %s, logical block %llu\n", 185 bdevname(bh->b_bdev, b), 186 (unsigned long long)bh->b_blocknr); 187 } 188 189 static void ext4_end_bio(struct bio *bio, int error) 190 { 191 ext4_io_end_t *io_end = bio->bi_private; 192 struct workqueue_struct *wq; 193 struct inode *inode; 194 unsigned long flags; 195 int i; 196 197 BUG_ON(!io_end); 198 bio->bi_private = NULL; 199 bio->bi_end_io = NULL; 200 if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 201 error = 0; 202 bio_put(bio); 203 204 for (i = 0; i < io_end->num_io_pages; i++) { 205 struct page *page = io_end->pages[i]->p_page; 206 struct buffer_head *bh, *head; 207 int partial_write = 0; 208 209 head = page_buffers(page); 210 if (error) 211 SetPageError(page); 212 BUG_ON(!head); 213 if (head->b_size == PAGE_CACHE_SIZE) 214 clear_buffer_dirty(head); 215 else { 216 loff_t offset; 217 loff_t io_end_offset = io_end->offset + io_end->size; 218 219 offset = (sector_t) page->index << PAGE_CACHE_SHIFT; 220 bh = head; 221 do { 222 if ((offset >= io_end->offset) && 223 (offset+bh->b_size <= io_end_offset)) { 224 if (error) 225 buffer_io_error(bh); 226 227 clear_buffer_dirty(bh); 228 } 229 if (buffer_delay(bh)) 230 partial_write = 1; 231 else if (!buffer_mapped(bh)) 232 clear_buffer_dirty(bh); 233 else if (buffer_dirty(bh)) 234 partial_write = 1; 235 offset += bh->b_size; 236 bh = bh->b_this_page; 237 } while (bh != head); 238 } 239 240 /* 241 * If this is a partial write which happened to make 242 * all buffers uptodate then we can optimize away a 243 * bogus readpage() for the next read(). Here we 244 * 'discover' whether the page went uptodate as a 245 * result of this (potentially partial) write. 246 */ 247 if (!partial_write) 248 SetPageUptodate(page); 249 250 put_io_page(io_end->pages[i]); 251 } 252 io_end->num_io_pages = 0; 253 inode = io_end->inode; 254 255 if (error) { 256 io_end->flag |= EXT4_IO_END_ERROR; 257 ext4_warning(inode->i_sb, "I/O error writing to inode %lu " 258 "(offset %llu size %ld starting block %llu)", 259 inode->i_ino, 260 (unsigned long long) io_end->offset, 261 (long) io_end->size, 262 (unsigned long long) 263 bio->bi_sector >> (inode->i_blkbits - 9)); 264 } 265 266 /* Add the io_end to per-inode completed io list*/ 267 spin_lock_irqsave(&EXT4_I(inode)->i_completed_io_lock, flags); 268 list_add_tail(&io_end->list, &EXT4_I(inode)->i_completed_io_list); 269 spin_unlock_irqrestore(&EXT4_I(inode)->i_completed_io_lock, flags); 270 271 wq = EXT4_SB(inode->i_sb)->dio_unwritten_wq; 272 /* queue the work to convert unwritten extents to written */ 273 queue_work(wq, &io_end->work); 274 } 275 276 void ext4_io_submit(struct ext4_io_submit *io) 277 { 278 struct bio *bio = io->io_bio; 279 280 if (bio) { 281 bio_get(io->io_bio); 282 submit_bio(io->io_op, io->io_bio); 283 BUG_ON(bio_flagged(io->io_bio, BIO_EOPNOTSUPP)); 284 bio_put(io->io_bio); 285 } 286 io->io_bio = 0; 287 io->io_op = 0; 288 io->io_end = 0; 289 } 290 291 static int io_submit_init(struct ext4_io_submit *io, 292 struct inode *inode, 293 struct writeback_control *wbc, 294 struct buffer_head *bh) 295 { 296 ext4_io_end_t *io_end; 297 struct page *page = bh->b_page; 298 int nvecs = bio_get_nr_vecs(bh->b_bdev); 299 struct bio *bio; 300 301 io_end = ext4_init_io_end(inode, GFP_NOFS); 302 if (!io_end) 303 return -ENOMEM; 304 do { 305 bio = bio_alloc(GFP_NOIO, nvecs); 306 nvecs >>= 1; 307 } while (bio == NULL); 308 309 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 310 bio->bi_bdev = bh->b_bdev; 311 bio->bi_private = io->io_end = io_end; 312 bio->bi_end_io = ext4_end_bio; 313 314 io_end->offset = (page->index << PAGE_CACHE_SHIFT) + bh_offset(bh); 315 316 io->io_bio = bio; 317 io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? 318 WRITE_SYNC_PLUG : WRITE); 319 io->io_next_block = bh->b_blocknr; 320 return 0; 321 } 322 323 static int io_submit_add_bh(struct ext4_io_submit *io, 324 struct ext4_io_page *io_page, 325 struct inode *inode, 326 struct writeback_control *wbc, 327 struct buffer_head *bh) 328 { 329 ext4_io_end_t *io_end; 330 int ret; 331 332 if (buffer_new(bh)) { 333 clear_buffer_new(bh); 334 unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr); 335 } 336 337 if (!buffer_mapped(bh) || buffer_delay(bh)) { 338 if (!buffer_mapped(bh)) 339 clear_buffer_dirty(bh); 340 if (io->io_bio) 341 ext4_io_submit(io); 342 return 0; 343 } 344 345 if (io->io_bio && bh->b_blocknr != io->io_next_block) { 346 submit_and_retry: 347 ext4_io_submit(io); 348 } 349 if (io->io_bio == NULL) { 350 ret = io_submit_init(io, inode, wbc, bh); 351 if (ret) 352 return ret; 353 } 354 io_end = io->io_end; 355 if ((io_end->num_io_pages >= MAX_IO_PAGES) && 356 (io_end->pages[io_end->num_io_pages-1] != io_page)) 357 goto submit_and_retry; 358 if (buffer_uninit(bh)) 359 io->io_end->flag |= EXT4_IO_END_UNWRITTEN; 360 io->io_end->size += bh->b_size; 361 io->io_next_block++; 362 ret = bio_add_page(io->io_bio, bh->b_page, bh->b_size, bh_offset(bh)); 363 if (ret != bh->b_size) 364 goto submit_and_retry; 365 if ((io_end->num_io_pages == 0) || 366 (io_end->pages[io_end->num_io_pages-1] != io_page)) { 367 io_end->pages[io_end->num_io_pages++] = io_page; 368 atomic_inc(&io_page->p_count); 369 } 370 return 0; 371 } 372 373 int ext4_bio_write_page(struct ext4_io_submit *io, 374 struct page *page, 375 int len, 376 struct writeback_control *wbc) 377 { 378 struct inode *inode = page->mapping->host; 379 unsigned block_start, block_end, blocksize; 380 struct ext4_io_page *io_page; 381 struct buffer_head *bh, *head; 382 int ret = 0; 383 384 blocksize = 1 << inode->i_blkbits; 385 386 BUG_ON(PageWriteback(page)); 387 set_page_writeback(page); 388 ClearPageError(page); 389 390 io_page = kmem_cache_alloc(io_page_cachep, GFP_NOFS); 391 if (!io_page) { 392 set_page_dirty(page); 393 unlock_page(page); 394 return -ENOMEM; 395 } 396 io_page->p_page = page; 397 atomic_set(&io_page->p_count, 1); 398 get_page(page); 399 400 for (bh = head = page_buffers(page), block_start = 0; 401 bh != head || !block_start; 402 block_start = block_end, bh = bh->b_this_page) { 403 block_end = block_start + blocksize; 404 if (block_start >= len) { 405 clear_buffer_dirty(bh); 406 set_buffer_uptodate(bh); 407 continue; 408 } 409 ret = io_submit_add_bh(io, io_page, inode, wbc, bh); 410 if (ret) { 411 /* 412 * We only get here on ENOMEM. Not much else 413 * we can do but mark the page as dirty, and 414 * better luck next time. 415 */ 416 set_page_dirty(page); 417 break; 418 } 419 } 420 unlock_page(page); 421 /* 422 * If the page was truncated before we could do the writeback, 423 * or we had a memory allocation error while trying to write 424 * the first buffer head, we won't have submitted any pages for 425 * I/O. In that case we need to make sure we've cleared the 426 * PageWriteback bit from the page to prevent the system from 427 * wedging later on. 428 */ 429 put_io_page(io_page); 430 return ret; 431 } 432